Replace google places api with photon api by komoot

This commit is contained in:
Eugene Burmakin 2024-08-13 18:25:48 +02:00
parent 382f937f29
commit 52ee90ac9c
21 changed files with 255 additions and 70 deletions

View file

@ -4,4 +4,4 @@ DATABASE_PASSWORD=password
DATABASE_NAME=dawarich_development
DATABASE_PORT=5432
REDIS_URL=redis://localhost:6379/1
GOOGLE_PLACES_API_KEY=''
PHOTON_API_HOST='photon.chibi.rodeo'

View file

@ -11,7 +11,7 @@ The visit suggestion release.
1. With this release deployment, data migration will work, starting visits suggestion process for all users.
2. After initial visit suggestion process, new suggestions will be calculated every 24 hours, based on points for last 7 days.
3. If you have enabled reverse geocoding and provided Google Places API key, Dawarich will try to reverse geocode your visit and suggest specific places you might have visited, such as cafes, restaurants, parks, etc. If reverse geocoding is not enabled, or Google Places API key is not provided, Dawarich will not try to suggest places but you'll be able to rename the visit yourself.
3. If you have enabled reverse geocoding and provided Photon Api Host, Dawarich will try to reverse geocode your visit and suggest specific places you might have visited, such as cafes, restaurants, parks, etc. If reverse geocoding is not enabled, or Photon Api Host is not provided, Dawarich will not try to suggest places but you'll be able to rename the visit yourself.
4. You can confirm or decline the visit suggestion. If you confirm the visit, it will be added to your timeline. If you decline the visit, it will be removed from your timeline. You'll be able to see all your confirmed, declined and suggested visits on the Visits page.
- [x] Get places from Google Places API based on visit coordinates
@ -20,12 +20,12 @@ The visit suggestion release.
- [x] Draw visit radius based on radius of points in the visit
- [x] Add a possibility to rename the visit
- [x] Make it look acceptable
- [ ] Create only uniq google places suggestions
- [ ] Make visits suggestion an idempotent process
### Added
- `GOOGLE_PLACES_API_KEY` environment variable to the `docker-compose.yml` file to allow user to set the Google Places API key for reverse geocoding
- `PHOTON_API_HOST` environment variable to the `docker-compose.yml` file to allow user to set the Photon API hpst for reverse geocoding
- A "Map" button to each visit on the Visits page to allow user to see the visit on the map
- Visits suggestion functionality. Read more on that in the release description
- Tabs to the Visits page to allow user to switch between confirmed, declined and suggested visits

File diff suppressed because one or more lines are too long

View file

@ -22,7 +22,7 @@ class VisitsController < ApplicationController
def update
if @visit.update(visit_params)
redirect_to visits_url, notice: 'Visit was successfully updated.', status: :see_other
redirect_back(fallback_location: visits_path(status: :suggested))
else
render :edit, status: :unprocessable_entity
end

View file

@ -10,7 +10,7 @@ class Place < ApplicationRecord
has_many :place_visits, dependent: :destroy
has_many :suggested_visits, through: :place_visits, source: :visit
enum source: { manual: 0, google_places: 1 }
enum source: { manual: 0, photon: 1 }
def async_reverse_geocode
return unless REVERSE_GEOCODING_ENABLED

View file

@ -19,6 +19,8 @@ class Point < ApplicationRecord
scope :reverse_geocoded, -> { where.not(city: nil, country: nil) }
scope :not_reverse_geocoded, -> { where(city: nil, country: nil) }
scope :visited, -> { where.not(visit_id: nil) }
scope :not_visited, -> { where(visit_id: nil) }
after_create :async_reverse_geocode

View file

@ -26,7 +26,7 @@ class Visit < ApplicationRecord
radius = points.map { Geocoder::Calculations.distance_between(center, [_1.latitude, _1.longitude]) }.max
radius >= 15 ? radius : 15
radius && radius >= 15 ? radius : 15
end
def center

View file

@ -1,55 +1,56 @@
# frozen_string_literal: true
# This class uses Komoot's Photon API
class ReverseGeocoding::Places::FetchData
attr_reader :place
IGNORED_OSM_VALUES = %w[house residential yes detached].freeze
IGNORED_OSM_KEYS = %w[highway railway].freeze
def initialize(place_id)
@place = Place.find(place_id)
end
def call
if GOOGLE_PLACES_API_KEY.blank?
Rails.logger.warn('GOOGLE_PLACES_API_KEY is not set')
if ::PHOTON_API_HOST.blank?
Rails.logger.warn('PHOTON_API_HOST is not set')
return
end
# return if place.reverse_geocoded?
google_places = google_places_client.spots(place.latitude, place.longitude, radius: 10)
first_place = google_places.shift
first_place = reverse_geocoded_places.shift
update_place(first_place)
add_suggested_place_to_a_visit
google_places.each { |google_place| fetch_and_create_place(google_place) }
reverse_geocoded_places.each { |reverse_geocoded_place| fetch_and_create_place(reverse_geocoded_place) }
end
private
def google_places_client
@google_places_client ||= GooglePlaces::Client.new(GOOGLE_PLACES_API_KEY)
end
def update_place(reverse_geocoded_place)
return if reverse_geocoded_place.nil?
data = reverse_geocoded_place.data
def update_place(google_place)
place.update!(
name: google_place.name,
latitude: google_place.lat,
longitude: google_place.lng,
city: google_place.city,
country: google_place.country,
geodata: google_place.json_result_object,
source: :google_places,
name: place_name(data),
latitude: data['geometry']['coordinates'][1],
longitude: data['geometry']['coordinates'][0],
city: data['properties']['city'],
country: data['properties']['country'],
geodata: data,
source: Place.sources[:photon],
reverse_geocoded_at: Time.current
)
end
def fetch_and_create_place(google_place)
new_place = find_google_place(google_place)
def fetch_and_create_place(reverse_geocoded_place)
data = reverse_geocoded_place.data
new_place = find_place(data)
new_place.name = google_place.name
new_place.city = google_place.city
new_place.country = google_place.country
new_place.geodata = google_place.json_result_object
new_place.source = :google_places
new_place.name = place_name(data)
new_place.city = data['properties']['city']
new_place.country = data['properties']['country']
new_place.geodata = data
new_place.source = :photon
new_place.save!
@ -61,22 +62,45 @@ class ReverseGeocoding::Places::FetchData
end
def add_suggested_place_to_a_visit(suggested_place: place)
# 1. Find all visits that are close to the place
# 2. Add the place to the visit as a suggestion
visits = Place.near([suggested_place.latitude, suggested_place.longitude], 0.1).flat_map(&:visits)
# This is a very naive implementation, we should probably check if the place is already suggested
visits.each { |visit| visit.suggested_places << suggested_place }
visits.each do |visit|
next if visit.suggested_places.include?(suggested_place)
visit.suggested_places << suggested_place
end
end
def find_google_place(google_place)
place = Place.where("geodata ->> 'place_id' = ?", google_place['place_id']).first
def find_place(place_data)
found_place = Place.where(
"geodata->'properties'->>'osm_id' = ?", place_data['properties']['osm_id'].to_s
).first
return place if place.present?
return found_place if found_place.present?
Place.find_or_initialize_by(
latitude: google_place['geometry']['location']['lat'],
longitude: google_place['geometry']['location']['lng']
latitude: place_data['geometry']['coordinates'][1].to_f.round(5),
longitude: place_data['geometry']['coordinates'][0].to_f.round(5)
)
end
def place_name(data)
name = data.dig('properties', 'name')
type = data.dig('properties', 'osm_value')&.capitalize&.gsub('_', ' ')
address = "#{data.dig('properties', 'postcode')} #{data.dig('properties', 'street')}"
address += " #{data.dig('properties', 'housenumber')}" if data.dig('properties', 'housenumber').present?
name ||= address
"#{name} (#{type})"
end
def reverse_geocoded_places
data = Geocoder.search([place.latitude, place.longitude], limit: 10, distance_sort: true)
data.reject do |place|
place.data['properties']['osm_value'].in?(IGNORED_OSM_VALUES) ||
place.data['properties']['osm_key'].in?(IGNORED_OSM_KEYS)
end
end
end

View file

@ -6,7 +6,7 @@ class Visits::Suggest
def initialize(user, start_at:, end_at:)
@start_at = start_at.to_i
@end_at = end_at.to_i
@points = user.tracked_points.order(timestamp: :asc).where(timestamp: start_at..end_at)
@points = user.tracked_points.not_visited.order(timestamp: :asc).where(timestamp: start_at..end_at)
@user = user
end
@ -18,9 +18,9 @@ class Visits::Suggest
create_visits_notification(user)
return unless reverse_geocoding_enabled?
nil unless reverse_geocoding_enabled?
reverse_geocode(visits)
# reverse_geocode(visits)
end
private
@ -40,8 +40,7 @@ class Visits::Suggest
search_params = {
user_id: user.id,
duration: visit_data[:duration],
started_at: Time.zone.at(visit_data[:points].first.timestamp),
ended_at: Time.zone.at(visit_data[:points].last.timestamp)
started_at: Time.zone.at(visit_data[:points].first.timestamp)
}
if visit_data[:area].present?
@ -52,6 +51,7 @@ class Visits::Suggest
visit = Visit.find_or_initialize_by(search_params)
visit.name = visit_data[:place]&.name || visit_data[:area]&.name if visit.name.blank?
visit.ended_at = Time.zone.at(visit_data[:points].last.timestamp)
visit.save!
visit_data[:points].each { |point| point.update!(visit_id: visit.id) }
@ -62,12 +62,12 @@ class Visits::Suggest
end
end
def reverse_geocode(places)
places.each(&:async_reverse_geocode)
def reverse_geocode(visits)
visits.each(&:async_reverse_geocode)
end
def reverse_geocoding_enabled?
::REVERSE_GEOCODING_ENABLED && ::GOOGLE_PLACES_API_KEY.present?
::REVERSE_GEOCODING_ENABLED && ::PHOTON_API_HOST.present?
end
def create_visits_notification(user)
@ -84,8 +84,8 @@ class Visits::Suggest
def create_place(visit)
place = Place.find_or_initialize_by(
latitude: visit[:latitude],
longitude: visit[:longitude]
latitude: visit[:latitude].to_f.round(5),
longitude: visit[:longitude].to_f.round(5)
)
place.name = Place::DEFAULT_NAME

View file

@ -1,2 +1,2 @@
<%= button_to 'Confirm', visit_path(visit, 'visit[status]': :confirmed), method: :patch, data: { turbo: false }, class: 'btn btn-xs btn-success' %>
<%= button_to 'Decline', visit_path(visit, 'visit[status]': :declined), method: :patch, data: { turbo: false }, class: 'btn btn-xs btn-error mx-1' %>
<%= link_to 'Confirm', visit_path(visit, 'visit[status]': :confirmed), method: :patch, data: { turbo_method: :patch }, class: 'btn btn-xs btn-success' %>
<%= link_to 'Decline', visit_path(visit, 'visit[status]': :declined), method: :patch, data: { turbo_method: :patch }, class: 'btn btn-xs btn-error mx-1' %>

View file

@ -18,7 +18,14 @@
data-controller="visit-modal-places"
data-id="<%= visit.id %>">
<% if visit.suggested_places.any? %>
<%= select_tag :place_id, options_for_select(visit.suggested_places.map { |place| [place.name, place.id] }, visit.suggested_places.first.id), class: 'w-full', data: { action: 'change->visit-modal-places#selectPlace' } %>
<%= select_tag :place_id,
options_for_select(
visit.suggested_places.map { |place| [place.name, place.id] },
(visit.place_id || visit.suggested_places.first.id)
),
class: 'w-full select select-bordered',
data: { action: 'change->visit-modal-places#selectPlace' }
%>
<% end %>
</div>
</div>

View file

@ -4,7 +4,7 @@
<%= render 'visits/name', visit: visit %>
<div><%= "#{visit.started_at.strftime('%H:%M')} - #{visit.ended_at.strftime('%H:%M')}" %></div>
</div>
<div class="opacity-0 transition-opacity duration-300 group-hover:opacity-100 flex items-center ml-4">
<div class="opacity-0 transition-opacity duration-200 group-hover:opacity-100 flex items-center ml-4">
<%= render 'visits/buttons', visit: visit if visit.suggested? %>
<!-- The button to open modal -->
<label for="visit_details_popup_<%= visit.id %>" class='btn btn-xs btn-info'>Map</label>

View file

@ -2,4 +2,4 @@
MIN_MINUTES_SPENT_IN_CITY = ENV.fetch('MIN_MINUTES_SPENT_IN_CITY', 60).to_i
REVERSE_GEOCODING_ENABLED = ENV.fetch('REVERSE_GEOCODING_ENABLED', 'true') == 'true'
GOOGLE_PLACES_API_KEY = ''
PHOTON_API_HOST = ENV.fetch('PHOTON_API_HOST', nil)

View file

@ -13,12 +13,11 @@ config = {
expiration: 1.day # Defaults to `nil`
# prefix: "another_key:" # Defaults to `geocoder:`
},
always_raise: :all
always_raise: :all,
use_https: false,
lookup: :photon,
photon: { host: 'photon.chibi.rodeo' }
}
if GOOGLE_PLACES_API_KEY.present?
config[:lookup] = :google
config[:api_key] = GOOGLE_PLACES_API_KEY
end
Geocoder.configure(config)

View file

@ -47,7 +47,7 @@ services:
APPLICATION_HOSTS: localhost
TIME_ZONE: Europe/London
APPLICATION_PROTOCOL: http
GOOGLE_PLACES_API_KEY: ''
PHOTON_API_HOST: ''
logging:
driver: "json-file"
options:
@ -80,7 +80,7 @@ services:
APPLICATION_HOSTS: localhost
BACKGROUND_PROCESSING_CONCURRENCY: 10
APPLICATION_PROTOCOL: http
GOOGLE_PLACES_API_KEY: ''
PHOTON_API_HOST: ''
logging:
driver: "json-file"
options:

View file

@ -16,7 +16,7 @@ RSpec.describe Place, type: :model do
end
describe 'enums' do
it { is_expected.to define_enum_for(:source).with_values(%i[manual google_places]) }
it { is_expected.to define_enum_for(:source).with_values(%i[manual photon]) }
end
describe 'methods' do

View file

@ -107,10 +107,10 @@ RSpec.describe '/visits', type: :request do
expect(visit.reload.status).to eq('declined')
end
it 'redirects to the visit index page' do
it 'redirects to the visits index page' do
patch visit_url(visit), params: { visit: { status: :confirmed } }
expect(response).to redirect_to(visits_url)
expect(response).to redirect_to(visits_url(status: :suggested))
end
end
end

View file

@ -3,6 +3,28 @@
require 'rails_helper'
RSpec.describe Visits::GroupPoints do
describe '#call' do
describe '#group_points_by_radius' do
it 'groups points by radius' do
day_points = [
build(:point, latitude: 0, longitude: 0, timestamp: 1.day.ago),
build(:point, latitude: 0.00001, longitude: 0.00001, timestamp: 1.day.ago + 1.minute),
build(:point, latitude: 0.00002, longitude: 0.00002, timestamp: 1.day.ago + 2.minutes),
build(:point, latitude: 0.00003, longitude: 0.00003, timestamp: 1.day.ago + 3.minutes),
build(:point, latitude: 0.00004, longitude: 0.00004, timestamp: 1.day.ago + 4.minutes),
build(:point, latitude: 0.00005, longitude: 0.00005, timestamp: 1.day.ago + 5.minutes),
build(:point, latitude: 0.00006, longitude: 0.00006, timestamp: 1.day.ago + 6.minutes),
build(:point, latitude: 0.00007, longitude: 0.00007, timestamp: 1.day.ago + 7.minutes),
build(:point, latitude: 0.00008, longitude: 0.00008, timestamp: 1.day.ago + 8.minutes),
build(:point, latitude: 0.00009, longitude: 0.00009, timestamp: 1.day.ago + 9.minutes),
build(:point, latitude: 0.0001, longitude: 0.0009, timestamp: 1.day.ago + 9.minutes)
]
grouped_points = described_class.new(day_points).group_points_by_radius
expect(grouped_points.size).to eq(1)
expect(grouped_points.first.size).to eq(10)
# The last point is too far from the first point
expect(grouped_points.first).not_to include(day_points.last)
end
end
end

View file

@ -4,5 +4,42 @@ require 'rails_helper'
RSpec.describe Visits::Group do
describe '#call' do
let(:time_threshold_minutes) { 30 }
let(:merge_threshold_minutes) { 15 }
subject(:group) do
described_class.new(time_threshold_minutes:, merge_threshold_minutes:)
end
context 'when points are too far apart' do
it 'groups points into separate visits' do
points = [
build(:point, latitude: 0, longitude: 0, timestamp: 1.day.ago),
build(:point, latitude: 0.00001, longitude: 0.00001, timestamp: 1.day.ago + 5.minutes),
build(:point, latitude: 0.00002, longitude: 0.00002, timestamp: 1.day.ago + 10.minutes),
build(:point, latitude: 0.00003, longitude: 0.00003, timestamp: 1.day.ago + 15.minutes),
build(:point, latitude: 0.00004, longitude: 0.00004, timestamp: 1.day.ago + 20.minutes),
build(:point, latitude: 0.00005, longitude: 0.00005, timestamp: 1.day.ago + 25.minutes),
build(:point, latitude: 0.00006, longitude: 0.00006, timestamp: 1.day.ago + 30.minutes),
build(:point, latitude: 0.00007, longitude: 0.00007, timestamp: 1.day.ago + 35.minutes),
build(:point, latitude: 0.00008, longitude: 0.00008, timestamp: 1.day.ago + 40.minutes),
build(:point, latitude: 0.00009, longitude: 0.00009, timestamp: 1.day.ago + 45.minutes),
build(:point, latitude: 0.0001, longitude: 0.0001, timestamp: 1.day.ago + 50.minutes),
build(:point, latitude: 0.00011, longitude: 0.00011, timestamp: 1.day.ago + 55.minutes),
build(:point, latitude: 0.00011, longitude: 0.00011, timestamp: 1.day.ago + 95.minutes),
build(:point, latitude: 0.00011, longitude: 0.00011, timestamp: 1.day.ago + 100.minutes),
build(:point, latitude: 0.00011, longitude: 0.00011, timestamp: 1.day.ago + 105.minutes)
]
expect(group.call(points)).to \
eq({
"#{time_formatter(1.day.ago)} - #{time_formatter(1.day.ago + 55.minutes)}" => points[0..11],
"#{time_formatter(1.day.ago + 95.minutes)} - #{time_formatter(1.day.ago + 105.minutes)}" => points[12..-1]
})
end
end
end
def time_formatter(time)
Time.zone.at(time).strftime('%Y-%m-%d %H:%M')
end
end

View file

@ -4,5 +4,43 @@ require 'rails_helper'
RSpec.describe Visits::Prepare do
describe '#call' do
let(:points) do
[
build(:point, latitude: 0, longitude: 0, timestamp: 1.day.ago),
build(:point, latitude: 0.00001, longitude: 0.00001, timestamp: 1.day.ago + 5.minutes),
build(:point, latitude: 0.00002, longitude: 0.00002, timestamp: 1.day.ago + 10.minutes),
build(:point, latitude: 0.00003, longitude: 0.00003, timestamp: 1.day.ago + 15.minutes),
build(:point, latitude: 0.00004, longitude: 0.00004, timestamp: 1.day.ago + 20.minutes),
build(:point, latitude: 0.00005, longitude: 0.00005, timestamp: 1.day.ago + 25.minutes),
build(:point, latitude: 0.00006, longitude: 0.00006, timestamp: 1.day.ago + 30.minutes),
build(:point, latitude: 0.00007, longitude: 0.00007, timestamp: 1.day.ago + 35.minutes),
build(:point, latitude: 0.00008, longitude: 0.00008, timestamp: 1.day.ago + 40.minutes),
build(:point, latitude: 0.00009, longitude: 0.00009, timestamp: 1.day.ago + 45.minutes),
build(:point, latitude: 0.0001, longitude: 0.0001, timestamp: 1.day.ago + 50.minutes),
build(:point, latitude: 0.00011, longitude: 0.00011, timestamp: 1.day.ago + 55.minutes),
build(:point, latitude: 0.00011, longitude: 0.00011, timestamp: 1.day.ago + 95.minutes),
build(:point, latitude: 0.00011, longitude: 0.00011, timestamp: 1.day.ago + 100.minutes),
build(:point, latitude: 0.00011, longitude: 0.00011, timestamp: 1.day.ago + 105.minutes)
]
end
subject { described_class.new(points).call }
it 'returns correct visits' do
expect(subject).to eq [
{
date: 1.day.ago.to_date.to_s,
visits: [
{
latitude: 0.0,
longitude: 0.0,
radius: 10,
points:,
duration: 105
}
]
}
]
end
end
end

View file

@ -4,5 +4,61 @@ require 'rails_helper'
RSpec.describe Visits::Suggest do
describe '#call' do
let!(:user) { create(:user) }
let(:start_at) { 1.week.ago }
let(:end_at) { Time.current }
let!(:points) do
[
create(:point, user:, timestamp: start_at),
create(:point, user:, timestamp: start_at + 5.minutes),
create(:point, user:, timestamp: start_at + 10.minutes),
create(:point, user:, timestamp: start_at + 15.minutes),
create(:point, user:, timestamp: start_at + 20.minutes),
create(:point, user:, timestamp: start_at + 25.minutes),
create(:point, user:, timestamp: start_at + 30.minutes),
create(:point, user:, timestamp: start_at + 35.minutes),
create(:point, user:, timestamp: start_at + 40.minutes),
create(:point, user:, timestamp: start_at + 45.minutes),
create(:point, user:, timestamp: start_at + 50.minutes),
create(:point, user:, timestamp: start_at + 55.minutes),
create(:point, user:, timestamp: start_at + 95.minutes),
create(:point, user:, timestamp: start_at + 100.minutes),
create(:point, user:, timestamp: start_at + 105.minutes)
]
end
subject { described_class.new(user, start_at:, end_at:).call }
it 'creates places' do
expect { subject }.to change(Place, :count).by(1)
end
it 'creates visits' do
expect { subject }.to change(Visit, :count).by(1)
end
it 'creates visits notification' do
expect { subject }.to change(Notification, :count).by(1)
end
it 'does not reverse geocodes visits' do
expect_any_instance_of(Visit).to_not receive(:async_reverse_geocode).and_call_original
subject
end
context 'when reverse geocoding is enabled' do
before do
stub_const('REVERSE_GEOCODING_ENABLED', true)
stub_const('PHOTON_API_HOST', 'http://localhost:2323')
end
it 'reverse geocodes visits' do
expect_any_instance_of(Visit).to receive(:async_reverse_geocode).and_call_original
subject
end
end
end
end