Merge pull request #1185 from Freika/feature/store-geodata

Feature/store geodata
This commit is contained in:
Evgenii Burmakin 2025-05-16 20:12:46 +02:00 committed by GitHub
commit 52fe105230
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
64 changed files with 908 additions and 3225 deletions

1
.rspec
View file

@ -1 +1,2 @@
--require spec_helper
--profile

View file

@ -9,17 +9,26 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
## Geodata on demand
This release introduces a new environment variable `STORE_GEODATA` to control whether to store geodata in the database.
When `STORE_GEODATA` is disabled, each feature that uses geodata will now make a direct request to the geocoding service to calculate required data.
Geodata is being used:
This release introduces a new environment variable `STORE_GEODATA` with default value `true` to control whether to store geodata in the database or not. Currently, geodata is being used when:
- Fetching places geodata
- Fetching countries for a trip
- Suggesting place name for a visit
If you prefer to keep the old behavior, you can set `STORE_GEODATA` to `true`. By default, starting this release, it's set to `false`.
Opting out of storing geodata will make each feature that uses geodata to make a direct request to the geocoding service to calculate required data instead of using existing geodata from the database. Setting `STORE_GEODATA` to `false` can also use you some database space.
If you decide to opt out, you can safely delete your existing geodata from the database:
1. Get into the [console](https://dawarich.app/docs/FAQ/#how-to-enter-dawarich-console)
2. Run the following commands:
```ruby
Point.update_all(geodata: {}) # to remove existing geodata
ActiveRecord::Base.connection.execute("VACUUM FULL") # to free up some space
```
Note, that this will take some time to complete, depending on the number of points you have. This is not a required step.
If you're running your own Photon instance, you can safely set `STORE_GEODATA` to `false`, otherwise it'd be better to keep it enabled, because that way Dawarich will be using existing geodata for its calculations.
@ -37,6 +46,7 @@ If you're running your own Photon instance, you can safely set `STORE_GEODATA` t
- Fixed a bug with an attempt to write points with same lonlat and timestamp from iOS app. #1170
- Importing GeoJSON files now saves velocity if it was stored in either `velocity` or `speed` property.
- `rake points:migrate_to_lonlat` should work properly now. #1083 #1161
- PostGIS extension is now being enabled only if it's not already enabled. #1186
# 0.26.0 - 2025-05-08

File diff suppressed because one or more lines are too long

View file

@ -3,7 +3,7 @@
class Api::V1::Countries::BordersController < ApplicationController
def index
countries = Rails.cache.fetch('dawarich/countries_codes', expires_in: 1.day) do
Oj.load(File.read(Rails.root.join('lib/assets/countries.json')))
Oj.load(File.read(Rails.root.join('lib/assets/countries.geojson')))
end
render json: countries

View file

@ -2,6 +2,7 @@
class Api::V1::Overland::BatchesController < ApiController
before_action :authenticate_active_api_user!, only: %i[create]
before_action :validate_points_limit, only: %i[create]
def create
Overland::BatchCreatingJob.perform_later(batch_params, current_api_user.id)

View file

@ -2,6 +2,7 @@
class Api::V1::Owntracks::PointsController < ApiController
before_action :authenticate_active_api_user!, only: %i[create]
before_action :validate_points_limit, only: %i[create]
def create
Owntracks::PointCreatingJob.perform_later(point_params, current_api_user.id)

View file

@ -2,6 +2,7 @@
class Api::V1::PointsController < ApiController
before_action :authenticate_active_api_user!, only: %i[create update destroy]
before_action :validate_points_limit, only: %i[create]
def index
start_at = params[:start_at]&.to_datetime&.to_i

View file

@ -2,6 +2,7 @@
class Api::V1::SubscriptionsController < ApiController
skip_before_action :authenticate_api_key, only: %i[callback]
def callback
decoded_token = Subscription::DecodeJwtToken.new(params[:token]).call

View file

@ -41,4 +41,10 @@ class ApiController < ApplicationController
def required_params
[]
end
def validate_points_limit
limit_exceeded = PointsLimitExceeded.new(current_api_user).call
render json: { error: 'Points limit exceeded' }, status: :unauthorized if limit_exceeded
end
end

View file

@ -6,7 +6,7 @@ class ImportsController < ApplicationController
before_action :authenticate_user!
before_action :authenticate_active_user!, only: %i[new create]
before_action :set_import, only: %i[show edit update destroy]
before_action :validate_points_limit, only: %i[new create]
def index
@imports =
current_user
@ -102,4 +102,10 @@ class ImportsController < ApplicationController
import
end
def validate_points_limit
limit_exceeded = PointsLimitExceeded.new(current_user).call
redirect_to imports_path, alert: 'Points limit exceeded', status: :unprocessable_entity if limit_exceeded
end
end

View file

@ -15,6 +15,10 @@ class TripsController < ApplicationController
@trip.photo_previews
end
@photo_sources = @trip.photo_sources
if @trip.path.blank? || @trip.distance.blank? || @trip.visited_countries.blank?
Trips::CalculateAllJob.perform_later(@trip.id)
end
end
def new
@ -28,7 +32,7 @@ class TripsController < ApplicationController
@trip = current_user.trips.build(trip_params)
if @trip.save
redirect_to @trip, notice: 'Trip was successfully created.'
redirect_to @trip, notice: 'Trip was successfully created. Data is being calculated in the background.'
else
render :new, status: :unprocessable_entity
end

View file

@ -391,7 +391,7 @@ export default class extends BaseController {
const visitedCountries = this.getVisitedCountries(countryCodesMap)
const filteredFeatures = worldData.features.filter(feature =>
visitedCountries.includes(feature.properties.ISO_A2)
visitedCountries.includes(feature.properties["ISO3166-1-Alpha-2"])
)
this.scratchLayer.addData({

View file

@ -3,6 +3,7 @@
import BaseController from "./base_controller"
import L from "leaflet"
import { createAllMapLayers } from "../maps/layers"
export default class extends BaseController {
static values = {
@ -31,11 +32,13 @@ export default class extends BaseController {
attributionControl: true
})
// Add the tile layer
L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', {
maxZoom: 19,
attribution: "&copy; <a href='http://www.openstreetmap.org/copyright'>OpenStreetMap</a>"
}).addTo(this.map)
// Add base map layer
const selectedLayerName = this.hasUserSettingsValue ?
this.userSettingsValue.preferred_map_layer || "OpenStreetMap" :
"OpenStreetMap";
const maps = this.baseMaps();
const defaultLayer = maps[selectedLayerName] || Object.values(maps)[0];
defaultLayer.addTo(this.map);
// If we have coordinates, show the route
if (this.hasPathValue && this.pathValue) {
@ -45,8 +48,39 @@ export default class extends BaseController {
}
}
baseMaps() {
const selectedLayerName = this.hasUserSettingsValue ?
this.userSettingsValue.preferred_map_layer || "OpenStreetMap" :
"OpenStreetMap";
let maps = createAllMapLayers(this.map, selectedLayerName);
// Add custom map if it exists in settings
if (this.hasUserSettingsValue && this.userSettingsValue.maps && this.userSettingsValue.maps.url) {
const customLayer = L.tileLayer(this.userSettingsValue.maps.url, {
maxZoom: 19,
attribution: "&copy; OpenStreetMap contributors"
});
// If this is the preferred layer, add it to the map immediately
if (selectedLayerName === this.userSettingsValue.maps.name) {
customLayer.addTo(this.map);
// Remove any other base layers that might be active
Object.values(maps).forEach(layer => {
if (this.map.hasLayer(layer)) {
this.map.removeLayer(layer);
}
});
}
maps[this.userSettingsValue.maps.name] = customLayer;
}
return maps;
}
showRoute() {
const points = this.parseLineString(this.pathValue)
const points = this.getCoordinates(this.pathValue)
// Only create polyline if we have points
if (points.length > 0) {
@ -69,37 +103,34 @@ export default class extends BaseController {
}
}
parseLineString(linestring) {
getCoordinates(pathData) {
try {
// Remove 'LINESTRING (' from start and ')' from end
const coordsString = linestring
.replace(/LINESTRING\s*\(/, '') // Remove LINESTRING and opening parenthesis
.replace(/\)$/, '') // Remove closing parenthesis
.trim() // Remove any leading/trailing whitespace
// Parse the path data if it's a string
let coordinates = pathData;
if (typeof pathData === 'string') {
try {
coordinates = JSON.parse(pathData);
} catch (e) {
console.error("Error parsing path data as JSON:", e);
return [];
}
}
// Split into coordinate pairs and parse
const points = coordsString.split(',').map(pair => {
// Clean up any extra whitespace and remove any special characters
const cleanPair = pair.trim().replace(/[()"\s]+/g, ' ')
const [lng, lat] = cleanPair.split(' ').filter(Boolean).map(Number)
// Handle array format - convert from [lng, lat] to [lat, lng] for Leaflet
return coordinates.map(coord => {
const [lng, lat] = coord;
// Validate the coordinates
if (isNaN(lat) || isNaN(lng) || !lat || !lng) {
console.error("Invalid coordinates:", cleanPair)
return null
console.error("Invalid coordinates:", coord);
return null;
}
return [lat, lng] // Leaflet uses [lat, lng] order
}).filter(point => point !== null) // Remove any invalid points
// Validate we have points before returning
if (points.length === 0) {
return []
}
return points
return [lat, lng]; // Leaflet uses [lat, lng] order
}).filter(point => point !== null);
} catch (error) {
return []
console.error("Error processing coordinates:", error);
return [];
}
}

View file

@ -133,22 +133,31 @@ export default class extends BaseController {
// After map initialization, add the path if it exists
if (this.containerTarget.dataset.path) {
const pathData = this.containerTarget.dataset.path.replace(/^"|"$/g, ''); // Remove surrounding quotes
const coordinates = this.parseLineString(pathData);
try {
let coordinates;
const pathData = this.containerTarget.dataset.path.replace(/^"|"$/g, ''); // Remove surrounding quotes
const polyline = L.polyline(coordinates, {
color: 'blue',
opacity: 0.8,
weight: 3,
zIndexOffset: 400
});
// Try to parse as JSON first (new format)
coordinates = JSON.parse(pathData);
// Convert from [lng, lat] to [lat, lng] for Leaflet
coordinates = coordinates.map(coord => [coord[1], coord[0]]);
polyline.addTo(this.polylinesLayer);
this.polylinesLayer.addTo(this.map);
const polyline = L.polyline(coordinates, {
color: 'blue',
opacity: 0.8,
weight: 3,
zIndexOffset: 400
});
// Fit the map to the polyline bounds
if (coordinates.length > 0) {
this.map.fitBounds(polyline.getBounds(), { padding: [50, 50] });
polyline.addTo(this.polylinesLayer);
this.polylinesLayer.addTo(this.map);
// Fit the map to the polyline bounds
if (coordinates.length > 0) {
this.map.fitBounds(polyline.getBounds(), { padding: [50, 50] });
}
} catch (error) {
console.error("Error processing path data:", error);
}
}
}
@ -246,17 +255,4 @@ export default class extends BaseController {
this.fitMapToBounds()
}
}
// Add this method to parse the LineString format
parseLineString(lineString) {
// Remove LINESTRING and parentheses, then split into coordinate pairs
const coordsString = lineString.replace('LINESTRING (', '').replace(')', '');
const coords = coordsString.split(', ');
// Convert each coordinate pair to [lat, lng] format
return coords.map(coord => {
const [lng, lat] = coord.split(' ').map(Number);
return [lat, lng]; // Swap to lat, lng for Leaflet
});
}
}

View file

@ -0,0 +1,11 @@
# frozen_string_literal: true
class DataMigrations::SetPointsCountryIdsJob < ApplicationJob
queue_as :default
def perform(point_id)
point = Point.find(point_id)
point.country_id = Country.containing_point(point.lon, point.lat).id
point.save!
end
end

View file

@ -0,0 +1,11 @@
# frozen_string_literal: true
class DataMigrations::StartSettingsPointsCountryIdsJob < ApplicationJob
queue_as :default
def perform
Point.where(country_id: nil).find_each do |point|
DataMigrations::SetPointsCountryIdsJob.perform_later(point.id)
end
end
end

View file

@ -0,0 +1,11 @@
# frozen_string_literal: true
class Trips::CalculateAllJob < ApplicationJob
queue_as :default
def perform(trip_id)
Trips::CalculatePathJob.perform_later(trip_id)
Trips::CalculateDistanceJob.perform_later(trip_id)
Trips::CalculateCountriesJob.perform_later(trip_id)
end
end

View file

@ -0,0 +1,25 @@
# frozen_string_literal: true
class Trips::CalculateCountriesJob < ApplicationJob
queue_as :default
def perform(trip_id)
trip = Trip.find(trip_id)
trip.calculate_countries
trip.save!
broadcast_update(trip)
end
private
def broadcast_update(trip)
Turbo::StreamsChannel.broadcast_update_to(
"trip_#{trip.id}",
target: "trip_countries",
partial: "trips/countries",
locals: { trip: trip }
)
end
end

View file

@ -0,0 +1,25 @@
# frozen_string_literal: true
class Trips::CalculateDistanceJob < ApplicationJob
queue_as :default
def perform(trip_id)
trip = Trip.find(trip_id)
trip.calculate_distance
trip.save!
broadcast_update(trip)
end
private
def broadcast_update(trip)
Turbo::StreamsChannel.broadcast_update_to(
"trip_#{trip.id}",
target: "trip_distance",
partial: "trips/distance",
locals: { trip: trip }
)
end
end

View file

@ -0,0 +1,25 @@
# frozen_string_literal: true
class Trips::CalculatePathJob < ApplicationJob
queue_as :default
def perform(trip_id)
trip = Trip.find(trip_id)
trip.calculate_path
trip.save!
broadcast_update(trip)
end
private
def broadcast_update(trip)
Turbo::StreamsChannel.broadcast_update_to(
"trip_#{trip.id}",
target: "trip_path",
partial: "trips/path",
locals: { trip: trip }
)
end
end

View file

@ -1,13 +0,0 @@
# frozen_string_literal: true
class Trips::CreatePathJob < ApplicationJob
queue_as :default
def perform(trip_id)
trip = Trip.find(trip_id)
trip.calculate_trip_data
trip.save!
end
end

11
app/models/country.rb Normal file
View file

@ -0,0 +1,11 @@
# frozen_string_literal: true
class Country < ApplicationRecord
validates :name, :iso_a2, :iso_a3, :geom, presence: true
def self.containing_point(lon, lat)
where("ST_Contains(geom, ST_SetSRID(ST_MakePoint(?, ?), 4326))", lon, lat)
.select(:id, :name, :iso_a2, :iso_a3)
.first
end
end

View file

@ -29,6 +29,7 @@ class Point < ApplicationRecord
scope :not_visited, -> { where(visit_id: nil) }
after_create :async_reverse_geocode, if: -> { DawarichSettings.store_geodata? }
after_create :set_country
after_create_commit :broadcast_coordinates
def self.without_raw_data
@ -57,6 +58,10 @@ class Point < ApplicationRecord
lonlat.y
end
def found_in_country
Country.containing_point(lon, lat)
end
private
# rubocop:disable Metrics/MethodLength Metrics/AbcSize
@ -76,4 +81,9 @@ class Point < ApplicationRecord
)
end
# rubocop:enable Metrics/MethodLength
def set_country
self.country_id = found_in_country&.id
save! if changed?
end
end

View file

@ -7,12 +7,11 @@ class Trip < ApplicationRecord
validates :name, :started_at, :ended_at, presence: true
before_save :calculate_trip_data
after_create :enqueue_calculation_jobs
after_update :enqueue_calculation_jobs, if: -> { saved_change_to_started_at? || saved_change_to_ended_at? }
def calculate_trip_data
calculate_path
calculate_distance
calculate_countries
def enqueue_calculation_jobs
Trips::CalculateAllJob.perform_later(id)
end
def points
@ -33,6 +32,25 @@ class Trip < ApplicationRecord
@photo_sources ||= photos.map { _1[:source] }.uniq
end
def calculate_path
trip_path = Tracks::BuildPath.new(points.pluck(:lonlat)).call
self.path = trip_path
end
def calculate_distance
distance = Point.total_distance(points, DISTANCE_UNIT)
self.distance = distance.round
end
def calculate_countries
countries =
Country.where(id: points.pluck(:country_id).compact.uniq).pluck(:name)
self.visited_countries = countries
end
private
def photos
@ -47,22 +65,4 @@ class Trip < ApplicationRecord
# to show all photos in the same height
vertical_photos.count > horizontal_photos.count ? vertical_photos : horizontal_photos
end
def calculate_path
trip_path = Tracks::BuildPath.new(points.pluck(:lonlat)).call
self.path = trip_path
end
def calculate_distance
distance = Point.total_distance(points, DISTANCE_UNIT)
self.distance = distance.round
end
def calculate_countries
countries = Trips::Countries.new(self).call
self.visited_countries = countries
end
end

View file

@ -1,7 +1,7 @@
# frozen_string_literal: true
class Api::PointSerializer < PointSerializer
EXCLUDED_ATTRIBUTES = %w[created_at updated_at visit_id import_id user_id raw_data].freeze
EXCLUDED_ATTRIBUTES = %w[created_at updated_at visit_id import_id user_id raw_data country_id].freeze
def call
point.attributes.except(*EXCLUDED_ATTRIBUTES)

View file

@ -3,7 +3,7 @@
class PointSerializer
EXCLUDED_ATTRIBUTES = %w[
created_at updated_at visit_id id import_id user_id raw_data lonlat
reverse_geocoded_at
reverse_geocoded_at country_id
].freeze
def initialize(point)

View file

@ -0,0 +1,20 @@
# frozen_string_literal: true
class PointsLimitExceeded
def initialize(user)
@user = user
end
def call
return false if DawarichSettings.self_hosted?
return true if @user.points.count >= points_limit
false
end
private
def points_limit
DawarichSettings::BASIC_PAID_PLAN_LIMIT
end
end

View file

@ -56,7 +56,7 @@ class ReverseGeocoding::Places::FetchData
new_place.name = place_name(data)
new_place.city = data['properties']['city']
new_place.country = data['properties']['country']
new_place.country = data['properties']['country'] # TODO: Use country id
new_place.geodata = data
new_place.source = :photon
if new_place.lonlat.blank?

View file

@ -1,92 +0,0 @@
# frozen_string_literal: true
class Trips::Countries
FILE_PATH = Rails.root.join('lib/assets/countries.json')
def initialize(trip, batch_count = 2)
@trip = trip
@batch_count = batch_count
@factory = RGeo::Geographic.spherical_factory
@file = File.read(FILE_PATH)
@countries_features =
RGeo::GeoJSON.decode(@file, json_parser: :json, geo_factory: @factory)
end
def call
all_points = @trip.points.to_a
total_points = all_points.size
# Return empty hash if no points
return {} if total_points.zero?
batches = split_into_batches(all_points, @batch_count)
threads_results = process_batches_in_threads(batches, total_points)
merge_thread_results(threads_results).uniq.compact
end
private
def split_into_batches(points, batch_count)
batch_count = [batch_count, 1].max # Ensure batch_count is at least 1
batch_size = (points.size / batch_count.to_f).ceil
points.each_slice(batch_size).to_a
end
def process_batches_in_threads(batches, total_points)
threads_results = []
threads = []
batches.each do |batch|
threads << Thread.new do
threads_results << process_batch(batch)
end
end
threads.each(&:join)
threads_results
end
def merge_thread_results(threads_results)
countries = []
threads_results.each do |result|
countries.concat(result)
end
countries
end
def process_batch(points)
points.map do |point|
country_code = geocode_point(point)
next unless country_code
country_code
end
end
def geocode_point(point)
lonlat = point.lonlat
return nil unless lonlat
latitude = lonlat.y
longitude = lonlat.x
fetch_country_code(latitude, longitude)
end
def fetch_country_code(latitude, longitude)
results = Geocoder.search([latitude, longitude], limit: 1)
return nil unless results.any?
result = results.first
result.data['properties']['countrycode']
rescue StandardError => e
Rails.logger.error("Error geocoding point: #{e.message}")
ExceptionReporter.call(e)
nil
end
end

View file

@ -6,11 +6,21 @@
<h3 class='text-xl font-bold mt-4'>Usage examples</h3>
<h3 class='text-lg font-bold mt-4'>Dawarich iOS app</h3>
<p>Provide your instance URL:</p>
<p class='mb-2'><code><%= root_url %></code></p>
<p>And provide your API key:</p>
<p><code><%= current_user.api_key %></code></p>
<div class='divider'>OR</div>
<h3 class='text-lg font-bold mt-4'>OwnTracks</h3>
<p><code><%= api_v1_owntracks_points_url(api_key: current_user.api_key) %></code></p>
<div class='divider'>OR</div>
<h3 class='text-lg font-bold mt-4'>Overland</h3>
<p><code><%= api_v1_overland_batches_url(api_key: current_user.api_key) %></code></p>
</p>
<p class='py-2'>
<%= link_to "Generate new API key", generate_api_key_path, data: { confirm: "Are you sure? This will invalidate the current API key.", turbo_confirm: "Are you sure?", turbo_method: :post }, class: 'btn btn-primary' %>

View file

@ -0,0 +1,6 @@
<p class="py-6">
<p class='py-2'>
You have used <%= number_with_delimiter(current_user.points.count) %> points of <%= number_with_delimiter(DawarichSettings::BASIC_PAID_PLAN_LIMIT) %> available.
</p>
<progress class="progress progress-primary w-1/2 h-5" value="<%= current_user.points.count %>" max="<%= DawarichSettings::BASIC_PAID_PLAN_LIMIT %>"></progress>
</p>

View file

@ -1,10 +1,13 @@
<% content_for :title, 'Account' %>
<div class="hero min-h-content bg-base-200">
<div class="hero-content flex-col lg:flex-row-reverse w-full my-10">
<div class="hero-content flex-col lg:flex-row-reverse w-full my-5">
<div class="text-center lg:text-left">
<h1 class="text-5xl font-bold">Edit your account!</h1>
<h1 class="text-5xl font-bold mb-5">Edit your account!</h1>
<%= render 'devise/registrations/api_key' %>
<% if !DawarichSettings.self_hosted? %>
<%= render 'devise/registrations/points_usage' %>
<% end %>
</div>
<div class="card flex-shrink-0 w-full max-w-sm shadow-2xl bg-base-100 px-5 py-5">
<%= form_for(resource, as: resource_name, url: registration_path(resource_name), class: 'form-body', method: :put, data: { turbo_method: :put, turbo: false }) do |f| %>

View file

@ -0,0 +1,14 @@
<% if trip.countries.any? %>
<p class="text-md text-base-content/60">
<%= "#{trip.countries.join(', ')} (#{trip.distance} #{DISTANCE_UNIT})" %>
</p>
<% elsif trip.visited_countries.present? %>
<p class="text-md text-base-content/60">
<%= "#{trip.visited_countries.join(', ')} (#{trip.distance} #{DISTANCE_UNIT})" %>
</p>
<% else %>
<p class="text-md text-base-content/60">
<span>Countries are being calculated...</span>
<span class="loading loading-dots loading-sm"></span>
</p>
<% end %>

View file

@ -0,0 +1,6 @@
<% if trip.distance.present? %>
<span class="text-md"><%= trip.distance %> <%= DISTANCE_UNIT %></span>
<% else %>
<span class="text-md">Calculating...</span>
<span class="loading loading-dots loading-sm"></span>
<% end %>

View file

@ -0,0 +1,24 @@
<% if trip.path.present? %>
<div
id='map'
class="w-full h-full rounded-lg z-0"
data-controller="trips"
data-trips-target="container"
data-distance_unit="<%= DISTANCE_UNIT %>"
data-api_key="<%= current_user.api_key %>"
data-user_settings="<%= current_user.settings.to_json %>"
data-path="<%= trip.path.coordinates.to_json %>"
data-started_at="<%= trip.started_at %>"
data-ended_at="<%= trip.ended_at %>"
data-timezone="<%= Rails.configuration.time_zone %>">
<div data-trips-target="container" class="h-[25rem] w-full min-h-screen">
</div>
</div>
<% else %>
<div class="flex items-center justify-center h-full">
<div class="text-center">
<p class="text-base-content/60">Trip path is being calculated...</p>
<div class="loading loading-spinner loading-lg mt-4"></div>
</div>
</div>
<% end %>

View file

@ -13,7 +13,7 @@
class="rounded-lg z-0"
data-controller="trip-map"
data-trip-map-trip-id-value="<%= trip.id %>"
data-trip-map-path-value="<%= trip.path.to_json %>"
data-trip-map-path-value="<%= trip.path.coordinates.to_json %>"
data-trip-map-api-key-value="<%= current_user.api_key %>"
data-trip-map-user-settings-value="<%= current_user.settings.to_json %>"
data-trip-map-timezone-value="<%= Rails.configuration.time_zone %>"

View file

@ -1,36 +1,31 @@
<% content_for :title, @trip.name %>
<%= turbo_stream_from "trip_#{@trip.id}" %>
<div class="container mx-auto px-4 max-w-4xl my-5">
<div class="text-center mb-8">
<h1 class="text-4xl font-bold mb-2"><%= @trip.name %></h1>
<p class="text-md text-base-content/60">
<%= human_date(@trip.started_at) %> - <%= human_date(@trip.ended_at) %>
</p>
<% if @trip.countries.any? %>
<p class="text-lg text-base-content/60">
<%= "#{@trip.countries.join(', ')} (#{@trip.distance} #{DISTANCE_UNIT})" %>
</p>
<% if @trip.countries.any? || @trip.visited_countries.present? %>
<div id="trip_countries">
<%= render "trips/countries", trip: @trip %>
</div>
<% else %>
<div id="trip_countries">
<p class="text-md text-base-content/60">
<span>Countries are being calculated...</span>
<span class="loading loading-dots loading-sm"></span>
</p>
</div>
<% end %>
</div>
<div class="bg-base-100 my-8 p-4">
<div class="grid grid-cols-1 md:grid-cols-2 gap-6">
<div class="w-full">
<div
id='map'
class="w-full h-full rounded-lg z-0"
data-controller="trips"
data-trips-target="container"
data-distance_unit="<%= DISTANCE_UNIT %>"
data-api_key="<%= current_user.api_key %>"
data-user_settings="<%= current_user.settings.to_json %>"
data-path="<%= @trip.path.to_json %>"
data-started_at="<%= @trip.started_at %>"
data-ended_at="<%= @trip.ended_at %>"
data-timezone="<%= Rails.configuration.time_zone %>">
<div data-trips-target="container" class="h-[25rem] w-full min-h-screen">
</div>
</div>
<div class="w-full" id="trip_path">
<%= render "trips/path", trip: @trip, current_user: current_user %>
</div>
<div class="w-full">
<div>

View file

@ -17,7 +17,7 @@ NOMINATIM_API_KEY = ENV.fetch('NOMINATIM_API_KEY', nil)
NOMINATIM_API_USE_HTTPS = ENV.fetch('NOMINATIM_API_USE_HTTPS', 'true') == 'true'
GEOAPIFY_API_KEY = ENV.fetch('GEOAPIFY_API_KEY', nil)
STORE_GEODATA = ENV.fetch('STORE_GEODATA', 'false') == 'true'
STORE_GEODATA = ENV.fetch('STORE_GEODATA', 'true') == 'true'
# /Reverse geocoding settings
SENTRY_DSN = ENV.fetch('SENTRY_DSN', nil)

View file

@ -1,7 +1,10 @@
# frozen_string_literal: true
class DawarichSettings
BASIC_PAID_PLAN_LIMIT = 10_000_000 # 10 million points
class << self
def reverse_geocoding_enabled?
@reverse_geocoding_enabled ||= photon_enabled? || geoapify_enabled? || nominatim_enabled?
end

View file

@ -3,7 +3,7 @@
class CreatePathsForTrips < ActiveRecord::Migration[8.0]
def up
Trip.find_each do |trip|
Trips::CreatePathJob.perform_later(trip.id)
Trips::CalculatePathJob.perform_later(trip.id)
end
end

View file

@ -0,0 +1,11 @@
# frozen_string_literal: true
class SetPointsCountryIds < ActiveRecord::Migration[8.0]
def up
DataMigrations::StartSettingsPointsCountryIdsJob.perform_later
end
def down
raise ActiveRecord::IrreversibleMigration
end
end

View file

@ -1 +1 @@
DataMigrate::Data.define(version: 20_250_404_182_629)
DataMigrate::Data.define(version: 20250516181033)

View file

@ -2,6 +2,6 @@
class EnablePostgisExtension < ActiveRecord::Migration[8.0]
def change
enable_extension 'postgis'
enable_extension 'postgis' unless extension_enabled?('postgis')
end
end

View file

@ -0,0 +1,19 @@
# frozen_string_literal: true
class CreateCountries < ActiveRecord::Migration[8.0]
def change
create_table :countries do |t|
t.string :name, null: false
t.string :iso_a2, null: false
t.string :iso_a3, null: false
t.multi_polygon :geom, srid: 4326
t.timestamps
end
add_index :countries, :name
add_index :countries, :iso_a2
add_index :countries, :iso_a3
add_index :countries, :geom, using: :gist
end
end

View file

@ -0,0 +1,9 @@
# frozen_string_literal: true
class AddCountryIdToPoints < ActiveRecord::Migration[8.0]
disable_ddl_transaction!
def change
add_reference :points, :country, index: { algorithm: :concurrently }
end
end

17
db/schema.rb generated
View file

@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema[8.0].define(version: 2025_05_13_164521) do
ActiveRecord::Schema[8.0].define(version: 2025_05_15_192211) do
# These are extensions that must be enabled in order to support this database
enable_extension "pg_catalog.plpgsql"
enable_extension "postgis"
@ -64,6 +64,19 @@ ActiveRecord::Schema[8.0].define(version: 2025_05_13_164521) do
t.index ["user_id"], name: "index_areas_on_user_id"
end
create_table "countries", force: :cascade do |t|
t.string "name", null: false
t.string "iso_a2", null: false
t.string "iso_a3", null: false
t.geometry "geom", limit: {srid: 4326, type: "multi_polygon"}
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.index ["geom"], name: "index_countries_on_geom", using: :gist
t.index ["iso_a2"], name: "index_countries_on_iso_a2"
t.index ["iso_a3"], name: "index_countries_on_iso_a3"
t.index ["name"], name: "index_countries_on_name"
end
create_table "data_migrations", primary_key: "version", id: :string, force: :cascade do |t|
end
@ -166,12 +179,14 @@ ActiveRecord::Schema[8.0].define(version: 2025_05_13_164521) do
t.decimal "course_accuracy", precision: 8, scale: 5
t.string "external_track_id"
t.geography "lonlat", limit: {srid: 4326, type: "st_point", geographic: true}
t.bigint "country_id"
t.index ["altitude"], name: "index_points_on_altitude"
t.index ["battery"], name: "index_points_on_battery"
t.index ["battery_status"], name: "index_points_on_battery_status"
t.index ["city"], name: "index_points_on_city"
t.index ["connection"], name: "index_points_on_connection"
t.index ["country"], name: "index_points_on_country"
t.index ["country_id"], name: "index_points_on_country_id"
t.index ["external_track_id"], name: "index_points_on_external_track_id"
t.index ["geodata"], name: "index_points_on_geodata", using: :gin
t.index ["import_id"], name: "index_points_on_import_id"

View file

@ -1,14 +1,36 @@
# frozen_string_literal: true
return if User.any?
if User.none?
puts 'Creating user...'
puts 'Creating user...'
User.create!(
email: 'demo@dawarich.app',
password: 'password',
password_confirmation: 'password',
admin: true
)
User.create!(
email: 'demo@dawarich.app',
password: 'password',
password_confirmation: 'password',
admin: true
)
puts "User created: #{User.first.email} / password: 'password'"
end
puts "User created: #{User.first.email} / password: 'password'"
if Country.none?
puts 'Creating countries...'
countries_json = Oj.load(File.read(Rails.root.join('lib/assets/countries.geojson')))
factory = RGeo::Geos.factory(srid: 4326)
countries_multi_polygon = RGeo::GeoJSON.decode(countries_json.to_json, geo_factory: factory)
ActiveRecord::Base.transaction do
countries_multi_polygon.each do |country, index|
p "Creating #{country.properties['name']}..."
Country.create!(
name: country.properties['name'],
iso_a2: country.properties['ISO3166-1-Alpha-2'],
iso_a3: country.properties['ISO3166-1-Alpha-3'],
geom: country.geometry
)
end
end
end

View file

@ -25,6 +25,7 @@ RUN apk -U add --no-cache \
less \
yaml-dev \
gcompat \
libgeos-dev \
&& mkdir -p $APP_PATH
# Update gem system and install bundler

View file

@ -22,6 +22,7 @@ RUN apk -U add --no-cache \
less \
yaml-dev \
gcompat \
libgeos-dev \
&& mkdir -p $APP_PATH
# Update gem system and install bundler

View file

@ -70,7 +70,7 @@ services:
PROMETHEUS_EXPORTER_HOST: 0.0.0.0
PROMETHEUS_EXPORTER_PORT: 9394
SELF_HOSTED: "true"
STORE_GEODATA: "false"
STORE_GEODATA: "true"
logging:
driver: "json-file"
options:
@ -123,7 +123,7 @@ services:
PROMETHEUS_EXPORTER_HOST: dawarich_app
PROMETHEUS_EXPORTER_PORT: 9394
SELF_HOSTED: "true"
STORE_GEODATA: "false"
STORE_GEODATA: "true"
logging:
driver: "json-file"
options:

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,7 @@
{
"webcredentials": {
"apps": [
"2A275P77DQ.app.dawarich.Dawarich"
]
}
}

View file

@ -0,0 +1,10 @@
FactoryBot.define do
factory :country do
name { "Serranilla Bank" }
iso_a2 { "SB" }
iso_a3 { "SBX" }
geom {
"MULTIPOLYGON (((-78.637074 15.862087, -78.640411 15.864, -78.636871 15.867296, -78.637074 15.862087)))"
}
end
end

View file

@ -31,6 +31,7 @@ FactoryBot.define do
external_track_id { nil }
lonlat { "POINT(#{FFaker::Geolocation.lng} #{FFaker::Geolocation.lat})" }
user
country_id { nil }
trait :with_known_location do
lonlat { 'POINT(37.6173 55.755826)' }

View file

@ -0,0 +1,28 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe DataMigrations::SetPointsCountryIdsJob, type: :job do
describe '#perform' do
let(:point) { create(:point, lonlat: 'POINT(10.0 20.0)', country_id: nil) }
let(:country) { create(:country) }
before do
allow(Country).to receive(:containing_point)
.with(point.lon, point.lat)
.and_return(country)
end
it 'updates the point with the correct country_id' do
described_class.perform_now(point.id)
expect(point.reload.country_id).to eq(country.id)
end
end
describe 'queue' do
it 'uses the default queue' do
expect(described_class.queue_name).to eq('default')
end
end
end

View file

@ -0,0 +1,33 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe DataMigrations::StartSettingsPointsCountryIdsJob, type: :job do
describe '#perform' do
let!(:point_with_country) { create(:point, country_id: 1) }
let!(:point_without_country1) { create(:point, country_id: nil) }
let!(:point_without_country2) { create(:point, country_id: nil) }
it 'enqueues SetPointsCountryIdsJob for points without country_id' do
expect { described_class.perform_now }.to \
have_enqueued_job(DataMigrations::SetPointsCountryIdsJob)
.with(point_without_country1.id)
.and have_enqueued_job(DataMigrations::SetPointsCountryIdsJob)
.with(point_without_country2.id)
end
it 'does not enqueue jobs for points with country_id' do
point_with_country.update(country_id: 1)
expect { described_class.perform_now }.not_to \
have_enqueued_job(DataMigrations::SetPointsCountryIdsJob)
.with(point_with_country.id)
end
end
describe 'queue' do
it 'uses the default queue' do
expect(described_class.queue_name).to eq('default')
end
end
end

View file

@ -1,23 +0,0 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe Trips::CreatePathJob, type: :job do
let!(:trip) { create(:trip, :with_points) }
let(:points) { trip.points }
let(:trip_path) do
"LINESTRING (#{points.map do |point|
"#{point.lon.to_f.round(5)} #{point.lat.to_f.round(5)}"
end.join(', ')})"
end
before do
trip.update(path: nil, distance: nil)
end
it 'creates a path for a trip' do
described_class.perform_now(trip.id)
expect(trip.reload.path.to_s).to eq(trip_path)
end
end

View file

@ -0,0 +1,12 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe Country, type: :model do
describe 'validations' do
it { is_expected.to validate_presence_of(:name) }
it { is_expected.to validate_presence_of(:iso_a2) }
it { is_expected.to validate_presence_of(:iso_a3) }
it { is_expected.to validate_presence_of(:geom) }
end
end

View file

@ -13,6 +13,21 @@ RSpec.describe Point, type: :model do
it { is_expected.to validate_presence_of(:lonlat) }
end
describe 'callbacks' do
describe '#set_country' do
let(:point) { build(:point, lonlat: 'POINT(-79.85581250721961 15.854775993302411)') }
let(:country) { create(:country) }
it 'sets the country' do
expect(Country).to receive(:containing_point).with(-79.85581250721961, 15.854775993302411).and_return(country)
point.save!
expect(point.country_id).to eq(country.id)
end
end
end
describe 'scopes' do
describe '.reverse_geocoded' do
let(:point) { create(:point, :reverse_geocoded) }

View file

@ -3,10 +3,6 @@
require 'rails_helper'
RSpec.describe Trip, type: :model do
before do
allow_any_instance_of(Trips::Countries).to receive(:call).and_return([])
end
describe 'validations' do
it { is_expected.to validate_presence_of(:name) }
it { is_expected.to validate_presence_of(:started_at) }
@ -20,17 +16,18 @@ RSpec.describe Trip, type: :model do
describe 'callbacks' do
let(:user) { create(:user) }
let(:trip) { create(:trip, :with_points, user:) }
let(:calculated_distance) { trip.send(:calculate_distance) }
it 'sets the distance' do
expect(trip.distance).to eq(calculated_distance)
context 'when the trip is created' do
let(:trip) { build(:trip, :with_points, user:) }
it 'enqueues the calculation jobs' do
expect(Trips::CalculateAllJob).to receive(:perform_later)
trip.save
end
end
it 'sets the path' do
expect(trip.path).to be_present
end
context 'when DawarichSettings.store_geodata is enabled' do
context 'when DawarichSettings.store_geodata? is enabled' do
before do
allow(DawarichSettings).to receive(:store_geodata?).and_return(true)
end
@ -39,16 +36,6 @@ RSpec.describe Trip, type: :model do
expect(trip.countries).to eq(trip.points.pluck(:country).uniq.compact)
end
end
context 'when DawarichSettings.store_geodata is disabled' do
it 'sets the visited countries' do
countries_service = instance_double(Trips::Countries, call: [])
expect(Trips::Countries).to receive(:new).with(trip).and_return(countries_service)
expect(countries_service).to receive(:call)
trip.save
end
end
end
describe '#countries' do

View file

@ -0,0 +1,50 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe PointsLimitExceeded do
describe '#call' do
subject(:points_limit_exceeded) { described_class.new(user).call }
let(:user) { create(:user) }
context 'when app is self-hosted' do
before do
allow(DawarichSettings).to receive(:self_hosted?).and_return(true)
end
it { is_expected.to be false }
end
context 'when app is not self-hosted' do
before do
allow(DawarichSettings).to receive(:self_hosted?).and_return(false)
stub_const('DawarichSettings::BASIC_PAID_PLAN_LIMIT', 10)
end
context 'when user points count is equal to the limit' do
before do
allow(user.points).to receive(:count).and_return(10)
end
it { is_expected.to be true }
end
context 'when user points count exceeds the limit' do
before do
allow(user.points).to receive(:count).and_return(11)
end
it { is_expected.to be true }
end
context 'when user points count is below the limit' do
before do
allow(user.points).to receive(:count).and_return(9)
end
it { is_expected.to be false }
end
end
end
end

View file

@ -1,93 +0,0 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe Trips::Countries do
let(:trip) { instance_double('Trip') }
let(:point1) { instance_double('Point', lonlat: factory.point(10.0, 50.0)) }
let(:point2) { instance_double('Point', lonlat: factory.point(20.0, 60.0)) }
let(:point3) { instance_double('Point', lonlat: factory.point(30.0, 70.0)) }
let(:point4) { instance_double('Point', lonlat: nil) }
let(:factory) { RGeo::Geographic.spherical_factory }
let(:points) { [point1, point2, point3, point4] }
let(:geo_json_content) do
{
type: 'FeatureCollection',
features: [
{
type: 'Feature',
properties: { ADMIN: 'Germany', ISO_A3: 'DEU', ISO_A2: 'DE' },
geometry: { type: 'MultiPolygon', coordinates: [] }
}
]
}.to_json
end
before do
allow(trip).to receive(:points).and_return(points)
allow(File).to receive(:read).with(Trips::Countries::FILE_PATH).and_return(geo_json_content)
# Explicitly stub all Geocoder calls with specific coordinates
allow(Geocoder).to receive(:search).and_return(
[double(data: { 'properties' => { 'countrycode' => 'DE' } })]
)
allow(Geocoder).to receive(:search).with([50.0, 10.0], limit: 1).and_return(
[double(data: { 'properties' => { 'countrycode' => 'DE' } })]
)
allow(Geocoder).to receive(:search).with([60.0, 20.0], limit: 1).and_return(
[double(data: { 'properties' => { 'countrycode' => 'SE' } })]
)
allow(Geocoder).to receive(:search).with([70.0, 30.0], limit: 1).and_return(
[double(data: { 'properties' => { 'countrycode' => 'FI' } })]
)
allow(Rails.logger).to receive(:info)
allow(Rails.logger).to receive(:error)
end
describe '#call' do
it 'returns a hash with country counts' do
allow(Thread).to receive(:new).and_yield
result = described_class.new(trip).call
expect(result).to match_array(%w[DE SE FI])
end
it 'handles points without coordinates' do
allow(Thread).to receive(:new).and_yield
result = described_class.new(trip).call
expect(result.size).to eq(3) # Should only count the 3 valid points
end
it 'processes batches in multiple threads' do
expect(Thread).to receive(:new).at_least(:twice).and_yield
described_class.new(trip).call
end
it 'sorts countries by count in descending order' do
allow(Thread).to receive(:new).and_yield
allow(points).to receive(:to_a).and_return([point1, point1, point2, point3, point4])
result = described_class.new(trip).call
expect(result.first).to eq('DE')
end
context 'when an error occurs' do
before do
allow(Geocoder).to receive(:search).and_raise(Geocoder::Error, 'Error')
end
it 'calls the exception reporter' do
expect(ExceptionReporter).to receive(:call).with(Geocoder::Error).at_least(3).times
described_class.new(trip).call
end
end
end
end