Put countries into database

This commit is contained in:
Eugene Burmakin 2025-05-16 18:51:48 +02:00
parent 96108b12d0
commit 5be5c1e584
40 changed files with 547 additions and 3072 deletions

View file

@ -3,7 +3,7 @@
class Api::V1::Countries::BordersController < ApplicationController
def index
countries = Rails.cache.fetch('dawarich/countries_codes', expires_in: 1.day) do
Oj.load(File.read(Rails.root.join('lib/assets/countries.json')))
Oj.load(File.read(Rails.root.join('lib/assets/countries.geojson')))
end
render json: countries

View file

@ -2,6 +2,7 @@
class Api::V1::Overland::BatchesController < ApiController
before_action :authenticate_active_api_user!, only: %i[create]
before_action :validate_points_limit, only: %i[create]
def create
Overland::BatchCreatingJob.perform_later(batch_params, current_api_user.id)

View file

@ -2,6 +2,7 @@
class Api::V1::Owntracks::PointsController < ApiController
before_action :authenticate_active_api_user!, only: %i[create]
before_action :validate_points_limit, only: %i[create]
def create
Owntracks::PointCreatingJob.perform_later(point_params, current_api_user.id)

View file

@ -2,6 +2,7 @@
class Api::V1::PointsController < ApiController
before_action :authenticate_active_api_user!, only: %i[create update destroy]
before_action :validate_points_limit, only: %i[create]
def index
start_at = params[:start_at]&.to_datetime&.to_i

View file

@ -2,6 +2,7 @@
class Api::V1::SubscriptionsController < ApiController
skip_before_action :authenticate_api_key, only: %i[callback]
def callback
decoded_token = Subscription::DecodeJwtToken.new(params[:token]).call

View file

@ -41,4 +41,10 @@ class ApiController < ApplicationController
def required_params
[]
end
def validate_points_limit
limit_exceeded = PointsLimitExceeded.new(current_api_user).call
render json: { error: 'Points limit exceeded' }, status: :unauthorized if limit_exceeded
end
end

View file

@ -6,7 +6,7 @@ class ImportsController < ApplicationController
before_action :authenticate_user!
before_action :authenticate_active_user!, only: %i[new create]
before_action :set_import, only: %i[show edit update destroy]
before_action :validate_points_limit, only: %i[new create]
def index
@imports =
current_user
@ -102,4 +102,10 @@ class ImportsController < ApplicationController
import
end
def validate_points_limit
limit_exceeded = PointsLimitExceeded.new(current_user).call
redirect_to new_import_path, alert: 'Points limit exceeded', status: :unprocessable_entity if limit_exceeded
end
end

View file

@ -0,0 +1,9 @@
class DataMigrations::SetPointsCountryIdsJob < ApplicationJob
queue_as :default
def perform(point_id)
point = Point.find(point_id)
point.country_id = Country.containing_point(point.lon, point.lat).id
point.save!
end
end

View file

@ -0,0 +1,9 @@
class DataMigrations::StartSettingsPointsCountryIdsJob < ApplicationJob
queue_as :default
def perform
Point.where(country_id: nil).find_each do |point|
DataMigrations::SetPointsCountryIdsJob.perform_later(point.id)
end
end
end

11
app/models/country.rb Normal file
View file

@ -0,0 +1,11 @@
# frozen_string_literal: true
class Country < ApplicationRecord
validates :name, :iso_a2, :iso_a3, :geom, presence: true
def self.containing_point(lon, lat)
where("ST_Contains(geom, ST_SetSRID(ST_MakePoint(?, ?), 4326))", lon, lat)
.select(:id, :name, :iso_a2, :iso_a3)
.first
end
end

View file

@ -29,6 +29,7 @@ class Point < ApplicationRecord
scope :not_visited, -> { where(visit_id: nil) }
after_create :async_reverse_geocode, if: -> { DawarichSettings.store_geodata? }
after_create :set_country
after_create_commit :broadcast_coordinates
def self.without_raw_data
@ -57,6 +58,10 @@ class Point < ApplicationRecord
lonlat.y
end
def found_in_country
Country.containing_point(lon, lat)
end
private
# rubocop:disable Metrics/MethodLength Metrics/AbcSize
@ -76,4 +81,9 @@ class Point < ApplicationRecord
)
end
# rubocop:enable Metrics/MethodLength
def set_country
self.country_id = found_in_country&.id
save! if changed?
end
end

View file

@ -46,7 +46,8 @@ class Trip < ApplicationRecord
end
def calculate_countries
countries = Trips::Countries.new(self).call
countries =
Country.where(id: points.pluck(:country_id).compact.uniq).pluck(:name)
self.visited_countries = countries
end

View file

@ -1,7 +1,7 @@
# frozen_string_literal: true
class Api::PointSerializer < PointSerializer
EXCLUDED_ATTRIBUTES = %w[created_at updated_at visit_id import_id user_id raw_data].freeze
EXCLUDED_ATTRIBUTES = %w[created_at updated_at visit_id import_id user_id raw_data country_id].freeze
def call
point.attributes.except(*EXCLUDED_ATTRIBUTES)

View file

@ -3,7 +3,7 @@
class PointSerializer
EXCLUDED_ATTRIBUTES = %w[
created_at updated_at visit_id id import_id user_id raw_data lonlat
reverse_geocoded_at
reverse_geocoded_at country_id
].freeze
def initialize(point)

View file

@ -0,0 +1,18 @@
class PointsLimitExceeded
def initialize(user)
@user = user
end
def call
return false if DawarichSettings.self_hosted?
return true if @user.points.count >= points_limit
false
end
private
def points_limit
DawarichSettings::BASIC_PAID_PLAN_LIMIT
end
end

View file

@ -56,7 +56,7 @@ class ReverseGeocoding::Places::FetchData
new_place.name = place_name(data)
new_place.city = data['properties']['city']
new_place.country = data['properties']['country']
new_place.country = data['properties']['country'] # TODO: Use country id
new_place.geodata = data
new_place.source = :photon
if new_place.lonlat.blank?

View file

@ -1,92 +0,0 @@
# frozen_string_literal: true
class Trips::Countries
FILE_PATH = Rails.root.join('lib/assets/countries.json')
def initialize(trip, batch_count = 2)
@trip = trip
@batch_count = batch_count
@factory = RGeo::Geographic.spherical_factory
@file = File.read(FILE_PATH)
@countries_features =
RGeo::GeoJSON.decode(@file, json_parser: :json, geo_factory: @factory)
end
def call
all_points = @trip.points.to_a
total_points = all_points.size
# Return empty hash if no points
return {} if total_points.zero?
batches = split_into_batches(all_points, @batch_count)
threads_results = process_batches_in_threads(batches, total_points)
merge_thread_results(threads_results).uniq.compact
end
private
def split_into_batches(points, batch_count)
batch_count = [batch_count, 1].max # Ensure batch_count is at least 1
batch_size = (points.size / batch_count.to_f).ceil
points.each_slice(batch_size).to_a
end
def process_batches_in_threads(batches, total_points)
threads_results = []
threads = []
batches.each do |batch|
threads << Thread.new do
threads_results << process_batch(batch)
end
end
threads.each(&:join)
threads_results
end
def merge_thread_results(threads_results)
countries = []
threads_results.each do |result|
countries.concat(result)
end
countries
end
def process_batch(points)
points.map do |point|
country_code = geocode_point(point)
next unless country_code
country_code
end
end
def geocode_point(point)
lonlat = point.lonlat
return nil unless lonlat
latitude = lonlat.y
longitude = lonlat.x
fetch_country_code(latitude, longitude)
end
def fetch_country_code(latitude, longitude)
results = Geocoder.search([latitude, longitude], limit: 1)
return nil unless results.any?
result = results.first
result.data['properties']['countrycode']
rescue StandardError => e
Rails.logger.error("Error geocoding point: #{e.message}")
ExceptionReporter.call(e)
nil
end
end

View file

@ -0,0 +1,6 @@
<p class="py-6">
<p class='py-2'>
You have used <%= number_with_delimiter(current_user.points.count) %> points of <%= number_with_delimiter(DawarichSettings::BASIC_PAID_PLAN_LIMIT) %> available.
</p>
<progress class="progress progress-primary w-1/2 h-5" value="<%= current_user.points.count %>" max="<%= DawarichSettings::BASIC_PAID_PLAN_LIMIT %>"></progress>
</p>

View file

@ -5,6 +5,7 @@
<div class="text-center lg:text-left">
<h1 class="text-5xl font-bold">Edit your account!</h1>
<%= render 'devise/registrations/api_key' %>
<%= render 'devise/registrations/points_usage' %>
</div>
<div class="card flex-shrink-0 w-full max-w-sm shadow-2xl bg-base-100 px-5 py-5">
<%= form_for(resource, as: resource_name, url: registration_path(resource_name), class: 'form-body', method: :put, data: { turbo_method: :put, turbo: false }) do |f| %>

View file

@ -1,7 +1,10 @@
# frozen_string_literal: true
class DawarichSettings
BASIC_PAID_PLAN_LIMIT = 10_000_000 # 10 million points
class << self
def reverse_geocoding_enabled?
@reverse_geocoding_enabled ||= photon_enabled? || geoapify_enabled? || nominatim_enabled?
end

View file

@ -0,0 +1,11 @@
# frozen_string_literal: true
class SetPointsCountryIds < ActiveRecord::Migration[8.0]
def up
DataMigrations::StartSettingsPointsCountryIdsJob.perform_later
end
def down
raise ActiveRecord::IrreversibleMigration
end
end

View file

@ -1 +1 @@
DataMigrate::Data.define(version: 20_250_404_182_629)
DataMigrate::Data.define(version: 20250516180933)

View file

@ -0,0 +1,17 @@
class CreateCountries < ActiveRecord::Migration[8.0]
def change
create_table :countries do |t|
t.string :name, null: false
t.string :iso_a2, null: false
t.string :iso_a3, null: false
t.multi_polygon :geom, srid: 4326
t.timestamps
end
add_index :countries, :name
add_index :countries, :iso_a2
add_index :countries, :iso_a3
add_index :countries, :geom, using: :gist
end
end

View file

@ -0,0 +1,9 @@
# frozen_string_literal: true
class AddCountryIdToPoints < ActiveRecord::Migration[8.0]
disable_ddl_transaction!
def change
add_reference :points, :country, index: { algorithm: :concurrently }
end
end

17
db/schema.rb generated
View file

@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema[8.0].define(version: 2025_05_13_164521) do
ActiveRecord::Schema[8.0].define(version: 2025_05_15_192211) do
# These are extensions that must be enabled in order to support this database
enable_extension "pg_catalog.plpgsql"
enable_extension "postgis"
@ -64,6 +64,19 @@ ActiveRecord::Schema[8.0].define(version: 2025_05_13_164521) do
t.index ["user_id"], name: "index_areas_on_user_id"
end
create_table "countries", force: :cascade do |t|
t.string "name", null: false
t.string "iso_a2", null: false
t.string "iso_a3", null: false
t.geometry "geom", limit: {srid: 4326, type: "multi_polygon"}
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.index ["geom"], name: "index_countries_on_geom", using: :gist
t.index ["iso_a2"], name: "index_countries_on_iso_a2"
t.index ["iso_a3"], name: "index_countries_on_iso_a3"
t.index ["name"], name: "index_countries_on_name"
end
create_table "data_migrations", primary_key: "version", id: :string, force: :cascade do |t|
end
@ -166,12 +179,14 @@ ActiveRecord::Schema[8.0].define(version: 2025_05_13_164521) do
t.decimal "course_accuracy", precision: 8, scale: 5
t.string "external_track_id"
t.geography "lonlat", limit: {srid: 4326, type: "st_point", geographic: true}
t.bigint "country_id"
t.index ["altitude"], name: "index_points_on_altitude"
t.index ["battery"], name: "index_points_on_battery"
t.index ["battery_status"], name: "index_points_on_battery_status"
t.index ["city"], name: "index_points_on_city"
t.index ["connection"], name: "index_points_on_connection"
t.index ["country"], name: "index_points_on_country"
t.index ["country_id"], name: "index_points_on_country_id"
t.index ["external_track_id"], name: "index_points_on_external_track_id"
t.index ["geodata"], name: "index_points_on_geodata", using: :gin
t.index ["import_id"], name: "index_points_on_import_id"

View file

@ -1,14 +1,36 @@
# frozen_string_literal: true
return if User.any?
if User.none?
puts 'Creating user...'
puts 'Creating user...'
User.create!(
email: 'demo@dawarich.app',
password: 'password',
password_confirmation: 'password',
admin: true
)
User.create!(
email: 'demo@dawarich.app',
password: 'password',
password_confirmation: 'password',
admin: true
)
puts "User created: #{User.first.email} / password: 'password'"
end
puts "User created: #{User.first.email} / password: 'password'"
if Country.none?
puts 'Creating countries...'
countries_json = Oj.load(File.read(Rails.root.join('lib/assets/countries.geojson')))
factory = RGeo::Geos.factory(srid: 4326)
countries_multi_polygon = RGeo::GeoJSON.decode(countries_json.to_json, geo_factory: factory)
ActiveRecord::Base.transaction do
countries_multi_polygon.each do |country, index|
p "Creating #{country.properties['name']}..."
Country.create!(
name: country.properties['name'],
iso_a2: country.properties['ISO3166-1-Alpha-2'],
iso_a3: country.properties['ISO3166-1-Alpha-3'],
geom: country.geometry
)
end
end
end

View file

@ -25,6 +25,7 @@ RUN apk -U add --no-cache \
less \
yaml-dev \
gcompat \
libgeos-dev \
&& mkdir -p $APP_PATH
# Update gem system and install bundler

View file

@ -22,6 +22,7 @@ RUN apk -U add --no-cache \
less \
yaml-dev \
gcompat \
libgeos-dev \
&& mkdir -p $APP_PATH
# Update gem system and install bundler

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,7 @@
{
"webcredentials": {
"apps": [
"2A275P77DQ.app.dawarich.Dawarich"
]
}
}

View file

@ -0,0 +1,10 @@
FactoryBot.define do
factory :country do
name { "Serranilla Bank" }
iso_a2 { "SB" }
iso_a3 { "SBX" }
geom {
"MULTIPOLYGON (((-78.637074 15.862087, -78.640411 15.864, -78.636871 15.867296, -78.637074 15.862087)))"
}
end
end

View file

@ -31,6 +31,7 @@ FactoryBot.define do
external_track_id { nil }
lonlat { "POINT(#{FFaker::Geolocation.lng} #{FFaker::Geolocation.lat})" }
user
country_id { nil }
trait :with_known_location do
lonlat { 'POINT(37.6173 55.755826)' }

View file

@ -0,0 +1,28 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe DataMigrations::SetPointsCountryIdsJob, type: :job do
describe '#perform' do
let(:point) { create(:point, lonlat: 'POINT(10.0 20.0)', country_id: nil) }
let(:country) { create(:country) }
before do
allow(Country).to receive(:containing_point)
.with(point.lon, point.lat)
.and_return(country)
end
it 'updates the point with the correct country_id' do
described_class.perform_now(point.id)
expect(point.reload.country_id).to eq(country.id)
end
end
describe 'queue' do
it 'uses the default queue' do
expect(described_class.queue_name).to eq('default')
end
end
end

View file

@ -0,0 +1,33 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe DataMigrations::StartSettingsPointsCountryIdsJob, type: :job do
describe '#perform' do
let!(:point_with_country) { create(:point, country_id: 1) }
let!(:point_without_country1) { create(:point, country_id: nil) }
let!(:point_without_country2) { create(:point, country_id: nil) }
it 'enqueues SetPointsCountryIdsJob for points without country_id' do
expect { described_class.perform_now }.to \
have_enqueued_job(DataMigrations::SetPointsCountryIdsJob)
.with(point_without_country1.id)
.and have_enqueued_job(DataMigrations::SetPointsCountryIdsJob)
.with(point_without_country2.id)
end
it 'does not enqueue jobs for points with country_id' do
point_with_country.update(country_id: 1)
expect { described_class.perform_now }.not_to \
have_enqueued_job(DataMigrations::SetPointsCountryIdsJob)
.with(point_with_country.id)
end
end
describe 'queue' do
it 'uses the default queue' do
expect(described_class.queue_name).to eq('default')
end
end
end

View file

@ -0,0 +1,11 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe Country, type: :model do
describe 'validations' do
it { is_expected.to validate_presence_of(:name) }
it { is_expected.to validate_presence_of(:iso_a2) }
it { is_expected.to validate_presence_of(:iso_a3) }
it { is_expected.to validate_presence_of(:geom) }
end
end

View file

@ -13,6 +13,21 @@ RSpec.describe Point, type: :model do
it { is_expected.to validate_presence_of(:lonlat) }
end
describe 'callbacks' do
describe '#set_country' do
let(:point) { build(:point, lonlat: 'POINT(-79.85581250721961 15.854775993302411)') }
let(:country) { create(:country) }
it 'sets the country' do
expect(Country).to receive(:containing_point).with(-79.85581250721961, 15.854775993302411).and_return(country)
point.save!
expect(point.country_id).to eq(country.id)
end
end
end
describe 'scopes' do
describe '.reverse_geocoded' do
let(:point) { create(:point, :reverse_geocoded) }

View file

@ -3,10 +3,6 @@
require 'rails_helper'
RSpec.describe Trip, type: :model do
before do
allow_any_instance_of(Trips::Countries).to receive(:call).and_return([])
end
describe 'validations' do
it { is_expected.to validate_presence_of(:name) }
it { is_expected.to validate_presence_of(:started_at) }
@ -40,22 +36,6 @@ RSpec.describe Trip, type: :model do
expect(trip.countries).to eq(trip.points.pluck(:country).uniq.compact)
end
end
context 'when DawarichSettings.store_geodata? is disabled' do
let(:countries_service) { instance_double(Trips::Countries, call: []) }
let(:trip) { build(:trip, :with_points, user:) }
before do
allow(DawarichSettings).to receive(:store_geodata?).and_return(false)
end
it 'sets the visited countries' do
expect(Trips::Countries).to receive(:new).with(trip).and_return(countries_service)
expect(any_instance_of(Trips::Countries)).to receive(:call)
trip.save
end
end
end
describe '#countries' do

View file

@ -1,93 +0,0 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe Trips::Countries do
let(:trip) { instance_double('Trip') }
let(:point1) { instance_double('Point', lonlat: factory.point(10.0, 50.0)) }
let(:point2) { instance_double('Point', lonlat: factory.point(20.0, 60.0)) }
let(:point3) { instance_double('Point', lonlat: factory.point(30.0, 70.0)) }
let(:point4) { instance_double('Point', lonlat: nil) }
let(:factory) { RGeo::Geographic.spherical_factory }
let(:points) { [point1, point2, point3, point4] }
let(:geo_json_content) do
{
type: 'FeatureCollection',
features: [
{
type: 'Feature',
properties: { ADMIN: 'Germany', ISO_A3: 'DEU', ISO_A2: 'DE' },
geometry: { type: 'MultiPolygon', coordinates: [] }
}
]
}.to_json
end
before do
allow(trip).to receive(:points).and_return(points)
allow(File).to receive(:read).with(Trips::Countries::FILE_PATH).and_return(geo_json_content)
# Explicitly stub all Geocoder calls with specific coordinates
allow(Geocoder).to receive(:search).and_return(
[double(data: { 'properties' => { 'countrycode' => 'DE' } })]
)
allow(Geocoder).to receive(:search).with([50.0, 10.0], limit: 1).and_return(
[double(data: { 'properties' => { 'countrycode' => 'DE' } })]
)
allow(Geocoder).to receive(:search).with([60.0, 20.0], limit: 1).and_return(
[double(data: { 'properties' => { 'countrycode' => 'SE' } })]
)
allow(Geocoder).to receive(:search).with([70.0, 30.0], limit: 1).and_return(
[double(data: { 'properties' => { 'countrycode' => 'FI' } })]
)
allow(Rails.logger).to receive(:info)
allow(Rails.logger).to receive(:error)
end
describe '#call' do
it 'returns a hash with country counts' do
allow(Thread).to receive(:new).and_yield
result = described_class.new(trip).call
expect(result).to match_array(%w[DE SE FI])
end
it 'handles points without coordinates' do
allow(Thread).to receive(:new).and_yield
result = described_class.new(trip).call
expect(result.size).to eq(3) # Should only count the 3 valid points
end
it 'processes batches in multiple threads' do
expect(Thread).to receive(:new).at_least(:twice).and_yield
described_class.new(trip).call
end
it 'sorts countries by count in descending order' do
allow(Thread).to receive(:new).and_yield
allow(points).to receive(:to_a).and_return([point1, point1, point2, point3, point4])
result = described_class.new(trip).call
expect(result.first).to eq('DE')
end
context 'when an error occurs' do
before do
allow(Geocoder).to receive(:search).and_raise(Geocoder::Error, 'Error')
end
it 'calls the exception reporter' do
expect(ExceptionReporter).to receive(:call).with(Geocoder::Error).at_least(3).times
described_class.new(trip).call
end
end
end
end