diff --git a/CHANGELOG.md b/CHANGELOG.md index f7fab29c..bd34526c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,40 @@ and this project adheres to [Semantic Versioning](http://semver.org/). # 0.25.4 - 2025-04-02 +⚠️ This release includes a breaking change. ⚠️ + +Make sure to add `dawarich_storage` volume to your `docker-compose.yml` file. Example: + +```diff +... + + dawarich_app: + image: freikin/dawarich:latest + container_name: dawarich_app + volumes: + - dawarich_public:/var/app/public + - dawarich_watched:/var/app/tmp/imports/watched ++ - dawarich_storage:/var/app/storage + +... + + dawarich_sidekiq: + image: freikin/dawarich:latest + container_name: dawarich_sidekiq + volumes: + - dawarich_public:/var/app/public + - dawarich_watched:/var/app/tmp/imports/watched ++ - dawarich_storage:/var/app/storage + +volumes: + dawarich_db_data: + dawarich_shared: + dawarich_public: + dawarich_watched: ++ dawarich_storage: +``` + + In this release we're changing the way import files are being stored. Previously, they were being stored in the `raw_data` column of the `imports` table. Now, they are being attached to the import record. All new imports will be using the new storage, to migrate existing imports, you can use the `rake imports:migrate_to_new_storage` task. Run it in the container shell. This is an optional task, that will not affect your points or other data. @@ -14,17 +48,25 @@ Big imports might take a while to migrate, so be patient. If your hardware doesn't have enough memory to migrate the imports, you can delete your imports and re-import them. +## Added + +- Sentry is now can be used for error tracking. + ## Changed - Import files are now being attached to the import record instead of being stored in the `raw_data` database column. - Import files can now be stored in S3-compatible storage. - Export files are now being attached to the export record instead of being stored in the file system. - Export files can now be stored in S3-compatible storage. +- Users can now import Google's Records.json file via the UI instead of using the CLI. +- Optional telemetry sending is now disabled and will be removed in the future. ## Fixed - Moving points on the map now works correctly. #957 - `rake points:migrate_to_lonlat` task now also reindexes the points table. +- Fixed filling `lonlat` column for old places after reverse geocoding. +- Deleting an import now correctly recalculates stats. # 0.25.3 - 2025-03-22 diff --git a/Gemfile b/Gemfile index ed3e9e9d..3929ec9d 100644 --- a/Gemfile +++ b/Gemfile @@ -31,6 +31,8 @@ gem 'rgeo' gem 'rgeo-activerecord' gem 'rswag-api' gem 'rswag-ui' +gem 'sentry-ruby' +gem 'sentry-rails' gem 'sidekiq' gem 'sidekiq-cron' gem 'sidekiq-limit_fetch' diff --git a/Gemfile.lock b/Gemfile.lock index 35bfd90a..1ce8cf07 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -383,6 +383,12 @@ GEM rubocop-ast (>= 1.38.0, < 2.0) ruby-progressbar (1.13.0) securerandom (0.4.1) + sentry-rails (5.23.0) + railties (>= 5.0) + sentry-ruby (~> 5.23.0) + sentry-ruby (5.23.0) + bigdecimal + concurrent-ruby (~> 1.0, >= 1.0.2) shoulda-matchers (6.4.0) activesupport (>= 5.2.0) sidekiq (7.3.9) @@ -501,6 +507,8 @@ DEPENDENCIES rswag-specs rswag-ui rubocop-rails + sentry-rails + sentry-ruby shoulda-matchers sidekiq sidekiq-cron diff --git a/app/controllers/exports_controller.rb b/app/controllers/exports_controller.rb index 87029695..d6fe19e8 100644 --- a/app/controllers/exports_controller.rb +++ b/app/controllers/exports_controller.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class ExportsController < ApplicationController + include ActiveStorage::SetCurrent + before_action :authenticate_user! before_action :set_export, only: %i[destroy] diff --git a/app/controllers/imports_controller.rb b/app/controllers/imports_controller.rb index 24a90e51..caf874a9 100644 --- a/app/controllers/imports_controller.rb +++ b/app/controllers/imports_controller.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class ImportsController < ApplicationController + include ActiveStorage::SetCurrent + before_action :authenticate_user! before_action :authenticate_active_user!, only: %i[new create] before_action :set_import, only: %i[show destroy] @@ -9,7 +11,7 @@ class ImportsController < ApplicationController @imports = current_user .imports - .select(:id, :name, :source, :created_at, :points_count) + .select(:id, :name, :source, :created_at, :processed) .order(created_at: :desc) .page(params[:page]) end diff --git a/app/jobs/import/update_points_count_job.rb b/app/jobs/import/update_points_count_job.rb new file mode 100644 index 00000000..b0757024 --- /dev/null +++ b/app/jobs/import/update_points_count_job.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class Import::UpdatePointsCountJob < ApplicationJob + queue_as :imports + + def perform(import_id) + import = Import.find(import_id) + + import.update(processed: import.points.count) + rescue ActiveRecord::RecordNotFound + nil + end +end diff --git a/app/jobs/import/watcher_job.rb b/app/jobs/import/watcher_job.rb index 57ae24bd..a2f6676f 100644 --- a/app/jobs/import/watcher_job.rb +++ b/app/jobs/import/watcher_job.rb @@ -5,6 +5,8 @@ class Import::WatcherJob < ApplicationJob sidekiq_options retry: false def perform + return unless DawarichSettings.self_hosted? + Imports::Watcher.new.call end end diff --git a/app/models/export.rb b/app/models/export.rb index 21ab76a1..96060a04 100644 --- a/app/models/export.rb +++ b/app/models/export.rb @@ -11,9 +11,21 @@ class Export < ApplicationRecord has_one_attached :file after_commit -> { ExportJob.perform_later(id) }, on: :create - after_commit -> { file.purge }, on: :destroy + after_commit -> { remove_attached_file }, on: :destroy def process! Exports::Create.new(export: self).call end + + private + + def remove_attached_file + storage_config = Rails.application.config.active_storage + + if storage_config.service == :local + file.purge_later + else + file.purge + end + end end diff --git a/app/models/import.rb b/app/models/import.rb index 3965c219..2b302589 100644 --- a/app/models/import.rb +++ b/app/models/import.rb @@ -4,12 +4,10 @@ class Import < ApplicationRecord belongs_to :user has_many :points, dependent: :destroy - delegate :count, to: :points, prefix: true - has_one_attached :file after_commit -> { Import::ProcessJob.perform_later(id) }, on: :create - after_commit -> { file.purge }, on: :destroy + after_commit :remove_attached_file, on: :destroy enum :source, { google_semantic_history: 0, owntracks: 1, google_records: 2, @@ -38,4 +36,10 @@ class Import < ApplicationRecord file.attach(io: raw_file, filename: name, content_type: 'application/json') end + + private + + def remove_attached_file + file.purge_later + end end diff --git a/app/models/point.rb b/app/models/point.rb index 38970d91..a438bdb5 100644 --- a/app/models/point.rb +++ b/app/models/point.rb @@ -30,6 +30,7 @@ class Point < ApplicationRecord after_create :async_reverse_geocode after_create_commit :broadcast_coordinates + after_commit -> { Import::UpdatePointsCountJob.perform_later(import_id) }, on: :destroy, if: -> { import_id.present? } def self.without_raw_data select(column_names - ['raw_data']) diff --git a/app/services/google_maps/records_importer.rb b/app/services/google_maps/records_importer.rb index ec9555f7..3cecb1bd 100644 --- a/app/services/google_maps/records_importer.rb +++ b/app/services/google_maps/records_importer.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +# This class is used to import Google's Records.json file +# via the CLI, vs the UI, which uses the `GoogleMaps::RecordsStorage Importer` class. + class GoogleMaps::RecordsImporter include Imports::Broadcaster diff --git a/app/services/google_maps/records_storage_importer.rb b/app/services/google_maps/records_storage_importer.rb new file mode 100644 index 00000000..f56cc6fe --- /dev/null +++ b/app/services/google_maps/records_storage_importer.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +# This class is used to import Google's Records.json file +# via the UI, vs the CLI, which uses the `GoogleMaps::RecordsImporter` class. + +class GoogleMaps::RecordsStorageImporter + BATCH_SIZE = 1000 + + def initialize(import, user_id) + @import = import + @user = User.find_by(id: user_id) + end + + def call + process_file_in_batches + rescue Oj::ParseError => e + Rails.logger.error("JSON parsing error: #{e.message}") + raise + end + + private + + attr_reader :import, :user + + def process_file_in_batches + retries = 0 + max_retries = 3 + + begin + file = Timeout.timeout(300) do # 5 minutes timeout + import.file.download + end + + # Verify file size + expected_size = import.file.blob.byte_size + actual_size = file.size + + if expected_size != actual_size + raise "Incomplete download: expected #{expected_size} bytes, got #{actual_size} bytes" + end + + # Verify checksum + expected_checksum = import.file.blob.checksum + actual_checksum = Base64.strict_encode64(Digest::MD5.digest(file)) + + if expected_checksum != actual_checksum + raise "Checksum mismatch: expected #{expected_checksum}, got #{actual_checksum}" + end + + parsed_file = Oj.load(file, mode: :compat) + + return unless parsed_file.is_a?(Hash) && parsed_file['locations'] + + batch = [] + index = 0 + + parsed_file['locations'].each do |location| + batch << location + + next if batch.size < BATCH_SIZE + + index += BATCH_SIZE + + GoogleMaps::RecordsImporter.new(import, index).call(batch) + + batch = [] + end + rescue Timeout::Error => e + retries += 1 + if retries <= max_retries + Rails.logger.warn("Download timeout, attempt #{retries} of #{max_retries}") + retry + else + Rails.logger.error("Download failed after #{max_retries} attempts") + raise + end + rescue StandardError => e + Rails.logger.error("Download error: #{e.message}") + raise + end + end +end diff --git a/app/services/imports/create.rb b/app/services/imports/create.rb index 39b28eb7..7ad60d36 100644 --- a/app/services/imports/create.rb +++ b/app/services/imports/create.rb @@ -15,6 +15,7 @@ class Imports::Create schedule_stats_creating(user.id) schedule_visit_suggesting(user.id, import) + update_import_points_count(import) rescue StandardError => e create_import_failed_notification(import, user, e) end @@ -26,6 +27,7 @@ class Imports::Create case source when 'google_semantic_history' then GoogleMaps::SemanticHistoryParser when 'google_phone_takeout' then GoogleMaps::PhoneTakeoutParser + when 'google_records' then GoogleMaps::RecordsStorageImporter when 'owntracks' then OwnTracks::Importer when 'gpx' then Gpx::TrackImporter when 'geojson' then Geojson::ImportParser @@ -33,6 +35,10 @@ class Imports::Create end end + def update_import_points_count(import) + Import::UpdatePointsCountJob.perform_later(import.id) + end + def schedule_stats_creating(user_id) import.years_and_months_tracked.each do |year, month| Stats::CalculatingJob.perform_later(user_id, year, month) diff --git a/app/services/imports/destroy.rb b/app/services/imports/destroy.rb index 55efb008..fc0a6ff2 100644 --- a/app/services/imports/destroy.rb +++ b/app/services/imports/destroy.rb @@ -11,6 +11,6 @@ class Imports::Destroy def call @import.destroy! - BulkStatsCalculatingJob.perform_later(@user.id) + Stats::BulkCalculator.new(@user.id).call end end diff --git a/app/services/reverse_geocoding/places/fetch_data.rb b/app/services/reverse_geocoding/places/fetch_data.rb index 3b6309a1..1390c29a 100644 --- a/app/services/reverse_geocoding/places/fetch_data.rb +++ b/app/services/reverse_geocoding/places/fetch_data.rb @@ -19,6 +19,7 @@ class ReverseGeocoding::Places::FetchData first_place = reverse_geocoded_places.shift update_place(first_place) + reverse_geocoded_places.each { |reverse_geocoded_place| fetch_and_create_place(reverse_geocoded_place) } end @@ -49,6 +50,9 @@ class ReverseGeocoding::Places::FetchData new_place.country = data['properties']['country'] new_place.geodata = data new_place.source = :photon + if new_place.lonlat.blank? + new_place.lonlat = "POINT(#{data['geometry']['coordinates'][0]} #{data['geometry']['coordinates'][1]})" + end new_place.save! end @@ -88,7 +92,7 @@ class ReverseGeocoding::Places::FetchData limit: 10, distance_sort: true, radius: 1, - units: ::DISTANCE_UNIT, + units: ::DISTANCE_UNIT ) data.reject do |place| diff --git a/app/views/imports/_form.html.erb b/app/views/imports/_form.html.erb index ccfa8030..2e77a631 100644 --- a/app/views/imports/_form.html.erb +++ b/app/views/imports/_form.html.erb @@ -13,6 +13,24 @@

JSON files from your Takeout/Location History/Semantic Location History/YEAR

+
+
+ +

The Records.json file from your Google Takeout

+
+
+
+
+ +

A JSON file you received after your request for Takeout from your mobile device

+
+
-
-
- -

A JSON file you received after your request for Takeout from your mobile device

-
-