From 1671a781b0737267762740a133bf84fad0c9984f Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Thu, 16 Oct 2025 18:59:21 +0200 Subject: [PATCH 1/2] Improve performance of Google Maps imports by batching database inserts. --- CHANGELOG.md | 1 + app/services/geojson/importer.rb | 42 ++++++++- .../google_maps/phone_takeout_importer.rb | 57 +++++++----- app/services/google_maps/records_importer.rb | 10 +++ .../google_maps/semantic_history_importer.rb | 7 +- app/services/photos/importer.rb | 53 ++++++++--- .../google_maps/records_importer_spec.rb | 88 +++++++++++++++++++ .../semantic_history_importer_spec.rb | 1 + 8 files changed, 223 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41b47751..58831051 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ In this release we're introducing family features that allow users to create fam ## Changed - Minor versions of Dawarich are being built for ARM64 architecture as well again. #1840 +- Importing process for Google Maps Timeline exports is now significantly faster. # [0.33.1] - 2025-10-07 diff --git a/app/services/geojson/importer.rb b/app/services/geojson/importer.rb index 94230047..501c0533 100644 --- a/app/services/geojson/importer.rb +++ b/app/services/geojson/importer.rb @@ -5,6 +5,7 @@ class Geojson::Importer include Imports::FileLoader include PointValidation + BATCH_SIZE = 1000 attr_reader :import, :user_id, :file_path def initialize(import, user_id, file_path = nil) @@ -17,13 +18,46 @@ class Geojson::Importer json = load_json_data data = Geojson::Params.new(json).call - data.each.with_index(1) do |point, index| + points_data = data.map do |point| next if point[:lonlat].nil? - next if point_exists?(point, user_id) - Point.create!(point.merge(user_id:, import_id: import.id)) + point.merge( + user_id: user_id, + import_id: import.id, + created_at: Time.current, + updated_at: Time.current + ) + end - broadcast_import_progress(import, index) + points_data.compact.each_slice(BATCH_SIZE).with_index do |batch, batch_index| + bulk_insert_points(batch) + broadcast_import_progress(import, (batch_index + 1) * BATCH_SIZE) end end + + private + + def bulk_insert_points(batch) + unique_batch = batch.uniq { |record| [record[:lonlat], record[:timestamp], record[:user_id]] } + + # rubocop:disable Rails/SkipsModelValidations + Point.upsert_all( + unique_batch, + unique_by: %i[lonlat timestamp user_id], + returning: false, + on_duplicate: :skip + ) + # rubocop:enable Rails/SkipsModelValidations + rescue StandardError => e + create_notification("Failed to process GeoJSON batch: #{e.message}") + end + + def create_notification(message) + Notification.create!( + user_id: user_id, + title: 'GeoJSON Import Error', + content: message, + kind: :error + ) + end end diff --git a/app/services/google_maps/phone_takeout_importer.rb b/app/services/google_maps/phone_takeout_importer.rb index 51cfda5c..4e74bc54 100644 --- a/app/services/google_maps/phone_takeout_importer.rb +++ b/app/services/google_maps/phone_takeout_importer.rb @@ -12,30 +12,23 @@ class GoogleMaps::PhoneTakeoutImporter @file_path = file_path end + BATCH_SIZE = 1000 + def call - points_data = parse_json - - points_data.compact.each.with_index(1) do |point_data, index| - next if Point.exists?( - timestamp: point_data[:timestamp], - lonlat: point_data[:lonlat], - user_id: - ) - - Point.create( - lonlat: point_data[:lonlat], - timestamp: point_data[:timestamp], - raw_data: point_data[:raw_data], - accuracy: point_data[:accuracy], - altitude: point_data[:altitude], - velocity: point_data[:velocity], - import_id: import.id, - topic: 'Google Maps Phone Timeline Export', + points_data = parse_json.compact.map do |point_data| + point_data.merge( + import_id: import.id, + topic: 'Google Maps Phone Timeline Export', tracker_id: 'google-maps-phone-timeline-export', - user_id: + user_id: user_id, + created_at: Time.current, + updated_at: Time.current ) + end - broadcast_import_progress(import, index) + points_data.each_slice(BATCH_SIZE).with_index do |batch, batch_index| + bulk_insert_points(batch) + broadcast_import_progress(import, (batch_index + 1) * BATCH_SIZE) end end @@ -177,4 +170,28 @@ class GoogleMaps::PhoneTakeoutImporter point_hash(lat, lon, timestamp, segment) end end + + def bulk_insert_points(batch) + unique_batch = batch.uniq { |record| [record[:lonlat], record[:timestamp], record[:user_id]] } + + # rubocop:disable Rails/SkipsModelValidations + Point.upsert_all( + unique_batch, + unique_by: %i[lonlat timestamp user_id], + returning: false, + on_duplicate: :skip + ) + # rubocop:enable Rails/SkipsModelValidations + rescue StandardError => e + create_notification("Failed to process phone takeout batch: #{e.message}") + end + + def create_notification(message) + Notification.create!( + user_id: user_id, + title: 'Google Maps Phone Takeout Import Error', + content: message, + kind: :error + ) + end end diff --git a/app/services/google_maps/records_importer.rb b/app/services/google_maps/records_importer.rb index 3cecb1bd..3986414d 100644 --- a/app/services/google_maps/records_importer.rb +++ b/app/services/google_maps/records_importer.rb @@ -32,6 +32,10 @@ class GoogleMaps::RecordsImporter timestamp: parse_timestamp(location), altitude: location['altitude'], velocity: location['velocity'], + accuracy: location['accuracy'], + vertical_accuracy: location['verticalAccuracy'], + course: location['heading'], + battery: parse_battery_charging(location['batteryCharging']), raw_data: location, topic: 'Google Maps Timeline Export', tracker_id: 'google-maps-timeline-export', @@ -74,6 +78,12 @@ class GoogleMaps::RecordsImporter ) end + def parse_battery_charging(battery_charging) + return nil if battery_charging.nil? + + battery_charging ? 1 : 0 + end + def create_notification(message) Notification.create!( user: @import.user, diff --git a/app/services/google_maps/semantic_history_importer.rb b/app/services/google_maps/semantic_history_importer.rb index e5eeb0b9..d24faa91 100644 --- a/app/services/google_maps/semantic_history_importer.rb +++ b/app/services/google_maps/semantic_history_importer.rb @@ -43,6 +43,7 @@ class GoogleMaps::SemanticHistoryImporter { lonlat: point_data[:lonlat], timestamp: point_data[:timestamp], + accuracy: point_data[:accuracy], raw_data: point_data[:raw_data], topic: 'Google Maps Timeline Export', tracker_id: 'google-maps-timeline-export', @@ -86,6 +87,7 @@ class GoogleMaps::SemanticHistoryImporter longitude: activity['startLocation']['longitudeE7'], latitude: activity['startLocation']['latitudeE7'], timestamp: activity['duration']['startTimestamp'] || activity['duration']['startTimestampMs'], + accuracy: activity.dig('startLocation', 'accuracyMetres'), raw_data: activity ) end @@ -111,6 +113,7 @@ class GoogleMaps::SemanticHistoryImporter longitude: place_visit['location']['longitudeE7'], latitude: place_visit['location']['latitudeE7'], timestamp: place_visit['duration']['startTimestamp'] || place_visit['duration']['startTimestampMs'], + accuracy: place_visit.dig('location', 'accuracyMetres'), raw_data: place_visit ) elsif (candidate = place_visit.dig('otherCandidateLocations', 0)) @@ -125,14 +128,16 @@ class GoogleMaps::SemanticHistoryImporter longitude: candidate['longitudeE7'], latitude: candidate['latitudeE7'], timestamp: place_visit['duration']['startTimestamp'] || place_visit['duration']['startTimestampMs'], + accuracy: candidate['accuracyMetres'], raw_data: place_visit ) end - def build_point_from_location(longitude:, latitude:, timestamp:, raw_data:) + def build_point_from_location(longitude:, latitude:, timestamp:, raw_data:, accuracy: nil) { lonlat: "POINT(#{longitude.to_f / 10**7} #{latitude.to_f / 10**7})", timestamp: Timestamps.parse_timestamp(timestamp), + accuracy: accuracy, raw_data: raw_data } end diff --git a/app/services/photos/importer.rb b/app/services/photos/importer.rb index e307b6b1..48775ca6 100644 --- a/app/services/photos/importer.rb +++ b/app/services/photos/importer.rb @@ -4,6 +4,8 @@ class Photos::Importer include Imports::Broadcaster include Imports::FileLoader include PointValidation + + BATCH_SIZE = 1000 attr_reader :import, :user_id, :file_path def initialize(import, user_id, file_path = nil) @@ -14,25 +16,54 @@ class Photos::Importer def call json = load_json_data + points_data = json.map { |point| prepare_point_data(point) } - json.each.with_index(1) { |point, index| create_point(point, index) } + points_data.compact.each_slice(BATCH_SIZE).with_index do |batch, batch_index| + bulk_insert_points(batch) + broadcast_import_progress(import, (batch_index + 1) * BATCH_SIZE) + end end - def create_point(point, index) - return 0 unless valid?(point) - return 0 if point_exists?(point, point['timestamp']) + private - Point.create( - lonlat: point['lonlat'], + def prepare_point_data(point) + return nil unless valid?(point) + + { + lonlat: point['lonlat'], longitude: point['longitude'], - latitude: point['latitude'], + latitude: point['latitude'], timestamp: point['timestamp'].to_i, - raw_data: point, + raw_data: point, import_id: import.id, - user_id: - ) + user_id: user_id, + created_at: Time.current, + updated_at: Time.current + } + end - broadcast_import_progress(import, index) + def bulk_insert_points(batch) + unique_batch = batch.uniq { |record| [record[:lonlat], record[:timestamp], record[:user_id]] } + + # rubocop:disable Rails/SkipsModelValidations + Point.upsert_all( + unique_batch, + unique_by: %i[lonlat timestamp user_id], + returning: false, + on_duplicate: :skip + ) + # rubocop:enable Rails/SkipsModelValidations + rescue StandardError => e + create_notification("Failed to process photo location batch: #{e.message}") + end + + def create_notification(message) + Notification.create!( + user_id: user_id, + title: 'Photos Import Error', + content: message, + kind: :error + ) end def valid?(point) diff --git a/spec/services/google_maps/records_importer_spec.rb b/spec/services/google_maps/records_importer_spec.rb index e2761d4f..c0cf5033 100644 --- a/spec/services/google_maps/records_importer_spec.rb +++ b/spec/services/google_maps/records_importer_spec.rb @@ -17,6 +17,12 @@ RSpec.describe GoogleMaps::RecordsImporter do 'accuracy' => 10, 'altitude' => 100, 'verticalAccuracy' => 5, + 'heading' => 270, + 'velocity' => 15, + 'batteryCharging' => true, + 'source' => 'GPS', + 'deviceTag' => 1234567890, + 'platformType' => 'ANDROID', 'activity' => [ { 'timestampMs' => (time.to_f * 1000).to_i.to_s, @@ -111,5 +117,87 @@ RSpec.describe GoogleMaps::RecordsImporter do expect(created_point.timestamp).to eq(time.to_i) end end + + context 'with additional Records.json schema fields' do + let(:locations) do + [ + { + 'timestamp' => time.iso8601, + 'latitudeE7' => 123_456_789, + 'longitudeE7' => 123_456_789, + 'accuracy' => 20, + 'altitude' => 150, + 'verticalAccuracy' => 10, + 'heading' => 270, + 'velocity' => 10, + 'batteryCharging' => true, + 'source' => 'WIFI', + 'deviceTag' => 1234567890, + 'platformType' => 'ANDROID' + } + ] + end + + it 'extracts all supported fields' do + expect { parser }.to change(Point, :count).by(1) + + created_point = Point.last + expect(created_point.accuracy).to eq(20) + expect(created_point.altitude).to eq(150) + expect(created_point.vertical_accuracy).to eq(10) + expect(created_point.course).to eq(270) + expect(created_point.velocity).to eq('10') + expect(created_point.battery).to eq(1) # true -> 1 + end + + it 'stores all fields in raw_data' do + parser + created_point = Point.last + + expect(created_point.raw_data['source']).to eq('WIFI') + expect(created_point.raw_data['deviceTag']).to eq(1234567890) + expect(created_point.raw_data['platformType']).to eq('ANDROID') + end + end + + context 'with batteryCharging false' do + let(:locations) do + [ + { + 'timestamp' => time.iso8601, + 'latitudeE7' => 123_456_789, + 'longitudeE7' => 123_456_789, + 'batteryCharging' => false + } + ] + end + + it 'stores battery as 0' do + expect { parser }.to change(Point, :count).by(1) + expect(Point.last.battery).to eq(0) + end + end + + context 'with missing optional fields' do + let(:locations) do + [ + { + 'timestamp' => time.iso8601, + 'latitudeE7' => 123_456_789, + 'longitudeE7' => 123_456_789 + } + ] + end + + it 'handles missing fields gracefully' do + expect { parser }.to change(Point, :count).by(1) + + created_point = Point.last + expect(created_point.accuracy).to be_nil + expect(created_point.vertical_accuracy).to be_nil + expect(created_point.course).to be_nil + expect(created_point.battery).to be_nil + end + end end end diff --git a/spec/services/google_maps/semantic_history_importer_spec.rb b/spec/services/google_maps/semantic_history_importer_spec.rb index 44f47c85..a8faa4a9 100644 --- a/spec/services/google_maps/semantic_history_importer_spec.rb +++ b/spec/services/google_maps/semantic_history_importer_spec.rb @@ -149,5 +149,6 @@ RSpec.describe GoogleMaps::SemanticHistoryImporter do end end end + end end From cdd5525ff4854dab584ea6904ef3ef915fae8b13 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Thu, 16 Oct 2025 19:01:39 +0200 Subject: [PATCH 2/2] Update changelog --- CHANGELOG.md | 2 +- spec/services/google_maps/semantic_history_importer_spec.rb | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58831051..99b81c0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,7 @@ In this release we're introducing family features that allow users to create fam ## Changed - Minor versions of Dawarich are being built for ARM64 architecture as well again. #1840 -- Importing process for Google Maps Timeline exports is now significantly faster. +- Importing process for Google Maps Timeline exports, GeoJSON and geodata from photos is now significantly faster. # [0.33.1] - 2025-10-07 diff --git a/spec/services/google_maps/semantic_history_importer_spec.rb b/spec/services/google_maps/semantic_history_importer_spec.rb index a8faa4a9..44f47c85 100644 --- a/spec/services/google_maps/semantic_history_importer_spec.rb +++ b/spec/services/google_maps/semantic_history_importer_spec.rb @@ -149,6 +149,5 @@ RSpec.describe GoogleMaps::SemanticHistoryImporter do end end end - end end