Improve performance of Google Maps imports by batching database inserts.

CHANGELOG.md

@@ -21,6 +21,7 @@ In this release we're introducing family features that allow users to create fam
 ## Changed
 
 - Minor versions of Dawarich are being built for ARM64 architecture as well again. #1840
+- Importing process for Google Maps Timeline exports is now significantly faster.
 
 
 # [0.33.1] - 2025-10-07

app/services/geojson/importer.rb

@@ -5,6 +5,7 @@ class Geojson::Importer
   include Imports::FileLoader
   include PointValidation
 
+  BATCH_SIZE = 1000
   attr_reader :import, :user_id, :file_path
 
   def initialize(import, user_id, file_path = nil)
@@ -17,13 +18,46 @@ class Geojson::Importer
     json = load_json_data
     data = Geojson::Params.new(json).call
 
-    data.each.with_index(1) do |point, index|
+    points_data = data.map do |point|
       next if point[:lonlat].nil?
-      next if point_exists?(point, user_id)
 
-      Point.create!(point.merge(user_id:, import_id: import.id))
+      point.merge(
+        user_id: user_id,
+        import_id: import.id,
+        created_at: Time.current,
+        updated_at: Time.current
+      )
+    end
 
-      broadcast_import_progress(import, index)
+    points_data.compact.each_slice(BATCH_SIZE).with_index do |batch, batch_index|
+      bulk_insert_points(batch)
+      broadcast_import_progress(import, (batch_index + 1) * BATCH_SIZE)
     end
   end
+
+  private
+
+  def bulk_insert_points(batch)
+    unique_batch = batch.uniq { |record| [record[:lonlat], record[:timestamp], record[:user_id]] }
+
+    # rubocop:disable Rails/SkipsModelValidations
+    Point.upsert_all(
+      unique_batch,
+      unique_by: %i[lonlat timestamp user_id],
+      returning: false,
+      on_duplicate: :skip
+    )
+    # rubocop:enable Rails/SkipsModelValidations
+  rescue StandardError => e
+    create_notification("Failed to process GeoJSON batch: #{e.message}")
+  end
+
+  def create_notification(message)
+    Notification.create!(
+      user_id: user_id,
+      title: 'GeoJSON Import Error',
+      content: message,
+      kind: :error
+    )
+  end
 end

app/services/google_maps/phone_takeout_importer.rb

@@ -12,30 +12,23 @@ class GoogleMaps::PhoneTakeoutImporter
     @file_path = file_path
   end
 
+  BATCH_SIZE = 1000
+
   def call
-    points_data = parse_json
-
-    points_data.compact.each.with_index(1) do |point_data, index|
-      next if Point.exists?(
-        timestamp:  point_data[:timestamp],
-        lonlat:     point_data[:lonlat],
-        user_id:
-      )
-
-      Point.create(
-        lonlat:     point_data[:lonlat],
-        timestamp:  point_data[:timestamp],
-        raw_data:   point_data[:raw_data],
-        accuracy:   point_data[:accuracy],
-        altitude:   point_data[:altitude],
-        velocity:   point_data[:velocity],
-        import_id:  import.id,
-        topic:      'Google Maps Phone Timeline Export',
+    points_data = parse_json.compact.map do |point_data|
+      point_data.merge(
+        import_id: import.id,
+        topic: 'Google Maps Phone Timeline Export',
         tracker_id: 'google-maps-phone-timeline-export',
-        user_id:
+        user_id: user_id,
+        created_at: Time.current,
+        updated_at: Time.current
       )
+    end
 
-      broadcast_import_progress(import, index)
+    points_data.each_slice(BATCH_SIZE).with_index do |batch, batch_index|
+      bulk_insert_points(batch)
+      broadcast_import_progress(import, (batch_index + 1) * BATCH_SIZE)
     end
   end
 
@@ -177,4 +170,28 @@ class GoogleMaps::PhoneTakeoutImporter
       point_hash(lat, lon, timestamp, segment)
     end
   end
+
+  def bulk_insert_points(batch)
+    unique_batch = batch.uniq { |record| [record[:lonlat], record[:timestamp], record[:user_id]] }
+
+    # rubocop:disable Rails/SkipsModelValidations
+    Point.upsert_all(
+      unique_batch,
+      unique_by: %i[lonlat timestamp user_id],
+      returning: false,
+      on_duplicate: :skip
+    )
+    # rubocop:enable Rails/SkipsModelValidations
+  rescue StandardError => e
+    create_notification("Failed to process phone takeout batch: #{e.message}")
+  end
+
+  def create_notification(message)
+    Notification.create!(
+      user_id: user_id,
+      title: 'Google Maps Phone Takeout Import Error',
+      content: message,
+      kind: :error
+    )
+  end
 end

app/services/google_maps/records_importer.rb

@@ -32,6 +32,10 @@ class GoogleMaps::RecordsImporter
       timestamp: parse_timestamp(location),
       altitude: location['altitude'],
       velocity: location['velocity'],
+      accuracy: location['accuracy'],
+      vertical_accuracy: location['verticalAccuracy'],
+      course: location['heading'],
+      battery: parse_battery_charging(location['batteryCharging']),
       raw_data: location,
       topic: 'Google Maps Timeline Export',
       tracker_id: 'google-maps-timeline-export',
@@ -74,6 +78,12 @@ class GoogleMaps::RecordsImporter
     )
   end
 
+  def parse_battery_charging(battery_charging)
+    return nil if battery_charging.nil?
+
+    battery_charging ? 1 : 0
+  end
+
   def create_notification(message)
     Notification.create!(
       user: @import.user,

app/services/google_maps/semantic_history_importer.rb

@@ -43,6 +43,7 @@ class GoogleMaps::SemanticHistoryImporter
     {
       lonlat: point_data[:lonlat],
       timestamp: point_data[:timestamp],
+      accuracy: point_data[:accuracy],
       raw_data: point_data[:raw_data],
       topic: 'Google Maps Timeline Export',
       tracker_id: 'google-maps-timeline-export',
@@ -86,6 +87,7 @@ class GoogleMaps::SemanticHistoryImporter
         longitude: activity['startLocation']['longitudeE7'],
         latitude: activity['startLocation']['latitudeE7'],
         timestamp: activity['duration']['startTimestamp'] || activity['duration']['startTimestampMs'],
+        accuracy: activity.dig('startLocation', 'accuracyMetres'),
         raw_data: activity
       )
     end
@@ -111,6 +113,7 @@ class GoogleMaps::SemanticHistoryImporter
         longitude: place_visit['location']['longitudeE7'],
         latitude: place_visit['location']['latitudeE7'],
         timestamp: place_visit['duration']['startTimestamp'] || place_visit['duration']['startTimestampMs'],
+        accuracy: place_visit.dig('location', 'accuracyMetres'),
         raw_data: place_visit
       )
     elsif (candidate = place_visit.dig('otherCandidateLocations', 0))
@@ -125,14 +128,16 @@ class GoogleMaps::SemanticHistoryImporter
       longitude: candidate['longitudeE7'],
       latitude: candidate['latitudeE7'],
       timestamp: place_visit['duration']['startTimestamp'] || place_visit['duration']['startTimestampMs'],
+      accuracy: candidate['accuracyMetres'],
       raw_data: place_visit
     )
   end
 
-  def build_point_from_location(longitude:, latitude:, timestamp:, raw_data:)
+  def build_point_from_location(longitude:, latitude:, timestamp:, raw_data:, accuracy: nil)
     {
       lonlat: "POINT(#{longitude.to_f / 10**7} #{latitude.to_f / 10**7})",
       timestamp: Timestamps.parse_timestamp(timestamp),
+      accuracy: accuracy,
       raw_data: raw_data
     }
   end

app/services/photos/importer.rb

@@ -4,6 +4,8 @@ class Photos::Importer
   include Imports::Broadcaster
   include Imports::FileLoader
   include PointValidation
+
+  BATCH_SIZE = 1000
   attr_reader :import, :user_id, :file_path
 
   def initialize(import, user_id, file_path = nil)
@@ -14,25 +16,54 @@ class Photos::Importer
 
   def call
     json = load_json_data
+    points_data = json.map { |point| prepare_point_data(point) }
 
-    json.each.with_index(1) { |point, index| create_point(point, index) }
+    points_data.compact.each_slice(BATCH_SIZE).with_index do |batch, batch_index|
+      bulk_insert_points(batch)
+      broadcast_import_progress(import, (batch_index + 1) * BATCH_SIZE)
+    end
   end
 
-  def create_point(point, index)
-    return 0 unless valid?(point)
-    return 0 if point_exists?(point, point['timestamp'])
+  private
 
-    Point.create(
-      lonlat:    point['lonlat'],
+  def prepare_point_data(point)
+    return nil unless valid?(point)
+
+    {
+      lonlat: point['lonlat'],
       longitude: point['longitude'],
-      latitude:  point['latitude'],
+      latitude: point['latitude'],
       timestamp: point['timestamp'].to_i,
-      raw_data:  point,
+      raw_data: point,
       import_id: import.id,
-      user_id:
-    )
+      user_id: user_id,
+      created_at: Time.current,
+      updated_at: Time.current
+    }
+  end
 
-    broadcast_import_progress(import, index)
+  def bulk_insert_points(batch)
+    unique_batch = batch.uniq { |record| [record[:lonlat], record[:timestamp], record[:user_id]] }
+
+    # rubocop:disable Rails/SkipsModelValidations
+    Point.upsert_all(
+      unique_batch,
+      unique_by: %i[lonlat timestamp user_id],
+      returning: false,
+      on_duplicate: :skip
+    )
+    # rubocop:enable Rails/SkipsModelValidations
+  rescue StandardError => e
+    create_notification("Failed to process photo location batch: #{e.message}")
+  end
+
+  def create_notification(message)
+    Notification.create!(
+      user_id: user_id,
+      title: 'Photos Import Error',
+      content: message,
+      kind: :error
+    )
   end
 
   def valid?(point)

spec/services/google_maps/records_importer_spec.rb

@@ -17,6 +17,12 @@ RSpec.describe GoogleMaps::RecordsImporter do
           'accuracy' => 10,
           'altitude' => 100,
           'verticalAccuracy' => 5,
+          'heading' => 270,
+          'velocity' => 15,
+          'batteryCharging' => true,
+          'source' => 'GPS',
+          'deviceTag' => 1234567890,
+          'platformType' => 'ANDROID',
           'activity' => [
             {
               'timestampMs' => (time.to_f * 1000).to_i.to_s,
@@ -111,5 +117,87 @@ RSpec.describe GoogleMaps::RecordsImporter do
         expect(created_point.timestamp).to eq(time.to_i)
       end
     end
+
+    context 'with additional Records.json schema fields' do
+      let(:locations) do
+        [
+          {
+            'timestamp' => time.iso8601,
+            'latitudeE7' => 123_456_789,
+            'longitudeE7' => 123_456_789,
+            'accuracy' => 20,
+            'altitude' => 150,
+            'verticalAccuracy' => 10,
+            'heading' => 270,
+            'velocity' => 10,
+            'batteryCharging' => true,
+            'source' => 'WIFI',
+            'deviceTag' => 1234567890,
+            'platformType' => 'ANDROID'
+          }
+        ]
+      end
+
+      it 'extracts all supported fields' do
+        expect { parser }.to change(Point, :count).by(1)
+
+        created_point = Point.last
+        expect(created_point.accuracy).to eq(20)
+        expect(created_point.altitude).to eq(150)
+        expect(created_point.vertical_accuracy).to eq(10)
+        expect(created_point.course).to eq(270)
+        expect(created_point.velocity).to eq('10')
+        expect(created_point.battery).to eq(1) # true -> 1
+      end
+
+      it 'stores all fields in raw_data' do
+        parser
+        created_point = Point.last
+
+        expect(created_point.raw_data['source']).to eq('WIFI')
+        expect(created_point.raw_data['deviceTag']).to eq(1234567890)
+        expect(created_point.raw_data['platformType']).to eq('ANDROID')
+      end
+    end
+
+    context 'with batteryCharging false' do
+      let(:locations) do
+        [
+          {
+            'timestamp' => time.iso8601,
+            'latitudeE7' => 123_456_789,
+            'longitudeE7' => 123_456_789,
+            'batteryCharging' => false
+          }
+        ]
+      end
+
+      it 'stores battery as 0' do
+        expect { parser }.to change(Point, :count).by(1)
+        expect(Point.last.battery).to eq(0)
+      end
+    end
+
+    context 'with missing optional fields' do
+      let(:locations) do
+        [
+          {
+            'timestamp' => time.iso8601,
+            'latitudeE7' => 123_456_789,
+            'longitudeE7' => 123_456_789
+          }
+        ]
+      end
+
+      it 'handles missing fields gracefully' do
+        expect { parser }.to change(Point, :count).by(1)
+
+        created_point = Point.last
+        expect(created_point.accuracy).to be_nil
+        expect(created_point.vertical_accuracy).to be_nil
+        expect(created_point.course).to be_nil
+        expect(created_point.battery).to be_nil
+      end
+    end
   end
 end

spec/services/google_maps/semantic_history_importer_spec.rb

@@ -149,5 +149,6 @@ RSpec.describe GoogleMaps::SemanticHistoryImporter do
         end
       end
     end
+
   end
 end