dawarich/app/services/google_maps/records_parser.rb

93 lines
2.4 KiB
Ruby
Raw Normal View History

2024-05-18 09:00:44 -04:00
# frozen_string_literal: true
class GoogleMaps::RecordsParser
2025-01-21 13:14:36 -05:00
include Imports::Broadcaster
2025-01-21 04:07:54 -05:00
BATCH_SIZE = 1000
2025-01-21 13:14:36 -05:00
attr_reader :import, :current_index
2024-05-18 09:00:44 -04:00
2025-01-21 13:14:36 -05:00
def initialize(import, current_index = 0)
2024-05-18 09:00:44 -04:00
@import = import
2025-01-21 04:07:54 -05:00
@batch = []
2025-01-21 13:14:36 -05:00
@current_index = current_index
2024-05-18 09:00:44 -04:00
end
2025-01-21 13:14:36 -05:00
def call(locations)
Array(locations).each do |location|
@batch << prepare_location_data(location)
next unless @batch.size >= BATCH_SIZE
2025-01-21 13:14:36 -05:00
bulk_insert_points(@batch)
broadcast_import_progress(import, current_index)
@batch = []
2025-01-21 04:07:54 -05:00
end
2024-06-10 13:08:27 -04:00
2025-01-21 13:14:36 -05:00
return unless @batch.any?
bulk_insert_points(@batch)
broadcast_import_progress(import, current_index)
2024-05-18 09:00:44 -04:00
end
private
2025-01-21 13:14:36 -05:00
# rubocop:disable Metrics/MethodLength
def prepare_location_data(location)
{
2025-01-21 13:14:36 -05:00
latitude: location['latitudeE7'].to_f / 10**7,
longitude: location['longitudeE7'].to_f / 10**7,
2025-01-21 13:32:12 -05:00
timestamp: parse_timestamp(location),
2025-01-21 13:14:36 -05:00
altitude: location['altitude'],
velocity: location['velocity'],
raw_data: location,
2025-01-21 04:07:54 -05:00
topic: 'Google Maps Timeline Export',
tracker_id: 'google-maps-timeline-export',
2025-01-21 13:14:36 -05:00
import_id: @import.id,
user_id: @import.user_id,
2025-01-21 04:07:54 -05:00
created_at: Time.current,
updated_at: Time.current
}
2024-05-18 09:00:44 -04:00
end
2025-01-21 04:07:54 -05:00
2025-01-21 13:14:36 -05:00
# rubocop:enable Metrics/MethodLength
def bulk_insert_points(batch)
# Deduplicate records within the batch before upserting
# Use all fields in the unique constraint for deduplication
unique_batch = deduplicate_batch(batch)
# Sort the batch to ensure consistent ordering and prevent deadlocks
# sorted_batch = sort_batch(unique_batch)
2025-01-21 04:07:54 -05:00
Point.upsert_all(
2025-01-21 13:14:36 -05:00
unique_batch,
2025-01-21 04:07:54 -05:00
unique_by: %i[latitude longitude timestamp user_id],
2025-01-21 13:14:36 -05:00
returning: false,
on_duplicate: :skip
2025-01-21 04:07:54 -05:00
)
2025-01-21 13:14:36 -05:00
rescue StandardError => e
Rails.logger.error("Batch insert failed for import #{@import.id}: #{e.message}")
# Create notification for the user
Notification.create!(
user: @import.user,
title: 'Google Maps Import Error',
content: "Failed to process location batch: #{e.message}",
kind: :error
)
end
def deduplicate_batch(batch)
batch.uniq do |record|
[
record[:latitude].round(7),
record[:longitude].round(7),
record[:timestamp],
record[:user_id]
]
end
2025-01-21 04:07:54 -05:00
end
2025-01-21 13:32:12 -05:00
def parse_timestamp(location)
Timestamps.parse_timestamp(location['timestamp'] || location['timestampMs'])
end
2024-05-18 09:00:44 -04:00
end