From 049812823f6595f83d13a191f92b68a041f8e419 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Tue, 21 Jan 2025 10:07:54 +0100 Subject: [PATCH 1/7] Stream google records import --- app/jobs/import/google_takeout_job.rb | 7 ++- app/services/google_maps/records_parser.rb | 45 ++++++++++------- app/services/tasks/imports/google_records.rb | 52 ++++++++++++++++---- 3 files changed, 72 insertions(+), 32 deletions(-) diff --git a/app/jobs/import/google_takeout_job.rb b/app/jobs/import/google_takeout_job.rb index d962a304..4cb74ae2 100644 --- a/app/jobs/import/google_takeout_job.rb +++ b/app/jobs/import/google_takeout_job.rb @@ -4,11 +4,10 @@ class Import::GoogleTakeoutJob < ApplicationJob queue_as :imports sidekiq_options retry: false - def perform(import_id, json_string) + def perform(import_id, json_array) import = Import.find(import_id) + records = Oj.load(json_array) - json = Oj.load(json_string) - - GoogleMaps::RecordsParser.new(import).call(json) + GoogleMaps::RecordsParser.new(import).call(records) end end diff --git a/app/services/google_maps/records_parser.rb b/app/services/google_maps/records_parser.rb index 04ee4621..e2fe2e05 100644 --- a/app/services/google_maps/records_parser.rb +++ b/app/services/google_maps/records_parser.rb @@ -1,32 +1,25 @@ # frozen_string_literal: true class GoogleMaps::RecordsParser + BATCH_SIZE = 1000 attr_reader :import def initialize(import) @import = import + @batch = [] end - def call(json) - data = parse_json(json) + def call(records) + Array(records).each do |record| + @batch << parse_json(record) - return if Point.exists?( - latitude: data[:latitude], - longitude: data[:longitude], - timestamp: data[:timestamp], - user_id: import.user_id - ) + if @batch.size >= BATCH_SIZE + bulk_insert_points + @batch = [] + end + end - Point.create( - latitude: data[:latitude], - longitude: data[:longitude], - timestamp: data[:timestamp], - raw_data: data[:raw_data], - topic: 'Google Maps Timeline Export', - tracker_id: 'google-maps-timeline-export', - import_id: import.id, - user_id: import.user_id - ) + bulk_insert_points if @batch.any? end private @@ -38,7 +31,21 @@ class GoogleMaps::RecordsParser timestamp: Timestamps.parse_timestamp(json['timestamp'] || json['timestampMs']), altitude: json['altitude'], velocity: json['velocity'], - raw_data: json + raw_data: json, + topic: 'Google Maps Timeline Export', + tracker_id: 'google-maps-timeline-export', + import_id: import.id, + user_id: import.user_id, + created_at: Time.current, + updated_at: Time.current } end + + def bulk_insert_points + Point.upsert_all( + @batch, + unique_by: %i[latitude longitude timestamp user_id], + returning: false + ) + end end diff --git a/app/services/tasks/imports/google_records.rb b/app/services/tasks/imports/google_records.rb index 8f8839e3..9b91f76f 100644 --- a/app/services/tasks/imports/google_records.rb +++ b/app/services/tasks/imports/google_records.rb @@ -4,6 +4,8 @@ # the main source of user's location history data. class Tasks::Imports::GoogleRecords + BATCH_SIZE = 1000 # Adjust based on your needs + def initialize(file_path, user_email) @file_path = file_path @user = User.find_by(email: user_email) @@ -14,10 +16,11 @@ class Tasks::Imports::GoogleRecords import_id = create_import log_start - file_content = read_file - json_data = Oj.load(file_content) - schedule_import_jobs(json_data, import_id) + process_file_in_batches(import_id) log_success + rescue Oj::ParseError => e + Rails.logger.error("JSON parsing error: #{e.message}") + raise end private @@ -26,14 +29,45 @@ class Tasks::Imports::GoogleRecords @user.imports.create(name: @file_path, source: :google_records).id end - def read_file - File.read(@file_path) + def process_file_in_batches(import_id) + batch = [] + + Oj.load_file(@file_path, mode: :compat) do |record| + next unless record.is_a?(Hash) && record['locations'] + + record['locations'].each do |location| + batch << prepare_location_data(location, import_id) + + if batch.size >= BATCH_SIZE + bulk_insert_locations(batch) + batch = [] + end + end + end + + # Process any remaining records + bulk_insert_locations(batch) if batch.any? end - def schedule_import_jobs(json_data, import_id) - json_data['locations'].each do |json| - Import::GoogleTakeoutJob.perform_later(import_id, json.to_json) - end + def prepare_location_data(location, import_id) + { + import_id: import_id, + latitude: location['latitudeE7']&.to_f&. / 1e7, + longitude: location['longitudeE7']&.to_f&. / 1e7, + timestamp: Time.at(location['timestampMs'].to_i / 1000), + accuracy: location['accuracy'], + source_data: location.to_json, + created_at: Time.current, + updated_at: Time.current + } + end + + def bulk_insert_locations(batch) + Location.upsert_all( + batch, + unique_by: %i[import_id timestamp], + returning: false + ) end def log_start From 0ff47f3ac7e3e9b1f450749fea6fab513c45801b Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Tue, 21 Jan 2025 16:03:52 +0100 Subject: [PATCH 2/7] Fix Google Records import --- app/services/tasks/imports/google_records.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/app/services/tasks/imports/google_records.rb b/app/services/tasks/imports/google_records.rb index 9b91f76f..fc74a823 100644 --- a/app/services/tasks/imports/google_records.rb +++ b/app/services/tasks/imports/google_records.rb @@ -39,22 +39,22 @@ class Tasks::Imports::GoogleRecords batch << prepare_location_data(location, import_id) if batch.size >= BATCH_SIZE - bulk_insert_locations(batch) + bulk_insert_points(batch) batch = [] end end end # Process any remaining records - bulk_insert_locations(batch) if batch.any? + bulk_insert_points(batch) if batch.any? end def prepare_location_data(location, import_id) { import_id: import_id, - latitude: location['latitudeE7']&.to_f&. / 1e7, - longitude: location['longitudeE7']&.to_f&. / 1e7, - timestamp: Time.at(location['timestampMs'].to_i / 1000), + latitude: location['latitudeE7']&.to_f&.div(1e7), + longitude: location['longitudeE7']&.to_f&.div(1e7), + timestamp: Time.zone.at(location['timestampMs'].to_i / 1000), accuracy: location['accuracy'], source_data: location.to_json, created_at: Time.current, @@ -62,8 +62,8 @@ class Tasks::Imports::GoogleRecords } end - def bulk_insert_locations(batch) - Location.upsert_all( + def bulk_insert_points(batch) + Point.upsert_all( batch, unique_by: %i[import_id timestamp], returning: false From b43810b1fb23bba9d9cda7beadf2a99b0540f094 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Tue, 21 Jan 2025 19:14:36 +0100 Subject: [PATCH 3/7] Import Google Records JSON in batches --- app/jobs/import/google_takeout_job.rb | 6 +- app/models/import.rb | 2 + app/services/google_maps/records_parser.rb | 83 ++++++++++++++------ app/services/tasks/imports/google_records.rb | 42 +++------- config/initializers/sidekiq.rb | 1 + 5 files changed, 77 insertions(+), 57 deletions(-) diff --git a/app/jobs/import/google_takeout_job.rb b/app/jobs/import/google_takeout_job.rb index 4cb74ae2..3bfe965c 100644 --- a/app/jobs/import/google_takeout_job.rb +++ b/app/jobs/import/google_takeout_job.rb @@ -4,10 +4,10 @@ class Import::GoogleTakeoutJob < ApplicationJob queue_as :imports sidekiq_options retry: false - def perform(import_id, json_array) + def perform(import_id, locations, current_index) + locations_batch = Oj.load(locations) import = Import.find(import_id) - records = Oj.load(json_array) - GoogleMaps::RecordsParser.new(import).call(records) + GoogleMaps::RecordsParser.new(import, current_index).call(locations_batch) end end diff --git a/app/models/import.rb b/app/models/import.rb index f396c555..045e8b5f 100644 --- a/app/models/import.rb +++ b/app/models/import.rb @@ -4,6 +4,8 @@ class Import < ApplicationRecord belongs_to :user has_many :points, dependent: :destroy + delegate :count, to: :points, prefix: true + include ImportUploader::Attachment(:raw) enum :source, { diff --git a/app/services/google_maps/records_parser.rb b/app/services/google_maps/records_parser.rb index e2fe2e05..0762e0d6 100644 --- a/app/services/google_maps/records_parser.rb +++ b/app/services/google_maps/records_parser.rb @@ -1,51 +1,88 @@ # frozen_string_literal: true class GoogleMaps::RecordsParser - BATCH_SIZE = 1000 - attr_reader :import + include Imports::Broadcaster - def initialize(import) + BATCH_SIZE = 1000 + attr_reader :import, :current_index + + def initialize(import, current_index = 0) @import = import @batch = [] + @current_index = current_index end - def call(records) - Array(records).each do |record| - @batch << parse_json(record) + def call(locations) + Array(locations).each do |location| + @batch << prepare_location_data(location) + next unless @batch.size >= BATCH_SIZE - if @batch.size >= BATCH_SIZE - bulk_insert_points - @batch = [] - end + bulk_insert_points(@batch) + broadcast_import_progress(import, current_index) + @batch = [] end - bulk_insert_points if @batch.any? + return unless @batch.any? + + bulk_insert_points(@batch) + broadcast_import_progress(import, current_index) end private - def parse_json(json) + # rubocop:disable Metrics/MethodLength + def prepare_location_data(location) { - latitude: json['latitudeE7'].to_f / 10**7, - longitude: json['longitudeE7'].to_f / 10**7, - timestamp: Timestamps.parse_timestamp(json['timestamp'] || json['timestampMs']), - altitude: json['altitude'], - velocity: json['velocity'], - raw_data: json, + latitude: location['latitudeE7'].to_f / 10**7, + longitude: location['longitudeE7'].to_f / 10**7, + timestamp: Timestamps.parse_timestamp(location['timestamp'] || location['timestampMs']), + altitude: location['altitude'], + velocity: location['velocity'], + raw_data: location, topic: 'Google Maps Timeline Export', tracker_id: 'google-maps-timeline-export', - import_id: import.id, - user_id: import.user_id, + import_id: @import.id, + user_id: @import.user_id, created_at: Time.current, updated_at: Time.current } end - def bulk_insert_points + # rubocop:enable Metrics/MethodLength + def bulk_insert_points(batch) + # Deduplicate records within the batch before upserting + # Use all fields in the unique constraint for deduplication + unique_batch = deduplicate_batch(batch) + + # Sort the batch to ensure consistent ordering and prevent deadlocks + # sorted_batch = sort_batch(unique_batch) + Point.upsert_all( - @batch, + unique_batch, unique_by: %i[latitude longitude timestamp user_id], - returning: false + returning: false, + on_duplicate: :skip + ) + rescue StandardError => e + Rails.logger.error("Batch insert failed for import #{@import.id}: #{e.message}") + + # Create notification for the user + Notification.create!( + user: @import.user, + title: 'Google Maps Import Error', + content: "Failed to process location batch: #{e.message}", + kind: :error ) end + + def deduplicate_batch(batch) + batch.uniq do |record| + [ + record[:latitude].round(7), + record[:longitude].round(7), + record[:timestamp], + record[:user_id] + ] + end + end end diff --git a/app/services/tasks/imports/google_records.rb b/app/services/tasks/imports/google_records.rb index fc74a823..83174128 100644 --- a/app/services/tasks/imports/google_records.rb +++ b/app/services/tasks/imports/google_records.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true -# This class is named based on Google Takeout's Records.json file, -# the main source of user's location history data. +# This class is named based on Google Takeout's Records.json file class Tasks::Imports::GoogleRecords BATCH_SIZE = 1000 # Adjust based on your needs @@ -35,39 +34,20 @@ class Tasks::Imports::GoogleRecords Oj.load_file(@file_path, mode: :compat) do |record| next unless record.is_a?(Hash) && record['locations'] - record['locations'].each do |location| - batch << prepare_location_data(location, import_id) + index = 0 - if batch.size >= BATCH_SIZE - bulk_insert_points(batch) - batch = [] - end + record['locations'].each do |location| + batch << location + + next unless batch.size >= BATCH_SIZE + + index += BATCH_SIZE + Import::GoogleTakeoutJob.perform_later(import_id, Oj.dump(batch), index) + batch = [] end end - # Process any remaining records - bulk_insert_points(batch) if batch.any? - end - - def prepare_location_data(location, import_id) - { - import_id: import_id, - latitude: location['latitudeE7']&.to_f&.div(1e7), - longitude: location['longitudeE7']&.to_f&.div(1e7), - timestamp: Time.zone.at(location['timestampMs'].to_i / 1000), - accuracy: location['accuracy'], - source_data: location.to_json, - created_at: Time.current, - updated_at: Time.current - } - end - - def bulk_insert_points(batch) - Point.upsert_all( - batch, - unique_by: %i[import_id timestamp], - returning: false - ) + Import::GoogleTakeoutJob.perform_later(import_id, Oj.dump(batch)) if batch.any? end def log_start diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index d9dec786..ab3f00c5 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -2,6 +2,7 @@ Sidekiq.configure_server do |config| config.redis = { url: ENV['REDIS_URL'] } + config.logger = Sidekiq::Logger.new($stdout) if ENV['PROMETHEUS_EXPORTER_ENABLED'].to_s == 'true' require 'prometheus_exporter/instrumentation' From 510868a5940047e330b1f1e4f1c5ae633f2d0f58 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Tue, 21 Jan 2025 19:32:12 +0100 Subject: [PATCH 4/7] Fix failed specs --- app/services/google_maps/records_parser.rb | 6 +- .../google_maps/records_parser_spec.rb | 65 ++++++++++++++----- .../tasks/imports/google_records_spec.rb | 4 +- spec/tasks/import_spec.rb | 2 +- 4 files changed, 58 insertions(+), 19 deletions(-) diff --git a/app/services/google_maps/records_parser.rb b/app/services/google_maps/records_parser.rb index 0762e0d6..32f02723 100644 --- a/app/services/google_maps/records_parser.rb +++ b/app/services/google_maps/records_parser.rb @@ -35,7 +35,7 @@ class GoogleMaps::RecordsParser { latitude: location['latitudeE7'].to_f / 10**7, longitude: location['longitudeE7'].to_f / 10**7, - timestamp: Timestamps.parse_timestamp(location['timestamp'] || location['timestampMs']), + timestamp: parse_timestamp(location), altitude: location['altitude'], velocity: location['velocity'], raw_data: location, @@ -85,4 +85,8 @@ class GoogleMaps::RecordsParser ] end end + + def parse_timestamp(location) + Timestamps.parse_timestamp(location['timestamp'] || location['timestampMs']) + end end diff --git a/spec/services/google_maps/records_parser_spec.rb b/spec/services/google_maps/records_parser_spec.rb index 96495dad..da99f501 100644 --- a/spec/services/google_maps/records_parser_spec.rb +++ b/spec/services/google_maps/records_parser_spec.rb @@ -4,21 +4,36 @@ require 'rails_helper' RSpec.describe GoogleMaps::RecordsParser do describe '#call' do - subject(:parser) { described_class.new(import).call(json) } + subject(:parser) { described_class.new(import).call(locations) } let(:import) { create(:import) } let(:time) { DateTime.new(2025, 1, 1, 12, 0, 0) } - let(:json) do - { - 'latitudeE7' => 123_456_789, - 'longitudeE7' => 123_456_789, - 'altitude' => 0, - 'velocity' => 0 - } + let(:locations) do + [ + { + 'timestampMs' => (time.to_f * 1000).to_i.to_s, + 'latitudeE7' => 123_456_789, + 'longitudeE7' => 123_456_789, + 'accuracy' => 10, + 'altitude' => 100, + 'verticalAccuracy' => 5, + 'activity' => [ + { + 'timestampMs' => (time.to_f * 1000).to_i.to_s, + 'activity' => [ + { + 'type' => 'STILL', + 'confidence' => 100 + } + ] + } + ] + } + ] end context 'with regular timestamp' do - let(:json) { super().merge('timestamp' => time.to_s) } + let(:locations) { super()[0].merge('timestamp' => time.to_s).to_json } it 'creates a point' do expect { parser }.to change(Point, :count).by(1) @@ -26,11 +41,23 @@ RSpec.describe GoogleMaps::RecordsParser do end context 'when point already exists' do - let(:json) { super().merge('timestamp' => time.to_s) } + let(:locations) do + [ + super()[0].merge( + 'timestamp' => time.to_s, + 'latitudeE7' => 123_456_789, + 'longitudeE7' => 123_456_789 + ) + ] + end before do create( - :point, user: import.user, import:, latitude: 12.3456789, longitude: 12.3456789, + :point, + user: import.user, + import: import, + latitude: 12.3456789, + longitude: 12.3456789, timestamp: time.to_i ) end @@ -41,7 +68,9 @@ RSpec.describe GoogleMaps::RecordsParser do end context 'with timestampMs in milliseconds' do - let(:json) { super().merge('timestampMs' => (time.to_f * 1000).to_i.to_s) } + let(:locations) do + [super()[0].merge('timestampMs' => (time.to_f * 1000).to_i.to_s)] + end it 'creates a point using milliseconds timestamp' do expect { parser }.to change(Point, :count).by(1) @@ -49,7 +78,9 @@ RSpec.describe GoogleMaps::RecordsParser do end context 'with ISO 8601 timestamp' do - let(:json) { super().merge('timestamp' => time.iso8601) } + let(:locations) do + [super()[0].merge('timestamp' => time.iso8601)] + end it 'parses ISO 8601 timestamp correctly' do expect { parser }.to change(Point, :count).by(1) @@ -59,7 +90,9 @@ RSpec.describe GoogleMaps::RecordsParser do end context 'with timestamp in milliseconds' do - let(:json) { super().merge('timestamp' => (time.to_f * 1000).to_i.to_s) } + let(:locations) do + [super()[0].merge('timestamp' => (time.to_f * 1000).to_i.to_s)] + end it 'parses millisecond timestamp correctly' do expect { parser }.to change(Point, :count).by(1) @@ -69,7 +102,9 @@ RSpec.describe GoogleMaps::RecordsParser do end context 'with timestamp in seconds' do - let(:json) { super().merge('timestamp' => time.to_i.to_s) } + let(:locations) do + [super()[0].merge('timestamp' => time.to_i.to_s)] + end it 'parses second timestamp correctly' do expect { parser }.to change(Point, :count).by(1) diff --git a/spec/services/tasks/imports/google_records_spec.rb b/spec/services/tasks/imports/google_records_spec.rb index 0310dbd1..29fddfdf 100644 --- a/spec/services/tasks/imports/google_records_spec.rb +++ b/spec/services/tasks/imports/google_records_spec.rb @@ -5,10 +5,10 @@ require 'rails_helper' RSpec.describe Tasks::Imports::GoogleRecords do describe '#call' do let(:user) { create(:user) } - let(:file_path) { Rails.root.join('spec/fixtures/files/google/records.json') } + let(:file_path) { Rails.root.join('spec/fixtures/files/google/records.json').to_s } it 'schedules the Import::GoogleTakeoutJob' do - expect(Import::GoogleTakeoutJob).to receive(:perform_later).exactly(3).times + expect(Import::GoogleTakeoutJob).to receive(:perform_later).exactly(1).time described_class.new(file_path, user.email).call end diff --git a/spec/tasks/import_spec.rb b/spec/tasks/import_spec.rb index 4cd785db..0e718f76 100644 --- a/spec/tasks/import_spec.rb +++ b/spec/tasks/import_spec.rb @@ -3,7 +3,7 @@ require 'rails_helper' describe 'import.rake' do - let(:file_path) { Rails.root.join('spec/fixtures/files/google/records.json') } + let(:file_path) { Rails.root.join('spec/fixtures/files/google/records.json').to_s } let(:user) { create(:user) } it 'calls importing class' do From 591543fe98ab58993d60d5d7eaa5c025c9d87aed Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Tue, 21 Jan 2025 19:42:04 +0100 Subject: [PATCH 5/7] Rename GoogleMaps::RecordsParser to GoogleMaps::RecordsImporter --- app/jobs/import/google_takeout_job.rb | 2 +- ...{records_parser.rb => records_importer.rb} | 48 ++++++++----------- ...arser_spec.rb => records_importer_spec.rb} | 2 +- 3 files changed, 22 insertions(+), 30 deletions(-) rename app/services/google_maps/{records_parser.rb => records_importer.rb} (63%) rename spec/services/google_maps/{records_parser_spec.rb => records_importer_spec.rb} (98%) diff --git a/app/jobs/import/google_takeout_job.rb b/app/jobs/import/google_takeout_job.rb index 3bfe965c..02702cf7 100644 --- a/app/jobs/import/google_takeout_job.rb +++ b/app/jobs/import/google_takeout_job.rb @@ -8,6 +8,6 @@ class Import::GoogleTakeoutJob < ApplicationJob locations_batch = Oj.load(locations) import = Import.find(import_id) - GoogleMaps::RecordsParser.new(import, current_index).call(locations_batch) + GoogleMaps::RecordsImporter.new(import, current_index).call(locations_batch) end end diff --git a/app/services/google_maps/records_parser.rb b/app/services/google_maps/records_importer.rb similarity index 63% rename from app/services/google_maps/records_parser.rb rename to app/services/google_maps/records_importer.rb index 32f02723..c7edfb1f 100644 --- a/app/services/google_maps/records_parser.rb +++ b/app/services/google_maps/records_importer.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class GoogleMaps::RecordsParser +class GoogleMaps::RecordsImporter include Imports::Broadcaster BATCH_SIZE = 1000 @@ -13,19 +13,11 @@ class GoogleMaps::RecordsParser end def call(locations) - Array(locations).each do |location| - @batch << prepare_location_data(location) - next unless @batch.size >= BATCH_SIZE - - bulk_insert_points(@batch) + Array(locations).each_slice(BATCH_SIZE) do |location_batch| + batch = location_batch.map { prepare_location_data(_1) } + bulk_insert_points(batch) broadcast_import_progress(import, current_index) - @batch = [] end - - return unless @batch.any? - - bulk_insert_points(@batch) - broadcast_import_progress(import, current_index) end private @@ -47,32 +39,21 @@ class GoogleMaps::RecordsParser updated_at: Time.current } end - # rubocop:enable Metrics/MethodLength + def bulk_insert_points(batch) - # Deduplicate records within the batch before upserting - # Use all fields in the unique constraint for deduplication unique_batch = deduplicate_batch(batch) - # Sort the batch to ensure consistent ordering and prevent deadlocks - # sorted_batch = sort_batch(unique_batch) - + # rubocop:disable Rails/SkipsModelValidations Point.upsert_all( unique_batch, unique_by: %i[latitude longitude timestamp user_id], returning: false, on_duplicate: :skip ) + # rubocop:enable Rails/SkipsModelValidations rescue StandardError => e - Rails.logger.error("Batch insert failed for import #{@import.id}: #{e.message}") - - # Create notification for the user - Notification.create!( - user: @import.user, - title: 'Google Maps Import Error', - content: "Failed to process location batch: #{e.message}", - kind: :error - ) + create_notification("Failed to process location batch: #{e.message}") end def deduplicate_batch(batch) @@ -87,6 +68,17 @@ class GoogleMaps::RecordsParser end def parse_timestamp(location) - Timestamps.parse_timestamp(location['timestamp'] || location['timestampMs']) + Timestamps.parse_timestamp( + location['timestamp'] || location['timestampMs'] + ) + end + + def create_notification(message) + Notification.create!( + user: @import.user, + title: 'Google\'s Records.json Import Error', + content: message, + kind: :error + ) end end diff --git a/spec/services/google_maps/records_parser_spec.rb b/spec/services/google_maps/records_importer_spec.rb similarity index 98% rename from spec/services/google_maps/records_parser_spec.rb rename to spec/services/google_maps/records_importer_spec.rb index da99f501..8ce4d69d 100644 --- a/spec/services/google_maps/records_parser_spec.rb +++ b/spec/services/google_maps/records_importer_spec.rb @@ -2,7 +2,7 @@ require 'rails_helper' -RSpec.describe GoogleMaps::RecordsParser do +RSpec.describe GoogleMaps::RecordsImporter do describe '#call' do subject(:parser) { described_class.new(import).call(locations) } From 9cf3685eb6516b4ae68e410268be5bb29fb5766e Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Tue, 21 Jan 2025 19:46:23 +0100 Subject: [PATCH 6/7] Update app version and changelog --- .app_version | 2 +- CHANGELOG.md | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.app_version b/.app_version index fda96dcf..9e40e75c 100644 --- a/.app_version +++ b/.app_version @@ -1 +1 @@ -0.23.2 +0.23.3 diff --git a/CHANGELOG.md b/CHANGELOG.md index 04da2e91..51f3f561 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -# 0.23.2 - 2025-01-21 +# 0.23.3 - 2025-01-21 + +### Fixed + +- Drastically improved performance for Google's Records.json import. ### Fixed From 96b3a1b02200e0f4f4bd985ad9c1be266564e352 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Tue, 21 Jan 2025 23:28:02 +0100 Subject: [PATCH 7/7] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51f3f561..f50e154a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Fixed -- Drastically improved performance for Google's Records.json import. +- Drastically improved performance for Google's Records.json import. It will now take less than 5 minutes to import 500,000 points, which previously took a few hours. ### Fixed