From 96c9f1030c70402500dff773799f3dbbbfa621e9 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Fri, 22 Aug 2025 20:13:10 +0200 Subject: [PATCH] Move json loading to a module --- app/controllers/imports_controller.rb | 18 ++---------- app/services/geojson/importer.rb | 12 +------- .../google_maps/phone_takeout_importer.rb | 8 ++---- .../google_maps/records_storage_importer.rb | 28 ++++--------------- .../google_maps/semantic_history_importer.rb | 11 +------- app/services/gpx/track_importer.rb | 7 ++--- app/services/imports/file_loader.rb | 26 +++++++++++++++++ app/services/own_tracks/importer.rb | 7 ++--- app/services/photos/importer.rb | 8 ++---- 9 files changed, 45 insertions(+), 80 deletions(-) create mode 100644 app/services/imports/file_loader.rb diff --git a/app/controllers/imports_controller.rb b/app/controllers/imports_controller.rb index bbbfafb3..8240c278 100644 --- a/app/controllers/imports_controller.rb +++ b/app/controllers/imports_controller.rb @@ -104,12 +104,7 @@ class ImportsController < ApplicationController import = current_user.imports.build(name: blob.filename.to_s) import.file.attach(blob) - - # Auto-detect source if not already set - if import.source.blank? - detected_source = detect_import_source(import.file) - import.source = detected_source if detected_source - end + import.source = detect_import_source(import.file) if import.source.blank? import.save! @@ -117,16 +112,9 @@ class ImportsController < ApplicationController end def detect_import_source(file_attachment) - # Download file to temporary location for source detection temp_file_path = Imports::SecureFileDownloader.new(file_attachment).download_to_temp_file - - # Detect source using optimized file-based detection - detector = Imports::SourceDetector.new_from_file(temp_file_path) - detected_source = detector.detect_source - - Rails.logger.info "Auto-detected import source: #{detected_source || 'unknown'} for file: #{file_attachment.filename}" - - detected_source + + Imports::SourceDetector.new_from_file(temp_file_path).detect_source rescue StandardError => e Rails.logger.warn "Failed to auto-detect import source for #{file_attachment.filename}: #{e.message}" nil diff --git a/app/services/geojson/importer.rb b/app/services/geojson/importer.rb index 2f52d4c3..94230047 100644 --- a/app/services/geojson/importer.rb +++ b/app/services/geojson/importer.rb @@ -2,6 +2,7 @@ class Geojson::Importer include Imports::Broadcaster + include Imports::FileLoader include PointValidation attr_reader :import, :user_id, :file_path @@ -25,15 +26,4 @@ class Geojson::Importer broadcast_import_progress(import, index) end end - - private - - def load_json_data - if file_path && File.exist?(file_path) - Oj.load_file(file_path, mode: :compat) - else - file_content = Imports::SecureFileDownloader.new(import.file).download_with_verification - Oj.load(file_content, mode: :compat) - end - end end diff --git a/app/services/google_maps/phone_takeout_importer.rb b/app/services/google_maps/phone_takeout_importer.rb index 67305e42..51cfda5c 100644 --- a/app/services/google_maps/phone_takeout_importer.rb +++ b/app/services/google_maps/phone_takeout_importer.rb @@ -2,6 +2,7 @@ class GoogleMaps::PhoneTakeoutImporter include Imports::Broadcaster + include Imports::FileLoader attr_reader :import, :user_id, :file_path @@ -47,12 +48,7 @@ class GoogleMaps::PhoneTakeoutImporter raw_signals = [] raw_array = [] - json = if file_path && File.exist?(file_path) - Oj.load_file(file_path, mode: :compat) - else - file_content = Imports::SecureFileDownloader.new(import.file).download_with_verification - Oj.load(file_content, mode: :compat) - end + json = load_json_data if json.is_a?(Array) raw_array = parse_raw_array(json) diff --git a/app/services/google_maps/records_storage_importer.rb b/app/services/google_maps/records_storage_importer.rb index 3f4520fb..f47c15fd 100644 --- a/app/services/google_maps/records_storage_importer.rb +++ b/app/services/google_maps/records_storage_importer.rb @@ -4,6 +4,8 @@ # via the UI, vs the CLI, which uses the `GoogleMaps::RecordsImporter` class. class GoogleMaps::RecordsStorageImporter + include Imports::FileLoader + BATCH_SIZE = 1000 def initialize(import, user_id, file_path = nil) @@ -24,31 +26,13 @@ class GoogleMaps::RecordsStorageImporter attr_reader :import, :user, :file_path def process_file_in_batches - locations = if file_path && File.exist?(file_path) - # Use streaming for large files - parse_file_from_path(file_path) - else - # Fallback to traditional method - file_content = Imports::SecureFileDownloader.new(import.file).download_with_verification - parse_file(file_content) - end + parsed_file = load_json_data + return unless parsed_file.is_a?(Hash) && parsed_file['locations'] + + locations = parsed_file['locations'] process_locations_in_batches(locations) if locations.present? end - def parse_file_from_path(file_path) - parsed_file = Oj.load_file(file_path, mode: :compat) - return nil unless parsed_file.is_a?(Hash) && parsed_file['locations'] - - parsed_file['locations'] - end - - def parse_file(file_content) - parsed_file = Oj.load(file_content, mode: :compat) - return nil unless parsed_file.is_a?(Hash) && parsed_file['locations'] - - parsed_file['locations'] - end - def process_locations_in_batches(locations) batch = [] index = 0 diff --git a/app/services/google_maps/semantic_history_importer.rb b/app/services/google_maps/semantic_history_importer.rb index 08be9919..05601a7a 100644 --- a/app/services/google_maps/semantic_history_importer.rb +++ b/app/services/google_maps/semantic_history_importer.rb @@ -2,6 +2,7 @@ class GoogleMaps::SemanticHistoryImporter include Imports::Broadcaster + include Imports::FileLoader BATCH_SIZE = 1000 attr_reader :import, :user_id, :file_path @@ -69,16 +70,6 @@ class GoogleMaps::SemanticHistoryImporter end.compact end - def load_json_data - if file_path && File.exist?(file_path) - # Use streaming JSON loading for better memory efficiency - Oj.load_file(file_path, mode: :compat) - else - # Fallback to traditional method - file_content = Imports::SecureFileDownloader.new(import.file).download_with_verification - Oj.load(file_content, mode: :compat) - end - end def parse_timeline_object(timeline_object) if timeline_object['activitySegment'].present? diff --git a/app/services/gpx/track_importer.rb b/app/services/gpx/track_importer.rb index 49c6cdc9..2a25cc99 100644 --- a/app/services/gpx/track_importer.rb +++ b/app/services/gpx/track_importer.rb @@ -4,6 +4,7 @@ require 'rexml/document' class Gpx::TrackImporter include Imports::Broadcaster + include Imports::FileLoader attr_reader :import, :user_id, :file_path @@ -14,11 +15,7 @@ class Gpx::TrackImporter end def call - file_content = if file_path && File.exist?(file_path) - File.read(file_path) - else - Imports::SecureFileDownloader.new(import.file).download_with_verification - end + file_content = load_file_content json = Hash.from_xml(file_content) tracks = json['gpx']['trk'] diff --git a/app/services/imports/file_loader.rb b/app/services/imports/file_loader.rb new file mode 100644 index 00000000..b26d4188 --- /dev/null +++ b/app/services/imports/file_loader.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Imports + module FileLoader + extend ActiveSupport::Concern + + private + + def load_json_data + if file_path && File.exist?(file_path) + Oj.load_file(file_path, mode: :compat) + else + file_content = Imports::SecureFileDownloader.new(import.file).download_with_verification + Oj.load(file_content, mode: :compat) + end + end + + def load_file_content + if file_path && File.exist?(file_path) + File.read(file_path) + else + Imports::SecureFileDownloader.new(import.file).download_with_verification + end + end + end +end diff --git a/app/services/own_tracks/importer.rb b/app/services/own_tracks/importer.rb index 8f08517a..70fcf2e4 100644 --- a/app/services/own_tracks/importer.rb +++ b/app/services/own_tracks/importer.rb @@ -2,6 +2,7 @@ class OwnTracks::Importer include Imports::Broadcaster + include Imports::FileLoader attr_reader :import, :user_id, :file_path @@ -12,11 +13,7 @@ class OwnTracks::Importer end def call - file_content = if file_path && File.exist?(file_path) - File.read(file_path) - else - Imports::SecureFileDownloader.new(import.file).download_with_verification - end + file_content = load_file_content parsed_data = OwnTracks::RecParser.new(file_content).call points_data = parsed_data.map do |point| diff --git a/app/services/photos/importer.rb b/app/services/photos/importer.rb index e16f56c7..e307b6b1 100644 --- a/app/services/photos/importer.rb +++ b/app/services/photos/importer.rb @@ -2,6 +2,7 @@ class Photos::Importer include Imports::Broadcaster + include Imports::FileLoader include PointValidation attr_reader :import, :user_id, :file_path @@ -12,12 +13,7 @@ class Photos::Importer end def call - json = if file_path && File.exist?(file_path) - Oj.load_file(file_path, mode: :compat) - else - file_content = Imports::SecureFileDownloader.new(import.file).download_with_verification - Oj.load(file_content, mode: :compat) - end + json = load_json_data json.each.with_index(1) { |point, index| create_point(point, index) } end