diff --git a/CHANGELOG.md b/CHANGELOG.md index 2be0bd66..0740f2c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - User can now delete a visit by clicking on the delete button in the visit popup. - Import failure now throws an internal server error. +## Changed + +- Source of imports is now being detected automatically. + # [0.30.9] - 2025-08-19 diff --git a/app/controllers/api/v1/visits_controller.rb b/app/controllers/api/v1/visits_controller.rb index d30db35b..248e5ea7 100644 --- a/app/controllers/api/v1/visits_controller.rb +++ b/app/controllers/api/v1/visits_controller.rb @@ -12,14 +12,14 @@ class Api::V1::VisitsController < ApiController def create service = Visits::Create.new(current_api_user, visit_params) - - if service.call + + result = service.call + + if result render json: Api::VisitSerializer.new(service.visit).call else - render json: { - error: 'Failed to create visit', - errors: service.errors - }, status: :unprocessable_entity + error_message = service.errors || 'Failed to create visit' + render json: { error: error_message }, status: :unprocessable_entity end end @@ -77,11 +77,11 @@ class Api::V1::VisitsController < ApiController def destroy visit = current_api_user.visits.find(params[:id]) - + if visit.destroy head :no_content else - render json: { + render json: { error: 'Failed to delete visit', errors: visit.errors.full_messages }, status: :unprocessable_entity diff --git a/app/controllers/imports_controller.rb b/app/controllers/imports_controller.rb index 2d7feef1..a0f798ff 100644 --- a/app/controllers/imports_controller.rb +++ b/app/controllers/imports_controller.rb @@ -43,8 +43,7 @@ class ImportsController < ApplicationController raw_files = Array(files_params).reject(&:blank?) if raw_files.empty? - redirect_to new_import_path, alert: 'No files were selected for upload', status: :unprocessable_entity - return + redirect_to new_import_path, alert: 'No files were selected for upload', status: :unprocessable_entity and return end created_imports = [] @@ -59,11 +58,11 @@ class ImportsController < ApplicationController if created_imports.any? redirect_to imports_url, notice: "#{created_imports.size} files are queued to be imported in background", - status: :see_other + status: :see_other and return else redirect_to new_import_path, alert: 'No valid file references were found. Please upload files using the file selector.', - status: :unprocessable_entity + status: :unprocessable_entity and return end rescue StandardError => e if created_imports.present? @@ -95,7 +94,7 @@ class ImportsController < ApplicationController end def import_params - params.require(:import).permit(:name, :source, files: []) + params.require(:import).permit(:name, files: []) end def create_import_from_signed_id(signed_id) @@ -103,18 +102,29 @@ class ImportsController < ApplicationController blob = ActiveStorage::Blob.find_signed(signed_id) - import = current_user.imports.build( - name: blob.filename.to_s, - source: params[:import][:source] - ) - + import = current_user.imports.build(name: blob.filename.to_s) import.file.attach(blob) + import.source = detect_import_source(import.file) if import.source.blank? import.save! import end + def detect_import_source(file_attachment) + temp_file_path = Imports::SecureFileDownloader.new(file_attachment).download_to_temp_file + + Imports::SourceDetector.new_from_file_header(temp_file_path).detect_source + rescue StandardError => e + Rails.logger.warn "Failed to auto-detect import source for #{file_attachment.filename}: #{e.message}" + nil + ensure + # Cleanup temp file + if temp_file_path && File.exist?(temp_file_path) + File.unlink(temp_file_path) + end + end + def validate_points_limit limit_exceeded = PointsLimitExceeded.new(current_user).call diff --git a/app/services/geojson/importer.rb b/app/services/geojson/importer.rb index 9967cd49..94230047 100644 --- a/app/services/geojson/importer.rb +++ b/app/services/geojson/importer.rb @@ -2,19 +2,19 @@ class Geojson::Importer include Imports::Broadcaster + include Imports::FileLoader include PointValidation - attr_reader :import, :user_id + attr_reader :import, :user_id, :file_path - def initialize(import, user_id) + def initialize(import, user_id, file_path = nil) @import = import @user_id = user_id + @file_path = file_path end def call - file_content = Imports::SecureFileDownloader.new(import.file).download_with_verification - json = Oj.load(file_content) - + json = load_json_data data = Geojson::Params.new(json).call data.each.with_index(1) do |point, index| diff --git a/app/services/google_maps/phone_takeout_importer.rb b/app/services/google_maps/phone_takeout_importer.rb index 90f75f72..51cfda5c 100644 --- a/app/services/google_maps/phone_takeout_importer.rb +++ b/app/services/google_maps/phone_takeout_importer.rb @@ -2,12 +2,14 @@ class GoogleMaps::PhoneTakeoutImporter include Imports::Broadcaster + include Imports::FileLoader - attr_reader :import, :user_id + attr_reader :import, :user_id, :file_path - def initialize(import, user_id) + def initialize(import, user_id, file_path = nil) @import = import @user_id = user_id + @file_path = file_path end def call @@ -46,9 +48,7 @@ class GoogleMaps::PhoneTakeoutImporter raw_signals = [] raw_array = [] - file_content = Imports::SecureFileDownloader.new(import.file).download_with_verification - - json = Oj.load(file_content) + json = load_json_data if json.is_a?(Array) raw_array = parse_raw_array(json) diff --git a/app/services/google_maps/records_storage_importer.rb b/app/services/google_maps/records_storage_importer.rb index 28c80bc8..f47c15fd 100644 --- a/app/services/google_maps/records_storage_importer.rb +++ b/app/services/google_maps/records_storage_importer.rb @@ -4,11 +4,14 @@ # via the UI, vs the CLI, which uses the `GoogleMaps::RecordsImporter` class. class GoogleMaps::RecordsStorageImporter + include Imports::FileLoader + BATCH_SIZE = 1000 - def initialize(import, user_id) + def initialize(import, user_id, file_path = nil) @import = import @user = User.find_by(id: user_id) + @file_path = file_path end def call @@ -20,21 +23,16 @@ class GoogleMaps::RecordsStorageImporter private - attr_reader :import, :user + attr_reader :import, :user, :file_path def process_file_in_batches - file_content = Imports::SecureFileDownloader.new(import.file).download_with_verification - locations = parse_file(file_content) + parsed_file = load_json_data + return unless parsed_file.is_a?(Hash) && parsed_file['locations'] + + locations = parsed_file['locations'] process_locations_in_batches(locations) if locations.present? end - def parse_file(file_content) - parsed_file = Oj.load(file_content, mode: :compat) - return nil unless parsed_file.is_a?(Hash) && parsed_file['locations'] - - parsed_file['locations'] - end - def process_locations_in_batches(locations) batch = [] index = 0 diff --git a/app/services/google_maps/semantic_history_importer.rb b/app/services/google_maps/semantic_history_importer.rb index ae6209b4..e5eeb0b9 100644 --- a/app/services/google_maps/semantic_history_importer.rb +++ b/app/services/google_maps/semantic_history_importer.rb @@ -2,13 +2,15 @@ class GoogleMaps::SemanticHistoryImporter include Imports::Broadcaster + include Imports::FileLoader BATCH_SIZE = 1000 - attr_reader :import, :user_id + attr_reader :import, :user_id, :file_path - def initialize(import, user_id) + def initialize(import, user_id, file_path = nil) @import = import @user_id = user_id + @file_path = file_path @current_index = 0 end @@ -61,8 +63,7 @@ class GoogleMaps::SemanticHistoryImporter end def points_data - file_content = Imports::SecureFileDownloader.new(import.file).download_with_verification - json = Oj.load(file_content) + json = load_json_data json['timelineObjects'].flat_map do |timeline_object| parse_timeline_object(timeline_object) diff --git a/app/services/gpx/track_importer.rb b/app/services/gpx/track_importer.rb index e0207292..2a25cc99 100644 --- a/app/services/gpx/track_importer.rb +++ b/app/services/gpx/track_importer.rb @@ -4,16 +4,18 @@ require 'rexml/document' class Gpx::TrackImporter include Imports::Broadcaster + include Imports::FileLoader - attr_reader :import, :user_id + attr_reader :import, :user_id, :file_path - def initialize(import, user_id) + def initialize(import, user_id, file_path = nil) @import = import @user_id = user_id + @file_path = file_path end def call - file_content = Imports::SecureFileDownloader.new(import.file).download_with_verification + file_content = load_file_content json = Hash.from_xml(file_content) tracks = json['gpx']['trk'] diff --git a/app/services/imports/create.rb b/app/services/imports/create.rb index 766c832a..cac5ad34 100644 --- a/app/services/imports/create.rb +++ b/app/services/imports/create.rb @@ -14,7 +14,10 @@ class Imports::Create import.update!(status: :processing) broadcast_status_update - importer(import.source).new(import, user.id).call + temp_file_path = Imports::SecureFileDownloader.new(import.file).download_to_temp_file + + source = import.source.presence || detect_source_from_file(temp_file_path) + importer(source).new(import, user.id, temp_file_path).call schedule_stats_creating(user.id) schedule_visit_suggesting(user.id, import) @@ -27,6 +30,10 @@ class Imports::Create create_import_failed_notification(import, user, e) ensure + if temp_file_path && File.exist?(temp_file_path) + File.unlink(temp_file_path) + end + if import.processing? import.update!(status: :completed) broadcast_status_update @@ -81,6 +88,11 @@ class Imports::Create ).call end + def detect_source_from_file(temp_file_path) + detector = Imports::SourceDetector.new_from_file_header(temp_file_path) + detector.detect_source! + end + def import_failed_message(import, error) if DawarichSettings.self_hosted? "Import \"#{import.name}\" failed: #{error.message}, stacktrace: #{error.backtrace.join("\n")}" diff --git a/app/services/imports/file_loader.rb b/app/services/imports/file_loader.rb new file mode 100644 index 00000000..b26d4188 --- /dev/null +++ b/app/services/imports/file_loader.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Imports + module FileLoader + extend ActiveSupport::Concern + + private + + def load_json_data + if file_path && File.exist?(file_path) + Oj.load_file(file_path, mode: :compat) + else + file_content = Imports::SecureFileDownloader.new(import.file).download_with_verification + Oj.load(file_content, mode: :compat) + end + end + + def load_file_content + if file_path && File.exist?(file_path) + File.read(file_path) + else + Imports::SecureFileDownloader.new(import.file).download_with_verification + end + end + end +end diff --git a/app/services/imports/secure_file_downloader.rb b/app/services/imports/secure_file_downloader.rb index f4bd2091..d92e64be 100644 --- a/app/services/imports/secure_file_downloader.rb +++ b/app/services/imports/secure_file_downloader.rb @@ -9,6 +9,63 @@ class Imports::SecureFileDownloader end def download_with_verification + file_content = download_to_string + verify_file_integrity(file_content) + file_content + end + + def download_to_temp_file + retries = 0 + temp_file = nil + + begin + Timeout.timeout(DOWNLOAD_TIMEOUT) do + temp_file = create_temp_file + + # Download directly to temp file + storage_attachment.download do |chunk| + temp_file.write(chunk) + end + temp_file.rewind + + # If file is empty, try alternative download method + if temp_file.size == 0 + Rails.logger.warn('No content received from block download, trying alternative method') + temp_file.write(storage_attachment.blob.download) + temp_file.rewind + end + end + rescue Timeout::Error => e + retries += 1 + if retries <= MAX_RETRIES + Rails.logger.warn("Download timeout, attempt #{retries} of #{MAX_RETRIES}") + cleanup_temp_file(temp_file) + retry + else + Rails.logger.error("Download failed after #{MAX_RETRIES} attempts") + cleanup_temp_file(temp_file) + raise + end + rescue StandardError => e + Rails.logger.error("Download error: #{e.message}") + cleanup_temp_file(temp_file) + raise + end + + raise 'Download completed but no content was received' if temp_file.size == 0 + + verify_temp_file_integrity(temp_file) + temp_file.path + ensure + # Keep temp file open so it can be read by other processes + # Caller is responsible for cleanup + end + + private + + attr_reader :storage_attachment + + def download_to_string retries = 0 file_content = nil @@ -51,13 +108,23 @@ class Imports::SecureFileDownloader raise 'Download completed but no content was received' if file_content.nil? || file_content.empty? - verify_file_integrity(file_content) file_content end - private + def create_temp_file + extension = File.extname(storage_attachment.filename.to_s) + basename = File.basename(storage_attachment.filename.to_s, extension) + Tempfile.new(["#{basename}_#{Time.now.to_i}", extension], binmode: true) + end - attr_reader :storage_attachment + def cleanup_temp_file(temp_file) + return unless temp_file + + temp_file.close unless temp_file.closed? + temp_file.unlink if File.exist?(temp_file.path) + rescue StandardError => e + Rails.logger.warn("Failed to cleanup temp file: #{e.message}") + end def verify_file_integrity(file_content) return if file_content.nil? || file_content.empty? @@ -78,4 +145,26 @@ class Imports::SecureFileDownloader raise "Checksum mismatch: expected #{expected_checksum}, got #{actual_checksum}" end + + def verify_temp_file_integrity(temp_file) + return if temp_file.nil? || temp_file.size == 0 + + # Verify file size + expected_size = storage_attachment.blob.byte_size + actual_size = temp_file.size + + if expected_size != actual_size + raise "Incomplete download: expected #{expected_size} bytes, got #{actual_size} bytes" + end + + # Verify checksum + expected_checksum = storage_attachment.blob.checksum + temp_file.rewind + actual_checksum = Base64.strict_encode64(Digest::MD5.digest(temp_file.read)) + temp_file.rewind + + return unless expected_checksum != actual_checksum + + raise "Checksum mismatch: expected #{expected_checksum}, got #{actual_checksum}" + end end diff --git a/app/services/imports/source_detector.rb b/app/services/imports/source_detector.rb new file mode 100644 index 00000000..d122892f --- /dev/null +++ b/app/services/imports/source_detector.rb @@ -0,0 +1,235 @@ +# frozen_string_literal: true + +class Imports::SourceDetector + class UnknownSourceError < StandardError; end + + DETECTION_RULES = { + google_semantic_history: { + required_keys: ['timelineObjects'], + nested_patterns: [ + ['timelineObjects', 0, 'activitySegment'], + ['timelineObjects', 0, 'placeVisit'] + ] + }, + google_records: { + required_keys: ['locations'], + nested_patterns: [ + ['locations', 0, 'latitudeE7'], + ['locations', 0, 'longitudeE7'] + ] + }, + google_phone_takeout: { + alternative_patterns: [ + # Pattern 1: Object with semanticSegments + { + required_keys: ['semanticSegments'], + nested_patterns: [['semanticSegments', 0, 'startTime']] + }, + # Pattern 2: Object with rawSignals + { + required_keys: ['rawSignals'] + }, + # Pattern 3: Array format with visit/activity objects + { + structure: :array, + nested_patterns: [ + [0, 'visit', 'topCandidate', 'placeLocation'], + [0, 'activity'] + ] + } + ] + }, + geojson: { + required_keys: ['type', 'features'], + required_values: { 'type' => 'FeatureCollection' }, + nested_patterns: [ + ['features', 0, 'type'], + ['features', 0, 'geometry'], + ['features', 0, 'properties'] + ] + }, + owntracks: { + structure: :rec_file_lines, + line_pattern: /"_type":"location"/ + } + }.freeze + + def initialize(file_content, filename = nil, file_path = nil) + @file_content = file_content + @filename = filename + @file_path = file_path + end + + def self.new_from_file_header(file_path) + filename = File.basename(file_path) + + # For detection, read only first 2KB to optimize performance + header_content = File.open(file_path, 'rb') { |f| f.read(2048) } + + new(header_content, filename, file_path) + end + + def detect_source + return :gpx if gpx_file? + return :owntracks if owntracks_file? + + json_data = parse_json + return nil unless json_data + + DETECTION_RULES.each do |format, rules| + next if format == :owntracks # Already handled above + + if matches_format?(json_data, rules) + return format + end + end + + nil + end + + def detect_source! + format = detect_source + raise UnknownSourceError, 'Unable to detect file format' unless format + + format + end + + private + + attr_reader :file_content, :filename, :file_path + + def gpx_file? + return false unless filename + + # Must have .gpx extension AND contain GPX XML structure + return false unless filename.downcase.end_with?('.gpx') + + # Check content for GPX structure + content_to_check = if file_path && File.exist?(file_path) + # Read first 1KB for GPX detection + File.open(file_path, 'rb') { |f| f.read(1024) } + else + file_content + end + + content_to_check.strip.start_with?('= current.length + current = current[key] + elsif current.is_a?(Hash) + return false unless current.key?(key) + current = current[key] + else + return false + end + end + + !current.nil? + end +end diff --git a/app/services/own_tracks/importer.rb b/app/services/own_tracks/importer.rb index bc63f5f6..70fcf2e4 100644 --- a/app/services/own_tracks/importer.rb +++ b/app/services/own_tracks/importer.rb @@ -2,16 +2,18 @@ class OwnTracks::Importer include Imports::Broadcaster + include Imports::FileLoader - attr_reader :import, :user_id + attr_reader :import, :user_id, :file_path - def initialize(import, user_id) + def initialize(import, user_id, file_path = nil) @import = import @user_id = user_id + @file_path = file_path end def call - file_content = Imports::SecureFileDownloader.new(import.file).download_with_verification + file_content = load_file_content parsed_data = OwnTracks::RecParser.new(file_content).call points_data = parsed_data.map do |point| diff --git a/app/services/photos/importer.rb b/app/services/photos/importer.rb index f3ce8fc4..e307b6b1 100644 --- a/app/services/photos/importer.rb +++ b/app/services/photos/importer.rb @@ -2,17 +2,18 @@ class Photos::Importer include Imports::Broadcaster + include Imports::FileLoader include PointValidation - attr_reader :import, :user_id + attr_reader :import, :user_id, :file_path - def initialize(import, user_id) + def initialize(import, user_id, file_path = nil) @import = import @user_id = user_id + @file_path = file_path end def call - file_content = Imports::SecureFileDownloader.new(import.file).download_with_verification - json = Oj.load(file_content) + json = load_json_data json.each.with_index(1) { |point, index| create_point(point, index) } end diff --git a/app/services/visits/create.rb b/app/services/visits/create.rb index deef97f6..c2f47310 100644 --- a/app/services/visits/create.rb +++ b/app/services/visits/create.rb @@ -8,6 +8,7 @@ module Visits @user = user @params = params.respond_to?(:with_indifferent_access) ? params.with_indifferent_access : params @visit = nil + @errors = nil end def call @@ -15,10 +16,19 @@ module Visits place = find_or_create_place return false unless place - create_visit(place) + visit = create_visit(place) + visit end + rescue ActiveRecord::RecordInvalid => e + ExceptionReporter.call(e, "Failed to create visit: #{e.message}") + + @errors = "Failed to create visit: #{e.message}" + + false rescue StandardError => e - ExceptionReporter.call(e, 'Failed to create visit') + ExceptionReporter.call(e, "Failed to create visit: #{e.message}") + + @errors = "Failed to create visit: #{e.message}" false end @@ -56,7 +66,7 @@ module Visits place rescue StandardError => e - ExceptionReporter.call(e, 'Failed to create place') + ExceptionReporter.call(e, "Failed to create place: #{e.message}") nil end diff --git a/app/views/imports/_form.html.erb b/app/views/imports/_form.html.erb index 3f2857fb..978baa68 100644 --- a/app/views/imports/_form.html.erb +++ b/app/views/imports/_form.html.erb @@ -4,68 +4,6 @@ direct_upload_user_trial_value: current_user.trial?, direct_upload_target: "form" } do |form| %> -
- -
-
-
- -

JSON files from your Takeout/Location History/Semantic Location History/YEAR

-
-
-
-
- -

The Records.json file from your Google Takeout

-
-
-
-
- -

A JSON file you received after your request for Takeout from your mobile device

-
-
-
-
- -

A .REC file you could find in your volumes/owntracks-recorder/store/rec/USER/TOPIC directory

-
-
-
-
- -

A valid GeoJSON file. For example, a file, exported from a Dawarich instance

-
-
-
-
- -

GPX track file

-
-
-
-
-