From b049c11542cbe25369c095ade1e7e55989376bcd Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Sat, 23 Aug 2025 16:07:15 +0200 Subject: [PATCH 1/9] Fix import detection --- CHANGELOG.md | 10 ++++++++ app/controllers/imports_controller.rb | 24 +++++++++---------- .../settings/background_jobs_controller.rb | 7 ++++-- app/models/import.rb | 2 +- app/services/imports/create.rb | 13 +++++++++- app/services/imports/source_detector.rb | 12 +++++----- ...5940_remove_default_from_imports_source.rb | 5 ++++ db/schema.rb | 7 +++--- spec/factories/imports.rb | 2 +- ...json => phone-takeout_w_3_duplicates.json} | 2 -- .../phone_takeout_importer_spec.rb | 2 +- spec/services/imports/create_spec.rb | 5 ++-- spec/services/imports/source_detector_spec.rb | 4 ++-- 13 files changed, 61 insertions(+), 34 deletions(-) create mode 100644 db/migrate/20250823125940_remove_default_from_imports_source.rb rename spec/fixtures/files/google/{phone-takeout.json => phone-takeout_w_3_duplicates.json} (99%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bf98f0c..64d88779 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +# [UNRELEASED] - + +## Changed + +- If user already have import with the same name, it will be appended with timestamp during the import process. + +## Fixed + +- Some types of imports were not being detected correctly and were failing to import. + # [0.30.10] - 2025-08-22 ## Added diff --git a/app/controllers/imports_controller.rb b/app/controllers/imports_controller.rb index a0f798ff..3ee75a95 100644 --- a/app/controllers/imports_controller.rb +++ b/app/controllers/imports_controller.rb @@ -102,27 +102,25 @@ class ImportsController < ApplicationController blob = ActiveStorage::Blob.find_signed(signed_id) - import = current_user.imports.build(name: blob.filename.to_s) + import_name = generate_unique_import_name(blob.filename.to_s) + import = current_user.imports.build(name: import_name) import.file.attach(blob) - import.source = detect_import_source(import.file) if import.source.blank? import.save! import end - def detect_import_source(file_attachment) - temp_file_path = Imports::SecureFileDownloader.new(file_attachment).download_to_temp_file + def generate_unique_import_name(original_name) + return original_name unless current_user.imports.exists?(name: original_name) - Imports::SourceDetector.new_from_file_header(temp_file_path).detect_source - rescue StandardError => e - Rails.logger.warn "Failed to auto-detect import source for #{file_attachment.filename}: #{e.message}" - nil - ensure - # Cleanup temp file - if temp_file_path && File.exist?(temp_file_path) - File.unlink(temp_file_path) - end + # Extract filename and extension + basename = File.basename(original_name, File.extname(original_name)) + extension = File.extname(original_name) + + # Add current datetime + timestamp = Time.current.strftime('%Y%m%d_%H%M%S') + "#{basename}_#{timestamp}#{extension}" end def validate_points_limit diff --git a/app/controllers/settings/background_jobs_controller.rb b/app/controllers/settings/background_jobs_controller.rb index 31bda769..fdc1fac3 100644 --- a/app/controllers/settings/background_jobs_controller.rb +++ b/app/controllers/settings/background_jobs_controller.rb @@ -1,12 +1,15 @@ # frozen_string_literal: true class Settings::BackgroundJobsController < ApplicationController - before_action :authenticate_self_hosted! + before_action :authenticate_self_hosted!, unless: lambda { + %w[start_immich_import start_photoprism_import].include?(params[:job_name]) + } + before_action :authenticate_admin!, unless: lambda { %w[start_immich_import start_photoprism_import].include?(params[:job_name]) } - def index;end + def index; end def create EnqueueBackgroundJob.perform_later(params[:job_name], current_user.id) diff --git a/app/models/import.rb b/app/models/import.rb index 74024798..8635f2a9 100644 --- a/app/models/import.rb +++ b/app/models/import.rb @@ -21,7 +21,7 @@ class Import < ApplicationRecord google_semantic_history: 0, owntracks: 1, google_records: 2, google_phone_takeout: 3, gpx: 4, immich_api: 5, geojson: 6, photoprism_api: 7, user_data_archive: 8 - } + }, allow_nil: true def process! if user_data_archive? diff --git a/app/services/imports/create.rb b/app/services/imports/create.rb index 8c8a93f2..a6675c92 100644 --- a/app/services/imports/create.rb +++ b/app/services/imports/create.rb @@ -16,7 +16,12 @@ class Imports::Create temp_file_path = Imports::SecureFileDownloader.new(import.file).download_to_temp_file - source = import.source.presence || detect_source_from_file(temp_file_path) + source = if import.source.nil? || should_detect_source? + detect_source_from_file(temp_file_path) + else + import.source + end + importer(source).new(import, user.id, temp_file_path).call schedule_stats_creating(user.id) @@ -90,8 +95,14 @@ class Imports::Create ).call end + def should_detect_source? + # Don't override API-based sources that can't be reliably detected + !%w[immich_api photoprism_api].include?(import.source) + end + def detect_source_from_file(temp_file_path) detector = Imports::SourceDetector.new_from_file_header(temp_file_path) + detector.detect_source! end diff --git a/app/services/imports/source_detector.rb b/app/services/imports/source_detector.rb index d122892f..7acbb081 100644 --- a/app/services/imports/source_detector.rb +++ b/app/services/imports/source_detector.rb @@ -62,10 +62,10 @@ class Imports::SourceDetector def self.new_from_file_header(file_path) filename = File.basename(file_path) - + # For detection, read only first 2KB to optimize performance header_content = File.open(file_path, 'rb') { |f| f.read(2048) } - + new(header_content, filename, file_path) end @@ -103,7 +103,7 @@ class Imports::SourceDetector # Must have .gpx extension AND contain GPX XML structure return false unless filename.downcase.end_with?('.gpx') - + # Check content for GPX structure content_to_check = if file_path && File.exist?(file_path) # Read first 1KB for GPX detection @@ -111,7 +111,7 @@ class Imports::SourceDetector else file_content end - + content_to_check.strip.start_with?(' Date: Sat, 23 Aug 2025 16:13:52 +0200 Subject: [PATCH 2/9] Fix schema --- db/schema.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/db/schema.rb b/db/schema.rb index 6cb87072..02a2c3be 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -230,7 +230,7 @@ ActiveRecord::Schema[8.0].define(version: 2025_08_23_125940) do t.datetime "end_at", null: false t.bigint "user_id", null: false t.geometry "original_path", limit: {srid: 0, type: "line_string"}, null: false - t.decimal "distance", precision: 8, scale: 2 + t.integer "distance" t.float "avg_speed" t.integer "duration" t.integer "elevation_gain" @@ -274,7 +274,6 @@ ActiveRecord::Schema[8.0].define(version: 2025_08_23_125940) do t.string "last_sign_in_ip" t.integer "status", default: 0 t.datetime "active_until" - t.integer "points_count", default: 0, null: false t.index ["email"], name: "index_users_on_email", unique: true t.index ["reset_password_token"], name: "index_users_on_reset_password_token", unique: true end From 432e1d26697ee95ebcc64616833826d090597dda Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Sat, 23 Aug 2025 16:28:25 +0200 Subject: [PATCH 3/9] Return self-hosted validation --- app/controllers/settings/background_jobs_controller.rb | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/app/controllers/settings/background_jobs_controller.rb b/app/controllers/settings/background_jobs_controller.rb index fdc1fac3..b9f3a597 100644 --- a/app/controllers/settings/background_jobs_controller.rb +++ b/app/controllers/settings/background_jobs_controller.rb @@ -1,10 +1,7 @@ # frozen_string_literal: true class Settings::BackgroundJobsController < ApplicationController - before_action :authenticate_self_hosted!, unless: lambda { - %w[start_immich_import start_photoprism_import].include?(params[:job_name]) - } - + before_action :authenticate_self_hosted! before_action :authenticate_admin!, unless: lambda { %w[start_immich_import start_photoprism_import].include?(params[:job_name]) } From 5a85a56897b832195010a1614834045f8199364a Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Sat, 23 Aug 2025 16:37:53 +0200 Subject: [PATCH 4/9] Update imports table --- app/services/imports/create.rb | 1 + app/views/imports/index.html.erb | 9 +++++---- spec/services/imports/create_spec.rb | 6 ++++++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/app/services/imports/create.rb b/app/services/imports/create.rb index a6675c92..9b5417d0 100644 --- a/app/services/imports/create.rb +++ b/app/services/imports/create.rb @@ -22,6 +22,7 @@ class Imports::Create import.source end + import.update!(source: source) importer(source).new(import, user.id, temp_file_path).call schedule_stats_creating(user.id) diff --git a/app/views/imports/index.html.erb b/app/views/imports/index.html.erb index f86c2c5d..cfcf0bef 100644 --- a/app/views/imports/index.html.erb +++ b/app/views/imports/index.html.erb @@ -36,7 +36,7 @@
- +
@@ -55,7 +55,8 @@ <% @imports.each do |import| %> + data-points-total="<%= import.processed %>" + class="hover"> <% end %> diff --git a/spec/services/imports/create_spec.rb b/spec/services/imports/create_spec.rb index a1c3129a..756268f9 100644 --- a/spec/services/imports/create_spec.rb +++ b/spec/services/imports/create_spec.rb @@ -21,6 +21,12 @@ RSpec.describe Imports::Create do expect(import.reload.status).to eq('processing').or eq('completed') end + it 'updates the import source' do + service.call + + expect(import.reload.source).to eq('owntracks') + end + context 'when import succeeds' do it 'sets status to completed' do service.call From b9ab38ec6c4d3a165dd5c62b2e5ea2dfa9378172 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Sat, 23 Aug 2025 16:39:42 +0200 Subject: [PATCH 5/9] Fix unified imports source handling --- spec/services/users/import_data/imports_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/services/users/import_data/imports_spec.rb b/spec/services/users/import_data/imports_spec.rb index f9ef66e9..9485bb95 100644 --- a/spec/services/users/import_data/imports_spec.rb +++ b/spec/services/users/import_data/imports_spec.rb @@ -221,7 +221,7 @@ RSpec.describe Users::ImportData::Imports, type: :service do created_imports = user.imports.pluck(:name, :source) expect(created_imports).to contain_exactly( ['Valid Import', 'owntracks'], - ['Missing Source Import', 'google_semantic_history'] + ['Missing Source Import', nil] ) end From a6f4a931aff0595d16c04b89a357a3accc523e84 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Sat, 23 Aug 2025 18:46:00 +0200 Subject: [PATCH 6/9] Make sure no errors raised if source is nil or unknown --- app/helpers/trips_helper.rb | 2 ++ app/serializers/api/photo_serializer.rb | 2 ++ app/services/imports/create.rb | 2 ++ app/services/imports/watcher.rb | 4 +++- app/services/photos/thumbnail.rb | 1 + 5 files changed, 10 insertions(+), 1 deletion(-) diff --git a/app/helpers/trips_helper.rb b/app/helpers/trips_helper.rb index 89f7771a..e035b53b 100644 --- a/app/helpers/trips_helper.rb +++ b/app/helpers/trips_helper.rb @@ -21,6 +21,8 @@ module TripsHelper immich_search_url(settings['immich_url'], start_date, end_date) when 'photoprism' photoprism_search_url(settings['photoprism_url'], start_date, end_date) + else + nil # return nil for nil or unknown source end end diff --git a/app/serializers/api/photo_serializer.rb b/app/serializers/api/photo_serializer.rb index c0a1119a..c21e9c6f 100644 --- a/app/serializers/api/photo_serializer.rb +++ b/app/serializers/api/photo_serializer.rb @@ -68,6 +68,8 @@ class Api::PhotoSerializer photo.dig('exifInfo', 'orientation') == '6' ? 'portrait' : 'landscape' when 'photoprism' photo['Portrait'] ? 'portrait' : 'landscape' + else + 'landscape' # default orientation for nil or unknown source end end end diff --git a/app/services/imports/create.rb b/app/services/imports/create.rb index 9b5417d0..1f952715 100644 --- a/app/services/imports/create.rb +++ b/app/services/imports/create.rb @@ -49,6 +49,8 @@ class Imports::Create private def importer(source) + return raise ArgumentError, 'Import source cannot be nil' if source.nil? + case source.to_s when 'google_semantic_history' then GoogleMaps::SemanticHistoryImporter when 'google_phone_takeout' then GoogleMaps::PhoneTakeoutImporter diff --git a/app/services/imports/watcher.rb b/app/services/imports/watcher.rb index 79e0a59c..6467ec06 100644 --- a/app/services/imports/watcher.rb +++ b/app/services/imports/watcher.rb @@ -70,12 +70,14 @@ class Imports::Watcher end def mime_type(source) - case source.to_sym + case source&.to_sym when :gpx then 'application/xml' when :json, :geojson, :google_phone_takeout, :google_records, :google_semantic_history 'application/json' when :owntracks 'application/octet-stream' + when nil + 'application/octet-stream' # fallback MIME type for nil source else raise UnsupportedSourceError, "Unsupported source: #{source}" end diff --git a/app/services/photos/thumbnail.rb b/app/services/photos/thumbnail.rb index 4f927aad..9ad4e57e 100644 --- a/app/services/photos/thumbnail.rb +++ b/app/services/photos/thumbnail.rb @@ -10,6 +10,7 @@ class Photos::Thumbnail end def call + raise ArgumentError, 'Photo source cannot be nil' if source.nil? raise unsupported_source_error unless SUPPORTED_SOURCES.include?(source) HTTParty.get(request_url, headers: headers) From d400d3c9fd08b6202bfcb790132359c8200a872d Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Sat, 23 Aug 2025 19:09:18 +0200 Subject: [PATCH 7/9] Fix minor issues --- app/helpers/trips_helper.rb | 2 -- app/services/imports/create.rb | 2 +- app/services/photos/thumbnail.rb | 4 ++-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/app/helpers/trips_helper.rb b/app/helpers/trips_helper.rb index e035b53b..89f7771a 100644 --- a/app/helpers/trips_helper.rb +++ b/app/helpers/trips_helper.rb @@ -21,8 +21,6 @@ module TripsHelper immich_search_url(settings['immich_url'], start_date, end_date) when 'photoprism' photoprism_search_url(settings['photoprism_url'], start_date, end_date) - else - nil # return nil for nil or unknown source end end diff --git a/app/services/imports/create.rb b/app/services/imports/create.rb index 1f952715..67d05abb 100644 --- a/app/services/imports/create.rb +++ b/app/services/imports/create.rb @@ -49,7 +49,7 @@ class Imports::Create private def importer(source) - return raise ArgumentError, 'Import source cannot be nil' if source.nil? + raise ArgumentError, 'Import source cannot be nil' if source.nil? case source.to_s when 'google_semantic_history' then GoogleMaps::SemanticHistoryImporter diff --git a/app/services/photos/thumbnail.rb b/app/services/photos/thumbnail.rb index 9ad4e57e..ad2da385 100644 --- a/app/services/photos/thumbnail.rb +++ b/app/services/photos/thumbnail.rb @@ -11,7 +11,7 @@ class Photos::Thumbnail def call raise ArgumentError, 'Photo source cannot be nil' if source.nil? - raise unsupported_source_error unless SUPPORTED_SOURCES.include?(source) + unsupported_source_error unless SUPPORTED_SOURCES.include?(source) HTTParty.get(request_url, headers: headers) end @@ -52,7 +52,7 @@ class Photos::Thumbnail request_headers end - def unsupported_source_error + unsupported_source_error unless SUPPORTED_SOURCES.include?(source) raise ArgumentError, "Unsupported source: #{source}" end end From 625b8e614f1c2e377b1117733b5aa00a643506ac Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Sat, 23 Aug 2025 21:57:25 +0200 Subject: [PATCH 8/9] Fix failing spec --- CHANGELOG.md | 2 +- app/services/photos/thumbnail.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64d88779..abdfeb77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## Fixed -- Some types of imports were not being detected correctly and were failing to import. +- Some types of imports were not being detected correctly and were failing to import. #1678 # [0.30.10] - 2025-08-22 diff --git a/app/services/photos/thumbnail.rb b/app/services/photos/thumbnail.rb index ad2da385..c7046a2d 100644 --- a/app/services/photos/thumbnail.rb +++ b/app/services/photos/thumbnail.rb @@ -52,7 +52,7 @@ class Photos::Thumbnail request_headers end - unsupported_source_error unless SUPPORTED_SOURCES.include?(source) + def unsupported_source_error raise ArgumentError, "Unsupported source: #{source}" end end From 86bfe1b1d9678f6a4f6b7fdd19b3e65dca3fb724 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Sat, 23 Aug 2025 22:29:53 +0200 Subject: [PATCH 9/9] Update app version --- .app_version | 2 +- CHANGELOG.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.app_version b/.app_version index bca57db5..3df680e9 100644 --- a/.app_version +++ b/.app_version @@ -1 +1 @@ -0.30.10 +0.30.11 diff --git a/CHANGELOG.md b/CHANGELOG.md index abdfeb77..3d7cca45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -# [UNRELEASED] - +# [0.30.11] - 2025-08-23 ## Changed
Name
<%= link_to import.name, import, class: 'underline hover:no-underline' %> (<%= import.source %>) @@ -72,9 +73,9 @@ <%= human_datetime(import.created_at) %> <% if import.file.present? %> - <%= link_to 'Download', rails_blob_path(import.file, disposition: 'attachment'), class: "px-4 py-2 bg-blue-500 text-white rounded-md", download: import.name %> + <%= link_to 'Download', rails_blob_path(import.file, disposition: 'attachment'), class: "btn btn-outline btn-sm btn-info", download: import.name %> <% end %> - <%= link_to 'Delete', import, data: { confirm: "Are you sure?", turbo_confirm: "Are you sure?", turbo_method: :delete }, method: :delete, class: "px-4 py-2 bg-red-500 text-white rounded-md" %> + <%= link_to 'Delete', import, data: { confirm: "Are you sure?", turbo_confirm: "Are you sure?", turbo_method: :delete }, method: :delete, class: "btn btn-outline btn-sm btn-error" %>