diff --git a/app/controllers/imports_controller.rb b/app/controllers/imports_controller.rb index 3b363e6b..24a90e51 100644 --- a/app/controllers/imports_controller.rb +++ b/app/controllers/imports_controller.rb @@ -23,19 +23,17 @@ class ImportsController < ApplicationController def create files = import_params[:files].reject(&:blank?) - import_ids = files.map do |file| - import = current_user.imports.create( + files.each do |file| + import = current_user.imports.build( name: file.original_filename, source: params[:import][:source] ) - import.file.attach(file) + import.file.attach(io: file, filename: file.original_filename, content_type: file.content_type) - import.id + import.save! end - import_ids.each { Import::ProcessJob.perform_later(_1) } - redirect_to imports_url, notice: "#{files.size} files are queued to be imported in background", status: :see_other rescue StandardError => e Import.where(user: current_user, name: files.map(&:original_filename)).destroy_all diff --git a/app/services/immich/import_geodata.rb b/app/services/immich/import_geodata.rb index 63a9232b..770a8087 100644 --- a/app/services/immich/import_geodata.rb +++ b/app/services/immich/import_geodata.rb @@ -20,10 +20,13 @@ class Immich::ImportGeodata create_import_failed_notification(import.name) and return unless import.new_record? - import.raw_data = immich_data_json - import.save! + import.file.attach( + io: StringIO.new(immich_data_json.to_json), + filename: file_name, + content_type: 'application/json' + ) - Import::ProcessJob.perform_later(import.id) + import.save! end private diff --git a/app/services/imports/create.rb b/app/services/imports/create.rb index e34661b1..39b28eb7 100644 --- a/app/services/imports/create.rb +++ b/app/services/imports/create.rb @@ -14,7 +14,7 @@ class Imports::Create create_import_finished_notification(import, user) schedule_stats_creating(user.id) - # schedule_visit_suggesting(user.id, import) # Disabled until places & visits are reworked + schedule_visit_suggesting(user.id, import) rescue StandardError => e create_import_failed_notification(import, user, e) end @@ -44,7 +44,7 @@ class Imports::Create start_at = Time.zone.at(points.first.timestamp) end_at = Time.zone.at(points.last.timestamp) - VisitSuggestingJob.perform_later(user_ids: [user_id], start_at:, end_at:) + VisitSuggestingJob.perform_later(user_id:, start_at:, end_at:) end def create_import_finished_notification(import, user) diff --git a/app/services/imports/watcher.rb b/app/services/imports/watcher.rb index f29cacb8..79e0a59c 100644 --- a/app/services/imports/watcher.rb +++ b/app/services/imports/watcher.rb @@ -16,7 +16,7 @@ class Imports::Watcher file_names = file_names(user_directory_path) file_names.each do |file_name| - process_file(user, user_directory_path, file_name) + create_import(user, user_directory_path, file_name) end end end @@ -26,49 +26,29 @@ class Imports::Watcher def user_directories Dir.entries(WATCHED_DIR_PATH).select do |entry| path = File.join(WATCHED_DIR_PATH, entry) + File.directory?(path) && !['.', '..'].include?(entry) end end - def find_user(file_name) - email = file_name.split('_').first - - User.find_by(email:) - end - def file_names(directory_path) Dir.entries(directory_path).select { |file| SUPPORTED_FORMATS.include?(File.extname(file)) } end - def process_file(user, directory_path, file_name) + def create_import(user, directory_path, file_name) file_path = File.join(directory_path, file_name) import = Import.find_or_initialize_by(user:, name: file_name) return if import.persisted? import.source = source(file_name) - import.raw_data = raw_data(file_path, import.source) + import.file.attach( + io: File.open(file_path), + filename: file_name, + content_type: mime_type(import.source) + ) import.save! - - Import::ProcessJob.perform_later(import.id) - end - - def find_or_initialize_import(user, file_name) - import_name = file_name.split('_')[1..].join('_') - - Import.find_or_initialize_by(user:, name: import_name) - end - - def set_import_attributes(import, file_path, file_name) - source = source(file_name) - - import.source = source - import.raw_data = raw_data(file_path, source) - - import.save! - - import.id end def source(file_name) @@ -89,16 +69,13 @@ class Imports::Watcher end end - def raw_data(file_path, source) - file = File.read(file_path) - + def mime_type(source) case source.to_sym - when :gpx - Hash.from_xml(file) + when :gpx then 'application/xml' when :json, :geojson, :google_phone_takeout, :google_records, :google_semantic_history - JSON.parse(file) + 'application/json' when :owntracks - OwnTracks::RecParser.new(file).call + 'application/octet-stream' else raise UnsupportedSourceError, "Unsupported source: #{source}" end diff --git a/app/services/photoprism/import_geodata.rb b/app/services/photoprism/import_geodata.rb index 2ad5fa6e..2d0e7a68 100644 --- a/app/services/photoprism/import_geodata.rb +++ b/app/services/photoprism/import_geodata.rb @@ -23,8 +23,13 @@ class Photoprism::ImportGeodata import = find_or_create_import(json_data) return create_import_failed_notification(import.name) unless import.new_record? - import.update!(raw_data: json_data) - Import::ProcessJob.perform_later(import.id) + import.file.attach( + io: StringIO.new(json_data.to_json), + filename: file_name(json_data), + content_type: 'application/json' + ) + + import.save! end def find_or_create_import(json_data) diff --git a/spec/jobs/import/process_job_spec.rb b/spec/jobs/import/process_job_spec.rb index d805ac9f..bd102947 100644 --- a/spec/jobs/import/process_job_spec.rb +++ b/spec/jobs/import/process_job_spec.rb @@ -7,7 +7,12 @@ RSpec.describe Import::ProcessJob, type: :job do subject(:perform) { described_class.new.perform(import.id) } let(:user) { create(:user) } - let!(:import) { create(:import, user:, name: 'owntracks_export.json') } + let!(:import) { create(:import, user:, name: '2024-03.rec') } + let(:file_path) { Rails.root.join('spec/fixtures/files/owntracks/2024-03.rec') } + + before do + import.file.attach(io: File.open(file_path), filename: '2024-03.rec', content_type: 'application/octet-stream') + end it 'creates points' do expect { perform }.to change { Point.count }.by(9) diff --git a/spec/services/imports/create_spec.rb b/spec/services/imports/create_spec.rb index 961d014c..02da61b9 100644 --- a/spec/services/imports/create_spec.rb +++ b/spec/services/imports/create_spec.rb @@ -9,6 +9,13 @@ RSpec.describe Imports::Create do describe '#call' do context 'when source is google_semantic_history' do let(:import) { create(:import, source: 'google_semantic_history') } + let(:file_path) { Rails.root.join('spec/fixtures/files/google/semantic_history.json') } + let(:file) { Rack::Test::UploadedFile.new(file_path, 'application/json') } + + before do + import.file.attach(io: File.open(file_path), filename: 'semantic_history.json', + content_type: 'application/json') + end it 'calls the GoogleMaps::SemanticHistoryParser' do expect(GoogleMaps::SemanticHistoryParser).to \ @@ -29,6 +36,12 @@ RSpec.describe Imports::Create do context 'when source is owntracks' do let(:import) { create(:import, source: 'owntracks') } + let(:file_path) { Rails.root.join('spec/fixtures/files/owntracks/2024-03.rec') } + let(:file) { Rack::Test::UploadedFile.new(file_path, 'application/octet-stream') } + + before do + import.file.attach(io: File.open(file_path), filename: '2024-03.rec', content_type: 'application/octet-stream') + end it 'calls the OwnTracks::Importer' do expect(OwnTracks::Importer).to \ @@ -50,7 +63,7 @@ RSpec.describe Imports::Create do end end - xit 'schedules visit suggesting' do + it 'schedules visit suggesting' do Sidekiq::Testing.inline! do expect { service.call }.to have_enqueued_job(VisitSuggestingJob) end @@ -72,6 +85,13 @@ RSpec.describe Imports::Create do context 'when source is gpx' do let(:import) { create(:import, source: 'gpx') } + let(:file_path) { Rails.root.join('spec/fixtures/files/gpx/gpx_track_single_segment.gpx') } + let(:file) { Rack::Test::UploadedFile.new(file_path, 'application/octet-stream') } + + before do + import.file.attach(io: File.open(file_path), filename: 'gpx_track_single_segment.gpx', + content_type: 'application/octet-stream') + end it 'calls the Gpx::TrackImporter' do expect(Gpx::TrackImporter).to \