diff --git a/app/controllers/imports_controller.rb b/app/controllers/imports_controller.rb index fb36bc4a..ca3c34c8 100644 --- a/app/controllers/imports_controller.rb +++ b/app/controllers/imports_controller.rb @@ -32,11 +32,8 @@ class ImportsController < ApplicationController raw_data = case params[:import][:source] - when 'google_semantic_history' then GoogleMaps::SemanticHistoryParser.new(import, current_user.id).call - when 'owntracks' then OwnTracks::RecParser.new(file).call - when 'geojson' then Geojson::ImportParser.new(import, current_user.id).call - when 'google_phone_takeout' then GoogleMaps::PhoneTakeoutParser.new(import, current_user.id).call when 'gpx' then Hash.from_xml(file) + when 'owntracks' then OwnTracks::RecParser.new(file).call else JSON.parse(file) end diff --git a/app/services/imports/watcher.rb b/app/services/imports/watcher.rb index 43f3653b..15854f80 100644 --- a/app/services/imports/watcher.rb +++ b/app/services/imports/watcher.rb @@ -36,7 +36,7 @@ class Imports::Watcher def file_names(directory_path) Dir.entries(directory_path).select do |file| - ['.gpx', '.json'].include?(File.extname(file)) + ['.gpx', '.json', '.rec'].include?(File.extname(file)) end end @@ -73,8 +73,18 @@ class Imports::Watcher def source(file_name) case file_name.split('.').last - when 'json' then :geojson - when 'gpx' then :gpx + when 'json' + if file_name.match?(/location-history/i) + :google_phone_takeout + elsif file_name.match?(/Records/i) + :google_records + elsif file_name.match?(/\d{4}_\w+/i) + :google_semantic_history + else + :geojson + end + when 'rec' then :owntracks + when 'gpx' then :gpx else raise UnsupportedSourceError, 'Unsupported source ' end end @@ -82,6 +92,19 @@ class Imports::Watcher def raw_data(file_path, source) file = File.read(file_path) - source.to_sym == :gpx ? Hash.from_xml(file) : JSON.parse(file) + case source + when :google_phone_takeout + GoogleMaps::PhoneTakeoutParser.new(file).call + when :google_semantic_history + GoogleMaps::SemanticHistoryParser.new(file).call + when :google_records + GoogleMaps::RecordsParser.new(file).call + when :owntracks + OwnTracks::RecParser.new(file).call + when :gpx + Hash.from_xml(file) + else + JSON.parse(file) + end end end diff --git a/spec/features/import_process_spec.rb b/spec/features/import_process_spec.rb deleted file mode 100644 index a71edb0b..00000000 --- a/spec/features/import_process_spec.rb +++ /dev/null @@ -1,20 +0,0 @@ -# spec/features/import_process_spec.rb -require 'rails_helper' - -RSpec.feature 'Import Process', type: :feature do - let(:user) { create(:user) } - - before do - sign_in user - end - - scenario 'User imports a Google Phone Takeout file' do - visit new_import_path - - choose 'Google Phone Takeout' - attach_file 'import_files', Rails.root.join('spec/fixtures/files/google/phone-takeout.json') - click_button 'Create Import' - - expect(page).to have_content('files are queued to be imported in background') - end -end \ No newline at end of file diff --git a/spec/fixtures/files/watched/user@domain.com/export_same_points.json b/spec/fixtures/files/watched/user@domain.com/export_same_points.json deleted file mode 100644 index 2ecfb883..00000000 --- a/spec/fixtures/files/watched/user@domain.com/export_same_points.json +++ /dev/null @@ -1 +0,0 @@ -{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}}]} diff --git a/spec/fixtures/files/watched/user@domain.com/gpx_track_single_segment.gpx b/spec/fixtures/files/watched/user@domain.com/gpx_track_single_segment.gpx deleted file mode 100644 index 3176fb4f..00000000 --- a/spec/fixtures/files/watched/user@domain.com/gpx_track_single_segment.gpx +++ /dev/null @@ -1,1239 +0,0 @@ - - - - La Zubia - balcon - - Garmin Connect - - - - - La Zubia - balcon - - - 824.93 - - - - 822.91 - - - - 819.57 - - - - 815.2 - - - - 811.41 - - - - 808.11 - - - - 805.33 - - - - 802.85 - - - - 800.8 - - - - 798.9 - - - - 797.19 - - - - 795.8 - - - - 794.31 - - - - 793.25 - - - - 792.19 - - - - 791.44 - - - - 791.24 - - - - 791.47 - - - - 792.04 - - - - 792.18 - - - - 793.94 - - - - 795.29 - - - - 796.89 - - - - 798.7 - - - - 801.44 - - - - 803.97 - - - - 806.6 - - - - 809.27 - - - - 811.96 - - - - 814.62 - - - - 817.54 - - - - 820.18 - - - - 822.76 - - - - 825.25 - - - - 827.89 - - - - 830.82 - - - - 833.17 - - - - 835.42 - - - - 837.9 - - - - 839.89 - - - - 841.98 - - - - 844.17 - - - - 846.01 - - - - 847.32 - - - - 848.51 - - - - 849.54 - - - - 850.3 - - - - 850.74 - - - - 851.11 - - - - 851.31 - - - - 851.37 - - - - 851.36 - - - - 851.21 - - - - 851.04 - - - - 850.86 - - - - 850.41 - - - - 849.94 - - - - 849.54 - - - - 849.08 - - - - 848.67 - - - - 848.36 - - - - 848.08 - - - - 847.87 - - - - 847.77 - - - - 847.74 - - - - 847.75 - - - - 847.81 - - - - 847.96 - - - - 848.17 - - - - 848.37 - - - - 848.68 - - - - 849.01 - - - - 849.24 - - - - 849.47 - - - - 849.7 - - - - 849.88 - - - - 850.1 - - - - 850.25 - - - - 850.38 - - - - 850.47 - - - - 850.46 - - - - 850.35 - - - - 850.35 - - - - 850.02 - - - - 849.6 - - - - 849.05 - - - - 848.37 - - - - 847.54 - - - - 846.57 - - - - 845.55 - - - - 844.29 - - - - 842.85 - - - - 841.43 - - - - 839.98 - - - - 838.63 - - - - 837.18 - - - - 835.48 - - - - 833.92 - - - - 832.43 - - - - 831.06 - - - - 829.84 - - - - 829.04 - - - - 828.42 - - - - 828.15 - - - - 828.11 - - - - 828.51 - - - - 829.55 - - - - 830.31 - - - - 831.12 - - - - 831.93 - - - - 832.91 - - - - 833.85 - - - - 834.91 - - - - 836.07 - - - - 837.2 - - - - 838.38 - - - - 839.56 - - - - 840.58 - - - - 841.58 - - - - 842.46 - - - - 843.23 - - - - 843.46 - - - - 843.41 - - - - 842.64 - - - - 841.84 - - - - 840.81 - - - - 839.56 - - - - 837.86 - - - - 836.03 - - - - 833.91 - - - - 831.55 - - - - 828.71 - - - - 825.47 - - - - 820.96 - - - - 817.85 - - - - 814.71 - - - - 811.52 - - - - 808.25 - - - - 805.03 - - - - 801.68 - - - - 798.27 - - - - 794.91 - - - - 791.73 - - - - 788.61 - - - - 785.48 - - - - 782.4 - - - - 779.42 - - - - 776.47 - - - - 773.67 - - - - 770.99 - - - - 768.4 - - - - 765.66 - - - - 763.1 - - - - 760.26 - - - - 757.88 - - - - 755.75 - - - - 753.7 - - - - 751.75 - - - - 749.94 - - - - 748.17 - - - - 746.34 - - - - 744.47 - - - - 743.18 - - - - 742.0 - - - - 741.01 - - - - 740.17 - - - - 739.53 - - - - 738.88 - - - - 738.42 - - - - 738.16 - - - - 738.01 - - - - 738.01 - - - - 738.11 - - - - 738.36 - - - - 738.8 - - - - 739.13 - - - - 739.78 - - - - 740.12 - - - - 740.55 - - - - 740.93 - - - - 741.31 - - - - 741.6 - - - - 741.82 - - - - 741.89 - - - - 741.94 - - - - 741.89 - - - - 742.0 - - - - 742.05 - - - - 742.17 - - - - 742.28 - - - - 742.49 - - - - 742.74 - - - - 742.86 - - - - 743.34 - - - - 744.01 - - - - 744.96 - - - - 746.14 - - - - 747.41 - - - - 748.68 - - - - 750.03 - - - - 751.57 - - - - 753.47 - - - - 755.4 - - - - 757.49 - - - - 759.68 - - - - 762.09 - - - - 764.56 - - - - 767.4 - - - - 770.3 - - - - 773.45 - - - - 776.83 - - - - 780.51 - - - - 783.74 - - - - 786.94 - - - - 790.76 - - - - 794.06 - - - - 797.36 - - - - 800.75 - - - - 804.12 - - - - 807.53 - - - - 811.02 - - - - 814.61 - - - - 818.13 - - - - 821.6 - - - - 825.29 - - - - 828.89 - - - - 832.37 - - - - 836.28 - - - - 839.49 - - - - 842.19 - - - - 844.74 - - - - 847.21 - - - - 849.34 - - - - 851.3 - - - - 852.93 - - - - 854.35 - - - - 855.69 - - - - 856.86 - - - - 857.72 - - - - 858.43 - - - - 858.78 - - - - 859.01 - - - - 859.0 - - - - 858.97 - - - - 859.21 - - - - 859.45 - - - - 859.73 - - - - 860.06 - - - - 860.45 - - - - 861.08 - - - - 861.61 - - - - 862.29 - - - - 863.0 - - - - 863.9 - - - - 864.96 - - - - 866.07 - - - - 867.3 - - - - 869.0 - - - - 870.45 - - - - 871.97 - - - - 873.37 - - - - 874.8 - - - - 876.17 - - - - 877.6 - - - - 879.15 - - - - 880.87 - - - - 882.54 - - - - 884.28 - - - - 886.01 - - - - 887.84 - - - - 889.62 - - - - 891.29 - - - - 892.83 - - - - 893.87 - - - - 894.78 - - - - 895.66 - - - - 896.51 - - - - 896.83 - - - - 896.95 - - - - 896.98 - - - - 896.67 - - - - 896.92 - - - - 897.13 - - - - 897.08 - - - - 897.65 - - - - 898.62 - - - - 899.59 - - - - 900.3 - - - - 901.06 - - - - 901.98 - - - - 902.94 - - - - 904.14 - - - - 905.06 - - - - 905.5 - - - - 905.8 - - - - 905.47 - - - - 905.91 - - - - 906.01 - - - - 905.66 - - - - 904.85 - - - - 904.4 - - - - 903.49 - - - - 903.02 - - - - 901.8 - - - - 901.42 - - - - - \ No newline at end of file diff --git a/spec/services/imports/watcher_spec.rb b/spec/services/imports/watcher_spec.rb index 83dacdf2..d745c352 100644 --- a/spec/services/imports/watcher_spec.rb +++ b/spec/services/imports/watcher_spec.rb @@ -5,14 +5,18 @@ require 'rails_helper' RSpec.describe Imports::Watcher do describe '#call' do subject(:service) { described_class.new.call } - let(:watched_dir_path) { Rails.root.join('spec/fixtures/files/watched') } let(:user) { create(:user, email: 'user@domain.com') } before do + FileUtils.mkdir_p(watched_dir_path.join(user.email)) stub_const('Imports::Watcher::WATCHED_DIR_PATH', watched_dir_path) end + after do + FileUtils.rm_rf(watched_dir_path) + end + context 'when there are no files in the watched directory' do it 'does not call ImportJob' do expect(ImportJob).not_to receive(:perform_later) @@ -22,26 +26,42 @@ RSpec.describe Imports::Watcher do end context 'when there are files in the watched directory' do - Sidekiq::Testing.inline! do - context 'when the file has a valid user email' do - it 'creates an import for the user' do - expect { service }.to change(user.imports, :count).by(2) - end + context 'when the file has a valid user email' do + it 'creates an import for the user' do + Sidekiq::Testing.inline! + File.write(watched_dir_path.join(user.email, 'location-history.json'), '{"type": "FeatureCollection"}') + File.write(watched_dir_path.join(user.email, 'Records.json'), '{"type": "FeatureCollection"}') + File.write(watched_dir_path.join(user.email, '2023_January.json'), '{"type": "FeatureCollection"}') + File.write(watched_dir_path.join(user.email, 'owntracks.rec'), '{"type": "FeatureCollection"}') + File.write(watched_dir_path.join(user.email, 'gpx_track_single_segment.gpx'), '{"type": "FeatureCollection"}') + + expect { service }.to change(user.imports, :count).by(5) end + end - context 'when the file has an invalid user email' do - it 'does not create an import' do - expect { service }.not_to change(Import, :count) - end + context 'when the file has an invalid user email' do + it 'does not create an import' do + FileUtils.mkdir_p(watched_dir_path.join('invalid@domain.com')) + File.write(watched_dir_path.join('invalid@domain.com', 'location-history.json'), '{"type": "FeatureCollection"}') + File.write(watched_dir_path.join('invalid@domain.com', 'Records.json'), '{"type": "FeatureCollection"}') + File.write(watched_dir_path.join('invalid@domain.com', '2023_January.json'), '{"type": "FeatureCollection"}') + File.write(watched_dir_path.join('invalid@domain.com', 'owntracks.rec'), '{"type": "FeatureCollection"}') + File.write(watched_dir_path.join('invalid@domain.com', 'gpx_track_single_segment.gpx'), '{"type": "FeatureCollection"}') + + expect { service }.not_to change(Import, :count) end + end - context 'when the import already exists' do - it 'does not create a new import' do - create(:import, user:, name: 'export_same_points.json') - create(:import, user:, name: 'gpx_track_single_segment.gpx') - - expect { service }.not_to change(Import, :count) - end + context 'when the import already exists' do + it 'does not create a new import' do + create(:import, user:, name: 'export_same_points.json') + create(:import, user:, name: 'gpx_track_single_segment.gpx') + create(:import, user:, name: 'location-history.json') + create(:import, user:, name: 'Records.json') + create(:import, user:, name: '2023_January.json') + create(:import, user:, name: 'data.geojson') + + expect { service }.not_to change(Import, :count) end end end