From c8960d46f2cd2fe4cb98139bbba850902585f685 Mon Sep 17 00:00:00 2001 From: GED Date: Thu, 12 Dec 2024 22:05:36 +0100 Subject: [PATCH 01/11] feat: added google_phone_takeout and google_semantic_histo into imports_controller for creating a new import --- Gemfile.lock | 195 +++++++++++++------------- app/controllers/imports_controller.rb | 5 +- spec/features/import_process_spec.rb | 20 +++ 3 files changed, 124 insertions(+), 96 deletions(-) create mode 100644 spec/features/import_process_spec.rb diff --git a/Gemfile.lock b/Gemfile.lock index cdff83dc..e041e5ac 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -10,66 +10,65 @@ GIT GEM remote: https://rubygems.org/ specs: - actioncable (7.2.2) - actionpack (= 7.2.2) - activesupport (= 7.2.2) + actioncable (8.0.0.1) + actionpack (= 8.0.0.1) + activesupport (= 8.0.0.1) nio4r (~> 2.0) websocket-driver (>= 0.6.1) zeitwerk (~> 2.6) - actionmailbox (7.2.2) - actionpack (= 7.2.2) - activejob (= 7.2.2) - activerecord (= 7.2.2) - activestorage (= 7.2.2) - activesupport (= 7.2.2) + actionmailbox (8.0.0.1) + actionpack (= 8.0.0.1) + activejob (= 8.0.0.1) + activerecord (= 8.0.0.1) + activestorage (= 8.0.0.1) + activesupport (= 8.0.0.1) mail (>= 2.8.0) - actionmailer (7.2.2) - actionpack (= 7.2.2) - actionview (= 7.2.2) - activejob (= 7.2.2) - activesupport (= 7.2.2) + actionmailer (8.0.0.1) + actionpack (= 8.0.0.1) + actionview (= 8.0.0.1) + activejob (= 8.0.0.1) + activesupport (= 8.0.0.1) mail (>= 2.8.0) rails-dom-testing (~> 2.2) - actionpack (7.2.2) - actionview (= 7.2.2) - activesupport (= 7.2.2) + actionpack (8.0.0.1) + actionview (= 8.0.0.1) + activesupport (= 8.0.0.1) nokogiri (>= 1.8.5) - racc - rack (>= 2.2.4, < 3.2) + rack (>= 2.2.4) rack-session (>= 1.0.1) rack-test (>= 0.6.3) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) useragent (~> 0.16) - actiontext (7.2.2) - actionpack (= 7.2.2) - activerecord (= 7.2.2) - activestorage (= 7.2.2) - activesupport (= 7.2.2) + actiontext (8.0.0.1) + actionpack (= 8.0.0.1) + activerecord (= 8.0.0.1) + activestorage (= 8.0.0.1) + activesupport (= 8.0.0.1) globalid (>= 0.6.0) nokogiri (>= 1.8.5) - actionview (7.2.2) - activesupport (= 7.2.2) + actionview (8.0.0.1) + activesupport (= 8.0.0.1) builder (~> 3.1) erubi (~> 1.11) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - activejob (7.2.2) - activesupport (= 7.2.2) + activejob (8.0.0.1) + activesupport (= 8.0.0.1) globalid (>= 0.3.6) - activemodel (7.2.2) - activesupport (= 7.2.2) - activerecord (7.2.2) - activemodel (= 7.2.2) - activesupport (= 7.2.2) + activemodel (8.0.0.1) + activesupport (= 8.0.0.1) + activerecord (8.0.0.1) + activemodel (= 8.0.0.1) + activesupport (= 8.0.0.1) timeout (>= 0.4.0) - activestorage (7.2.2) - actionpack (= 7.2.2) - activejob (= 7.2.2) - activerecord (= 7.2.2) - activesupport (= 7.2.2) + activestorage (8.0.0.1) + actionpack (= 8.0.0.1) + activejob (= 8.0.0.1) + activerecord (= 8.0.0.1) + activesupport (= 8.0.0.1) marcel (~> 1.0) - activesupport (7.2.2) + activesupport (8.0.0.1) base64 benchmark (>= 0.3) bigdecimal @@ -81,6 +80,7 @@ GEM minitest (>= 5.1) securerandom (>= 0.3) tzinfo (~> 2.0, >= 2.0.5) + uri (>= 0.13.1) addressable (2.8.7) public_suffix (>= 2.0.2, < 7.0) ast (2.4.2) @@ -109,7 +109,7 @@ GEM data_migrate (11.2.0) activerecord (>= 6.1) railties (>= 6.1) - date (3.4.0) + date (3.4.1) debug (1.9.2) irb (~> 1.10) reline (>= 0.3.8) @@ -149,7 +149,7 @@ GEM rake groupdate (6.5.1) activesupport (>= 7) - hashdiff (1.1.1) + hashdiff (1.1.2) httparty (0.22.0) csv mini_mime (>= 1.0.0) @@ -160,13 +160,14 @@ GEM actionpack (>= 6.0.0) activesupport (>= 6.0.0) railties (>= 6.0.0) - io-console (0.7.2) - irb (1.14.1) + io-console (0.8.0) + irb (1.14.2) rdoc (>= 4.0.0) reline (>= 0.4.2) - json (2.7.4) - json-schema (5.0.1) + json (2.9.0) + json-schema (5.1.1) addressable (~> 2.8) + bigdecimal (~> 3.1) kaminari (1.2.2) activesupport (>= 4.1.0) kaminari-actionview (= 1.2.2) @@ -180,7 +181,7 @@ GEM kaminari-core (= 1.2.2) kaminari-core (1.2.2) language_server-protocol (3.17.0.3) - logger (1.6.1) + logger (1.6.2) lograge (0.14.0) actionpack (>= 4) activesupport (>= 4) @@ -197,11 +198,11 @@ GEM marcel (1.0.4) method_source (1.1.0) mini_mime (1.1.5) - minitest (5.25.2) - msgpack (1.7.3) + minitest (5.25.4) + msgpack (1.7.5) multi_xml (0.7.1) bigdecimal (~> 3.1) - net-imap (0.5.0) + net-imap (0.5.1) date net-protocol net-pop (0.1.2) @@ -211,26 +212,26 @@ GEM net-smtp (0.5.0) net-protocol nio4r (2.7.4) - nokogiri (1.16.7-aarch64-linux) + nokogiri (1.17.1-aarch64-linux) racc (~> 1.4) - nokogiri (1.16.7-arm-linux) + nokogiri (1.17.1-arm-linux) racc (~> 1.4) - nokogiri (1.16.7-arm64-darwin) + nokogiri (1.17.1-arm64-darwin) racc (~> 1.4) - nokogiri (1.16.7-x86-linux) + nokogiri (1.17.1-x86-linux) racc (~> 1.4) - nokogiri (1.16.7-x86_64-darwin) + nokogiri (1.17.1-x86_64-darwin) racc (~> 1.4) - nokogiri (1.16.7-x86_64-linux) + nokogiri (1.17.1-x86_64-linux) racc (~> 1.4) oj (3.16.7) bigdecimal (>= 3.0) ostruct (>= 0.2) optimist (3.2.0) orm_adapter (0.5.0) - ostruct (0.6.0) + ostruct (0.6.1) parallel (1.26.3) - parser (3.3.5.0) + parser (3.3.6.0) ast (~> 2.4.1) racc patience_diff (1.2.0) @@ -246,7 +247,8 @@ GEM pry (>= 0.13, < 0.15) pry-rails (0.3.11) pry (>= 0.13.0) - psych (5.2.0) + psych (5.2.1) + date stringio public_suffix (6.0.1) puma (6.5.0) @@ -262,30 +264,30 @@ GEM rack (>= 1.3) rackup (2.2.1) rack (>= 3) - rails (7.2.2) - actioncable (= 7.2.2) - actionmailbox (= 7.2.2) - actionmailer (= 7.2.2) - actionpack (= 7.2.2) - actiontext (= 7.2.2) - actionview (= 7.2.2) - activejob (= 7.2.2) - activemodel (= 7.2.2) - activerecord (= 7.2.2) - activestorage (= 7.2.2) - activesupport (= 7.2.2) + rails (8.0.0.1) + actioncable (= 8.0.0.1) + actionmailbox (= 8.0.0.1) + actionmailer (= 8.0.0.1) + actionpack (= 8.0.0.1) + actiontext (= 8.0.0.1) + actionview (= 8.0.0.1) + activejob (= 8.0.0.1) + activemodel (= 8.0.0.1) + activerecord (= 8.0.0.1) + activestorage (= 8.0.0.1) + activesupport (= 8.0.0.1) bundler (>= 1.15.0) - railties (= 7.2.2) + railties (= 8.0.0.1) rails-dom-testing (2.2.0) activesupport (>= 5.0.0) minitest nokogiri (>= 1.6) - rails-html-sanitizer (1.6.0) + rails-html-sanitizer (1.6.1) loofah (~> 2.21) - nokogiri (~> 1.14) - railties (7.2.2) - actionpack (= 7.2.2) - activesupport (= 7.2.2) + nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0) + railties (8.0.0.1) + actionpack (= 8.0.0.1) + activesupport (= 8.0.0.1) irb (~> 1.13) rackup (>= 1.0.0) rake (>= 12.2) @@ -297,17 +299,17 @@ GEM psych (>= 4.0.0) redis (5.3.0) redis-client (>= 0.22.0) - redis-client (0.22.2) + redis-client (0.23.0) connection_pool - regexp_parser (2.9.2) - reline (0.5.11) + regexp_parser (2.9.3) + reline (0.5.12) io-console (~> 0.5) request_store (1.7.0) rack (>= 1.4) responders (3.1.1) actionpack (>= 5.2) railties (>= 5.2) - rexml (3.3.8) + rexml (3.3.9) rspec-core (3.13.2) rspec-support (~> 3.13.0) rspec-expectations (3.13.3) @@ -324,7 +326,7 @@ GEM rspec-expectations (~> 3.13) rspec-mocks (~> 3.13) rspec-support (~> 3.13) - rspec-support (3.13.1) + rspec-support (3.13.2) rswag-api (2.16.0) activesupport (>= 5.2, < 8.1) railties (>= 5.2, < 8.1) @@ -336,17 +338,17 @@ GEM rswag-ui (2.16.0) actionpack (>= 5.2, < 8.1) railties (>= 5.2, < 8.1) - rubocop (1.67.0) + rubocop (1.69.2) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 2.4, < 3.0) - rubocop-ast (>= 1.32.2, < 2.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.36.2, < 2.0) ruby-progressbar (~> 1.7) - unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.32.3) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.36.2) parser (>= 3.3.1.0) rubocop-rails (2.27.0) activesupport (>= 4.2.0) @@ -354,7 +356,7 @@ GEM rubocop (>= 1.52.0, < 2.0) rubocop-ast (>= 1.31.1, < 2.0) ruby-progressbar (1.13.0) - securerandom (0.3.2) + securerandom (0.4.0) shoulda-matchers (6.4.0) activesupport (>= 5.2.0) shrine (3.6.0) @@ -395,12 +397,12 @@ GEM tailwindcss-rails (3.0.0) railties (>= 7.0.0) tailwindcss-ruby - tailwindcss-ruby (3.4.14) - tailwindcss-ruby (3.4.14-aarch64-linux) - tailwindcss-ruby (3.4.14-arm-linux) - tailwindcss-ruby (3.4.14-arm64-darwin) - tailwindcss-ruby (3.4.14-x86_64-darwin) - tailwindcss-ruby (3.4.14-x86_64-linux) + tailwindcss-ruby (3.4.16) + tailwindcss-ruby (3.4.16-aarch64-linux) + tailwindcss-ruby (3.4.16-arm-linux) + tailwindcss-ruby (3.4.16-arm64-darwin) + tailwindcss-ruby (3.4.16-x86_64-darwin) + tailwindcss-ruby (3.4.16-x86_64-linux) thor (1.3.2) timeout (0.4.2) turbo-rails (2.0.11) @@ -409,8 +411,11 @@ GEM tzinfo (2.0.6) concurrent-ruby (~> 1.0) unicode (0.4.4.5) - unicode-display_width (2.6.0) - useragent (0.16.10) + unicode-display_width (3.1.2) + unicode-emoji (~> 4.0, >= 4.0.4) + unicode-emoji (4.0.4) + uri (1.0.2) + useragent (0.16.11) warden (1.2.9) rack (>= 2.0.9) webmock (3.24.0) diff --git a/app/controllers/imports_controller.rb b/app/controllers/imports_controller.rb index ca3c34c8..fb36bc4a 100644 --- a/app/controllers/imports_controller.rb +++ b/app/controllers/imports_controller.rb @@ -32,8 +32,11 @@ class ImportsController < ApplicationController raw_data = case params[:import][:source] - when 'gpx' then Hash.from_xml(file) + when 'google_semantic_history' then GoogleMaps::SemanticHistoryParser.new(import, current_user.id).call when 'owntracks' then OwnTracks::RecParser.new(file).call + when 'geojson' then Geojson::ImportParser.new(import, current_user.id).call + when 'google_phone_takeout' then GoogleMaps::PhoneTakeoutParser.new(import, current_user.id).call + when 'gpx' then Hash.from_xml(file) else JSON.parse(file) end diff --git a/spec/features/import_process_spec.rb b/spec/features/import_process_spec.rb new file mode 100644 index 00000000..a71edb0b --- /dev/null +++ b/spec/features/import_process_spec.rb @@ -0,0 +1,20 @@ +# spec/features/import_process_spec.rb +require 'rails_helper' + +RSpec.feature 'Import Process', type: :feature do + let(:user) { create(:user) } + + before do + sign_in user + end + + scenario 'User imports a Google Phone Takeout file' do + visit new_import_path + + choose 'Google Phone Takeout' + attach_file 'import_files', Rails.root.join('spec/fixtures/files/google/phone-takeout.json') + click_button 'Create Import' + + expect(page).to have_content('files are queued to be imported in background') + end +end \ No newline at end of file From e67693f43ccc8415f143abfe1a48f421e3a31525 Mon Sep 17 00:00:00 2001 From: GED Date: Sat, 14 Dec 2024 10:27:19 +0100 Subject: [PATCH 02/11] Revert changes to Gemfile.lock --- Gemfile.lock | 195 +++++++++++++++++++++++++-------------------------- 1 file changed, 95 insertions(+), 100 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index e041e5ac..cdff83dc 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -10,65 +10,66 @@ GIT GEM remote: https://rubygems.org/ specs: - actioncable (8.0.0.1) - actionpack (= 8.0.0.1) - activesupport (= 8.0.0.1) + actioncable (7.2.2) + actionpack (= 7.2.2) + activesupport (= 7.2.2) nio4r (~> 2.0) websocket-driver (>= 0.6.1) zeitwerk (~> 2.6) - actionmailbox (8.0.0.1) - actionpack (= 8.0.0.1) - activejob (= 8.0.0.1) - activerecord (= 8.0.0.1) - activestorage (= 8.0.0.1) - activesupport (= 8.0.0.1) + actionmailbox (7.2.2) + actionpack (= 7.2.2) + activejob (= 7.2.2) + activerecord (= 7.2.2) + activestorage (= 7.2.2) + activesupport (= 7.2.2) mail (>= 2.8.0) - actionmailer (8.0.0.1) - actionpack (= 8.0.0.1) - actionview (= 8.0.0.1) - activejob (= 8.0.0.1) - activesupport (= 8.0.0.1) + actionmailer (7.2.2) + actionpack (= 7.2.2) + actionview (= 7.2.2) + activejob (= 7.2.2) + activesupport (= 7.2.2) mail (>= 2.8.0) rails-dom-testing (~> 2.2) - actionpack (8.0.0.1) - actionview (= 8.0.0.1) - activesupport (= 8.0.0.1) + actionpack (7.2.2) + actionview (= 7.2.2) + activesupport (= 7.2.2) nokogiri (>= 1.8.5) - rack (>= 2.2.4) + racc + rack (>= 2.2.4, < 3.2) rack-session (>= 1.0.1) rack-test (>= 0.6.3) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) useragent (~> 0.16) - actiontext (8.0.0.1) - actionpack (= 8.0.0.1) - activerecord (= 8.0.0.1) - activestorage (= 8.0.0.1) - activesupport (= 8.0.0.1) + actiontext (7.2.2) + actionpack (= 7.2.2) + activerecord (= 7.2.2) + activestorage (= 7.2.2) + activesupport (= 7.2.2) globalid (>= 0.6.0) nokogiri (>= 1.8.5) - actionview (8.0.0.1) - activesupport (= 8.0.0.1) + actionview (7.2.2) + activesupport (= 7.2.2) builder (~> 3.1) erubi (~> 1.11) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - activejob (8.0.0.1) - activesupport (= 8.0.0.1) + activejob (7.2.2) + activesupport (= 7.2.2) globalid (>= 0.3.6) - activemodel (8.0.0.1) - activesupport (= 8.0.0.1) - activerecord (8.0.0.1) - activemodel (= 8.0.0.1) - activesupport (= 8.0.0.1) + activemodel (7.2.2) + activesupport (= 7.2.2) + activerecord (7.2.2) + activemodel (= 7.2.2) + activesupport (= 7.2.2) timeout (>= 0.4.0) - activestorage (8.0.0.1) - actionpack (= 8.0.0.1) - activejob (= 8.0.0.1) - activerecord (= 8.0.0.1) - activesupport (= 8.0.0.1) + activestorage (7.2.2) + actionpack (= 7.2.2) + activejob (= 7.2.2) + activerecord (= 7.2.2) + activesupport (= 7.2.2) marcel (~> 1.0) - activesupport (8.0.0.1) + activesupport (7.2.2) base64 benchmark (>= 0.3) bigdecimal @@ -80,7 +81,6 @@ GEM minitest (>= 5.1) securerandom (>= 0.3) tzinfo (~> 2.0, >= 2.0.5) - uri (>= 0.13.1) addressable (2.8.7) public_suffix (>= 2.0.2, < 7.0) ast (2.4.2) @@ -109,7 +109,7 @@ GEM data_migrate (11.2.0) activerecord (>= 6.1) railties (>= 6.1) - date (3.4.1) + date (3.4.0) debug (1.9.2) irb (~> 1.10) reline (>= 0.3.8) @@ -149,7 +149,7 @@ GEM rake groupdate (6.5.1) activesupport (>= 7) - hashdiff (1.1.2) + hashdiff (1.1.1) httparty (0.22.0) csv mini_mime (>= 1.0.0) @@ -160,14 +160,13 @@ GEM actionpack (>= 6.0.0) activesupport (>= 6.0.0) railties (>= 6.0.0) - io-console (0.8.0) - irb (1.14.2) + io-console (0.7.2) + irb (1.14.1) rdoc (>= 4.0.0) reline (>= 0.4.2) - json (2.9.0) - json-schema (5.1.1) + json (2.7.4) + json-schema (5.0.1) addressable (~> 2.8) - bigdecimal (~> 3.1) kaminari (1.2.2) activesupport (>= 4.1.0) kaminari-actionview (= 1.2.2) @@ -181,7 +180,7 @@ GEM kaminari-core (= 1.2.2) kaminari-core (1.2.2) language_server-protocol (3.17.0.3) - logger (1.6.2) + logger (1.6.1) lograge (0.14.0) actionpack (>= 4) activesupport (>= 4) @@ -198,11 +197,11 @@ GEM marcel (1.0.4) method_source (1.1.0) mini_mime (1.1.5) - minitest (5.25.4) - msgpack (1.7.5) + minitest (5.25.2) + msgpack (1.7.3) multi_xml (0.7.1) bigdecimal (~> 3.1) - net-imap (0.5.1) + net-imap (0.5.0) date net-protocol net-pop (0.1.2) @@ -212,26 +211,26 @@ GEM net-smtp (0.5.0) net-protocol nio4r (2.7.4) - nokogiri (1.17.1-aarch64-linux) + nokogiri (1.16.7-aarch64-linux) racc (~> 1.4) - nokogiri (1.17.1-arm-linux) + nokogiri (1.16.7-arm-linux) racc (~> 1.4) - nokogiri (1.17.1-arm64-darwin) + nokogiri (1.16.7-arm64-darwin) racc (~> 1.4) - nokogiri (1.17.1-x86-linux) + nokogiri (1.16.7-x86-linux) racc (~> 1.4) - nokogiri (1.17.1-x86_64-darwin) + nokogiri (1.16.7-x86_64-darwin) racc (~> 1.4) - nokogiri (1.17.1-x86_64-linux) + nokogiri (1.16.7-x86_64-linux) racc (~> 1.4) oj (3.16.7) bigdecimal (>= 3.0) ostruct (>= 0.2) optimist (3.2.0) orm_adapter (0.5.0) - ostruct (0.6.1) + ostruct (0.6.0) parallel (1.26.3) - parser (3.3.6.0) + parser (3.3.5.0) ast (~> 2.4.1) racc patience_diff (1.2.0) @@ -247,8 +246,7 @@ GEM pry (>= 0.13, < 0.15) pry-rails (0.3.11) pry (>= 0.13.0) - psych (5.2.1) - date + psych (5.2.0) stringio public_suffix (6.0.1) puma (6.5.0) @@ -264,30 +262,30 @@ GEM rack (>= 1.3) rackup (2.2.1) rack (>= 3) - rails (8.0.0.1) - actioncable (= 8.0.0.1) - actionmailbox (= 8.0.0.1) - actionmailer (= 8.0.0.1) - actionpack (= 8.0.0.1) - actiontext (= 8.0.0.1) - actionview (= 8.0.0.1) - activejob (= 8.0.0.1) - activemodel (= 8.0.0.1) - activerecord (= 8.0.0.1) - activestorage (= 8.0.0.1) - activesupport (= 8.0.0.1) + rails (7.2.2) + actioncable (= 7.2.2) + actionmailbox (= 7.2.2) + actionmailer (= 7.2.2) + actionpack (= 7.2.2) + actiontext (= 7.2.2) + actionview (= 7.2.2) + activejob (= 7.2.2) + activemodel (= 7.2.2) + activerecord (= 7.2.2) + activestorage (= 7.2.2) + activesupport (= 7.2.2) bundler (>= 1.15.0) - railties (= 8.0.0.1) + railties (= 7.2.2) rails-dom-testing (2.2.0) activesupport (>= 5.0.0) minitest nokogiri (>= 1.6) - rails-html-sanitizer (1.6.1) + rails-html-sanitizer (1.6.0) loofah (~> 2.21) - nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0) - railties (8.0.0.1) - actionpack (= 8.0.0.1) - activesupport (= 8.0.0.1) + nokogiri (~> 1.14) + railties (7.2.2) + actionpack (= 7.2.2) + activesupport (= 7.2.2) irb (~> 1.13) rackup (>= 1.0.0) rake (>= 12.2) @@ -299,17 +297,17 @@ GEM psych (>= 4.0.0) redis (5.3.0) redis-client (>= 0.22.0) - redis-client (0.23.0) + redis-client (0.22.2) connection_pool - regexp_parser (2.9.3) - reline (0.5.12) + regexp_parser (2.9.2) + reline (0.5.11) io-console (~> 0.5) request_store (1.7.0) rack (>= 1.4) responders (3.1.1) actionpack (>= 5.2) railties (>= 5.2) - rexml (3.3.9) + rexml (3.3.8) rspec-core (3.13.2) rspec-support (~> 3.13.0) rspec-expectations (3.13.3) @@ -326,7 +324,7 @@ GEM rspec-expectations (~> 3.13) rspec-mocks (~> 3.13) rspec-support (~> 3.13) - rspec-support (3.13.2) + rspec-support (3.13.1) rswag-api (2.16.0) activesupport (>= 5.2, < 8.1) railties (>= 5.2, < 8.1) @@ -338,17 +336,17 @@ GEM rswag-ui (2.16.0) actionpack (>= 5.2, < 8.1) railties (>= 5.2, < 8.1) - rubocop (1.69.2) + rubocop (1.67.0) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 2.9.3, < 3.0) - rubocop-ast (>= 1.36.2, < 2.0) + regexp_parser (>= 2.4, < 3.0) + rubocop-ast (>= 1.32.2, < 2.0) ruby-progressbar (~> 1.7) - unicode-display_width (>= 2.4.0, < 4.0) - rubocop-ast (1.36.2) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.32.3) parser (>= 3.3.1.0) rubocop-rails (2.27.0) activesupport (>= 4.2.0) @@ -356,7 +354,7 @@ GEM rubocop (>= 1.52.0, < 2.0) rubocop-ast (>= 1.31.1, < 2.0) ruby-progressbar (1.13.0) - securerandom (0.4.0) + securerandom (0.3.2) shoulda-matchers (6.4.0) activesupport (>= 5.2.0) shrine (3.6.0) @@ -397,12 +395,12 @@ GEM tailwindcss-rails (3.0.0) railties (>= 7.0.0) tailwindcss-ruby - tailwindcss-ruby (3.4.16) - tailwindcss-ruby (3.4.16-aarch64-linux) - tailwindcss-ruby (3.4.16-arm-linux) - tailwindcss-ruby (3.4.16-arm64-darwin) - tailwindcss-ruby (3.4.16-x86_64-darwin) - tailwindcss-ruby (3.4.16-x86_64-linux) + tailwindcss-ruby (3.4.14) + tailwindcss-ruby (3.4.14-aarch64-linux) + tailwindcss-ruby (3.4.14-arm-linux) + tailwindcss-ruby (3.4.14-arm64-darwin) + tailwindcss-ruby (3.4.14-x86_64-darwin) + tailwindcss-ruby (3.4.14-x86_64-linux) thor (1.3.2) timeout (0.4.2) turbo-rails (2.0.11) @@ -411,11 +409,8 @@ GEM tzinfo (2.0.6) concurrent-ruby (~> 1.0) unicode (0.4.4.5) - unicode-display_width (3.1.2) - unicode-emoji (~> 4.0, >= 4.0.4) - unicode-emoji (4.0.4) - uri (1.0.2) - useragent (0.16.11) + unicode-display_width (2.6.0) + useragent (0.16.10) warden (1.2.9) rack (>= 2.0.9) webmock (3.24.0) From ab84d282d38af6ceccb8807e43e92d2aa575fa28 Mon Sep 17 00:00:00 2001 From: GED Date: Mon, 16 Dec 2024 12:13:38 +0100 Subject: [PATCH 03/11] feat: added formats(google_phone_takeout, google_records, google_semantic_history, owntracks to the watcher --- app/controllers/imports_controller.rb | 5 +- app/services/imports/watcher.rb | 31 +- spec/features/import_process_spec.rb | 20 - .../user@domain.com/export_same_points.json | 1 - .../gpx_track_single_segment.gpx | 1239 ----------------- spec/services/imports/watcher_spec.rb | 54 +- 6 files changed, 65 insertions(+), 1285 deletions(-) delete mode 100644 spec/features/import_process_spec.rb delete mode 100644 spec/fixtures/files/watched/user@domain.com/export_same_points.json delete mode 100644 spec/fixtures/files/watched/user@domain.com/gpx_track_single_segment.gpx diff --git a/app/controllers/imports_controller.rb b/app/controllers/imports_controller.rb index fb36bc4a..ca3c34c8 100644 --- a/app/controllers/imports_controller.rb +++ b/app/controllers/imports_controller.rb @@ -32,11 +32,8 @@ class ImportsController < ApplicationController raw_data = case params[:import][:source] - when 'google_semantic_history' then GoogleMaps::SemanticHistoryParser.new(import, current_user.id).call - when 'owntracks' then OwnTracks::RecParser.new(file).call - when 'geojson' then Geojson::ImportParser.new(import, current_user.id).call - when 'google_phone_takeout' then GoogleMaps::PhoneTakeoutParser.new(import, current_user.id).call when 'gpx' then Hash.from_xml(file) + when 'owntracks' then OwnTracks::RecParser.new(file).call else JSON.parse(file) end diff --git a/app/services/imports/watcher.rb b/app/services/imports/watcher.rb index 43f3653b..15854f80 100644 --- a/app/services/imports/watcher.rb +++ b/app/services/imports/watcher.rb @@ -36,7 +36,7 @@ class Imports::Watcher def file_names(directory_path) Dir.entries(directory_path).select do |file| - ['.gpx', '.json'].include?(File.extname(file)) + ['.gpx', '.json', '.rec'].include?(File.extname(file)) end end @@ -73,8 +73,18 @@ class Imports::Watcher def source(file_name) case file_name.split('.').last - when 'json' then :geojson - when 'gpx' then :gpx + when 'json' + if file_name.match?(/location-history/i) + :google_phone_takeout + elsif file_name.match?(/Records/i) + :google_records + elsif file_name.match?(/\d{4}_\w+/i) + :google_semantic_history + else + :geojson + end + when 'rec' then :owntracks + when 'gpx' then :gpx else raise UnsupportedSourceError, 'Unsupported source ' end end @@ -82,6 +92,19 @@ class Imports::Watcher def raw_data(file_path, source) file = File.read(file_path) - source.to_sym == :gpx ? Hash.from_xml(file) : JSON.parse(file) + case source + when :google_phone_takeout + GoogleMaps::PhoneTakeoutParser.new(file).call + when :google_semantic_history + GoogleMaps::SemanticHistoryParser.new(file).call + when :google_records + GoogleMaps::RecordsParser.new(file).call + when :owntracks + OwnTracks::RecParser.new(file).call + when :gpx + Hash.from_xml(file) + else + JSON.parse(file) + end end end diff --git a/spec/features/import_process_spec.rb b/spec/features/import_process_spec.rb deleted file mode 100644 index a71edb0b..00000000 --- a/spec/features/import_process_spec.rb +++ /dev/null @@ -1,20 +0,0 @@ -# spec/features/import_process_spec.rb -require 'rails_helper' - -RSpec.feature 'Import Process', type: :feature do - let(:user) { create(:user) } - - before do - sign_in user - end - - scenario 'User imports a Google Phone Takeout file' do - visit new_import_path - - choose 'Google Phone Takeout' - attach_file 'import_files', Rails.root.join('spec/fixtures/files/google/phone-takeout.json') - click_button 'Create Import' - - expect(page).to have_content('files are queued to be imported in background') - end -end \ No newline at end of file diff --git a/spec/fixtures/files/watched/user@domain.com/export_same_points.json b/spec/fixtures/files/watched/user@domain.com/export_same_points.json deleted file mode 100644 index 2ecfb883..00000000 --- a/spec/fixtures/files/watched/user@domain.com/export_same_points.json +++ /dev/null @@ -1 +0,0 @@ -{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}}]} diff --git a/spec/fixtures/files/watched/user@domain.com/gpx_track_single_segment.gpx b/spec/fixtures/files/watched/user@domain.com/gpx_track_single_segment.gpx deleted file mode 100644 index 3176fb4f..00000000 --- a/spec/fixtures/files/watched/user@domain.com/gpx_track_single_segment.gpx +++ /dev/null @@ -1,1239 +0,0 @@ - - - - La Zubia - balcon - - Garmin Connect - - - - - La Zubia - balcon - - - 824.93 - - - - 822.91 - - - - 819.57 - - - - 815.2 - - - - 811.41 - - - - 808.11 - - - - 805.33 - - - - 802.85 - - - - 800.8 - - - - 798.9 - - - - 797.19 - - - - 795.8 - - - - 794.31 - - - - 793.25 - - - - 792.19 - - - - 791.44 - - - - 791.24 - - - - 791.47 - - - - 792.04 - - - - 792.18 - - - - 793.94 - - - - 795.29 - - - - 796.89 - - - - 798.7 - - - - 801.44 - - - - 803.97 - - - - 806.6 - - - - 809.27 - - - - 811.96 - - - - 814.62 - - - - 817.54 - - - - 820.18 - - - - 822.76 - - - - 825.25 - - - - 827.89 - - - - 830.82 - - - - 833.17 - - - - 835.42 - - - - 837.9 - - - - 839.89 - - - - 841.98 - - - - 844.17 - - - - 846.01 - - - - 847.32 - - - - 848.51 - - - - 849.54 - - - - 850.3 - - - - 850.74 - - - - 851.11 - - - - 851.31 - - - - 851.37 - - - - 851.36 - - - - 851.21 - - - - 851.04 - - - - 850.86 - - - - 850.41 - - - - 849.94 - - - - 849.54 - - - - 849.08 - - - - 848.67 - - - - 848.36 - - - - 848.08 - - - - 847.87 - - - - 847.77 - - - - 847.74 - - - - 847.75 - - - - 847.81 - - - - 847.96 - - - - 848.17 - - - - 848.37 - - - - 848.68 - - - - 849.01 - - - - 849.24 - - - - 849.47 - - - - 849.7 - - - - 849.88 - - - - 850.1 - - - - 850.25 - - - - 850.38 - - - - 850.47 - - - - 850.46 - - - - 850.35 - - - - 850.35 - - - - 850.02 - - - - 849.6 - - - - 849.05 - - - - 848.37 - - - - 847.54 - - - - 846.57 - - - - 845.55 - - - - 844.29 - - - - 842.85 - - - - 841.43 - - - - 839.98 - - - - 838.63 - - - - 837.18 - - - - 835.48 - - - - 833.92 - - - - 832.43 - - - - 831.06 - - - - 829.84 - - - - 829.04 - - - - 828.42 - - - - 828.15 - - - - 828.11 - - - - 828.51 - - - - 829.55 - - - - 830.31 - - - - 831.12 - - - - 831.93 - - - - 832.91 - - - - 833.85 - - - - 834.91 - - - - 836.07 - - - - 837.2 - - - - 838.38 - - - - 839.56 - - - - 840.58 - - - - 841.58 - - - - 842.46 - - - - 843.23 - - - - 843.46 - - - - 843.41 - - - - 842.64 - - - - 841.84 - - - - 840.81 - - - - 839.56 - - - - 837.86 - - - - 836.03 - - - - 833.91 - - - - 831.55 - - - - 828.71 - - - - 825.47 - - - - 820.96 - - - - 817.85 - - - - 814.71 - - - - 811.52 - - - - 808.25 - - - - 805.03 - - - - 801.68 - - - - 798.27 - - - - 794.91 - - - - 791.73 - - - - 788.61 - - - - 785.48 - - - - 782.4 - - - - 779.42 - - - - 776.47 - - - - 773.67 - - - - 770.99 - - - - 768.4 - - - - 765.66 - - - - 763.1 - - - - 760.26 - - - - 757.88 - - - - 755.75 - - - - 753.7 - - - - 751.75 - - - - 749.94 - - - - 748.17 - - - - 746.34 - - - - 744.47 - - - - 743.18 - - - - 742.0 - - - - 741.01 - - - - 740.17 - - - - 739.53 - - - - 738.88 - - - - 738.42 - - - - 738.16 - - - - 738.01 - - - - 738.01 - - - - 738.11 - - - - 738.36 - - - - 738.8 - - - - 739.13 - - - - 739.78 - - - - 740.12 - - - - 740.55 - - - - 740.93 - - - - 741.31 - - - - 741.6 - - - - 741.82 - - - - 741.89 - - - - 741.94 - - - - 741.89 - - - - 742.0 - - - - 742.05 - - - - 742.17 - - - - 742.28 - - - - 742.49 - - - - 742.74 - - - - 742.86 - - - - 743.34 - - - - 744.01 - - - - 744.96 - - - - 746.14 - - - - 747.41 - - - - 748.68 - - - - 750.03 - - - - 751.57 - - - - 753.47 - - - - 755.4 - - - - 757.49 - - - - 759.68 - - - - 762.09 - - - - 764.56 - - - - 767.4 - - - - 770.3 - - - - 773.45 - - - - 776.83 - - - - 780.51 - - - - 783.74 - - - - 786.94 - - - - 790.76 - - - - 794.06 - - - - 797.36 - - - - 800.75 - - - - 804.12 - - - - 807.53 - - - - 811.02 - - - - 814.61 - - - - 818.13 - - - - 821.6 - - - - 825.29 - - - - 828.89 - - - - 832.37 - - - - 836.28 - - - - 839.49 - - - - 842.19 - - - - 844.74 - - - - 847.21 - - - - 849.34 - - - - 851.3 - - - - 852.93 - - - - 854.35 - - - - 855.69 - - - - 856.86 - - - - 857.72 - - - - 858.43 - - - - 858.78 - - - - 859.01 - - - - 859.0 - - - - 858.97 - - - - 859.21 - - - - 859.45 - - - - 859.73 - - - - 860.06 - - - - 860.45 - - - - 861.08 - - - - 861.61 - - - - 862.29 - - - - 863.0 - - - - 863.9 - - - - 864.96 - - - - 866.07 - - - - 867.3 - - - - 869.0 - - - - 870.45 - - - - 871.97 - - - - 873.37 - - - - 874.8 - - - - 876.17 - - - - 877.6 - - - - 879.15 - - - - 880.87 - - - - 882.54 - - - - 884.28 - - - - 886.01 - - - - 887.84 - - - - 889.62 - - - - 891.29 - - - - 892.83 - - - - 893.87 - - - - 894.78 - - - - 895.66 - - - - 896.51 - - - - 896.83 - - - - 896.95 - - - - 896.98 - - - - 896.67 - - - - 896.92 - - - - 897.13 - - - - 897.08 - - - - 897.65 - - - - 898.62 - - - - 899.59 - - - - 900.3 - - - - 901.06 - - - - 901.98 - - - - 902.94 - - - - 904.14 - - - - 905.06 - - - - 905.5 - - - - 905.8 - - - - 905.47 - - - - 905.91 - - - - 906.01 - - - - 905.66 - - - - 904.85 - - - - 904.4 - - - - 903.49 - - - - 903.02 - - - - 901.8 - - - - 901.42 - - - - - \ No newline at end of file diff --git a/spec/services/imports/watcher_spec.rb b/spec/services/imports/watcher_spec.rb index 83dacdf2..d745c352 100644 --- a/spec/services/imports/watcher_spec.rb +++ b/spec/services/imports/watcher_spec.rb @@ -5,14 +5,18 @@ require 'rails_helper' RSpec.describe Imports::Watcher do describe '#call' do subject(:service) { described_class.new.call } - let(:watched_dir_path) { Rails.root.join('spec/fixtures/files/watched') } let(:user) { create(:user, email: 'user@domain.com') } before do + FileUtils.mkdir_p(watched_dir_path.join(user.email)) stub_const('Imports::Watcher::WATCHED_DIR_PATH', watched_dir_path) end + after do + FileUtils.rm_rf(watched_dir_path) + end + context 'when there are no files in the watched directory' do it 'does not call ImportJob' do expect(ImportJob).not_to receive(:perform_later) @@ -22,26 +26,42 @@ RSpec.describe Imports::Watcher do end context 'when there are files in the watched directory' do - Sidekiq::Testing.inline! do - context 'when the file has a valid user email' do - it 'creates an import for the user' do - expect { service }.to change(user.imports, :count).by(2) - end + context 'when the file has a valid user email' do + it 'creates an import for the user' do + Sidekiq::Testing.inline! + File.write(watched_dir_path.join(user.email, 'location-history.json'), '{"type": "FeatureCollection"}') + File.write(watched_dir_path.join(user.email, 'Records.json'), '{"type": "FeatureCollection"}') + File.write(watched_dir_path.join(user.email, '2023_January.json'), '{"type": "FeatureCollection"}') + File.write(watched_dir_path.join(user.email, 'owntracks.rec'), '{"type": "FeatureCollection"}') + File.write(watched_dir_path.join(user.email, 'gpx_track_single_segment.gpx'), '{"type": "FeatureCollection"}') + + expect { service }.to change(user.imports, :count).by(5) end + end - context 'when the file has an invalid user email' do - it 'does not create an import' do - expect { service }.not_to change(Import, :count) - end + context 'when the file has an invalid user email' do + it 'does not create an import' do + FileUtils.mkdir_p(watched_dir_path.join('invalid@domain.com')) + File.write(watched_dir_path.join('invalid@domain.com', 'location-history.json'), '{"type": "FeatureCollection"}') + File.write(watched_dir_path.join('invalid@domain.com', 'Records.json'), '{"type": "FeatureCollection"}') + File.write(watched_dir_path.join('invalid@domain.com', '2023_January.json'), '{"type": "FeatureCollection"}') + File.write(watched_dir_path.join('invalid@domain.com', 'owntracks.rec'), '{"type": "FeatureCollection"}') + File.write(watched_dir_path.join('invalid@domain.com', 'gpx_track_single_segment.gpx'), '{"type": "FeatureCollection"}') + + expect { service }.not_to change(Import, :count) end + end - context 'when the import already exists' do - it 'does not create a new import' do - create(:import, user:, name: 'export_same_points.json') - create(:import, user:, name: 'gpx_track_single_segment.gpx') - - expect { service }.not_to change(Import, :count) - end + context 'when the import already exists' do + it 'does not create a new import' do + create(:import, user:, name: 'export_same_points.json') + create(:import, user:, name: 'gpx_track_single_segment.gpx') + create(:import, user:, name: 'location-history.json') + create(:import, user:, name: 'Records.json') + create(:import, user:, name: '2023_January.json') + create(:import, user:, name: 'data.geojson') + + expect { service }.not_to change(Import, :count) end end end From 409e3b745d88cefe1b86a2d35d4b681430d70711 Mon Sep 17 00:00:00 2001 From: GED Date: Mon, 16 Dec 2024 20:53:48 +0100 Subject: [PATCH 04/11] feat: added certain files for watcher_spec with valid and invalid users --- app/services/imports/watcher.rb | 9 +- .../location-history.json | 1 + .../watched/user@domain.com/2023_January.json | 1 + .../watched/user@domain.com/Records.json | 1 + .../user@domain.com/export_same_points.json | 1 + .../gpx_track_single_segment.gpx | 1239 +++++++++++++++++ .../user@domain.com/location-history.json | 1 + .../watched/user@domain.com/owntracks.rec | 16 + spec/services/imports/watcher_spec.rb | 31 +- 9 files changed, 1271 insertions(+), 29 deletions(-) create mode 100644 spec/fixtures/files/watched/invalid_user@domain.com/location-history.json create mode 100644 spec/fixtures/files/watched/user@domain.com/2023_January.json create mode 100644 spec/fixtures/files/watched/user@domain.com/Records.json create mode 100644 spec/fixtures/files/watched/user@domain.com/export_same_points.json create mode 100644 spec/fixtures/files/watched/user@domain.com/gpx_track_single_segment.gpx create mode 100644 spec/fixtures/files/watched/user@domain.com/location-history.json create mode 100644 spec/fixtures/files/watched/user@domain.com/owntracks.rec diff --git a/app/services/imports/watcher.rb b/app/services/imports/watcher.rb index 15854f80..a8808ffd 100644 --- a/app/services/imports/watcher.rb +++ b/app/services/imports/watcher.rb @@ -4,6 +4,7 @@ class Imports::Watcher class UnsupportedSourceError < StandardError; end WATCHED_DIR_PATH = Rails.root.join('tmp/imports/watched') + SUPPORTED_FORMATS = %w[.gpx .json .rec].freeze def call user_directories.each do |user_email| @@ -36,7 +37,7 @@ class Imports::Watcher def file_names(directory_path) Dir.entries(directory_path).select do |file| - ['.gpx', '.json', '.rec'].include?(File.extname(file)) + SUPPORTED_FORMATS.include?(File.extname(file)) end end @@ -82,7 +83,7 @@ class Imports::Watcher :google_semantic_history else :geojson - end + end when 'rec' then :owntracks when 'gpx' then :gpx else raise UnsupportedSourceError, 'Unsupported source ' @@ -103,8 +104,6 @@ class Imports::Watcher OwnTracks::RecParser.new(file).call when :gpx Hash.from_xml(file) - else - JSON.parse(file) end end -end +end \ No newline at end of file diff --git a/spec/fixtures/files/watched/invalid_user@domain.com/location-history.json b/spec/fixtures/files/watched/invalid_user@domain.com/location-history.json new file mode 100644 index 00000000..1c51592c --- /dev/null +++ b/spec/fixtures/files/watched/invalid_user@domain.com/location-history.json @@ -0,0 +1 @@ +{"type": "FeatureCollection"} \ No newline at end of file diff --git a/spec/fixtures/files/watched/user@domain.com/2023_January.json b/spec/fixtures/files/watched/user@domain.com/2023_January.json new file mode 100644 index 00000000..1c51592c --- /dev/null +++ b/spec/fixtures/files/watched/user@domain.com/2023_January.json @@ -0,0 +1 @@ +{"type": "FeatureCollection"} \ No newline at end of file diff --git a/spec/fixtures/files/watched/user@domain.com/Records.json b/spec/fixtures/files/watched/user@domain.com/Records.json new file mode 100644 index 00000000..1c51592c --- /dev/null +++ b/spec/fixtures/files/watched/user@domain.com/Records.json @@ -0,0 +1 @@ +{"type": "FeatureCollection"} \ No newline at end of file diff --git a/spec/fixtures/files/watched/user@domain.com/export_same_points.json b/spec/fixtures/files/watched/user@domain.com/export_same_points.json new file mode 100644 index 00000000..2ecfb883 --- /dev/null +++ b/spec/fixtures/files/watched/user@domain.com/export_same_points.json @@ -0,0 +1 @@ +{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}}]} diff --git a/spec/fixtures/files/watched/user@domain.com/gpx_track_single_segment.gpx b/spec/fixtures/files/watched/user@domain.com/gpx_track_single_segment.gpx new file mode 100644 index 00000000..3176fb4f --- /dev/null +++ b/spec/fixtures/files/watched/user@domain.com/gpx_track_single_segment.gpx @@ -0,0 +1,1239 @@ + + + + La Zubia - balcon + + Garmin Connect + + + + + La Zubia - balcon + + + 824.93 + + + + 822.91 + + + + 819.57 + + + + 815.2 + + + + 811.41 + + + + 808.11 + + + + 805.33 + + + + 802.85 + + + + 800.8 + + + + 798.9 + + + + 797.19 + + + + 795.8 + + + + 794.31 + + + + 793.25 + + + + 792.19 + + + + 791.44 + + + + 791.24 + + + + 791.47 + + + + 792.04 + + + + 792.18 + + + + 793.94 + + + + 795.29 + + + + 796.89 + + + + 798.7 + + + + 801.44 + + + + 803.97 + + + + 806.6 + + + + 809.27 + + + + 811.96 + + + + 814.62 + + + + 817.54 + + + + 820.18 + + + + 822.76 + + + + 825.25 + + + + 827.89 + + + + 830.82 + + + + 833.17 + + + + 835.42 + + + + 837.9 + + + + 839.89 + + + + 841.98 + + + + 844.17 + + + + 846.01 + + + + 847.32 + + + + 848.51 + + + + 849.54 + + + + 850.3 + + + + 850.74 + + + + 851.11 + + + + 851.31 + + + + 851.37 + + + + 851.36 + + + + 851.21 + + + + 851.04 + + + + 850.86 + + + + 850.41 + + + + 849.94 + + + + 849.54 + + + + 849.08 + + + + 848.67 + + + + 848.36 + + + + 848.08 + + + + 847.87 + + + + 847.77 + + + + 847.74 + + + + 847.75 + + + + 847.81 + + + + 847.96 + + + + 848.17 + + + + 848.37 + + + + 848.68 + + + + 849.01 + + + + 849.24 + + + + 849.47 + + + + 849.7 + + + + 849.88 + + + + 850.1 + + + + 850.25 + + + + 850.38 + + + + 850.47 + + + + 850.46 + + + + 850.35 + + + + 850.35 + + + + 850.02 + + + + 849.6 + + + + 849.05 + + + + 848.37 + + + + 847.54 + + + + 846.57 + + + + 845.55 + + + + 844.29 + + + + 842.85 + + + + 841.43 + + + + 839.98 + + + + 838.63 + + + + 837.18 + + + + 835.48 + + + + 833.92 + + + + 832.43 + + + + 831.06 + + + + 829.84 + + + + 829.04 + + + + 828.42 + + + + 828.15 + + + + 828.11 + + + + 828.51 + + + + 829.55 + + + + 830.31 + + + + 831.12 + + + + 831.93 + + + + 832.91 + + + + 833.85 + + + + 834.91 + + + + 836.07 + + + + 837.2 + + + + 838.38 + + + + 839.56 + + + + 840.58 + + + + 841.58 + + + + 842.46 + + + + 843.23 + + + + 843.46 + + + + 843.41 + + + + 842.64 + + + + 841.84 + + + + 840.81 + + + + 839.56 + + + + 837.86 + + + + 836.03 + + + + 833.91 + + + + 831.55 + + + + 828.71 + + + + 825.47 + + + + 820.96 + + + + 817.85 + + + + 814.71 + + + + 811.52 + + + + 808.25 + + + + 805.03 + + + + 801.68 + + + + 798.27 + + + + 794.91 + + + + 791.73 + + + + 788.61 + + + + 785.48 + + + + 782.4 + + + + 779.42 + + + + 776.47 + + + + 773.67 + + + + 770.99 + + + + 768.4 + + + + 765.66 + + + + 763.1 + + + + 760.26 + + + + 757.88 + + + + 755.75 + + + + 753.7 + + + + 751.75 + + + + 749.94 + + + + 748.17 + + + + 746.34 + + + + 744.47 + + + + 743.18 + + + + 742.0 + + + + 741.01 + + + + 740.17 + + + + 739.53 + + + + 738.88 + + + + 738.42 + + + + 738.16 + + + + 738.01 + + + + 738.01 + + + + 738.11 + + + + 738.36 + + + + 738.8 + + + + 739.13 + + + + 739.78 + + + + 740.12 + + + + 740.55 + + + + 740.93 + + + + 741.31 + + + + 741.6 + + + + 741.82 + + + + 741.89 + + + + 741.94 + + + + 741.89 + + + + 742.0 + + + + 742.05 + + + + 742.17 + + + + 742.28 + + + + 742.49 + + + + 742.74 + + + + 742.86 + + + + 743.34 + + + + 744.01 + + + + 744.96 + + + + 746.14 + + + + 747.41 + + + + 748.68 + + + + 750.03 + + + + 751.57 + + + + 753.47 + + + + 755.4 + + + + 757.49 + + + + 759.68 + + + + 762.09 + + + + 764.56 + + + + 767.4 + + + + 770.3 + + + + 773.45 + + + + 776.83 + + + + 780.51 + + + + 783.74 + + + + 786.94 + + + + 790.76 + + + + 794.06 + + + + 797.36 + + + + 800.75 + + + + 804.12 + + + + 807.53 + + + + 811.02 + + + + 814.61 + + + + 818.13 + + + + 821.6 + + + + 825.29 + + + + 828.89 + + + + 832.37 + + + + 836.28 + + + + 839.49 + + + + 842.19 + + + + 844.74 + + + + 847.21 + + + + 849.34 + + + + 851.3 + + + + 852.93 + + + + 854.35 + + + + 855.69 + + + + 856.86 + + + + 857.72 + + + + 858.43 + + + + 858.78 + + + + 859.01 + + + + 859.0 + + + + 858.97 + + + + 859.21 + + + + 859.45 + + + + 859.73 + + + + 860.06 + + + + 860.45 + + + + 861.08 + + + + 861.61 + + + + 862.29 + + + + 863.0 + + + + 863.9 + + + + 864.96 + + + + 866.07 + + + + 867.3 + + + + 869.0 + + + + 870.45 + + + + 871.97 + + + + 873.37 + + + + 874.8 + + + + 876.17 + + + + 877.6 + + + + 879.15 + + + + 880.87 + + + + 882.54 + + + + 884.28 + + + + 886.01 + + + + 887.84 + + + + 889.62 + + + + 891.29 + + + + 892.83 + + + + 893.87 + + + + 894.78 + + + + 895.66 + + + + 896.51 + + + + 896.83 + + + + 896.95 + + + + 896.98 + + + + 896.67 + + + + 896.92 + + + + 897.13 + + + + 897.08 + + + + 897.65 + + + + 898.62 + + + + 899.59 + + + + 900.3 + + + + 901.06 + + + + 901.98 + + + + 902.94 + + + + 904.14 + + + + 905.06 + + + + 905.5 + + + + 905.8 + + + + 905.47 + + + + 905.91 + + + + 906.01 + + + + 905.66 + + + + 904.85 + + + + 904.4 + + + + 903.49 + + + + 903.02 + + + + 901.8 + + + + 901.42 + + + + + \ No newline at end of file diff --git a/spec/fixtures/files/watched/user@domain.com/location-history.json b/spec/fixtures/files/watched/user@domain.com/location-history.json new file mode 100644 index 00000000..1c51592c --- /dev/null +++ b/spec/fixtures/files/watched/user@domain.com/location-history.json @@ -0,0 +1 @@ +{"type": "FeatureCollection"} \ No newline at end of file diff --git a/spec/fixtures/files/watched/user@domain.com/owntracks.rec b/spec/fixtures/files/watched/user@domain.com/owntracks.rec new file mode 100644 index 00000000..5ddc6ef7 --- /dev/null +++ b/spec/fixtures/files/watched/user@domain.com/owntracks.rec @@ -0,0 +1,16 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [102.0, 0.5] + }, + "properties": { + "timestamp": "2023-01-01T00:00:00Z", + "accuracy": 10 + } + } + ] +} \ No newline at end of file diff --git a/spec/services/imports/watcher_spec.rb b/spec/services/imports/watcher_spec.rb index d745c352..834e2059 100644 --- a/spec/services/imports/watcher_spec.rb +++ b/spec/services/imports/watcher_spec.rb @@ -5,16 +5,13 @@ require 'rails_helper' RSpec.describe Imports::Watcher do describe '#call' do subject(:service) { described_class.new.call } + let(:watched_dir_path) { Rails.root.join('spec/fixtures/files/watched') } let(:user) { create(:user, email: 'user@domain.com') } before do - FileUtils.mkdir_p(watched_dir_path.join(user.email)) stub_const('Imports::Watcher::WATCHED_DIR_PATH', watched_dir_path) - end - - after do - FileUtils.rm_rf(watched_dir_path) + Sidekiq::Testing.inline! end context 'when there are no files in the watched directory' do @@ -28,39 +25,25 @@ RSpec.describe Imports::Watcher do context 'when there are files in the watched directory' do context 'when the file has a valid user email' do it 'creates an import for the user' do - Sidekiq::Testing.inline! - File.write(watched_dir_path.join(user.email, 'location-history.json'), '{"type": "FeatureCollection"}') - File.write(watched_dir_path.join(user.email, 'Records.json'), '{"type": "FeatureCollection"}') - File.write(watched_dir_path.join(user.email, '2023_January.json'), '{"type": "FeatureCollection"}') - File.write(watched_dir_path.join(user.email, 'owntracks.rec'), '{"type": "FeatureCollection"}') - File.write(watched_dir_path.join(user.email, 'gpx_track_single_segment.gpx'), '{"type": "FeatureCollection"}') - - expect { service }.to change(user.imports, :count).by(5) + expect { service }.to change(user.imports, :count).by(6) end end context 'when the file has an invalid user email' do it 'does not create an import' do - FileUtils.mkdir_p(watched_dir_path.join('invalid@domain.com')) - File.write(watched_dir_path.join('invalid@domain.com', 'location-history.json'), '{"type": "FeatureCollection"}') - File.write(watched_dir_path.join('invalid@domain.com', 'Records.json'), '{"type": "FeatureCollection"}') - File.write(watched_dir_path.join('invalid@domain.com', '2023_January.json'), '{"type": "FeatureCollection"}') - File.write(watched_dir_path.join('invalid@domain.com', 'owntracks.rec'), '{"type": "FeatureCollection"}') - File.write(watched_dir_path.join('invalid@domain.com', 'gpx_track_single_segment.gpx'), '{"type": "FeatureCollection"}') - - expect { service }.not_to change(Import, :count) + expect { service }.not_to change(Import, :count) end end context 'when the import already exists' do it 'does not create a new import' do + create(:import, user:, name: '2023_January.json') create(:import, user:, name: 'export_same_points.json') create(:import, user:, name: 'gpx_track_single_segment.gpx') create(:import, user:, name: 'location-history.json') + create(:import, user:, name: 'owntracks.rec') create(:import, user:, name: 'Records.json') - create(:import, user:, name: '2023_January.json') - create(:import, user:, name: 'data.geojson') - + expect { service }.not_to change(Import, :count) end end From a3614abdfa0122546468c581b02125bbb9b287a6 Mon Sep 17 00:00:00 2001 From: GED Date: Tue, 17 Dec 2024 11:19:15 +0100 Subject: [PATCH 05/11] fix: raw_data parsing json since all formats are expecting import and user_id, but not a file --- app/services/imports/watcher.rb | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/app/services/imports/watcher.rb b/app/services/imports/watcher.rb index a8808ffd..8d9c9c1d 100644 --- a/app/services/imports/watcher.rb +++ b/app/services/imports/watcher.rb @@ -48,6 +48,7 @@ class Imports::Watcher return if import.persisted? import.source = source(file_name) + import.raw_data = raw_data(file_path, import.source) import.save! @@ -73,7 +74,7 @@ class Imports::Watcher end def source(file_name) - case file_name.split('.').last + case file_name.split('.').last.downcase when 'json' if file_name.match?(/location-history/i) :google_phone_takeout @@ -93,17 +94,15 @@ class Imports::Watcher def raw_data(file_path, source) file = File.read(file_path) - case source - when :google_phone_takeout - GoogleMaps::PhoneTakeoutParser.new(file).call - when :google_semantic_history - GoogleMaps::SemanticHistoryParser.new(file).call - when :google_records - GoogleMaps::RecordsParser.new(file).call - when :owntracks - OwnTracks::RecParser.new(file).call + case source.to_sym when :gpx Hash.from_xml(file) + when :json, :geojson, :google_phone_takeout, :google_records, :google_semantic_history + JSON.parse(file) + when :owntracks + OwnTracks::RecParser.new(file).call + else + raise UnsupportedSourceError, "Unsupported source: #{source}" end end end \ No newline at end of file From ef8882037f4f7d053f244f744665f1da03cb49bf Mon Sep 17 00:00:00 2001 From: GED Date: Tue, 17 Dec 2024 11:29:19 +0100 Subject: [PATCH 06/11] refactor: rubocop --- app/services/imports/watcher.rb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/app/services/imports/watcher.rb b/app/services/imports/watcher.rb index 8d9c9c1d..98ef06da 100644 --- a/app/services/imports/watcher.rb +++ b/app/services/imports/watcher.rb @@ -48,7 +48,6 @@ class Imports::Watcher return if import.persisted? import.source = source(file_name) - import.raw_data = raw_data(file_path, import.source) import.save! @@ -75,7 +74,7 @@ class Imports::Watcher def source(file_name) case file_name.split('.').last.downcase - when 'json' + when 'json' if file_name.match?(/location-history/i) :google_phone_takeout elsif file_name.match?(/Records/i) @@ -84,7 +83,7 @@ class Imports::Watcher :google_semantic_history else :geojson - end + end when 'rec' then :owntracks when 'gpx' then :gpx else raise UnsupportedSourceError, 'Unsupported source ' @@ -105,4 +104,4 @@ class Imports::Watcher raise UnsupportedSourceError, "Unsupported source: #{source}" end end -end \ No newline at end of file +end From 56363c78a0b1e72ed9383d962fe0c88afb850c21 Mon Sep 17 00:00:00 2001 From: GED Date: Tue, 17 Dec 2024 18:30:55 +0100 Subject: [PATCH 07/11] feat: added code (imitation of the real points) to the files for testing --- .../watched/user@domain.com/2023_January.json | 58 +++++++++++- .../watched/user@domain.com/Records.json | 27 +++++- .../user@domain.com/location-history.json | 90 ++++++++++++++++++- .../watched/user@domain.com/owntracks.rec | 29 +++--- spec/services/imports/watcher_spec.rb | 2 + 5 files changed, 187 insertions(+), 19 deletions(-) diff --git a/spec/fixtures/files/watched/user@domain.com/2023_January.json b/spec/fixtures/files/watched/user@domain.com/2023_January.json index 1c51592c..237a8e2e 100644 --- a/spec/fixtures/files/watched/user@domain.com/2023_January.json +++ b/spec/fixtures/files/watched/user@domain.com/2023_January.json @@ -1 +1,57 @@ -{"type": "FeatureCollection"} \ No newline at end of file +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 14.3439906, + 50.0506312 + ] + }, + "properties": { + "timestamp": "2023-01-01T08:00:00Z" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 14.3439906, + 50.0506312 + ] + }, + "properties": { + "timestamp": "2023-01-01T10:00:00Z" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 14.42076, + 50.08804 + ] + }, + "properties": { + "timestamp": "2023-01-02T12:00:00Z" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 14.42076, + 50.08804 + ] + }, + "properties": { + "timestamp": "2023-01-02T14:00:00Z" + } + } + ] +} \ No newline at end of file diff --git a/spec/fixtures/files/watched/user@domain.com/Records.json b/spec/fixtures/files/watched/user@domain.com/Records.json index 1c51592c..67be4f16 100644 --- a/spec/fixtures/files/watched/user@domain.com/Records.json +++ b/spec/fixtures/files/watched/user@domain.com/Records.json @@ -1 +1,26 @@ -{"type": "FeatureCollection"} \ No newline at end of file +{ + "locations": [ + { + "latitudeE7": 533690550, + "longitudeE7": 836950010, + "accuracy": 150, + "source": "UNKNOWN", + "timestamp": "2012-12-15T14:21:29.460Z" + }, + { + "latitudeE7": 533563380, + "longitudeE7": 837616500, + "accuracy": 18000, + "source": "UNKNOWN", + "timestamp": "2013-01-04T10:22:43.225Z" + }, + { + "latitudeE7": 533690589, + "longitudeE7": 836951347, + "accuracy": 22, + "source": "WIFI", + "deviceTag": 1184882232, + "timestamp": "2013-03-01T05:17:39.849Z" + } + ] +} \ No newline at end of file diff --git a/spec/fixtures/files/watched/user@domain.com/location-history.json b/spec/fixtures/files/watched/user@domain.com/location-history.json index 1c51592c..88805bc2 100644 --- a/spec/fixtures/files/watched/user@domain.com/location-history.json +++ b/spec/fixtures/files/watched/user@domain.com/location-history.json @@ -1 +1,89 @@ -{"type": "FeatureCollection"} \ No newline at end of file +[ + { + "endTime": "2023-08-27T17:04:26.999-05:00", + "startTime": "2023-08-27T15:48:56.000-05:00", + "visit": { + "hierarchyLevel": "0", + "topCandidate": { + "probability": "0.785181", + "semanticType": "Unknown", + "placeID": "ChIJxxP_Qwb2aIYRTwDNDLkUmD0", + "placeLocation": "geo:27.720022,-97.347951" + }, + "probability": "0.710000" + } + }, + { + "endTime": "2023-08-27T22:00:00.000Z", + "startTime": "2023-08-27T20:00:00.000Z", + "timelinePath": [ + { + "point": "geo:27.720007,-97.348044", + "durationMinutesOffsetFromStartTime": "49" + } + ] + }, + { + "endTime": "2023-09-02T23:25:59.000-06:00", + "startTime": "2023-08-27T14:48:56.000-06:00", + "timelineMemory": { + "destinations": [ + { + "identifier": "ChIJs9KSYYBfaIYRj5AOiZNQ0a4" + }, + { + "identifier": "ChIJw6lCfj2sZ4YRl6q2LNNyojk" + }, + { + "identifier": "ChIJA89FstRIAYcRr9I2aBzR89A" + }, + { + "identifier": "ChIJtWVg4r5DFIcRr0zkOeDPEfY" + } + ], + "distanceFromOriginKms": "1594" + } + }, + { + "endTime": "2023-08-28T00:00:00.000Z", + "startTime": "2023-08-27T22:00:00.000Z", + "timelinePath": [ + { + "point": "geo:27.701123,-97.362988", + "durationMinutesOffsetFromStartTime": "4" + }, + { + "point": "geo:27.701123,-97.362988", + "durationMinutesOffsetFromStartTime": "4" + }, + { + "point": "geo:27.687173,-97.363743", + "durationMinutesOffsetFromStartTime": "7" + }, + { + "point": "geo:27.686129,-97.381865", + "durationMinutesOffsetFromStartTime": "10" + }, + { + "point": "geo:27.686129,-97.381865", + "durationMinutesOffsetFromStartTime": "10" + }, + { + "point": "geo:27.686129,-97.381865", + "durationMinutesOffsetFromStartTime": "108" + }, + { + "point": "geo:27.696576,-97.376949", + "durationMinutesOffsetFromStartTime": "109" + }, + { + "point": "geo:27.709617,-97.375988", + "durationMinutesOffsetFromStartTime": "112" + }, + { + "point": "geo:27.709617,-97.375988", + "durationMinutesOffsetFromStartTime": "112" + } + ] + } +] \ No newline at end of file diff --git a/spec/fixtures/files/watched/user@domain.com/owntracks.rec b/spec/fixtures/files/watched/user@domain.com/owntracks.rec index 5ddc6ef7..473591f7 100644 --- a/spec/fixtures/files/watched/user@domain.com/owntracks.rec +++ b/spec/fixtures/files/watched/user@domain.com/owntracks.rec @@ -1,16 +1,13 @@ -{ - "type": "FeatureCollection", - "features": [ - { - "type": "Feature", - "geometry": { - "type": "Point", - "coordinates": [102.0, 0.5] - }, - "properties": { - "timestamp": "2023-01-01T00:00:00Z", - "accuracy": 10 - } - } - ] -} \ No newline at end of file +2024-03-01T09:03:09Z * {"bs":2,"p":100.266,"batt":94,"_type":"location","tid":"RO","topic":"owntracks/test/iPhone 12 Pro","alt":36,"lon":13.332,"vel":0,"t":"p","BSSID":"b0:f2:8:45:94:33","SSID":"Home Wifi","conn":"w","vac":4,"acc":10,"tst":1709283789,"lat":52.225,"m":1,"inrids":["5f1d1b"],"inregions":["home"],"_http":true} +2024-03-01T17:46:02Z * {"bs":1,"p":100.28,"batt":94,"_type":"location","tid":"RO","topic":"owntracks/test/iPhone 12 Pro","alt":36,"lon":13.333,"t":"p","vel":0,"BSSID":"b0:f2:8:45:94:33","conn":"w","SSID":"Home Wifi","vac":3,"cog":98,"acc":9,"tst":1709315162,"lat":52.226,"m":1,"inrids":["5f1d1b"],"inregions":["home"],"_http":true} +2024-03-01T18:26:55Z * {"lon":13.334,"acc":5,"wtst":1696359532,"event":"leave","rid":"5f1d1b","desc":"home","topic":"owntracks/test/iPhone 12 Pro/event","lat":52.227,"t":"c","tst":1709317615,"tid":"RO","_type":"transition","_http":true} +2024-03-01T18:26:55Z * {"cog":40,"batt":85,"lon":13.335,"acc":5,"bs":1,"p":100.279,"vel":3,"vac":3,"lat":52.228,"topic":"owntracks/test/iPhone 12 Pro","t":"c","conn":"m","m":1,"tst":1709317615,"alt":36,"_type":"location","tid":"RO","_http":true} +2024-03-01T18:28:30Z * {"cog":38,"batt":85,"lon":13.336,"acc":5,"bs":1,"p":100.349,"vel":3,"vac":3,"lat":52.229,"topic":"owntracks/test/iPhone 12 Pro","t":"v","conn":"m","m":1,"tst":1709317710,"alt":35,"_type":"location","tid":"RO","_http":true} +2024-03-01T18:33:03Z * {"cog":18,"batt":85,"lon":13.337,"acc":5,"bs":1,"p":100.347,"vel":4,"vac":3,"lat":52.230,"topic":"owntracks/test/iPhone 12 Pro","conn":"m","m":1,"tst":1709317983,"alt":36,"_type":"location","tid":"RO","_http":true} +2024-03-01T18:40:11Z * {"cog":43,"batt":85,"lon":13.338,"acc":5,"bs":1,"p":100.348,"vel":6,"vac":3,"lat":52.231,"topic":"owntracks/test/iPhone 12 Pro","conn":"m","m":1,"tst":1709318411,"alt":37,"_type":"location","tid":"RO","_http":true} +2024-03-01T18:42:57Z * {"cog":320,"batt":85,"lon":13.339,"acc":5,"bs":1,"p":100.353,"vel":3,"vac":3,"lat":52.232,"topic":"owntracks/test/iPhone 12 Pro","t":"v","conn":"m","m":1,"tst":1709318577,"alt":37,"_type":"location","tid":"RO","_http":true} +2024-03-01T18:40:08Z lwt {"_type":"lwt","tst":1717459208} +2024-03-01T18:40:09Z waypoints {"_type":"waypoint","desc":"Home","lat":52.232,"lon":13.339,"rad":50,"tst":1717459768} +2024-03-01T18:40:10Z event {"_type":"transition","acc":5,"desc":"Home","event":"enter","lat":52.232,"lon":13.339,"t":"l","tid":"s8","tst":1717460098,"wtst":1717459768} +2024-03-01T18:40:11Z * {"cog":43,"batt":85,"lon":13.338,"acc":5,"bs":1,"p":100.348,"vel":6,"vac":3,"lat":52.231,"topic":"owntracks/test/iPhone 12 Pro","conn":"m","m":1,"tst":1709318411,"alt":37,"_type":"location","tid":"RO","_http":true} +2024-03-01T18:40:11Z * {"cog":43,"batt":85,"lon":13.341,"acc":5,"bs":1,"p":100.348,"created_at":1709318940,"vel":6,"vac":3,"lat":52.234,"topic":"owntracks/test/iPhone 12 Pro","conn":"m","m":1,"tst":1709318411,"alt":37,"_type":"location","tid":"RO","_http":true} diff --git a/spec/services/imports/watcher_spec.rb b/spec/services/imports/watcher_spec.rb index 834e2059..05ed8759 100644 --- a/spec/services/imports/watcher_spec.rb +++ b/spec/services/imports/watcher_spec.rb @@ -14,6 +14,8 @@ RSpec.describe Imports::Watcher do Sidekiq::Testing.inline! end + after { Sidekiq::Testing.fake! } + context 'when there are no files in the watched directory' do it 'does not call ImportJob' do expect(ImportJob).not_to receive(:perform_later) From 49be1fe5a151c2398358ee4ee33101ffd97276fa Mon Sep 17 00:00:00 2001 From: GED Date: Mon, 23 Dec 2024 10:11:11 +0100 Subject: [PATCH 08/11] feat: added a real draft for location_history --- .byebug_history | 7 ++ .../location-history.json | 90 ++++++++++++++++++- 2 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 .byebug_history diff --git a/.byebug_history b/.byebug_history new file mode 100644 index 00000000..662c1215 --- /dev/null +++ b/.byebug_history @@ -0,0 +1,7 @@ +continue + +continue +continie +continue +Point.count +Point diff --git a/spec/fixtures/files/watched/invalid_user@domain.com/location-history.json b/spec/fixtures/files/watched/invalid_user@domain.com/location-history.json index 1c51592c..88805bc2 100644 --- a/spec/fixtures/files/watched/invalid_user@domain.com/location-history.json +++ b/spec/fixtures/files/watched/invalid_user@domain.com/location-history.json @@ -1 +1,89 @@ -{"type": "FeatureCollection"} \ No newline at end of file +[ + { + "endTime": "2023-08-27T17:04:26.999-05:00", + "startTime": "2023-08-27T15:48:56.000-05:00", + "visit": { + "hierarchyLevel": "0", + "topCandidate": { + "probability": "0.785181", + "semanticType": "Unknown", + "placeID": "ChIJxxP_Qwb2aIYRTwDNDLkUmD0", + "placeLocation": "geo:27.720022,-97.347951" + }, + "probability": "0.710000" + } + }, + { + "endTime": "2023-08-27T22:00:00.000Z", + "startTime": "2023-08-27T20:00:00.000Z", + "timelinePath": [ + { + "point": "geo:27.720007,-97.348044", + "durationMinutesOffsetFromStartTime": "49" + } + ] + }, + { + "endTime": "2023-09-02T23:25:59.000-06:00", + "startTime": "2023-08-27T14:48:56.000-06:00", + "timelineMemory": { + "destinations": [ + { + "identifier": "ChIJs9KSYYBfaIYRj5AOiZNQ0a4" + }, + { + "identifier": "ChIJw6lCfj2sZ4YRl6q2LNNyojk" + }, + { + "identifier": "ChIJA89FstRIAYcRr9I2aBzR89A" + }, + { + "identifier": "ChIJtWVg4r5DFIcRr0zkOeDPEfY" + } + ], + "distanceFromOriginKms": "1594" + } + }, + { + "endTime": "2023-08-28T00:00:00.000Z", + "startTime": "2023-08-27T22:00:00.000Z", + "timelinePath": [ + { + "point": "geo:27.701123,-97.362988", + "durationMinutesOffsetFromStartTime": "4" + }, + { + "point": "geo:27.701123,-97.362988", + "durationMinutesOffsetFromStartTime": "4" + }, + { + "point": "geo:27.687173,-97.363743", + "durationMinutesOffsetFromStartTime": "7" + }, + { + "point": "geo:27.686129,-97.381865", + "durationMinutesOffsetFromStartTime": "10" + }, + { + "point": "geo:27.686129,-97.381865", + "durationMinutesOffsetFromStartTime": "10" + }, + { + "point": "geo:27.686129,-97.381865", + "durationMinutesOffsetFromStartTime": "108" + }, + { + "point": "geo:27.696576,-97.376949", + "durationMinutesOffsetFromStartTime": "109" + }, + { + "point": "geo:27.709617,-97.375988", + "durationMinutesOffsetFromStartTime": "112" + }, + { + "point": "geo:27.709617,-97.375988", + "durationMinutesOffsetFromStartTime": "112" + } + ] + } +] \ No newline at end of file From 1f223142f38ffae340d04716b2dfdf8b4c720a4b Mon Sep 17 00:00:00 2001 From: GED Date: Tue, 24 Dec 2024 13:08:14 +0100 Subject: [PATCH 09/11] error: watcher does not create points --- app/jobs/import_job.rb | 4 + .../google_maps/semantic_history_parser.rb | 2 + app/services/imports/watcher.rb | 7 +- .../watched/user@domain.com/2023_January.json | 13 + .../watched/user@domain.com/Records.json | 14 + .../user@domain.com/export_same_points.json | 176 ++- .../gpx_track_single_segment.gpx | 1200 ----------------- spec/services/imports/watcher_spec.rb | 6 + 8 files changed, 220 insertions(+), 1202 deletions(-) diff --git a/app/jobs/import_job.rb b/app/jobs/import_job.rb index a07cfa46..298c2770 100644 --- a/app/jobs/import_job.rb +++ b/app/jobs/import_job.rb @@ -4,9 +4,13 @@ class ImportJob < ApplicationJob queue_as :imports def perform(user_id, import_id) + puts "ImportJob started for user_id: #{user_id}, import_id: #{import_id}" + user = User.find(user_id) import = user.imports.find(import_id) import.process! + puts "ImportJob finished for user_id: #{user_id}, import_id: #{import_id}" + end end diff --git a/app/services/google_maps/semantic_history_parser.rb b/app/services/google_maps/semantic_history_parser.rb index 83d2486b..b7181eff 100644 --- a/app/services/google_maps/semantic_history_parser.rb +++ b/app/services/google_maps/semantic_history_parser.rb @@ -33,6 +33,8 @@ class GoogleMaps::SemanticHistoryParser ) broadcast_import_progress(import, index) + puts "Created point: #{location['latitudeE7'] / 1e7}, #{location['longitudeE7'] / 1e7} at #{location['timestamp']}" + end end diff --git a/app/services/imports/watcher.rb b/app/services/imports/watcher.rb index 98ef06da..b3922baa 100644 --- a/app/services/imports/watcher.rb +++ b/app/services/imports/watcher.rb @@ -10,11 +10,13 @@ class Imports::Watcher user_directories.each do |user_email| user = User.find_by(email: user_email) next unless user +puts "Processing directory for user: #{user.email}" user_directory_path = File.join(WATCHED_DIR_PATH, user_email) file_names = file_names(user_directory_path) file_names.each do |file_name| + puts "Processing file: #{file_name}" process_file(user, user_directory_path, file_name) end end @@ -51,8 +53,11 @@ class Imports::Watcher import.raw_data = raw_data(file_path, import.source) import.save! + puts "Import saved for file: #{file_name}" + ImportJob.perform_later(user.id, import.id) + puts "ImportJob enqueued for user_id: #{user.id}, import_id: #{import.id}" end def find_or_initialize_import(user, file_name) @@ -104,4 +109,4 @@ class Imports::Watcher raise UnsupportedSourceError, "Unsupported source: #{source}" end end -end +end \ No newline at end of file diff --git a/spec/fixtures/files/watched/user@domain.com/2023_January.json b/spec/fixtures/files/watched/user@domain.com/2023_January.json index 237a8e2e..1a25ab3c 100644 --- a/spec/fixtures/files/watched/user@domain.com/2023_January.json +++ b/spec/fixtures/files/watched/user@domain.com/2023_January.json @@ -52,6 +52,19 @@ "properties": { "timestamp": "2023-01-02T14:00:00Z" } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 14.42076, + 50.08804 + ] + }, + "properties": { + "timestamp": "2023-01-02T16:00:00Z" + } } ] } \ No newline at end of file diff --git a/spec/fixtures/files/watched/user@domain.com/Records.json b/spec/fixtures/files/watched/user@domain.com/Records.json index 67be4f16..930ca395 100644 --- a/spec/fixtures/files/watched/user@domain.com/Records.json +++ b/spec/fixtures/files/watched/user@domain.com/Records.json @@ -21,6 +21,20 @@ "source": "WIFI", "deviceTag": 1184882232, "timestamp": "2013-03-01T05:17:39.849Z" + }, + { + "latitudeE7": 533700000, + "longitudeE7": 836960000, + "accuracy": 50, + "source": "GPS", + "timestamp": "2013-04-01T12:00:00.000Z" + }, + { + "latitudeE7": 533710000, + "longitudeE7": 836970000, + "accuracy": 30, + "source": "GPS", + "timestamp": "2013-05-01T08:30:00.000Z" } ] } \ No newline at end of file diff --git a/spec/fixtures/files/watched/user@domain.com/export_same_points.json b/spec/fixtures/files/watched/user@domain.com/export_same_points.json index 2ecfb883..7951ef5a 100644 --- a/spec/fixtures/files/watched/user@domain.com/export_same_points.json +++ b/spec/fixtures/files/watched/user@domain.com/export_same_points.json @@ -1 +1,175 @@ -{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}}]} +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + "37.6173", + "55.755826" + ] + }, + "properties": { + "battery_status": "unplugged", + "ping": "MyString", + "battery": 1, + "tracker_id": "MyString", + "topic": "MyString", + "altitude": 1, + "longitude": "37.6173", + "velocity": "0", + "trigger": "background_event", + "bssid": "MyString", + "ssid": "MyString", + "connection": "wifi", + "vertical_accuracy": 1, + "accuracy": 1, + "timestamp": 1609459200, + "latitude": "55.755826", + "mode": 1, + "inrids": [], + "in_regions": [], + "city": null, + "country": null, + "geodata": {} + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + "37.6173", + "55.755826" + ] + }, + "properties": { + "battery_status": "unplugged", + "ping": "MyString", + "battery": 1, + "tracker_id": "MyString", + "topic": "MyString", + "altitude": 1, + "longitude": "37.6173", + "velocity": "0", + "trigger": "background_event", + "bssid": "MyString", + "ssid": "MyString", + "connection": "wifi", + "vertical_accuracy": 1, + "accuracy": 1, + "timestamp": 1609459200, + "latitude": "55.755826", + "mode": 1, + "inrids": [], + "in_regions": [], + "city": null, + "country": null, + "geodata": {} + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + "37.6173", + "55.755826" + ] + }, + "properties": { + "battery_status": "unplugged", + "ping": "MyString", + "battery": 1, + "tracker_id": "MyString", + "topic": "MyString", + "altitude": 1, + "longitude": "37.6173", + "velocity": "0", + "trigger": "background_event", + "bssid": "MyString", + "ssid": "MyString", + "connection": "wifi", + "vertical_accuracy": 1, + "accuracy": 1, + "timestamp": 1609459200, + "latitude": "55.755826", + "mode": 1, + "inrids": [], + "in_regions": [], + "city": null, + "country": null, + "geodata": {} + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + "37.6173", + "55.755826" + ] + }, + "properties": { + "battery_status": "unplugged", + "ping": "MyString", + "battery": 1, + "tracker_id": "MyString", + "topic": "MyString", + "altitude": 1, + "longitude": "37.6173", + "velocity": "0", + "trigger": "background_event", + "bssid": "MyString", + "ssid": "MyString", + "connection": "wifi", + "vertical_accuracy": 1, + "accuracy": 1, + "timestamp": 1609459200, + "latitude": "55.755826", + "mode": 1, + "inrids": [], + "in_regions": [], + "city": null, + "country": null, + "geodata": {} + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + "37.6173", + "55.755826" + ] + }, + "properties": { + "battery_status": "unplugged", + "ping": "MyString", + "battery": 1, + "tracker_id": "MyString", + "topic": "MyString", + "altitude": 1, + "longitude": "37.6173", + "velocity": "0", + "trigger": "background_event", + "bssid": "MyString", + "ssid": "MyString", + "connection": "wifi", + "vertical_accuracy": 1, + "accuracy": 1, + "timestamp": 1609459200, + "latitude": "55.755826", + "mode": 1, + "inrids": [], + "in_regions": [], + "city": null, + "country": null, + "geodata": {} + } + } + ] +} \ No newline at end of file diff --git a/spec/fixtures/files/watched/user@domain.com/gpx_track_single_segment.gpx b/spec/fixtures/files/watched/user@domain.com/gpx_track_single_segment.gpx index 3176fb4f..7c73a8ac 100644 --- a/spec/fixtures/files/watched/user@domain.com/gpx_track_single_segment.gpx +++ b/spec/fixtures/files/watched/user@domain.com/gpx_track_single_segment.gpx @@ -34,1206 +34,6 @@ 811.41 - - 808.11 - - - - 805.33 - - - - 802.85 - - - - 800.8 - - - - 798.9 - - - - 797.19 - - - - 795.8 - - - - 794.31 - - - - 793.25 - - - - 792.19 - - - - 791.44 - - - - 791.24 - - - - 791.47 - - - - 792.04 - - - - 792.18 - - - - 793.94 - - - - 795.29 - - - - 796.89 - - - - 798.7 - - - - 801.44 - - - - 803.97 - - - - 806.6 - - - - 809.27 - - - - 811.96 - - - - 814.62 - - - - 817.54 - - - - 820.18 - - - - 822.76 - - - - 825.25 - - - - 827.89 - - - - 830.82 - - - - 833.17 - - - - 835.42 - - - - 837.9 - - - - 839.89 - - - - 841.98 - - - - 844.17 - - - - 846.01 - - - - 847.32 - - - - 848.51 - - - - 849.54 - - - - 850.3 - - - - 850.74 - - - - 851.11 - - - - 851.31 - - - - 851.37 - - - - 851.36 - - - - 851.21 - - - - 851.04 - - - - 850.86 - - - - 850.41 - - - - 849.94 - - - - 849.54 - - - - 849.08 - - - - 848.67 - - - - 848.36 - - - - 848.08 - - - - 847.87 - - - - 847.77 - - - - 847.74 - - - - 847.75 - - - - 847.81 - - - - 847.96 - - - - 848.17 - - - - 848.37 - - - - 848.68 - - - - 849.01 - - - - 849.24 - - - - 849.47 - - - - 849.7 - - - - 849.88 - - - - 850.1 - - - - 850.25 - - - - 850.38 - - - - 850.47 - - - - 850.46 - - - - 850.35 - - - - 850.35 - - - - 850.02 - - - - 849.6 - - - - 849.05 - - - - 848.37 - - - - 847.54 - - - - 846.57 - - - - 845.55 - - - - 844.29 - - - - 842.85 - - - - 841.43 - - - - 839.98 - - - - 838.63 - - - - 837.18 - - - - 835.48 - - - - 833.92 - - - - 832.43 - - - - 831.06 - - - - 829.84 - - - - 829.04 - - - - 828.42 - - - - 828.15 - - - - 828.11 - - - - 828.51 - - - - 829.55 - - - - 830.31 - - - - 831.12 - - - - 831.93 - - - - 832.91 - - - - 833.85 - - - - 834.91 - - - - 836.07 - - - - 837.2 - - - - 838.38 - - - - 839.56 - - - - 840.58 - - - - 841.58 - - - - 842.46 - - - - 843.23 - - - - 843.46 - - - - 843.41 - - - - 842.64 - - - - 841.84 - - - - 840.81 - - - - 839.56 - - - - 837.86 - - - - 836.03 - - - - 833.91 - - - - 831.55 - - - - 828.71 - - - - 825.47 - - - - 820.96 - - - - 817.85 - - - - 814.71 - - - - 811.52 - - - - 808.25 - - - - 805.03 - - - - 801.68 - - - - 798.27 - - - - 794.91 - - - - 791.73 - - - - 788.61 - - - - 785.48 - - - - 782.4 - - - - 779.42 - - - - 776.47 - - - - 773.67 - - - - 770.99 - - - - 768.4 - - - - 765.66 - - - - 763.1 - - - - 760.26 - - - - 757.88 - - - - 755.75 - - - - 753.7 - - - - 751.75 - - - - 749.94 - - - - 748.17 - - - - 746.34 - - - - 744.47 - - - - 743.18 - - - - 742.0 - - - - 741.01 - - - - 740.17 - - - - 739.53 - - - - 738.88 - - - - 738.42 - - - - 738.16 - - - - 738.01 - - - - 738.01 - - - - 738.11 - - - - 738.36 - - - - 738.8 - - - - 739.13 - - - - 739.78 - - - - 740.12 - - - - 740.55 - - - - 740.93 - - - - 741.31 - - - - 741.6 - - - - 741.82 - - - - 741.89 - - - - 741.94 - - - - 741.89 - - - - 742.0 - - - - 742.05 - - - - 742.17 - - - - 742.28 - - - - 742.49 - - - - 742.74 - - - - 742.86 - - - - 743.34 - - - - 744.01 - - - - 744.96 - - - - 746.14 - - - - 747.41 - - - - 748.68 - - - - 750.03 - - - - 751.57 - - - - 753.47 - - - - 755.4 - - - - 757.49 - - - - 759.68 - - - - 762.09 - - - - 764.56 - - - - 767.4 - - - - 770.3 - - - - 773.45 - - - - 776.83 - - - - 780.51 - - - - 783.74 - - - - 786.94 - - - - 790.76 - - - - 794.06 - - - - 797.36 - - - - 800.75 - - - - 804.12 - - - - 807.53 - - - - 811.02 - - - - 814.61 - - - - 818.13 - - - - 821.6 - - - - 825.29 - - - - 828.89 - - - - 832.37 - - - - 836.28 - - - - 839.49 - - - - 842.19 - - - - 844.74 - - - - 847.21 - - - - 849.34 - - - - 851.3 - - - - 852.93 - - - - 854.35 - - - - 855.69 - - - - 856.86 - - - - 857.72 - - - - 858.43 - - - - 858.78 - - - - 859.01 - - - - 859.0 - - - - 858.97 - - - - 859.21 - - - - 859.45 - - - - 859.73 - - - - 860.06 - - - - 860.45 - - - - 861.08 - - - - 861.61 - - - - 862.29 - - - - 863.0 - - - - 863.9 - - - - 864.96 - - - - 866.07 - - - - 867.3 - - - - 869.0 - - - - 870.45 - - - - 871.97 - - - - 873.37 - - - - 874.8 - - - - 876.17 - - - - 877.6 - - - - 879.15 - - - - 880.87 - - - - 882.54 - - - - 884.28 - - - - 886.01 - - - - 887.84 - - - - 889.62 - - - - 891.29 - - - - 892.83 - - - - 893.87 - - - - 894.78 - - - - 895.66 - - - - 896.51 - - - - 896.83 - - - - 896.95 - - - - 896.98 - - - - 896.67 - - - - 896.92 - - - - 897.13 - - - - 897.08 - - - - 897.65 - - - - 898.62 - - - - 899.59 - - - - 900.3 - - - - 901.06 - - - - 901.98 - - - - 902.94 - - - - 904.14 - - - - 905.06 - - - - 905.5 - - - - 905.8 - - - - 905.47 - - - - 905.91 - - - - 906.01 - - - - 905.66 - - - - 904.85 - - - - 904.4 - - - - 903.49 - - - - 903.02 - - - - 901.8 - - - - 901.42 - - \ No newline at end of file diff --git a/spec/services/imports/watcher_spec.rb b/spec/services/imports/watcher_spec.rb index 05ed8759..65bcd19a 100644 --- a/spec/services/imports/watcher_spec.rb +++ b/spec/services/imports/watcher_spec.rb @@ -29,6 +29,12 @@ RSpec.describe Imports::Watcher do it 'creates an import for the user' do expect { service }.to change(user.imports, :count).by(6) end + + it 'creates points for the user' do + initial_point_count = Point.count + service + expect(Point.count).to be > initial_point_count + end end context 'when the file has an invalid user email' do From 242d3c23dc7ea2be637b116b2c373db0f563a856 Mon Sep 17 00:00:00 2001 From: GED Date: Wed, 25 Dec 2024 05:04:19 +0100 Subject: [PATCH 10/11] Remove .byebug_history and add it to .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index d53fcdf6..3c01a870 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,5 @@ !/app/assets/builds/.keep .DS_Store .env + +.byebug_history From af8b3b4f5da50f00608d41399876e194dc753c8a Mon Sep 17 00:00:00 2001 From: GED Date: Wed, 25 Dec 2024 05:06:56 +0100 Subject: [PATCH 11/11] fix: remove debbuging messages --- .byebug_history | 7 ------- app/jobs/import_job.rb | 2 -- app/services/google_maps/semantic_history_parser.rb | 2 -- app/services/imports/watcher.rb | 2 +- 4 files changed, 1 insertion(+), 12 deletions(-) delete mode 100644 .byebug_history diff --git a/.byebug_history b/.byebug_history deleted file mode 100644 index 662c1215..00000000 --- a/.byebug_history +++ /dev/null @@ -1,7 +0,0 @@ -continue - -continue -continie -continue -Point.count -Point diff --git a/app/jobs/import_job.rb b/app/jobs/import_job.rb index 298c2770..5cad02b6 100644 --- a/app/jobs/import_job.rb +++ b/app/jobs/import_job.rb @@ -4,13 +4,11 @@ class ImportJob < ApplicationJob queue_as :imports def perform(user_id, import_id) - puts "ImportJob started for user_id: #{user_id}, import_id: #{import_id}" user = User.find(user_id) import = user.imports.find(import_id) import.process! - puts "ImportJob finished for user_id: #{user_id}, import_id: #{import_id}" end end diff --git a/app/services/google_maps/semantic_history_parser.rb b/app/services/google_maps/semantic_history_parser.rb index b7181eff..83d2486b 100644 --- a/app/services/google_maps/semantic_history_parser.rb +++ b/app/services/google_maps/semantic_history_parser.rb @@ -33,8 +33,6 @@ class GoogleMaps::SemanticHistoryParser ) broadcast_import_progress(import, index) - puts "Created point: #{location['latitudeE7'] / 1e7}, #{location['longitudeE7'] / 1e7} at #{location['timestamp']}" - end end diff --git a/app/services/imports/watcher.rb b/app/services/imports/watcher.rb index b3922baa..4ea6c07b 100644 --- a/app/services/imports/watcher.rb +++ b/app/services/imports/watcher.rb @@ -57,7 +57,7 @@ puts "Processing directory for user: #{user.email}" ImportJob.perform_later(user.id, import.id) - puts "ImportJob enqueued for user_id: #{user.id}, import_id: #{import.id}" + puts "ImportJob enqueued for user_id: #{user.id}, import_id: #{import.id}" end def find_or_initialize_import(user, file_name)