Merge pull request #301 from Freika/feature/imports_watchdog

Imports watcher

.app_version

@@ -1 +1 @@
-0.14.7
+0.15.0

.gitignore

@@ -24,12 +24,22 @@
 /tmp/storage/*
 !/tmp/storage/
 !/tmp/storage/.keep
+/tmp/imports/*
+!/tmp/imports/
+/tmp/imports/watched/*
+!/tmp/imports/watched/
+!/tmp/imports/watched/.keep
+!/tmp/imports/watched/put-your-files-here.txt
+
 
 /public/assets
 
-# We need directories for import and export files, but not the files themselves.
+# Ignore all files under /public/exports except the .keep file
 /public/exports/*
 !/public/exports/.keep
+!/public/exports/
+
+# Ignore all files under /public/imports, but keep .keep files and the watched directory
 /public/imports/*
 !/public/imports/.keep
 

CHANGELOG.md

@@ -5,6 +5,21 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+# 0.15.0 - 2024-10-03
+
+### Added
+
+- You can now put your GPX and GeoJSON files to `tmp/imports/watched` directory and Dawarich will automatically import them. This is useful if you have a service that can put files to the directory automatically. The directory is being watched every 60 minutes for new files.
+
+### Changed
+
+- Monkey patch for Geocoder to support http along with https for Photon API host was removed becausee it was breaking the reverse geocoding process. Now you can use only https for the Photon API host. This might be changed in the future
+- Disable retries for some background jobs
+
+### Fixed
+
+- Stats update is now being correctly triggered every 6 hours
+
 # [0.14.7] - 2024-10-01
 
 ### Fixed

app/jobs/area_visits_calculating_job.rb

@@ -2,6 +2,7 @@
 
 class AreaVisitsCalculatingJob < ApplicationJob
   queue_as :default
+  sidekiq_options retry: false
 
   def perform(user_id)
     user = User.find(user_id)

app/jobs/area_visits_calculation_scheduling_job.rb

@@ -2,6 +2,7 @@
 
 class AreaVisitsCalculationSchedulingJob < ApplicationJob
   queue_as :default
+  sidekiq_options retry: false
 
   def perform
     User.find_each { AreaVisitsCalculatingJob.perform_later(_1.id) }

app/jobs/enqueue_background_job.rb

@@ -6,7 +6,7 @@ class EnqueueBackgroundJob < ApplicationJob
   def perform(job_name, user_id)
     case job_name
     when 'start_immich_import'
-      ImportImmichGeodataJob.perform_later(user_id)
+      Import::ImmichGeodataJob.perform_later(user_id)
     when 'start_reverse_geocoding', 'continue_reverse_geocoding'
       Jobs::Create.new(job_name, user_id).call
     else

app/jobs/import/google_takeout_job.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class ImportGoogleTakeoutJob < ApplicationJob
+class Import::GoogleTakeoutJob < ApplicationJob
   queue_as :imports
   sidekiq_options retry: false
 

app/jobs/import/immich_geodata_job.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
-class ImportImmichGeodataJob < ApplicationJob
+class Import::ImmichGeodataJob < ApplicationJob
   queue_as :imports
+  sidekiq_options retry: false
 
   def perform(user_id)
     user = User.find(user_id)

app/jobs/import/watcher_job.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class Import::WatcherJob < ApplicationJob
+  queue_as :imports
+  sidekiq_options retry: false
+
+  def perform
+    Imports::Watcher.new.call
+  end
+end

app/jobs/stat_creating_job.rb

@@ -4,6 +4,8 @@ class StatCreatingJob < ApplicationJob
   queue_as :stats
 
   def perform(user_ids = nil)
+    user_ids = user_ids.nil? ? User.pluck(:id) : Array(user_ids)
+
     CreateStats.new(user_ids).call
   end
 end

app/jobs/visit_suggesting_job.rb

@@ -2,6 +2,7 @@
 
 class VisitSuggestingJob < ApplicationJob
   queue_as :visit_suggesting
+  sidekiq_options retry: false
 
   def perform(user_ids: [], start_at: 1.day.ago, end_at: Time.current)
     users = user_ids.any? ? User.where(id: user_ids) : User.all

app/services/imports/watcher.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+class Imports::Watcher
+  class UnsupportedSourceError < StandardError; end
+
+  WATCHED_DIR_PATH = Rails.root.join('tmp/imports/watched')
+
+  def call
+    %w[*.gpx *.json].each do |pattern|
+      Dir[WATCHED_DIR_PATH.join(pattern)].each do |file_path|
+        # valid file_name example: "email@dawarich.app_2024-01-01-2024-01-31.json"
+        file_name = File.basename(file_path)
+
+        user = find_user(file_name)
+        next unless user
+
+        import = find_or_initialize_import(user, file_name)
+
+        next if import.persisted?
+
+        import_id = set_import_attributes(import, file_path, file_name)
+
+        ImportJob.perform_later(user.id, import_id)
+      end
+    end
+  end
+
+  private
+
+  def find_user(file_name)
+    email = file_name.split('_').first
+
+    User.find_by(email:)
+  end
+
+  def find_or_initialize_import(user, file_name)
+    import_name = file_name.split('_')[1..].join('_')
+
+    Import.find_or_initialize_by(user:, name: import_name)
+  end
+
+  def set_import_attributes(import, file_path, file_name)
+    source = source(file_name)
+
+    import.source = source
+    import.raw_data = raw_data(file_path, source)
+
+    import.save!
+
+    import.id
+  end
+
+  def source(file_name)
+    case file_name.split('.').last
+    when 'json' then :geojson
+    when 'gpx'  then :gpx
+    else raise UnsupportedSourceError, 'Unsupported source '
+    end
+  end
+
+  def raw_data(file_path, source)
+    file = File.read(file_path)
+
+    source == :gpx ? Hash.from_xml(file) : JSON.parse(file)
+  end
+end

app/services/tasks/imports/google_records.rb

@@ -32,7 +32,7 @@ class Tasks::Imports::GoogleRecords
 
   def schedule_import_jobs(json_data, import_id)
     json_data['locations'].each do |json|
-      ImportGoogleTakeoutJob.perform_later(import_id, json.to_json)
+      Import::GoogleTakeoutJob.perform_later(import_id, json.to_json)
     end
   end
 

config/initializers/00_monkey_patching.rb

@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-# By default, Geocoder supports only https protocol when talking to Photon API.
-# This is kinda inconvenient when you're running a local instance of Photon
-# and want to use http protocol. This monkey patch allows you to do that.
-
-module Geocoder::Lookup
-  class Photon < Base
-    private
-
-    def supported_protocols
-      %i[https http]
-    end
-  end
-end

config/schedule.yml

@@ -1,16 +1,21 @@
 # config/schedule.yml
 
 stat_creating_job:
-  cron: "0 */6 * * *" # every 6 hours
+  cron: "0 */6 * * *" # every 6 hour
   class: "StatCreatingJob"
-  queue: default
+  queue: stats
 
 area_visits_calculation_scheduling_job:
   cron: "0 0 * * *" # every day at 0:00
   class: "AreaVisitsCalculationSchedulingJob"
-  queue: default
+  queue: visit_suggesting
 
 visit_suggesting_job:
   cron: "0 1 * * *" # every day at 1:00
   class: "VisitSuggestingJob"
-  queue: default
+  queue: visit_suggesting
+
+watcher_job:
+  cron: "0 */1 * * *" # every 1 hour
+  class: "Import::WatcherJob"
+  queue: imports

spec/jobs/import/immich_geodata_job_spec.rb

@@ -2,14 +2,14 @@
 
 require 'rails_helper'
 
-RSpec.describe ImportImmichGeodataJob, type: :job do
+RSpec.describe Import::ImmichGeodataJob, type: :job do
   describe '#perform' do
     let(:user) { create(:user) }
 
     it 'calls Immich::ImportGeodata' do
       expect_any_instance_of(Immich::ImportGeodata).to receive(:call)
 
-      ImportImmichGeodataJob.perform_now(user.id)
+      described_class.perform_now(user.id)
     end
   end
 end

spec/jobs/import/watcher_job_spec.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe Import::WatcherJob, type: :job do
+  describe '#perform' do
+    it 'calls Imports::Watcher' do
+      expect_any_instance_of(Imports::Watcher).to receive(:call)
+
+      described_class.perform_now
+    end
+  end
+end

spec/services/imports/watcher_spec.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe Imports::Watcher do
+  describe '#call' do
+    subject(:service) { described_class.new.call }
+
+    let(:watched_dir_path) { Rails.root.join('spec/fixtures/files/watched') }
+    let(:user) { create(:user, email: 'user@domain.com') }
+
+    before do
+      stub_const('Imports::Watcher::WATCHED_DIR_PATH', watched_dir_path)
+    end
+
+    context 'when there are no files in the watched directory' do
+      it 'does not call ImportJob' do
+        expect(ImportJob).not_to receive(:perform_later)
+
+        service
+      end
+    end
+
+    context 'when there are files in the watched directory' do
+      Sidekiq::Testing.inline! do
+        context 'when the file has a valid user email' do
+          it 'creates an import for the user' do
+            expect { service }.to change(user.imports, :count).by(2)
+          end
+        end
+
+        context 'when the file has an invalid user email' do
+          it 'does not create an import' do
+            expect { service }.not_to change(Import, :count)
+          end
+        end
+
+        context 'when the import already exists' do
+          it 'does not create a new import' do
+            create(:import, user:, name: 'export_same_points.json')
+            create(:import, user:, name: 'gpx_track_single_segment.gpx')
+
+            expect { service }.not_to change(Import, :count)
+          end
+        end
+      end
+    end
+  end
+end

spec/services/tasks/imports/google_records_spec.rb

@@ -7,8 +7,8 @@ RSpec.describe Tasks::Imports::GoogleRecords do
     let(:user) { create(:user) }
     let(:file_path) { Rails.root.join('spec/fixtures/files/google/records.json') }
 
-    it 'schedules the ImportGoogleTakeoutJob' do
-      expect(ImportGoogleTakeoutJob).to receive(:perform_later).exactly(3).times
+    it 'schedules the Import::GoogleTakeoutJob' do
+      expect(Import::GoogleTakeoutJob).to receive(:perform_later).exactly(3).times
 
       described_class.new(file_path, user.email).call
     end

tmp/imports/watched/put-your-files-here.txt

@@ -0,0 +1,5 @@
+The /public/imporst/watched/ directory is watched by Dawarich. Any files you put in this directory will be imported into the database. The name of the file must start with an email of the user you want to import the file for. The email must be followed by an underscore symbol (_) and the name of the file.
+
+For example, if you want to import a file for the user with the email address "email@dawarich.app", you would name the file "email@dawarich.app_2024-05-01_2024-05-31.gpx". The file will be imported into the database and the user will receive a notification in the app.
+
+Both GeoJSON and GPX files are supported.