Implement the import of geodata from Immich to Dawarich

.app_version

@@ -1 +1 @@
-0.10.0
+0.11.0

CHANGELOG.md

@@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [0.11.0] — 2024-08-21
+
+### Added
+
+- A user can now trigger the import of their geodata from Immich to Dawarich by clicking the "Import Immich data" button in the Imports page.
+- A user can now provide a url and an api key for their Immich instance and then trigger the import of their geodata from Immich to Dawarich. This can be done in the Settings page.
+
+### Changed
+
+- Table columns on the Exports page were reordered to make it more user-friendly.
+- Exports are now being named with this pattern: "export_from_dd.mm.yyyy_to_dd.mm.yyyy.json" where "dd.mm.yyyy" is the date range of the export.
+- Notification about any error now will include the stacktrace.
+
 ## [0.10.0] — 2024-08-20
 
 ### Added

Gemfile

@@ -9,6 +9,7 @@ gem 'chartkick'
 gem 'data_migrate'
 gem 'devise'
 gem 'geocoder'
+gem 'httparty'
 gem 'importmap-rails'
 gem 'kaminari'
 gem 'lograge'

Gemfile.lock

@@ -138,6 +138,10 @@ GEM
     globalid (1.2.1)
       activesupport (>= 6.1)
     hashdiff (1.1.0)
+    httparty (0.22.0)
+      csv
+      mini_mime (>= 1.0.0)
+      multi_xml (>= 0.5.2)
     i18n (1.14.5)
       concurrent-ruby (~> 1.0)
     importmap-rails (2.0.1)
@@ -182,6 +186,8 @@ GEM
     mini_mime (1.1.5)
     minitest (5.25.1)
     msgpack (1.7.2)
+    multi_xml (0.7.1)
+      bigdecimal (~> 3.1)
     mutex_m (0.2.0)
     net-imap (0.4.12)
       date
@@ -425,6 +431,7 @@ DEPENDENCIES
   ffaker
   foreman
   geocoder
+  httparty
   importmap-rails
   kaminari
   lograge

app/controllers/exports_controller.rb

@@ -9,7 +9,7 @@ class ExportsController < ApplicationController
   end
 
   def create
-    export_name = "#{params[:start_at].to_date}_#{params[:end_at].to_date}"
+    export_name = "export_from_#{params[:start_at].to_date}_to_#{params[:end_at].to_date}"
     export = current_user.exports.create(name: export_name, status: :created)
 
     ExportJob.perform_later(export.id, params[:start_at], params[:end_at])

app/controllers/settings/background_jobs_controller.rb

@@ -9,7 +9,7 @@ class Settings::BackgroundJobsController < ApplicationController
   end
 
   def create
-    EnqueueReverseGeocodingJob.perform_later(params[:job_name], current_user.id)
+    EnqueueBackgroundJob.perform_later(params[:job_name], current_user.id)
 
     flash.now[:notice] = 'Job was successfully created.'
 

app/controllers/settings_controller.rb

@@ -3,8 +3,7 @@
 class SettingsController < ApplicationController
   before_action :authenticate_user!
 
-  def index
+  def index; end
-  end
 
   def update
     current_user.update(settings: settings_params)
@@ -31,7 +30,8 @@ class SettingsController < ApplicationController
   def settings_params
     params.require(:settings).permit(
       :meters_between_routes, :minutes_between_routes, :fog_of_war_meters,
-      :time_threshold_minutes, :merge_threshold_minutes, :route_opacity
+      :time_threshold_minutes, :merge_threshold_minutes, :route_opacity,
+      :immich_url, :immich_api_key
     )
   end
 end

app/jobs/enqueue_background_job.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class EnqueueBackgroundJob < ApplicationJob
+  queue_as :reverse_geocoding
+
+  def perform(job_name, user_id)
+    case job_name
+    when 'start_immich_import'
+      ImportImmichGeodataJob.perform_later(user_id)
+    when 'start_reverse_geocoding', 'continue_reverse_geocoding'
+      Jobs::Create.new(job_name, user_id).call
+    else
+      raise ArgumentError, "Unknown job name: #{job_name}"
+    end
+  end
+end

app/jobs/enqueue_reverse_geocoding_job.rb

@@ -1,9 +0,0 @@
-# frozen_string_literal: true
-
-class EnqueueReverseGeocodingJob < ApplicationJob
-  queue_as :reverse_geocoding
-
-  def perform(job_name, user_id)
-    Jobs::Create.new(job_name, user_id).call
-  end
-end

app/jobs/import_immich_geodata_job.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class ImportImmichGeodataJob < ApplicationJob
+  queue_as :imports
+
+  def perform(user_id)
+    user = User.find(user_id)
+
+    Immich::ImportGeodata.new(user).call
+  end
+end

app/jobs/import_job.rb

@@ -26,19 +26,21 @@ class ImportJob < ApplicationJob
       user:,
       kind: :error,
       title: 'Import failed',
-      content: "Import \"#{import.name}\" failed: #{e.message}"
+      content: "Import \"#{import.name}\" failed: #{e.message}, stacktrace: #{e.backtrace.join("\n")}"
     ).call
   end
 
   private
 
   def parser(source)
+    # Bad classes naming by the way, they are not parsers, they are point creators
     case source
     when 'google_semantic_history'  then GoogleMaps::SemanticHistoryParser
     when 'google_records'           then GoogleMaps::RecordsParser
     when 'google_phone_takeout'     then GoogleMaps::PhoneTakeoutParser
     when 'owntracks'                then OwnTracks::ExportParser
     when 'gpx'                      then Gpx::TrackParser
+    when 'immich_api'               then Immich::ImportParser
     end
   end
 end

app/models/import.rb

@@ -8,5 +8,8 @@ class Import < ApplicationRecord
 
   include ImportUploader::Attachment(:raw)
 
-  enum source: { google_semantic_history: 0, owntracks: 1, google_records: 2, google_phone_takeout: 3, gpx: 4 }
+  enum source: {
+    google_semantic_history: 0, owntracks: 1, google_records: 2,
+    google_phone_takeout: 3, gpx: 4, immich_api: 5
+  }
 end

app/services/create_stats.rb

@@ -29,7 +29,12 @@ class CreateStats
 
       Notifications::Create.new(user:, kind: :info, title: 'Stats updated', content: 'Stats updated').call
     rescue StandardError => e
-      Notifications::Create.new(user:, kind: :error, title: 'Stats update failed', content: e.message).call
+      Notifications::Create.new(
+        user:,
+        kind: :error,
+        title: 'Stats update failed',
+        content: "#{e.message}, stacktrace: #{e.backtrace.join("\n")}"
+      ).call
     end
   end
 

app/services/exports/create.rb

@@ -37,7 +37,7 @@ class Exports::Create
       user:,
       kind: :error,
       title: 'Export failed',
-      content: "Export \"#{export.name}\" failed: #{e.message}"
+      content: "Export \"#{export.name}\" failed: #{e.message}, stacktrace: #{e.backtrace.join("\n")}"
     ).call
 
     export.update!(status: :failed)

app/services/immich/import_geodata.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+class Immich::ImportGeodata
+  attr_reader :user, :immich_api_base_url, :immich_api_key
+
+  def initialize(user)
+    @user = user
+    @immich_api_base_url = "#{user.settings['immich_url']}/api"
+    @immich_api_key = user.settings['immich_api_key']
+  end
+
+  def call
+    raise ArgumentError, 'Immich API key is missing' if immich_api_key.blank?
+    raise ArgumentError, 'Immich URL is missing'     if user.settings['immich_url'].blank?
+
+    immich_data       = retrieve_immich_data
+    immich_data_json  = parse_immich_data(immich_data)
+    file_name         = file_name(immich_data_json)
+    import            = user.imports.find_or_initialize_by(name: file_name, source: :immich_api)
+
+    create_import_failed_notification and return unless import.new_record?
+
+    import.raw_data = immich_data_json
+    import.save!
+    ImportJob.perform_later(user.id, import.id)
+  end
+
+  private
+
+  def headers
+    {
+      'x-api-key' => immich_api_key,
+      'accept' => 'application/json'
+    }
+  end
+
+  def retrieve_immich_data
+    (1..12).flat_map do |month_number|
+      (1..31).map do |day|
+        url = "#{immich_api_base_url}/assets/memory-lane?day=#{day}&month=#{month_number}"
+
+        JSON.parse(HTTParty.get(url, headers:).body)
+      end
+    end
+  end
+
+  def valid?(asset)
+    asset.dig('exifInfo', 'latitude') &&
+      asset.dig('exifInfo', 'longitude') &&
+      asset.dig('exifInfo', 'dateTimeOriginal')
+  end
+
+  def parse_immich_data(immich_data)
+    geodata = []
+
+    immich_data.each do |memory_lane|
+      log_no_data and next if memory_lane_invalid?(memory_lane)
+
+      assets = extract_assets(memory_lane)
+
+      assets.each { |asset| geodata << extract_geodata(asset) if valid?(asset) }
+    end
+
+    geodata.sort_by { |data| data[:timestamp] }
+  end
+
+  def memory_lane_invalid?(memory_lane)
+    memory_lane.is_a?(Hash) && memory_lane['statusCode'] == 404
+  end
+
+  def extract_assets(memory_lane)
+    memory_lane.flat_map { |lane| lane['assets'] }.compact
+  end
+
+  def extract_geodata(asset)
+    {
+      latitude: asset.dig('exifInfo', 'latitude'),
+      longitude: asset.dig('exifInfo', 'longitude'),
+      timestamp: Time.zone.parse(asset.dig('exifInfo', 'dateTimeOriginal')).to_i
+    }
+  end
+
+  def log_no_data
+    Rails.logger.debug 'No data found'
+  end
+
+  def create_import_failed_notification
+    Notifications::Create.new(
+      user:,
+      kind: :info,
+      title: 'Import was not created',
+      content: 'Import with the same name already exists. If you want to proceed, delete the existing import and try again.'
+    ).call
+  end
+
+  def file_name(immich_data_json)
+    from              = Time.zone.at(immich_data_json.first[:timestamp]).to_date
+    to                = Time.zone.at(immich_data_json.last[:timestamp]).to_date
+
+    "immich-geodata-#{user.email}-from-#{from}-to-#{to}.json"
+  end
+end

app/services/immich/import_parser.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+class Immich::ImportParser
+  attr_reader :import, :json, :user_id
+
+  def initialize(import, user_id)
+    @import = import
+    @json = import.raw_data
+    @user_id = user_id
+  end
+
+  def call
+    json.each { |point| create_point(point) }
+  end
+
+  def create_point(point)
+    return 0 if point['latitude'].blank? || point['longitude'].blank? || point['timestamp'].blank?
+    return 0 if point_exists?(point, point['timestamp'])
+
+    Point.create(
+      latitude:   point['latitude'].to_d,
+      longitude:  point['longitude'].to_d,
+      timestamp:  point['timestamp'],
+      raw_data:   point,
+      import_id:  import.id,
+      user_id:
+    )
+
+    1
+  end
+
+  def point_exists?(point, timestamp)
+    Point.exists?(
+      latitude:   point['latitude'].to_d,
+      longitude:  point['longitude'].to_d,
+      timestamp:,
+      user_id:
+    )
+  end
+end

app/views/exports/index.html.erb

@@ -28,15 +28,16 @@
           <thead>
             <tr>
               <th>Name</th>
+              <th>Created at</th>
               <th>Status</th>
               <th>Actions</th>
-              <th>Created at</th>
             </tr>
           </thead>
           <tbody>
             <% @exports.each do |export| %>
               <tr>
                 <td><%= export.name %></td>
+                <td><%= export.created_at.strftime('%Y-%m-%d %H:%M:%S') %></td>
                 <td><%= export.status %></td>
                 <td>
                   <% if export.completed? %>
@@ -44,7 +45,6 @@
                   <% end %>
                   <%= link_to 'Delete', export, data: { confirm: "Are you sure?", turbo_confirm: "Are you sure?", turbo_method: :delete }, method: :delete, class: "px-4 py-2 bg-red-500 text-white rounded-md" %>
                 </td>
-                <td><%= export.created_at.strftime('%Y-%m-%d %H:%M:%S') %></td>
               </tr>
             <% end %>
           </tbody>

app/views/imports/index.html.erb

@@ -1,9 +1,15 @@
 <% content_for :title, 'Imports' %>
 
 <div class="w-full">
-  <div class="flex justify-between items-center">
+  <div class="flex justify-between items-center mb-3">
     <h1 class="font-bold text-4xl">Imports</h1>
     <%= link_to "New import", new_import_path, class: "rounded-lg py-3 px-5 bg-blue-600 text-white block font-medium" %>
+
+    <% if current_user.settings['immich_url'] && current_user.settings['immich_api_key'] %>
+      <%= link_to 'Import Immich data', settings_background_jobs_path(job_name: 'start_immich_import'), method: :post, data: { confirm: 'Are you sure?', turbo_confirm: 'Are you sure?', turbo_method: :post }, class: 'rounded-lg py-3 px-5 bg-blue-600 text-white block font-medium' %>
+    <% else %>
+      <a href='' class="rounded-lg py-3 px-5 bg-blue-900 text-gray block font-medium tooltip cursor-not-allowed" data-tip="You need to provide your Immich instance data in the Settings">Import Immich data</a>
+    <% end %>
   </div>
 
   <div id="imports" class="min-w-full">

app/views/settings/index.html.erb

@@ -164,6 +164,14 @@
           <% end %>
           <%= f.number_field :route_opacity, value: current_user.settings['route_opacity'], class: "input input-bordered" %>
         </div>
+        <div class="form-control my-2">
+          <%= f.label :immich_url %>
+          <%= f.text_field :immich_url, value: current_user.settings['immich_url'], class: "input input-bordered", placeholder: 'http://192.168.0.1:2283' %>
+        </div>
+        <div class="form-control my-2">
+          <%= f.label :immich_api_key %>
+          <%= f.text_field :immich_api_key, value: current_user.settings['immich_api_key'], class: "input input-bordered", placeholder: 'xxxxxxxxxxxxxx' %>
+        </div>
         <div class="form-control my-2">
           <%= f.submit "Update", class: "btn btn-primary" %>
         </div>

app/views/visits/index.html.erb

@@ -50,7 +50,7 @@
                 <div class="flex items-center justify-between">
                   <div>
                     <div class="text-lg font-black <%= 'underline decoration-dotted' if visit.pending? %>">
-                      <%= visit.area.name %>
+                      <%= visit&.area&.name %>
                     </div>
                     <div>
                       <%= "#{visit.started_at.strftime('%H:%M')} - #{visit.ended_at.strftime('%H:%M')}" %>

spec/fixtures/files/immich/geodata.json

@@ -0,0 +1,12 @@
+[
+  {
+    "latitude": 59.0000,
+    "longitude": 30.0000,
+    "timestamp": 978296400
+  },
+  {
+    "latitude": 55.0001,
+    "longitude": 37.0001,
+    "timestamp": 978296400
+  }
+]

spec/fixtures/files/immich/response.json

@@ -0,0 +1,24 @@
+[
+  [
+    {
+      "assets": [
+        {
+          "exifInfo": {
+            "dateTimeOriginal": "2022-12-31T23:17:06.170Z",
+            "latitude": 52.0000,
+            "longitude": 13.0000
+          }
+        },
+        {
+          "exifInfo": {
+            "dateTimeOriginal": "2022-12-31T23:21:53.140Z",
+            "latitude": 52.0000,
+            "longitude": 13.0000
+          }
+        }
+      ],
+      "title": "1 year ago",
+      "yearsAgo": 1
+    }
+  ]
+]

spec/jobs/enqueue_background_job_spec.rb

@@ -2,13 +2,13 @@
 
 require 'rails_helper'
 
-RSpec.describe EnqueueReverseGeocodingJob, type: :job do
+RSpec.describe EnqueueBackgroundJob, type: :job do
   let(:job_name) { 'start_reverse_geocoding' }
   let(:user_id) { 1 }
 
   it 'calls job creation service' do
     expect(Jobs::Create).to receive(:new).with(job_name, user_id).and_return(double(call: nil))
 
-    EnqueueReverseGeocodingJob.perform_now(job_name, user_id)
+    EnqueueBackgroundJob.perform_now(job_name, user_id)
   end
 end

spec/jobs/import_immich_geodata_job_spec.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe ImportImmichGeodataJob, type: :job do
+  describe '#perform' do
+    let(:user) { create(:user) }
+
+    it 'calls Immich::ImportGeodata' do
+      expect_any_instance_of(Immich::ImportGeodata).to receive(:call)
+
+      ImportImmichGeodataJob.perform_now(user.id)
+    end
+  end
+end

spec/models/import_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'rails_helper'
 
 RSpec.describe Import, type: :model do
@@ -13,7 +15,8 @@ RSpec.describe Import, type: :model do
         owntracks: 1,
         google_records: 2,
         google_phone_takeout: 3,
-        gpx: 4
+        gpx: 4,
+        immich_api: 5
       )
     end
   end

spec/requests/settings/background_jobs_spec.rb

@@ -46,7 +46,7 @@ RSpec.describe '/settings/background_jobs', type: :request do
           it 'enqueues a new job' do
             expect do
               post settings_background_jobs_url, params:
-            end.to have_enqueued_job(EnqueueReverseGeocodingJob)
+            end.to have_enqueued_job(EnqueueBackgroundJob)
           end
 
           it 'redirects to the created settings_background_job' do

spec/services/immich/import_geodata_spec.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe Immich::ImportGeodata do
+  describe '#call' do
+    subject(:service) { described_class.new(user).call }
+
+    let(:user) do
+      create(:user, settings: { 'immich_url' => 'http://immich.app', 'immich_api_key' => '123456' })
+    end
+    let(:immich_data) do
+      [
+        {
+          "assets": [
+            {
+              "exifInfo": {
+                "dateTimeOriginal": '2022-12-31T23:17:06.170Z',
+                "latitude": 52.0000,
+                "longitude": 13.0000
+              }
+            },
+            {
+              "exifInfo": {
+                "dateTimeOriginal": '2022-12-31T23:21:53.140Z',
+                "latitude": 52.0000,
+                "longitude": 13.0000
+              }
+            }
+          ],
+          "title": '1 year ago',
+          "yearsAgo": 1
+        }
+      ].to_json
+    end
+
+    context 'when user has immich_url and immich_api_key' do
+      before do
+        stub_request(
+          :any,
+          %r{http://immich\.app/api/assets/memory-lane\?day=(1[0-9]|2[0-9]|3[01]|[1-9])&month=(1[0-2]|[1-9])}
+        ).to_return(status: 200, body: immich_data, headers: {})
+      end
+
+      it 'creates import' do
+        expect { service }.to change { Import.count }.by(1)
+      end
+
+      it 'enqueues ImportJob' do
+        expect(ImportJob).to receive(:perform_later)
+
+        service
+      end
+
+      context 'when import already exists' do
+        before { service }
+
+        it 'does not create new import' do
+          expect { service }.not_to(change { Import.count })
+        end
+
+        it 'does not enqueue ImportJob' do
+          expect(ImportJob).to_not receive(:perform_later)
+
+          service
+        end
+      end
+    end
+
+    context 'when user has no immich_url' do
+      before do
+        user.settings['immich_url'] = nil
+        user.save
+      end
+
+      it 'raises ArgumentError' do
+        expect { service }.to raise_error(ArgumentError, 'Immich URL is missing')
+      end
+    end
+
+    context 'when user has no immich_api_key' do
+      before do
+        user.settings['immich_api_key'] = nil
+        user.save
+      end
+
+      it 'raises ArgumentError' do
+        expect { service }.to raise_error(ArgumentError, 'Immich API key is missing')
+      end
+    end
+  end
+end

spec/services/immich/import_parser_spec.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe Immich::ImportParser do
+  describe '#call' do
+    subject(:service) { described_class.new(import, user.id).call }
+
+    let(:user) do
+      create(:user, settings: { 'immich_url' => 'http://immich.app', 'immich_api_key' => '123456' })
+    end
+
+    let(:immich_data) do
+      JSON.parse(File.read(Rails.root.join('spec/fixtures/files/immich/geodata.json')))
+    end
+    let(:import) { create(:import, user:, raw_data: immich_data) }
+
+    context 'when there are no points' do
+      it 'creates new points' do
+        expect { service }.to change { Point.count }.by(2)
+      end
+
+      it 'creates points with correct attributes' do
+        service
+
+        expect(Point.first.latitude.to_f).to eq(59.0000)
+        expect(Point.first.longitude.to_f).to eq(30.0000)
+        expect(Point.first.timestamp).to eq(978_296_400)
+        expect(Point.first.import_id).to eq(import.id)
+
+        expect(Point.second.latitude.to_f).to eq(55.0001)
+        expect(Point.second.longitude.to_f).to eq(37.0001)
+        expect(Point.second.timestamp).to eq(978_296_400)
+        expect(Point.second.import_id).to eq(import.id)
+      end
+    end
+
+    context 'when there are points with the same coordinates' do
+      let!(:existing_point) do
+        create(:point, latitude: 59.0000, longitude: 30.0000, timestamp: 978_296_400, user:)
+      end
+
+      it 'creates only new points' do
+        expect { service }.to change { Point.count }.by(1)
+      end
+    end
+  end
+end