Extract latitude and longitude from raw_data

.gitignore

@@ -68,3 +68,5 @@
 
 /config/credentials/production.key
 /config/credentials/production.yml.enc
+
+Makefile

CHANGELOG.md

@@ -5,13 +5,17 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
 
-# 0.25.5 - UNRELEASED
+# 0.25.5 - 2025-04-13
 
 ## Removed
 
 - Optional telemetry was removed from the app.
 - Sidekiq Web UI is now protected by basic auth in non-self-hosted mode. (Needs to be tested)
 
+## Changed
+
+- `rake points:migrate_to_lonlat` task now also tries to extract latitude and longitude from `raw_data` column before using `longitude` and `latitude` columns to fill `lonlat` column.
+
 
 # 0.25.4 - 2025-04-02
 

app/services/photos/import_parser.rb

@@ -3,17 +3,20 @@
 class Photos::ImportParser
   include Imports::Broadcaster
   include PointValidation
-  attr_reader :import, :json, :user_id
+  attr_reader :import, :user_id
 
   def initialize(import, user_id)
     @import = import
-    @json = import.raw_data
     @user_id = user_id
   end
 
   def call
+    import.file.download do |file|
+      json = Oj.load(file)
+
       json.each.with_index(1) { |point, index| create_point(point, index) }
     end
+  end
 
   def create_point(point, index)
     return 0 if point['latitude'].blank? || point['longitude'].blank? || point['timestamp'].blank?

app/services/points/raw_data_lonlat_extractor.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+class Points::RawDataLonlatExtractor
+  def initialize(point)
+    @point = point
+  end
+
+  def call
+    lonlat = extract_lonlat(@point)
+
+    @point.update(
+      longitude: lonlat[0],
+      latitude: lonlat[1]
+    )
+  end
+
+  private
+
+  # rubocop:disable Metrics/MethodLength
+  def extract_lonlat(point)
+    if point.raw_data['activitySegment']['waypointPath']['waypoints'][0]
+      # google_semantic_history_parser
+      [
+        point.raw_data['activitySegment']['waypointPath']['waypoints'][0]['lngE7'].to_f / 10**7,
+        point.raw_data['activitySegment']['waypointPath']['waypoints'][0]['latE7'].to_f / 10**7
+      ]
+    elsif point.raw_data['longitudeE7'] && point.raw_data['latitudeE7']
+      # google records
+      [
+        point.raw_data['longitudeE7'].to_f / 10**7,
+        point.raw_data['latitudeE7'].to_f / 10**7
+      ]
+    elsif point.raw_data['position']['LatLng']
+      # google phone export
+      raw_coordinates = point.raw_data['position']['LatLng']
+      if raw_coordinates.include?('°')
+        raw_coordinates.split(', ').map { _1.chomp('°') }
+      else
+        raw_coordinates.delete('geo:').split(',')
+      end
+    elsif point.raw_data['lon'] && point.raw_data['lat']
+      # gpx_track_importer, owntracks
+      [point.raw_data['lon'], point.raw_data['lat']]
+    elsif point.raw_data['geometry']['coordinates'][0] && point.raw_data['geometry']['coordinates'][1]
+      # geojson
+      [
+        point.raw_data['geometry']['coordinates'][0],
+        point.raw_data['geometry']['coordinates'][1]
+      ]
+    elsif point.raw_data['longitude'] && point.raw_data['latitude']
+      # immich_api, photoprism_api
+      [point.raw_data['longitude'], point.raw_data['latitude']]
+    end
+  end
+  # rubocop:enable Metrics/MethodLength
+end

app/views/shared/_navbar.html.erb

@@ -98,8 +98,8 @@
           </div>
           <ul tabindex="0" class="dropdown-content z-[5000] menu p-2 shadow-lg bg-base-100 rounded-box min-w-52" data-notifications-target="list">
             <li><%= link_to 'See all', notifications_path %></li>
-            <div class="divider p-0 m-0"></div>
             <% @unread_notifications.first(10).each do |notification| %>
+              <div class="divider p-0 m-0"></div>
               <li class='notification-item'>
                 <%= link_to notification do %>
                   <%= notification.title %>
@@ -126,7 +126,7 @@
         </li>
       <% else %>
         <li><%= link_to 'Login', new_user_session_path %></li>
-        <% if !SELF_HOSTED && devise_mapping.registerable? && controller_name != 'registrations' %>
+        <% if !SELF_HOSTED && devise_mapping&.registerable? %>
           <li><%= link_to 'Register', new_user_registration_path %></li>
         <% end %>
       <% end %>

lib/tasks/points.rake

@@ -5,6 +5,12 @@ namespace :points do
   task migrate_to_lonlat: :environment do
     puts 'Updating points to use lonlat...'
 
+    points = Point.where(longitude: nil, latitude: nil).select(:id, :longitude, :latitude, :raw_data)
+
+    points.find_each do |point|
+      Points::RawDataLonlatExtractor.new(point).call
+    end
+
     ActiveRecord::Base.connection.execute('REINDEX TABLE points;')
 
     ActiveRecord::Base.transaction do

spec/services/photos/import_parser_spec.rb

@@ -13,7 +13,14 @@ RSpec.describe Photos::ImportParser do
     let(:immich_data) do
       JSON.parse(File.read(Rails.root.join('spec/fixtures/files/immich/geodata.json')))
     end
-    let(:import) { create(:import, user:, raw_data: immich_data) }
+    let(:import) { create(:import, user:) }
+
+    let(:file_path) { Rails.root.join('spec/fixtures/files/immich/geodata.json') }
+    let(:file) { Rack::Test::UploadedFile.new(file_path, 'text/plain') }
+
+    before do
+      import.file.attach(io: File.open(file_path), filename: 'immich_geodata.json', content_type: 'application/json')
+    end
 
     context 'when there are no points' do
       it 'creates new points' do