Support properties->date field for timestamp in GeoJSON imports (#2159)

* Support properties->date field for timestamp in GeoJSON imports

* Fix GeoJSON date parsing

CHANGELOG.md

@@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+# [0.37.4] - Unreleased
+
+## Fixed
+
+- GeoJSON formatted points now have correct timestamp parsed from raw_data['properties']['date'] field.
+- Reduce number of iterations during cache cleaning to improve performance.
+
 # [0.37.3] - Unreleased
 
 ## Fixed

app/services/cache/clean.rb

@@ -6,10 +6,14 @@ class Cache::Clean
       Rails.logger.info('Cleaning cache...')
       delete_control_flag
       delete_version_cache
-      delete_years_tracked_cache
-      delete_points_geocoded_stats_cache
-      delete_countries_cities_cache
-      delete_total_distance_cache
+
+      User.find_each do |user|
+        delete_years_tracked_cache(user)
+        delete_points_geocoded_stats_cache(user)
+        delete_countries_cities_cache(user)
+        delete_total_distance_cache(user)
+      end
+
       Rails.logger.info('Cache cleaned')
     end
 
@@ -23,29 +27,21 @@ class Cache::Clean
       Rails.cache.delete(CheckAppVersion::VERSION_CACHE_KEY)
     end
 
-    def delete_years_tracked_cache
-      User.find_each do |user|
-        Rails.cache.delete("dawarich/user_#{user.id}_years_tracked")
-      end
+    def delete_years_tracked_cache(user)
+      Rails.cache.delete("dawarich/user_#{user.id}_years_tracked")
     end
 
-    def delete_points_geocoded_stats_cache
-      User.find_each do |user|
-        Rails.cache.delete("dawarich/user_#{user.id}_points_geocoded_stats")
-      end
+    def delete_points_geocoded_stats_cache(user)
+      Rails.cache.delete("dawarich/user_#{user.id}_points_geocoded_stats")
     end
 
-    def delete_countries_cities_cache
-      User.find_each do |user|
-        Rails.cache.delete("dawarich/user_#{user.id}_countries_visited")
-        Rails.cache.delete("dawarich/user_#{user.id}_cities_visited")
-      end
+    def delete_countries_cities_cache(user)
+      Rails.cache.delete("dawarich/user_#{user.id}_countries_visited")
+      Rails.cache.delete("dawarich/user_#{user.id}_cities_visited")
     end
 
-    def delete_total_distance_cache
-      User.find_each do |user|
-        Rails.cache.delete("dawarich/user_#{user.id}_total_distance")
-      end
+    def delete_total_distance_cache(user)
+      Rails.cache.delete("dawarich/user_#{user.id}_total_distance")
     end
   end
 end

app/services/geojson/params.rb

@@ -80,18 +80,35 @@ class Geojson::Params
   end
 
   def timestamp(feature)
-    return Time.zone.at(feature[3]) if feature.is_a?(Array)
+    if feature.is_a?(Array)
+      return parse_array_timestamp(feature[3]) if feature[3].present?
 
+      return nil
+    end
+
+    numeric_timestamp(feature) || parse_string_timestamp(feature)
+  end
+
+  def parse_array_timestamp(value)
+    return value.to_i if value.is_a?(Numeric)
+
+    Time.zone.parse(value.to_s).utc.to_i if value.present?
+  end
+
+  def numeric_timestamp(feature)
     value = feature.dig(:properties, :timestamp) ||
             feature.dig(:geometry, :coordinates, 3)
 
-    return Time.zone.at(value.to_i) if value.is_a?(Numeric)
+    value.to_i if value.is_a?(Numeric)
+  end
 
-    ### GPSLogger for Android case ###
-    time = feature.dig(:properties, :time)
+  def parse_string_timestamp(feature)
+    ### GPSLogger for Android / Google Takeout case ###
+    time = feature.dig(:properties, :time) ||
+           feature.dig(:properties, :date)
+    ### /GPSLogger for Android / Google Takeout case ###
 
-    Time.zone.parse(time).to_i if time.present?
-    ### /GPSLogger for Android case ###
+    Time.zone.parse(time).utc.to_i if time.present?
   end
 
   def speed(feature)

app/services/google_maps/phone_takeout_importer.rb

@@ -74,17 +74,17 @@ class GoogleMaps::PhoneTakeoutImporter
 
   def parse_visit_place_location(data_point)
     lat, lon = parse_coordinates(data_point['visit']['topCandidate']['placeLocation'])
-    timestamp = DateTime.parse(data_point['startTime']).to_i
+    timestamp = DateTime.parse(data_point['startTime']).utc.to_i
 
     point_hash(lat, lon, timestamp, data_point)
   end
 
   def parse_activity(data_point)
     start_lat, start_lon = parse_coordinates(data_point['activity']['start'])
-    start_timestamp = DateTime.parse(data_point['startTime']).to_i
+    start_timestamp = DateTime.parse(data_point['startTime']).utc.to_i
 
     end_lat, end_lon = parse_coordinates(data_point['activity']['end'])
-    end_timestamp = DateTime.parse(data_point['endTime']).to_i
+    end_timestamp = DateTime.parse(data_point['endTime']).utc.to_i
 
     [
       point_hash(start_lat, start_lon, start_timestamp, data_point),
@@ -107,16 +107,16 @@ class GoogleMaps::PhoneTakeoutImporter
 
   def parse_semantic_visit(segment)
     lat, lon = parse_coordinates(segment['visit']['topCandidate']['placeLocation']['latLng'])
-    timestamp = DateTime.parse(segment['startTime']).to_i
+    timestamp = DateTime.parse(segment['startTime']).utc.to_i
 
     point_hash(lat, lon, timestamp, segment)
   end
 
   def parse_semantic_activity(segment)
     start_lat, start_lon = parse_coordinates(segment['activity']['start']['latLng'])
-    start_timestamp = DateTime.parse(segment['startTime']).to_i
+    start_timestamp = DateTime.parse(segment['startTime']).utc.to_i
     end_lat, end_lon = parse_coordinates(segment['activity']['end']['latLng'])
-    end_timestamp = DateTime.parse(segment['endTime']).to_i
+    end_timestamp = DateTime.parse(segment['endTime']).utc.to_i
 
     [
       point_hash(start_lat, start_lon, start_timestamp, segment),
@@ -127,7 +127,7 @@ class GoogleMaps::PhoneTakeoutImporter
   def parse_semantic_timeline_path(segment)
     segment['timelinePath'].map do |point|
       lat, lon = parse_coordinates(point['point'])
-      timestamp = DateTime.parse(point['time']).to_i
+      timestamp = DateTime.parse(point['time']).utc.to_i
 
       point_hash(lat, lon, timestamp, segment)
     end
@@ -165,7 +165,7 @@ class GoogleMaps::PhoneTakeoutImporter
       next unless segment.dig('position', 'LatLng')
 
       lat, lon = parse_coordinates(segment['position']['LatLng'])
-      timestamp = DateTime.parse(segment['position']['timestamp']).to_i
+      timestamp = DateTime.parse(segment['position']['timestamp']).utc.to_i
 
       point_hash(lat, lon, timestamp, segment)
     end

app/services/gpx/track_importer.rb

@@ -44,7 +44,7 @@ class Gpx::TrackImporter
     {
       lonlat: "POINT(#{point['lon'].to_d} #{point['lat'].to_d})",
       altitude: point['ele'].to_i,
-      timestamp: Time.parse(point['time']).to_i,
+      timestamp: Time.parse(point['time']).utc.to_i,
       import_id: import.id,
       velocity: speed(point),
       raw_data: point,

app/services/immich/import_geodata.rb

@@ -56,7 +56,7 @@ class Immich::ImportGeodata
       latitude: asset['exifInfo']['latitude'],
       longitude: asset['exifInfo']['longitude'],
       lonlat: "SRID=4326;POINT(#{asset['exifInfo']['longitude']} #{asset['exifInfo']['latitude']})",
-      timestamp: Time.zone.parse(asset['exifInfo']['dateTimeOriginal']).to_i
+      timestamp: Time.zone.parse(asset['exifInfo']['dateTimeOriginal']).utc.to_i
     }
   end
 

app/services/kml/importer.rb

@@ -192,7 +192,7 @@ class Kml::Importer
   end
 
   def build_gx_track_point(timestamp_str, coord_str, index)
-    time = Time.parse(timestamp_str).to_i
+    time = Time.parse(timestamp_str).utc.to_i
     coord_parts = coord_str.split(/\s+/)
     return nil if coord_parts.size < 2
 
@@ -239,7 +239,7 @@ class Kml::Importer
     node = find_timestamp_node(placemark)
     raise 'No timestamp found in placemark' unless node
 
-    Time.parse(node.text).to_i
+    Time.parse(node.text).utc.to_i
   rescue StandardError => e
     Rails.logger.error("Failed to parse timestamp: #{e.message}")
     raise e

app/services/photoprism/import_geodata.rb

@@ -66,7 +66,7 @@ class Photoprism::ImportGeodata
       latitude: asset['Lat'],
       longitude: asset['Lng'],
       lonlat: "SRID=4326;POINT(#{asset['Lng']} #{asset['Lat']})",
-      timestamp: Time.zone.parse(asset['TakenAt']).to_i
+      timestamp: Time.zone.parse(asset['TakenAt']).utc.to_i
     }
   end
 

app/services/users/import_data/notifications.rb

@@ -110,8 +110,8 @@ class Users::ImportData::Notifications
 
   def normalize_timestamp(timestamp)
     case timestamp
-    when String then Time.parse(timestamp).to_i
-    when Time, DateTime then timestamp.to_i
+    when String then Time.parse(timestamp).utc.to_i
+    when Time, DateTime then timestamp.utc.to_i
     else
       timestamp.to_s
     end

db/migrate/20260113230537_set_points_timestamp_from_geojson_date.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class SetPointsTimestampFromGeojsonDate < ActiveRecord::Migration[8.0]
+  def change
+    Point.where(timestamp: nil).find_each do |point|
+      geojson = point.raw_data
+
+      next unless geojson && geojson['properties'] && geojson['properties']['date']
+
+      begin
+        parsed_time = Time.zone.parse(geojson['properties']['date']).utc.to_i
+
+        point.update!(timestamp: parsed_time)
+      rescue ArgumentError => e
+        Rails.logger.warn("Failed to parse date for Point ID #{point.id}: #{e.message}")
+      end
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[8.0].define(version: 2026_01_03_114630) do
+ActiveRecord::Schema[8.0].define(version: 2026_01_13_230537) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_catalog.plpgsql"
   enable_extension "postgis"
@@ -507,6 +507,7 @@ ActiveRecord::Schema[8.0].define(version: 2026_01_03_114630) do
     t.index ["area_id"], name: "index_visits_on_area_id"
     t.index ["place_id"], name: "index_visits_on_place_id"
     t.index ["started_at"], name: "index_visits_on_started_at"
+    t.index ["user_id", "status", "started_at"], name: "index_visits_on_user_id_and_status_and_started_at"
     t.index ["user_id"], name: "index_visits_on_user_id"
   end
 

spec/fixtures/files/geojson/google_takeout_example.json

@@ -0,0 +1,15 @@
+{
+  "type": "FeatureCollection",
+  "features": [
+    {
+      "type": "Feature",
+      "geometry": {
+        "type": "Point",
+        "coordinates": [28, 36]
+      },
+      "properties": {
+        "date": "2016-06-21T06:09:33Z"
+      }
+    }
+  ]
+}

spec/services/cache/clean_spec.rb

@@ -93,14 +93,15 @@ RSpec.describe Cache::Clean do
       # Create a user that will be found during the cleaning process
       user3 = nil
 
-      allow(User).to receive(:find_each).and_yield(user1).and_yield(user2) do |&block|
+      allow(User).to receive(:find_each) do |&block|
+        # Yield existing users
+        block.call(user1)
+        block.call(user2)
+
         # Create a new user while iterating - this should not cause errors
         user3 = create(:user)
         Rails.cache.write("dawarich/user_#{user3.id}_years_tracked", { 2023 => ['May'] })
         Rails.cache.write("dawarich/user_#{user3.id}_points_geocoded_stats", { geocoded: 1, without_data: 0 })
-
-        # Continue with the original block
-        [user1, user2].each(&block)
       end
 
       expect { described_class.call }.not_to raise_error

spec/services/geojson/params_spec.rb

@@ -21,7 +21,7 @@ RSpec.describe Geojson::Params do
         lonlat: 'POINT(0.1 0.1)',
         battery_status: nil,
         battery: nil,
-        timestamp: Time.zone.at(1_609_459_201),
+        timestamp: 1_609_459_201,
         altitude: 1,
         velocity: 1.5,
         tracker_id: nil,
@@ -102,5 +102,37 @@ RSpec.describe Geojson::Params do
         )
       end
     end
+
+    context 'when the json is exported from Google Takeout' do
+      let(:file_path) { Rails.root.join('spec/fixtures/files/geojson/google_takeout_example.json') }
+
+      it 'returns the correct data for each point' do
+        expect(subject.first).to eq(
+          lonlat: 'POINT(28 36)',
+          battery_status: nil,
+          battery: nil,
+          timestamp: Time.parse('2016-06-21T06:09:33Z').to_i,
+          altitude: nil,
+          velocity: 0.0,
+          tracker_id: nil,
+          ssid: nil,
+          accuracy: nil,
+          vertical_accuracy: nil,
+          raw_data: {
+            'geometry' => {
+              'coordinates' => [
+                28,
+                36
+              ],
+              'type' => 'Point'
+            },
+            'properties' => {
+              'date' => '2016-06-21T06:09:33Z'
+            },
+            'type' => 'Feature'
+          }
+        )
+      end
+    end
   end
 end