From 096a7a6ffa2c3c0cd77ddcd86e1df2537aa3dc84 Mon Sep 17 00:00:00 2001 From: Evgenii Burmakin Date: Wed, 14 Jan 2026 00:17:27 +0100 Subject: [PATCH] Support properties->date field for timestamp in GeoJSON imports (#2159) * Support properties->date field for timestamp in GeoJSON imports * Fix GeoJSON date parsing --- CHANGELOG.md | 7 ++++ app/services/cache/clean.rb | 38 +++++++++---------- app/services/geojson/params.rb | 29 +++++++++++--- .../google_maps/phone_takeout_importer.rb | 16 ++++---- app/services/gpx/track_importer.rb | 2 +- app/services/immich/import_geodata.rb | 2 +- app/services/kml/importer.rb | 4 +- app/services/photoprism/import_geodata.rb | 2 +- .../users/import_data/notifications.rb | 4 +- ..._set_points_timestamp_from_geojson_date.rb | 19 ++++++++++ db/schema.rb | 3 +- .../files/geojson/google_takeout_example.json | 15 ++++++++ spec/services/cache/clean_spec.rb | 9 +++-- spec/services/geojson/params_spec.rb | 34 ++++++++++++++++- 14 files changed, 136 insertions(+), 48 deletions(-) create mode 100644 db/migrate/20260113230537_set_points_timestamp_from_geojson_date.rb create mode 100644 spec/fixtures/files/geojson/google_takeout_example.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 0066e76e..dab14730 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +# [0.37.4] - Unreleased + +## Fixed + +- GeoJSON formatted points now have correct timestamp parsed from raw_data['properties']['date'] field. +- Reduce number of iterations during cache cleaning to improve performance. + # [0.37.3] - Unreleased ## Fixed diff --git a/app/services/cache/clean.rb b/app/services/cache/clean.rb index af8354b7..d8a10557 100644 --- a/app/services/cache/clean.rb +++ b/app/services/cache/clean.rb @@ -6,10 +6,14 @@ class Cache::Clean Rails.logger.info('Cleaning cache...') delete_control_flag delete_version_cache - delete_years_tracked_cache - delete_points_geocoded_stats_cache - delete_countries_cities_cache - delete_total_distance_cache + + User.find_each do |user| + delete_years_tracked_cache(user) + delete_points_geocoded_stats_cache(user) + delete_countries_cities_cache(user) + delete_total_distance_cache(user) + end + Rails.logger.info('Cache cleaned') end @@ -23,29 +27,21 @@ class Cache::Clean Rails.cache.delete(CheckAppVersion::VERSION_CACHE_KEY) end - def delete_years_tracked_cache - User.find_each do |user| - Rails.cache.delete("dawarich/user_#{user.id}_years_tracked") - end + def delete_years_tracked_cache(user) + Rails.cache.delete("dawarich/user_#{user.id}_years_tracked") end - def delete_points_geocoded_stats_cache - User.find_each do |user| - Rails.cache.delete("dawarich/user_#{user.id}_points_geocoded_stats") - end + def delete_points_geocoded_stats_cache(user) + Rails.cache.delete("dawarich/user_#{user.id}_points_geocoded_stats") end - def delete_countries_cities_cache - User.find_each do |user| - Rails.cache.delete("dawarich/user_#{user.id}_countries_visited") - Rails.cache.delete("dawarich/user_#{user.id}_cities_visited") - end + def delete_countries_cities_cache(user) + Rails.cache.delete("dawarich/user_#{user.id}_countries_visited") + Rails.cache.delete("dawarich/user_#{user.id}_cities_visited") end - def delete_total_distance_cache - User.find_each do |user| - Rails.cache.delete("dawarich/user_#{user.id}_total_distance") - end + def delete_total_distance_cache(user) + Rails.cache.delete("dawarich/user_#{user.id}_total_distance") end end end diff --git a/app/services/geojson/params.rb b/app/services/geojson/params.rb index 00b9907c..21ab96e4 100644 --- a/app/services/geojson/params.rb +++ b/app/services/geojson/params.rb @@ -80,18 +80,35 @@ class Geojson::Params end def timestamp(feature) - return Time.zone.at(feature[3]) if feature.is_a?(Array) + if feature.is_a?(Array) + return parse_array_timestamp(feature[3]) if feature[3].present? + return nil + end + + numeric_timestamp(feature) || parse_string_timestamp(feature) + end + + def parse_array_timestamp(value) + return value.to_i if value.is_a?(Numeric) + + Time.zone.parse(value.to_s).utc.to_i if value.present? + end + + def numeric_timestamp(feature) value = feature.dig(:properties, :timestamp) || feature.dig(:geometry, :coordinates, 3) - return Time.zone.at(value.to_i) if value.is_a?(Numeric) + value.to_i if value.is_a?(Numeric) + end - ### GPSLogger for Android case ### - time = feature.dig(:properties, :time) + def parse_string_timestamp(feature) + ### GPSLogger for Android / Google Takeout case ### + time = feature.dig(:properties, :time) || + feature.dig(:properties, :date) + ### /GPSLogger for Android / Google Takeout case ### - Time.zone.parse(time).to_i if time.present? - ### /GPSLogger for Android case ### + Time.zone.parse(time).utc.to_i if time.present? end def speed(feature) diff --git a/app/services/google_maps/phone_takeout_importer.rb b/app/services/google_maps/phone_takeout_importer.rb index 4e74bc54..a0afe807 100644 --- a/app/services/google_maps/phone_takeout_importer.rb +++ b/app/services/google_maps/phone_takeout_importer.rb @@ -74,17 +74,17 @@ class GoogleMaps::PhoneTakeoutImporter def parse_visit_place_location(data_point) lat, lon = parse_coordinates(data_point['visit']['topCandidate']['placeLocation']) - timestamp = DateTime.parse(data_point['startTime']).to_i + timestamp = DateTime.parse(data_point['startTime']).utc.to_i point_hash(lat, lon, timestamp, data_point) end def parse_activity(data_point) start_lat, start_lon = parse_coordinates(data_point['activity']['start']) - start_timestamp = DateTime.parse(data_point['startTime']).to_i + start_timestamp = DateTime.parse(data_point['startTime']).utc.to_i end_lat, end_lon = parse_coordinates(data_point['activity']['end']) - end_timestamp = DateTime.parse(data_point['endTime']).to_i + end_timestamp = DateTime.parse(data_point['endTime']).utc.to_i [ point_hash(start_lat, start_lon, start_timestamp, data_point), @@ -107,16 +107,16 @@ class GoogleMaps::PhoneTakeoutImporter def parse_semantic_visit(segment) lat, lon = parse_coordinates(segment['visit']['topCandidate']['placeLocation']['latLng']) - timestamp = DateTime.parse(segment['startTime']).to_i + timestamp = DateTime.parse(segment['startTime']).utc.to_i point_hash(lat, lon, timestamp, segment) end def parse_semantic_activity(segment) start_lat, start_lon = parse_coordinates(segment['activity']['start']['latLng']) - start_timestamp = DateTime.parse(segment['startTime']).to_i + start_timestamp = DateTime.parse(segment['startTime']).utc.to_i end_lat, end_lon = parse_coordinates(segment['activity']['end']['latLng']) - end_timestamp = DateTime.parse(segment['endTime']).to_i + end_timestamp = DateTime.parse(segment['endTime']).utc.to_i [ point_hash(start_lat, start_lon, start_timestamp, segment), @@ -127,7 +127,7 @@ class GoogleMaps::PhoneTakeoutImporter def parse_semantic_timeline_path(segment) segment['timelinePath'].map do |point| lat, lon = parse_coordinates(point['point']) - timestamp = DateTime.parse(point['time']).to_i + timestamp = DateTime.parse(point['time']).utc.to_i point_hash(lat, lon, timestamp, segment) end @@ -165,7 +165,7 @@ class GoogleMaps::PhoneTakeoutImporter next unless segment.dig('position', 'LatLng') lat, lon = parse_coordinates(segment['position']['LatLng']) - timestamp = DateTime.parse(segment['position']['timestamp']).to_i + timestamp = DateTime.parse(segment['position']['timestamp']).utc.to_i point_hash(lat, lon, timestamp, segment) end diff --git a/app/services/gpx/track_importer.rb b/app/services/gpx/track_importer.rb index 2a25cc99..fc6b4646 100644 --- a/app/services/gpx/track_importer.rb +++ b/app/services/gpx/track_importer.rb @@ -44,7 +44,7 @@ class Gpx::TrackImporter { lonlat: "POINT(#{point['lon'].to_d} #{point['lat'].to_d})", altitude: point['ele'].to_i, - timestamp: Time.parse(point['time']).to_i, + timestamp: Time.parse(point['time']).utc.to_i, import_id: import.id, velocity: speed(point), raw_data: point, diff --git a/app/services/immich/import_geodata.rb b/app/services/immich/import_geodata.rb index 9f9679ee..d27a3bee 100644 --- a/app/services/immich/import_geodata.rb +++ b/app/services/immich/import_geodata.rb @@ -56,7 +56,7 @@ class Immich::ImportGeodata latitude: asset['exifInfo']['latitude'], longitude: asset['exifInfo']['longitude'], lonlat: "SRID=4326;POINT(#{asset['exifInfo']['longitude']} #{asset['exifInfo']['latitude']})", - timestamp: Time.zone.parse(asset['exifInfo']['dateTimeOriginal']).to_i + timestamp: Time.zone.parse(asset['exifInfo']['dateTimeOriginal']).utc.to_i } end diff --git a/app/services/kml/importer.rb b/app/services/kml/importer.rb index 39df1683..5bdf8fe1 100644 --- a/app/services/kml/importer.rb +++ b/app/services/kml/importer.rb @@ -192,7 +192,7 @@ class Kml::Importer end def build_gx_track_point(timestamp_str, coord_str, index) - time = Time.parse(timestamp_str).to_i + time = Time.parse(timestamp_str).utc.to_i coord_parts = coord_str.split(/\s+/) return nil if coord_parts.size < 2 @@ -239,7 +239,7 @@ class Kml::Importer node = find_timestamp_node(placemark) raise 'No timestamp found in placemark' unless node - Time.parse(node.text).to_i + Time.parse(node.text).utc.to_i rescue StandardError => e Rails.logger.error("Failed to parse timestamp: #{e.message}") raise e diff --git a/app/services/photoprism/import_geodata.rb b/app/services/photoprism/import_geodata.rb index c31946c1..e2de583b 100644 --- a/app/services/photoprism/import_geodata.rb +++ b/app/services/photoprism/import_geodata.rb @@ -66,7 +66,7 @@ class Photoprism::ImportGeodata latitude: asset['Lat'], longitude: asset['Lng'], lonlat: "SRID=4326;POINT(#{asset['Lng']} #{asset['Lat']})", - timestamp: Time.zone.parse(asset['TakenAt']).to_i + timestamp: Time.zone.parse(asset['TakenAt']).utc.to_i } end diff --git a/app/services/users/import_data/notifications.rb b/app/services/users/import_data/notifications.rb index e485d0aa..e5d80f6a 100644 --- a/app/services/users/import_data/notifications.rb +++ b/app/services/users/import_data/notifications.rb @@ -110,8 +110,8 @@ class Users::ImportData::Notifications def normalize_timestamp(timestamp) case timestamp - when String then Time.parse(timestamp).to_i - when Time, DateTime then timestamp.to_i + when String then Time.parse(timestamp).utc.to_i + when Time, DateTime then timestamp.utc.to_i else timestamp.to_s end diff --git a/db/migrate/20260113230537_set_points_timestamp_from_geojson_date.rb b/db/migrate/20260113230537_set_points_timestamp_from_geojson_date.rb new file mode 100644 index 00000000..aad838dd --- /dev/null +++ b/db/migrate/20260113230537_set_points_timestamp_from_geojson_date.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class SetPointsTimestampFromGeojsonDate < ActiveRecord::Migration[8.0] + def change + Point.where(timestamp: nil).find_each do |point| + geojson = point.raw_data + + next unless geojson && geojson['properties'] && geojson['properties']['date'] + + begin + parsed_time = Time.zone.parse(geojson['properties']['date']).utc.to_i + + point.update!(timestamp: parsed_time) + rescue ArgumentError => e + Rails.logger.warn("Failed to parse date for Point ID #{point.id}: #{e.message}") + end + end + end +end diff --git a/db/schema.rb b/db/schema.rb index d7baaeb4..c21c782c 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2026_01_03_114630) do +ActiveRecord::Schema[8.0].define(version: 2026_01_13_230537) do # These are extensions that must be enabled in order to support this database enable_extension "pg_catalog.plpgsql" enable_extension "postgis" @@ -507,6 +507,7 @@ ActiveRecord::Schema[8.0].define(version: 2026_01_03_114630) do t.index ["area_id"], name: "index_visits_on_area_id" t.index ["place_id"], name: "index_visits_on_place_id" t.index ["started_at"], name: "index_visits_on_started_at" + t.index ["user_id", "status", "started_at"], name: "index_visits_on_user_id_and_status_and_started_at" t.index ["user_id"], name: "index_visits_on_user_id" end diff --git a/spec/fixtures/files/geojson/google_takeout_example.json b/spec/fixtures/files/geojson/google_takeout_example.json new file mode 100644 index 00000000..55589395 --- /dev/null +++ b/spec/fixtures/files/geojson/google_takeout_example.json @@ -0,0 +1,15 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [28, 36] + }, + "properties": { + "date": "2016-06-21T06:09:33Z" + } + } + ] +} diff --git a/spec/services/cache/clean_spec.rb b/spec/services/cache/clean_spec.rb index 7be77a20..5a796480 100644 --- a/spec/services/cache/clean_spec.rb +++ b/spec/services/cache/clean_spec.rb @@ -93,14 +93,15 @@ RSpec.describe Cache::Clean do # Create a user that will be found during the cleaning process user3 = nil - allow(User).to receive(:find_each).and_yield(user1).and_yield(user2) do |&block| + allow(User).to receive(:find_each) do |&block| + # Yield existing users + block.call(user1) + block.call(user2) + # Create a new user while iterating - this should not cause errors user3 = create(:user) Rails.cache.write("dawarich/user_#{user3.id}_years_tracked", { 2023 => ['May'] }) Rails.cache.write("dawarich/user_#{user3.id}_points_geocoded_stats", { geocoded: 1, without_data: 0 }) - - # Continue with the original block - [user1, user2].each(&block) end expect { described_class.call }.not_to raise_error diff --git a/spec/services/geojson/params_spec.rb b/spec/services/geojson/params_spec.rb index 345ddbe3..41c5a7bb 100644 --- a/spec/services/geojson/params_spec.rb +++ b/spec/services/geojson/params_spec.rb @@ -21,7 +21,7 @@ RSpec.describe Geojson::Params do lonlat: 'POINT(0.1 0.1)', battery_status: nil, battery: nil, - timestamp: Time.zone.at(1_609_459_201), + timestamp: 1_609_459_201, altitude: 1, velocity: 1.5, tracker_id: nil, @@ -102,5 +102,37 @@ RSpec.describe Geojson::Params do ) end end + + context 'when the json is exported from Google Takeout' do + let(:file_path) { Rails.root.join('spec/fixtures/files/geojson/google_takeout_example.json') } + + it 'returns the correct data for each point' do + expect(subject.first).to eq( + lonlat: 'POINT(28 36)', + battery_status: nil, + battery: nil, + timestamp: Time.parse('2016-06-21T06:09:33Z').to_i, + altitude: nil, + velocity: 0.0, + tracker_id: nil, + ssid: nil, + accuracy: nil, + vertical_accuracy: nil, + raw_data: { + 'geometry' => { + 'coordinates' => [ + 28, + 36 + ], + 'type' => 'Point' + }, + 'properties' => { + 'date' => '2016-06-21T06:09:33Z' + }, + 'type' => 'Feature' + } + ) + end + end end end