From 90bfd13f9511c2be41980603e93b06faf6e90229 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Thu, 5 Sep 2024 23:12:21 +0200 Subject: [PATCH 1/4] Improve GeoJSON import service to work with FeatureCollection --- app/services/geojson/params.rb | 61 ++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/app/services/geojson/params.rb b/app/services/geojson/params.rb index c87ad072..8f5b728e 100644 --- a/app/services/geojson/params.rb +++ b/app/services/geojson/params.rb @@ -8,31 +8,56 @@ class Geojson::Params end def call - json['features'].map do |point| - next if point[:geometry].nil? || point.dig(:properties, :timestamp).nil? - - { - latitude: point[:geometry][:coordinates][1], - longitude: point[:geometry][:coordinates][0], - battery_status: point[:properties][:battery_state], - battery: battery_level(point[:properties][:battery_level]), - timestamp: Time.zone.at(point[:properties][:timestamp]), - altitude: point[:properties][:altitude], - velocity: point[:properties][:speed], - tracker_id: point[:properties][:device_id], - ssid: point[:properties][:wifi], - accuracy: point[:properties][:horizontal_accuracy], - vertical_accuracy: point[:properties][:vertical_accuracy], - raw_data: point - } - end.compact + case json['type'] + when 'Feature' then process_feature(json) + when 'FeatureCollection' then process_feature_collection(json) + end end private + def process_feature(json) + json['features'].map do |point| + next if point[:geometry].nil? || point.dig(:properties, :timestamp).nil? + + build_point(point) + end.compact + end + + def process_feature_collection(json) + json['features'].map { |feature| process_feature(feature) } + end + + def build_point(point) + { + latitude: point[:geometry][:coordinates][1], + longitude: point[:geometry][:coordinates][0], + battery_status: point[:properties][:battery_state], + battery: battery_level(point[:properties][:battery_level]), + timestamp: timestamp(point), + altitude: altitude(point), + velocity: point[:properties][:speed], + tracker_id: point[:properties][:device_id], + ssid: point[:properties][:wifi], + accuracy: point[:properties][:horizontal_accuracy], + vertical_accuracy: point[:properties][:vertical_accuracy], + raw_data: point + } + end + def battery_level(level) value = (level.to_f * 100).to_i value.positive? ? value : nil end + + def altitude(point) + point.dig(:properties, :altitude) || point.dig(:geometry, :coordinates, 2) + end + + def timestamp(point) + value = point.dig(:properties, :timestamp) || point.dig(:geometry, :coordinates, 3) + + Time.zone.at(value) + end end From 3592f4649274fa2f70d0e2238a13365403d3c840 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Fri, 6 Sep 2024 00:07:06 +0200 Subject: [PATCH 2/4] Rework GeoJSON import to support FeatureCollection as a root object and remove points counter for imports --- .app_version | 2 +- CHANGELOG.md | 7 ++ app/jobs/import_job.rb | 58 +-------------- app/models/import.rb | 4 + app/services/geojson/params.rb | 73 +++++++++++++------ .../google_maps/phone_takeout_parser.rb | 13 +--- .../google_maps/semantic_history_parser.rb | 9 --- app/services/gpx/track_parser.rb | 23 ++---- app/services/immich/import_parser.rb | 2 - app/services/imports/create.rb | 66 +++++++++++++++++ app/services/own_tracks/export_parser.rb | 9 --- spec/services/gpx/track_parser_spec.rb | 3 - swagger/v1/swagger.yaml | 2 +- 13 files changed, 139 insertions(+), 132 deletions(-) create mode 100644 app/services/imports/create.rb diff --git a/.app_version b/.app_version index c317a918..9beb74d4 100644 --- a/.app_version +++ b/.app_version @@ -1 +1 @@ -0.13.1 +0.13.2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 22a5a8d8..8b4b7efb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [0.13.2] — 2024-09-06 + +### Fixed + +- GeoJSON import now correctly imports files with FeatureCollection as a root object + + ## [0.13.1] — 2024-09-05 ### Added diff --git a/app/jobs/import_job.rb b/app/jobs/import_job.rb index 5a13d76d..a07cfa46 100644 --- a/app/jobs/import_job.rb +++ b/app/jobs/import_job.rb @@ -7,62 +7,6 @@ class ImportJob < ApplicationJob user = User.find(user_id) import = user.imports.find(import_id) - result = parser(import.source).new(import, user_id).call - - import.update( - raw_points: result[:raw_points], doubles: result[:doubles], processed: result[:processed] - ) - - create_import_finished_notification(import, user) - - schedule_stats_creating(user_id) - schedule_visit_suggesting(user_id, import) - rescue StandardError => e - create_import_failed_notification(import, user, e) - end - - private - - def parser(source) - # Bad classes naming by the way, they are not parsers, they are point creators - case source - when 'google_semantic_history' then GoogleMaps::SemanticHistoryParser - when 'google_records' then GoogleMaps::RecordsParser - when 'google_phone_takeout' then GoogleMaps::PhoneTakeoutParser - when 'owntracks' then OwnTracks::ExportParser - when 'gpx' then Gpx::TrackParser - when 'immich_api' then Immich::ImportParser - when 'geojson' then Geojson::ImportParser - end - end - - def schedule_stats_creating(user_id) - StatCreatingJob.perform_later(user_id) - end - - def schedule_visit_suggesting(user_id, import) - points = import.points.order(:timestamp) - start_at = Time.zone.at(points.first.timestamp) - end_at = Time.zone.at(points.last.timestamp) - - VisitSuggestingJob.perform_later(user_ids: [user_id], start_at:, end_at:) - end - - def create_import_finished_notification(import, user) - Notifications::Create.new( - user:, - kind: :info, - title: 'Import finished', - content: "Import \"#{import.name}\" successfully finished." - ).call - end - - def create_import_failed_notification(import, user, error) - Notifications::Create.new( - user:, - kind: :error, - title: 'Import failed', - content: "Import \"#{import.name}\" failed: #{error.message}, stacktrace: #{error.backtrace.join("\n")}" - ).call + import.process! end end diff --git a/app/models/import.rb b/app/models/import.rb index 22da8972..c6e5a8a6 100644 --- a/app/models/import.rb +++ b/app/models/import.rb @@ -12,4 +12,8 @@ class Import < ApplicationRecord google_semantic_history: 0, owntracks: 1, google_records: 2, google_phone_takeout: 3, gpx: 4, immich_api: 5, geojson: 6 } + + def process! + Imports::Create.new(user, self).call + end end diff --git a/app/services/geojson/params.rb b/app/services/geojson/params.rb index 8f5b728e..f8838951 100644 --- a/app/services/geojson/params.rb +++ b/app/services/geojson/params.rb @@ -11,37 +11,63 @@ class Geojson::Params case json['type'] when 'Feature' then process_feature(json) when 'FeatureCollection' then process_feature_collection(json) - end + end.flatten end private def process_feature(json) - json['features'].map do |point| - next if point[:geometry].nil? || point.dig(:properties, :timestamp).nil? - - build_point(point) - end.compact + case json[:geometry][:type] + when 'Point' + build_point(json) + when 'LineString' + build_line(json) + when 'MultiLineString' + build_multi_line(json) + end end def process_feature_collection(json) json['features'].map { |feature| process_feature(feature) } end - def build_point(point) + def build_point(feature) { - latitude: point[:geometry][:coordinates][1], - longitude: point[:geometry][:coordinates][0], - battery_status: point[:properties][:battery_state], - battery: battery_level(point[:properties][:battery_level]), - timestamp: timestamp(point), - altitude: altitude(point), - velocity: point[:properties][:speed], - tracker_id: point[:properties][:device_id], - ssid: point[:properties][:wifi], - accuracy: point[:properties][:horizontal_accuracy], - vertical_accuracy: point[:properties][:vertical_accuracy], - raw_data: point + latitude: feature[:geometry][:coordinates][1], + longitude: feature[:geometry][:coordinates][0], + battery_status: feature[:properties][:battery_state], + battery: battery_level(feature[:properties][:battery_level]), + timestamp: timestamp(feature), + altitude: altitude(feature), + velocity: feature[:properties][:speed], + tracker_id: feature[:properties][:device_id], + ssid: feature[:properties][:wifi], + accuracy: feature[:properties][:horizontal_accuracy], + vertical_accuracy: feature[:properties][:vertical_accuracy], + raw_data: feature + } + end + + def build_line(feature) + feature[:geometry][:coordinates].map do |point| + build_line_point(feature, point) + end + end + + def build_multi_line(feature) + feature[:geometry][:coordinates].map do |line| + line.map do |point| + build_line_point(feature, point) + end + end + end + + def build_line_point(feature, point) + { + latitude: point[1], + longitude: point[0], + timestamp: timestamp(point), + raw_data: point } end @@ -51,13 +77,14 @@ class Geojson::Params value.positive? ? value : nil end - def altitude(point) - point.dig(:properties, :altitude) || point.dig(:geometry, :coordinates, 2) + def altitude(feature) + feature.dig(:properties, :altitude) || feature.dig(:geometry, :coordinates, 2) end - def timestamp(point) - value = point.dig(:properties, :timestamp) || point.dig(:geometry, :coordinates, 3) + def timestamp(feature) + return Time.zone.at(feature[3]) if feature.is_a?(Array) + value = feature.dig(:properties, :timestamp) || feature.dig(:geometry, :coordinates, 3) Time.zone.at(value) end end diff --git a/app/services/google_maps/phone_takeout_parser.rb b/app/services/google_maps/phone_takeout_parser.rb index 3e42d0ed..d3713a4c 100644 --- a/app/services/google_maps/phone_takeout_parser.rb +++ b/app/services/google_maps/phone_takeout_parser.rb @@ -11,8 +11,6 @@ class GoogleMaps::PhoneTakeoutParser def call points_data = parse_json - points = 0 - points_data.compact.each do |point_data| next if Point.exists?( timestamp: point_data[:timestamp], @@ -34,14 +32,7 @@ class GoogleMaps::PhoneTakeoutParser tracker_id: 'google-maps-phone-timeline-export', user_id: ) - - points += 1 end - - doubles = points_data.size - points - processed = points + doubles - - { raw_points: points_data.size, points:, doubles:, processed: } end private @@ -58,7 +49,9 @@ class GoogleMaps::PhoneTakeoutParser if import.raw_data.is_a?(Array) raw_array = parse_raw_array(import.raw_data) else - semantic_segments = parse_semantic_segments(import.raw_data['semanticSegments']) if import.raw_data['semanticSegments'] + if import.raw_data['semanticSegments'] + semantic_segments = parse_semantic_segments(import.raw_data['semanticSegments']) + end raw_signals = parse_raw_signals(import.raw_data['rawSignals']) if import.raw_data['rawSignals'] end diff --git a/app/services/google_maps/semantic_history_parser.rb b/app/services/google_maps/semantic_history_parser.rb index b4e0901b..4231c965 100644 --- a/app/services/google_maps/semantic_history_parser.rb +++ b/app/services/google_maps/semantic_history_parser.rb @@ -11,8 +11,6 @@ class GoogleMaps::SemanticHistoryParser def call points_data = parse_json - points = 0 - points_data.each do |point_data| next if Point.exists?( timestamp: point_data[:timestamp], @@ -31,14 +29,7 @@ class GoogleMaps::SemanticHistoryParser import_id: import.id, user_id: ) - - points += 1 end - - doubles = points_data.size - points - processed = points + doubles - - { raw_points: points_data.size, points:, doubles:, processed: } end private diff --git a/app/services/gpx/track_parser.rb b/app/services/gpx/track_parser.rb index db7b8bb3..3fde5a4b 100644 --- a/app/services/gpx/track_parser.rb +++ b/app/services/gpx/track_parser.rb @@ -13,32 +13,23 @@ class Gpx::TrackParser tracks = json['gpx']['trk'] tracks_arr = tracks.is_a?(Array) ? tracks : [tracks] - tracks_arr - .map { parse_track(_1) } - .flatten - .reduce { |result, points| result.merge(points) { _2 + _3 } } + tracks_arr.map { parse_track(_1) }.flatten end private def parse_track(track) segments = track['trkseg'] - segments_arr = segments.is_a?(Array) ? segments : [segments] + segments_array = segments.is_a?(Array) ? segments : [segments] - segments_arr.map do |segment| - trackpoints = segment['trkpt'] - - points = trackpoints.reduce(0) { _1 + create_point(_2) } - doubles = trackpoints.size - points - processed = points + doubles - - { raw_points: trackpoints.size, points:, doubles:, processed: } + segments_array.map do |segment| + segment['trkpt'].each { create_point(_1) } end end def create_point(point) - return 0 if point['lat'].blank? || point['lon'].blank? || point['time'].blank? - return 0 if point_exists?(point) + return if point['lat'].blank? || point['lon'].blank? || point['time'].blank? + return if point_exists?(point) Point.create( latitude: point['lat'].to_d, @@ -49,8 +40,6 @@ class Gpx::TrackParser raw_data: point, user_id: ) - - 1 end def point_exists?(point) diff --git a/app/services/immich/import_parser.rb b/app/services/immich/import_parser.rb index 2ef62cb9..8037d7a4 100644 --- a/app/services/immich/import_parser.rb +++ b/app/services/immich/import_parser.rb @@ -11,8 +11,6 @@ class Immich::ImportParser def call json.each { |point| create_point(point) } - - { raw_points: 0, points: 0, doubles: 0, processed: 0 } end def create_point(point) diff --git a/app/services/imports/create.rb b/app/services/imports/create.rb new file mode 100644 index 00000000..1be0d098 --- /dev/null +++ b/app/services/imports/create.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +class Imports::Create + attr_reader :user, :import + + def initialize(user, import) + @user = user + @import = import + end + + def call + parser(import.source).new(import, user.id).call + + create_import_finished_notification(import, user) + + schedule_stats_creating(user.id) + schedule_visit_suggesting(user.id, import) + rescue StandardError => e + create_import_failed_notification(import, user, e) + end + + private + + def parser(source) + # Bad classes naming by the way, they are not parsers, they are point creators + case source + when 'google_semantic_history' then GoogleMaps::SemanticHistoryParser + when 'google_records' then GoogleMaps::RecordsParser + when 'google_phone_takeout' then GoogleMaps::PhoneTakeoutParser + when 'owntracks' then OwnTracks::ExportParser + when 'gpx' then Gpx::TrackParser + when 'immich_api' then Immich::ImportParser + when 'geojson' then Geojson::ImportParser + end + end + + def schedule_stats_creating(user_id) + StatCreatingJob.perform_later(user_id) + end + + def schedule_visit_suggesting(user_id, import) + points = import.points.order(:timestamp) + start_at = Time.zone.at(points.first.timestamp) + end_at = Time.zone.at(points.last.timestamp) + + VisitSuggestingJob.perform_later(user_ids: [user_id], start_at:, end_at:) + end + + def create_import_finished_notification(import, user) + Notifications::Create.new( + user:, + kind: :info, + title: 'Import finished', + content: "Import \"#{import.name}\" successfully finished." + ).call + end + + def create_import_failed_notification(import, user, error) + Notifications::Create.new( + user:, + kind: :error, + title: 'Import failed', + content: "Import \"#{import.name}\" failed: #{error.message}, stacktrace: #{error.backtrace.join("\n")}" + ).call + end +end diff --git a/app/services/own_tracks/export_parser.rb b/app/services/own_tracks/export_parser.rb index c64fa179..c93a7c81 100644 --- a/app/services/own_tracks/export_parser.rb +++ b/app/services/own_tracks/export_parser.rb @@ -12,8 +12,6 @@ class OwnTracks::ExportParser def call points_data = parse_json - points = 0 - points_data.each do |point_data| next if Point.exists?( timestamp: point_data[:timestamp], @@ -28,14 +26,7 @@ class OwnTracks::ExportParser end point.save - - points += 1 end - - doubles = points_data.size - points - processed = points + doubles - - { raw_points: points_data.size, points:, doubles:, processed: } end private diff --git a/spec/services/gpx/track_parser_spec.rb b/spec/services/gpx/track_parser_spec.rb index d4011deb..b9282f84 100644 --- a/spec/services/gpx/track_parser_spec.rb +++ b/spec/services/gpx/track_parser_spec.rb @@ -15,7 +15,6 @@ RSpec.describe Gpx::TrackParser do context 'when file has a single segment' do it 'creates points' do expect { parser }.to change { Point.count }.by(301) - expect(parser).to eq({ doubles: 4, points: 301, processed: 305, raw_points: 305 }) end end @@ -24,7 +23,6 @@ RSpec.describe Gpx::TrackParser do it 'creates points' do expect { parser }.to change { Point.count }.by(558) - expect(parser).to eq({ doubles: 0, points: 558, processed: 558, raw_points: 558 }) end end end @@ -34,7 +32,6 @@ RSpec.describe Gpx::TrackParser do it 'creates points' do expect { parser }.to change { Point.count }.by(407) - expect(parser).to eq({ doubles: 0, points: 407, processed: 407, raw_points: 407 }) end end end diff --git a/swagger/v1/swagger.yaml b/swagger/v1/swagger.yaml index aff1d958..3b175419 100644 --- a/swagger/v1/swagger.yaml +++ b/swagger/v1/swagger.yaml @@ -113,7 +113,7 @@ paths: - Health responses: '200': - description: areas found + description: Healthy "/api/v1/overland/batches": post: summary: Creates a batch of points From f81e1192834031b5f0a9ef22460c740df5dd55a5 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Fri, 6 Sep 2024 00:19:53 +0200 Subject: [PATCH 3/4] Display number of points found on the Points page --- app/controllers/points_controller.rb | 2 ++ app/views/points/index.html.erb | 3 +++ 2 files changed, 5 insertions(+) diff --git a/app/controllers/points_controller.rb b/app/controllers/points_controller.rb index 7043a48a..20aa37bc 100644 --- a/app/controllers/points_controller.rb +++ b/app/controllers/points_controller.rb @@ -17,6 +17,8 @@ class PointsController < ApplicationController @start_at = Time.zone.at(start_at) @end_at = Time.zone.at(end_at) + + @points_number = @points.except(:limit, :offset).size end def bulk_destroy diff --git a/app/views/points/index.html.erb b/app/views/points/index.html.erb index 2f57e6bd..f83be405 100644 --- a/app/views/points/index.html.erb +++ b/app/views/points/index.html.erb @@ -43,6 +43,9 @@
<%= f.submit "Delete Selected", class: "px-4 py-2 bg-red-500 text-white rounded-md", data: { confirm: "Are you sure?", turbo_confirm: "Are you sure?" } %> +
+ <%= @points_number %> points found +
Order by: <%= link_to 'Newest', points_path(order_by: :desc), class: 'btn btn-xs btn-primary mx-1' %> From f2ea6c5ffd079a47d9e15ac5ac0be032f71f5b84 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Fri, 6 Sep 2024 00:20:42 +0200 Subject: [PATCH 4/4] Update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b4b7efb..2f63d60d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - GeoJSON import now correctly imports files with FeatureCollection as a root object +### Changed + +- The Points page now have number of points found for provided date range ## [0.13.1] — 2024-09-05