diff --git a/.app_version b/.app_version index c317a918..9beb74d4 100644 --- a/.app_version +++ b/.app_version @@ -1 +1 @@ -0.13.1 +0.13.2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 22a5a8d8..2f63d60d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [0.13.2] — 2024-09-06 + +### Fixed + +- GeoJSON import now correctly imports files with FeatureCollection as a root object + +### Changed + +- The Points page now have number of points found for provided date range + ## [0.13.1] — 2024-09-05 ### Added diff --git a/app/controllers/points_controller.rb b/app/controllers/points_controller.rb index 7043a48a..20aa37bc 100644 --- a/app/controllers/points_controller.rb +++ b/app/controllers/points_controller.rb @@ -17,6 +17,8 @@ class PointsController < ApplicationController @start_at = Time.zone.at(start_at) @end_at = Time.zone.at(end_at) + + @points_number = @points.except(:limit, :offset).size end def bulk_destroy diff --git a/app/jobs/import_job.rb b/app/jobs/import_job.rb index 5a13d76d..a07cfa46 100644 --- a/app/jobs/import_job.rb +++ b/app/jobs/import_job.rb @@ -7,62 +7,6 @@ class ImportJob < ApplicationJob user = User.find(user_id) import = user.imports.find(import_id) - result = parser(import.source).new(import, user_id).call - - import.update( - raw_points: result[:raw_points], doubles: result[:doubles], processed: result[:processed] - ) - - create_import_finished_notification(import, user) - - schedule_stats_creating(user_id) - schedule_visit_suggesting(user_id, import) - rescue StandardError => e - create_import_failed_notification(import, user, e) - end - - private - - def parser(source) - # Bad classes naming by the way, they are not parsers, they are point creators - case source - when 'google_semantic_history' then GoogleMaps::SemanticHistoryParser - when 'google_records' then GoogleMaps::RecordsParser - when 'google_phone_takeout' then GoogleMaps::PhoneTakeoutParser - when 'owntracks' then OwnTracks::ExportParser - when 'gpx' then Gpx::TrackParser - when 'immich_api' then Immich::ImportParser - when 'geojson' then Geojson::ImportParser - end - end - - def schedule_stats_creating(user_id) - StatCreatingJob.perform_later(user_id) - end - - def schedule_visit_suggesting(user_id, import) - points = import.points.order(:timestamp) - start_at = Time.zone.at(points.first.timestamp) - end_at = Time.zone.at(points.last.timestamp) - - VisitSuggestingJob.perform_later(user_ids: [user_id], start_at:, end_at:) - end - - def create_import_finished_notification(import, user) - Notifications::Create.new( - user:, - kind: :info, - title: 'Import finished', - content: "Import \"#{import.name}\" successfully finished." - ).call - end - - def create_import_failed_notification(import, user, error) - Notifications::Create.new( - user:, - kind: :error, - title: 'Import failed', - content: "Import \"#{import.name}\" failed: #{error.message}, stacktrace: #{error.backtrace.join("\n")}" - ).call + import.process! end end diff --git a/app/models/import.rb b/app/models/import.rb index 22da8972..c6e5a8a6 100644 --- a/app/models/import.rb +++ b/app/models/import.rb @@ -12,4 +12,8 @@ class Import < ApplicationRecord google_semantic_history: 0, owntracks: 1, google_records: 2, google_phone_takeout: 3, gpx: 4, immich_api: 5, geojson: 6 } + + def process! + Imports::Create.new(user, self).call + end end diff --git a/app/services/geojson/params.rb b/app/services/geojson/params.rb index c87ad072..f8838951 100644 --- a/app/services/geojson/params.rb +++ b/app/services/geojson/params.rb @@ -8,31 +8,83 @@ class Geojson::Params end def call - json['features'].map do |point| - next if point[:geometry].nil? || point.dig(:properties, :timestamp).nil? - - { - latitude: point[:geometry][:coordinates][1], - longitude: point[:geometry][:coordinates][0], - battery_status: point[:properties][:battery_state], - battery: battery_level(point[:properties][:battery_level]), - timestamp: Time.zone.at(point[:properties][:timestamp]), - altitude: point[:properties][:altitude], - velocity: point[:properties][:speed], - tracker_id: point[:properties][:device_id], - ssid: point[:properties][:wifi], - accuracy: point[:properties][:horizontal_accuracy], - vertical_accuracy: point[:properties][:vertical_accuracy], - raw_data: point - } - end.compact + case json['type'] + when 'Feature' then process_feature(json) + when 'FeatureCollection' then process_feature_collection(json) + end.flatten end private + def process_feature(json) + case json[:geometry][:type] + when 'Point' + build_point(json) + when 'LineString' + build_line(json) + when 'MultiLineString' + build_multi_line(json) + end + end + + def process_feature_collection(json) + json['features'].map { |feature| process_feature(feature) } + end + + def build_point(feature) + { + latitude: feature[:geometry][:coordinates][1], + longitude: feature[:geometry][:coordinates][0], + battery_status: feature[:properties][:battery_state], + battery: battery_level(feature[:properties][:battery_level]), + timestamp: timestamp(feature), + altitude: altitude(feature), + velocity: feature[:properties][:speed], + tracker_id: feature[:properties][:device_id], + ssid: feature[:properties][:wifi], + accuracy: feature[:properties][:horizontal_accuracy], + vertical_accuracy: feature[:properties][:vertical_accuracy], + raw_data: feature + } + end + + def build_line(feature) + feature[:geometry][:coordinates].map do |point| + build_line_point(feature, point) + end + end + + def build_multi_line(feature) + feature[:geometry][:coordinates].map do |line| + line.map do |point| + build_line_point(feature, point) + end + end + end + + def build_line_point(feature, point) + { + latitude: point[1], + longitude: point[0], + timestamp: timestamp(point), + raw_data: point + } + end + def battery_level(level) value = (level.to_f * 100).to_i value.positive? ? value : nil end + + def altitude(feature) + feature.dig(:properties, :altitude) || feature.dig(:geometry, :coordinates, 2) + end + + def timestamp(feature) + return Time.zone.at(feature[3]) if feature.is_a?(Array) + + value = feature.dig(:properties, :timestamp) || feature.dig(:geometry, :coordinates, 3) + Time.zone.at(value) + end end diff --git a/app/services/google_maps/phone_takeout_parser.rb b/app/services/google_maps/phone_takeout_parser.rb index 3e42d0ed..d3713a4c 100644 --- a/app/services/google_maps/phone_takeout_parser.rb +++ b/app/services/google_maps/phone_takeout_parser.rb @@ -11,8 +11,6 @@ class GoogleMaps::PhoneTakeoutParser def call points_data = parse_json - points = 0 - points_data.compact.each do |point_data| next if Point.exists?( timestamp: point_data[:timestamp], @@ -34,14 +32,7 @@ class GoogleMaps::PhoneTakeoutParser tracker_id: 'google-maps-phone-timeline-export', user_id: ) - - points += 1 end - - doubles = points_data.size - points - processed = points + doubles - - { raw_points: points_data.size, points:, doubles:, processed: } end private @@ -58,7 +49,9 @@ class GoogleMaps::PhoneTakeoutParser if import.raw_data.is_a?(Array) raw_array = parse_raw_array(import.raw_data) else - semantic_segments = parse_semantic_segments(import.raw_data['semanticSegments']) if import.raw_data['semanticSegments'] + if import.raw_data['semanticSegments'] + semantic_segments = parse_semantic_segments(import.raw_data['semanticSegments']) + end raw_signals = parse_raw_signals(import.raw_data['rawSignals']) if import.raw_data['rawSignals'] end diff --git a/app/services/google_maps/semantic_history_parser.rb b/app/services/google_maps/semantic_history_parser.rb index b4e0901b..4231c965 100644 --- a/app/services/google_maps/semantic_history_parser.rb +++ b/app/services/google_maps/semantic_history_parser.rb @@ -11,8 +11,6 @@ class GoogleMaps::SemanticHistoryParser def call points_data = parse_json - points = 0 - points_data.each do |point_data| next if Point.exists?( timestamp: point_data[:timestamp], @@ -31,14 +29,7 @@ class GoogleMaps::SemanticHistoryParser import_id: import.id, user_id: ) - - points += 1 end - - doubles = points_data.size - points - processed = points + doubles - - { raw_points: points_data.size, points:, doubles:, processed: } end private diff --git a/app/services/gpx/track_parser.rb b/app/services/gpx/track_parser.rb index db7b8bb3..3fde5a4b 100644 --- a/app/services/gpx/track_parser.rb +++ b/app/services/gpx/track_parser.rb @@ -13,32 +13,23 @@ class Gpx::TrackParser tracks = json['gpx']['trk'] tracks_arr = tracks.is_a?(Array) ? tracks : [tracks] - tracks_arr - .map { parse_track(_1) } - .flatten - .reduce { |result, points| result.merge(points) { _2 + _3 } } + tracks_arr.map { parse_track(_1) }.flatten end private def parse_track(track) segments = track['trkseg'] - segments_arr = segments.is_a?(Array) ? segments : [segments] + segments_array = segments.is_a?(Array) ? segments : [segments] - segments_arr.map do |segment| - trackpoints = segment['trkpt'] - - points = trackpoints.reduce(0) { _1 + create_point(_2) } - doubles = trackpoints.size - points - processed = points + doubles - - { raw_points: trackpoints.size, points:, doubles:, processed: } + segments_array.map do |segment| + segment['trkpt'].each { create_point(_1) } end end def create_point(point) - return 0 if point['lat'].blank? || point['lon'].blank? || point['time'].blank? - return 0 if point_exists?(point) + return if point['lat'].blank? || point['lon'].blank? || point['time'].blank? + return if point_exists?(point) Point.create( latitude: point['lat'].to_d, @@ -49,8 +40,6 @@ class Gpx::TrackParser raw_data: point, user_id: ) - - 1 end def point_exists?(point) diff --git a/app/services/immich/import_parser.rb b/app/services/immich/import_parser.rb index 2ef62cb9..8037d7a4 100644 --- a/app/services/immich/import_parser.rb +++ b/app/services/immich/import_parser.rb @@ -11,8 +11,6 @@ class Immich::ImportParser def call json.each { |point| create_point(point) } - - { raw_points: 0, points: 0, doubles: 0, processed: 0 } end def create_point(point) diff --git a/app/services/imports/create.rb b/app/services/imports/create.rb new file mode 100644 index 00000000..1be0d098 --- /dev/null +++ b/app/services/imports/create.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +class Imports::Create + attr_reader :user, :import + + def initialize(user, import) + @user = user + @import = import + end + + def call + parser(import.source).new(import, user.id).call + + create_import_finished_notification(import, user) + + schedule_stats_creating(user.id) + schedule_visit_suggesting(user.id, import) + rescue StandardError => e + create_import_failed_notification(import, user, e) + end + + private + + def parser(source) + # Bad classes naming by the way, they are not parsers, they are point creators + case source + when 'google_semantic_history' then GoogleMaps::SemanticHistoryParser + when 'google_records' then GoogleMaps::RecordsParser + when 'google_phone_takeout' then GoogleMaps::PhoneTakeoutParser + when 'owntracks' then OwnTracks::ExportParser + when 'gpx' then Gpx::TrackParser + when 'immich_api' then Immich::ImportParser + when 'geojson' then Geojson::ImportParser + end + end + + def schedule_stats_creating(user_id) + StatCreatingJob.perform_later(user_id) + end + + def schedule_visit_suggesting(user_id, import) + points = import.points.order(:timestamp) + start_at = Time.zone.at(points.first.timestamp) + end_at = Time.zone.at(points.last.timestamp) + + VisitSuggestingJob.perform_later(user_ids: [user_id], start_at:, end_at:) + end + + def create_import_finished_notification(import, user) + Notifications::Create.new( + user:, + kind: :info, + title: 'Import finished', + content: "Import \"#{import.name}\" successfully finished." + ).call + end + + def create_import_failed_notification(import, user, error) + Notifications::Create.new( + user:, + kind: :error, + title: 'Import failed', + content: "Import \"#{import.name}\" failed: #{error.message}, stacktrace: #{error.backtrace.join("\n")}" + ).call + end +end diff --git a/app/services/own_tracks/export_parser.rb b/app/services/own_tracks/export_parser.rb index c64fa179..c93a7c81 100644 --- a/app/services/own_tracks/export_parser.rb +++ b/app/services/own_tracks/export_parser.rb @@ -12,8 +12,6 @@ class OwnTracks::ExportParser def call points_data = parse_json - points = 0 - points_data.each do |point_data| next if Point.exists?( timestamp: point_data[:timestamp], @@ -28,14 +26,7 @@ class OwnTracks::ExportParser end point.save - - points += 1 end - - doubles = points_data.size - points - processed = points + doubles - - { raw_points: points_data.size, points:, doubles:, processed: } end private diff --git a/app/views/points/index.html.erb b/app/views/points/index.html.erb index 2f57e6bd..f83be405 100644 --- a/app/views/points/index.html.erb +++ b/app/views/points/index.html.erb @@ -43,6 +43,9 @@