Rework GeoJSON import to support FeatureCollection as a root object and remove points counter for imports

.app_version

@@ -1 +1 @@
-0.13.1
+0.13.2

CHANGELOG.md

@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [0.13.2] — 2024-09-06
+
+### Fixed
+
+- GeoJSON import now correctly imports files with FeatureCollection as a root object
+
+
 ## [0.13.1] — 2024-09-05
 
 ### Added

app/jobs/import_job.rb

@@ -7,62 +7,6 @@ class ImportJob < ApplicationJob
     user = User.find(user_id)
     import = user.imports.find(import_id)
 
-    result = parser(import.source).new(import, user_id).call
-
-    import.update(
-      raw_points: result[:raw_points], doubles: result[:doubles], processed: result[:processed]
-    )
-
-    create_import_finished_notification(import, user)
-
-    schedule_stats_creating(user_id)
-    schedule_visit_suggesting(user_id, import)
-  rescue StandardError => e
-    create_import_failed_notification(import, user, e)
-  end
-
-  private
-
-  def parser(source)
-    # Bad classes naming by the way, they are not parsers, they are point creators
-    case source
-    when 'google_semantic_history'  then GoogleMaps::SemanticHistoryParser
-    when 'google_records'           then GoogleMaps::RecordsParser
-    when 'google_phone_takeout'     then GoogleMaps::PhoneTakeoutParser
-    when 'owntracks'                then OwnTracks::ExportParser
-    when 'gpx'                      then Gpx::TrackParser
-    when 'immich_api'               then Immich::ImportParser
-    when 'geojson'                  then Geojson::ImportParser
-    end
-  end
-
-  def schedule_stats_creating(user_id)
-    StatCreatingJob.perform_later(user_id)
-  end
-
-  def schedule_visit_suggesting(user_id, import)
-    points = import.points.order(:timestamp)
-    start_at = Time.zone.at(points.first.timestamp)
-    end_at = Time.zone.at(points.last.timestamp)
-
-    VisitSuggestingJob.perform_later(user_ids: [user_id], start_at:, end_at:)
-  end
-
-  def create_import_finished_notification(import, user)
-    Notifications::Create.new(
-      user:,
-      kind: :info,
-      title: 'Import finished',
-      content: "Import \"#{import.name}\" successfully finished."
-    ).call
-  end
-
-  def create_import_failed_notification(import, user, error)
-    Notifications::Create.new(
-      user:,
-      kind: :error,
-      title: 'Import failed',
-      content: "Import \"#{import.name}\" failed: #{error.message}, stacktrace: #{error.backtrace.join("\n")}"
-    ).call
+    import.process!
   end
 end

app/models/import.rb

@@ -12,4 +12,8 @@ class Import < ApplicationRecord
     google_semantic_history: 0, owntracks: 1, google_records: 2,
     google_phone_takeout: 3, gpx: 4, immich_api: 5, geojson: 6
   }
+
+  def process!
+    Imports::Create.new(user, self).call
+  end
 end

app/services/geojson/params.rb

@@ -11,37 +11,63 @@ class Geojson::Params
     case json['type']
     when 'Feature' then process_feature(json)
     when 'FeatureCollection' then process_feature_collection(json)
-    end
+    end.flatten
   end
 
   private
 
   def process_feature(json)
-    json['features'].map do |point|
-      next if point[:geometry].nil? || point.dig(:properties, :timestamp).nil?
-
-      build_point(point)
-    end.compact
+    case json[:geometry][:type]
+    when 'Point'
+      build_point(json)
+    when 'LineString'
+      build_line(json)
+    when 'MultiLineString'
+      build_multi_line(json)
+    end
   end
 
   def process_feature_collection(json)
     json['features'].map { |feature| process_feature(feature) }
   end
 
-  def build_point(point)
+  def build_point(feature)
     {
-      latitude:           point[:geometry][:coordinates][1],
-      longitude:          point[:geometry][:coordinates][0],
-      battery_status:     point[:properties][:battery_state],
-      battery:            battery_level(point[:properties][:battery_level]),
-      timestamp:          timestamp(point),
-      altitude:           altitude(point),
-      velocity:           point[:properties][:speed],
-      tracker_id:         point[:properties][:device_id],
-      ssid:               point[:properties][:wifi],
-      accuracy:           point[:properties][:horizontal_accuracy],
-      vertical_accuracy:  point[:properties][:vertical_accuracy],
-      raw_data:           point
+      latitude:           feature[:geometry][:coordinates][1],
+      longitude:          feature[:geometry][:coordinates][0],
+      battery_status:     feature[:properties][:battery_state],
+      battery:            battery_level(feature[:properties][:battery_level]),
+      timestamp:          timestamp(feature),
+      altitude:           altitude(feature),
+      velocity:           feature[:properties][:speed],
+      tracker_id:         feature[:properties][:device_id],
+      ssid:               feature[:properties][:wifi],
+      accuracy:           feature[:properties][:horizontal_accuracy],
+      vertical_accuracy:  feature[:properties][:vertical_accuracy],
+      raw_data:           feature
+    }
+  end
+
+  def build_line(feature)
+    feature[:geometry][:coordinates].map do |point|
+      build_line_point(feature, point)
+    end
+  end
+
+  def build_multi_line(feature)
+    feature[:geometry][:coordinates].map do |line|
+      line.map do |point|
+        build_line_point(feature, point)
+      end
+    end
+  end
+
+  def build_line_point(feature, point)
+    {
+      latitude:  point[1],
+      longitude: point[0],
+      timestamp: timestamp(point),
+      raw_data:  point
     }
   end
 
@@ -51,13 +77,14 @@ class Geojson::Params
     value.positive? ? value : nil
   end
 
-  def altitude(point)
-    point.dig(:properties, :altitude) || point.dig(:geometry, :coordinates, 2)
+  def altitude(feature)
+    feature.dig(:properties, :altitude) || feature.dig(:geometry, :coordinates, 2)
   end
 
-  def timestamp(point)
-    value = point.dig(:properties, :timestamp) || point.dig(:geometry, :coordinates, 3)
+  def timestamp(feature)
+    return Time.zone.at(feature[3]) if feature.is_a?(Array)
 
+    value = feature.dig(:properties, :timestamp) || feature.dig(:geometry, :coordinates, 3)
     Time.zone.at(value)
   end
 end

app/services/google_maps/phone_takeout_parser.rb

@@ -11,8 +11,6 @@ class GoogleMaps::PhoneTakeoutParser
   def call
     points_data = parse_json
 
-    points = 0
-
     points_data.compact.each do |point_data|
       next if Point.exists?(
         timestamp:  point_data[:timestamp],
@@ -34,14 +32,7 @@ class GoogleMaps::PhoneTakeoutParser
         tracker_id: 'google-maps-phone-timeline-export',
         user_id:
       )
-
-      points += 1
     end
-
-    doubles = points_data.size - points
-    processed = points + doubles
-
-    { raw_points: points_data.size, points:, doubles:, processed: }
   end
 
   private
@@ -58,7 +49,9 @@ class GoogleMaps::PhoneTakeoutParser
     if import.raw_data.is_a?(Array)
       raw_array = parse_raw_array(import.raw_data)
     else
-      semantic_segments = parse_semantic_segments(import.raw_data['semanticSegments']) if import.raw_data['semanticSegments']
+      if import.raw_data['semanticSegments']
+        semantic_segments = parse_semantic_segments(import.raw_data['semanticSegments'])
+      end
       raw_signals = parse_raw_signals(import.raw_data['rawSignals']) if import.raw_data['rawSignals']
     end
 

app/services/google_maps/semantic_history_parser.rb

@@ -11,8 +11,6 @@ class GoogleMaps::SemanticHistoryParser
   def call
     points_data = parse_json
 
-    points = 0
-
     points_data.each do |point_data|
       next if Point.exists?(
         timestamp: point_data[:timestamp],
@@ -31,14 +29,7 @@ class GoogleMaps::SemanticHistoryParser
         import_id: import.id,
         user_id:
       )
-
-      points += 1
     end
-
-    doubles = points_data.size - points
-    processed = points + doubles
-
-    { raw_points: points_data.size, points:, doubles:, processed: }
   end
 
   private

app/services/gpx/track_parser.rb

@@ -13,32 +13,23 @@ class Gpx::TrackParser
     tracks = json['gpx']['trk']
     tracks_arr = tracks.is_a?(Array) ? tracks : [tracks]
 
-    tracks_arr
-      .map { parse_track(_1) }
-      .flatten
-      .reduce { |result, points| result.merge(points) { _2 + _3 } }
+    tracks_arr.map { parse_track(_1) }.flatten
   end
 
   private
 
   def parse_track(track)
     segments = track['trkseg']
-    segments_arr = segments.is_a?(Array) ? segments : [segments]
+    segments_array = segments.is_a?(Array) ? segments : [segments]
 
-    segments_arr.map do |segment|
-      trackpoints = segment['trkpt']
-
-      points = trackpoints.reduce(0) { _1 + create_point(_2) }
-      doubles = trackpoints.size - points
-      processed = points + doubles
-
-      { raw_points: trackpoints.size, points:, doubles:, processed: }
+    segments_array.map do |segment|
+      segment['trkpt'].each { create_point(_1) }
     end
   end
 
   def create_point(point)
-    return 0 if point['lat'].blank? || point['lon'].blank? || point['time'].blank?
-    return 0 if point_exists?(point)
+    return if point['lat'].blank? || point['lon'].blank? || point['time'].blank?
+    return if point_exists?(point)
 
     Point.create(
       latitude:   point['lat'].to_d,
@@ -49,8 +40,6 @@ class Gpx::TrackParser
       raw_data: point,
       user_id:
     )
-
-    1
   end
 
   def point_exists?(point)

app/services/immich/import_parser.rb

@@ -11,8 +11,6 @@ class Immich::ImportParser
 
   def call
     json.each { |point| create_point(point) }
-
-    { raw_points: 0, points: 0, doubles: 0, processed: 0 }
   end
 
   def create_point(point)

app/services/imports/create.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+class Imports::Create
+  attr_reader :user, :import
+
+  def initialize(user, import)
+    @user = user
+    @import = import
+  end
+
+  def call
+    parser(import.source).new(import, user.id).call
+
+    create_import_finished_notification(import, user)
+
+    schedule_stats_creating(user.id)
+    schedule_visit_suggesting(user.id, import)
+  rescue StandardError => e
+    create_import_failed_notification(import, user, e)
+  end
+
+  private
+
+  def parser(source)
+    # Bad classes naming by the way, they are not parsers, they are point creators
+    case source
+    when 'google_semantic_history'  then GoogleMaps::SemanticHistoryParser
+    when 'google_records'           then GoogleMaps::RecordsParser
+    when 'google_phone_takeout'     then GoogleMaps::PhoneTakeoutParser
+    when 'owntracks'                then OwnTracks::ExportParser
+    when 'gpx'                      then Gpx::TrackParser
+    when 'immich_api'               then Immich::ImportParser
+    when 'geojson'                  then Geojson::ImportParser
+    end
+  end
+
+  def schedule_stats_creating(user_id)
+    StatCreatingJob.perform_later(user_id)
+  end
+
+  def schedule_visit_suggesting(user_id, import)
+    points = import.points.order(:timestamp)
+    start_at = Time.zone.at(points.first.timestamp)
+    end_at = Time.zone.at(points.last.timestamp)
+
+    VisitSuggestingJob.perform_later(user_ids: [user_id], start_at:, end_at:)
+  end
+
+  def create_import_finished_notification(import, user)
+    Notifications::Create.new(
+      user:,
+      kind: :info,
+      title: 'Import finished',
+      content: "Import \"#{import.name}\" successfully finished."
+    ).call
+  end
+
+  def create_import_failed_notification(import, user, error)
+    Notifications::Create.new(
+      user:,
+      kind: :error,
+      title: 'Import failed',
+      content: "Import \"#{import.name}\" failed: #{error.message}, stacktrace: #{error.backtrace.join("\n")}"
+    ).call
+  end
+end

app/services/own_tracks/export_parser.rb

@@ -12,8 +12,6 @@ class OwnTracks::ExportParser
   def call
     points_data = parse_json
 
-    points = 0
-
     points_data.each do |point_data|
       next if Point.exists?(
         timestamp: point_data[:timestamp],
@@ -28,14 +26,7 @@ class OwnTracks::ExportParser
       end
 
       point.save
-
-      points += 1
     end
-
-    doubles = points_data.size - points
-    processed = points + doubles
-
-    { raw_points: points_data.size, points:, doubles:, processed: }
   end
 
   private

spec/services/gpx/track_parser_spec.rb

@@ -15,7 +15,6 @@ RSpec.describe Gpx::TrackParser do
       context 'when file has a single segment' do
         it 'creates points' do
           expect { parser }.to change { Point.count }.by(301)
-          expect(parser).to eq({ doubles: 4, points: 301, processed: 305, raw_points: 305 })
         end
       end
 
@@ -24,7 +23,6 @@ RSpec.describe Gpx::TrackParser do
 
         it 'creates points' do
           expect { parser }.to change { Point.count }.by(558)
-          expect(parser).to eq({ doubles: 0, points: 558, processed: 558, raw_points: 558 })
         end
       end
     end
@@ -34,7 +32,6 @@ RSpec.describe Gpx::TrackParser do
 
       it 'creates points' do
         expect { parser }.to change { Point.count }.by(407)
-        expect(parser).to eq({ doubles: 0, points: 407, processed: 407, raw_points: 407 })
       end
     end
   end

swagger/v1/swagger.yaml

@@ -113,7 +113,7 @@ paths:
       - Health
       responses:
         '200':
-          description: areas found
+          description: Healthy
   "/api/v1/overland/batches":
     post:
       summary: Creates a batch of points