Merge pull request #228 from Freika/fix/geojson-import

GeoJSON import update

.app_version

@@ -1 +1 @@
-0.13.1
+0.13.2

CHANGELOG.md

@@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [0.13.2] — 2024-09-06
+
+### Fixed
+
+- GeoJSON import now correctly imports files with FeatureCollection as a root object
+
+### Changed
+
+- The Points page now have number of points found for provided date range
+
 ## [0.13.1] — 2024-09-05
 
 ### Added

app/controllers/points_controller.rb

@@ -17,6 +17,8 @@ class PointsController < ApplicationController
 
     @start_at = Time.zone.at(start_at)
     @end_at = Time.zone.at(end_at)
+
+    @points_number = @points.except(:limit, :offset).size
   end
 
   def bulk_destroy

app/jobs/import_job.rb

@@ -7,62 +7,6 @@ class ImportJob < ApplicationJob
     user = User.find(user_id)
     import = user.imports.find(import_id)
 
-    result = parser(import.source).new(import, user_id).call
-
-    import.update(
-      raw_points: result[:raw_points], doubles: result[:doubles], processed: result[:processed]
-    )
-
-    create_import_finished_notification(import, user)
-
-    schedule_stats_creating(user_id)
-    schedule_visit_suggesting(user_id, import)
-  rescue StandardError => e
-    create_import_failed_notification(import, user, e)
-  end
-
-  private
-
-  def parser(source)
-    # Bad classes naming by the way, they are not parsers, they are point creators
-    case source
-    when 'google_semantic_history'  then GoogleMaps::SemanticHistoryParser
-    when 'google_records'           then GoogleMaps::RecordsParser
-    when 'google_phone_takeout'     then GoogleMaps::PhoneTakeoutParser
-    when 'owntracks'                then OwnTracks::ExportParser
-    when 'gpx'                      then Gpx::TrackParser
-    when 'immich_api'               then Immich::ImportParser
-    when 'geojson'                  then Geojson::ImportParser
-    end
-  end
-
-  def schedule_stats_creating(user_id)
-    StatCreatingJob.perform_later(user_id)
-  end
-
-  def schedule_visit_suggesting(user_id, import)
-    points = import.points.order(:timestamp)
-    start_at = Time.zone.at(points.first.timestamp)
-    end_at = Time.zone.at(points.last.timestamp)
-
-    VisitSuggestingJob.perform_later(user_ids: [user_id], start_at:, end_at:)
-  end
-
-  def create_import_finished_notification(import, user)
-    Notifications::Create.new(
-      user:,
-      kind: :info,
-      title: 'Import finished',
-      content: "Import \"#{import.name}\" successfully finished."
-    ).call
-  end
-
-  def create_import_failed_notification(import, user, error)
-    Notifications::Create.new(
-      user:,
-      kind: :error,
-      title: 'Import failed',
-      content: "Import \"#{import.name}\" failed: #{error.message}, stacktrace: #{error.backtrace.join("\n")}"
-    ).call
+    import.process!
   end
 end

app/models/import.rb

@@ -12,4 +12,8 @@ class Import < ApplicationRecord
     google_semantic_history: 0, owntracks: 1, google_records: 2,
     google_phone_takeout: 3, gpx: 4, immich_api: 5, geojson: 6
   }
+
+  def process!
+    Imports::Create.new(user, self).call
+  end
 end

app/services/geojson/params.rb

@@ -8,31 +8,83 @@ class Geojson::Params
   end
 
   def call
-    json['features'].map do |point|
-      next if point[:geometry].nil? || point.dig(:properties, :timestamp).nil?
-
-      {
-        latitude:           point[:geometry][:coordinates][1],
-        longitude:          point[:geometry][:coordinates][0],
-        battery_status:     point[:properties][:battery_state],
-        battery:            battery_level(point[:properties][:battery_level]),
-        timestamp:          Time.zone.at(point[:properties][:timestamp]),
-        altitude:           point[:properties][:altitude],
-        velocity:           point[:properties][:speed],
-        tracker_id:         point[:properties][:device_id],
-        ssid:               point[:properties][:wifi],
-        accuracy:           point[:properties][:horizontal_accuracy],
-        vertical_accuracy:  point[:properties][:vertical_accuracy],
-        raw_data:           point
-      }
-    end.compact
+    case json['type']
+    when 'Feature' then process_feature(json)
+    when 'FeatureCollection' then process_feature_collection(json)
+    end.flatten
   end
 
   private
 
+  def process_feature(json)
+    case json[:geometry][:type]
+    when 'Point'
+      build_point(json)
+    when 'LineString'
+      build_line(json)
+    when 'MultiLineString'
+      build_multi_line(json)
+    end
+  end
+
+  def process_feature_collection(json)
+    json['features'].map { |feature| process_feature(feature) }
+  end
+
+  def build_point(feature)
+    {
+      latitude:           feature[:geometry][:coordinates][1],
+      longitude:          feature[:geometry][:coordinates][0],
+      battery_status:     feature[:properties][:battery_state],
+      battery:            battery_level(feature[:properties][:battery_level]),
+      timestamp:          timestamp(feature),
+      altitude:           altitude(feature),
+      velocity:           feature[:properties][:speed],
+      tracker_id:         feature[:properties][:device_id],
+      ssid:               feature[:properties][:wifi],
+      accuracy:           feature[:properties][:horizontal_accuracy],
+      vertical_accuracy:  feature[:properties][:vertical_accuracy],
+      raw_data:           feature
+    }
+  end
+
+  def build_line(feature)
+    feature[:geometry][:coordinates].map do |point|
+      build_line_point(feature, point)
+    end
+  end
+
+  def build_multi_line(feature)
+    feature[:geometry][:coordinates].map do |line|
+      line.map do |point|
+        build_line_point(feature, point)
+      end
+    end
+  end
+
+  def build_line_point(feature, point)
+    {
+      latitude:  point[1],
+      longitude: point[0],
+      timestamp: timestamp(point),
+      raw_data:  point
+    }
+  end
+
   def battery_level(level)
     value = (level.to_f * 100).to_i
 
     value.positive? ? value : nil
   end
+
+  def altitude(feature)
+    feature.dig(:properties, :altitude) || feature.dig(:geometry, :coordinates, 2)
+  end
+
+  def timestamp(feature)
+    return Time.zone.at(feature[3]) if feature.is_a?(Array)
+
+    value = feature.dig(:properties, :timestamp) || feature.dig(:geometry, :coordinates, 3)
+    Time.zone.at(value)
+  end
 end

app/services/google_maps/phone_takeout_parser.rb

@@ -11,8 +11,6 @@ class GoogleMaps::PhoneTakeoutParser
   def call
     points_data = parse_json
 
-    points = 0
-
     points_data.compact.each do |point_data|
       next if Point.exists?(
         timestamp:  point_data[:timestamp],
@@ -34,14 +32,7 @@ class GoogleMaps::PhoneTakeoutParser
         tracker_id: 'google-maps-phone-timeline-export',
         user_id:
       )
-
-      points += 1
     end
-
-    doubles = points_data.size - points
-    processed = points + doubles
-
-    { raw_points: points_data.size, points:, doubles:, processed: }
   end
 
   private
@@ -58,7 +49,9 @@ class GoogleMaps::PhoneTakeoutParser
     if import.raw_data.is_a?(Array)
       raw_array = parse_raw_array(import.raw_data)
     else
-      semantic_segments = parse_semantic_segments(import.raw_data['semanticSegments']) if import.raw_data['semanticSegments']
+      if import.raw_data['semanticSegments']
+        semantic_segments = parse_semantic_segments(import.raw_data['semanticSegments'])
+      end
       raw_signals = parse_raw_signals(import.raw_data['rawSignals']) if import.raw_data['rawSignals']
     end
 

app/services/google_maps/semantic_history_parser.rb

@@ -11,8 +11,6 @@ class GoogleMaps::SemanticHistoryParser
   def call
     points_data = parse_json
 
-    points = 0
-
     points_data.each do |point_data|
       next if Point.exists?(
         timestamp: point_data[:timestamp],
@@ -31,14 +29,7 @@ class GoogleMaps::SemanticHistoryParser
         import_id: import.id,
         user_id:
       )
-
-      points += 1
     end
-
-    doubles = points_data.size - points
-    processed = points + doubles
-
-    { raw_points: points_data.size, points:, doubles:, processed: }
   end
 
   private

app/services/gpx/track_parser.rb

@@ -13,32 +13,23 @@ class Gpx::TrackParser
     tracks = json['gpx']['trk']
     tracks_arr = tracks.is_a?(Array) ? tracks : [tracks]
 
-    tracks_arr
-      .map { parse_track(_1) }
-      .flatten
-      .reduce { |result, points| result.merge(points) { _2 + _3 } }
+    tracks_arr.map { parse_track(_1) }.flatten
   end
 
   private
 
   def parse_track(track)
     segments = track['trkseg']
-    segments_arr = segments.is_a?(Array) ? segments : [segments]
+    segments_array = segments.is_a?(Array) ? segments : [segments]
 
-    segments_arr.map do |segment|
-      trackpoints = segment['trkpt']
-
-      points = trackpoints.reduce(0) { _1 + create_point(_2) }
-      doubles = trackpoints.size - points
-      processed = points + doubles
-
-      { raw_points: trackpoints.size, points:, doubles:, processed: }
+    segments_array.map do |segment|
+      segment['trkpt'].each { create_point(_1) }
     end
   end
 
   def create_point(point)
-    return 0 if point['lat'].blank? || point['lon'].blank? || point['time'].blank?
-    return 0 if point_exists?(point)
+    return if point['lat'].blank? || point['lon'].blank? || point['time'].blank?
+    return if point_exists?(point)
 
     Point.create(
       latitude:   point['lat'].to_d,
@@ -49,8 +40,6 @@ class Gpx::TrackParser
       raw_data: point,
       user_id:
     )
-
-    1
   end
 
   def point_exists?(point)

app/services/immich/import_parser.rb

@@ -11,8 +11,6 @@ class Immich::ImportParser
 
   def call
     json.each { |point| create_point(point) }
-
-    { raw_points: 0, points: 0, doubles: 0, processed: 0 }
   end
 
   def create_point(point)

app/services/imports/create.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+class Imports::Create
+  attr_reader :user, :import
+
+  def initialize(user, import)
+    @user = user
+    @import = import
+  end
+
+  def call
+    parser(import.source).new(import, user.id).call
+
+    create_import_finished_notification(import, user)
+
+    schedule_stats_creating(user.id)
+    schedule_visit_suggesting(user.id, import)
+  rescue StandardError => e
+    create_import_failed_notification(import, user, e)
+  end
+
+  private
+
+  def parser(source)
+    # Bad classes naming by the way, they are not parsers, they are point creators
+    case source
+    when 'google_semantic_history'  then GoogleMaps::SemanticHistoryParser
+    when 'google_records'           then GoogleMaps::RecordsParser
+    when 'google_phone_takeout'     then GoogleMaps::PhoneTakeoutParser
+    when 'owntracks'                then OwnTracks::ExportParser
+    when 'gpx'                      then Gpx::TrackParser
+    when 'immich_api'               then Immich::ImportParser
+    when 'geojson'                  then Geojson::ImportParser
+    end
+  end
+
+  def schedule_stats_creating(user_id)
+    StatCreatingJob.perform_later(user_id)
+  end
+
+  def schedule_visit_suggesting(user_id, import)
+    points = import.points.order(:timestamp)
+    start_at = Time.zone.at(points.first.timestamp)
+    end_at = Time.zone.at(points.last.timestamp)
+
+    VisitSuggestingJob.perform_later(user_ids: [user_id], start_at:, end_at:)
+  end
+
+  def create_import_finished_notification(import, user)
+    Notifications::Create.new(
+      user:,
+      kind: :info,
+      title: 'Import finished',
+      content: "Import \"#{import.name}\" successfully finished."
+    ).call
+  end
+
+  def create_import_failed_notification(import, user, error)
+    Notifications::Create.new(
+      user:,
+      kind: :error,
+      title: 'Import failed',
+      content: "Import \"#{import.name}\" failed: #{error.message}, stacktrace: #{error.backtrace.join("\n")}"
+    ).call
+  end
+end

app/services/own_tracks/export_parser.rb

@@ -12,8 +12,6 @@ class OwnTracks::ExportParser
   def call
     points_data = parse_json
 
-    points = 0
-
     points_data.each do |point_data|
       next if Point.exists?(
         timestamp: point_data[:timestamp],
@@ -28,14 +26,7 @@ class OwnTracks::ExportParser
       end
 
       point.save
-
-      points += 1
     end
-
-    doubles = points_data.size - points
-    processed = points + doubles
-
-    { raw_points: points_data.size, points:, doubles:, processed: }
   end
 
   private

app/views/points/index.html.erb

@@ -43,6 +43,9 @@
 
         <div class="flex justify-between my-5">
           <%= f.submit "Delete Selected", class: "px-4 py-2 bg-red-500 text-white rounded-md", data: { confirm: "Are you sure?", turbo_confirm: "Are you sure?" } %>
+          <div class="flex justify-center">
+            <%= @points_number %> points found
+          </div>
           <div class="flex justify-end">
             <span class="mr-2">Order by:</span>
             <%= link_to 'Newest', points_path(order_by: :desc), class: 'btn btn-xs btn-primary mx-1' %>

spec/services/gpx/track_parser_spec.rb

@@ -15,7 +15,6 @@ RSpec.describe Gpx::TrackParser do
       context 'when file has a single segment' do
         it 'creates points' do
           expect { parser }.to change { Point.count }.by(301)
-          expect(parser).to eq({ doubles: 4, points: 301, processed: 305, raw_points: 305 })
         end
       end
 
@@ -24,7 +23,6 @@ RSpec.describe Gpx::TrackParser do
 
         it 'creates points' do
           expect { parser }.to change { Point.count }.by(558)
-          expect(parser).to eq({ doubles: 0, points: 558, processed: 558, raw_points: 558 })
         end
       end
     end
@@ -34,7 +32,6 @@ RSpec.describe Gpx::TrackParser do
 
       it 'creates points' do
         expect { parser }.to change { Point.count }.by(407)
-        expect(parser).to eq({ doubles: 0, points: 407, processed: 407, raw_points: 407 })
       end
     end
   end

swagger/v1/swagger.yaml

@@ -113,7 +113,7 @@ paths:
       - Health
       responses:
         '200':
-          description: areas found
+          description: Healthy
   "/api/v1/overland/batches":
     post:
       summary: Creates a batch of points