From 6c0a954e8ec172338db89cba17118c8f03f117f5 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Mon, 20 Jan 2025 16:37:15 +0100 Subject: [PATCH] Implement dawarich points parsing --- app/jobs/points/create_job.rb | 26 ++++ app/services/points/params.rb | 41 ++++++ ...dd_course_and_course_accuracy_to_points.rb | 8 ++ ...0152540_add_external_track_id_to_points.rb | 11 ++ db/schema.rb | 6 +- .../files/points/geojson_example.json | 136 ++++++++++++++++++ spec/jobs/points/create_job_spec.rb | 5 + spec/services/points/params_spec.rb | 66 +++++++++ 8 files changed, 298 insertions(+), 1 deletion(-) create mode 100644 app/jobs/points/create_job.rb create mode 100644 app/services/points/params.rb create mode 100644 db/migrate/20250120152014_add_course_and_course_accuracy_to_points.rb create mode 100644 db/migrate/20250120152540_add_external_track_id_to_points.rb create mode 100644 spec/fixtures/files/points/geojson_example.json create mode 100644 spec/jobs/points/create_job_spec.rb create mode 100644 spec/services/points/params_spec.rb diff --git a/app/jobs/points/create_job.rb b/app/jobs/points/create_job.rb new file mode 100644 index 00000000..f079046d --- /dev/null +++ b/app/jobs/points/create_job.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class Points::CreateJob < ApplicationJob + queue_as :default + + def perform(params, user_id) + data = Overland::Params.new(params).call + + data.each do |location| + next if point_exists?(location, user_id) + + Point.create!(location.merge(user_id:)) + end + end + + private + + def point_exists?(params, user_id) + Point.exists?( + latitude: params[:latitude], + longitude: params[:longitude], + timestamp: params[:timestamp], + user_id: + ) + end +end diff --git a/app/services/points/params.rb b/app/services/points/params.rb new file mode 100644 index 00000000..1e2873ca --- /dev/null +++ b/app/services/points/params.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +class Points::Params + attr_reader :data, :points + + def initialize(json) + @data = json.with_indifferent_access + @points = @data[:locations] + end + + def call + points.map do |point| + next if point[:geometry].nil? || point.dig(:properties, :timestamp).nil? + + { + latitude: point[:geometry][:coordinates][1], + longitude: point[:geometry][:coordinates][0], + battery_status: point[:properties][:battery_state], + battery: battery_level(point[:properties][:battery_level]), + timestamp: DateTime.parse(point[:properties][:timestamp]), + altitude: point[:properties][:altitude], + tracker_id: point[:properties][:device_id], + velocity: point[:properties][:speed], + ssid: point[:properties][:wifi], + accuracy: point[:properties][:horizontal_accuracy], + vertical_accuracy: point[:properties][:vertical_accuracy], + course_accuracy: point[:properties][:course_accuracy], + course: point[:properties][:course], + raw_data: point + } + end.compact + end + + private + + def battery_level(level) + value = (level.to_f * 100).to_i + + value.positive? ? value : nil + end +end diff --git a/db/migrate/20250120152014_add_course_and_course_accuracy_to_points.rb b/db/migrate/20250120152014_add_course_and_course_accuracy_to_points.rb new file mode 100644 index 00000000..78e1feb0 --- /dev/null +++ b/db/migrate/20250120152014_add_course_and_course_accuracy_to_points.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +class AddCourseAndCourseAccuracyToPoints < ActiveRecord::Migration[8.0] + def change + add_column :points, :course, :decimal, precision: 8, scale: 5 + add_column :points, :course_accuracy, :decimal, precision: 8, scale: 5 + end +end diff --git a/db/migrate/20250120152540_add_external_track_id_to_points.rb b/db/migrate/20250120152540_add_external_track_id_to_points.rb new file mode 100644 index 00000000..4531b19d --- /dev/null +++ b/db/migrate/20250120152540_add_external_track_id_to_points.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class AddExternalTrackIdToPoints < ActiveRecord::Migration[8.0] + disable_ddl_transaction! + + def change + add_column :points, :external_track_id, :string + + add_index :points, :external_track_id, algorithm: :concurrently + end +end diff --git a/db/schema.rb b/db/schema.rb index 16db4226..31ce24e6 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2024_12_11_113119) do +ActiveRecord::Schema[8.0].define(version: 2025_01_20_152540) do # These are extensions that must be enabled in order to support this database enable_extension "pg_catalog.plpgsql" @@ -156,12 +156,16 @@ ActiveRecord::Schema[8.0].define(version: 2024_12_11_113119) do t.jsonb "geodata", default: {}, null: false t.bigint "visit_id" t.datetime "reverse_geocoded_at" + t.decimal "course", precision: 8, scale: 5 + t.decimal "course_accuracy", precision: 8, scale: 5 + t.string "external_track_id" t.index ["altitude"], name: "index_points_on_altitude" t.index ["battery"], name: "index_points_on_battery" t.index ["battery_status"], name: "index_points_on_battery_status" t.index ["city"], name: "index_points_on_city" t.index ["connection"], name: "index_points_on_connection" t.index ["country"], name: "index_points_on_country" + t.index ["external_track_id"], name: "index_points_on_external_track_id" t.index ["geodata"], name: "index_points_on_geodata", using: :gin t.index ["import_id"], name: "index_points_on_import_id" t.index ["latitude", "longitude"], name: "index_points_on_latitude_and_longitude" diff --git a/spec/fixtures/files/points/geojson_example.json b/spec/fixtures/files/points/geojson_example.json new file mode 100644 index 00000000..c1cac9e4 --- /dev/null +++ b/spec/fixtures/files/points/geojson_example.json @@ -0,0 +1,136 @@ +{ + "locations" : [ + { + "type" : "Feature", + "geometry" : { + "type" : "Point", + "coordinates" : [ + -122.40530871, + 37.744304130000003 + ] + }, + "properties" : { + "horizontal_accuracy" : 5, + "track_id" : "799F32F5-89BB-45FB-A639-098B1B95B09F", + "speed_accuracy" : 0, + "vertical_accuracy" : -1, + "course_accuracy" : 0, + "altitude" : 0, + "speed" : 92.087999999999994, + "course" : 27.07, + "timestamp" : "2025-01-17T21:03:01Z", + "device_id" : "8D5D4197-245B-4619-A88B-2049100ADE46" + } + }, + { + "type" : "Feature", + "properties" : { + "timestamp" : "2025-01-17T21:03:02Z", + "horizontal_accuracy" : 5, + "course" : 24.260000000000002, + "speed_accuracy" : 0, + "device_id" : "8D5D4197-245B-4619-A88B-2049100ADE46", + "vertical_accuracy" : -1, + "altitude" : 0, + "track_id" : "799F32F5-89BB-45FB-A639-098B1B95B09F", + "speed" : 92.448000000000008, + "course_accuracy" : 0 + }, + "geometry" : { + "type" : "Point", + "coordinates" : [ + -122.40518926999999, + 37.744513759999997 + ] + } + }, + { + "type" : "Feature", + "properties" : { + "altitude" : 0, + "horizontal_accuracy" : 5, + "speed" : 123.76800000000001, + "course_accuracy" : 0, + "speed_accuracy" : 0, + "course" : 309.73000000000002, + "track_id" : "F63A3CF9-2FF8-4076-8F59-5BB1EDC23888", + "device_id" : "8D5D4197-245B-4619-A88B-2049100ADE46", + "timestamp" : "2025-01-17T21:18:38Z", + "vertical_accuracy" : -1 + }, + "geometry" : { + "type" : "Point", + "coordinates" : [ + -122.28487643, + 37.454486080000002 + ] + } + }, + { + "type" : "Feature", + "properties" : { + "track_id" : "F63A3CF9-2FF8-4076-8F59-5BB1EDC23888", + "device_id" : "8D5D4197-245B-4619-A88B-2049100ADE46", + "speed_accuracy" : 0, + "course_accuracy" : 0, + "speed" : 123.3, + "horizontal_accuracy" : 5, + "course" : 309.38, + "altitude" : 0, + "timestamp" : "2025-01-17T21:18:39Z", + "vertical_accuracy" : -1 + }, + "geometry" : { + "coordinates" : [ + -122.28517332, + 37.454684899999997 + ], + "type" : "Point" + } + }, + { + "geometry" : { + "coordinates" : [ + -122.28547306, + 37.454883219999999 + ], + "type" : "Point" + }, + "properties" : { + "course_accuracy" : 0, + "device_id" : "8D5D4197-245B-4619-A88B-2049100ADE46", + "vertical_accuracy" : -1, + "course" : 309.73000000000002, + "speed_accuracy" : 0, + "timestamp" : "2025-01-17T21:18:40Z", + "horizontal_accuracy" : 5, + "speed" : 125.06400000000001, + "track_id" : "F63A3CF9-2FF8-4076-8F59-5BB1EDC23888", + "altitude" : 0 + }, + "type" : "Feature" + }, + { + "geometry" : { + "type" : "Point", + "coordinates" : [ + -122.28577665, + 37.455080109999997 + ] + }, + "properties" : { + "course_accuracy" : 0, + "speed_accuracy" : 0, + "speed" : 124.05600000000001, + "track_id" : "F63A3CF9-2FF8-4076-8F59-5BB1EDC23888", + "course" : 309.73000000000002, + "device_id" : "8D5D4197-245B-4619-A88B-2049100ADE46", + "altitude" : 0, + "horizontal_accuracy" : 5, + "vertical_accuracy" : -1, + "timestamp" : "2025-01-17T21:18:41Z" + }, + "type" : "Feature" + } + ] +} diff --git a/spec/jobs/points/create_job_spec.rb b/spec/jobs/points/create_job_spec.rb new file mode 100644 index 00000000..70baa6e5 --- /dev/null +++ b/spec/jobs/points/create_job_spec.rb @@ -0,0 +1,5 @@ +require 'rails_helper' + +RSpec.describe Points::CreateJob, type: :job do + pending "add some examples to (or delete) #{__FILE__}" +end diff --git a/spec/services/points/params_spec.rb b/spec/services/points/params_spec.rb new file mode 100644 index 00000000..6fb3f486 --- /dev/null +++ b/spec/services/points/params_spec.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe Points::Params do + describe '#call' do + let(:file_path) { 'spec/fixtures/files/points/geojson_example.json' } + let(:file) { File.open(file_path) } + let(:json) { JSON.parse(file.read) } + let(:expected_json) do + { + latitude: 37.74430413, + longitude: -122.40530871, + battery_status: nil, + battery: nil, + timestamp: DateTime.parse('2025-01-17T21:03:01Z'), + altitude: 0, + tracker_id: '8D5D4197-245B-4619-A88B-2049100ADE46', + velocity: 92.088, + ssid: nil, + accuracy: 5, + vertical_accuracy: -1, + course_accuracy: 0, + course: 27.07, + raw_data: { + type: 'Feature', + geometry: { + type: 'Point', + coordinates: [-122.40530871, 37.74430413] + }, + properties: { + horizontal_accuracy: 5, + track_id: '799F32F5-89BB-45FB-A639-098B1B95B09F', + speed_accuracy: 0, + vertical_accuracy: -1, + course_accuracy: 0, + altitude: 0, + speed: 92.088, + course: 27.07, + timestamp: '2025-01-17T21:03:01Z', + device_id: '8D5D4197-245B-4619-A88B-2049100ADE46' + } + }.with_indifferent_access + } + end + + subject(:params) { described_class.new(json).call } + + it 'returns an array of points' do + expect(params).to be_an(Array) + expect(params.first).to eq(expected_json) + end + + it 'returns the correct number of points' do + expect(params.size).to eq(6) + end + + it 'returns correct keys' do + expect(params.first.keys).to eq(expected_json.keys) + end + + it 'returns the correct values' do + expect(params.first).to eq(expected_json) + end + end +end