From 5c16cc6dca15d1ce095dbfae72b7dc1a8746be72 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Thu, 15 Aug 2024 19:47:59 +0200 Subject: [PATCH 1/5] Fix owntracks params --- app/services/own_tracks/export_parser.rb | 16 +++--- app/services/own_tracks/params.rb | 12 ++--- app/services/reverse_geocoding/fetch_data.rb | 2 +- .../services/own_tracks/export_parser_spec.rb | 53 +++++++++++++++++++ spec/services/own_tracks/params_spec.rb | 30 ++++++----- 5 files changed, 82 insertions(+), 31 deletions(-) diff --git a/app/services/own_tracks/export_parser.rb b/app/services/own_tracks/export_parser.rb index acad0954..c64fa179 100644 --- a/app/services/own_tracks/export_parser.rb +++ b/app/services/own_tracks/export_parser.rb @@ -22,16 +22,12 @@ class OwnTracks::ExportParser user_id: ) - Point.create( - latitude: point_data[:latitude], - longitude: point_data[:longitude], - timestamp: point_data[:timestamp], - raw_data: point_data[:raw_data], - topic: point_data[:topic], - tracker_id: point_data[:tracker_id], - import_id: import.id, - user_id: - ) + point = Point.new(point_data).tap do |p| + p.user_id = user_id + p.import_id = import.id + end + + point.save points += 1 end diff --git a/app/services/own_tracks/params.rb b/app/services/own_tracks/params.rb index d2f40628..16ef464d 100644 --- a/app/services/own_tracks/params.rb +++ b/app/services/own_tracks/params.rb @@ -36,18 +36,18 @@ class OwnTracks::Params def battery_status return 'unknown' if params[:bs].nil? - case params[:bs] - when 'u' then 'unplugged' - when 'c' then 'charging' - when 'f' then 'full' + case params[:bs].to_i + when 1 then 'unplugged' + when 2 then 'charging' + when 3 then 'full' else 'unknown' end end def trigger - return 'unknown' if params[:m].nil? + return 'unknown' if params[:t].nil? - case params[:m] + case params[:t] when 'p' then 'background_event' when 'c' then 'circular_region_event' when 'b' then 'beacon_event' diff --git a/app/services/reverse_geocoding/fetch_data.rb b/app/services/reverse_geocoding/fetch_data.rb index 7be065fd..709c9117 100644 --- a/app/services/reverse_geocoding/fetch_data.rb +++ b/app/services/reverse_geocoding/fetch_data.rb @@ -21,6 +21,6 @@ class ReverseGeocoding::FetchData private def reverse_geocoded? - point.city.present? && point.country.present? || point.geodata.present? + point.geodata.present? end end diff --git a/spec/services/own_tracks/export_parser_spec.rb b/spec/services/own_tracks/export_parser_spec.rb index 21ecec64..8a98f6f8 100644 --- a/spec/services/own_tracks/export_parser_spec.rb +++ b/spec/services/own_tracks/export_parser_spec.rb @@ -13,6 +13,59 @@ RSpec.describe OwnTracks::ExportParser do it 'creates points' do expect { parser }.to change { Point.count }.by(9) end + + it 'correctly writes attributes' do + parser + + expect(Point.first.attributes).to include( + 'latitude' => 40.7128, + 'longitude' => -74.006, + 'battery_status' => 'charging', + 'battery' => 85, + 'ping' => nil, + 'altitude' => 41, + 'accuracy' => 8, + 'vertical_accuracy' => 3, + 'velocity' => nil, + 'connection' => 'wifi', + 'ssid' => 'Home Wifi', + 'bssid' => 'b0:f2:8:45:94:33', + 'trigger' => 'background_event', + 'tracker_id' => 'RO', + 'timestamp' => 1_706_965_203, + 'inrids' => ['5f1d1b'], + 'in_regions' => ['home'], + 'topic' => 'owntracks/test/iPhone 12 Pro', + 'visit_id' => nil, + 'user_id' => user.id, + 'country' => nil, + 'raw_data' => { + 'batt' => 85, + 'lon' => -74.006, + 'acc' => 8, + 'bs' => 2, + 'inrids' => ['5f1d1b'], + 'BSSID' => 'b0:f2:8:45:94:33', + 'SSID' => 'Home Wifi', + 'vac' => 3, + 'inregions' => ['home'], + 'lat' => 40.7128, + 'topic' => 'owntracks/test/iPhone 12 Pro', + 't' => 'p', + 'conn' => 'w', + 'm' => 1, + 'tst' => 1_706_965_203, + 'alt' => 41, + '_type' => 'location', + 'tid' => 'RO', + '_http' => true, + 'ghash' => 'u33d773', + 'isorcv' => '2024-02-03T13:00:03Z', + 'isotst' => '2024-02-03T13:00:03Z', + 'disptst' => '2024-02-03 13:00:03' + } + ) + end end end end diff --git a/spec/services/own_tracks/params_spec.rb b/spec/services/own_tracks/params_spec.rb index 39b3926d..b1ed55cf 100644 --- a/spec/services/own_tracks/params_spec.rb +++ b/spec/services/own_tracks/params_spec.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'rails_helper' RSpec.describe OwnTracks::Params do @@ -15,7 +17,7 @@ RSpec.describe OwnTracks::Params do { latitude: 40.7128, longitude: -74.006, - battery_status: 'unknown', + battery_status: 'charging', battery: 85, ping: nil, altitude: 41, @@ -25,9 +27,9 @@ RSpec.describe OwnTracks::Params do connection: 'wifi', ssid: 'Home Wifi', bssid: 'b0:f2:8:45:94:33', - trigger: 'unknown', + trigger: 'background_event', tracker_id: 'RO', - timestamp: 1706965203, + timestamp: 1_706_965_203, inrids: ['5f1d1b'], in_regions: ['home'], topic: 'owntracks/test/iPhone 12 Pro', @@ -46,7 +48,7 @@ RSpec.describe OwnTracks::Params do 't' => 'p', 'conn' => 'w', 'm' => 1, - 'tst' => 1706965203, + 'tst' => 1_706_965_203, 'alt' => 41, '_type' => 'location', 'tid' => 'RO', @@ -64,7 +66,7 @@ RSpec.describe OwnTracks::Params do end context 'when battery status is unplugged' do - let(:raw_point_params) { super().merge(bs: 'u') } + let(:raw_point_params) { super().merge(bs: 1) } it 'returns parsed params' do expect(params[:battery_status]).to eq('unplugged') @@ -72,7 +74,7 @@ RSpec.describe OwnTracks::Params do end context 'when battery status is charging' do - let(:raw_point_params) { super().merge(bs: 'c') } + let(:raw_point_params) { super().merge(bs: 2) } it 'returns parsed params' do expect(params[:battery_status]).to eq('charging') @@ -80,7 +82,7 @@ RSpec.describe OwnTracks::Params do end context 'when battery status is full' do - let(:raw_point_params) { super().merge(bs: 'f') } + let(:raw_point_params) { super().merge(bs: 3) } it 'returns parsed params' do expect(params[:battery_status]).to eq('full') @@ -96,7 +98,7 @@ RSpec.describe OwnTracks::Params do end context 'when trigger is circular_region_event' do - let(:raw_point_params) { super().merge(m: 'c') } + let(:raw_point_params) { super().merge(t: 'c') } it 'returns parsed params' do expect(params[:trigger]).to eq('circular_region_event') @@ -104,7 +106,7 @@ RSpec.describe OwnTracks::Params do end context 'when trigger is beacon_event' do - let(:raw_point_params) { super().merge(m: 'b') } + let(:raw_point_params) { super().merge(t: 'b') } it 'returns parsed params' do expect(params[:trigger]).to eq('beacon_event') @@ -112,7 +114,7 @@ RSpec.describe OwnTracks::Params do end context 'when trigger is report_location_message_event' do - let(:raw_point_params) { super().merge(m: 'r') } + let(:raw_point_params) { super().merge(t: 'r') } it 'returns parsed params' do expect(params[:trigger]).to eq('report_location_message_event') @@ -120,7 +122,7 @@ RSpec.describe OwnTracks::Params do end context 'when trigger is manual_event' do - let(:raw_point_params) { super().merge(m: 'u') } + let(:raw_point_params) { super().merge(t: 'u') } it 'returns parsed params' do expect(params[:trigger]).to eq('manual_event') @@ -128,7 +130,7 @@ RSpec.describe OwnTracks::Params do end context 'when trigger is timer_based_event' do - let(:raw_point_params) { super().merge(m: 't') } + let(:raw_point_params) { super().merge(t: 't') } it 'returns parsed params' do expect(params[:trigger]).to eq('timer_based_event') @@ -136,7 +138,7 @@ RSpec.describe OwnTracks::Params do end context 'when trigger is settings_monitoring_event' do - let(:raw_point_params) { super().merge(m: 'v') } + let(:raw_point_params) { super().merge(t: 'v') } it 'returns parsed params' do expect(params[:trigger]).to eq('settings_monitoring_event') @@ -184,7 +186,7 @@ RSpec.describe OwnTracks::Params do end context 'when trigger is unknown' do - let(:raw_point_params) { super().merge(m: 'unknown') } + before { raw_point_params[:t] = 'unknown' } it 'returns parsed params' do expect(params[:trigger]).to eq('unknown') From 2b4a16d6cb9ce2843c5a9529558346861dc93eb7 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Thu, 15 Aug 2024 19:57:52 +0200 Subject: [PATCH 2/5] Add data migration to fill owntracks points data --- ...0240815174852_add_owntracks_points_data.rb | 33 +++++++++++++++++++ db/schema.rb | 5 ++- 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 db/data/20240815174852_add_owntracks_points_data.rb diff --git a/db/data/20240815174852_add_owntracks_points_data.rb b/db/data/20240815174852_add_owntracks_points_data.rb new file mode 100644 index 00000000..80123c61 --- /dev/null +++ b/db/data/20240815174852_add_owntracks_points_data.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +class AddOwntracksPointsData < ActiveRecord::Migration[7.1] + def up + Import.owntracks.each do |import| + import.points.each do |point| + params = OwnTracks::Params.new(point.raw_data).call + + point.update!( + battery: params[:battery], + ping: params[:ping], + altitude: params[:altitude], + accuracy: params[:accuracy], + vertical_accuracy: params[:vertical_accuracy], + velocity: params[:velocity], + ssid: params[:ssid], + bssid: params[:bssid], + tracker_id: params[:tracker_id], + inrids: params[:inrids], + in_regions: params[:in_regions], + topic: params[:topic], + battery_status: params[:battery_status], + connection: params[:connection], + trigger: params[:trigger] + ) + end + end + end + + def down + raise ActiveRecord::IrreversibleMigration + end +end diff --git a/db/schema.rb b/db/schema.rb index 30119084..7bf01c80 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2024_07_21_183116) do +ActiveRecord::Schema[7.1].define(version: 2024_08_08_121027) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -53,6 +53,9 @@ ActiveRecord::Schema[7.1].define(version: 2024_07_21_183116) do t.index ["user_id"], name: "index_areas_on_user_id" end + create_table "data_migrations", primary_key: "version", id: :string, force: :cascade do |t| + end + create_table "exports", force: :cascade do |t| t.string "name", null: false t.string "url" From a0aa68498552f1af9f488e2486d0e470c2a98b16 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Thu, 15 Aug 2024 20:23:05 +0200 Subject: [PATCH 3/5] Update owntracks points data --- ...0240815174852_add_owntracks_points_data.rb | 63 ++++++++++++++----- 1 file changed, 46 insertions(+), 17 deletions(-) diff --git a/db/data/20240815174852_add_owntracks_points_data.rb b/db/data/20240815174852_add_owntracks_points_data.rb index 80123c61..f6df6178 100644 --- a/db/data/20240815174852_add_owntracks_points_data.rb +++ b/db/data/20240815174852_add_owntracks_points_data.rb @@ -2,32 +2,61 @@ class AddOwntracksPointsData < ActiveRecord::Migration[7.1] def up + Rails.logger.info("Updating #{Import.owntracks.count} owntracks imports points") + + import_points = 0 Import.owntracks.each do |import| import.points.each do |point| params = OwnTracks::Params.new(point.raw_data).call - point.update!( - battery: params[:battery], - ping: params[:ping], - altitude: params[:altitude], - accuracy: params[:accuracy], - vertical_accuracy: params[:vertical_accuracy], - velocity: params[:velocity], - ssid: params[:ssid], - bssid: params[:bssid], - tracker_id: params[:tracker_id], - inrids: params[:inrids], - in_regions: params[:in_regions], - topic: params[:topic], - battery_status: params[:battery_status], - connection: params[:connection], - trigger: params[:trigger] - ) + update_point(point, params) + + import_points += 1 end end + + Rails.logger.info("#{import_points} points updated from owntracks imports") + + # Getting points by owntracks-specific data + points = Point.where("raw_data -> 'm' is not null and raw_data -> 'acc' is not null") + + Rails.logger.info("Updating #{points.count} points") + + points_updated = 0 + points.each do |point| + params = OwnTracks::Params.new(point.raw_data).call + + update_point(point, params) + + points_updated += 1 + end + + Rails.logger.info("#{points_updated} points updated") end def down raise ActiveRecord::IrreversibleMigration end + + private + + def update_point(point, params) + point.update!( + battery: params[:battery], + ping: params[:ping], + altitude: params[:altitude], + accuracy: params[:accuracy], + vertical_accuracy: params[:vertical_accuracy], + velocity: params[:velocity], + ssid: params[:ssid], + bssid: params[:bssid], + tracker_id: params[:tracker_id], + inrids: params[:inrids], + in_regions: params[:in_regions], + topic: params[:topic], + battery_status: params[:battery_status], + connection: params[:connection], + trigger: params[:trigger] + ) + end end From bbbf591b6ba6c065f953c19f929143e016015af4 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Thu, 15 Aug 2024 20:24:22 +0200 Subject: [PATCH 4/5] Update changelog --- .app_version | 2 +- CHANGELOG.md | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.app_version b/.app_version index 8225a4ba..583b27ac 100644 --- a/.app_version +++ b/.app_version @@ -1 +1 @@ -0.9.11 +0.9.12 diff --git a/CHANGELOG.md b/CHANGELOG.md index 35bec667..2dc3bf7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [0.9.12] — 2024-08-15 + +### Fixed + +- Owntracks points are now being saved to the database with the full attributes +- Existing owntracks points also filled with missing data + ## [0.9.11] — 2024-08-14 ### Fixed From 27fca4c0da73113febc439b9bc9df8c5b38e0c9f Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Thu, 15 Aug 2024 20:25:40 +0200 Subject: [PATCH 5/5] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2dc3bf7d..1f3969c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Owntracks points are now being saved to the database with the full attributes - Existing owntracks points also filled with missing data +- Definition of "reverse geocoded points" is now correctly based on the number of points that have full reverse geocoding data instead of the number of points that have only country and city ## [0.9.11] — 2024-08-14