Merge remote-tracking branch 'origin/dev' into feature/user-features-access

This commit is contained in:
Eugene Burmakin 2025-02-24 21:40:36 +01:00
commit 173af225d7
73 changed files with 1085 additions and 409 deletions

2
.gitignore vendored
View file

@ -63,3 +63,5 @@
.ash_history
.cache/
.dotnet/
.cursorrules
.cursormemory.md

View file

@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](http://keepachangelog.com/)
and this project adheres to [Semantic Versioning](http://semver.org/).
# 0.24.2 - 2025-02-15
# 0.24.2 - 2025-02-24
## Added
@ -12,8 +12,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
## Fixed
- Fixed a bug where background jobs to import Immich and Photoprism geolocation data data could not be created by non-admin users.
- Fixed a bug where upon point deletion there was an error it was not being removed from the map, while it was actually deleted from the database. #883
- Fixed a bug where non-admin users could not import Immich and Photoprism geolocation data.
- Fixed a bug where upon point deletion it was not being removed from the map, while it was actually deleted from the database. #883
- Fixed a bug where upon import deletion stats were not being recalculated. #824
### Changed
@ -21,6 +21,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
- Restrict access to Sidekiq in non self-hosted mode.
- Restrict access to background jobs in non self-hosted mode.
- Restrict access to users management in non self-hosted mode.
- Points are now using `lonlat` column for storing longitude and latitude.
- Semantic history points are now being imported much faster.
- GPX files are now being imported much faster.
- Distance calculation are now using Postgis functions and expected to be more accurate.
# 0.24.1 - 2025-02-13

View file

@ -9,7 +9,7 @@ gem 'bootsnap', require: false
gem 'chartkick'
gem 'data_migrate'
gem 'devise'
gem 'geocoder', git: 'https://github.com/alexreisner/geocoder.git', ref: '04ee293'
gem 'geocoder'
gem 'gpx'
gem 'groupdate'
gem 'httparty'
@ -19,11 +19,12 @@ gem 'lograge'
gem 'oj'
gem 'pg'
gem 'prometheus_exporter'
gem 'activerecord-postgis-adapter', github: 'StoneGod/activerecord-postgis-adapter', branch: 'rails-8'
gem 'activerecord-postgis-adapter'
gem 'puma'
gem 'pundit'
gem 'rails', '~> 8.0'
gem 'rgeo'
gem 'rgeo-activerecord'
gem 'rswag-api'
gem 'rswag-ui'
gem 'shrine', '~> 3.6'

View file

@ -1,21 +1,3 @@
GIT
remote: https://github.com/StoneGod/activerecord-postgis-adapter.git
revision: 147fd43191ef703e2a1b3654f31d9139201a87e8
branch: rails-8
specs:
activerecord-postgis-adapter (10.0.1)
activerecord (~> 8.0.0)
rgeo-activerecord (~> 8.0.0)
GIT
remote: https://github.com/alexreisner/geocoder.git
revision: 04ee2936a30b30a23ded5231d7faf6cf6c27c099
ref: 04ee293
specs:
geocoder (1.8.3)
base64 (>= 0.1.0)
csv (>= 3.0.0)
GEM
remote: https://rubygems.org/
specs:
@ -71,6 +53,9 @@ GEM
activemodel (= 8.0.1)
activesupport (= 8.0.1)
timeout (>= 0.4.0)
activerecord-postgis-adapter (11.0.0)
activerecord (~> 8.0.0)
rgeo-activerecord (~> 8.0.0)
activestorage (8.0.1)
actionpack (= 8.0.1)
activejob (= 8.0.1)
@ -153,6 +138,9 @@ GEM
fugit (1.11.1)
et-orbi (~> 1, >= 1.2.11)
raabro (~> 1.4)
geocoder (1.8.5)
base64 (>= 0.1.0)
csv (>= 3.0.0)
globalid (1.2.1)
activesupport (>= 6.1)
gpx (1.2.0)
@ -192,7 +180,7 @@ GEM
kaminari-core (= 1.2.2)
kaminari-core (1.2.2)
language_server-protocol (3.17.0.4)
logger (1.6.5)
logger (1.6.6)
lograge (0.14.0)
actionpack (>= 4)
activesupport (>= 4)
@ -461,7 +449,7 @@ PLATFORMS
x86_64-linux
DEPENDENCIES
activerecord-postgis-adapter!
activerecord-postgis-adapter
bootsnap
chartkick
data_migrate
@ -473,7 +461,7 @@ DEPENDENCIES
fakeredis
ffaker
foreman
geocoder!
geocoder
gpx
groupdate
httparty
@ -490,6 +478,7 @@ DEPENDENCIES
rails (~> 8.0)
redis
rgeo
rgeo-activerecord
rspec-rails
rswag-api
rswag-specs

View file

@ -7,8 +7,8 @@ class MapController < ApplicationController
@points = points.where('timestamp >= ? AND timestamp <= ?', start_at, end_at)
@coordinates =
@points.pluck(:latitude, :longitude, :battery, :altitude, :timestamp, :velocity, :id, :country)
.map { [_1.to_f, _2.to_f, _3.to_s, _4.to_s, _5.to_s, _6.to_s, _7.to_s, _8.to_s] }
@points.pluck(:lonlat, :battery, :altitude, :timestamp, :velocity, :id, :country)
.map { |lonlat, *rest| [lonlat.y, lonlat.x, *rest.map(&:to_s)] }
@distance = distance
@start_at = Time.zone.at(start_at)
@end_at = Time.zone.at(end_at)

View file

@ -0,0 +1,13 @@
# frozen_string_literal: true
class DataMigrations::MigratePointsLatlonJob < ApplicationJob
queue_as :default
def perform(user_id)
user = User.find(user_id)
# rubocop:disable Rails/SkipsModelValidations
user.tracked_points.update_all('lonlat = ST_SetSRID(ST_MakePoint(longitude, latitude), 4326)')
# rubocop:enable Rails/SkipsModelValidations
end
end

View file

@ -9,7 +9,7 @@ class Points::CreateJob < ApplicationJob
data.each_slice(1000) do |location_batch|
Point.upsert_all(
location_batch,
unique_by: %i[latitude longitude timestamp user_id],
unique_by: %i[lonlat timestamp user_id],
returning: false
)
end

View file

@ -0,0 +1,110 @@
# frozen_string_literal: true
module Distanceable
extend ActiveSupport::Concern
DISTANCE_UNITS = {
km: 1000, # to meters
mi: 1609.34, # to meters
m: 1, # already in meters
ft: 0.3048, # to meters
yd: 0.9144 # to meters
}.freeze
module ClassMethods
def total_distance(points = nil, unit = :km)
# Handle method being called directly on relation vs with array
if points.nil?
calculate_distance_for_relation(unit)
else
calculate_distance_for_array(points, unit)
end
end
private
def calculate_distance_for_relation(unit)
unless DISTANCE_UNITS.key?(unit.to_sym)
raise ArgumentError, "Invalid unit. Supported units are: #{DISTANCE_UNITS.keys.join(', ')}"
end
distance_in_meters = connection.select_value(<<-SQL.squish)
WITH points_with_previous AS (
SELECT
lonlat,
LAG(lonlat) OVER (ORDER BY timestamp) as prev_lonlat
FROM (#{to_sql}) AS points
)
SELECT COALESCE(
SUM(
ST_Distance(
lonlat::geography,
prev_lonlat::geography
)
),
0
)
FROM points_with_previous
WHERE prev_lonlat IS NOT NULL
SQL
distance_in_meters.to_f / DISTANCE_UNITS[unit.to_sym]
end
def calculate_distance_for_array(points, unit = :km)
unless DISTANCE_UNITS.key?(unit.to_sym)
raise ArgumentError, "Invalid unit. Supported units are: #{DISTANCE_UNITS.keys.join(', ')}"
end
return 0 if points.length < 2
total_meters = points.each_cons(2).sum do |point1, point2|
connection.select_value(<<-SQL.squish)
SELECT ST_Distance(
ST_GeomFromEWKT('#{point1.lonlat}')::geography,
ST_GeomFromEWKT('#{point2.lonlat}')::geography
)
SQL
end
total_meters.to_f / DISTANCE_UNITS[unit.to_sym]
end
end
def distance_to(other_point, unit = :km)
unless DISTANCE_UNITS.key?(unit.to_sym)
raise ArgumentError, "Invalid unit. Supported units are: #{DISTANCE_UNITS.keys.join(', ')}"
end
# Extract coordinates based on what type other_point is
other_lonlat = extract_point(other_point)
return nil if other_lonlat.nil?
# Calculate distance in meters using PostGIS
distance_in_meters = self.class.connection.select_value(<<-SQL.squish)
SELECT ST_Distance(
ST_GeomFromEWKT('#{lonlat}')::geography,
ST_GeomFromEWKT('#{other_lonlat}')::geography
)
SQL
# Convert to requested unit
distance_in_meters.to_f / DISTANCE_UNITS[unit.to_sym]
end
private
def extract_point(point)
case point
when Array
unless point.length == 2
raise ArgumentError,
'Coordinates array must contain exactly 2 elements [latitude, longitude]'
end
RGeo::Geographic.spherical_factory(srid: 4326).point(point[1], point[0])
when self.class
point.lonlat
end
end
end

View file

@ -0,0 +1,74 @@
# frozen_string_literal: true
module Nearable
extend ActiveSupport::Concern
DISTANCE_UNITS = {
km: 1000, # to meters
mi: 1609.34, # to meters
m: 1, # already in meters
ft: 0.3048, # to meters
yd: 0.9144 # to meters
}.freeze
class_methods do
# rubocop:disable Metrics/MethodLength
def near(*args)
latitude, longitude, radius, unit = extract_coordinates_and_options(*args)
unless DISTANCE_UNITS.key?(unit.to_sym)
raise ArgumentError, "Invalid unit. Supported units are: #{DISTANCE_UNITS.keys.join(', ')}"
end
# Convert radius to meters for ST_DWithin
radius_in_meters = radius * DISTANCE_UNITS[unit.to_sym]
# Create a point from the given coordinates
point = "SRID=4326;POINT(#{longitude} #{latitude})"
where(<<-SQL.squish)
ST_DWithin(
lonlat::geography,
ST_GeomFromEWKT('#{point}')::geography,
#{radius_in_meters}
)
SQL
end
def with_distance(*args)
latitude, longitude, unit = extract_coordinates_and_options(*args)
unless DISTANCE_UNITS.key?(unit.to_sym)
raise ArgumentError, "Invalid unit. Supported units are: #{DISTANCE_UNITS.keys.join(', ')}"
end
point = "SRID=4326;POINT(#{longitude} #{latitude})"
conversion_factor = 1.0 / DISTANCE_UNITS[unit.to_sym]
select(<<-SQL.squish)
#{table_name}.*,
ST_Distance(
lonlat::geography,
ST_GeomFromEWKT('#{point}')::geography
) * #{conversion_factor} as distance_in_#{unit}
SQL
end
# rubocop:enable Metrics/MethodLength
private
def extract_coordinates_and_options(*args)
coords = args.first
if !coords.is_a?(Array) || coords.length != 2
raise ArgumentError,
'First argument must be coordinates array containing exactly 2 elements [latitude, longitude]'
end
[coords[0], coords[1], *args[1..]].tap do |extracted|
# Set default values for missing options
extracted[2] ||= 1 if extracted.length < 3 # default radius
extracted[3] ||= :km if extracted.length < 4 # default unit
end
end
end
end

View file

@ -1,18 +1,20 @@
# frozen_string_literal: true
class Point < ApplicationRecord
reverse_geocoded_by :latitude, :longitude
include Nearable
include Distanceable
belongs_to :import, optional: true, counter_cache: true
belongs_to :visit, optional: true
belongs_to :user
validates :latitude, :longitude, :timestamp, presence: true
validates :timestamp, uniqueness: {
scope: %i[latitude longitude user_id],
validates :timestamp, :lonlat, presence: true
validates :lonlat, uniqueness: {
scope: %i[timestamp user_id],
message: 'already has a point at this location and time for this user',
index: true
}
enum :battery_status, { unknown: 0, unplugged: 1, charging: 2, full: 3 }, suffix: true
enum :trigger, {
unknown: 0, background_event: 1, circular_region_event: 2, beacon_event: 3,
@ -47,14 +49,23 @@ class Point < ApplicationRecord
reverse_geocoded_at.present?
end
def lon
lonlat.x
end
def lat
lonlat.y
end
private
# rubocop:disable Metrics/MethodLength Metrics/AbcSize
def broadcast_coordinates
PointsChannel.broadcast_to(
user,
[
latitude.to_f,
longitude.to_f,
lat,
lon,
battery.to_s,
altitude.to_s,
timestamp.to_s,
@ -64,4 +75,5 @@ class Point < ApplicationRecord
]
)
end
# rubocop:enable Metrics/MethodLength
end

View file

@ -37,7 +37,7 @@ class Stat < ApplicationRecord
def calculate_daily_distances(monthly_points)
timespan.to_a.map.with_index(1) do |day, index|
daily_points = filter_points_for_day(monthly_points, day)
distance = calculate_distance(daily_points)
distance = Point.total_distance(daily_points, DISTANCE_UNIT)
[index, distance.round(2)]
end
end
@ -48,10 +48,4 @@ class Stat < ApplicationRecord
points.select { |p| p.timestamp.between?(beginning_of_day, end_of_day) }
end
def calculate_distance(points)
points.each_cons(2).sum do |point1, point2|
DistanceCalculator.new(point1, point2).call
end
end
end

View file

@ -14,7 +14,6 @@ class Trip < ApplicationRecord
calculate_distance
end
def points
user.tracked_points.where(timestamp: started_at.to_i..ended_at.to_i).order(:timestamp)
end
@ -47,18 +46,13 @@ class Trip < ApplicationRecord
end
def calculate_path
trip_path = Tracks::BuildPath.new(points.pluck(:latitude, :longitude)).call
trip_path = Tracks::BuildPath.new(points.pluck(:lonlat)).call
self.path = trip_path
end
def calculate_distance
distance = 0
points.each_cons(2) do |point1, point2|
distance += DistanceCalculator.new(point1, point2).call
end
distance = Point.total_distance(points, DISTANCE_UNIT)
self.distance = distance.round
end

View file

@ -29,7 +29,7 @@ class Visit < ApplicationRecord
return area&.radius if area.present?
radius = points.map do |point|
Geocoder::Calculations.distance_between(center, [point.latitude, point.longitude])
Geocoder::Calculations.distance_between(center, [point.lat, point.lon])
end.max
radius && radius >= 15 ? radius : 15

View file

@ -8,8 +8,8 @@ class Api::SlimPointSerializer
def call
{
id: point.id,
latitude: point.latitude,
longitude: point.longitude,
latitude: point.lat.to_s,
longitude: point.lon.to_s,
timestamp: point.timestamp
}
end

View file

@ -22,8 +22,8 @@ class ExportSerializer
def export_point(point)
{
lat: point.latitude,
lon: point.longitude,
lat: point.lat.to_s,
lon: point.lon.to_s,
bs: battery_status(point),
batt: point.battery,
p: point.ping,

View file

@ -1,14 +1,20 @@
# frozen_string_literal: true
class PointSerializer
EXCLUDED_ATTRIBUTES = %w[created_at updated_at visit_id id import_id user_id raw_data].freeze
EXCLUDED_ATTRIBUTES = %w[
created_at updated_at visit_id id import_id user_id raw_data lonlat
reverse_geocoded_at
].freeze
def initialize(point)
@point = point
end
def call
point.attributes.except(*EXCLUDED_ATTRIBUTES)
point.attributes.except(*EXCLUDED_ATTRIBUTES).tap do |attributes|
attributes['latitude'] = point.lat.to_s
attributes['longitude'] = point.lon.to_s
end
end
private

View file

@ -14,7 +14,7 @@ class Points::GeojsonSerializer
type: 'Feature',
geometry: {
type: 'Point',
coordinates: [point.longitude, point.latitude]
coordinates: [point.lon.to_s, point.lat.to_s]
},
properties: PointSerializer.new(point).call
}

View file

@ -17,8 +17,8 @@ class Points::GpxSerializer
points.each do |point|
track_segment.points << GPX::TrackPoint.new(
lat: point.latitude.to_f,
lon: point.longitude.to_f,
lat: point.lat,
lon: point.lon,
elevation: point.altitude.to_f,
time: point.recorded_at
)

View file

@ -38,7 +38,7 @@ class Areas::Visits::Create
end
points = Point.where(user_id: user.id)
.near([area.latitude, area.longitude], area_radius, units: DISTANCE_UNIT)
.near([area.latitude, area.longitude], area_radius, DISTANCE_UNIT)
.order(timestamp: :asc)
# check if all points within the area are assigned to a visit

View file

@ -1,18 +0,0 @@
# frozen_string_literal: true
class DistanceCalculator
def initialize(point1, point2)
@point1 = point1
@point2 = point2
end
def call
Geocoder::Calculations.distance_between(
point1.to_coordinates, point2.to_coordinates, units: ::DISTANCE_UNIT
)
end
private
attr_reader :point1, :point2
end

View file

@ -27,8 +27,7 @@ class Geojson::ImportParser
def point_exists?(params, user_id)
Point.exists?(
latitude: params[:latitude],
longitude: params[:longitude],
lonlat: params[:lonlat],
timestamp: params[:timestamp],
user_id:
)

View file

@ -33,8 +33,7 @@ class Geojson::Params
def build_point(feature)
{
latitude: feature[:geometry][:coordinates][1],
longitude: feature[:geometry][:coordinates][0],
lonlat: "POINT(#{feature[:geometry][:coordinates][0]} #{feature[:geometry][:coordinates][1]})",
battery_status: feature[:properties][:battery_state],
battery: battery_level(feature[:properties][:battery_level]),
timestamp: timestamp(feature),
@ -64,8 +63,7 @@ class Geojson::Params
def build_line_point(point)
{
latitude: point[1],
longitude: point[0],
lonlat: "POINT(#{point[0]} #{point[1]})",
timestamp: timestamp(point),
raw_data: point
}

View file

@ -16,14 +16,12 @@ class GoogleMaps::PhoneTakeoutParser
points_data.compact.each.with_index(1) do |point_data, index|
next if Point.exists?(
timestamp: point_data[:timestamp],
latitude: point_data[:latitude],
longitude: point_data[:longitude],
lonlat: point_data[:lonlat],
user_id:
)
Point.create(
latitude: point_data[:latitude],
longitude: point_data[:longitude],
lonlat: point_data[:lonlat],
timestamp: point_data[:timestamp],
raw_data: point_data[:raw_data],
accuracy: point_data[:accuracy],
@ -72,8 +70,7 @@ class GoogleMaps::PhoneTakeoutParser
def point_hash(lat, lon, timestamp, raw_data)
{
latitude: lat.to_f,
longitude: lon.to_f,
lonlat: "POINT(#{lon.to_f} #{lat.to_f})",
timestamp:,
raw_data:,
accuracy: raw_data['accuracyMeters'],

View file

@ -25,8 +25,7 @@ class GoogleMaps::RecordsImporter
# rubocop:disable Metrics/MethodLength
def prepare_location_data(location)
{
latitude: location['latitudeE7'].to_f / 10**7,
longitude: location['longitudeE7'].to_f / 10**7,
lonlat: "POINT(#{location['longitudeE7'].to_f / 10**7} #{location['latitudeE7'].to_f / 10**7})",
timestamp: parse_timestamp(location),
altitude: location['altitude'],
velocity: location['velocity'],
@ -47,7 +46,7 @@ class GoogleMaps::RecordsImporter
# rubocop:disable Rails/SkipsModelValidations
Point.upsert_all(
unique_batch,
unique_by: %i[latitude longitude timestamp user_id],
unique_by: %i[lonlat timestamp user_id],
returning: false,
on_duplicate: :skip
)
@ -59,8 +58,7 @@ class GoogleMaps::RecordsImporter
def deduplicate_batch(batch)
batch.uniq do |record|
[
record[:latitude].round(7),
record[:longitude].round(7),
record[:lonlat],
record[:timestamp],
record[:user_id]
]

View file

@ -3,87 +3,135 @@
class GoogleMaps::SemanticHistoryParser
include Imports::Broadcaster
BATCH_SIZE = 1000
attr_reader :import, :user_id
def initialize(import, user_id)
@import = import
@user_id = user_id
@current_index = 0
end
def call
points_data = parse_json
points_data.each.with_index(1) do |point_data, index|
next if Point.exists?(
timestamp: point_data[:timestamp],
latitude: point_data[:latitude],
longitude: point_data[:longitude],
user_id:
)
Point.create(
latitude: point_data[:latitude],
longitude: point_data[:longitude],
timestamp: point_data[:timestamp],
raw_data: point_data[:raw_data],
topic: 'Google Maps Timeline Export',
tracker_id: 'google-maps-timeline-export',
import_id: import.id,
user_id:
)
broadcast_import_progress(import, index)
points_data.each_slice(BATCH_SIZE) do |batch|
@current_index += batch.size
process_batch(batch)
broadcast_import_progress(import, @current_index)
end
end
private
def process_batch(batch)
records = batch.map { |point_data| prepare_point_data(point_data) }
# rubocop:disable Rails/SkipsModelValidations
Point.upsert_all(
records,
unique_by: %i[lonlat timestamp user_id],
returning: false,
on_duplicate: :skip
)
# rubocop:enable Rails/SkipsModelValidations
rescue StandardError => e
create_notification("Failed to process location batch: #{e.message}")
end
def prepare_point_data(point_data)
{
lonlat: point_data[:lonlat],
timestamp: point_data[:timestamp],
raw_data: point_data[:raw_data],
topic: 'Google Maps Timeline Export',
tracker_id: 'google-maps-timeline-export',
import_id: import.id,
user_id: user_id,
created_at: Time.current,
updated_at: Time.current
}
end
def create_notification(message)
Notification.create!(
user_id: user_id,
title: 'Google Maps Timeline Import Error',
content: message,
kind: :error
)
end
def parse_json
import.raw_data['timelineObjects'].flat_map do |timeline_object|
if timeline_object['activitySegment'].present?
if timeline_object['activitySegment']['startLocation'].blank?
next if timeline_object['activitySegment']['waypointPath'].blank?
parse_timeline_object(timeline_object)
end.compact
end
timeline_object['activitySegment']['waypointPath']['waypoints'].map do |waypoint|
{
latitude: waypoint['latE7'].to_f / 10**7,
longitude: waypoint['lngE7'].to_f / 10**7,
timestamp: Timestamps.parse_timestamp(timeline_object['activitySegment']['duration']['startTimestamp'] || timeline_object['activitySegment']['duration']['startTimestampMs']),
raw_data: timeline_object
}
end
else
{
latitude: timeline_object['activitySegment']['startLocation']['latitudeE7'].to_f / 10**7,
longitude: timeline_object['activitySegment']['startLocation']['longitudeE7'].to_f / 10**7,
timestamp: Timestamps.parse_timestamp(timeline_object['activitySegment']['duration']['startTimestamp'] || timeline_object['activitySegment']['duration']['startTimestampMs']),
raw_data: timeline_object
}
end
elsif timeline_object['placeVisit'].present?
if timeline_object.dig('placeVisit', 'location', 'latitudeE7').present? &&
timeline_object.dig('placeVisit', 'location', 'longitudeE7').present?
{
latitude: timeline_object['placeVisit']['location']['latitudeE7'].to_f / 10**7,
longitude: timeline_object['placeVisit']['location']['longitudeE7'].to_f / 10**7,
timestamp: Timestamps.parse_timestamp(timeline_object['placeVisit']['duration']['startTimestamp'] || timeline_object['placeVisit']['duration']['startTimestampMs']),
raw_data: timeline_object
}
elsif timeline_object.dig('placeVisit', 'otherCandidateLocations')&.any?
point = timeline_object['placeVisit']['otherCandidateLocations'][0]
def parse_timeline_object(timeline_object)
if timeline_object['activitySegment'].present?
parse_activity_segment(timeline_object['activitySegment'])
elsif timeline_object['placeVisit'].present?
parse_place_visit(timeline_object['placeVisit'])
end
end
next unless point['latitudeE7'].present? && point['longitudeE7'].present?
def parse_activity_segment(activity)
if activity['startLocation'].blank?
parse_waypoints(activity)
else
build_point_from_location(
longitude: activity['startLocation']['longitudeE7'],
latitude: activity['startLocation']['latitudeE7'],
timestamp: activity['duration']['startTimestamp'] || activity['duration']['startTimestampMs'],
raw_data: activity
)
end
end
{
latitude: point['latitudeE7'].to_f / 10**7,
longitude: point['longitudeE7'].to_f / 10**7,
timestamp: Timestamps.parse_timestamp(timeline_object['placeVisit']['duration']['startTimestamp'] || timeline_object['placeVisit']['duration']['startTimestampMs']),
raw_data: timeline_object
}
else
next
end
end
end.reject(&:blank?)
def parse_waypoints(activity)
return if activity['waypointPath'].blank?
activity['waypointPath']['waypoints'].map do |waypoint|
build_point_from_location(
longitude: waypoint['lngE7'],
latitude: waypoint['latE7'],
timestamp: activity['duration']['startTimestamp'] || activity['duration']['startTimestampMs'],
raw_data: activity
)
end
end
def parse_place_visit(place_visit)
if place_visit.dig('location', 'latitudeE7').present? &&
place_visit.dig('location', 'longitudeE7').present?
build_point_from_location(
longitude: place_visit['location']['longitudeE7'],
latitude: place_visit['location']['latitudeE7'],
timestamp: place_visit['duration']['startTimestamp'] || place_visit['duration']['startTimestampMs'],
raw_data: place_visit
)
elsif (candidate = place_visit.dig('otherCandidateLocations', 0))
parse_candidate_location(candidate, place_visit)
end
end
def parse_candidate_location(candidate, place_visit)
return unless candidate['latitudeE7'].present? && candidate['longitudeE7'].present?
build_point_from_location(
longitude: candidate['longitudeE7'],
latitude: candidate['latitudeE7'],
timestamp: place_visit['duration']['startTimestamp'] || place_visit['duration']['startTimestampMs'],
raw_data: place_visit
)
end
def build_point_from_location(longitude:, latitude:, timestamp:, raw_data:)
{
lonlat: "POINT(#{longitude.to_f / 10**7} #{latitude.to_f / 10**7})",
timestamp: Timestamps.parse_timestamp(timestamp),
raw_data: raw_data
}
end
end

View file

@ -0,0 +1,84 @@
# frozen_string_literal: true
class Gpx::TrackImporter
include Imports::Broadcaster
attr_reader :import, :json, :user_id
def initialize(import, user_id)
@import = import
@json = import.raw_data
@user_id = user_id
end
def call
tracks = json['gpx']['trk']
tracks_arr = tracks.is_a?(Array) ? tracks : [tracks]
points = tracks_arr.map { parse_track(_1) }.flatten.compact
points_data = points.map.with_index(1) { |point, index| prepare_point(point, index) }.compact
bulk_insert_points(points_data)
end
private
def parse_track(track)
return if track['trkseg'].blank?
segments = track['trkseg']
segments_array = segments.is_a?(Array) ? segments : [segments]
segments_array.compact.map { |segment| segment['trkpt'] }
end
def prepare_point(point, index)
return if point['lat'].blank? || point['lon'].blank? || point['time'].blank?
{
lonlat: "POINT(#{point['lon'].to_d} #{point['lat'].to_d})",
altitude: point['ele'].to_i,
timestamp: Time.parse(point['time']).to_i,
import_id: import.id,
velocity: speed(point),
raw_data: point,
user_id: user_id,
created_at: Time.current,
updated_at: Time.current
}
end
def bulk_insert_points(batch)
unique_batch = batch.uniq { |record| [record[:lonlat], record[:timestamp], record[:user_id]] }
# rubocop:disable Rails/SkipsModelValidations
Point.upsert_all(
unique_batch,
unique_by: %i[lonlat timestamp user_id],
returning: false,
on_duplicate: :skip
)
# rubocop:enable Rails/SkipsModelValidations
broadcast_import_progress(import, unique_batch.size)
rescue StandardError => e
create_notification("Failed to process GPX track: #{e.message}")
end
def create_notification(message)
Notification.create!(
user_id: user_id,
title: 'GPX Import Error',
content: message,
kind: :error
)
end
def speed(point)
return if point['extensions'].blank?
(
point.dig('extensions', 'speed') || point.dig('extensions', 'TrackPointExtension', 'speed')
).to_f.round(1)
end
end

View file

@ -1,68 +0,0 @@
# frozen_string_literal: true
class Gpx::TrackParser
include Imports::Broadcaster
attr_reader :import, :json, :user_id
def initialize(import, user_id)
@import = import
@json = import.raw_data
@user_id = user_id
end
def call
tracks = json['gpx']['trk']
tracks_arr = tracks.is_a?(Array) ? tracks : [tracks]
tracks_arr.map { parse_track(_1) }.flatten.compact.each.with_index(1) do |point, index|
create_point(point, index)
end
end
private
def parse_track(track)
return if track['trkseg'].blank?
segments = track['trkseg']
segments_array = segments.is_a?(Array) ? segments : [segments]
segments_array.compact.map { |segment| segment['trkpt'] }
end
def create_point(point, index)
return if point['lat'].blank? || point['lon'].blank? || point['time'].blank?
return if point_exists?(point)
Point.create(
latitude: point['lat'].to_d,
longitude: point['lon'].to_d,
altitude: point['ele'].to_i,
timestamp: Time.parse(point['time']).to_i,
import_id: import.id,
velocity: speed(point),
raw_data: point,
user_id:
)
broadcast_import_progress(import, index)
end
def point_exists?(point)
Point.exists?(
latitude: point['lat'].to_d,
longitude: point['lon'].to_d,
timestamp: Time.parse(point['time']).to_i,
user_id:
)
end
def speed(point)
return if point['extensions'].blank?
(
point.dig('extensions', 'speed') || point.dig('extensions', 'TrackPointExtension', 'speed')
).to_f.round(1)
end
end

View file

@ -26,8 +26,8 @@ class Imports::Create
case source
when 'google_semantic_history' then GoogleMaps::SemanticHistoryParser
when 'google_phone_takeout' then GoogleMaps::PhoneTakeoutParser
when 'owntracks' then OwnTracks::ExportParser
when 'gpx' then Gpx::TrackParser
when 'owntracks' then OwnTracks::Importer
when 'gpx' then Gpx::TrackImporter
when 'geojson' then Geojson::ImportParser
when 'immich_api', 'photoprism_api' then Photos::ImportParser
end

View file

@ -13,8 +13,7 @@ class Overland::Params
next if point[:geometry].nil? || point.dig(:properties, :timestamp).nil?
{
latitude: point[:geometry][:coordinates][1],
longitude: point[:geometry][:coordinates][0],
lonlat: "POINT(#{point[:geometry][:coordinates][0]} #{point[:geometry][:coordinates][1]})",
battery_status: point[:properties][:battery_state],
battery: battery_level(point[:properties][:battery_level]),
timestamp: DateTime.parse(point[:properties][:timestamp]),

View file

@ -1,35 +0,0 @@
# frozen_string_literal: true
class OwnTracks::ExportParser
include Imports::Broadcaster
attr_reader :import, :data, :user_id
def initialize(import, user_id)
@import = import
@data = import.raw_data
@user_id = user_id
end
def call
points_data = data.map { |point| OwnTracks::Params.new(point).call }
points_data.each.with_index(1) do |point_data, index|
next if Point.exists?(
timestamp: point_data[:timestamp],
latitude: point_data[:latitude],
longitude: point_data[:longitude],
user_id:
)
point = Point.new(point_data).tap do |p|
p.user_id = user_id
p.import_id = import.id
end
point.save
broadcast_import_progress(import, index)
end
end
end

View file

@ -0,0 +1,52 @@
# frozen_string_literal: true
class OwnTracks::Importer
include Imports::Broadcaster
attr_reader :import, :data, :user_id
def initialize(import, user_id)
@import = import
@data = import.raw_data
@user_id = user_id
end
def call
points_data = data.map.with_index(1) do |point, index|
OwnTracks::Params.new(point).call.merge(
import_id: import.id,
user_id: user_id,
created_at: Time.current,
updated_at: Time.current
)
end
bulk_insert_points(points_data)
end
private
def bulk_insert_points(batch)
unique_batch = batch.uniq { |record| [record[:lonlat], record[:timestamp], record[:user_id]] }
# rubocop:disable Rails/SkipsModelValidations
Point.upsert_all(
unique_batch,
unique_by: %i[lonlat timestamp user_id],
returning: false,
on_duplicate: :skip
)
# rubocop:enable Rails/SkipsModelValidations
rescue StandardError => e
create_notification("Failed to process OwnTracks data: #{e.message}")
end
def create_notification(message)
Notification.create!(
user_id: user_id,
title: 'OwnTracks Import Error',
content: message,
kind: :error
)
end
end

View file

@ -7,10 +7,11 @@ class OwnTracks::Params
@params = params.to_h.deep_symbolize_keys
end
# rubocop:disable Metrics/MethodLength
# rubocop:disable Metrics/AbcSize
def call
{
latitude: params[:lat],
longitude: params[:lon],
lonlat: "POINT(#{params[:lon]} #{params[:lat]})",
battery: params[:batt],
ping: params[:p],
altitude: params[:alt],
@ -30,6 +31,8 @@ class OwnTracks::Params
raw_data: params.deep_stringify_keys
}
end
# rubocop:enable Metrics/AbcSize
# rubocop:enable Metrics/MethodLength
private

View file

@ -20,8 +20,7 @@ class Photos::ImportParser
return 0 if point_exists?(point, point['timestamp'])
Point.create(
latitude: point['latitude'].to_d,
longitude: point['longitude'].to_d,
lonlat: "POINT(#{point['longitude']} #{point['latitude']})",
timestamp: point['timestamp'],
raw_data: point,
import_id: import.id,
@ -33,8 +32,7 @@ class Photos::ImportParser
def point_exists?(point, timestamp)
Point.exists?(
latitude: point['latitude'].to_d,
longitude: point['longitude'].to_d,
lonlat: "POINT(#{point['longitude']} #{point['latitude']})",
timestamp:,
user_id:
)

View file

@ -18,7 +18,7 @@ class ReverseGeocoding::Points::FetchData
private
def update_point_with_geocoding_data
response = Geocoder.search([point.latitude, point.longitude]).first
response = Geocoder.search([point.lat, point.lon]).first
return if response.blank? || response.data['error'].present?
point.update!(

View file

@ -46,7 +46,7 @@ class Stats::CalculateMonth
.tracked_points
.without_raw_data
.where(timestamp: start_timestamp..end_timestamp)
.select(:latitude, :longitude, :timestamp, :city, :country)
.select(:lonlat, :timestamp, :city, :country)
.order(timestamp: :asc)
end

View file

@ -7,7 +7,7 @@ class Tracks::BuildPath
def call
factory.line_string(
coordinates.map { |point| factory.point(point[1].to_f.round(5), point[0].to_f.round(5)) }
coordinates.map { |point| factory.point(point.lon.to_f.round(5), point.lat.to_f.round(5)) }
)
end

View file

@ -16,7 +16,9 @@ class Visits::Prepare
grouped_points = Visits::GroupPoints.new(day_points).group_points_by_radius
day_result = prepare_day_result(grouped_points)
# Iterate through the day_result, check if there are any points outside of visits that are between two consecutive visits. If there are none, merge the visits.
# Iterate through the day_result, check if there are any points outside
# of visits that are between two consecutive visits. If there are none,
# merge the visits.
day_result.each_cons(2) do |visit1, visit2|
next if visit1[:points].last == visit2[:points].first
@ -65,8 +67,8 @@ class Visits::Prepare
center_point = group.first
{
latitude: center_point.latitude,
longitude: center_point.longitude,
latitude: center_point.lat,
longitude: center_point.lon,
radius: calculate_radius(center_point, group),
points: group,
duration: (group.last.timestamp - group.first.timestamp).to_i / 60,

View file

@ -0,0 +1,13 @@
# frozen_string_literal: true
class MigratePointsLatlon < ActiveRecord::Migration[8.0]
def up
User.find_each do |user|
DataMigrations::MigratePointsLatlonJob.perform_later(user.id)
end
end
def down
raise ActiveRecord::IrreversibleMigration
end
end

View file

@ -0,0 +1,7 @@
# frozen_string_literal: true
class AddLonlatToPoints < ActiveRecord::Migration[8.0]
def change
add_column :points, :lonlat, :st_point, geographic: true
end
end

View file

@ -0,0 +1,9 @@
# frozen_string_literal: true
class AddLonlatIndex < ActiveRecord::Migration[8.0]
disable_ddl_transaction!
def change
add_index :points, :lonlat, using: :gist, algorithm: :concurrently
end
end

View file

@ -0,0 +1,28 @@
# frozen_string_literal: true
class RemovePointsLatitudeLongitudeUniquenessIndex < ActiveRecord::Migration[8.0]
disable_ddl_transaction!
def up
return unless index_exists?(
:points, %i[latitude longitude timestamp user_id],
name: 'unique_points_lat_long_timestamp_user_id_index'
)
remove_index :points,
name: 'unique_points_lat_long_timestamp_user_id_index',
algorithm: :concurrently
end
def down
return if index_exists?(
:points, %i[latitude longitude timestamp user_id],
name: 'unique_points_lat_long_timestamp_user_id_index'
)
add_index :points, %i[latitude longitude timestamp user_id],
unique: true,
name: 'unique_points_lat_long_timestamp_user_id_index',
algorithm: :concurrently
end
end

View file

@ -0,0 +1,13 @@
# frozen_string_literal: true
class AddUniqueLonLatIndexToPoints < ActiveRecord::Migration[8.0]
disable_ddl_transaction!
def change
return if index_exists?(:points, %i[lonlat timestamp user_id], name: 'index_points_on_lonlat_timestamp_user_id')
add_index :points, %i[lonlat timestamp user_id], unique: true,
name: 'index_points_on_lonlat_timestamp_user_id',
algorithm: :concurrently
end
end

6
db/schema.rb generated
View file

@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema[8.0].define(version: 2025_02_19_195822) do
ActiveRecord::Schema[8.0].define(version: 2025_02_21_194509) do
# These are extensions that must be enabled in order to support this database
enable_extension "pg_catalog.plpgsql"
enable_extension "postgis"
@ -160,6 +160,7 @@ ActiveRecord::Schema[8.0].define(version: 2025_02_19_195822) do
t.decimal "course", precision: 8, scale: 5
t.decimal "course_accuracy", precision: 8, scale: 5
t.string "external_track_id"
t.geography "lonlat", limit: {srid: 4326, type: "st_point", geographic: true}
t.index ["altitude"], name: "index_points_on_altitude"
t.index ["battery"], name: "index_points_on_battery"
t.index ["battery_status"], name: "index_points_on_battery_status"
@ -169,8 +170,9 @@ ActiveRecord::Schema[8.0].define(version: 2025_02_19_195822) do
t.index ["external_track_id"], name: "index_points_on_external_track_id"
t.index ["geodata"], name: "index_points_on_geodata", using: :gin
t.index ["import_id"], name: "index_points_on_import_id"
t.index ["latitude", "longitude", "timestamp", "user_id"], name: "unique_points_lat_long_timestamp_user_id_index", unique: true
t.index ["latitude", "longitude"], name: "index_points_on_latitude_and_longitude"
t.index ["lonlat", "timestamp", "user_id"], name: "index_points_on_lonlat_timestamp_user_id", unique: true
t.index ["lonlat"], name: "index_points_on_lonlat", using: :gist
t.index ["reverse_geocoded_at"], name: "index_points_on_reverse_geocoded_at"
t.index ["timestamp"], name: "index_points_on_timestamp"
t.index ["trigger"], name: "index_points_on_trigger"

View file

@ -29,11 +29,11 @@ FactoryBot.define do
course { nil }
course_accuracy { nil }
external_track_id { nil }
lonlat { "POINT(#{FFaker::Geolocation.lng} #{FFaker::Geolocation.lat})" }
user
trait :with_known_location do
latitude { 55.755826 }
longitude { 37.6173 }
lonlat { 'POINT(37.6173 55.755826)' }
end
trait :with_geodata do

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -0,0 +1,16 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe DataMigrations::MigratePointsLatlonJob, type: :job do
describe '#perform' do
it 'updates the lonlat column for all tracked points' do
user = create(:user)
point = create(:point, latitude: 2.0, longitude: 1.0, user: user)
expect { subject.perform(user.id) }.to change {
point.reload.lonlat
}.to(RGeo::Geographic.spherical_factory.point(1.0, 2.0))
end
end
end

View file

@ -26,7 +26,7 @@ RSpec.describe ImportJob, type: :job do
context 'when there is an error' do
before do
allow_any_instance_of(OwnTracks::ExportParser).to receive(:call).and_raise(StandardError)
allow_any_instance_of(OwnTracks::Importer).to receive(:call).and_raise(StandardError)
end
it 'does not create points' do

View file

@ -7,7 +7,7 @@ RSpec.describe Trips::CreatePathJob, type: :job do
let(:points) { trip.points }
let(:trip_path) do
"LINESTRING (#{points.map do |point|
"#{point.longitude.to_f.round(5)} #{point.latitude.to_f.round(5)}"
"#{point.lon.to_f.round(5)} #{point.lat.to_f.round(5)}"
end.join(', ')})"
end

View file

@ -9,9 +9,8 @@ RSpec.describe Point, type: :model do
end
describe 'validations' do
it { is_expected.to validate_presence_of(:latitude) }
it { is_expected.to validate_presence_of(:longitude) }
it { is_expected.to validate_presence_of(:timestamp) }
it { is_expected.to validate_presence_of(:lonlat) }
end
describe 'scopes' do
@ -63,5 +62,21 @@ RSpec.describe Point, type: :model do
end
end
end
describe '#lon' do
let(:point) { create(:point, lonlat: 'POINT(1 2)') }
it 'returns longitude' do
expect(point.lon).to eq(1)
end
end
describe '#lat' do
let(:point) { create(:point, lonlat: 'POINT(1 2)') }
it 'returns latitude' do
expect(point.lat).to eq(2)
end
end
end
end

View file

@ -25,11 +25,11 @@ RSpec.describe Stat, type: :model do
context 'when there are points' do
let!(:points) do
create(:point, user:, latitude: 1, longitude: 1, timestamp: DateTime.new(year, 1, 1, 1))
create(:point, user:, latitude: 2, longitude: 2, timestamp: DateTime.new(year, 1, 1, 2))
create(:point, user:, lonlat: 'POINT(1 1)', timestamp: DateTime.new(year, 1, 1, 1))
create(:point, user:, lonlat: 'POINT(2 2)', timestamp: DateTime.new(year, 1, 1, 2))
end
before { expected_distance[0][1] = 157.23 }
before { expected_distance[0][1] = 156.88 }
it 'returns distance by day' do
expect(subject).to eq(expected_distance)

View file

@ -6,8 +6,15 @@ RSpec.describe Api::SlimPointSerializer do
describe '#call' do
subject(:serializer) { described_class.new(point).call }
let(:point) { create(:point) }
let(:expected_json) { point.attributes.slice('id', 'latitude', 'longitude', 'timestamp') }
let!(:point) { create(:point, :with_known_location) }
let(:expected_json) do
{
id: point.id,
latitude: point.lat.to_s,
longitude: point.lon.to_s,
timestamp: point.timestamp
}
end
it 'returns JSON with correct attributes' do
expect(serializer.to_json).to eq(expected_json.to_json)

View file

@ -18,8 +18,8 @@ RSpec.describe ExportSerializer do
user_email => {
'dawarich-export' => [
{
lat: points.first.latitude,
lon: points.first.longitude,
lat: points.first.lat.to_s,
lon: points.first.lon.to_s,
bs: 'u',
batt: points.first.battery,
p: points.first.ping,
@ -39,8 +39,8 @@ RSpec.describe ExportSerializer do
raw_data: points.first.raw_data
},
{
lat: points.second.latitude,
lon: points.second.longitude,
lat: points.second.lat.to_s,
lon: points.second.lon.to_s,
bs: 'u',
batt: points.second.battery,
p: points.second.ping,

View file

@ -8,15 +8,37 @@ RSpec.describe PointSerializer do
let(:point) { create(:point) }
let(:expected_json) do
point.attributes.except(*PointSerializer::EXCLUDED_ATTRIBUTES)
{
'battery_status' => point.battery_status,
'ping' => point.ping,
'battery' => point.battery,
'tracker_id' => point.tracker_id,
'topic' => point.topic,
'altitude' => point.altitude,
'longitude' => point.lon.to_s,
'velocity' => point.velocity,
'trigger' => point.trigger,
'bssid' => point.bssid,
'ssid' => point.ssid,
'connection' => point.connection,
'vertical_accuracy' => point.vertical_accuracy,
'accuracy' => point.accuracy,
'timestamp' => point.timestamp,
'latitude' => point.lat.to_s,
'mode' => point.mode,
'inrids' => point.inrids,
'in_regions' => point.in_regions,
'city' => point.city,
'country' => point.country,
'geodata' => point.geodata,
'course' => point.course,
'course_accuracy' => point.course_accuracy,
'external_track_id' => point.external_track_id
}
end
it 'returns JSON' do
it 'returns JSON with correct attributes' do
expect(serializer.to_json).to eq(expected_json.to_json)
end
it 'does not include excluded attributes' do
expect(serializer).not_to include(*PointSerializer::EXCLUDED_ATTRIBUTES)
end
end
end

View file

@ -20,7 +20,7 @@ RSpec.describe Points::GeojsonSerializer do
type: 'Feature',
geometry: {
type: 'Point',
coordinates: [point.longitude, point.latitude]
coordinates: [point.lon.to_s, point.lat.to_s]
},
properties: PointSerializer.new(point).call
}

View file

@ -24,8 +24,8 @@ RSpec.describe Points::GpxSerializer do
serializer.tracks[0].points.each_with_index do |track_point, index|
point = points[index]
expect(track_point.lat).to eq(point.latitude)
expect(track_point.lon).to eq(point.longitude)
expect(track_point.lat.to_s).to eq(point.lat.to_s)
expect(track_point.lon.to_s).to eq(point.lon.to_s)
expect(track_point.time).to eq(point.recorded_at)
end
end

View file

@ -23,14 +23,14 @@ RSpec.describe Areas::Visits::Create do
context 'when there are points' do
let(:home_visit_date) { DateTime.new(2021, 1, 1, 10, 0, 0, Time.zone.formatted_offset) }
let!(:home_point1) { create(:point, user:, latitude: 0, longitude: 0, timestamp: home_visit_date) }
let!(:home_point2) { create(:point, user:, latitude: 0, longitude: 0, timestamp: home_visit_date + 10.minutes) }
let!(:home_point3) { create(:point, user:, latitude: 0, longitude: 0, timestamp: home_visit_date + 20.minutes) }
let!(:home_point1) { create(:point, user:, lonlat: 'POINT(0 0)', timestamp: home_visit_date) }
let!(:home_point2) { create(:point, user:, lonlat: 'POINT(0 0)', timestamp: home_visit_date + 10.minutes) }
let!(:home_point3) { create(:point, user:, lonlat: 'POINT(0 0)', timestamp: home_visit_date + 20.minutes) }
let(:work_visit_date) { DateTime.new(2021, 1, 1, 12, 0, 0, Time.zone.formatted_offset) }
let!(:work_point1) { create(:point, user:, latitude: 1, longitude: 1, timestamp: work_visit_date) }
let!(:work_point2) { create(:point, user:, latitude: 1, longitude: 1, timestamp: work_visit_date + 10.minutes) }
let!(:work_point3) { create(:point, user:, latitude: 1, longitude: 1, timestamp: work_visit_date + 20.minutes) }
let!(:work_point1) { create(:point, user:, lonlat: 'POINT(1 1)', timestamp: work_visit_date) }
let!(:work_point2) { create(:point, user:, lonlat: 'POINT(1 1)', timestamp: work_visit_date + 10.minutes) }
let!(:work_point3) { create(:point, user:, lonlat: 'POINT(1 1)', timestamp: work_visit_date + 20.minutes) }
it 'creates visits' do
expect { create_visits }.to change { Visit.count }.by(2)
@ -47,7 +47,7 @@ RSpec.describe Areas::Visits::Create do
end
context 'when there are points outside the time threshold' do
let(:home_point4) { create(:point, user:, latitude: 0, longitude: 0, timestamp: home_visit_date + 40.minutes) }
let(:home_point4) { create(:point, user:, lonlat: 'POINT(0 0)', timestamp: home_visit_date + 40.minutes) }
it 'does not create visits' do
expect { create_visits }.to change { Visit.count }.by(2)

View file

@ -18,8 +18,7 @@ RSpec.describe Geojson::Params do
it 'returns the correct data for each point' do
expect(subject.first).to eq(
latitude: '0.0',
longitude: '0.0',
lonlat: 'POINT(0.1 0.1)',
battery_status: nil,
battery: nil,
timestamp: Time.zone.at(1_609_459_201),
@ -34,8 +33,8 @@ RSpec.describe Geojson::Params do
'geometry' => {
'type' => 'Point',
'coordinates' => [
'0.0',
'0.0'
'0.1',
'0.1'
]
},
'properties' => {
@ -72,8 +71,7 @@ RSpec.describe Geojson::Params do
it 'returns the correct data for each point' do
expect(subject.first).to eq(
latitude: 10.758321212464024,
longitude: 106.64234449272531,
lonlat: 'POINT(106.64234449272531 10.758321212464024)',
battery_status: nil,
battery: nil,
timestamp: Time.parse('2024-11-03T16:30:11.331+07:00').to_i,

View file

@ -9,6 +9,7 @@ RSpec.describe GoogleMaps::PhoneTakeoutParser do
let(:user) { create(:user) }
context 'when file content is an object' do
# This file contains 3 duplicates
let(:file_path) { Rails.root.join('spec/fixtures/files/google/phone-takeout.json') }
let(:raw_data) { JSON.parse(File.read(file_path)) }
let(:import) { create(:import, user:, name: 'phone_takeout.json', raw_data:) }
@ -21,6 +22,7 @@ RSpec.describe GoogleMaps::PhoneTakeoutParser do
end
context 'when file content is an array' do
# This file contains 4 duplicates
let(:file_path) { Rails.root.join('spec/fixtures/files/google/location-history.json') }
let(:raw_data) { JSON.parse(File.read(file_path)) }
let(:import) { create(:import, user:, name: 'phone_takeout.json', raw_data:) }
@ -33,12 +35,12 @@ RSpec.describe GoogleMaps::PhoneTakeoutParser do
it 'creates points with correct data' do
parser
expect(Point.all[6].latitude).to eq(27.696576.to_d)
expect(Point.all[6].longitude).to eq(-97.376949.to_d)
expect(Point.all[6].lat).to eq(27.696576)
expect(Point.all[6].lon).to eq(-97.376949)
expect(Point.all[6].timestamp).to eq(1_693_180_140)
expect(Point.last.latitude).to eq(27.709617.to_d)
expect(Point.last.longitude).to eq(-97.375988.to_d)
expect(Point.last.lat).to eq(27.709617)
expect(Point.last.lon).to eq(-97.375988)
expect(Point.last.timestamp).to eq(1_693_180_320)
end
end

View file

@ -56,8 +56,7 @@ RSpec.describe GoogleMaps::RecordsImporter do
:point,
user: import.user,
import: import,
latitude: 12.3456789,
longitude: 12.3456789,
lonlat: 'POINT(12.3456789 12.3456789)',
timestamp: time.to_i
)
end

View file

@ -80,7 +80,7 @@ RSpec.describe GoogleMaps::SemanticHistoryParser do
{
'activitySegment' => {
'startLocation' => { 'latitudeE7' => 123_456_789, 'longitudeE7' => 123_456_789 },
'duration' => { 'startTimestamp' => (time.to_i).to_s }
'duration' => { 'startTimestamp' => time.to_i.to_s }
}
}
end

View file

@ -2,7 +2,7 @@
require 'rails_helper'
RSpec.describe Gpx::TrackParser do
RSpec.describe Gpx::TrackImporter do
describe '#call' do
subject(:parser) { described_class.new(import, user.id).call }
@ -17,7 +17,7 @@ RSpec.describe Gpx::TrackParser do
end
it 'broadcasts importing progress' do
expect_any_instance_of(Imports::Broadcaster).to receive(:broadcast_import_progress).exactly(10).times
expect_any_instance_of(Imports::Broadcaster).to receive(:broadcast_import_progress).exactly(1).time
parser
end
@ -31,7 +31,7 @@ RSpec.describe Gpx::TrackParser do
end
it 'broadcasts importing progress' do
expect_any_instance_of(Imports::Broadcaster).to receive(:broadcast_import_progress).exactly(43).times
expect_any_instance_of(Imports::Broadcaster).to receive(:broadcast_import_progress).exactly(1).time
parser
end
@ -45,7 +45,7 @@ RSpec.describe Gpx::TrackParser do
end
it 'broadcasts importing progress' do
expect_any_instance_of(Imports::Broadcaster).to receive(:broadcast_import_progress).exactly(34).times
expect_any_instance_of(Imports::Broadcaster).to receive(:broadcast_import_progress).exactly(1).time
parser
end
@ -53,8 +53,8 @@ RSpec.describe Gpx::TrackParser do
it 'creates points with correct data' do
parser
expect(Point.first.latitude).to eq(37.17221.to_d)
expect(Point.first.longitude).to eq(-3.55468.to_d)
expect(Point.first.lat).to eq(37.1722103)
expect(Point.first.lon).to eq(-3.55468)
expect(Point.first.altitude).to eq(1066)
expect(Point.first.timestamp).to eq(Time.zone.parse('2024-04-21T10:19:55Z').to_i)
expect(Point.first.velocity).to eq('2.9')
@ -67,8 +67,8 @@ RSpec.describe Gpx::TrackParser do
it 'creates points with correct data' do
parser
expect(Point.first.latitude).to eq(10.758321.to_d)
expect(Point.first.longitude).to eq(106.642344.to_d)
expect(Point.first.lat).to eq(10.758321212464024)
expect(Point.first.lon).to eq(106.64234449272531)
expect(Point.first.altitude).to eq(17)
expect(Point.first.timestamp).to eq(1_730_626_211)
expect(Point.first.velocity).to eq('2.8')

View file

@ -30,8 +30,8 @@ RSpec.describe Imports::Create do
context 'when source is owntracks' do
let(:import) { create(:import, source: 'owntracks') }
it 'calls the OwnTracks::ExportParser' do
expect(OwnTracks::ExportParser).to \
it 'calls the OwnTracks::Importer' do
expect(OwnTracks::Importer).to \
receive(:new).with(import, user.id).and_return(double(call: true))
service.call
end
@ -59,7 +59,7 @@ RSpec.describe Imports::Create do
context 'when import fails' do
before do
allow(OwnTracks::ExportParser).to receive(:new).with(import, user.id).and_raise(StandardError)
allow(OwnTracks::Importer).to receive(:new).with(import, user.id).and_raise(StandardError)
end
it 'creates a failed notification' do
@ -73,8 +73,8 @@ RSpec.describe Imports::Create do
context 'when source is gpx' do
let(:import) { create(:import, source: 'gpx') }
it 'calls the Gpx::TrackParser' do
expect(Gpx::TrackParser).to \
it 'calls the Gpx::TrackImporter' do
expect(Gpx::TrackImporter).to \
receive(:new).with(import, user.id).and_return(double(call: true))
service.call
end

View file

@ -11,8 +11,7 @@ RSpec.describe Overland::Params do
let(:expected_json) do
{
latitude: 37.3318,
longitude: -122.030581,
lonlat: 'POINT(-122.030581 37.3318)',
battery_status: 'charging',
battery: 89,
altitude: 0,
@ -31,8 +30,6 @@ RSpec.describe Overland::Params do
it 'returns a hash with the correct keys' do
expect(params[0].keys).to match_array(
%i[
latitude
longitude
battery_status
battery
altitude
@ -43,6 +40,7 @@ RSpec.describe Overland::Params do
tracker_id
timestamp
raw_data
lonlat
]
)
end

View file

@ -2,7 +2,7 @@
require 'rails_helper'
RSpec.describe OwnTracks::ExportParser do
RSpec.describe OwnTracks::Importer do
describe '#call' do
subject(:parser) { described_class.new(import, user.id).call }
@ -17,9 +17,10 @@ RSpec.describe OwnTracks::ExportParser do
it 'correctly writes attributes' do
parser
expect(Point.first.attributes).to include(
'latitude' => 52.225,
'longitude' => 13.332,
point = Point.first
expect(point.lonlat.x).to be_within(0.001).of(13.332)
expect(point.lonlat.y).to be_within(0.001).of(52.225)
expect(point.attributes.except('lonlat')).to include(
'battery_status' => 'charging',
'battery' => 94,
'ping' => '100.266',

View file

@ -13,8 +13,7 @@ RSpec.describe OwnTracks::Params do
let(:expected_json) do
{
latitude: 52.225,
longitude: 13.332,
lonlat: 'POINT(13.332 52.225)',
battery: 94,
ping: 100.266,
altitude: 36,

View file

@ -23,13 +23,13 @@ RSpec.describe Photos::ImportParser do
it 'creates points with correct attributes' do
service
expect(Point.first.latitude.to_f).to eq(59.0000)
expect(Point.first.longitude.to_f).to eq(30.0000)
expect(Point.first.lat.to_f).to eq(59.0000)
expect(Point.first.lon.to_f).to eq(30.0000)
expect(Point.first.timestamp).to eq(978_296_400)
expect(Point.first.import_id).to eq(import.id)
expect(Point.second.latitude.to_f).to eq(55.0001)
expect(Point.second.longitude.to_f).to eq(37.0001)
expect(Point.second.lat.to_f).to eq(55.0001)
expect(Point.second.lon.to_f).to eq(37.0001)
expect(Point.second.timestamp).to eq(978_296_400)
expect(Point.second.import_id).to eq(import.id)
end
@ -37,7 +37,7 @@ RSpec.describe Photos::ImportParser do
context 'when there are points with the same coordinates' do
let!(:existing_point) do
create(:point, latitude: 59.0000, longitude: 30.0000, timestamp: 978_296_400, user:)
create(:point, lonlat: 'POINT(30.0000 59.0000)', timestamp: 978_296_400, user:)
end
it 'creates only new points' do

View file

@ -27,7 +27,7 @@ RSpec.describe ReverseGeocoding::Points::FetchData do
it 'calls Geocoder' do
fetch_data
expect(Geocoder).to have_received(:search).with([point.latitude, point.longitude])
expect(Geocoder).to have_received(:search).with([point.lat, point.lon])
end
end

View file

@ -26,24 +26,21 @@ RSpec.describe Stats::CalculateMonth do
user:,
import:,
timestamp: timestamp1,
latitude: 52.107902115161316,
longitude: 14.452712811406352)
lonlat: 'POINT(14.452712811406352 52.107902115161316)')
end
let!(:point2) do
create(:point,
user:,
import:,
timestamp: timestamp2,
latitude: 51.9746598171507,
longitude: 12.291519487061901)
lonlat: 'POINT(12.291519487061901 51.9746598171507)')
end
let!(:point3) do
create(:point,
user:,
import:,
timestamp: timestamp3,
latitude: 52.72859111523629,
longitude: 9.77973105800526)
lonlat: 'POINT(9.77973105800526 52.72859111523629)')
end
context 'when units are kilometers' do
@ -56,7 +53,7 @@ RSpec.describe Stats::CalculateMonth do
it 'calculates distance' do
calculate_stats
expect(user.stats.last.distance).to eq(338)
expect(user.stats.last.distance).to eq(339)
end
context 'when there is an error' do
@ -84,7 +81,7 @@ RSpec.describe Stats::CalculateMonth do
it 'calculates distance' do
calculate_stats
expect(user.stats.last.distance).to eq(210)
expect(user.stats.last.distance).to eq(211)
end
context 'when there is an error' do

View file

@ -6,9 +6,9 @@ RSpec.describe Tracks::BuildPath do
describe '#call' do
let(:coordinates) do
[
[45.123456, -122.654321], # [lat, lng]
[45.234567, -122.765432],
[45.345678, -122.876543]
RGeo::Geographic.spherical_factory.point(-122.654321, 45.123456),
RGeo::Geographic.spherical_factory.point(-122.765432, 45.234567),
RGeo::Geographic.spherical_factory.point(-122.876543, 45.345678)
]
end
@ -26,9 +26,9 @@ RSpec.describe Tracks::BuildPath do
it 'correctly converts coordinates to points with rounded values' do
points = result.points
coordinates.each_with_index do |(lat, lng), index|
expect(points[index].x).to eq(lng.to_f.round(5))
expect(points[index].y).to eq(lat.to_f.round(5))
coordinates.each_with_index do |coordinate_pair, index|
expect(points[index].x).to eq(coordinate_pair.lon.to_f.round(5))
expect(points[index].y).to eq(coordinate_pair.lat.to_f.round(5))
end
end
end

View file

@ -6,17 +6,17 @@ RSpec.describe Visits::GroupPoints do
describe '#group_points_by_radius' do
it 'groups points by radius' do
day_points = [
build(:point, latitude: 0, longitude: 0, timestamp: 1.day.ago),
build(:point, latitude: 0.00001, longitude: 0.00001, timestamp: 1.day.ago + 1.minute),
build(:point, latitude: 0.00002, longitude: 0.00002, timestamp: 1.day.ago + 2.minutes),
build(:point, latitude: 0.00003, longitude: 0.00003, timestamp: 1.day.ago + 3.minutes),
build(:point, latitude: 0.00004, longitude: 0.00004, timestamp: 1.day.ago + 4.minutes),
build(:point, latitude: 0.00005, longitude: 0.00005, timestamp: 1.day.ago + 5.minutes),
build(:point, latitude: 0.00006, longitude: 0.00006, timestamp: 1.day.ago + 6.minutes),
build(:point, latitude: 0.00007, longitude: 0.00007, timestamp: 1.day.ago + 7.minutes),
build(:point, latitude: 0.00008, longitude: 0.00008, timestamp: 1.day.ago + 8.minutes),
build(:point, latitude: 0.00009, longitude: 0.00009, timestamp: 1.day.ago + 9.minutes),
build(:point, latitude: 0.0001, longitude: 0.0009, timestamp: 1.day.ago + 9.minutes)
build(:point, lonlat: 'POINT(0 0)', timestamp: 1.day.ago),
build(:point, lonlat: 'POINT(0.00001 0.00001)', timestamp: 1.day.ago + 1.minute),
build(:point, lonlat: 'POINT(0.00002 0.00002)', timestamp: 1.day.ago + 2.minutes),
build(:point, lonlat: 'POINT(0.00003 0.00003)', timestamp: 1.day.ago + 3.minutes),
build(:point, lonlat: 'POINT(0.00004 0.00004)', timestamp: 1.day.ago + 4.minutes),
build(:point, lonlat: 'POINT(0.00005 0.00005)', timestamp: 1.day.ago + 5.minutes),
build(:point, lonlat: 'POINT(0.00006 0.00006)', timestamp: 1.day.ago + 6.minutes),
build(:point, lonlat: 'POINT(0.00007 0.00007)', timestamp: 1.day.ago + 7.minutes),
build(:point, lonlat: 'POINT(0.00008 0.00008)', timestamp: 1.day.ago + 8.minutes),
build(:point, lonlat: 'POINT(0.00009 0.00009)', timestamp: 1.day.ago + 9.minutes),
build(:point, lonlat: 'POINT(0.001 0.001)', timestamp: 1.day.ago + 10.minutes)
]
grouped_points = described_class.new(day_points).group_points_by_radius

View file

@ -14,21 +14,21 @@ RSpec.describe Visits::Group do
context 'when points are too far apart' do
it 'groups points into separate visits' do
points = [
build(:point, latitude: 0, longitude: 0, timestamp: 1.day.ago),
build(:point, latitude: 0.00001, longitude: 0.00001, timestamp: 1.day.ago + 5.minutes),
build(:point, latitude: 0.00002, longitude: 0.00002, timestamp: 1.day.ago + 10.minutes),
build(:point, latitude: 0.00003, longitude: 0.00003, timestamp: 1.day.ago + 15.minutes),
build(:point, latitude: 0.00004, longitude: 0.00004, timestamp: 1.day.ago + 20.minutes),
build(:point, latitude: 0.00005, longitude: 0.00005, timestamp: 1.day.ago + 25.minutes),
build(:point, latitude: 0.00006, longitude: 0.00006, timestamp: 1.day.ago + 30.minutes),
build(:point, latitude: 0.00007, longitude: 0.00007, timestamp: 1.day.ago + 35.minutes),
build(:point, latitude: 0.00008, longitude: 0.00008, timestamp: 1.day.ago + 40.minutes),
build(:point, latitude: 0.00009, longitude: 0.00009, timestamp: 1.day.ago + 45.minutes),
build(:point, latitude: 0.0001, longitude: 0.0001, timestamp: 1.day.ago + 50.minutes),
build(:point, latitude: 0.00011, longitude: 0.00011, timestamp: 1.day.ago + 55.minutes),
build(:point, latitude: 0.00011, longitude: 0.00011, timestamp: 1.day.ago + 95.minutes),
build(:point, latitude: 0.00011, longitude: 0.00011, timestamp: 1.day.ago + 100.minutes),
build(:point, latitude: 0.00011, longitude: 0.00011, timestamp: 1.day.ago + 105.minutes)
build(:point, lonlat: 'POINT(0 0)', timestamp: 1.day.ago),
build(:point, lonlat: 'POINT(0.00001 0.00001)', timestamp: 1.day.ago + 5.minutes),
build(:point, lonlat: 'POINT(0.00002 0.00002)', timestamp: 1.day.ago + 10.minutes),
build(:point, lonlat: 'POINT(0.00003 0.00003)', timestamp: 1.day.ago + 15.minutes),
build(:point, lonlat: 'POINT(0.00004 0.00004)', timestamp: 1.day.ago + 20.minutes),
build(:point, lonlat: 'POINT(0.00005 0.00005)', timestamp: 1.day.ago + 25.minutes),
build(:point, lonlat: 'POINT(0.00006 0.00006)', timestamp: 1.day.ago + 30.minutes),
build(:point, lonlat: 'POINT(0.00007 0.00007)', timestamp: 1.day.ago + 35.minutes),
build(:point, lonlat: 'POINT(0.00008 0.00008)', timestamp: 1.day.ago + 40.minutes),
build(:point, lonlat: 'POINT(0.00009 0.00009)', timestamp: 1.day.ago + 45.minutes),
build(:point, lonlat: 'POINT(0.0001 0.0001)', timestamp: 1.day.ago + 50.minutes),
build(:point, lonlat: 'POINT(0.00011 0.00011)', timestamp: 1.day.ago + 55.minutes),
build(:point, lonlat: 'POINT(0.00011 0.00011)', timestamp: 1.day.ago + 95.minutes),
build(:point, lonlat: 'POINT(0.00011 0.00011)', timestamp: 1.day.ago + 100.minutes),
build(:point, lonlat: 'POINT(0.00011 0.00011)', timestamp: 1.day.ago + 105.minutes)
]
expect(group.call(points)).to \
eq({

View file

@ -7,21 +7,21 @@ RSpec.describe Visits::Prepare do
let(:static_time) { Time.zone.local(2021, 1, 1, 0, 0, 0) }
let(:points) do
[
build(:point, latitude: 0, longitude: 0, timestamp: static_time),
build(:point, latitude: 0.00001, longitude: 0.00001, timestamp: static_time + 5.minutes),
build(:point, latitude: 0.00002, longitude: 0.00002, timestamp: static_time + 10.minutes),
build(:point, latitude: 0.00003, longitude: 0.00003, timestamp: static_time + 15.minutes),
build(:point, latitude: 0.00004, longitude: 0.00004, timestamp: static_time + 20.minutes),
build(:point, latitude: 0.00005, longitude: 0.00005, timestamp: static_time + 25.minutes),
build(:point, latitude: 0.00006, longitude: 0.00006, timestamp: static_time + 30.minutes),
build(:point, latitude: 0.00007, longitude: 0.00007, timestamp: static_time + 35.minutes),
build(:point, latitude: 0.00008, longitude: 0.00008, timestamp: static_time + 40.minutes),
build(:point, latitude: 0.00009, longitude: 0.00009, timestamp: static_time + 45.minutes),
build(:point, latitude: 0.0001, longitude: 0.0001, timestamp: static_time + 50.minutes),
build(:point, latitude: 0.00011, longitude: 0.00011, timestamp: static_time + 55.minutes),
build(:point, latitude: 0.00011, longitude: 0.00011, timestamp: static_time + 95.minutes),
build(:point, latitude: 0.00011, longitude: 0.00011, timestamp: static_time + 100.minutes),
build(:point, latitude: 0.00011, longitude: 0.00011, timestamp: static_time + 105.minutes)
build(:point, lonlat: 'POINT(0 0)', timestamp: static_time),
build(:point, lonlat: 'POINT(0.00001 0.00001)', timestamp: static_time + 5.minutes),
build(:point, lonlat: 'POINT(0.00002 0.00002)', timestamp: static_time + 10.minutes),
build(:point, lonlat: 'POINT(0.00003 0.00003)', timestamp: static_time + 15.minutes),
build(:point, lonlat: 'POINT(0.00004 0.00004)', timestamp: static_time + 20.minutes),
build(:point, lonlat: 'POINT(0.00005 0.00005)', timestamp: static_time + 25.minutes),
build(:point, lonlat: 'POINT(0.00006 0.00006)', timestamp: static_time + 30.minutes),
build(:point, lonlat: 'POINT(0.00007 0.00007)', timestamp: static_time + 35.minutes),
build(:point, lonlat: 'POINT(0.00008 0.00008)', timestamp: static_time + 40.minutes),
build(:point, lonlat: 'POINT(0.00009 0.00009)', timestamp: static_time + 45.minutes),
build(:point, lonlat: 'POINT(0.0001 0.0001)', timestamp: static_time + 50.minutes),
build(:point, lonlat: 'POINT(0.00011 0.00011)', timestamp: static_time + 55.minutes),
build(:point, lonlat: 'POINT(0.00011 0.00011)', timestamp: static_time + 95.minutes),
build(:point, lonlat: 'POINT(0.00011 0.00011)', timestamp: static_time + 100.minutes),
build(:point, lonlat: 'POINT(0.00011 0.00011)', timestamp: static_time + 105.minutes)
]
end