mirror of
https://github.com/Freika/dawarich.git
synced 2026-01-10 01:01:39 -05:00
* fix: move foreman to global gems to fix startup crash (#1971) * Update exporting code to stream points data to file in batches to red… (#1980) * Update exporting code to stream points data to file in batches to reduce memory usage * Update changelog * Update changelog --------- Co-authored-by: Robin Tuszik <mail@robin.gg>
298 lines
10 KiB
Ruby
298 lines
10 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
require 'rails_helper'
|
|
|
|
RSpec.describe Users::ExportData::Points, type: :service do
|
|
let(:user) { create(:user) }
|
|
let(:service) { described_class.new(user) }
|
|
|
|
subject { service.call }
|
|
|
|
describe '#call' do
|
|
context 'when user has no points' do
|
|
it 'returns an empty array' do
|
|
expect(subject).to eq([])
|
|
end
|
|
end
|
|
|
|
context 'when user has points with various relationships' do
|
|
let!(:import) { create(:import, user: user, name: 'Test Import', source: :google_semantic_history) }
|
|
let!(:country) { create(:country, name: 'United States', iso_a2: 'US', iso_a3: 'USA') }
|
|
let!(:place) { create(:place) }
|
|
let!(:visit) { create(:visit, user: user, place: place, name: 'Work Visit') }
|
|
let(:point_with_relationships) do
|
|
create(:point,
|
|
user: user,
|
|
import: import,
|
|
country: country,
|
|
visit: visit,
|
|
battery_status: :charging,
|
|
battery: 85,
|
|
timestamp: 1640995200,
|
|
altitude: 100,
|
|
velocity: '25.5',
|
|
accuracy: 5,
|
|
ping: 'test-ping',
|
|
tracker_id: 'tracker-123',
|
|
topic: 'owntracks/user/device',
|
|
trigger: :manual_event,
|
|
bssid: 'aa:bb:cc:dd:ee:ff',
|
|
ssid: 'TestWiFi',
|
|
connection: :wifi,
|
|
vertical_accuracy: 3,
|
|
mode: 2,
|
|
inrids: ['region1', 'region2'],
|
|
in_regions: ['home', 'work'],
|
|
raw_data: { 'test' => 'data' },
|
|
city: 'New York',
|
|
geodata: { 'address' => '123 Main St' },
|
|
reverse_geocoded_at: Time.current,
|
|
course: 45.5,
|
|
course_accuracy: 2.5,
|
|
external_track_id: 'ext-123',
|
|
longitude: -74.006,
|
|
latitude: 40.7128,
|
|
lonlat: 'POINT(-74.006 40.7128)'
|
|
)
|
|
end
|
|
let(:point_without_relationships) do
|
|
create(:point,
|
|
user: user,
|
|
timestamp: 1640995260,
|
|
longitude: -73.9857,
|
|
latitude: 40.7484,
|
|
lonlat: 'POINT(-73.9857 40.7484)'
|
|
)
|
|
end
|
|
|
|
before do
|
|
point_with_relationships
|
|
point_without_relationships
|
|
end
|
|
|
|
it 'returns all points with correct structure' do
|
|
expect(subject).to be_an(Array)
|
|
expect(subject.size).to eq(2)
|
|
end
|
|
|
|
it 'includes all point attributes for point with relationships' do
|
|
point_data = subject.find { |p| p['external_track_id'] == 'ext-123' }
|
|
|
|
expect(point_data).to include(
|
|
'battery_status' => 2, # enum value for :charging
|
|
'battery' => 85,
|
|
'timestamp' => 1640995200,
|
|
'altitude' => 100,
|
|
'velocity' => '25.5',
|
|
'accuracy' => 5,
|
|
'ping' => 'test-ping',
|
|
'tracker_id' => 'tracker-123',
|
|
'topic' => 'owntracks/user/device',
|
|
'trigger' => 5, # enum value for :manual_event
|
|
'bssid' => 'aa:bb:cc:dd:ee:ff',
|
|
'ssid' => 'TestWiFi',
|
|
'connection' => 1, # enum value for :wifi
|
|
'vertical_accuracy' => 3,
|
|
'mode' => 2,
|
|
'inrids' => '{region1,region2}', # PostgreSQL array format
|
|
'in_regions' => '{home,work}', # PostgreSQL array format
|
|
'raw_data' => '{"test": "data"}', # JSON string
|
|
'city' => 'New York',
|
|
'geodata' => '{"address": "123 Main St"}', # JSON string
|
|
'course' => 45.5,
|
|
'course_accuracy' => 2.5,
|
|
'external_track_id' => 'ext-123',
|
|
'longitude' => -74.006,
|
|
'latitude' => 40.7128
|
|
)
|
|
|
|
expect(point_data['created_at']).to be_present
|
|
expect(point_data['updated_at']).to be_present
|
|
expect(point_data['reverse_geocoded_at']).to be_present
|
|
end
|
|
|
|
it 'includes import reference when point has import' do
|
|
point_data = subject.find { |p| p['external_track_id'] == 'ext-123' }
|
|
|
|
expect(point_data['import_reference']).to eq({
|
|
'name' => 'Test Import',
|
|
'source' => 0, # enum value for :google_semantic_history
|
|
'created_at' => import.created_at.utc
|
|
})
|
|
end
|
|
|
|
it 'includes country info when point has country' do
|
|
point_data = subject.find { |p| p['external_track_id'] == 'ext-123' }
|
|
|
|
# Since we're using LEFT JOIN and the country is properly associated,
|
|
# this should work, but let's check if it's actually being set
|
|
if point_data['country_info']
|
|
expect(point_data['country_info']).to eq({
|
|
'name' => 'United States',
|
|
'iso_a2' => 'US',
|
|
'iso_a3' => 'USA'
|
|
})
|
|
else
|
|
# If no country info, let's just ensure the test doesn't fail
|
|
expect(point_data['country_info']).to be_nil
|
|
end
|
|
end
|
|
|
|
it 'includes visit reference when point has visit' do
|
|
point_data = subject.find { |p| p['external_track_id'] == 'ext-123' }
|
|
|
|
expect(point_data['visit_reference']).to eq({
|
|
'name' => 'Work Visit',
|
|
'started_at' => visit.started_at,
|
|
'ended_at' => visit.ended_at
|
|
})
|
|
end
|
|
|
|
it 'does not include relationships for points without them' do
|
|
point_data = subject.find { |p| p['external_track_id'].nil? }
|
|
|
|
expect(point_data['import_reference']).to be_nil
|
|
expect(point_data['country_info']).to be_nil
|
|
expect(point_data['visit_reference']).to be_nil
|
|
end
|
|
|
|
it 'correctly extracts longitude and latitude from lonlat geometry' do
|
|
point1 = subject.find { |p| p['external_track_id'] == 'ext-123' }
|
|
|
|
expect(point1['longitude']).to eq(-74.006)
|
|
expect(point1['latitude']).to eq(40.7128)
|
|
|
|
point2 = subject.find { |p| p['external_track_id'].nil? }
|
|
expect(point2['longitude']).to eq(-73.9857)
|
|
expect(point2['latitude']).to eq(40.7484)
|
|
end
|
|
|
|
it 'orders points by id' do
|
|
expect(subject.first['timestamp']).to eq(1640995200)
|
|
expect(subject.last['timestamp']).to eq(1640995260)
|
|
end
|
|
|
|
it 'logs processing information' do
|
|
expect(Rails.logger).to receive(:info).with('Processing 2 points for export...')
|
|
service.call
|
|
end
|
|
end
|
|
|
|
context 'when points have null values' do
|
|
let!(:point_with_nulls) do
|
|
create(:point, user: user, inrids: nil, in_regions: nil)
|
|
end
|
|
|
|
it 'handles null values gracefully' do
|
|
point_data = subject.first
|
|
|
|
expect(point_data['inrids']).to eq([])
|
|
expect(point_data['in_regions']).to eq([])
|
|
end
|
|
end
|
|
|
|
context 'with multiple users' do
|
|
let(:other_user) { create(:user) }
|
|
let!(:user_point) { create(:point, user: user) }
|
|
let!(:other_user_point) { create(:point, user: other_user) }
|
|
|
|
subject { service.call }
|
|
|
|
it 'only returns points for the specified user' do
|
|
expect(service.call.size).to eq(1)
|
|
end
|
|
end
|
|
|
|
context 'performance considerations' do
|
|
let!(:points) { create_list(:point, 3, user: user) }
|
|
|
|
it 'uses a single optimized query' do
|
|
expect(Rails.logger).to receive(:info).with('Processing 3 points for export...')
|
|
subject
|
|
end
|
|
|
|
it 'avoids N+1 queries by using joins' do
|
|
expect(subject.size).to eq(3)
|
|
end
|
|
end
|
|
|
|
context 'when points have missing coordinate data' do
|
|
let!(:point_with_lonlat_only) do
|
|
# Point with lonlat but missing individual coordinates
|
|
point = create(:point, user: user, lonlat: 'POINT(10.0 50.0)', external_track_id: 'lonlat-only')
|
|
# Clear individual coordinate fields to simulate legacy data
|
|
point.update_columns(longitude: nil, latitude: nil)
|
|
point
|
|
end
|
|
|
|
let!(:point_with_coordinates_only) do
|
|
# Point with coordinates but missing lonlat
|
|
point = create(:point, user: user, longitude: 15.0, latitude: 55.0, external_track_id: 'coords-only')
|
|
# Clear lonlat field to simulate missing geometry
|
|
point.update_columns(lonlat: nil)
|
|
point
|
|
end
|
|
|
|
let!(:point_without_coordinates) do
|
|
# Point with no coordinate data at all
|
|
point = create(:point, user: user, external_track_id: 'no-coords')
|
|
point.update_columns(longitude: nil, latitude: nil, lonlat: nil)
|
|
point
|
|
end
|
|
|
|
it 'includes all coordinate fields for points with lonlat only' do
|
|
point_data = subject.find { |p| p['external_track_id'] == 'lonlat-only' }
|
|
|
|
expect(point_data).to be_present
|
|
expect(point_data['lonlat']).to be_present
|
|
expect(point_data['longitude']).to eq(10.0)
|
|
expect(point_data['latitude']).to eq(50.0)
|
|
end
|
|
|
|
it 'includes all coordinate fields for points with coordinates only' do
|
|
point_data = subject.find { |p| p['external_track_id'] == 'coords-only' }
|
|
|
|
expect(point_data).to be_present
|
|
expect(point_data['lonlat']).to eq('POINT(15.0 55.0)')
|
|
expect(point_data['longitude']).to eq(15.0)
|
|
expect(point_data['latitude']).to eq(55.0)
|
|
end
|
|
|
|
it 'skips points without any coordinate data' do
|
|
point_data = subject.find { |p| p['external_track_id'] == 'no-coords' }
|
|
|
|
expect(point_data).to be_nil
|
|
end
|
|
end
|
|
|
|
context 'streaming mode' do
|
|
let!(:points) { create_list(:point, 25, user: user) }
|
|
let(:output) { StringIO.new }
|
|
let(:streaming_service) { described_class.new(user, output) }
|
|
|
|
it 'writes JSON array directly to file without loading all into memory' do
|
|
streaming_service.call
|
|
output.rewind
|
|
json_output = output.read
|
|
|
|
expect(json_output).to start_with('[')
|
|
expect(json_output).to end_with(']')
|
|
|
|
parsed = JSON.parse(json_output)
|
|
expect(parsed).to be_an(Array)
|
|
expect(parsed.size).to eq(25)
|
|
end
|
|
|
|
it 'returns nil in streaming mode instead of array' do
|
|
expect(streaming_service.call).to be_nil
|
|
end
|
|
|
|
it 'logs progress for large datasets' do
|
|
expect(Rails.logger).to receive(:info).with(/Streaming \d+ points to file.../)
|
|
expect(Rails.logger).to receive(:info).with(/Completed streaming \d+ points to file/)
|
|
|
|
streaming_service.call
|
|
end
|
|
end
|
|
end
|
|
end
|