mirror of
https://github.com/Freika/dawarich.git
synced 2026-01-13 02:31:39 -05:00
Merge pull request #539 from Eduard-Gimaev/import_google_formats
feat: added google_phone_takeout and google_semantic_histo into impor…
This commit is contained in:
Evgenii Burmakin
GitHub
commit
3644a4f7b2
11 changed files with 538 additions and 1222 deletions
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -52,6 +52,9 @@
|
|||
.DS_Store
|
||||
.env
|
||||
|
||||
.byebug_history
|
||||
|
||||
|
||||
.devcontainer/.onCreateCommandMarker
|
||||
.devcontainer/.postCreateCommandMarker
|
||||
.devcontainer/.updateContentCommandMarker
|
||||
|
|
|
|||
|
|
@ -4,9 +4,11 @@ class ImportJob < ApplicationJob
|
|||
queue_as :imports
|
||||
|
||||
def perform(user_id, import_id)
|
||||
|
||||
user = User.find(user_id)
|
||||
import = user.imports.find(import_id)
|
||||
|
||||
import.process!
|
||||
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -4,16 +4,19 @@ class Imports::Watcher
|
|||
class UnsupportedSourceError < StandardError; end
|
||||
|
||||
WATCHED_DIR_PATH = Rails.root.join('tmp/imports/watched')
|
||||
SUPPORTED_FORMATS = %w[.gpx .json .rec].freeze
|
||||
|
||||
def call
|
||||
user_directories.each do |user_email|
|
||||
user = User.find_by(email: user_email)
|
||||
next unless user
|
||||
puts "Processing directory for user: #{user.email}"
|
||||
|
||||
user_directory_path = File.join(WATCHED_DIR_PATH, user_email)
|
||||
file_names = file_names(user_directory_path)
|
||||
|
||||
file_names.each do |file_name|
|
||||
puts "Processing file: #{file_name}"
|
||||
process_file(user, user_directory_path, file_name)
|
||||
end
|
||||
end
|
||||
|
|
@ -36,7 +39,7 @@ class Imports::Watcher
|
|||
|
||||
def file_names(directory_path)
|
||||
Dir.entries(directory_path).select do |file|
|
||||
['.gpx', '.json'].include?(File.extname(file))
|
||||
SUPPORTED_FORMATS.include?(File.extname(file))
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -50,8 +53,11 @@ class Imports::Watcher
|
|||
import.raw_data = raw_data(file_path, import.source)
|
||||
|
||||
import.save!
|
||||
puts "Import saved for file: #{file_name}"
|
||||
|
||||
|
||||
ImportJob.perform_later(user.id, import.id)
|
||||
puts "ImportJob enqueued for user_id: #{user.id}, import_id: #{import.id}"
|
||||
end
|
||||
|
||||
def find_or_initialize_import(user, file_name)
|
||||
|
|
@ -72,9 +78,19 @@ class Imports::Watcher
|
|||
end
|
||||
|
||||
def source(file_name)
|
||||
case file_name.split('.').last
|
||||
when 'json' then :geojson
|
||||
when 'gpx' then :gpx
|
||||
case file_name.split('.').last.downcase
|
||||
when 'json'
|
||||
if file_name.match?(/location-history/i)
|
||||
:google_phone_takeout
|
||||
elsif file_name.match?(/Records/i)
|
||||
:google_records
|
||||
elsif file_name.match?(/\d{4}_\w+/i)
|
||||
:google_semantic_history
|
||||
else
|
||||
:geojson
|
||||
end
|
||||
when 'rec' then :owntracks
|
||||
when 'gpx' then :gpx
|
||||
else raise UnsupportedSourceError, 'Unsupported source '
|
||||
end
|
||||
end
|
||||
|
|
@ -82,6 +98,15 @@ class Imports::Watcher
|
|||
def raw_data(file_path, source)
|
||||
file = File.read(file_path)
|
||||
|
||||
source.to_sym == :gpx ? Hash.from_xml(file) : JSON.parse(file)
|
||||
case source.to_sym
|
||||
when :gpx
|
||||
Hash.from_xml(file)
|
||||
when :json, :geojson, :google_phone_takeout, :google_records, :google_semantic_history
|
||||
JSON.parse(file)
|
||||
when :owntracks
|
||||
OwnTracks::RecParser.new(file).call
|
||||
else
|
||||
raise UnsupportedSourceError, "Unsupported source: #{source}"
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
89
spec/fixtures/files/watched/invalid_user@domain.com/location-history.json
vendored
Normal file
89
spec/fixtures/files/watched/invalid_user@domain.com/location-history.json
vendored
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
[
|
||||
{
|
||||
"endTime": "2023-08-27T17:04:26.999-05:00",
|
||||
"startTime": "2023-08-27T15:48:56.000-05:00",
|
||||
"visit": {
|
||||
"hierarchyLevel": "0",
|
||||
"topCandidate": {
|
||||
"probability": "0.785181",
|
||||
"semanticType": "Unknown",
|
||||
"placeID": "ChIJxxP_Qwb2aIYRTwDNDLkUmD0",
|
||||
"placeLocation": "geo:27.720022,-97.347951"
|
||||
},
|
||||
"probability": "0.710000"
|
||||
}
|
||||
},
|
||||
{
|
||||
"endTime": "2023-08-27T22:00:00.000Z",
|
||||
"startTime": "2023-08-27T20:00:00.000Z",
|
||||
"timelinePath": [
|
||||
{
|
||||
"point": "geo:27.720007,-97.348044",
|
||||
"durationMinutesOffsetFromStartTime": "49"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"endTime": "2023-09-02T23:25:59.000-06:00",
|
||||
"startTime": "2023-08-27T14:48:56.000-06:00",
|
||||
"timelineMemory": {
|
||||
"destinations": [
|
||||
{
|
||||
"identifier": "ChIJs9KSYYBfaIYRj5AOiZNQ0a4"
|
||||
},
|
||||
{
|
||||
"identifier": "ChIJw6lCfj2sZ4YRl6q2LNNyojk"
|
||||
},
|
||||
{
|
||||
"identifier": "ChIJA89FstRIAYcRr9I2aBzR89A"
|
||||
},
|
||||
{
|
||||
"identifier": "ChIJtWVg4r5DFIcRr0zkOeDPEfY"
|
||||
}
|
||||
],
|
||||
"distanceFromOriginKms": "1594"
|
||||
}
|
||||
},
|
||||
{
|
||||
"endTime": "2023-08-28T00:00:00.000Z",
|
||||
"startTime": "2023-08-27T22:00:00.000Z",
|
||||
"timelinePath": [
|
||||
{
|
||||
"point": "geo:27.701123,-97.362988",
|
||||
"durationMinutesOffsetFromStartTime": "4"
|
||||
},
|
||||
{
|
||||
"point": "geo:27.701123,-97.362988",
|
||||
"durationMinutesOffsetFromStartTime": "4"
|
||||
},
|
||||
{
|
||||
"point": "geo:27.687173,-97.363743",
|
||||
"durationMinutesOffsetFromStartTime": "7"
|
||||
},
|
||||
{
|
||||
"point": "geo:27.686129,-97.381865",
|
||||
"durationMinutesOffsetFromStartTime": "10"
|
||||
},
|
||||
{
|
||||
"point": "geo:27.686129,-97.381865",
|
||||
"durationMinutesOffsetFromStartTime": "10"
|
||||
},
|
||||
{
|
||||
"point": "geo:27.686129,-97.381865",
|
||||
"durationMinutesOffsetFromStartTime": "108"
|
||||
},
|
||||
{
|
||||
"point": "geo:27.696576,-97.376949",
|
||||
"durationMinutesOffsetFromStartTime": "109"
|
||||
},
|
||||
{
|
||||
"point": "geo:27.709617,-97.375988",
|
||||
"durationMinutesOffsetFromStartTime": "112"
|
||||
},
|
||||
{
|
||||
"point": "geo:27.709617,-97.375988",
|
||||
"durationMinutesOffsetFromStartTime": "112"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
70
spec/fixtures/files/watched/user@domain.com/2023_January.json
vendored
Normal file
70
spec/fixtures/files/watched/user@domain.com/2023_January.json
vendored
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
{
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [
|
||||
14.3439906,
|
||||
50.0506312
|
||||
]
|
||||
},
|
||||
"properties": {
|
||||
"timestamp": "2023-01-01T08:00:00Z"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [
|
||||
14.3439906,
|
||||
50.0506312
|
||||
]
|
||||
},
|
||||
"properties": {
|
||||
"timestamp": "2023-01-01T10:00:00Z"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [
|
||||
14.42076,
|
||||
50.08804
|
||||
]
|
||||
},
|
||||
"properties": {
|
||||
"timestamp": "2023-01-02T12:00:00Z"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [
|
||||
14.42076,
|
||||
50.08804
|
||||
]
|
||||
},
|
||||
"properties": {
|
||||
"timestamp": "2023-01-02T14:00:00Z"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [
|
||||
14.42076,
|
||||
50.08804
|
||||
]
|
||||
},
|
||||
"properties": {
|
||||
"timestamp": "2023-01-02T16:00:00Z"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
40
spec/fixtures/files/watched/user@domain.com/Records.json
vendored
Normal file
40
spec/fixtures/files/watched/user@domain.com/Records.json
vendored
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
{
|
||||
"locations": [
|
||||
{
|
||||
"latitudeE7": 533690550,
|
||||
"longitudeE7": 836950010,
|
||||
"accuracy": 150,
|
||||
"source": "UNKNOWN",
|
||||
"timestamp": "2012-12-15T14:21:29.460Z"
|
||||
},
|
||||
{
|
||||
"latitudeE7": 533563380,
|
||||
"longitudeE7": 837616500,
|
||||
"accuracy": 18000,
|
||||
"source": "UNKNOWN",
|
||||
"timestamp": "2013-01-04T10:22:43.225Z"
|
||||
},
|
||||
{
|
||||
"latitudeE7": 533690589,
|
||||
"longitudeE7": 836951347,
|
||||
"accuracy": 22,
|
||||
"source": "WIFI",
|
||||
"deviceTag": 1184882232,
|
||||
"timestamp": "2013-03-01T05:17:39.849Z"
|
||||
},
|
||||
{
|
||||
"latitudeE7": 533700000,
|
||||
"longitudeE7": 836960000,
|
||||
"accuracy": 50,
|
||||
"source": "GPS",
|
||||
"timestamp": "2013-04-01T12:00:00.000Z"
|
||||
},
|
||||
{
|
||||
"latitudeE7": 533710000,
|
||||
"longitudeE7": 836970000,
|
||||
"accuracy": 30,
|
||||
"source": "GPS",
|
||||
"timestamp": "2013-05-01T08:30:00.000Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load diff
89
spec/fixtures/files/watched/user@domain.com/location-history.json
vendored
Normal file
89
spec/fixtures/files/watched/user@domain.com/location-history.json
vendored
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
[
|
||||
{
|
||||
"endTime": "2023-08-27T17:04:26.999-05:00",
|
||||
"startTime": "2023-08-27T15:48:56.000-05:00",
|
||||
"visit": {
|
||||
"hierarchyLevel": "0",
|
||||
"topCandidate": {
|
||||
"probability": "0.785181",
|
||||
"semanticType": "Unknown",
|
||||
"placeID": "ChIJxxP_Qwb2aIYRTwDNDLkUmD0",
|
||||
"placeLocation": "geo:27.720022,-97.347951"
|
||||
},
|
||||
"probability": "0.710000"
|
||||
}
|
||||
},
|
||||
{
|
||||
"endTime": "2023-08-27T22:00:00.000Z",
|
||||
"startTime": "2023-08-27T20:00:00.000Z",
|
||||
"timelinePath": [
|
||||
{
|
||||
"point": "geo:27.720007,-97.348044",
|
||||
"durationMinutesOffsetFromStartTime": "49"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"endTime": "2023-09-02T23:25:59.000-06:00",
|
||||
"startTime": "2023-08-27T14:48:56.000-06:00",
|
||||
"timelineMemory": {
|
||||
"destinations": [
|
||||
{
|
||||
"identifier": "ChIJs9KSYYBfaIYRj5AOiZNQ0a4"
|
||||
},
|
||||
{
|
||||
"identifier": "ChIJw6lCfj2sZ4YRl6q2LNNyojk"
|
||||
},
|
||||
{
|
||||
"identifier": "ChIJA89FstRIAYcRr9I2aBzR89A"
|
||||
},
|
||||
{
|
||||
"identifier": "ChIJtWVg4r5DFIcRr0zkOeDPEfY"
|
||||
}
|
||||
],
|
||||
"distanceFromOriginKms": "1594"
|
||||
}
|
||||
},
|
||||
{
|
||||
"endTime": "2023-08-28T00:00:00.000Z",
|
||||
"startTime": "2023-08-27T22:00:00.000Z",
|
||||
"timelinePath": [
|
||||
{
|
||||
"point": "geo:27.701123,-97.362988",
|
||||
"durationMinutesOffsetFromStartTime": "4"
|
||||
},
|
||||
{
|
||||
"point": "geo:27.701123,-97.362988",
|
||||
"durationMinutesOffsetFromStartTime": "4"
|
||||
},
|
||||
{
|
||||
"point": "geo:27.687173,-97.363743",
|
||||
"durationMinutesOffsetFromStartTime": "7"
|
||||
},
|
||||
{
|
||||
"point": "geo:27.686129,-97.381865",
|
||||
"durationMinutesOffsetFromStartTime": "10"
|
||||
},
|
||||
{
|
||||
"point": "geo:27.686129,-97.381865",
|
||||
"durationMinutesOffsetFromStartTime": "10"
|
||||
},
|
||||
{
|
||||
"point": "geo:27.686129,-97.381865",
|
||||
"durationMinutesOffsetFromStartTime": "108"
|
||||
},
|
||||
{
|
||||
"point": "geo:27.696576,-97.376949",
|
||||
"durationMinutesOffsetFromStartTime": "109"
|
||||
},
|
||||
{
|
||||
"point": "geo:27.709617,-97.375988",
|
||||
"durationMinutesOffsetFromStartTime": "112"
|
||||
},
|
||||
{
|
||||
"point": "geo:27.709617,-97.375988",
|
||||
"durationMinutesOffsetFromStartTime": "112"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
13
spec/fixtures/files/watched/user@domain.com/owntracks.rec
vendored
Normal file
13
spec/fixtures/files/watched/user@domain.com/owntracks.rec
vendored
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
2024-03-01T09:03:09Z * {"bs":2,"p":100.266,"batt":94,"_type":"location","tid":"RO","topic":"owntracks/test/iPhone 12 Pro","alt":36,"lon":13.332,"vel":0,"t":"p","BSSID":"b0:f2:8:45:94:33","SSID":"Home Wifi","conn":"w","vac":4,"acc":10,"tst":1709283789,"lat":52.225,"m":1,"inrids":["5f1d1b"],"inregions":["home"],"_http":true}
|
||||
2024-03-01T17:46:02Z * {"bs":1,"p":100.28,"batt":94,"_type":"location","tid":"RO","topic":"owntracks/test/iPhone 12 Pro","alt":36,"lon":13.333,"t":"p","vel":0,"BSSID":"b0:f2:8:45:94:33","conn":"w","SSID":"Home Wifi","vac":3,"cog":98,"acc":9,"tst":1709315162,"lat":52.226,"m":1,"inrids":["5f1d1b"],"inregions":["home"],"_http":true}
|
||||
2024-03-01T18:26:55Z * {"lon":13.334,"acc":5,"wtst":1696359532,"event":"leave","rid":"5f1d1b","desc":"home","topic":"owntracks/test/iPhone 12 Pro/event","lat":52.227,"t":"c","tst":1709317615,"tid":"RO","_type":"transition","_http":true}
|
||||
2024-03-01T18:26:55Z * {"cog":40,"batt":85,"lon":13.335,"acc":5,"bs":1,"p":100.279,"vel":3,"vac":3,"lat":52.228,"topic":"owntracks/test/iPhone 12 Pro","t":"c","conn":"m","m":1,"tst":1709317615,"alt":36,"_type":"location","tid":"RO","_http":true}
|
||||
2024-03-01T18:28:30Z * {"cog":38,"batt":85,"lon":13.336,"acc":5,"bs":1,"p":100.349,"vel":3,"vac":3,"lat":52.229,"topic":"owntracks/test/iPhone 12 Pro","t":"v","conn":"m","m":1,"tst":1709317710,"alt":35,"_type":"location","tid":"RO","_http":true}
|
||||
2024-03-01T18:33:03Z * {"cog":18,"batt":85,"lon":13.337,"acc":5,"bs":1,"p":100.347,"vel":4,"vac":3,"lat":52.230,"topic":"owntracks/test/iPhone 12 Pro","conn":"m","m":1,"tst":1709317983,"alt":36,"_type":"location","tid":"RO","_http":true}
|
||||
2024-03-01T18:40:11Z * {"cog":43,"batt":85,"lon":13.338,"acc":5,"bs":1,"p":100.348,"vel":6,"vac":3,"lat":52.231,"topic":"owntracks/test/iPhone 12 Pro","conn":"m","m":1,"tst":1709318411,"alt":37,"_type":"location","tid":"RO","_http":true}
|
||||
2024-03-01T18:42:57Z * {"cog":320,"batt":85,"lon":13.339,"acc":5,"bs":1,"p":100.353,"vel":3,"vac":3,"lat":52.232,"topic":"owntracks/test/iPhone 12 Pro","t":"v","conn":"m","m":1,"tst":1709318577,"alt":37,"_type":"location","tid":"RO","_http":true}
|
||||
2024-03-01T18:40:08Z lwt {"_type":"lwt","tst":1717459208}
|
||||
2024-03-01T18:40:09Z waypoints {"_type":"waypoint","desc":"Home","lat":52.232,"lon":13.339,"rad":50,"tst":1717459768}
|
||||
2024-03-01T18:40:10Z event {"_type":"transition","acc":5,"desc":"Home","event":"enter","lat":52.232,"lon":13.339,"t":"l","tid":"s8","tst":1717460098,"wtst":1717459768}
|
||||
2024-03-01T18:40:11Z * {"cog":43,"batt":85,"lon":13.338,"acc":5,"bs":1,"p":100.348,"vel":6,"vac":3,"lat":52.231,"topic":"owntracks/test/iPhone 12 Pro","conn":"m","m":1,"tst":1709318411,"alt":37,"_type":"location","tid":"RO","_http":true}
|
||||
2024-03-01T18:40:11Z * {"cog":43,"batt":85,"lon":13.341,"acc":5,"bs":1,"p":100.348,"created_at":1709318940,"vel":6,"vac":3,"lat":52.234,"topic":"owntracks/test/iPhone 12 Pro","conn":"m","m":1,"tst":1709318411,"alt":37,"_type":"location","tid":"RO","_http":true}
|
||||
|
|
@ -11,8 +11,11 @@ RSpec.describe Imports::Watcher do
|
|||
|
||||
before do
|
||||
stub_const('Imports::Watcher::WATCHED_DIR_PATH', watched_dir_path)
|
||||
Sidekiq::Testing.inline!
|
||||
end
|
||||
|
||||
after { Sidekiq::Testing.fake! }
|
||||
|
||||
context 'when there are no files in the watched directory' do
|
||||
it 'does not call ImportJob' do
|
||||
expect(ImportJob).not_to receive(:perform_later)
|
||||
|
|
@ -22,26 +25,34 @@ RSpec.describe Imports::Watcher do
|
|||
end
|
||||
|
||||
context 'when there are files in the watched directory' do
|
||||
Sidekiq::Testing.inline! do
|
||||
context 'when the file has a valid user email' do
|
||||
it 'creates an import for the user' do
|
||||
expect { service }.to change(user.imports, :count).by(2)
|
||||
end
|
||||
context 'when the file has a valid user email' do
|
||||
it 'creates an import for the user' do
|
||||
expect { service }.to change(user.imports, :count).by(6)
|
||||
end
|
||||
|
||||
context 'when the file has an invalid user email' do
|
||||
it 'does not create an import' do
|
||||
expect { service }.not_to change(Import, :count)
|
||||
end
|
||||
it 'creates points for the user' do
|
||||
initial_point_count = Point.count
|
||||
service
|
||||
expect(Point.count).to be > initial_point_count
|
||||
end
|
||||
end
|
||||
|
||||
context 'when the import already exists' do
|
||||
it 'does not create a new import' do
|
||||
create(:import, user:, name: 'export_same_points.json')
|
||||
create(:import, user:, name: 'gpx_track_single_segment.gpx')
|
||||
context 'when the file has an invalid user email' do
|
||||
it 'does not create an import' do
|
||||
expect { service }.not_to change(Import, :count)
|
||||
end
|
||||
end
|
||||
|
||||
expect { service }.not_to change(Import, :count)
|
||||
end
|
||||
context 'when the import already exists' do
|
||||
it 'does not create a new import' do
|
||||
create(:import, user:, name: '2023_January.json')
|
||||
create(:import, user:, name: 'export_same_points.json')
|
||||
create(:import, user:, name: 'gpx_track_single_segment.gpx')
|
||||
create(:import, user:, name: 'location-history.json')
|
||||
create(:import, user:, name: 'owntracks.rec')
|
||||
create(:import, user:, name: 'Records.json')
|
||||
|
||||
expect { service }.not_to change(Import, :count)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
Loading…
Reference in a new issue