Merge pull request #94 from Freika/google_phone_takeout_import

[0.8.2] Implement support for Google Phone Takeout import

.app_version

@@ -1 +1 @@
-0.8.1
+0.8.2

CHANGELOG.md

@@ -9,6 +9,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Added
 
+- Google Takeout geodata, taken from a [mobile devise](https://support.google.com/maps/thread/264641290/export-full-location-timeline-data-in-json-or-similar-format-in-the-new-version-of-timeline?hl=en), is now fully supported and can be imported to the Dawarich. The import process is the same as for other kinds of files, just select the JSON file and choose "Google Phone Takeout" as a source.
+
+### Fixed
+
+- Fixed a bug where an imported point was not being saved to the database if a point with the same timestamp and already existed in the database even if it was other user's point.
+
+---
+
+## [0.8.1] — 2024-06-30
+
+### Added
+
 - First user in the system can now create new users from the Settings page. This is useful for creating new users without the need to enable registrations. Default password for new users is `password`.
 
 ### Changed

app/controllers/imports_controller.rb

@@ -44,6 +44,7 @@ class ImportsController < ApplicationController
 
   def destroy
     @import.destroy!
+
     redirect_to imports_url, notice: 'Import was successfully destroyed.', status: :see_other
   end
 

app/services/google_maps/phone_takeout_parser.rb

@@ -13,14 +13,22 @@ class GoogleMaps::PhoneTakeoutParser
 
     points = 0
 
-    points_data.each do |point_data|
-      next if Point.exists?(timestamp: point_data[:timestamp])
+    points_data.compact.each do |point_data|
+      next if Point.exists?(
+        timestamp: point_data[:timestamp],
+        latitude: point_data[:latitude],
+        longitude: point_data[:longitude],
+        user_id:
+      )
 
       Point.create(
         latitude: point_data[:latitude],
         longitude: point_data[:longitude],
         timestamp: point_data[:timestamp],
         raw_data: point_data[:raw_data],
+        accuracy: point_data[:accuracy],
+        altitude: point_data[:altitude],
+        velocity: point_data[:velocity],
         topic: 'Google Maps Phone Timeline Export',
         tracker_id: 'google-maps-phone-timeline-export',
         import_id: import.id,
@@ -39,7 +47,7 @@ class GoogleMaps::PhoneTakeoutParser
   private
 
   def parse_json
-    import.raw_data['semanticSegments'].flat_map do |segment|
+    semantic_segments = import.raw_data['semanticSegments'].flat_map do |segment|
       if segment.key?('timelinePath')
         segment['timelinePath'].map do |point|
           lat, lon = parse_coordinates(point['point'])
@@ -52,8 +60,32 @@ class GoogleMaps::PhoneTakeoutParser
         timestamp = DateTime.parse(segment['startTime']).to_i
 
         point_hash(lat, lon, timestamp, segment)
+      else # activities
+        # Some activities don't have start latLng
+        next if segment.dig('activity', 'start', 'latLng').nil?
+
+        start_lat, start_lon = parse_coordinates(segment['activity']['start']['latLng'])
+        start_timestamp = DateTime.parse(segment['startTime']).to_i
+        end_lat, end_lon = parse_coordinates(segment['activity']['end']['latLng'])
+        end_timestamp = DateTime.parse(segment['endTime']).to_i
+
+        [
+          point_hash(start_lat, start_lon, start_timestamp, segment),
+          point_hash(end_lat, end_lon, end_timestamp, segment)
+        ]
       end
     end
+
+    raw_signals = import.raw_data['rawSignals'].flat_map do |segment|
+      next unless segment.dig('position', 'LatLng')
+
+      lat, lon = parse_coordinates(segment['position']['LatLng'])
+      timestamp = DateTime.parse(segment['position']['timestamp']).to_i
+
+      point_hash(lat, lon, timestamp, segment)
+    end
+
+    semantic_segments + raw_signals
   end
 
   def parse_coordinates(coordinates)
@@ -65,7 +97,10 @@ class GoogleMaps::PhoneTakeoutParser
       latitude: lat.to_f,
       longitude: lon.to_f,
       timestamp:,
-      raw_data:
+      raw_data:,
+      accuracy: raw_data['accuracyMeters'],
+      altitude: raw_data['altitudeMeters'],
+      velocitu: raw_data['speedMetersPerSecond']
     }
   end
 end

app/services/google_maps/records_parser.rb

@@ -10,7 +10,12 @@ class GoogleMaps::RecordsParser
   def call(json)
     data = parse_json(json)
 
-    return if Point.exists?(latitude: data[:latitude], longitude: data[:longitude], timestamp: data[:timestamp])
+    return if Point.exists?(
+      latitude: data[:latitude],
+      longitude: data[:longitude],
+      timestamp: data[:timestamp],
+      user_id: import.user_id
+    )
 
     Point.create(
       latitude: data[:latitude],

app/services/google_maps/semantic_history_parser.rb

@@ -14,7 +14,12 @@ class GoogleMaps::SemanticHistoryParser
     points = 0
 
     points_data.each do |point_data|
-      next if Point.exists?(timestamp: point_data[:timestamp])
+      next if Point.exists?(
+        timestamp: point_data[:timestamp],
+        latitude: point_data[:latitude],
+        longitude: point_data[:longitude],
+        user_id:
+      )
 
       Point.create(
         latitude: point_data[:latitude],

app/services/own_tracks/export_parser.rb

@@ -15,7 +15,12 @@ class OwnTracks::ExportParser
     points = 0
 
     points_data.each do |point_data|
-      next if Point.exists?(timestamp: point_data[:timestamp], tracker_id: point_data[:tracker_id])
+      next if Point.exists?(
+        timestamp: point_data[:timestamp],
+        latitude: point_data[:latitude],
+        longitude: point_data[:longitude],
+        user_id:
+      )
 
       Point.create(
         latitude: point_data[:latitude],

app/views/imports/_form.html.erb

@@ -1,23 +1,51 @@
 <%= form_with model: import, class: "contents" do |form| %>
-  <div class="form-control w-full max-w-xs">
+  <div class="form-control w-full">
     <label class="label">
       <span class="label-text">Select source</span>
     </label>
-    <div class="space-y-2">
-      <%= form.collection_radio_buttons :source, Import.sources.except('google_records'), :first, :first do |b| %>
+    <div class="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-4">
+      <div class="card bordered shadow-lg p-3 hover:shadow-blue-500/50">
         <div class="form-control">
           <label class="label cursor-pointer space-x-3">
-            <%= b.radio_button(class: "radio radio-primary") %>
-            <span class="label-text"><%= b.text.humanize %></span>
+            <%= form.radio_button :source, :google_semantic_history, class: "radio radio-primary" %>
+            <span class="label-text">Google Semantic History</span>
           </label>
+          <p class="text-sm mt-2">JSON files from your Takeout/Location History/Semantic Location History/YEAR</p>
         </div>
-      <% end %>
+      </div>
+      <div class="card bordered shadow-lg p-3 hover:shadow-blue-500/50">
+        <div class="form-control">
+          <label class="label cursor-pointer space-x-3">
+            <%= form.radio_button :source, :owntracks, class: "radio radio-primary" %>
+            <span class="label-text">Owntracks</span>
+          </label>
+          <p class="text-sm mt-2">A JSON file you exported by pressing Download button in top right corner of OwnTracks web interface</p>
+        </div>
+      </div>
+      <div class="card bordered shadow-lg p-3 hover:shadow-blue-500/50">
+        <div class="form-control">
+          <label class="label cursor-pointer space-x-3">
+            <%= form.radio_button :source, :google_phone_takeout, class: "radio radio-primary" %>
+            <span class="label-text">Google Phone Takeout</span>
+          </label>
+          <p class="text-sm mt-2">A JSON file you received after your request for Takeout from your mobile device</p>
+        </div>
+      </div>
+      <div class="card bordered shadow-lg p-3 hover:shadow-blue-500/50">
+        <div class="form-control">
+          <label class="label cursor-pointer space-x-3">
+            <%= form.radio_button :source, :gpx, class: "radio radio-primary" %>
+            <span class="label-text">GPX</span>
+          </label>
+          <p class="text-sm mt-2">GPX track file</p>
+        </div>
+      </div>
     </div>
   </div>
 
   <label class="form-control w-full max-w-xs my-5">
     <div class="label">
-      <span class="label-text">Select file(s)</span>
+      <span class="label-text">Select one or multiple files</span>
     </div>
     <%= form.file_field :files, multiple: true, class: "file-input file-input-bordered w-full max-w-xs" %>
   </label>

spec/fixtures/files/google/phone-takeout.json

@@ -34,6 +34,91 @@
     }
   ],
   "rawSignals": [
+    {
+      "wifiScan": {
+        "deliveryTime": "2024-06-06T11:44:37.000+01:00",
+        "devicesRecords": [
+          {
+            "mac": 70474800562644,
+            "rawRssi": -76
+          },
+          {
+            "mac": 70474800562645,
+            "rawRssi": -77
+          },
+          {
+            "mac": 193560579751752,
+            "rawRssi": -50
+          },
+          {
+            "mac": 193560579686216,
+            "rawRssi": -48
+          },
+          {
+            "mac": 70474800544725,
+            "rawRssi": -81
+          },
+          {
+            "mac": 70474801247336,
+            "rawRssi": -70
+          },
+          {
+            "mac": 70474800544724,
+            "rawRssi": -82
+          },
+          {
+            "mac": 70474801247337,
+            "rawRssi": -70
+          },
+          {
+            "mac": 70474801258069,
+            "rawRssi": -79
+          },
+          {
+            "mac": 114621892967568,
+            "rawRssi": -88
+          },
+          {
+            "mac": 70474801256596,
+            "rawRssi": -78
+          },
+          {
+            "mac": 70474801256597,
+            "rawRssi": -81
+          },
+          {
+            "mac": 70474801244137,
+            "rawRssi": -83
+          },
+          {
+            "mac": 70474801244136,
+            "rawRssi": -82
+          },
+          {
+            "mac": 70474801258068,
+            "rawRssi": -79
+          },
+          {
+            "mac": 70474801247316,
+            "rawRssi": -56
+          },
+          {
+            "mac": 70474801247317,
+            "rawRssi": -57
+          }
+        ]
+      }
+    },
+    {
+      "position": {
+        "LatLng": "48.833657°, 2.256223°",
+        "accuracyMeters": 13,
+        "altitudeMeters": 90.70000457763672,
+        "source": "WIFI",
+        "timestamp": "2024-06-06T11:44:37.000+01:00",
+        "speedMetersPerSecond": 0.07095485180616379
+      }
+    },
     {
       "activityRecord": {
         "probableActivities": [

spec/jobs/import_job_spec.rb

@@ -10,7 +10,7 @@ RSpec.describe ImportJob, type: :job do
     let(:import) { create(:import, user:, name: 'owntracks_export.json') }
 
     it 'creates points' do
-      expect { perform }.to change { Point.count }.by(8)
+      expect { perform }.to change { Point.count }.by(9)
     end
 
     it 'calls StatCreatingJob' do

spec/services/google_maps/phone_takeout_parser_spec.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe GoogleMaps::PhoneTakeoutParser do
+  describe '#call' do
+    subject(:parser) { described_class.new(import, user.id).call }
+
+    let(:user) { create(:user) }
+    let(:file_path) { Rails.root.join('spec/fixtures/files/google/phone-takeout.json') }
+    let(:raw_data) { JSON.parse(File.read(file_path)) }
+    let(:import) { create(:import, user:, name: 'phone_takeout.json', raw_data:) }
+
+    context 'when file exists' do
+      it 'creates points' do
+        expect { parser }.to change { Point.count }.by(4)
+      end
+    end
+  end
+end

spec/services/own_tracks/export_parser_spec.rb

@@ -11,7 +11,7 @@ RSpec.describe OwnTracks::ExportParser do
 
     context 'when file exists' do
       it 'creates points' do
-        expect { parser }.to change { Point.count }.by(8)
+        expect { parser }.to change { Point.count }.by(9)
       end
     end
   end