Implement support for Google Phone Takeout import

app/controllers/imports_controller.rb

@@ -44,6 +44,7 @@ class ImportsController < ApplicationController
 
   def destroy
     @import.destroy!
+
     redirect_to imports_url, notice: 'Import was successfully destroyed.', status: :see_other
   end
 

app/services/google_maps/phone_takeout_parser.rb

@@ -13,20 +13,29 @@ class GoogleMaps::PhoneTakeoutParser
 
     points = 0
 
-    points_data.each do |point_data|
-      next if Point.exists?(timestamp: point_data[:timestamp])
+    points_data.compact.each do |point_data|
+      next if Point.exists?(
+        timestamp: point_data[:timestamp],
+        latitude: point_data[:latitude],
+        longitude: point_data[:longitude],
+        user_id:
+      )
 
       Point.create(
         latitude: point_data[:latitude],
         longitude: point_data[:longitude],
         timestamp: point_data[:timestamp],
         raw_data: point_data[:raw_data],
+        accuracy: point_data[:accuracy],
+        altitude: point_data[:altitude],
+        velocity: point_data[:velocity],
         topic: 'Google Maps Phone Timeline Export',
         tracker_id: 'google-maps-phone-timeline-export',
         import_id: import.id,
         user_id:
       )
-
+    rescue
+      binding.pry
       points += 1
     end
 
@@ -39,7 +48,7 @@ class GoogleMaps::PhoneTakeoutParser
   private
 
   def parse_json
-    import.raw_data['semanticSegments'].flat_map do |segment|
+    semantic_segments = import.raw_data['semanticSegments'].flat_map do |segment|
       if segment.key?('timelinePath')
         segment['timelinePath'].map do |point|
           lat, lon = parse_coordinates(point['point'])
@@ -52,8 +61,32 @@ class GoogleMaps::PhoneTakeoutParser
         timestamp = DateTime.parse(segment['startTime']).to_i
 
         point_hash(lat, lon, timestamp, segment)
+      else # activities
+        # Some activities don't have start latLng
+        next if segment.dig('activity', 'start', 'latLng').nil?
+
+        start_lat, start_lon = parse_coordinates(segment['activity']['start']['latLng'])
+        start_timestamp = DateTime.parse(segment['startTime']).to_i
+        end_lat, end_lon = parse_coordinates(segment['activity']['end']['latLng'])
+        end_timestamp = DateTime.parse(segment['endTime']).to_i
+
+        [
+          point_hash(start_lat, start_lon, start_timestamp, segment),
+          point_hash(end_lat, end_lon, end_timestamp, segment)
+        ]
       end
     end
+
+    raw_signals = import.raw_data['rawSignals'].flat_map do |segment|
+      next unless segment.dig('position', 'LatLng')
+
+      lat, lon = parse_coordinates(segment['position']['LatLng'])
+      timestamp = DateTime.parse(segment['position']['timestamp']).to_i
+
+      point_hash(lat, lon, timestamp, segment)
+    end
+
+    semantic_segments + raw_signals
   end
 
   def parse_coordinates(coordinates)
@@ -65,7 +98,10 @@ class GoogleMaps::PhoneTakeoutParser
       latitude: lat.to_f,
       longitude: lon.to_f,
       timestamp:,
-      raw_data:
+      raw_data:,
+      accuracy: raw_data['accuracyMeters'],
+      altitude: raw_data['altitudeMeters'],
+      velocitu: raw_data['speedMetersPerSecond']
     }
   end
 end

app/services/google_maps/records_parser.rb

@@ -10,7 +10,12 @@ class GoogleMaps::RecordsParser
   def call(json)
     data = parse_json(json)
 
-    return if Point.exists?(latitude: data[:latitude], longitude: data[:longitude], timestamp: data[:timestamp])
+    return if Point.exists?(
+      latitude: data[:latitude],
+      longitude: data[:longitude],
+      timestamp: data[:timestamp],
+      user_id: import.user_id
+    )
 
     Point.create(
       latitude: data[:latitude],

app/services/google_maps/semantic_history_parser.rb

@@ -14,7 +14,12 @@ class GoogleMaps::SemanticHistoryParser
     points = 0
 
     points_data.each do |point_data|
-      next if Point.exists?(timestamp: point_data[:timestamp])
+      next if Point.exists?(
+        timestamp: point_data[:timestamp],
+        latitude: point_data[:latitude],
+        longitude: point_data[:longitude],
+        user_id:
+      )
 
       Point.create(
         latitude: point_data[:latitude],

app/services/own_tracks/export_parser.rb

@@ -15,7 +15,12 @@ class OwnTracks::ExportParser
     points = 0
 
     points_data.each do |point_data|
-      next if Point.exists?(timestamp: point_data[:timestamp], tracker_id: point_data[:tracker_id])
+      next if Point.exists?(
+        timestamp: point_data[:timestamp],
+        latitude: point_data[:latitude],
+        longitude: point_data[:longitude],
+        user_id:
+      )
 
       Point.create(
         latitude: point_data[:latitude],

app/views/imports/_form.html.erb

@@ -1,23 +1,51 @@
 <%= form_with model: import, class: "contents" do |form| %>
-  <div class="form-control w-full max-w-xs">
+  <div class="form-control w-full">
     <label class="label">
       <span class="label-text">Select source</span>
     </label>
-    <div class="space-y-2">
-      <%= form.collection_radio_buttons :source, Import.sources.except('google_records'), :first, :first do |b| %>
+    <div class="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-4">
+      <div class="card bordered shadow-lg p-3 hover:shadow-blue-500/50">
         <div class="form-control">
           <label class="label cursor-pointer space-x-3">
-            <%= b.radio_button(class: "radio radio-primary") %>
-            <span class="label-text"><%= b.text.humanize %></span>
+            <%= form.radio_button :source, 0, class: "radio radio-primary" %>
+            <span class="label-text">Google Semantic History</span>
           </label>
+          <p class="text-sm mt-2">JSON files from your Takeout/Location History/Semantic Location History/YEAR</p>
         </div>
-      <% end %>
+      </div>
+      <div class="card bordered shadow-lg p-3 hover:shadow-blue-500/50">
+        <div class="form-control">
+          <label class="label cursor-pointer space-x-3">
+            <%= form.radio_button :source, :owntracks, class: "radio radio-primary" %>
+            <span class="label-text">Owntracks</span>
+          </label>
+          <p class="text-sm mt-2">A JSON file you exported by pressing Download button in top right corner of OwnTracks web interface</p>
+        </div>
+      </div>
+      <div class="card bordered shadow-lg p-3 hover:shadow-blue-500/50">
+        <div class="form-control">
+          <label class="label cursor-pointer space-x-3">
+            <%= form.radio_button :source, 3, class: "radio radio-primary" %>
+            <span class="label-text">Google Phone Takeout</span>
+          </label>
+          <p class="text-sm mt-2">A JSON file you received after your request for Takeout from your mobile device</p>
+        </div>
+      </div>
+      <div class="card bordered shadow-lg p-3 hover:shadow-blue-500/50">
+        <div class="form-control">
+          <label class="label cursor-pointer space-x-3">
+            <%= form.radio_button :source, 4, class: "radio radio-primary" %>
+            <span class="label-text">GPX</span>
+          </label>
+          <p class="text-sm mt-2">GPX track file</p>
+        </div>
+      </div>
     </div>
   </div>
 
   <label class="form-control w-full max-w-xs my-5">
     <div class="label">
-      <span class="label-text">Select file(s)</span>
+      <span class="label-text">Select one or multiple files</span>
     </div>
     <%= form.file_field :files, multiple: true, class: "file-input file-input-bordered w-full max-w-xs" %>
   </label>

spec/fixtures/files/google/phone-takeout.json

@@ -34,6 +34,91 @@
     }
   ],
   "rawSignals": [
+    {
+      "wifiScan": {
+        "deliveryTime": "2024-06-06T11:44:37.000+01:00",
+        "devicesRecords": [
+          {
+            "mac": 70474800562644,
+            "rawRssi": -76
+          },
+          {
+            "mac": 70474800562645,
+            "rawRssi": -77
+          },
+          {
+            "mac": 193560579751752,
+            "rawRssi": -50
+          },
+          {
+            "mac": 193560579686216,
+            "rawRssi": -48
+          },
+          {
+            "mac": 70474800544725,
+            "rawRssi": -81
+          },
+          {
+            "mac": 70474801247336,
+            "rawRssi": -70
+          },
+          {
+            "mac": 70474800544724,
+            "rawRssi": -82
+          },
+          {
+            "mac": 70474801247337,
+            "rawRssi": -70
+          },
+          {
+            "mac": 70474801258069,
+            "rawRssi": -79
+          },
+          {
+            "mac": 114621892967568,
+            "rawRssi": -88
+          },
+          {
+            "mac": 70474801256596,
+            "rawRssi": -78
+          },
+          {
+            "mac": 70474801256597,
+            "rawRssi": -81
+          },
+          {
+            "mac": 70474801244137,
+            "rawRssi": -83
+          },
+          {
+            "mac": 70474801244136,
+            "rawRssi": -82
+          },
+          {
+            "mac": 70474801258068,
+            "rawRssi": -79
+          },
+          {
+            "mac": 70474801247316,
+            "rawRssi": -56
+          },
+          {
+            "mac": 70474801247317,
+            "rawRssi": -57
+          }
+        ]
+      }
+    },
+    {
+      "position": {
+        "LatLng": "48.833657°, 2.256223°",
+        "accuracyMeters": 13,
+        "altitudeMeters": 90.70000457763672,
+        "source": "WIFI",
+        "timestamp": "2024-06-06T11:44:37.000+01:00",
+        "speedMetersPerSecond": 0.07095485180616379
+      }
+    },
     {
       "activityRecord": {
         "probableActivities": [

spec/jobs/import_job_spec.rb

@@ -10,7 +10,7 @@ RSpec.describe ImportJob, type: :job do
     let(:import) { create(:import, user:, name: 'owntracks_export.json') }
 
     it 'creates points' do
-      expect { perform }.to change { Point.count }.by(8)
+      expect { perform }.to change { Point.count }.by(9)
     end
 
     it 'calls StatCreatingJob' do

spec/services/google_maps/phone_takeout_parser_spec.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe GoogleMaps::PhoneTakeoutParser do
+  describe '#call' do
+    subject(:parser) { described_class.new(import, user.id).call }
+
+    let(:user) { create(:user) }
+    let(:file_path) { Rails.root.join('spec/fixtures/files/google/phone-takeout.json') }
+    let(:raw_data) { JSON.parse(File.read(file_path)) }
+    let(:import) { create(:import, user:, name: 'phone_takeout.json', raw_data:) }
+
+    context 'when file exists' do
+      it 'creates points' do
+        expect { parser }.to change { Point.count }.by(4)
+      end
+    end
+  end
+end

spec/services/own_tracks/export_parser_spec.rb

@@ -11,7 +11,7 @@ RSpec.describe OwnTracks::ExportParser do
 
     context 'when file exists' do
       it 'creates points' do
-        expect { parser }.to change { Point.count }.by(8)
+        expect { parser }.to change { Point.count }.by(9)
       end
     end
   end