Merge pull request #281 from saschazepter/fix/import-google-timeline

Allow different timestamp formats in Google exports to be parsed

app/services/google_maps/records_parser.rb

@@ -35,7 +35,7 @@ class GoogleMaps::RecordsParser
     {
       latitude: json['latitudeE7'].to_f / 10**7,
       longitude: json['longitudeE7'].to_f / 10**7,
-      timestamp: DateTime.parse(json['timestamp']).to_i,
+      timestamp: Timestamps::parse_timestamp(json['timestamp'] || json['timestampMs']),
       altitude: json['altitude'],
       velocity: json['velocity'],
       raw_data: json

app/services/google_maps/semantic_history_parser.rb

@@ -44,7 +44,7 @@ class GoogleMaps::SemanticHistoryParser
             {
               latitude: waypoint['latE7'].to_f / 10**7,
               longitude: waypoint['lngE7'].to_f / 10**7,
-              timestamp: DateTime.parse(timeline_object['activitySegment']['duration']['startTimestamp']).to_i,
+              timestamp: Timestamps::parse_timestamp(timeline_object['activitySegment']['duration']['startTimestamp'] || timeline_object['activitySegment']['duration']['startTimestampMs']),
               raw_data: timeline_object
             }
           end
@@ -52,7 +52,7 @@ class GoogleMaps::SemanticHistoryParser
           {
             latitude: timeline_object['activitySegment']['startLocation']['latitudeE7'].to_f / 10**7,
             longitude: timeline_object['activitySegment']['startLocation']['longitudeE7'].to_f / 10**7,
-            timestamp: DateTime.parse(timeline_object['activitySegment']['duration']['startTimestamp']),
+            timestamp: Timestamps::parse_timestamp(timeline_object['activitySegment']['duration']['startTimestamp'] || timeline_object['activitySegment']['duration']['startTimestampMs']),
             raw_data: timeline_object
           }
         end
@@ -62,7 +62,7 @@ class GoogleMaps::SemanticHistoryParser
           {
             latitude: timeline_object['placeVisit']['location']['latitudeE7'].to_f / 10**7,
             longitude: timeline_object['placeVisit']['location']['longitudeE7'].to_f / 10**7,
-            timestamp: DateTime.parse(timeline_object['placeVisit']['duration']['startTimestamp']),
+            timestamp: Timestamps::parse_timestamp(timeline_object['placeVisit']['duration']['startTimestamp'] || timeline_object['placeVisit']['duration']['startTimestampMs']),
             raw_data: timeline_object
           }
         elsif timeline_object.dig('placeVisit', 'otherCandidateLocations')&.any?
@@ -73,7 +73,7 @@ class GoogleMaps::SemanticHistoryParser
           {
             latitude: point['latitudeE7'].to_f / 10**7,
             longitude: point['longitudeE7'].to_f / 10**7,
-            timestamp: DateTime.parse(timeline_object['placeVisit']['duration']['startTimestamp']),
+            timestamp: Timestamps::parse_timestamp(timeline_object['placeVisit']['duration']['startTimestamp'] || timeline_object['placeVisit']['duration']['startTimestampMs']),
             raw_data: timeline_object
           }
         else

lib/timestamps.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Timestamps
+
+  def self.parse_timestamp(timestamp)
+    begin
+      # if the timestamp is in ISO 8601 format, try to parse it
+      DateTime.parse(timestamp).to_time.to_i
+    rescue
+      if timestamp.to_s.length > 10
+        # If the timestamp is in milliseconds, convert to seconds
+        timestamp.to_i / 1000
+      else
+        # If the timestamp is in seconds, return it without change
+        timestamp.to_i
+      end
+    end
+  end
+end

spec/services/google_maps/records_parser_spec.rb

@@ -7,21 +7,27 @@ RSpec.describe GoogleMaps::RecordsParser do
     subject(:parser) { described_class.new(import).call(json) }
 
     let(:import) { create(:import) }
+    let(:time) { Time.zone.now }
     let(:json) do
       {
         'latitudeE7' => 123_456_789,
         'longitudeE7' => 123_456_789,
-        'timestamp' => Time.zone.now.to_s,
         'altitude' => 0,
         'velocity' => 0
       }
     end
 
-    it 'creates a point' do
+    context 'with regular timestamp' do
-      expect { parser }.to change(Point, :count).by(1)
+      let(:json) { super().merge('timestamp' => time.to_s) }
+
+      it 'creates a point' do
+        expect { parser }.to change(Point, :count).by(1)
+      end
     end
 
     context 'when point already exists' do
+      let(:json) { super().merge('timestamp' => time.to_s) }
+
       before do
         create(
           :point, user: import.user, import:, latitude: 12.3456789, longitude: 12.3456789,
@@ -33,5 +39,43 @@ RSpec.describe GoogleMaps::RecordsParser do
         expect { parser }.not_to change(Point, :count)
       end
     end
+
+    context 'with timestampMs in milliseconds' do
+      let(:json) { super().merge('timestampMs' => (time.to_f * 1000).to_i.to_s) }
+
+      it 'creates a point using milliseconds timestamp' do
+        expect { parser }.to change(Point, :count).by(1)
+      end
+    end
+
+    context 'with ISO 8601 timestamp' do
+      let(:json) { super().merge('timestamp' => time.iso8601) }
+
+      it 'parses ISO 8601 timestamp correctly' do
+        expect { parser }.to change(Point, :count).by(1)
+        created_point = Point.last
+        expect(created_point.timestamp).to eq(time.to_i)
+      end
+    end
+
+    context 'with timestamp in milliseconds' do
+      let(:json) { super().merge('timestamp' => (time.to_f * 1000).to_i.to_s) }
+
+      it 'parses millisecond timestamp correctly' do
+        expect { parser }.to change(Point, :count).by(1)
+        created_point = Point.last
+        expect(created_point.timestamp).to eq(time.to_i)
+      end
+    end
+
+    context 'with timestamp in seconds' do
+      let(:json) { super().merge('timestamp' => time.to_i.to_s) }
+
+      it 'parses second timestamp correctly' do
+        expect { parser }.to change(Point, :count).by(1)
+        created_point = Point.last
+        expect(created_point.timestamp).to eq(time.to_i)
+      end
+    end
   end
-end
+end

spec/services/google_maps/semantic_history_parser_spec.rb

@@ -7,6 +7,7 @@ RSpec.describe GoogleMaps::SemanticHistoryParser do
     subject(:parser) { described_class.new(import, user.id).call }
 
     let(:user) { create(:user) }
+    let(:time) { Time.zone.now }
 
     context 'when activitySegment is present' do
       context 'when startLocation is blank' do
@@ -19,7 +20,7 @@ RSpec.describe GoogleMaps::SemanticHistoryParser do
                   { 'latE7' => 123_456_789, 'lngE7' => 123_456_789 }
                 ]
               },
-              'duration' => { 'startTimestamp' => Time.zone.now.to_s }
+              'duration' => { 'startTimestamp' => time.to_s }
             }
           }
         end
@@ -32,7 +33,7 @@ RSpec.describe GoogleMaps::SemanticHistoryParser do
           let(:activity_segment) do
             {
               'activitySegment' => {
-                'duration' => { 'startTimestamp' => Time.zone.now.to_s }
+                'duration' => { 'startTimestamp' => time.to_s }
               }
             }
           end
@@ -49,7 +50,7 @@ RSpec.describe GoogleMaps::SemanticHistoryParser do
           {
             'activitySegment' => {
               'startLocation' => { 'latitudeE7' => 123_456_789, 'longitudeE7' => 123_456_789 },
-              'duration' => { 'startTimestamp' => Time.zone.now.to_s }
+              'duration' => { 'startTimestamp' => time.to_s }
             }
           }
         end
@@ -57,6 +58,68 @@ RSpec.describe GoogleMaps::SemanticHistoryParser do
         it 'creates a point' do
           expect { parser }.to change(Point, :count).by(1)
         end
+
+        context 'with different timestamp formats' do
+          context 'when timestamp is in ISO format' do
+            let(:activity_segment) do
+              {
+                'activitySegment' => {
+                  'startLocation' => { 'latitudeE7' => 123_456_789, 'longitudeE7' => 123_456_789 },
+                  'duration' => { 'startTimestamp' => time.iso8601 }
+                }
+              }
+            end
+
+            it 'creates a point' do
+              expect { parser }.to change(Point, :count).by(1)
+            end
+          end
+
+          context 'when timestamp is in seconds format' do
+            let(:activity_segment) do
+              {
+                'activitySegment' => {
+                  'startLocation' => { 'latitudeE7' => 123_456_789, 'longitudeE7' => 123_456_789 },
+                  'duration' => { 'startTimestamp' => (time.to_i).to_s }
+                }
+              }
+            end
+
+            it 'creates a point' do
+              expect { parser }.to change(Point, :count).by(1)
+            end
+          end
+
+          context 'when timestamp is in milliseconds format' do
+            let(:activity_segment) do
+              {
+                'activitySegment' => {
+                  'startLocation' => { 'latitudeE7' => 123_456_789, 'longitudeE7' => 123_456_789 },
+                  'duration' => { 'startTimestamp' => (time.to_f * 1000).to_i.to_s }
+                }
+              }
+            end
+
+            it 'creates a point' do
+              expect { parser }.to change(Point, :count).by(1)
+            end
+          end
+
+          context 'when timestampMs is used' do
+            let(:activity_segment) do
+              {
+                'activitySegment' => {
+                  'startLocation' => { 'latitudeE7' => 123_456_789, 'longitudeE7' => 123_456_789 },
+                  'duration' => { 'timestampMs' => (time.to_f * 1000).to_i.to_s }
+                }
+              }
+            end
+
+            it 'creates a point' do
+              expect { parser }.to change(Point, :count).by(1)
+            end
+          end
+        end
       end
     end
 
@@ -67,7 +130,7 @@ RSpec.describe GoogleMaps::SemanticHistoryParser do
           {
             'placeVisit' => {
               'location' => { 'latitudeE7' => 123_456_789, 'longitudeE7' => 123_456_789 },
-              'duration' => { 'startTimestamp' => Time.zone.now.to_s }
+              'duration' => { 'startTimestamp' => time.to_s }
             }
           }
         end
@@ -75,6 +138,68 @@ RSpec.describe GoogleMaps::SemanticHistoryParser do
         it 'creates a point' do
           expect { parser }.to change(Point, :count).by(1)
         end
+
+        context 'with different timestamp formats' do
+          context 'when timestamp is in ISO format' do
+            let(:place_visit) do
+              {
+                'placeVisit' => {
+                  'location' => { 'latitudeE7' => 123_456_789, 'longitudeE7' => 123_456_789 },
+                  'duration' => { 'startTimestamp' => time.iso8601 }
+                }
+              }
+            end
+
+            it 'creates a point' do
+              expect { parser }.to change(Point, :count).by(1)
+            end
+          end
+
+          context 'when timestamp is in seconds format' do
+            let(:place_visit) do
+              {
+                'placeVisit' => {
+                  'location' => { 'latitudeE7' => 123_456_789, 'longitudeE7' => 123_456_789 },
+                  'duration' => { 'startTimestamp' => time.to_i.to_s }
+                }
+              }
+            end
+
+            it 'creates a point' do
+              expect { parser }.to change(Point, :count).by(1)
+            end
+          end
+
+          context 'when timestamp is in milliseconds format' do
+            let(:place_visit) do
+              {
+                'placeVisit' => {
+                  'location' => { 'latitudeE7' => 123_456_789, 'longitudeE7' => 123_456_789 },
+                  'duration' => { 'startTimestamp' => (time.to_f * 1000).to_i.to_s }
+                }
+              }
+            end
+
+            it 'creates a point' do
+              expect { parser }.to change(Point, :count).by(1)
+            end
+          end
+
+          context 'when timestampMs is used' do
+            let(:place_visit) do
+              {
+                'placeVisit' => {
+                  'location' => { 'latitudeE7' => 123_456_789, 'longitudeE7' => 123_456_789 },
+                  'duration' => { 'timestampMs' => (time.to_f * 1000).to_i.to_s }
+                }
+              }
+            end
+
+            it 'creates a point' do
+              expect { parser }.to change(Point, :count).by(1)
+            end
+          end
+        end
       end
 
       context 'when location with coordinates is blank' do
@@ -83,7 +208,7 @@ RSpec.describe GoogleMaps::SemanticHistoryParser do
           {
             'placeVisit' => {
               'location' => {},
-              'duration' => { 'startTimestamp' => Time.zone.now.to_s }
+              'duration' => { 'startTimestamp' => time.to_s }
             }
           }
         end
@@ -97,7 +222,7 @@ RSpec.describe GoogleMaps::SemanticHistoryParser do
             {
               'placeVisit' => {
                 'otherCandidateLocations' => [{ 'latitudeE7' => 123_456_789, 'longitudeE7' => 123_456_789 }],
-                'duration' => { 'startTimestamp' => Time.zone.now.to_s }
+                'duration' => { 'startTimestamp' => time.to_s }
               }
             }
           end