Introduce uniqueness index and validation for points

app/controllers/api/v1/points_controller.rb

@@ -21,6 +21,9 @@ class Api::V1::PointsController < ApiController
     render json: serialized_points
   end
 
+  def create
+  end
+
   def update
     point = current_api_user.tracked_points.find(params[:id])
 

app/controllers/map_controller.rb

@@ -6,7 +6,6 @@ class MapController < ApplicationController
   def index
     @points = points.where('timestamp >= ? AND timestamp <= ?', start_at, end_at)
 
-    @countries_and_cities = CountriesAndCities.new(@points).call
     @coordinates =
       @points.pluck(:latitude, :longitude, :battery, :altitude, :timestamp, :velocity, :id, :country)
              .map { [_1.to_f, _2.to_f, _3.to_s, _4.to_s, _5.to_s, _6.to_s, _7.to_s, _8.to_s] }

app/jobs/points/create_job.rb

@@ -4,12 +4,13 @@ class Points::CreateJob < ApplicationJob
   queue_as :default
 
   def perform(params, user_id)
-    data = Overland::Params.new(params).call
+    data = Points::Params.new(params, user_id).call
 
-    data.each do |location|
-      next if point_exists?(location, user_id)
-
-      Point.create!(location.merge(user_id:))
+    data.each_slice(1000) do |location_batch|
+      Point.upsert_all(
+        location_batch,
+        unique_by: %i[latitude longitude timestamp user_id]
+      )
     end
   end
 

app/models/point.rb

@@ -8,7 +8,11 @@ class Point < ApplicationRecord
   belongs_to :user
 
   validates :latitude, :longitude, :timestamp, presence: true
-
+  validates :timestamp, uniqueness: {
+    scope: %i[latitude longitude user_id],
+    message: 'already has a point at this location and time for this user',
+    index: true
+  }
   enum :battery_status, { unknown: 0, unplugged: 1, charging: 2, full: 3 }, suffix: true
   enum :trigger, {
     unknown: 0, background_event: 1, circular_region_event: 2, beacon_event: 3,

db/data/20250120154554_remove_duplicate_points.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class RemoveDuplicatePoints < ActiveRecord::Migration[8.0]
+  def up
+    # Find duplicate groups using a subquery
+    duplicate_groups =
+      Point.select('latitude, longitude, timestamp, user_id, COUNT(*) as count')
+           .group('latitude, longitude, timestamp, user_id')
+           .having('COUNT(*) > 1')
+
+    puts "Duplicate groups found: #{duplicate_groups.length}"
+
+    duplicate_groups.each do |group|
+      points = Point.where(
+        latitude: group.latitude,
+        longitude: group.longitude,
+        timestamp: group.timestamp,
+        user_id: group.user_id
+      ).order(id: :asc)
+
+      # Keep the latest record and destroy all others
+      latest = points.last
+      points.where.not(id: latest.id).destroy_all
+    end
+  end
+
+  def down
+    # This migration cannot be reversed
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

db/data_schema.rb

@@ -1 +1 @@
-DataMigrate::Data.define(version: 20250104204852)
+DataMigrate::Data.define(version: 20250120154554)

db/migrate/20250120154555_add_unique_index_to_points.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class AddUniqueIndexToPoints < ActiveRecord::Migration[8.0]
+  disable_ddl_transaction!
+
+  def up
+    add_index :points, %i[latitude longitude timestamp user_id],
+              unique: true,
+              name: 'unique_points_index',
+              algorithm: :concurrently
+  end
+
+  def down
+    remove_index :points, name: 'unique_points_index'
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[8.0].define(version: 2025_01_20_152540) do
+ActiveRecord::Schema[8.0].define(version: 2025_01_20_154555) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_catalog.plpgsql"
 
@@ -168,6 +168,7 @@ ActiveRecord::Schema[8.0].define(version: 2025_01_20_152540) do
     t.index ["external_track_id"], name: "index_points_on_external_track_id"
     t.index ["geodata"], name: "index_points_on_geodata", using: :gin
     t.index ["import_id"], name: "index_points_on_import_id"
+    t.index ["latitude", "longitude", "timestamp", "user_id"], name: "unique_points_index", unique: true
     t.index ["latitude", "longitude"], name: "index_points_on_latitude_and_longitude"
     t.index ["reverse_geocoded_at"], name: "index_points_on_reverse_geocoded_at"
     t.index ["timestamp"], name: "index_points_on_timestamp"

spec/factories/points.rb

@@ -25,6 +25,10 @@ FactoryBot.define do
     import_id       { '' }
     city            { nil }
     country         { nil }
+    reverse_geocoded_at { nil }
+    course          { nil }
+    course_accuracy { nil }
+    external_track_id { nil }
     user
 
     trait :with_known_location do

spec/factories/trips.rb

@@ -10,11 +10,15 @@ FactoryBot.define do
 
     trait :with_points do
       after(:build) do |trip|
-        create_list(
-          :point, 25,
-          user: trip.user,
-          timestamp: trip.started_at + (1..1000).to_a.sample.minutes
-        )
+        (1..25).map do |i|
+          create(
+            :point,
+            :with_geodata,
+            :reverse_geocoded,
+            timestamp: trip.started_at + i.minutes,
+            user: trip.user
+          )
+        end
       end
     end
   end

spec/jobs/bulk_stats_calculating_job_spec.rb

@@ -9,8 +9,17 @@ RSpec.describe BulkStatsCalculatingJob, type: :job do
 
     let(:timestamp) { DateTime.new(2024, 1, 1).to_i }
 
-    let!(:points1) { create_list(:point, 10, user_id: user1.id, timestamp:) }
-    let!(:points2) { create_list(:point, 10, user_id: user2.id, timestamp:) }
+    let!(:points1) do
+      (1..10).map do |i|
+        create(:point, user_id: user1.id, timestamp: timestamp + i.minutes)
+      end
+    end
+
+    let!(:points2) do
+      (1..10).map do |i|
+        create(:point, user_id: user2.id, timestamp: timestamp + i.minutes)
+      end
+    end
 
     it 'enqueues Stats::CalculatingJob for each user' do
       expect(Stats::CalculatingJob).to receive(:perform_later).with(user1.id, 2024, 1)

spec/models/import_spec.rb

@@ -26,7 +26,11 @@ RSpec.describe Import, type: :model do
   describe '#years_and_months_tracked' do
     let(:import) { create(:import) }
     let(:timestamp) { Time.zone.local(2024, 11, 1) }
-    let!(:points) { create_list(:point, 3, import:, timestamp:) }
+    let!(:points) do
+      (1..3).map do |i|
+        create(:point, import:, timestamp: timestamp + i.minutes)
+      end
+    end
 
     it 'returns years and months tracked' do
       expect(import.years_and_months_tracked).to eq([[2024, 11]])

spec/models/stat_spec.rb

@@ -89,8 +89,14 @@ RSpec.describe Stat, type: :model do
       subject { stat.points.to_a }
 
       let(:stat) { create(:stat, year:, month: 1, user:) }
-      let(:timestamp) { DateTime.new(year, 1, 1, 5, 0, 0) }
-      let!(:points) { create_list(:point, 3, user:, timestamp:) }
+      let(:base_timestamp) { DateTime.new(year, 1, 1, 5, 0, 0) }
+      let!(:points) do
+        [
+          create(:point, user:, timestamp: base_timestamp),
+          create(:point, user:, timestamp: base_timestamp + 1.hour),
+          create(:point, user:, timestamp: base_timestamp + 2.hours)
+        ]
+      end
 
       it 'returns points' do
         expect(subject).to eq(points)

spec/models/user_spec.rb

@@ -115,7 +115,11 @@ RSpec.describe User, type: :model do
     end
 
     describe '#years_tracked' do
-      let!(:points) { create_list(:point, 3, user:, timestamp: DateTime.new(2024, 1, 1, 5, 0, 0)) }
+      let!(:points) do
+        (1..3).map do |i|
+          create(:point, user:, timestamp: DateTime.new(2024, 1, 1, 5, 0, 0) + i.minutes)
+        end
+      end
 
       it 'returns years tracked' do
         expect(user.years_tracked).to eq([{ year: 2024, months: ['Jan'] }])

spec/requests/api/v1/points_spec.rb

@@ -4,7 +4,11 @@ require 'rails_helper'
 
 RSpec.describe 'Api::V1::Points', type: :request do
   let!(:user) { create(:user) }
-  let!(:points) { create_list(:point, 150, user:) }
+  let!(:points) do
+    (1..15).map do |i|
+      create(:point, user:, timestamp: 1.day.ago + i.minutes)
+    end
+  end
 
   describe 'GET /index' do
     context 'when regular version of points is requested' do
@@ -21,7 +25,7 @@ RSpec.describe 'Api::V1::Points', type: :request do
 
         json_response = JSON.parse(response.body)
 
-        expect(json_response.size).to eq(100)
+        expect(json_response.size).to eq(15)
       end
 
       it 'returns a list of points with pagination' do
@@ -31,7 +35,7 @@ RSpec.describe 'Api::V1::Points', type: :request do
 
         json_response = JSON.parse(response.body)
 
-        expect(json_response.size).to eq(10)
+        expect(json_response.size).to eq(5)
       end
 
       it 'returns a list of points with pagination headers' do
@@ -40,7 +44,7 @@ RSpec.describe 'Api::V1::Points', type: :request do
         expect(response).to have_http_status(:ok)
 
         expect(response.headers['X-Current-Page']).to eq('2')
-        expect(response.headers['X-Total-Pages']).to eq('15')
+        expect(response.headers['X-Total-Pages']).to eq('2')
       end
     end
 
@@ -58,7 +62,7 @@ RSpec.describe 'Api::V1::Points', type: :request do
 
         json_response = JSON.parse(response.body)
 
-        expect(json_response.size).to eq(100)
+        expect(json_response.size).to eq(15)
       end
 
       it 'returns a list of points with pagination' do
@@ -68,7 +72,7 @@ RSpec.describe 'Api::V1::Points', type: :request do
 
         json_response = JSON.parse(response.body)
 
-        expect(json_response.size).to eq(10)
+        expect(json_response.size).to eq(5)
       end
 
       it 'returns a list of points with pagination headers' do
@@ -77,7 +81,7 @@ RSpec.describe 'Api::V1::Points', type: :request do
         expect(response).to have_http_status(:ok)
 
         expect(response.headers['X-Current-Page']).to eq('2')
-        expect(response.headers['X-Total-Pages']).to eq('15')
+        expect(response.headers['X-Total-Pages']).to eq('2')
       end
 
       it 'returns a list of points with slim attributes' do

spec/requests/api/v1/stats_spec.rb

@@ -10,14 +10,20 @@ RSpec.describe 'Api::V1::Stats', type: :request do
     let!(:stats_in_2020) { create_list(:stat, 12, year: 2020, user:) }
     let!(:stats_in_2021) { create_list(:stat, 12, year: 2021, user:) }
     let!(:points_in_2020) do
-      create_list(:point, 85, :with_geodata, :reverse_geocoded, timestamp: Time.zone.local(2020), user:)
+      (1..85).map do |i|
+        create(:point, :with_geodata, :reverse_geocoded, timestamp: Time.zone.local(2020, 1, 1).to_i + i.hours, user:)
+      end
+    end
+    let!(:points_in_2021) do
+      (1..95).map do |i|
+        create(:point, :with_geodata, :reverse_geocoded, timestamp: Time.zone.local(2021, 1, 1).to_i + i.hours, user:)
+      end
     end
-    let!(:points_in_2021) { create_list(:point, 95, timestamp: Time.zone.local(2021), user:) }
     let(:expected_json) do
       {
         totalDistanceKm: stats_in_2020.map(&:distance).sum + stats_in_2021.map(&:distance).sum,
         totalPointsTracked: points_in_2020.count + points_in_2021.count,
-        totalReverseGeocodedPoints: points_in_2020.count,
+        totalReverseGeocodedPoints: points_in_2020.count + points_in_2021.count,
         totalCountriesVisited: 1,
         totalCitiesVisited: 1,
         yearlyStats: [

spec/requests/exports_spec.rb

@@ -37,7 +37,11 @@ RSpec.describe '/exports', type: :request do
     before { sign_in user }
 
     context 'with valid parameters' do
-      let(:points) { create_list(:point, 10, user:, timestamp: 1.day.ago) }
+      let(:points) do
+        (1..10).map do |i|
+          create(:point, user:, timestamp: 1.day.ago + i.minutes)
+        end
+      end
 
       it 'creates a new Export' do
         expect { post exports_url, params: }.to change(Export, :count).by(1)

spec/requests/map_spec.rb

@@ -11,7 +11,11 @@ RSpec.describe 'Map', type: :request do
   describe 'GET /index' do
     context 'when user signed in' do
       let(:user) { create(:user) }
-      let(:points) { create_list(:point, 10, user:, timestamp: 1.day.ago) }
+      let(:points) do
+        (1..10).map do |i|
+          create(:point, user:, timestamp: 1.day.ago + i.minutes)
+        end
+      end
 
       before { sign_in user }
 

spec/serializers/export_serializer_spec.rb

@@ -7,7 +7,12 @@ RSpec.describe ExportSerializer do
     subject(:serializer) { described_class.new(points, user_email).call }
 
     let(:user_email) { 'ab@cd.com' }
-    let(:points) { create_list(:point, 2) }
+    let(:points) do
+      (1..2).map do |i|
+        create(:point, timestamp: 1.day.ago + i.minutes)
+      end
+    end
+
     let(:expected_json) do
       {
         user_email => {

spec/serializers/points/geojson_serializer_spec.rb

@@ -6,7 +6,12 @@ RSpec.describe Points::GeojsonSerializer do
   describe '#call' do
     subject(:serializer) { described_class.new(points).call }
 
-    let(:points) { create_list(:point, 3) }
+    let(:points) do
+      (1..3).map do |i|
+        create(:point, timestamp: 1.day.ago + i.minutes)
+      end
+    end
+
     let(:expected_json) do
       {
         type: 'FeatureCollection',

spec/serializers/points/gpx_serializer_spec.rb

@@ -6,7 +6,11 @@ RSpec.describe Points::GpxSerializer do
   describe '#call' do
     subject(:serializer) { described_class.new(points, 'some_name').call }
 
-    let(:points) { create_list(:point, 3) }
+    let(:points) do
+      (1..3).map do |i|
+        create(:point, timestamp: 1.day.ago + i.minutes)
+      end
+    end
 
     it 'returns GPX file' do
       expect(serializer).to be_a(GPX::GPXFile)

spec/serializers/stats_serializer_spec.rb

@@ -29,16 +29,20 @@ RSpec.describe StatsSerializer do
       let!(:stats_in_2020) { create_list(:stat, 12, year: 2020, user:) }
       let!(:stats_in_2021) { create_list(:stat, 12, year: 2021, user:) }
       let!(:points_in_2020) do
-        create_list(:point, 85, :with_geodata, :reverse_geocoded, timestamp: Time.zone.local(2020), user:)
+        (1..85).map do |i|
+          create(:point, :with_geodata, :reverse_geocoded, timestamp: Time.zone.local(2020, 1, 1).to_i + i.hours, user:)
+        end
       end
       let!(:points_in_2021) do
-        create_list(:point, 95, timestamp: Time.zone.local(2021), user:)
+        (1..95).map do |i|
+          create(:point, :with_geodata, :reverse_geocoded, timestamp: Time.zone.local(2021, 1, 1).to_i + i.hours, user:)
+        end
       end
       let(:expected_json) do
         {
           "totalDistanceKm": stats_in_2020.map(&:distance).sum + stats_in_2021.map(&:distance).sum,
           "totalPointsTracked": points_in_2020.count + points_in_2021.count,
-          "totalReverseGeocodedPoints": points_in_2020.count,
+          "totalReverseGeocodedPoints": points_in_2020.count + points_in_2021.count,
           "totalCountriesVisited": 1,
           "totalCitiesVisited": 1,
           "yearlyStats": [

spec/services/exports/create_spec.rb

@@ -15,7 +15,12 @@ RSpec.describe Exports::Create do
     let(:export_content)  { Points::GeojsonSerializer.new(points).call }
     let(:reverse_geocoded_at) { Time.zone.local(2021, 1, 1) }
     let!(:points) do
-      create_list(:point, 10, :with_known_location, user:, timestamp: start_at.to_datetime.to_i, reverse_geocoded_at:)
+      10.times.map do |i|
+        create(:point, :with_known_location,
+               user: user,
+               timestamp: start_at.to_datetime.to_i + i,
+               reverse_geocoded_at: reverse_geocoded_at)
+      end
     end
 
     before do

spec/services/google_maps/records_parser_spec.rb

@@ -7,7 +7,7 @@ RSpec.describe GoogleMaps::RecordsParser do
     subject(:parser) { described_class.new(import).call(json) }
 
     let(:import) { create(:import) }
-    let(:time) { Time.zone.now }
+    let(:time) { DateTime.new(2025, 1, 1, 12, 0, 0) }
     let(:json) do
       {
         'latitudeE7' => 123_456_789,
@@ -31,7 +31,7 @@ RSpec.describe GoogleMaps::RecordsParser do
       before do
         create(
           :point, user: import.user, import:, latitude: 12.3456789, longitude: 12.3456789,
-          timestamp: Time.zone.now.to_i
+          timestamp: time.to_i
         )
       end
 
@@ -78,4 +78,4 @@ RSpec.describe GoogleMaps::RecordsParser do
       end
     end
   end
-end
+end

spec/services/jobs/create_spec.rb

@@ -8,7 +8,12 @@ RSpec.describe Jobs::Create do
 
     context 'when job_name is start_reverse_geocoding' do
       let(:user) { create(:user) }
-      let(:points) { create_list(:point, 4, user:) }
+      let(:points) do
+        (1..4).map do |i|
+          create(:point, user:, timestamp: 1.day.ago + i.minutes)
+        end
+      end
+
       let(:job_name) { 'start_reverse_geocoding' }
 
       it 'enqueues reverse geocoding for all user points' do
@@ -24,8 +29,17 @@ RSpec.describe Jobs::Create do
 
     context 'when job_name is continue_reverse_geocoding' do
       let(:user) { create(:user) }
-      let(:points_without_address) { create_list(:point, 4, user:, country: nil, city: nil) }
-      let(:points_with_address) { create_list(:point, 5, user:, country: 'Country', city: 'City') }
+      let(:points_without_address) do
+        (1..4).map do |i|
+          create(:point, user:, country: nil, city: nil, timestamp: 1.day.ago + i.minutes)
+        end
+      end
+
+      let(:points_with_address) do
+        (1..5).map do |i|
+          create(:point, user:, country: 'Country', city: 'City', timestamp: 1.day.ago + i.minutes)
+        end
+      end
 
       let(:job_name) { 'continue_reverse_geocoding' }
 

spec/swagger/api/v1/points_controller_spec.rb

@@ -58,7 +58,11 @@ describe 'Points API', type: :request do
         let(:api_key)   { user.api_key }
         let(:start_at)  { Time.zone.now - 1.day }
         let(:end_at)    { Time.zone.now }
-        let(:points)    { create_list(:point, 10, user:, timestamp: 2.hours.ago) }
+        let(:points) do
+          (1..10).map do |i|
+            create(:point, user:, timestamp: 2.hours.ago + i.minutes)
+          end
+        end
 
         run_test!
       end

spec/swagger/api/v1/stats_controller_spec.rb

@@ -57,8 +57,18 @@ describe 'Stats API', type: :request do
         let!(:user) { create(:user) }
         let!(:stats_in_2020) { create_list(:stat, 12, year: 2020, user:) }
         let!(:stats_in_2021) { create_list(:stat, 12, year: 2021, user:) }
-        let!(:points_in_2020) { create_list(:point, 85, :with_geodata, timestamp: Time.zone.local(2020), user:) }
-        let!(:points_in_2021) { create_list(:point, 95, timestamp: Time.zone.local(2021), user:) }
+        let!(:points_in_2020) do
+          (1..85).map do |i|
+            create(:point, :with_geodata, :reverse_geocoded, timestamp: Time.zone.local(2020, 1, 1).to_i + i.hours,
+user:)
+          end
+        end
+        let!(:points_in_2021) do
+          (1..95).map do |i|
+            create(:point, :with_geodata, :reverse_geocoded, timestamp: Time.zone.local(2021, 1, 1).to_i + i.hours,
+user:)
+          end
+        end
         let(:api_key) { user.api_key }
 
         run_test!