Introduce points counter cache to optimize performance

This commit is contained in:
Eugene Burmakin 2025-08-21 22:32:29 +02:00
parent 8a716aaae1
commit 6e4934a93d
35 changed files with 117 additions and 74 deletions

View file

@ -8,7 +8,7 @@ class Api::V1::Countries::VisitedCitiesController < ApiController
end_at = DateTime.parse(params[:end_at]).to_i
points = current_api_user
.tracked_points
.points
.where(timestamp: start_at..end_at)
render json: { data: CountriesAndCities.new(points).call }

View file

@ -10,7 +10,7 @@ class Api::V1::PointsController < ApiController
order = params[:order] || 'desc'
points = current_api_user
.tracked_points
.points
.where(timestamp: start_at..end_at)
.order(timestamp: order)
.page(params[:page])
@ -31,7 +31,7 @@ class Api::V1::PointsController < ApiController
end
def update
point = current_api_user.tracked_points.find(params[:id])
point = current_api_user.points.find(params[:id])
point.update(lonlat: "POINT(#{point_params[:longitude]} #{point_params[:latitude]})")
@ -39,7 +39,7 @@ class Api::V1::PointsController < ApiController
end
def destroy
point = current_api_user.tracked_points.find(params[:id])
point = current_api_user.points.find(params[:id])
point.destroy
render json: { message: 'Point deleted successfully' }

View file

@ -6,6 +6,6 @@ class HomeController < ApplicationController
redirect_to map_url if current_user
@points = current_user.tracked_points.without_raw_data if current_user
@points = current_user.points.without_raw_data if current_user
end
end

View file

@ -88,6 +88,6 @@ class MapController < ApplicationController
end
def points_from_user
current_user.tracked_points.without_raw_data.order(timestamp: :asc)
current_user.points.without_raw_data.order(timestamp: :asc)
end
end

View file

@ -24,7 +24,7 @@ class PointsController < ApplicationController
alert: 'No points selected.',
status: :see_other and return if point_ids.blank?
current_user.tracked_points.where(id: point_ids).destroy_all
current_user.points.where(id: point_ids).destroy_all
redirect_to points_url(preserved_params),
notice: 'Points were successfully destroyed.',
@ -58,7 +58,7 @@ class PointsController < ApplicationController
end
def user_points
current_user.tracked_points
current_user.points
end
def order_by

View file

@ -86,7 +86,7 @@ module ApplicationHelper
end
def points_exist?(year, month, user)
user.tracked_points.where(
user.points.where(
timestamp: DateTime.new(year, month).beginning_of_month..DateTime.new(year, month).end_of_month
).exists?
end

View file

@ -18,7 +18,7 @@ class BulkVisitsSuggestingJob < ApplicationJob
users.active.find_each do |user|
next unless user.safe_settings.visits_suggestions_enabled?
next if user.tracked_points.empty?
next unless user.points_count.positive?
schedule_chunked_jobs(user, time_chunks)
end

View file

@ -7,7 +7,7 @@ class DataMigrations::MigratePointsLatlonJob < ApplicationJob
user = User.find(user_id)
# rubocop:disable Rails/SkipsModelValidations
user.tracked_points.update_all('lonlat = ST_SetSRID(ST_MakePoint(longitude, latitude), 4326)')
user.points.update_all('lonlat = ST_SetSRID(ST_MakePoint(longitude, latitude), 4326)')
# rubocop:enable Rails/SkipsModelValidations
end
end

View file

@ -0,0 +1,28 @@
# frozen_string_literal: true
class DataMigrations::PrefillPointsCounterCacheJob < ApplicationJob
queue_as :data_migrations
def perform(user_id = nil)
if user_id
prefill_counter_for_user(user_id)
else
User.find_each(batch_size: 100) do |user|
prefill_counter_for_user(user.id)
end
end
end
private
def prefill_counter_for_user(user_id)
user = User.find(user_id)
points_count = user.points.count
User.where(id: user_id).update_all(points_count: points_count)
Rails.logger.info "Updated points_count for user #{user_id}: #{points_count}"
rescue ActiveRecord::RecordNotFound
Rails.logger.warn "User #{user_id} not found, skipping counter cache update"
end
end

View file

@ -23,9 +23,9 @@ class Tracks::CleanupJob < ApplicationJob
private
def users_with_old_untracked_points(older_than)
User.active.joins(:tracked_points)
.where(tracked_points: { track_id: nil, timestamp: ..older_than.to_i })
.having('COUNT(tracked_points.id) >= 2') # Only users with enough points for tracks
User.active.joins(:points)
.where(points: { track_id: nil, timestamp: ..older_than.to_i })
.having('COUNT(points.id) >= 2') # Only users with enough points for tracks
.group(:id)
end
end

View file

@ -6,7 +6,7 @@ class Point < ApplicationRecord
belongs_to :import, optional: true, counter_cache: true
belongs_to :visit, optional: true
belongs_to :user
belongs_to :user, counter_cache: true
belongs_to :country, optional: true
belongs_to :track, optional: true

View file

@ -24,7 +24,7 @@ class Stat < ApplicationRecord
end
def points
user.tracked_points
user.points
.without_raw_data
.where(timestamp: timespan)
.order(timestamp: :asc)

View file

@ -18,7 +18,7 @@ class Trip < ApplicationRecord
end
def points
user.tracked_points.where(timestamp: started_at.to_i..ended_at.to_i).order(:timestamp)
user.points.where(timestamp: started_at.to_i..ended_at.to_i).order(:timestamp)
end
def photo_previews

View file

@ -4,14 +4,13 @@ class User < ApplicationRecord # rubocop:disable Metrics/ClassLength
devise :database_authenticatable, :registerable,
:recoverable, :rememberable, :validatable, :trackable
has_many :tracked_points, class_name: 'Point', dependent: :destroy
has_many :points, dependent: :destroy, counter_cache: true
has_many :imports, dependent: :destroy
has_many :stats, dependent: :destroy
has_many :exports, dependent: :destroy
has_many :notifications, dependent: :destroy
has_many :areas, dependent: :destroy
has_many :visits, dependent: :destroy
has_many :points, through: :imports
has_many :places, through: :visits
has_many :trips, dependent: :destroy
has_many :tracks, dependent: :destroy
@ -35,7 +34,7 @@ class User < ApplicationRecord # rubocop:disable Metrics/ClassLength
end
def countries_visited
tracked_points
points
.where.not(country_name: [nil, ''])
.distinct
.pluck(:country_name)
@ -43,7 +42,7 @@ class User < ApplicationRecord # rubocop:disable Metrics/ClassLength
end
def cities_visited
tracked_points.where.not(city: [nil, '']).distinct.pluck(:city).compact
points.where.not(city: [nil, '']).distinct.pluck(:city).compact
end
def total_distance
@ -60,11 +59,11 @@ class User < ApplicationRecord # rubocop:disable Metrics/ClassLength
end
def total_reverse_geocoded_points
tracked_points.where.not(reverse_geocoded_at: nil).count
points.where.not(reverse_geocoded_at: nil).count
end
def total_reverse_geocoded_points_without_data
tracked_points.where(geodata: {}).count
points.where(geodata: {}).count
end
def immich_integration_configured?
@ -118,7 +117,7 @@ class User < ApplicationRecord # rubocop:disable Metrics/ClassLength
end
def trial_state?
tracked_points.none? && trial?
points_count.zero? && trial?
end
private

View file

@ -10,7 +10,7 @@ class StatsSerializer
def call
{
totalDistanceKm: total_distance_km,
totalPointsTracked: user.tracked_points.count,
totalPointsTracked: user.points_count,
totalReverseGeocodedPoints: reverse_geocoded_points,
totalCountriesVisited: user.countries_visited.count,
totalCitiesVisited: user.cities_visited.count,
@ -27,7 +27,7 @@ class StatsSerializer
end
def reverse_geocoded_points
user.tracked_points.reverse_geocoded.count
user.points.reverse_geocoded.count
end
def yearly_stats

View file

@ -35,7 +35,7 @@ class Exports::Create
def time_framed_points
user
.tracked_points
.points
.where(timestamp: start_at.to_i..end_at.to_i)
.order(timestamp: :asc)
end

View file

@ -14,9 +14,9 @@ class Jobs::Create
points =
case job_name
when 'start_reverse_geocoding'
user.tracked_points
user.points
when 'continue_reverse_geocoding'
user.tracked_points.not_reverse_geocoded
user.points.not_reverse_geocoded
else
raise InvalidJobName, 'Invalid job name'
end

View file

@ -9,7 +9,7 @@ class PointsLimitExceeded
return false if DawarichSettings.self_hosted?
Rails.cache.fetch(cache_key, expires_in: 1.day) do
@user.tracked_points.count >= points_limit
@user.points_count >= points_limit
end
end

View file

@ -47,7 +47,7 @@ class Stats::CalculateMonth
return @points if defined?(@points)
@points = user
.tracked_points
.points
.without_raw_data
.where(timestamp: start_timestamp..end_timestamp)
.select(:lonlat, :timestamp)
@ -60,7 +60,7 @@ class Stats::CalculateMonth
def toponyms
toponym_points = user
.tracked_points
.points
.without_raw_data
.where(timestamp: start_timestamp..end_timestamp)
.select(:city, :country_name)

View file

@ -86,7 +86,7 @@ class Tracks::Generator
end
def load_bulk_points
scope = user.tracked_points.order(:timestamp)
scope = user.points.order(:timestamp)
scope = scope.where(timestamp: timestamp_range) if time_range_defined?
scope
@ -95,7 +95,7 @@ class Tracks::Generator
def load_incremental_points
# For incremental mode, we process untracked points
# If end_at is specified, only process points up to that time
scope = user.tracked_points.where(track_id: nil).order(:timestamp)
scope = user.points.where(track_id: nil).order(:timestamp)
scope = scope.where(timestamp: ..end_at.to_i) if end_at.present?
scope
@ -104,7 +104,7 @@ class Tracks::Generator
def load_daily_points
day_range = daily_time_range
user.tracked_points.where(timestamp: day_range).order(:timestamp)
user.points.where(timestamp: day_range).order(:timestamp)
end
def create_track_from_segment(segment_data)
@ -195,8 +195,8 @@ class Tracks::Generator
def bulk_timestamp_range
return [start_at.to_i, end_at.to_i] if start_at && end_at
first_point = user.tracked_points.order(:timestamp).first
last_point = user.tracked_points.order(:timestamp).last
first_point = user.points.order(:timestamp).first
last_point = user.points.order(:timestamp).last
[first_point&.timestamp || 0, last_point&.timestamp || Time.current.to_i]
end
@ -207,7 +207,7 @@ class Tracks::Generator
end
def incremental_timestamp_range
first_point = user.tracked_points.where(track_id: nil).order(:timestamp).first
first_point = user.points.where(track_id: nil).order(:timestamp).first
end_timestamp = end_at ? end_at.to_i : Time.current.to_i
[first_point&.timestamp || 0, end_timestamp]

View file

@ -50,7 +50,7 @@ class Tracks::IncrementalProcessor
def find_previous_point
@previous_point ||=
user.tracked_points
user.points
.where('timestamp < ?', new_point.timestamp)
.order(:timestamp)
.last

View file

@ -331,7 +331,7 @@ class Users::ExportData
trips: user.trips.count,
stats: user.stats.count,
notifications: user.notifications.count,
points: user.tracked_points.count,
points: user.points_count,
visits: user.visits.count,
places: user.places.count
}

View file

@ -13,7 +13,7 @@ module Visits
@user = user
@start_at = start_at.to_i
@end_at = end_at.to_i
@points = user.tracked_points.not_visited
@points = user.points.not_visited
.order(timestamp: :asc)
.where(timestamp: start_at..end_at)
end

View file

@ -6,7 +6,7 @@ class Visits::Suggest
def initialize(user, start_at:, end_at:)
@start_at = start_at.to_i
@end_at = end_at.to_i
@points = user.tracked_points.not_visited.order(timestamp: :asc).where(timestamp: start_at..end_at)
@points = user.points.not_visited.order(timestamp: :asc).where(timestamp: start_at..end_at)
@user = user
end

View file

@ -1,6 +1,6 @@
<p class="py-6">
<p class='py-2'>
You have used <%= number_with_delimiter(current_user.tracked_points.count) %> points of <%= number_with_delimiter(DawarichSettings::BASIC_PAID_PLAN_LIMIT) %> available.
You have used <%= number_with_delimiter(current_user.points_count) %> points of <%= number_with_delimiter(DawarichSettings::BASIC_PAID_PLAN_LIMIT) %> available.
</p>
<progress class="progress progress-primary w-1/2 h-5" value="<%= current_user.tracked_points.count %>" max="<%= DawarichSettings::BASIC_PAID_PLAN_LIMIT %>"></progress>
<progress class="progress progress-primary w-1/2 h-5" value="<%= current_user.points_count %>" max="<%= DawarichSettings::BASIC_PAID_PLAN_LIMIT %>"></progress>
</p>

View file

@ -24,7 +24,7 @@
</div>
</td>
<td>
<%= number_with_delimiter user.tracked_points.count %>
<%= number_with_delimiter user.points_count %>
</td>
<td>
<%= human_datetime(user.created_at) %>

View file

@ -8,7 +8,7 @@ class CreateTracksFromPoints < ActiveRecord::Migration[8.0]
processed_users = 0
User.find_each do |user|
points_count = user.tracked_points.count
points_count = user.points.count
if points_count > 0
puts "Enqueuing track creation for user #{user.id} (#{points_count} points)"

View file

@ -0,0 +1,16 @@
class AddPointsCountToUsers < ActiveRecord::Migration[8.0]
def change
add_column :users, :points_count, :integer, default: 0, null: false
# Initialize counter cache for existing users using background job
reversible do |dir|
dir.up do
# Enqueue job to prefill counter cache in background
# This prevents the migration from blocking on large datasets
say_with_time "Enqueueing job to prefill points counter cache" do
DataMigrations::PrefillPointsCounterCacheJob.perform_later
end
end
end
end
end

5
db/schema.rb generated
View file

@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema[8.0].define(version: 2025_07_28_191359) do
ActiveRecord::Schema[8.0].define(version: 2025_08_21_192219) do
# These are extensions that must be enabled in order to support this database
enable_extension "pg_catalog.plpgsql"
enable_extension "postgis"
@ -230,7 +230,7 @@ ActiveRecord::Schema[8.0].define(version: 2025_07_28_191359) do
t.datetime "end_at", null: false
t.bigint "user_id", null: false
t.geometry "original_path", limit: {srid: 0, type: "line_string"}, null: false
t.integer "distance"
t.decimal "distance", precision: 8, scale: 2
t.float "avg_speed"
t.integer "duration"
t.integer "elevation_gain"
@ -274,6 +274,7 @@ ActiveRecord::Schema[8.0].define(version: 2025_07_28_191359) do
t.string "last_sign_in_ip"
t.integer "status", default: 0
t.datetime "active_until"
t.integer "points_count", default: 0, null: false
t.index ["email"], name: "index_users_on_email", unique: true
t.index ["reset_password_token"], name: "index_users_on_reset_password_token", unique: true
end

View file

@ -5,9 +5,8 @@ require 'rails_helper'
RSpec.describe User, type: :model do
describe 'associations' do
it { is_expected.to have_many(:imports).dependent(:destroy) }
it { is_expected.to have_many(:points).through(:imports) }
it { is_expected.to have_many(:stats) }
it { is_expected.to have_many(:tracked_points).class_name('Point').dependent(:destroy) }
it { is_expected.to have_many(:points).class_name('Point').dependent(:destroy) }
it { is_expected.to have_many(:exports).dependent(:destroy) }
it { is_expected.to have_many(:notifications).dependent(:destroy) }
it { is_expected.to have_many(:areas).dependent(:destroy) }
@ -124,7 +123,7 @@ RSpec.describe User, type: :model do
end
it 'returns true' do
user.tracked_points.destroy_all
user.points.destroy_all
expect(user.trial_state?).to be_truthy
end

View file

@ -24,20 +24,20 @@ RSpec.describe PointsLimitExceeded do
context 'when user points count is equal to the limit' do
before do
allow(user.tracked_points).to receive(:count).and_return(10)
allow(user.points).to receive(:count).and_return(10)
end
it { is_expected.to be true }
it 'caches the result' do
expect(user.tracked_points).to receive(:count).once
expect(user.points).to receive(:count).once
2.times { described_class.new(user).call }
end
end
context 'when user points count exceeds the limit' do
before do
allow(user.tracked_points).to receive(:count).and_return(11)
allow(user.points).to receive(:count).and_return(11)
end
it { is_expected.to be true }
@ -45,7 +45,7 @@ RSpec.describe PointsLimitExceeded do
context 'when user points count is below the limit' do
before do
allow(user.tracked_points).to receive(:count).and_return(9)
allow(user.points).to receive(:count).and_return(9)
end
it { is_expected.to be false }

View file

@ -45,7 +45,7 @@ RSpec.describe Users::ExportData, type: :service do
allow(user).to receive(:trips).and_return(double(count: 8))
allow(user).to receive(:stats).and_return(double(count: 24))
allow(user).to receive(:notifications).and_return(double(count: 10))
allow(user).to receive(:tracked_points).and_return(double(count: 15000))
allow(user).to receive(:points).and_return(double(count: 15000))
allow(user).to receive(:visits).and_return(double(count: 45))
allow(user).to receive(:places).and_return(double(count: 20))
@ -187,7 +187,7 @@ RSpec.describe Users::ExportData, type: :service do
allow(user).to receive(:trips).and_return(double(count: 8))
allow(user).to receive(:stats).and_return(double(count: 24))
allow(user).to receive(:notifications).and_return(double(count: 10))
allow(user).to receive(:tracked_points).and_return(double(count: 15000))
allow(user).to receive(:points).and_return(double(count: 15000))
allow(user).to receive(:visits).and_return(double(count: 45))
allow(user).to receive(:places).and_return(double(count: 20))
@ -267,7 +267,7 @@ RSpec.describe Users::ExportData, type: :service do
allow(user).to receive(:trips).and_return(double(count: 8))
allow(user).to receive(:stats).and_return(double(count: 24))
allow(user).to receive(:notifications).and_return(double(count: 10))
allow(user).to receive(:tracked_points).and_return(double(count: 15000))
allow(user).to receive(:points).and_return(double(count: 15000))
allow(user).to receive(:visits).and_return(double(count: 45))
allow(user).to receive(:places).and_return(double(count: 20))
@ -374,7 +374,7 @@ RSpec.describe Users::ExportData, type: :service do
allow(user).to receive(:trips).and_return(double(count: 8))
allow(user).to receive(:stats).and_return(double(count: 24))
allow(user).to receive(:notifications).and_return(double(count: 10))
allow(user).to receive(:tracked_points).and_return(double(count: 15000))
allow(user).to receive(:points).and_return(double(count: 15000))
allow(user).to receive(:visits).and_return(double(count: 45))
allow(user).to receive(:places).and_return(double(count: 20))
allow(Rails.logger).to receive(:info)

View file

@ -312,33 +312,33 @@ RSpec.describe 'Users Export-Import Integration', type: :service do
trips: user.trips.count,
stats: user.stats.count,
notifications: user.notifications.count,
points: user.tracked_points.count,
points: user.points.count,
visits: user.visits.count,
places: user.places.count
}
end
def verify_relationships_preserved(original_user, target_user)
original_points_with_imports = original_user.tracked_points.where.not(import_id: nil).count
target_points_with_imports = target_user.tracked_points.where.not(import_id: nil).count
original_points_with_imports = original_user.points.where.not(import_id: nil).count
target_points_with_imports = target_user.points.where.not(import_id: nil).count
expect(target_points_with_imports).to eq(original_points_with_imports)
original_points_with_countries = original_user.tracked_points.where.not(country_id: nil).count
target_points_with_countries = target_user.tracked_points.where.not(country_id: nil).count
original_points_with_countries = original_user.points.where.not(country_id: nil).count
target_points_with_countries = target_user.points.where.not(country_id: nil).count
expect(target_points_with_countries).to eq(original_points_with_countries)
original_points_with_visits = original_user.tracked_points.where.not(visit_id: nil).count
target_points_with_visits = target_user.tracked_points.where.not(visit_id: nil).count
original_points_with_visits = original_user.points.where.not(visit_id: nil).count
target_points_with_visits = target_user.points.where.not(visit_id: nil).count
expect(target_points_with_visits).to eq(original_points_with_visits)
original_visits_with_places = original_user.visits.where.not(place_id: nil).count
target_visits_with_places = target_user.visits.where.not(place_id: nil).count
expect(target_visits_with_places).to eq(original_visits_with_places)
original_office_points = original_user.tracked_points.where(
original_office_points = original_user.points.where(
latitude: 40.7589, longitude: -73.9851
).first
target_office_points = target_user.tracked_points.where(
target_office_points = target_user.points.where(
latitude: 40.7589, longitude: -73.9851
).first

View file

@ -35,13 +35,13 @@ RSpec.describe Users::ImportData::Points, type: :service do
it 'assigns the correct country association' do
service.call
point = user.tracked_points.last
point = user.points.last
expect(point.country).to eq(country)
end
it 'excludes the string country field from attributes' do
service.call
point = user.tracked_points.last
point = user.points.last
# The country association should be set, not the string attribute
expect(point.read_attribute(:country)).to be_nil
expect(point.country).to eq(country)
@ -68,7 +68,7 @@ RSpec.describe Users::ImportData::Points, type: :service do
it 'does not create country and leaves country_id nil' do
expect { service.call }.not_to change(Country, :count)
point = user.tracked_points.last
point = user.points.last
expect(point.country_id).to be_nil
expect(point.city).to eq('Berlin')
end
@ -126,10 +126,10 @@ RSpec.describe Users::ImportData::Points, type: :service do
it 'imports valid points and reconstructs lonlat when needed' do
expect(service.call).to eq(2) # Two valid points (original + reconstructed)
expect(user.tracked_points.count).to eq(2)
expect(user.points.count).to eq(2)
# Check that lonlat was reconstructed properly
munich_point = user.tracked_points.find_by(city: 'Munich')
munich_point = user.points.find_by(city: 'Munich')
expect(munich_point).to be_present
expect(munich_point.lonlat.to_s).to match(/POINT\s*\(11\.582\s+48\.1351\)/)
end

View file

@ -27,7 +27,7 @@ RSpec.describe Visits::SmartDetect do
let(:created_visits) { [instance_double(Visit)] }
before do
allow(user).to receive_message_chain(:tracked_points, :not_visited, :order, :where).and_return(points)
allow(user).to receive_message_chain(:points, :not_visited, :order, :where).and_return(points)
allow(Visits::Detector).to receive(:new).with(points).and_return(visit_detector)
allow(Visits::Merger).to receive(:new).with(points).and_return(visit_merger)
allow(Visits::Creator).to receive(:new).with(user).and_return(visit_creator)