Merge pull request #1553 from Freika/fix/stats-calculation-performance

Fix stats calculation performance

.app_version

@@ -1 +1 @@
-0.30.1
+0.30.2

CHANGELOG.md

@@ -4,13 +4,25 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
-# [0.30.1] - 2025-07-21
+# [0.30.2] - 2025-07-22
+
+## Fixed
+
+- Stats calculation is now significantly faster.
+
+
+# [0.30.1] - 2025-07-22
 
 ## Fixed
 
 - Points limit exceeded check is now cached.
 - Reverse geocoding for places is now significantly faster.
 
+## Changed
+
+- Stats page should load faster now.
+- Track creation is temporarily disabled.
+
 
 # [0.30.0] - 2025-07-21
 

Gemfile.lock

@@ -126,7 +126,7 @@ GEM
       rack-test (>= 0.6.3)
       regexp_parser (>= 1.5, < 3.0)
       xpath (~> 3.2)
-    chartkick (5.1.5)
+    chartkick (5.2.0)
     coderay (1.1.3)
     concurrent-ruby (1.3.5)
     connection_pool (2.5.3)
@@ -144,7 +144,7 @@ GEM
     database_consistency (2.0.4)
       activerecord (>= 3.2)
     date (3.4.1)
-    debug (1.10.0)
+    debug (1.11.0)
       irb (~> 1.10)
       reline (>= 0.3.8)
     devise (4.9.4)
@@ -160,7 +160,7 @@ GEM
       dotenv (= 3.1.8)
       railties (>= 6.1)
     drb (2.2.3)
-    erb (5.0.1)
+    erb (5.0.2)
     erubi (1.13.1)
     et-orbi (1.2.11)
       tzinfo
@@ -194,7 +194,7 @@ GEM
       actionpack (>= 6.0.0)
       activesupport (>= 6.0.0)
       railties (>= 6.0.0)
-    io-console (0.8.0)
+    io-console (0.8.1)
     irb (1.15.2)
       pp (>= 0.6.0)
       rdoc (>= 4.0.0)
@@ -243,7 +243,7 @@ GEM
     multi_json (1.15.0)
     multi_xml (0.7.1)
       bigdecimal (~> 3.1)
-    net-imap (0.5.8)
+    net-imap (0.5.9)
       date
       net-protocol
     net-pop (0.1.2)
@@ -253,23 +253,23 @@ GEM
     net-smtp (0.5.1)
       net-protocol
     nio4r (2.7.4)
-    nokogiri (1.18.8)
+    nokogiri (1.18.9)
       mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
-    nokogiri (1.18.8-aarch64-linux-gnu)
+    nokogiri (1.18.9-aarch64-linux-gnu)
       racc (~> 1.4)
-    nokogiri (1.18.8-arm-linux-gnu)
+    nokogiri (1.18.9-arm-linux-gnu)
       racc (~> 1.4)
-    nokogiri (1.18.8-arm64-darwin)
+    nokogiri (1.18.9-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.18.8-x86_64-darwin)
+    nokogiri (1.18.9-x86_64-darwin)
       racc (~> 1.4)
-    nokogiri (1.18.8-x86_64-linux-gnu)
+    nokogiri (1.18.9-x86_64-linux-gnu)
       racc (~> 1.4)
     oj (3.16.11)
       bigdecimal (>= 3.0)
       ostruct (>= 0.2)
-    optimist (3.2.0)
+    optimist (3.2.1)
     orm_adapter (0.5.0)
     ostruct (0.6.1)
     parallel (1.27.0)
@@ -342,7 +342,7 @@ GEM
       zeitwerk (~> 2.6)
     rainbow (3.1.1)
     rake (13.3.0)
-    rdoc (6.14.1)
+    rdoc (6.14.2)
       erb
       psych (>= 4.0.0)
     redis (5.4.0)
@@ -350,7 +350,7 @@ GEM
     redis-client (0.24.0)
       connection_pool
     regexp_parser (2.10.0)
-    reline (0.6.1)
+    reline (0.6.2)
       io-console (~> 0.5)
     request_store (1.7.0)
       rack (>= 1.4)
@@ -462,7 +462,7 @@ GEM
     stringio (3.1.7)
     strong_migrations (2.3.0)
       activerecord (>= 7)
-    super_diff (0.15.0)
+    super_diff (0.16.0)
       attr_extras (>= 6.2.4)
       diff-lcs
       patience_diff
@@ -475,7 +475,7 @@ GEM
     tailwindcss-ruby (3.4.17-arm64-darwin)
     tailwindcss-ruby (3.4.17-x86_64-darwin)
     tailwindcss-ruby (3.4.17-x86_64-linux)
-    thor (1.3.2)
+    thor (1.4.0)
     timeout (0.4.3)
     turbo-rails (2.0.16)
       actionpack (>= 7.1.0)
@@ -496,7 +496,7 @@ GEM
       hashdiff (>= 0.4.0, < 2.0.0)
     webrick (1.9.1)
     websocket (1.2.11)
-    websocket-driver (0.7.7)
+    websocket-driver (0.8.0)
       base64
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)

app/controllers/stats_controller.rb

@@ -5,15 +5,15 @@ class StatsController < ApplicationController
   before_action :authenticate_active_user!, only: %i[update update_all]
 
   def index
-    @stats = current_user.stats.group_by(&:year).transform_values { |stats| stats.sort_by(&:updated_at).reverse }.sort.reverse
-    @points_total = current_user.tracked_points.count
-    @points_reverse_geocoded = current_user.total_reverse_geocoded_points
-    @points_reverse_geocoded_without_data = current_user.total_reverse_geocoded_points_without_data
+    @stats = build_stats
+    assign_points_statistics
+    @year_distances = precompute_year_distances
   end
 
   def show
     @year = params[:year].to_i
     @stats = current_user.stats.where(year: @year).order(:month)
+    @year_distances = { @year => Stat.year_distance(@year, current_user) }
   end
 
   def update
@@ -43,4 +43,30 @@ class StatsController < ApplicationController
 
     redirect_to stats_path, notice: 'Stats are being updated', status: :see_other
   end
+
+  private
+
+  def assign_points_statistics
+    points_stats = ::StatsQuery.new(current_user).points_stats
+
+    @points_total = points_stats[:total]
+    @points_reverse_geocoded = points_stats[:geocoded]
+    @points_reverse_geocoded_without_data = points_stats[:without_data]
+  end
+
+  def precompute_year_distances
+    year_distances = {}
+
+    @stats.each do |year, _stats|
+      year_distances[year] = Stat.year_distance(year, current_user)
+    end
+
+    year_distances
+  end
+
+  def build_stats
+    current_user.stats.group_by(&:year).transform_values do |stats|
+      stats.sort_by(&:updated_at).reverse
+    end.sort.reverse
+  end
 end

app/javascript/controllers/maps_controller.js

@@ -219,7 +219,7 @@ export default class extends BaseController {
     this.setupTracksSubscription();
 
     // Handle routes/tracks mode selection
-    this.addRoutesTracksSelector();
+    // this.addRoutesTracksSelector(); # Temporarily disabled
     this.switchRouteMode('routes', true);
 
     // Initialize layers based on settings

app/jobs/tracks/create_job.rb

@@ -27,6 +27,8 @@ class Tracks::CreateJob < ApplicationJob
   end
 
   def create_error_notification(user, error)
+    return unless DawarichSettings.self_hosted?
+
     Notifications::Create.new(
       user: user,
       kind: :error,

app/jobs/trips/calculate_all_job.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class Trips::CalculateAllJob < ApplicationJob
-  queue_as :default
+  queue_as :trips
 
   def perform(trip_id, distance_unit = 'km')
     Trips::CalculatePathJob.perform_later(trip_id)

app/jobs/trips/calculate_countries_job.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class Trips::CalculateCountriesJob < ApplicationJob
-  queue_as :default
+  queue_as :trips
 
   def perform(trip_id, distance_unit)
     trip = Trip.find(trip_id)

app/jobs/trips/calculate_distance_job.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class Trips::CalculateDistanceJob < ApplicationJob
-  queue_as :default
+  queue_as :trips
 
   def perform(trip_id, distance_unit)
     trip = Trip.find(trip_id)

app/jobs/trips/calculate_path_job.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class Trips::CalculatePathJob < ApplicationJob
-  queue_as :default
+  queue_as :trips
 
   def perform(trip_id)
     trip = Trip.find(trip_id)

app/models/country.rb

@@ -12,6 +12,8 @@ class Country < ApplicationRecord
   end
 
   def self.names_to_iso_a2
-    pluck(:name, :iso_a2).to_h
+    Rails.cache.fetch('countries_names_to_iso_a2', expires_in: 1.day) do
+      pluck(:name, :iso_a2).to_h
+    end
   end
 end

app/models/point.rb

@@ -33,8 +33,8 @@ class Point < ApplicationRecord
   after_create :async_reverse_geocode, if: -> { DawarichSettings.store_geodata? && !reverse_geocoded? }
   after_create :set_country
   after_create_commit :broadcast_coordinates
-  after_create_commit :trigger_incremental_track_generation, if: -> { import_id.nil? }
-  after_commit :recalculate_track, on: :update, if: -> { track.present? }
+  # after_create_commit :trigger_incremental_track_generation, if: -> { import_id.nil? }
+  # after_commit :recalculate_track, on: :update, if: -> { track.present? }
 
   def self.without_raw_data
     select(column_names - ['raw_data'])

app/models/stat.rb

@@ -37,18 +37,16 @@ class Stat < ApplicationRecord
   end
 
   def calculate_daily_distances(monthly_points)
-    timespan.to_a.map.with_index(1) do |day, index|
-      daily_points = filter_points_for_day(monthly_points, day)
-      # Calculate distance in meters for consistent storage
-      distance_meters = Point.total_distance(daily_points, :m)
-      [index, distance_meters.round]
-    end
+    Stats::DailyDistanceQuery.new(monthly_points, timespan, user_timezone).call
   end
 
-  def filter_points_for_day(points, day)
-    beginning_of_day = day.beginning_of_day.to_i
-    end_of_day = day.end_of_day.to_i
+  private
 
-    points.select { |p| p.timestamp.between?(beginning_of_day, end_of_day) }
+  def user_timezone
+    # Future: Once user.timezone column exists, uncomment the line below
+    # user.timezone.presence || Time.zone.name
+
+    # For now, use application timezone
+    Time.zone.name
   end
 end

app/queries/stats/daily_distance_query.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+class Stats::DailyDistanceQuery
+  def initialize(monthly_points, timespan, timezone = nil)
+    @monthly_points = monthly_points
+    @timespan = timespan
+    @timezone = validate_timezone(timezone)
+  end
+
+  def call
+    daily_distances = daily_distances(monthly_points)
+    distance_by_day_map = distance_by_day_map(daily_distances)
+
+    convert_to_daily_distances(distance_by_day_map)
+  end
+
+  private
+
+  attr_reader :monthly_points, :timespan, :timezone
+
+  def daily_distances(monthly_points)
+    Stat.connection.select_all(<<-SQL.squish)
+      WITH points_with_distances AS (
+        SELECT
+          EXTRACT(day FROM (to_timestamp(timestamp) AT TIME ZONE 'UTC' AT TIME ZONE '#{timezone}')) as day_of_month,
+          CASE
+            WHEN LAG(lonlat) OVER (
+              PARTITION BY EXTRACT(day FROM (to_timestamp(timestamp) AT TIME ZONE 'UTC' AT TIME ZONE '#{timezone}'))
+              ORDER BY timestamp
+            ) IS NOT NULL THEN
+              ST_Distance(
+                lonlat::geography,
+                LAG(lonlat) OVER (
+                  PARTITION BY EXTRACT(day FROM (to_timestamp(timestamp) AT TIME ZONE 'UTC' AT TIME ZONE '#{timezone}'))
+                  ORDER BY timestamp
+                )::geography
+              )
+            ELSE 0
+          END as segment_distance
+        FROM (#{monthly_points.to_sql}) as points
+      )
+      SELECT
+        day_of_month,
+        ROUND(COALESCE(SUM(segment_distance), 0)) as distance_meters
+      FROM points_with_distances
+      GROUP BY day_of_month
+      ORDER BY day_of_month
+    SQL
+  end
+
+  def distance_by_day_map(daily_distances)
+    daily_distances.index_by do |row|
+      row['day_of_month'].to_i
+    end
+  end
+
+  def convert_to_daily_distances(distance_by_day_map)
+    timespan.to_a.map.with_index(1) do |day, index|
+      distance_meters =
+        distance_by_day_map[day.day]&.fetch('distance_meters', 0) || 0
+
+      [index, distance_meters.to_i]
+    end
+  end
+
+  def validate_timezone(timezone)
+    return timezone if ActiveSupport::TimeZone.all.any? { |tz| tz.name == timezone }
+
+    'UTC'
+  end
+end

app/queries/stats_query.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+class StatsQuery
+  def initialize(user)
+    @user = user
+  end
+
+  def points_stats
+    sql = ActiveRecord::Base.sanitize_sql_array([
+      <<~SQL.squish,
+        SELECT
+          COUNT(id) as total,
+          COUNT(reverse_geocoded_at) as geocoded,
+          COUNT(CASE WHEN geodata = '{}'::jsonb THEN 1 END) as without_data
+        FROM points
+        WHERE user_id = ?
+      SQL
+      user.id
+    ])
+
+    result = Point.connection.select_one(sql)
+
+    {
+      total: result['total'].to_i,
+      geocoded: result['geocoded'].to_i,
+      without_data: result['without_data'].to_i
+    }
+  end
+
+  private
+
+  attr_reader :user
+end

app/services/stats/bulk_calculator.rb

@@ -21,7 +21,7 @@ module Stats
       last_calculated_at ||= DateTime.new(1970, 1, 1)
 
       time_diff = last_calculated_at.to_i..Time.current.to_i
-      Point.where(user_id:, timestamp: time_diff).pluck(:timestamp)
+      Point.where(user_id:, timestamp: time_diff).pluck(:timestamp).uniq
     end
 
     def extract_months(timestamps)

app/services/stats/calculate_month.rb

@@ -50,7 +50,7 @@ class Stats::CalculateMonth
               .tracked_points
               .without_raw_data
               .where(timestamp: start_timestamp..end_timestamp)
-              .select(:lonlat, :timestamp, :city, :country)
+              .select(:lonlat, :timestamp)
               .order(timestamp: :asc)
   end
 
@@ -59,7 +59,14 @@ class Stats::CalculateMonth
   end
 
   def toponyms
-    CountriesAndCities.new(points).call
+    toponym_points = user
+                      .tracked_points
+                      .without_raw_data
+                      .where(timestamp: start_timestamp..end_timestamp)
+                      .select(:city, :country)
+                      .distinct
+
+    CountriesAndCities.new(toponym_points).call
   end
 
   def create_stats_update_failed_notification(user, error)

app/views/stats/_stat.html.erb

@@ -20,7 +20,7 @@
 
   <%= area_chart(
     stat.daily_distance.map { |day, distance_meters|
-      [day, Stat.convert_distance(distance_meters, current_user.safe_settings.distance_unit).round(2)]
+      [day, Stat.convert_distance(distance_meters, current_user.safe_settings.distance_unit).round]
     },
     height: '200px',
     suffix: " #{current_user.safe_settings.distance_unit}",

app/views/stats/_year.html.erb

@@ -4,7 +4,7 @@
 </h2>
 <div class='my-10'>
   <%= column_chart(
-    Stat.year_distance(year, current_user),
+    @year_distances[year],
     height: '200px',
     suffix: " #{current_user.safe_settings.distance_unit}",
     xtitle: 'Days',

app/views/stats/index.html.erb

@@ -82,8 +82,8 @@
             </div>
           <% end %>
           <%= column_chart(
-            Stat.year_distance(year, current_user).map { |month_name, distance_meters|
-              [month_name, Stat.convert_distance(distance_meters, current_user.safe_settings.distance_unit).round(2)]
+            @year_distances[year].map { |month_name, distance_meters|
+              [month_name, Stat.convert_distance(distance_meters, current_user.safe_settings.distance_unit).round]
             },
             height: '200px',
             suffix: " #{current_user.safe_settings.distance_unit}",

config/schedule.yml

@@ -30,10 +30,10 @@ cache_preheating_job:
   class: "Cache::PreheatingJob"
   queue: default
 
-tracks_cleanup_job:
-  cron: "0 2 * * 0" # every Sunday at 02:00
-  class: "Tracks::CleanupJob"
-  queue: tracks
+# tracks_cleanup_job:
+#   cron: "0 2 * * 0" # every Sunday at 02:00
+#   class: "Tracks::CleanupJob"
+#   queue: tracks
 
 place_name_fetching_job:
   cron: "30 0 * * *" # every day at 00:30

config/sidekiq.yml

@@ -6,6 +6,7 @@
   - imports
   - exports
   - stats
+  - trips
   - tracks
   - reverse_geocoding
   - visit_suggesting

db/migrate/20250721204404_add_index_on_places_geodata_osm_id.rb

@@ -2,8 +2,8 @@ class AddIndexOnPlacesGeodataOsmId < ActiveRecord::Migration[8.0]
   disable_ddl_transaction!
 
   def change
-    add_index :places, "(geodata->'properties'->>'osm_id')", 
-              using: :btree, 
+    add_index :places, "(geodata->'properties'->>'osm_id')",
+              using: :btree,
               name: 'index_places_on_geodata_osm_id',
               algorithm: :concurrently
   end

spec/jobs/tracks/create_job_spec.rb

@@ -151,4 +151,50 @@ RSpec.describe Tracks::CreateJob, type: :job do
       expect(described_class.new.queue_name).to eq('tracks')
     end
   end
+
+  context 'when self-hosted' do
+    let(:generator_instance) { instance_double(Tracks::Generator) }
+    let(:notification_service) { instance_double(Notifications::Create) }
+    let(:error_message) { 'Something went wrong' }
+
+    before do
+      allow(DawarichSettings).to receive(:self_hosted?).and_return(true)
+      allow(Tracks::Generator).to receive(:new).and_return(generator_instance)
+      allow(generator_instance).to receive(:call).and_raise(StandardError, error_message)
+      allow(Notifications::Create).to receive(:new).and_return(notification_service)
+      allow(notification_service).to receive(:call)
+    end
+
+    it 'creates a failure notification when self-hosted' do
+      described_class.new.perform(user.id)
+
+      expect(Notifications::Create).to have_received(:new).with(
+        user: user,
+        kind: :error,
+        title: 'Track Generation Failed',
+        content: "Failed to generate tracks from your location data: #{error_message}"
+      )
+      expect(notification_service).to have_received(:call)
+    end
+  end
+
+  context 'when not self-hosted' do
+    let(:generator_instance) { instance_double(Tracks::Generator) }
+    let(:notification_service) { instance_double(Notifications::Create) }
+    let(:error_message) { 'Something went wrong' }
+
+    before do
+      allow(DawarichSettings).to receive(:self_hosted?).and_return(false)
+      allow(Tracks::Generator).to receive(:new).and_return(generator_instance)
+      allow(generator_instance).to receive(:call).and_raise(StandardError, error_message)
+      allow(Notifications::Create).to receive(:new).and_return(notification_service)
+      allow(notification_service).to receive(:call)
+    end
+
+    it 'does not create a failure notification' do
+      described_class.new.perform(user.id)
+
+      expect(notification_service).not_to have_received(:call)
+    end
+  end
 end

spec/models/point_spec.rb

@@ -30,7 +30,7 @@ RSpec.describe Point, type: :model do
       end
     end
 
-    describe '#recalculate_track' do
+    xdescribe '#recalculate_track' do
       let(:point) { create(:point, track: track) }
       let(:track) { create(:track) }
 
@@ -121,7 +121,7 @@ RSpec.describe Point, type: :model do
       end
     end
 
-    describe '#trigger_incremental_track_generation' do
+    xdescribe '#trigger_incremental_track_generation' do
       let(:point) do
         create(:point, track: track, import_id: nil, timestamp: 1.hour.ago.to_i, reverse_geocoded_at: 1.hour.ago)
       end

spec/queries/stats_query_spec.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe StatsQuery do
+  describe '#points_stats' do
+    subject(:points_stats) { described_class.new(user).points_stats }
+
+    let(:user) { create(:user) }
+    let!(:import) { create(:import, user: user) }
+
+    context 'when user has no points' do
+      it 'returns zero counts for all statistics' do
+        expect(points_stats).to eq({
+          total: 0,
+          geocoded: 0,
+          without_data: 0
+        })
+      end
+    end
+
+    context 'when user has points' do
+      let!(:geocoded_point_with_data) do
+        create(:point,
+               user: user,
+               import: import,
+               reverse_geocoded_at: Time.current,
+               geodata: { 'address' => '123 Main St' })
+      end
+
+      let!(:geocoded_point_without_data) do
+        create(:point,
+               user: user,
+               import: import,
+               reverse_geocoded_at: Time.current,
+               geodata: {})
+      end
+
+      let!(:non_geocoded_point) do
+        create(:point,
+               user: user,
+               import: import,
+               reverse_geocoded_at: nil,
+               geodata: { 'some' => 'data' })
+      end
+
+      it 'returns correct counts for all statistics' do
+        expect(points_stats).to eq({
+          total: 3,
+          geocoded: 2,
+          without_data: 1
+        })
+      end
+
+      context 'when another user has points' do
+        let(:other_user) { create(:user) }
+        let!(:other_import) { create(:import, user: other_user) }
+        let!(:other_point) do
+          create(:point,
+                 user: other_user,
+                 import: other_import,
+                 reverse_geocoded_at: Time.current,
+                 geodata: { 'address' => 'Other Address' })
+        end
+
+        it 'only counts points for the specified user' do
+          expect(points_stats).to eq({
+            total: 3,
+            geocoded: 2,
+            without_data: 1
+          })
+        end
+      end
+    end
+
+    context 'when all points are geocoded with data' do
+      before do
+        create_list(:point, 5,
+                    user: user,
+                    import: import,
+                    reverse_geocoded_at: Time.current,
+                    geodata: { 'address' => 'Some Address' })
+      end
+
+      it 'returns correct statistics' do
+        expect(points_stats).to eq({
+          total: 5,
+          geocoded: 5,
+          without_data: 0
+        })
+      end
+    end
+
+    context 'when all points are without geodata' do
+      before do
+        create_list(:point, 3,
+                    user: user,
+                    import: import,
+                    reverse_geocoded_at: Time.current,
+                    geodata: {})
+      end
+
+      it 'returns correct statistics' do
+        expect(points_stats).to eq({
+          total: 3,
+          geocoded: 3,
+          without_data: 3
+        })
+      end
+    end
+
+    context 'when all points are not geocoded' do
+      before do
+        create_list(:point, 4,
+                    user: user,
+                    import: import,
+                    reverse_geocoded_at: nil,
+                    geodata: { 'some' => 'data' })
+      end
+
+      it 'returns correct statistics' do
+        expect(points_stats).to eq({
+          total: 4,
+          geocoded: 0,
+          without_data: 0
+        })
+      end
+    end
+  end
+end