Merge 35f4c0f1f6 into 4044e77fcd

Fix stats calculation performance

.app_version

@@ -1 +1 @@
-0.30.1
+0.30.2

CHANGELOG.md

@@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+# [0.30.2] - 2025-07-22
+
+## Fixed
+
+- Stats calculation is now significantly faster.
+
+
 # [0.30.1] - 2025-07-22
 
 ## Fixed

app/controllers/stats_controller.rb

@@ -13,6 +13,7 @@ class StatsController < ApplicationController
   def show
     @year = params[:year].to_i
     @stats = current_user.stats.where(year: @year).order(:month)
+    @year_distances = { @year => Stat.year_distance(@year, current_user) }
   end
 
   def update

app/models/stat.rb

@@ -37,18 +37,16 @@ class Stat < ApplicationRecord
   end
 
   def calculate_daily_distances(monthly_points)
-    timespan.to_a.map.with_index(1) do |day, index|
-      daily_points = filter_points_for_day(monthly_points, day)
-      # Calculate distance in meters for consistent storage
-      distance_meters = Point.total_distance(daily_points, :m)
-      [index, distance_meters.round]
-    end
+    Stats::DailyDistanceQuery.new(monthly_points, timespan, user_timezone).call
   end
 
-  def filter_points_for_day(points, day)
-    beginning_of_day = day.beginning_of_day.to_i
-    end_of_day = day.end_of_day.to_i
+  private
 
-    points.select { |p| p.timestamp.between?(beginning_of_day, end_of_day) }
+  def user_timezone
+    # Future: Once user.timezone column exists, uncomment the line below
+    # user.timezone.presence || Time.zone.name
+
+    # For now, use application timezone
+    Time.zone.name
   end
 end

app/queries/stats/daily_distance_query.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+class Stats::DailyDistanceQuery
+  def initialize(monthly_points, timespan, timezone = nil)
+    @monthly_points = monthly_points
+    @timespan = timespan
+    @timezone = validate_timezone(timezone)
+  end
+
+  def call
+    daily_distances = daily_distances(monthly_points)
+    distance_by_day_map = distance_by_day_map(daily_distances)
+
+    convert_to_daily_distances(distance_by_day_map)
+  end
+
+  private
+
+  attr_reader :monthly_points, :timespan, :timezone
+
+  def daily_distances(monthly_points)
+    Stat.connection.select_all(<<-SQL.squish)
+      WITH points_with_distances AS (
+        SELECT
+          EXTRACT(day FROM (to_timestamp(timestamp) AT TIME ZONE 'UTC' AT TIME ZONE '#{timezone}')) as day_of_month,
+          CASE
+            WHEN LAG(lonlat) OVER (
+              PARTITION BY EXTRACT(day FROM (to_timestamp(timestamp) AT TIME ZONE 'UTC' AT TIME ZONE '#{timezone}'))
+              ORDER BY timestamp
+            ) IS NOT NULL THEN
+              ST_Distance(
+                lonlat::geography,
+                LAG(lonlat) OVER (
+                  PARTITION BY EXTRACT(day FROM (to_timestamp(timestamp) AT TIME ZONE 'UTC' AT TIME ZONE '#{timezone}'))
+                  ORDER BY timestamp
+                )::geography
+              )
+            ELSE 0
+          END as segment_distance
+        FROM (#{monthly_points.to_sql}) as points
+      )
+      SELECT
+        day_of_month,
+        ROUND(COALESCE(SUM(segment_distance), 0)) as distance_meters
+      FROM points_with_distances
+      GROUP BY day_of_month
+      ORDER BY day_of_month
+    SQL
+  end
+
+  def distance_by_day_map(daily_distances)
+    daily_distances.index_by do |row|
+      row['day_of_month'].to_i
+    end
+  end
+
+  def convert_to_daily_distances(distance_by_day_map)
+    timespan.to_a.map.with_index(1) do |day, index|
+      distance_meters =
+        distance_by_day_map[day.day]&.fetch('distance_meters', 0) || 0
+
+      [index, distance_meters.to_i]
+    end
+  end
+
+  def validate_timezone(timezone)
+    return timezone if ActiveSupport::TimeZone.all.any? { |tz| tz.name == timezone }
+
+    'UTC'
+  end
+end

app/services/stats/bulk_calculator.rb

@@ -21,7 +21,7 @@ module Stats
       last_calculated_at ||= DateTime.new(1970, 1, 1)
 
       time_diff = last_calculated_at.to_i..Time.current.to_i
-      Point.where(user_id:, timestamp: time_diff).pluck(:timestamp)
+      Point.where(user_id:, timestamp: time_diff).pluck(:timestamp).uniq
     end
 
     def extract_months(timestamps)

app/services/stats/calculate_month.rb

@@ -50,7 +50,7 @@ class Stats::CalculateMonth
               .tracked_points
               .without_raw_data
               .where(timestamp: start_timestamp..end_timestamp)
-              .select(:lonlat, :timestamp, :city, :country)
+              .select(:lonlat, :timestamp)
               .order(timestamp: :asc)
   end
 
@@ -59,7 +59,14 @@ class Stats::CalculateMonth
   end
 
   def toponyms
-    CountriesAndCities.new(points).call
+    toponym_points = user
+                      .tracked_points
+                      .without_raw_data
+                      .where(timestamp: start_timestamp..end_timestamp)
+                      .select(:city, :country)
+                      .distinct
+
+    CountriesAndCities.new(toponym_points).call
   end
 
   def create_stats_update_failed_notification(user, error)

app/views/stats/_stat.html.erb

@@ -20,7 +20,7 @@
 
   <%= area_chart(
     stat.daily_distance.map { |day, distance_meters|
-      [day, Stat.convert_distance(distance_meters, current_user.safe_settings.distance_unit).round(2)]
+      [day, Stat.convert_distance(distance_meters, current_user.safe_settings.distance_unit).round]
     },
     height: '200px',
     suffix: " #{current_user.safe_settings.distance_unit}",

app/views/stats/index.html.erb

@@ -83,7 +83,7 @@
           <% end %>
           <%= column_chart(
             @year_distances[year].map { |month_name, distance_meters|
-              [month_name, Stat.convert_distance(distance_meters, current_user.safe_settings.distance_unit).round(2)]
+              [month_name, Stat.convert_distance(distance_meters, current_user.safe_settings.distance_unit).round]
             },
             height: '200px',
             suffix: " #{current_user.safe_settings.distance_unit}",

config/cable.yml

@@ -1,11 +1,13 @@
 development:
   adapter: redis
-  url: <%= "#{ENV.fetch("REDIS_URL")}/#{ENV.fetch('RAILS_WS_DB', 2)}" %>
+  url: <%= "#{ENV.fetch("REDIS_URL")}" %>
+  db: <%= "#{ENV.fetch('RAILS_WS_DB', 2)}" %>
 
 test:
   adapter: test
 
 production:
   adapter: redis
-  url: <%= "#{ENV.fetch("REDIS_URL")}/#{ENV.fetch('RAILS_WS_DB', 2)}" %>
+  url: <%= "#{ENV.fetch("REDIS_URL")}" %>
+  db: <%= "#{ENV.fetch('RAILS_WS_DB', 2)}" %>
   channel_prefix: dawarich_production

config/environments/development.rb

@@ -26,7 +26,7 @@ Rails.application.configure do
 
   # Enable/disable caching. By default caching is disabled.
   # Run rails dev:cache to toggle caching.
-  config.cache_store = :redis_cache_store, { url: "#{ENV['REDIS_URL']}/#{ENV.fetch('RAILS_CACHE_DB', 0)}" }
+  config.cache_store = :redis_cache_store, { url: ENV['REDIS_URL'], db: ENV.fetch('RAILS_CACHE_DB', 0) }
 
   if Rails.root.join('tmp/caching-dev.txt').exist?
     config.action_controller.perform_caching = true

config/environments/production.rb

@@ -73,7 +73,7 @@ Rails.application.configure do
   config.log_level = ENV.fetch('RAILS_LOG_LEVEL', 'info')
 
   # Use a different cache store in production.
-  config.cache_store = :redis_cache_store, { url: "#{ENV['REDIS_URL']}/#{ENV.fetch('RAILS_CACHE_DB', 0)}" }
+  config.cache_store = :redis_cache_store, { url: ENV['REDIS_URL'], db: ENV.fetch('RAILS_CACHE_DB', 0) }
 
   # Use a real queuing backend for Active Job (and separate queues per environment).
   config.active_job.queue_adapter = :sidekiq

config/initializers/geocoder.rb

@@ -4,7 +4,7 @@ settings = {
   debug_mode: true,
   timeout: 5,
   units: :km,
-  cache: Redis.new(url: "#{ENV['REDIS_URL']}/#{ENV.fetch('RAILS_CACHE_DB', 0)}"),
+  cache: Redis.new(url: ENV['REDIS_URL'], db: ENV.fetch('RAILS_CACHE_DB', 0)),
   always_raise: :all,
   http_headers: {
     'User-Agent' => "Dawarich #{APP_VERSION} (https://dawarich.app)"

config/initializers/sidekiq.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 Sidekiq.configure_server do |config|
-  config.redis = { url: "#{ENV['REDIS_URL']}/#{ENV.fetch('RAILS_JOB_QUEUE_DB', 1)}" }
+  config.redis = { url: ENV['REDIS_URL'], db: ENV.fetch('RAILS_JOB_QUEUE_DB', 1) }
   config.logger = Sidekiq::Logger.new($stdout)
 
   if ENV['PROMETHEUS_EXPORTER_ENABLED'].to_s == 'true'
@@ -24,7 +24,7 @@ Sidekiq.configure_server do |config|
 end
 
 Sidekiq.configure_client do |config|
-  config.redis = { url: "#{ENV['REDIS_URL']}/#{ENV.fetch('RAILS_JOB_QUEUE_DB', 1)}" }
+  config.redis = { url: ENV['REDIS_URL'], db: ENV.fetch('RAILS_JOB_QUEUE_DB', 1) }
 end
 
 Sidekiq::Queue['reverse_geocoding'].limit = 1 if Sidekiq.server? && DawarichSettings.photon_uses_komoot_io?

db/migrate/20250721204404_add_index_on_places_geodata_osm_id.rb

@@ -2,8 +2,8 @@ class AddIndexOnPlacesGeodataOsmId < ActiveRecord::Migration[8.0]
   disable_ddl_transaction!
 
   def change
-    add_index :places, "(geodata->'properties'->>'osm_id')", 
-              using: :btree, 
+    add_index :places, "(geodata->'properties'->>'osm_id')",
+              using: :btree,
               name: 'index_places_on_geodata_osm_id',
               algorithm: :concurrently
   end