Calculate only necessary stats

app/controllers/stats_controller.rb

@@ -13,7 +13,11 @@ class StatsController < ApplicationController
   end
 
   def update
-    Stats::CalculatingJob.perform_later(current_user.id)
+    current_user.years_tracked.each do |year|
+      (1..12).each do |month|
+        Stats::CalculatingJob.perform_later(current_user.id, year, month)
+      end
+    end
 
     redirect_to stats_path, notice: 'Stats are being updated', status: :see_other
   end

app/jobs/bulk_stats_calculating_job.rb

@@ -7,7 +7,7 @@ class BulkStatsCalculatingJob < ApplicationJob
     user_ids = User.pluck(:id)
 
     user_ids.each do |user_id|
-      Stats::CalculatingJob.perform_later(user_id)
+      Stats::BulkCalculator.new(user_id).call
     end
   end
 end

app/jobs/cache/preheating_job.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class Cache::PreheatingJob < ApplicationJob
+  queue_as :default
+
+  def perform
+    User.find_each do |user|
+      Rails.cache.fetch("dawarich/user_#{user.id}_years_tracked", expires_in: 1.day) do
+        user.years_tracked
+      end
+    end
+  end
+end

app/jobs/stats/calculating_job.rb

@@ -3,8 +3,8 @@
 class Stats::CalculatingJob < ApplicationJob
   queue_as :stats
 
-  def perform(user_id, start_at: nil, end_at: nil)
-    Stats::Calculate.new(user_id, start_at:, end_at:).call
+  def perform(user_id, year, month)
+    Stats::CalculateMonth.new(user_id, year, month).call
 
     create_stats_updated_notification(user_id)
   rescue StandardError => e

app/models/import.rb

@@ -16,4 +16,10 @@ class Import < ApplicationRecord
   def process!
     Imports::Create.new(user, self).call
   end
+
+  def years_and_months_tracked
+    points.order(:timestamp).map do |point|
+      [Time.zone.at(point.timestamp).year, Time.zone.at(point.timestamp).month]
+    end.uniq
+  end
 end

app/models/stat.rb

@@ -6,39 +6,16 @@ class Stat < ApplicationRecord
   belongs_to :user
 
   def distance_by_day
-    timespan.to_a.map.with_index(1) do |day, index|
-      beginning_of_day = day.beginning_of_day.to_i
-      end_of_day = day.end_of_day.to_i
-
-      # We have to filter by user as well
-      points = user
-               .tracked_points
-               .without_raw_data
-               .order(timestamp: :asc)
-               .where(timestamp: beginning_of_day..end_of_day)
-
-      data = { day: index, distance: 0 }
-
-      points.each_cons(2) do |point1, point2|
-        distance = Geocoder::Calculations.distance_between(
-          point1.to_coordinates, point2.to_coordinates, units: ::DISTANCE_UNIT
-        )
-
-        data[:distance] += distance
-      end
-
-      [data[:day], data[:distance].round(2)]
-    end
+    monthly_points = points
+    calculate_daily_distances(monthly_points)
   end
 
   def self.year_distance(year, user)
-    stats = where(year:, user:).order(:month)
-
-    (1..12).to_a.map do |month|
-      month_stat = stats.select { |stat| stat.month == month }.first
+    stats_by_month = where(year:, user:).order(:month).index_by(&:month)
 
+    (1..12).map do |month|
       month_name = Date::MONTHNAMES[month]
-      distance = month_stat&.distance || 0
+      distance = stats_by_month[month]&.distance || 0
 
       [month_name, distance]
     end
@@ -64,9 +41,37 @@ class Stat < ApplicationRecord
     (starting_year..Time.current.year).to_a.reverse
   end
 
+  def points
+    user.tracked_points
+        .without_raw_data
+        .where(timestamp: timespan)
+        .order(timestamp: :asc)
+  end
+
   private
 
   def timespan
     DateTime.new(year, month).beginning_of_month..DateTime.new(year, month).end_of_month
   end
+
+  def calculate_daily_distances(monthly_points)
+    timespan.to_a.map.with_index(1) do |day, index|
+      daily_points = filter_points_for_day(monthly_points, day)
+      distance = calculate_distance(daily_points)
+      [index, distance.round(2)]
+    end
+  end
+
+  def filter_points_for_day(points, day)
+    beginning_of_day = day.beginning_of_day.to_i
+    end_of_day = day.end_of_day.to_i
+
+    points.select { |p| p.timestamp.between?(beginning_of_day, end_of_day) }
+  end
+
+  def calculate_distance(points)
+    points.each_cons(2).sum do |point1, point2|
+      DistanceCalculator.new(point1, point2).call
+    end
+  end
 end

app/models/user.rb

@@ -62,6 +62,17 @@ class User < ApplicationRecord
     settings['photoprism_url'].present? && settings['photoprism_api_key'].present?
   end
 
+  def years_tracked
+    Rails.cache.fetch("dawarich/user_#{id}_years_tracked", expires_in: 1.day) do
+      tracked_points
+        .pluck(:timestamp)
+        .map { |ts| Time.zone.at(ts).year }
+        .uniq
+        .sort
+        .reverse
+    end
+  end
+
   private
 
   def create_api_key

app/services/distance_calculator.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class DistanceCalculator
+  def initialize(point1, point2)
+    @point1 = point1
+    @point2 = point2
+  end
+
+  def call
+    Geocoder::Calculations.distance_between(
+      point1.to_coordinates, point2.to_coordinates, units: ::DISTANCE_UNIT
+    )
+  end
+
+  private
+
+  attr_reader :point1, :point2
+end

app/services/imports/create.rb

@@ -34,10 +34,9 @@ class Imports::Create
   end
 
   def schedule_stats_creating(user_id)
-    start_at = import.points.order(:timestamp).first.recorded_at
-    end_at = import.points.order(:timestamp).last.recorded_at
-
-    Stats::CalculatingJob.perform_later(user_id, start_at:, end_at:)
+    import.years_and_months_tracked.each do |year, month|
+      Stats::CalculatingJob.perform_later(user_id, year, month)
+    end
   end
 
   def schedule_visit_suggesting(user_id, import)

app/services/stats/bulk_calculator.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Stats
+  class BulkCalculator
+    def initialize(user_id)
+      @user_id = user_id
+    end
+
+    def call
+      months = extract_months(fetch_timestamps)
+
+      schedule_calculations(months)
+    end
+
+    private
+
+    attr_reader :user_id
+
+    def fetch_timestamps
+      last_calculated_at = Stat.where(user_id:).maximum(:updated_at)
+      last_calculated_at ||= DateTime.new(1970, 1, 1)
+
+      time_diff = last_calculated_at.to_i..Time.current.to_i
+      Point.where(user_id:, timestamp: time_diff).pluck(:timestamp)
+    end
+
+    def extract_months(timestamps)
+      timestamps.group_by do |timestamp|
+        time = Time.zone.at(timestamp)
+        [time.year, time.month]
+      end.keys
+    end
+
+    def schedule_calculations(months)
+      months.each do |year, month|
+        Stats::CalculatingJob.perform_later(user_id, year, month)
+      end
+    end
+  end
+end

app/services/stats/calculate.rb

@@ -1,69 +0,0 @@
-# frozen_string_literal: true
-
-class Stats::Calculate
-  def initialize(user_id, start_at: nil, end_at: nil)
-    @user = User.find(user_id)
-    @start_at = start_at || DateTime.new(1970, 1, 1)
-    @end_at = end_at || Time.current
-  end
-
-  def call
-    points = points(start_timestamp, end_timestamp)
-    points_by_month = points.group_by_month(&:recorded_at)
-
-    points_by_month.each do |month, month_points|
-      update_month_stats(month_points, month.year, month.month)
-    end
-  rescue StandardError => e
-    create_stats_update_failed_notification(user, e)
-  end
-
-  private
-
-  attr_reader :user, :start_at, :end_at
-
-  def start_timestamp = start_at.to_i
-  def end_timestamp   = end_at.to_i
-
-  def update_month_stats(month_points, year, month)
-    return if month_points.empty?
-
-    stat = current_stat(year, month)
-    distance_by_day = stat.distance_by_day
-
-    stat.daily_distance = distance_by_day
-    stat.distance = distance(distance_by_day)
-    stat.toponyms = toponyms(month_points)
-    stat.save
-  end
-
-  def points(start_at, end_at)
-    user
-      .tracked_points
-      .without_raw_data
-      .where(timestamp: start_at..end_at)
-      .order(:timestamp)
-      .select(:latitude, :longitude, :timestamp, :city, :country)
-  end
-
-  def distance(distance_by_day)
-    distance_by_day.sum { |day| day[1] }
-  end
-
-  def toponyms(points)
-    CountriesAndCities.new(points).call
-  end
-
-  def current_stat(year, month)
-    Stat.find_or_initialize_by(year:, month:, user:)
-  end
-
-  def create_stats_update_failed_notification(user, error)
-    Notifications::Create.new(
-      user:,
-      kind: :error,
-      title: 'Stats update failed',
-      content: "#{error.message}, stacktrace: #{error.backtrace.join("\n")}"
-    ).call
-  end
-end

app/services/stats/calculate_month.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+class Stats::CalculateMonth
+  def initialize(user_id, year, month)
+    @user = User.find(user_id)
+    @year = year
+    @month = month
+  end
+
+  def call
+    return if points.empty?
+
+    update_month_stats(year, month)
+  rescue StandardError => e
+    create_stats_update_failed_notification(user, e)
+  end
+
+  private
+
+  attr_reader :user, :year, :month
+
+  def start_timestamp = DateTime.new(year, month, 1).to_i
+
+  def end_timestamp
+    DateTime.new(year, month, -1).to_i # -1 returns last day of month
+  end
+
+  def update_month_stats(year, month)
+    Stat.transaction do
+      stat = Stat.find_or_initialize_by(year:, month:, user:)
+      distance_by_day = stat.distance_by_day
+
+      stat.assign_attributes(
+        daily_distance: distance_by_day,
+        distance: distance(distance_by_day),
+        toponyms: toponyms
+      )
+      stat.save
+    end
+  end
+
+  def points
+    return @points if defined?(@points)
+
+    @points = user
+              .tracked_points
+              .without_raw_data
+              .where(timestamp: start_timestamp..end_timestamp)
+              .select(:latitude, :longitude, :timestamp, :city, :country)
+              .order(timestamp: :asc)
+  end
+
+  def distance(distance_by_day)
+    distance_by_day.sum { |day| day[1] }
+  end
+
+  def toponyms
+    CountriesAndCities.new(points).call
+  end
+
+  def create_stats_update_failed_notification(user, error)
+    Notifications::Create.new(
+      user:,
+      kind: :error,
+      title: 'Stats update failed',
+      content: "#{error.message}, stacktrace: #{error.backtrace.join("\n")}"
+    ).call
+  end
+end

config/routes.rb

@@ -40,7 +40,7 @@ Rails.application.routes.draw do
   post 'notifications/mark_as_read', to: 'notifications#mark_as_read', as: :mark_notifications_as_read
   resources :stats, only: :index do
     collection do
-      post :update
+      post :update, constraints: { year: /\d{4}/, month: /\d{1,2}/ }
     end
   end
   get 'stats/:year', to: 'stats#show', constraints: { year: /\d{4}/ }

config/schedule.yml

@@ -30,3 +30,8 @@ telemetry_sending_job:
   cron: "0 */1 * * *" # every 1 hour
   class: "TelemetrySendingJob"
   queue: default
+
+cache_preheating_job:
+  cron: "0 0 * * *" # every day at 0:00
+  class: "Cache::PreheatingJob"
+  queue: default

db/migrate/[timestamp]_add_index_to_points_timestamp.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class AddIndexToPointsTimestamp < ActiveRecord::Migration[7.2]
+  disable_ddl_transaction!
+
+  def change
+    add_index :points, %i[user_id timestamp], algorithm: :concurrently
+  end
+end

db/schema.rb

@@ -198,6 +198,21 @@ ActiveRecord::Schema[7.2].define(version: 2024_12_05_160055) do
     t.index ["user_id"], name: "index_trips_on_user_id"
   end
 
+  create_table "user_digests", force: :cascade do |t|
+    t.bigint "user_id", null: false
+    t.integer "kind", default: 0, null: false
+    t.datetime "start_at", null: false
+    t.datetime "end_at"
+    t.integer "distance", default: 0, null: false
+    t.text "countries", default: [], array: true
+    t.text "cities", default: [], array: true
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["distance"], name: "index_user_digests_on_distance"
+    t.index ["kind"], name: "index_user_digests_on_kind"
+    t.index ["user_id"], name: "index_user_digests_on_user_id"
+  end
+
   create_table "users", force: :cascade do |t|
     t.string "email", default: "", null: false
     t.string "encrypted_password", default: "", null: false
@@ -245,6 +260,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_12_05_160055) do
   add_foreign_key "points", "visits"
   add_foreign_key "stats", "users"
   add_foreign_key "trips", "users"
+  add_foreign_key "user_digests", "users"
   add_foreign_key "visits", "areas"
   add_foreign_key "visits", "places"
   add_foreign_key "visits", "users"

spec/jobs/bulk_stats_calculating_job_spec.rb

@@ -4,14 +4,17 @@ require 'rails_helper'
 
 RSpec.describe BulkStatsCalculatingJob, type: :job do
   describe '#perform' do
-    it 'enqueues Stats::CalculatingJob for each user' do
-      user1 = create(:user)
-      user2 = create(:user)
-      user3 = create(:user)
+    let(:user1) { create(:user) }
+    let(:user2) { create(:user) }
 
-      expect(Stats::CalculatingJob).to receive(:perform_later).with(user1.id)
-      expect(Stats::CalculatingJob).to receive(:perform_later).with(user2.id)
-      expect(Stats::CalculatingJob).to receive(:perform_later).with(user3.id)
+    let(:timestamp) { DateTime.new(2024, 1, 1).to_i }
+
+    let!(:points1) { create_list(:point, 10, user_id: user1.id, timestamp:) }
+    let!(:points2) { create_list(:point, 10, user_id: user2.id, timestamp:) }
+
+    it 'enqueues Stats::CalculatingJob for each user' do
+      expect(Stats::CalculatingJob).to receive(:perform_later).with(user1.id, 2024, 1)
+      expect(Stats::CalculatingJob).to receive(:perform_later).with(user2.id, 2024, 1)
 
       BulkStatsCalculatingJob.perform_now
     end

spec/jobs/cache/preheating_job_spec.rb

@@ -0,0 +1,5 @@
+require 'rails_helper'
+
+RSpec.describe Cache::PreheatingJob, type: :job do
+  pending "add some examples to (or delete) #{__FILE__}"
+end

spec/jobs/import_job_spec.rb

@@ -8,16 +8,14 @@ RSpec.describe ImportJob, type: :job do
 
     let(:user) { create(:user) }
     let!(:import) { create(:import, user:, name: 'owntracks_export.json') }
-    let!(:import_points) { create_list(:point, 9, import: import) }
-    let(:start_at) { Time.zone.at(1_709_283_789) } # Timestamp of the first point in the "2024-03.rec" fixture file
-    let(:end_at) { import.points.reload.order(:timestamp).last.recorded_at }
 
     it 'creates points' do
       expect { perform }.to change { Point.count }.by(9)
     end
 
     it 'calls Stats::CalculatingJob' do
-      expect(Stats::CalculatingJob).to receive(:perform_later).with(user.id, start_at:, end_at:)
+      # Timestamp of the first point in the "2024-03.rec" fixture file
+      expect(Stats::CalculatingJob).to receive(:perform_later).with(user.id, 2024, 3)
 
       perform
     end

spec/jobs/stats/calculating_job_spec.rb

@@ -5,24 +5,36 @@ require 'rails_helper'
 RSpec.describe Stats::CalculatingJob, type: :job do
   describe '#perform' do
     let!(:user) { create(:user) }
-    let(:start_at) { nil }
-    let(:end_at) { nil }
 
-    subject { described_class.perform_now(user.id) }
+    subject { described_class.perform_now(user.id, 2024, 1) }
 
     before do
-      allow(Stats::Calculate).to receive(:new).and_call_original
-      allow_any_instance_of(Stats::Calculate).to receive(:call)
+      allow(Stats::CalculateMonth).to receive(:new).and_call_original
+      allow_any_instance_of(Stats::CalculateMonth).to receive(:call)
     end
 
-    it 'calls Stats::Calculate service' do
+    it 'calls Stats::CalculateMonth service' do
       subject
 
-      expect(Stats::Calculate).to have_received(:new).with(user.id, { start_at:, end_at: })
+      expect(Stats::CalculateMonth).to have_received(:new).with(user.id, 2024, 1)
     end
 
-    it 'created notifications' do
-      expect { subject }.to change { Notification.count }.by(1)
+    context 'when Stats::CalculateMonth raises an error' do
+      before do
+        allow_any_instance_of(Stats::CalculateMonth).to receive(:call).and_raise(StandardError)
+      end
+
+      it 'creates an error notification' do
+        expect { subject }.to change { Notification.count }.by(1)
+        expect(Notification.last.kind).to eq('error')
+      end
+    end
+
+    context 'when Stats::CalculateMonth does not raise an error' do
+      it 'creates an info notification' do
+        expect { subject }.to change { Notification.count }.by(1)
+        expect(Notification.last.kind).to eq('info')
+      end
     end
   end
 end

spec/requests/stats_spec.rb

@@ -54,8 +54,14 @@ RSpec.describe '/stats', type: :request do
     describe 'POST /update' do
       let(:stat) { create(:stat, user:, year: 2024) }
 
-      it 'enqueues Stats::CalculatingJob' do
-        expect { post stats_url(stat.year) }.to have_enqueued_job(Stats::CalculatingJob)
+      it 'enqueues Stats::CalculatingJob for each tracked year and month' do
+        allow(user).to receive(:years_tracked).and_return([2024])
+
+        post stats_url
+
+        (1..12).each do |month|
+          expect(Stats::CalculatingJob).to have_been_enqueued.with(user.id, 2024, month)
+        end
       end
     end
   end

spec/services/imports/create_spec.rb

@@ -11,7 +11,8 @@ RSpec.describe Imports::Create do
       let(:import) { create(:import, source: 'google_semantic_history') }
 
       it 'calls the GoogleMaps::SemanticHistoryParser' do
-        expect(GoogleMaps::SemanticHistoryParser).to receive(:new).with(import, user.id).and_return(double(call: true))
+        expect(GoogleMaps::SemanticHistoryParser).to \
+          receive(:new).with(import, user.id).and_return(double(call: true))
         service.call
       end
     end
@@ -20,7 +21,8 @@ RSpec.describe Imports::Create do
       let(:import) { create(:import, source: 'google_phone_takeout') }
 
       it 'calls the GoogleMaps::PhoneTakeoutParser' do
-        expect(GoogleMaps::PhoneTakeoutParser).to receive(:new).with(import, user.id).and_return(double(call: true))
+        expect(GoogleMaps::PhoneTakeoutParser).to \
+          receive(:new).with(import, user.id).and_return(double(call: true))
         service.call
       end
     end
@@ -29,7 +31,8 @@ RSpec.describe Imports::Create do
       let(:import) { create(:import, source: 'owntracks') }
 
       it 'calls the OwnTracks::ExportParser' do
-        expect(OwnTracks::ExportParser).to receive(:new).with(import, user.id).and_return(double(call: true))
+        expect(OwnTracks::ExportParser).to \
+          receive(:new).with(import, user.id).and_return(double(call: true))
         service.call
       end
 
@@ -42,7 +45,8 @@ RSpec.describe Imports::Create do
 
         it 'schedules stats creating' do
           Sidekiq::Testing.inline! do
-            expect { service.call }.to have_enqueued_job(Stats::CalculatingJob)
+            expect { service.call }.to \
+              have_enqueued_job(Stats::CalculatingJob).with(user.id, 2024, 3)
           end
         end
 
@@ -70,7 +74,8 @@ RSpec.describe Imports::Create do
       let(:import) { create(:import, source: 'gpx') }
 
       it 'calls the Gpx::TrackParser' do
-        expect(Gpx::TrackParser).to receive(:new).with(import, user.id).and_return(double(call: true))
+        expect(Gpx::TrackParser).to \
+          receive(:new).with(import, user.id).and_return(double(call: true))
         service.call
       end
     end
@@ -79,7 +84,8 @@ RSpec.describe Imports::Create do
       let(:import) { create(:import, source: 'geojson') }
 
       it 'calls the Geojson::ImportParser' do
-        expect(Geojson::ImportParser).to receive(:new).with(import, user.id).and_return(double(call: true))
+        expect(Geojson::ImportParser).to \
+          receive(:new).with(import, user.id).and_return(double(call: true))
         service.call
       end
     end
@@ -88,7 +94,8 @@ RSpec.describe Imports::Create do
       let(:import) { create(:import, source: 'immich_api') }
 
       it 'calls the Photos::ImportParser' do
-        expect(Photos::ImportParser).to receive(:new).with(import, user.id).and_return(double(call: true))
+        expect(Photos::ImportParser).to \
+          receive(:new).with(import, user.id).and_return(double(call: true))
         service.call
       end
     end
@@ -97,7 +104,8 @@ RSpec.describe Imports::Create do
       let(:import) { create(:import, source: 'photoprism_api') }
 
       it 'calls the Photos::ImportParser' do
-        expect(Photos::ImportParser).to receive(:new).with(import, user.id).and_return(double(call: true))
+        expect(Photos::ImportParser).to \
+          receive(:new).with(import, user.id).and_return(double(call: true))
         service.call
       end
     end

spec/services/stats/calculate_month_spec.rb

@@ -2,11 +2,13 @@
 
 require 'rails_helper'
 
-RSpec.describe Stats::Calculate do
+RSpec.describe Stats::CalculateMonth do
   describe '#call' do
-    subject(:calculate_stats) { described_class.new(user.id).call }
+    subject(:calculate_stats) { described_class.new(user.id, year, month).call }
 
     let(:user) { create(:user) }
+    let(:year) { 2021 }
+    let(:month) { 1 }
 
     context 'when there are no points' do
       it 'does not create stats' do
@@ -15,9 +17,9 @@ RSpec.describe Stats::Calculate do
     end
 
     context 'when there are points' do
-      let(:timestamp1) { DateTime.new(2021, 1, 1, 12).to_i }
-      let(:timestamp2) { DateTime.new(2021, 1, 1, 13).to_i }
-      let(:timestamp3) { DateTime.new(2021, 1, 1, 14).to_i }
+      let(:timestamp1) { DateTime.new(year, month, 1, 12).to_i }
+      let(:timestamp2) { DateTime.new(year, month, 1, 13).to_i }
+      let(:timestamp3) { DateTime.new(year, month, 1, 14).to_i }
       let!(:import) { create(:import, user:) }
       let!(:point1) do
         create(:point,