diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 6ce83808..4a1f0622 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -13,7 +13,11 @@ class StatsController < ApplicationController end def update - Stats::CalculatingJob.perform_later(current_user.id) + current_user.years_tracked.each do |year| + (1..12).each do |month| + Stats::CalculatingJob.perform_later(current_user.id, year, month) + end + end redirect_to stats_path, notice: 'Stats are being updated', status: :see_other end diff --git a/app/jobs/bulk_stats_calculating_job.rb b/app/jobs/bulk_stats_calculating_job.rb index a118aa9b..8cc2ba46 100644 --- a/app/jobs/bulk_stats_calculating_job.rb +++ b/app/jobs/bulk_stats_calculating_job.rb @@ -7,7 +7,7 @@ class BulkStatsCalculatingJob < ApplicationJob user_ids = User.pluck(:id) user_ids.each do |user_id| - Stats::CalculatingJob.perform_later(user_id) + Stats::BulkCalculator.new(user_id).call end end end diff --git a/app/jobs/cache/preheating_job.rb b/app/jobs/cache/preheating_job.rb new file mode 100644 index 00000000..75353ed8 --- /dev/null +++ b/app/jobs/cache/preheating_job.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class Cache::PreheatingJob < ApplicationJob + queue_as :default + + def perform + User.find_each do |user| + Rails.cache.fetch("dawarich/user_#{user.id}_years_tracked", expires_in: 1.day) do + user.years_tracked + end + end + end +end diff --git a/app/jobs/stats/calculating_job.rb b/app/jobs/stats/calculating_job.rb index a0faa50c..26f4756e 100644 --- a/app/jobs/stats/calculating_job.rb +++ b/app/jobs/stats/calculating_job.rb @@ -3,8 +3,8 @@ class Stats::CalculatingJob < ApplicationJob queue_as :stats - def perform(user_id, start_at: nil, end_at: nil) - Stats::Calculate.new(user_id, start_at:, end_at:).call + def perform(user_id, year, month) + Stats::CalculateMonth.new(user_id, year, month).call create_stats_updated_notification(user_id) rescue StandardError => e diff --git a/app/models/import.rb b/app/models/import.rb index 067baf12..2040d738 100644 --- a/app/models/import.rb +++ b/app/models/import.rb @@ -16,4 +16,10 @@ class Import < ApplicationRecord def process! Imports::Create.new(user, self).call end + + def years_and_months_tracked + points.order(:timestamp).map do |point| + [Time.zone.at(point.timestamp).year, Time.zone.at(point.timestamp).month] + end.uniq + end end diff --git a/app/models/stat.rb b/app/models/stat.rb index ee3081a7..8bfe5b36 100644 --- a/app/models/stat.rb +++ b/app/models/stat.rb @@ -6,39 +6,16 @@ class Stat < ApplicationRecord belongs_to :user def distance_by_day - timespan.to_a.map.with_index(1) do |day, index| - beginning_of_day = day.beginning_of_day.to_i - end_of_day = day.end_of_day.to_i - - # We have to filter by user as well - points = user - .tracked_points - .without_raw_data - .order(timestamp: :asc) - .where(timestamp: beginning_of_day..end_of_day) - - data = { day: index, distance: 0 } - - points.each_cons(2) do |point1, point2| - distance = Geocoder::Calculations.distance_between( - point1.to_coordinates, point2.to_coordinates, units: ::DISTANCE_UNIT - ) - - data[:distance] += distance - end - - [data[:day], data[:distance].round(2)] - end + monthly_points = points + calculate_daily_distances(monthly_points) end def self.year_distance(year, user) - stats = where(year:, user:).order(:month) - - (1..12).to_a.map do |month| - month_stat = stats.select { |stat| stat.month == month }.first + stats_by_month = where(year:, user:).order(:month).index_by(&:month) + (1..12).map do |month| month_name = Date::MONTHNAMES[month] - distance = month_stat&.distance || 0 + distance = stats_by_month[month]&.distance || 0 [month_name, distance] end @@ -64,9 +41,37 @@ class Stat < ApplicationRecord (starting_year..Time.current.year).to_a.reverse end + def points + user.tracked_points + .without_raw_data + .where(timestamp: timespan) + .order(timestamp: :asc) + end + private def timespan DateTime.new(year, month).beginning_of_month..DateTime.new(year, month).end_of_month end + + def calculate_daily_distances(monthly_points) + timespan.to_a.map.with_index(1) do |day, index| + daily_points = filter_points_for_day(monthly_points, day) + distance = calculate_distance(daily_points) + [index, distance.round(2)] + end + end + + def filter_points_for_day(points, day) + beginning_of_day = day.beginning_of_day.to_i + end_of_day = day.end_of_day.to_i + + points.select { |p| p.timestamp.between?(beginning_of_day, end_of_day) } + end + + def calculate_distance(points) + points.each_cons(2).sum do |point1, point2| + DistanceCalculator.new(point1, point2).call + end + end end diff --git a/app/models/user.rb b/app/models/user.rb index 58ce091d..2060b2c3 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -62,6 +62,17 @@ class User < ApplicationRecord settings['photoprism_url'].present? && settings['photoprism_api_key'].present? end + def years_tracked + Rails.cache.fetch("dawarich/user_#{id}_years_tracked", expires_in: 1.day) do + tracked_points + .pluck(:timestamp) + .map { |ts| Time.zone.at(ts).year } + .uniq + .sort + .reverse + end + end + private def create_api_key diff --git a/app/services/distance_calculator.rb b/app/services/distance_calculator.rb new file mode 100644 index 00000000..d00d070b --- /dev/null +++ b/app/services/distance_calculator.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class DistanceCalculator + def initialize(point1, point2) + @point1 = point1 + @point2 = point2 + end + + def call + Geocoder::Calculations.distance_between( + point1.to_coordinates, point2.to_coordinates, units: ::DISTANCE_UNIT + ) + end + + private + + attr_reader :point1, :point2 +end diff --git a/app/services/imports/create.rb b/app/services/imports/create.rb index 7c34cc1f..af9b0d0c 100644 --- a/app/services/imports/create.rb +++ b/app/services/imports/create.rb @@ -34,10 +34,9 @@ class Imports::Create end def schedule_stats_creating(user_id) - start_at = import.points.order(:timestamp).first.recorded_at - end_at = import.points.order(:timestamp).last.recorded_at - - Stats::CalculatingJob.perform_later(user_id, start_at:, end_at:) + import.years_and_months_tracked.each do |year, month| + Stats::CalculatingJob.perform_later(user_id, year, month) + end end def schedule_visit_suggesting(user_id, import) diff --git a/app/services/stats/bulk_calculator.rb b/app/services/stats/bulk_calculator.rb new file mode 100644 index 00000000..aa74d60c --- /dev/null +++ b/app/services/stats/bulk_calculator.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Stats + class BulkCalculator + def initialize(user_id) + @user_id = user_id + end + + def call + months = extract_months(fetch_timestamps) + + schedule_calculations(months) + end + + private + + attr_reader :user_id + + def fetch_timestamps + last_calculated_at = Stat.where(user_id:).maximum(:updated_at) + last_calculated_at ||= DateTime.new(1970, 1, 1) + + time_diff = last_calculated_at.to_i..Time.current.to_i + Point.where(user_id:, timestamp: time_diff).pluck(:timestamp) + end + + def extract_months(timestamps) + timestamps.group_by do |timestamp| + time = Time.zone.at(timestamp) + [time.year, time.month] + end.keys + end + + def schedule_calculations(months) + months.each do |year, month| + Stats::CalculatingJob.perform_later(user_id, year, month) + end + end + end +end diff --git a/app/services/stats/calculate.rb b/app/services/stats/calculate.rb deleted file mode 100644 index 5f7c127f..00000000 --- a/app/services/stats/calculate.rb +++ /dev/null @@ -1,69 +0,0 @@ -# frozen_string_literal: true - -class Stats::Calculate - def initialize(user_id, start_at: nil, end_at: nil) - @user = User.find(user_id) - @start_at = start_at || DateTime.new(1970, 1, 1) - @end_at = end_at || Time.current - end - - def call - points = points(start_timestamp, end_timestamp) - points_by_month = points.group_by_month(&:recorded_at) - - points_by_month.each do |month, month_points| - update_month_stats(month_points, month.year, month.month) - end - rescue StandardError => e - create_stats_update_failed_notification(user, e) - end - - private - - attr_reader :user, :start_at, :end_at - - def start_timestamp = start_at.to_i - def end_timestamp = end_at.to_i - - def update_month_stats(month_points, year, month) - return if month_points.empty? - - stat = current_stat(year, month) - distance_by_day = stat.distance_by_day - - stat.daily_distance = distance_by_day - stat.distance = distance(distance_by_day) - stat.toponyms = toponyms(month_points) - stat.save - end - - def points(start_at, end_at) - user - .tracked_points - .without_raw_data - .where(timestamp: start_at..end_at) - .order(:timestamp) - .select(:latitude, :longitude, :timestamp, :city, :country) - end - - def distance(distance_by_day) - distance_by_day.sum { |day| day[1] } - end - - def toponyms(points) - CountriesAndCities.new(points).call - end - - def current_stat(year, month) - Stat.find_or_initialize_by(year:, month:, user:) - end - - def create_stats_update_failed_notification(user, error) - Notifications::Create.new( - user:, - kind: :error, - title: 'Stats update failed', - content: "#{error.message}, stacktrace: #{error.backtrace.join("\n")}" - ).call - end -end diff --git a/app/services/stats/calculate_month.rb b/app/services/stats/calculate_month.rb new file mode 100644 index 00000000..b99b2603 --- /dev/null +++ b/app/services/stats/calculate_month.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +class Stats::CalculateMonth + def initialize(user_id, year, month) + @user = User.find(user_id) + @year = year + @month = month + end + + def call + return if points.empty? + + update_month_stats(year, month) + rescue StandardError => e + create_stats_update_failed_notification(user, e) + end + + private + + attr_reader :user, :year, :month + + def start_timestamp = DateTime.new(year, month, 1).to_i + + def end_timestamp + DateTime.new(year, month, -1).to_i # -1 returns last day of month + end + + def update_month_stats(year, month) + Stat.transaction do + stat = Stat.find_or_initialize_by(year:, month:, user:) + distance_by_day = stat.distance_by_day + + stat.assign_attributes( + daily_distance: distance_by_day, + distance: distance(distance_by_day), + toponyms: toponyms + ) + stat.save + end + end + + def points + return @points if defined?(@points) + + @points = user + .tracked_points + .without_raw_data + .where(timestamp: start_timestamp..end_timestamp) + .select(:latitude, :longitude, :timestamp, :city, :country) + .order(timestamp: :asc) + end + + def distance(distance_by_day) + distance_by_day.sum { |day| day[1] } + end + + def toponyms + CountriesAndCities.new(points).call + end + + def create_stats_update_failed_notification(user, error) + Notifications::Create.new( + user:, + kind: :error, + title: 'Stats update failed', + content: "#{error.message}, stacktrace: #{error.backtrace.join("\n")}" + ).call + end +end diff --git a/config/routes.rb b/config/routes.rb index 2c40e93d..7639ce4b 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -40,7 +40,7 @@ Rails.application.routes.draw do post 'notifications/mark_as_read', to: 'notifications#mark_as_read', as: :mark_notifications_as_read resources :stats, only: :index do collection do - post :update + post :update, constraints: { year: /\d{4}/, month: /\d{1,2}/ } end end get 'stats/:year', to: 'stats#show', constraints: { year: /\d{4}/ } diff --git a/config/schedule.yml b/config/schedule.yml index 0b99f8c1..08de79bd 100644 --- a/config/schedule.yml +++ b/config/schedule.yml @@ -30,3 +30,8 @@ telemetry_sending_job: cron: "0 */1 * * *" # every 1 hour class: "TelemetrySendingJob" queue: default + +cache_preheating_job: + cron: "0 0 * * *" # every day at 0:00 + class: "Cache::PreheatingJob" + queue: default diff --git a/db/migrate/[timestamp]_add_index_to_points_timestamp.rb b/db/migrate/[timestamp]_add_index_to_points_timestamp.rb new file mode 100644 index 00000000..8e4bc3fa --- /dev/null +++ b/db/migrate/[timestamp]_add_index_to_points_timestamp.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddIndexToPointsTimestamp < ActiveRecord::Migration[7.2] + disable_ddl_transaction! + + def change + add_index :points, %i[user_id timestamp], algorithm: :concurrently + end +end diff --git a/db/schema.rb b/db/schema.rb index 2927e2d5..fea44510 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -198,6 +198,21 @@ ActiveRecord::Schema[7.2].define(version: 2024_12_05_160055) do t.index ["user_id"], name: "index_trips_on_user_id" end + create_table "user_digests", force: :cascade do |t| + t.bigint "user_id", null: false + t.integer "kind", default: 0, null: false + t.datetime "start_at", null: false + t.datetime "end_at" + t.integer "distance", default: 0, null: false + t.text "countries", default: [], array: true + t.text "cities", default: [], array: true + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["distance"], name: "index_user_digests_on_distance" + t.index ["kind"], name: "index_user_digests_on_kind" + t.index ["user_id"], name: "index_user_digests_on_user_id" + end + create_table "users", force: :cascade do |t| t.string "email", default: "", null: false t.string "encrypted_password", default: "", null: false @@ -245,6 +260,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_12_05_160055) do add_foreign_key "points", "visits" add_foreign_key "stats", "users" add_foreign_key "trips", "users" + add_foreign_key "user_digests", "users" add_foreign_key "visits", "areas" add_foreign_key "visits", "places" add_foreign_key "visits", "users" diff --git a/spec/jobs/bulk_stats_calculating_job_spec.rb b/spec/jobs/bulk_stats_calculating_job_spec.rb index be3cc1b4..15bbc9fb 100644 --- a/spec/jobs/bulk_stats_calculating_job_spec.rb +++ b/spec/jobs/bulk_stats_calculating_job_spec.rb @@ -4,14 +4,17 @@ require 'rails_helper' RSpec.describe BulkStatsCalculatingJob, type: :job do describe '#perform' do - it 'enqueues Stats::CalculatingJob for each user' do - user1 = create(:user) - user2 = create(:user) - user3 = create(:user) + let(:user1) { create(:user) } + let(:user2) { create(:user) } - expect(Stats::CalculatingJob).to receive(:perform_later).with(user1.id) - expect(Stats::CalculatingJob).to receive(:perform_later).with(user2.id) - expect(Stats::CalculatingJob).to receive(:perform_later).with(user3.id) + let(:timestamp) { DateTime.new(2024, 1, 1).to_i } + + let!(:points1) { create_list(:point, 10, user_id: user1.id, timestamp:) } + let!(:points2) { create_list(:point, 10, user_id: user2.id, timestamp:) } + + it 'enqueues Stats::CalculatingJob for each user' do + expect(Stats::CalculatingJob).to receive(:perform_later).with(user1.id, 2024, 1) + expect(Stats::CalculatingJob).to receive(:perform_later).with(user2.id, 2024, 1) BulkStatsCalculatingJob.perform_now end diff --git a/spec/jobs/cache/preheating_job_spec.rb b/spec/jobs/cache/preheating_job_spec.rb new file mode 100644 index 00000000..3180c856 --- /dev/null +++ b/spec/jobs/cache/preheating_job_spec.rb @@ -0,0 +1,5 @@ +require 'rails_helper' + +RSpec.describe Cache::PreheatingJob, type: :job do + pending "add some examples to (or delete) #{__FILE__}" +end diff --git a/spec/jobs/import_job_spec.rb b/spec/jobs/import_job_spec.rb index 45532506..aa6d3e22 100644 --- a/spec/jobs/import_job_spec.rb +++ b/spec/jobs/import_job_spec.rb @@ -8,16 +8,14 @@ RSpec.describe ImportJob, type: :job do let(:user) { create(:user) } let!(:import) { create(:import, user:, name: 'owntracks_export.json') } - let!(:import_points) { create_list(:point, 9, import: import) } - let(:start_at) { Time.zone.at(1_709_283_789) } # Timestamp of the first point in the "2024-03.rec" fixture file - let(:end_at) { import.points.reload.order(:timestamp).last.recorded_at } it 'creates points' do expect { perform }.to change { Point.count }.by(9) end it 'calls Stats::CalculatingJob' do - expect(Stats::CalculatingJob).to receive(:perform_later).with(user.id, start_at:, end_at:) + # Timestamp of the first point in the "2024-03.rec" fixture file + expect(Stats::CalculatingJob).to receive(:perform_later).with(user.id, 2024, 3) perform end diff --git a/spec/jobs/stats/calculating_job_spec.rb b/spec/jobs/stats/calculating_job_spec.rb index a8b95de5..fdab7593 100644 --- a/spec/jobs/stats/calculating_job_spec.rb +++ b/spec/jobs/stats/calculating_job_spec.rb @@ -5,24 +5,36 @@ require 'rails_helper' RSpec.describe Stats::CalculatingJob, type: :job do describe '#perform' do let!(:user) { create(:user) } - let(:start_at) { nil } - let(:end_at) { nil } - subject { described_class.perform_now(user.id) } + subject { described_class.perform_now(user.id, 2024, 1) } before do - allow(Stats::Calculate).to receive(:new).and_call_original - allow_any_instance_of(Stats::Calculate).to receive(:call) + allow(Stats::CalculateMonth).to receive(:new).and_call_original + allow_any_instance_of(Stats::CalculateMonth).to receive(:call) end - it 'calls Stats::Calculate service' do + it 'calls Stats::CalculateMonth service' do subject - expect(Stats::Calculate).to have_received(:new).with(user.id, { start_at:, end_at: }) + expect(Stats::CalculateMonth).to have_received(:new).with(user.id, 2024, 1) end - it 'created notifications' do - expect { subject }.to change { Notification.count }.by(1) + context 'when Stats::CalculateMonth raises an error' do + before do + allow_any_instance_of(Stats::CalculateMonth).to receive(:call).and_raise(StandardError) + end + + it 'creates an error notification' do + expect { subject }.to change { Notification.count }.by(1) + expect(Notification.last.kind).to eq('error') + end + end + + context 'when Stats::CalculateMonth does not raise an error' do + it 'creates an info notification' do + expect { subject }.to change { Notification.count }.by(1) + expect(Notification.last.kind).to eq('info') + end end end end diff --git a/spec/requests/stats_spec.rb b/spec/requests/stats_spec.rb index 3afb9516..40c19823 100644 --- a/spec/requests/stats_spec.rb +++ b/spec/requests/stats_spec.rb @@ -54,8 +54,14 @@ RSpec.describe '/stats', type: :request do describe 'POST /update' do let(:stat) { create(:stat, user:, year: 2024) } - it 'enqueues Stats::CalculatingJob' do - expect { post stats_url(stat.year) }.to have_enqueued_job(Stats::CalculatingJob) + it 'enqueues Stats::CalculatingJob for each tracked year and month' do + allow(user).to receive(:years_tracked).and_return([2024]) + + post stats_url + + (1..12).each do |month| + expect(Stats::CalculatingJob).to have_been_enqueued.with(user.id, 2024, month) + end end end end diff --git a/spec/services/imports/create_spec.rb b/spec/services/imports/create_spec.rb index d35e1898..85f2131a 100644 --- a/spec/services/imports/create_spec.rb +++ b/spec/services/imports/create_spec.rb @@ -11,7 +11,8 @@ RSpec.describe Imports::Create do let(:import) { create(:import, source: 'google_semantic_history') } it 'calls the GoogleMaps::SemanticHistoryParser' do - expect(GoogleMaps::SemanticHistoryParser).to receive(:new).with(import, user.id).and_return(double(call: true)) + expect(GoogleMaps::SemanticHistoryParser).to \ + receive(:new).with(import, user.id).and_return(double(call: true)) service.call end end @@ -20,7 +21,8 @@ RSpec.describe Imports::Create do let(:import) { create(:import, source: 'google_phone_takeout') } it 'calls the GoogleMaps::PhoneTakeoutParser' do - expect(GoogleMaps::PhoneTakeoutParser).to receive(:new).with(import, user.id).and_return(double(call: true)) + expect(GoogleMaps::PhoneTakeoutParser).to \ + receive(:new).with(import, user.id).and_return(double(call: true)) service.call end end @@ -29,7 +31,8 @@ RSpec.describe Imports::Create do let(:import) { create(:import, source: 'owntracks') } it 'calls the OwnTracks::ExportParser' do - expect(OwnTracks::ExportParser).to receive(:new).with(import, user.id).and_return(double(call: true)) + expect(OwnTracks::ExportParser).to \ + receive(:new).with(import, user.id).and_return(double(call: true)) service.call end @@ -42,7 +45,8 @@ RSpec.describe Imports::Create do it 'schedules stats creating' do Sidekiq::Testing.inline! do - expect { service.call }.to have_enqueued_job(Stats::CalculatingJob) + expect { service.call }.to \ + have_enqueued_job(Stats::CalculatingJob).with(user.id, 2024, 3) end end @@ -70,7 +74,8 @@ RSpec.describe Imports::Create do let(:import) { create(:import, source: 'gpx') } it 'calls the Gpx::TrackParser' do - expect(Gpx::TrackParser).to receive(:new).with(import, user.id).and_return(double(call: true)) + expect(Gpx::TrackParser).to \ + receive(:new).with(import, user.id).and_return(double(call: true)) service.call end end @@ -79,7 +84,8 @@ RSpec.describe Imports::Create do let(:import) { create(:import, source: 'geojson') } it 'calls the Geojson::ImportParser' do - expect(Geojson::ImportParser).to receive(:new).with(import, user.id).and_return(double(call: true)) + expect(Geojson::ImportParser).to \ + receive(:new).with(import, user.id).and_return(double(call: true)) service.call end end @@ -88,7 +94,8 @@ RSpec.describe Imports::Create do let(:import) { create(:import, source: 'immich_api') } it 'calls the Photos::ImportParser' do - expect(Photos::ImportParser).to receive(:new).with(import, user.id).and_return(double(call: true)) + expect(Photos::ImportParser).to \ + receive(:new).with(import, user.id).and_return(double(call: true)) service.call end end @@ -97,7 +104,8 @@ RSpec.describe Imports::Create do let(:import) { create(:import, source: 'photoprism_api') } it 'calls the Photos::ImportParser' do - expect(Photos::ImportParser).to receive(:new).with(import, user.id).and_return(double(call: true)) + expect(Photos::ImportParser).to \ + receive(:new).with(import, user.id).and_return(double(call: true)) service.call end end diff --git a/spec/services/stats/calculate_spec.rb b/spec/services/stats/calculate_month_spec.rb similarity index 87% rename from spec/services/stats/calculate_spec.rb rename to spec/services/stats/calculate_month_spec.rb index b02e9f7d..a5ec4383 100644 --- a/spec/services/stats/calculate_spec.rb +++ b/spec/services/stats/calculate_month_spec.rb @@ -2,11 +2,13 @@ require 'rails_helper' -RSpec.describe Stats::Calculate do +RSpec.describe Stats::CalculateMonth do describe '#call' do - subject(:calculate_stats) { described_class.new(user.id).call } + subject(:calculate_stats) { described_class.new(user.id, year, month).call } let(:user) { create(:user) } + let(:year) { 2021 } + let(:month) { 1 } context 'when there are no points' do it 'does not create stats' do @@ -15,9 +17,9 @@ RSpec.describe Stats::Calculate do end context 'when there are points' do - let(:timestamp1) { DateTime.new(2021, 1, 1, 12).to_i } - let(:timestamp2) { DateTime.new(2021, 1, 1, 13).to_i } - let(:timestamp3) { DateTime.new(2021, 1, 1, 14).to_i } + let(:timestamp1) { DateTime.new(year, month, 1, 12).to_i } + let(:timestamp2) { DateTime.new(year, month, 1, 13).to_i } + let(:timestamp3) { DateTime.new(year, month, 1, 14).to_i } let!(:import) { create(:import, user:) } let!(:point1) do create(:point,