diff --git a/.app_version b/.app_version index 61e6e92d..b72b05ed 100644 --- a/.app_version +++ b/.app_version @@ -1 +1 @@ -0.19.2 +0.19.3 diff --git a/CHANGELOG.md b/CHANGELOG.md index 88b0a520..b67dd939 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +# 0.19.3 - 2024-12-06 + +### Changed + +- Refactored stats calculation to calculate only necessary stats, instead of calculating all stats +- Stats are now being calculated every 1 hour instead of 6 hours +- List of years on the Map page is now being calculated based on user's points instead of stats. It's also being cached for 1 day due to the fact that it's usually a heavy operation based on the number of points. +- Reverse-geocoding points is now being performed in batches of 1,000 points to prevent memory exhaustion. + # 0.19.2 - 2024-12-04 ## The Telemetry release diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 6ce83808..4a1f0622 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -13,7 +13,11 @@ class StatsController < ApplicationController end def update - Stats::CalculatingJob.perform_later(current_user.id) + current_user.years_tracked.each do |year| + (1..12).each do |month| + Stats::CalculatingJob.perform_later(current_user.id, year, month) + end + end redirect_to stats_path, notice: 'Stats are being updated', status: :see_other end diff --git a/app/jobs/bulk_stats_calculating_job.rb b/app/jobs/bulk_stats_calculating_job.rb index a118aa9b..8cc2ba46 100644 --- a/app/jobs/bulk_stats_calculating_job.rb +++ b/app/jobs/bulk_stats_calculating_job.rb @@ -7,7 +7,7 @@ class BulkStatsCalculatingJob < ApplicationJob user_ids = User.pluck(:id) user_ids.each do |user_id| - Stats::CalculatingJob.perform_later(user_id) + Stats::BulkCalculator.new(user_id).call end end end diff --git a/app/jobs/cache/preheating_job.rb b/app/jobs/cache/preheating_job.rb new file mode 100644 index 00000000..bdf3ea99 --- /dev/null +++ b/app/jobs/cache/preheating_job.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class Cache::PreheatingJob < ApplicationJob + queue_as :default + + def perform + User.find_each do |user| + Rails.cache.write("dawarich/user_#{user.id}_years_tracked", user.years_tracked, expires_in: 1.day) + end + end +end diff --git a/app/jobs/stats/calculating_job.rb b/app/jobs/stats/calculating_job.rb index a0faa50c..26f4756e 100644 --- a/app/jobs/stats/calculating_job.rb +++ b/app/jobs/stats/calculating_job.rb @@ -3,8 +3,8 @@ class Stats::CalculatingJob < ApplicationJob queue_as :stats - def perform(user_id, start_at: nil, end_at: nil) - Stats::Calculate.new(user_id, start_at:, end_at:).call + def perform(user_id, year, month) + Stats::CalculateMonth.new(user_id, year, month).call create_stats_updated_notification(user_id) rescue StandardError => e diff --git a/app/models/import.rb b/app/models/import.rb index 067baf12..2040d738 100644 --- a/app/models/import.rb +++ b/app/models/import.rb @@ -16,4 +16,10 @@ class Import < ApplicationRecord def process! Imports::Create.new(user, self).call end + + def years_and_months_tracked + points.order(:timestamp).map do |point| + [Time.zone.at(point.timestamp).year, Time.zone.at(point.timestamp).month] + end.uniq + end end diff --git a/app/models/stat.rb b/app/models/stat.rb index ee3081a7..9376c991 100644 --- a/app/models/stat.rb +++ b/app/models/stat.rb @@ -6,39 +6,16 @@ class Stat < ApplicationRecord belongs_to :user def distance_by_day - timespan.to_a.map.with_index(1) do |day, index| - beginning_of_day = day.beginning_of_day.to_i - end_of_day = day.end_of_day.to_i - - # We have to filter by user as well - points = user - .tracked_points - .without_raw_data - .order(timestamp: :asc) - .where(timestamp: beginning_of_day..end_of_day) - - data = { day: index, distance: 0 } - - points.each_cons(2) do |point1, point2| - distance = Geocoder::Calculations.distance_between( - point1.to_coordinates, point2.to_coordinates, units: ::DISTANCE_UNIT - ) - - data[:distance] += distance - end - - [data[:day], data[:distance].round(2)] - end + monthly_points = points + calculate_daily_distances(monthly_points) end def self.year_distance(year, user) - stats = where(year:, user:).order(:month) - - (1..12).to_a.map do |month| - month_stat = stats.select { |stat| stat.month == month }.first + stats_by_month = where(year:, user:).order(:month).index_by(&:month) + (1..12).map do |month| month_name = Date::MONTHNAMES[month] - distance = month_stat&.distance || 0 + distance = stats_by_month[month]&.distance || 0 [month_name, distance] end @@ -58,10 +35,11 @@ class Stat < ApplicationRecord } end - def self.years - starting_year = select(:year).min&.year || Time.current.year - - (starting_year..Time.current.year).to_a.reverse + def points + user.tracked_points + .without_raw_data + .where(timestamp: timespan) + .order(timestamp: :asc) end private @@ -69,4 +47,25 @@ class Stat < ApplicationRecord def timespan DateTime.new(year, month).beginning_of_month..DateTime.new(year, month).end_of_month end + + def calculate_daily_distances(monthly_points) + timespan.to_a.map.with_index(1) do |day, index| + daily_points = filter_points_for_day(monthly_points, day) + distance = calculate_distance(daily_points) + [index, distance.round(2)] + end + end + + def filter_points_for_day(points, day) + beginning_of_day = day.beginning_of_day.to_i + end_of_day = day.end_of_day.to_i + + points.select { |p| p.timestamp.between?(beginning_of_day, end_of_day) } + end + + def calculate_distance(points) + points.each_cons(2).sum do |point1, point2| + DistanceCalculator.new(point1, point2).call + end + end end diff --git a/app/models/user.rb b/app/models/user.rb index 58ce091d..2060b2c3 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -62,6 +62,17 @@ class User < ApplicationRecord settings['photoprism_url'].present? && settings['photoprism_api_key'].present? end + def years_tracked + Rails.cache.fetch("dawarich/user_#{id}_years_tracked", expires_in: 1.day) do + tracked_points + .pluck(:timestamp) + .map { |ts| Time.zone.at(ts).year } + .uniq + .sort + .reverse + end + end + private def create_api_key diff --git a/app/services/distance_calculator.rb b/app/services/distance_calculator.rb new file mode 100644 index 00000000..d00d070b --- /dev/null +++ b/app/services/distance_calculator.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class DistanceCalculator + def initialize(point1, point2) + @point1 = point1 + @point2 = point2 + end + + def call + Geocoder::Calculations.distance_between( + point1.to_coordinates, point2.to_coordinates, units: ::DISTANCE_UNIT + ) + end + + private + + attr_reader :point1, :point2 +end diff --git a/app/services/imports/create.rb b/app/services/imports/create.rb index 7c34cc1f..af9b0d0c 100644 --- a/app/services/imports/create.rb +++ b/app/services/imports/create.rb @@ -34,10 +34,9 @@ class Imports::Create end def schedule_stats_creating(user_id) - start_at = import.points.order(:timestamp).first.recorded_at - end_at = import.points.order(:timestamp).last.recorded_at - - Stats::CalculatingJob.perform_later(user_id, start_at:, end_at:) + import.years_and_months_tracked.each do |year, month| + Stats::CalculatingJob.perform_later(user_id, year, month) + end end def schedule_visit_suggesting(user_id, import) diff --git a/app/services/jobs/create.rb b/app/services/jobs/create.rb index fdefe62d..ff8466be 100644 --- a/app/services/jobs/create.rb +++ b/app/services/jobs/create.rb @@ -21,6 +21,6 @@ class Jobs::Create raise InvalidJobName, 'Invalid job name' end - points.each(&:async_reverse_geocode) + points.find_each(batch_size: 1_000, &:async_reverse_geocode) end end diff --git a/app/services/stats/bulk_calculator.rb b/app/services/stats/bulk_calculator.rb new file mode 100644 index 00000000..aa74d60c --- /dev/null +++ b/app/services/stats/bulk_calculator.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Stats + class BulkCalculator + def initialize(user_id) + @user_id = user_id + end + + def call + months = extract_months(fetch_timestamps) + + schedule_calculations(months) + end + + private + + attr_reader :user_id + + def fetch_timestamps + last_calculated_at = Stat.where(user_id:).maximum(:updated_at) + last_calculated_at ||= DateTime.new(1970, 1, 1) + + time_diff = last_calculated_at.to_i..Time.current.to_i + Point.where(user_id:, timestamp: time_diff).pluck(:timestamp) + end + + def extract_months(timestamps) + timestamps.group_by do |timestamp| + time = Time.zone.at(timestamp) + [time.year, time.month] + end.keys + end + + def schedule_calculations(months) + months.each do |year, month| + Stats::CalculatingJob.perform_later(user_id, year, month) + end + end + end +end diff --git a/app/services/stats/calculate.rb b/app/services/stats/calculate.rb deleted file mode 100644 index 5f7c127f..00000000 --- a/app/services/stats/calculate.rb +++ /dev/null @@ -1,69 +0,0 @@ -# frozen_string_literal: true - -class Stats::Calculate - def initialize(user_id, start_at: nil, end_at: nil) - @user = User.find(user_id) - @start_at = start_at || DateTime.new(1970, 1, 1) - @end_at = end_at || Time.current - end - - def call - points = points(start_timestamp, end_timestamp) - points_by_month = points.group_by_month(&:recorded_at) - - points_by_month.each do |month, month_points| - update_month_stats(month_points, month.year, month.month) - end - rescue StandardError => e - create_stats_update_failed_notification(user, e) - end - - private - - attr_reader :user, :start_at, :end_at - - def start_timestamp = start_at.to_i - def end_timestamp = end_at.to_i - - def update_month_stats(month_points, year, month) - return if month_points.empty? - - stat = current_stat(year, month) - distance_by_day = stat.distance_by_day - - stat.daily_distance = distance_by_day - stat.distance = distance(distance_by_day) - stat.toponyms = toponyms(month_points) - stat.save - end - - def points(start_at, end_at) - user - .tracked_points - .without_raw_data - .where(timestamp: start_at..end_at) - .order(:timestamp) - .select(:latitude, :longitude, :timestamp, :city, :country) - end - - def distance(distance_by_day) - distance_by_day.sum { |day| day[1] } - end - - def toponyms(points) - CountriesAndCities.new(points).call - end - - def current_stat(year, month) - Stat.find_or_initialize_by(year:, month:, user:) - end - - def create_stats_update_failed_notification(user, error) - Notifications::Create.new( - user:, - kind: :error, - title: 'Stats update failed', - content: "#{error.message}, stacktrace: #{error.backtrace.join("\n")}" - ).call - end -end diff --git a/app/services/stats/calculate_month.rb b/app/services/stats/calculate_month.rb new file mode 100644 index 00000000..b99b2603 --- /dev/null +++ b/app/services/stats/calculate_month.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +class Stats::CalculateMonth + def initialize(user_id, year, month) + @user = User.find(user_id) + @year = year + @month = month + end + + def call + return if points.empty? + + update_month_stats(year, month) + rescue StandardError => e + create_stats_update_failed_notification(user, e) + end + + private + + attr_reader :user, :year, :month + + def start_timestamp = DateTime.new(year, month, 1).to_i + + def end_timestamp + DateTime.new(year, month, -1).to_i # -1 returns last day of month + end + + def update_month_stats(year, month) + Stat.transaction do + stat = Stat.find_or_initialize_by(year:, month:, user:) + distance_by_day = stat.distance_by_day + + stat.assign_attributes( + daily_distance: distance_by_day, + distance: distance(distance_by_day), + toponyms: toponyms + ) + stat.save + end + end + + def points + return @points if defined?(@points) + + @points = user + .tracked_points + .without_raw_data + .where(timestamp: start_timestamp..end_timestamp) + .select(:latitude, :longitude, :timestamp, :city, :country) + .order(timestamp: :asc) + end + + def distance(distance_by_day) + distance_by_day.sum { |day| day[1] } + end + + def toponyms + CountriesAndCities.new(points).call + end + + def create_stats_update_failed_notification(user, error) + Notifications::Create.new( + user:, + kind: :error, + title: 'Stats update failed', + content: "#{error.message}, stacktrace: #{error.backtrace.join("\n")}" + ).call + end +end diff --git a/app/views/shared/_right_sidebar.html.erb b/app/views/shared/_right_sidebar.html.erb index 87c99c04..797adaeb 100644 --- a/app/views/shared/_right_sidebar.html.erb +++ b/app/views/shared/_right_sidebar.html.erb @@ -4,7 +4,7 @@