Merge remote-tracking branch 'origin/master' into chore/add-telemetry-notification

This commit is contained in:
Eugene Burmakin 2024-12-06 17:47:58 +01:00
commit 10c3b14684
27 changed files with 320 additions and 172 deletions

View file

@ -1 +1 @@
0.19.2
0.19.3

View file

@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](http://keepachangelog.com/)
and this project adheres to [Semantic Versioning](http://semver.org/).
# 0.19.3 - 2024-12-06
### Changed
- Refactored stats calculation to calculate only necessary stats, instead of calculating all stats
- Stats are now being calculated every 1 hour instead of 6 hours
- List of years on the Map page is now being calculated based on user's points instead of stats. It's also being cached for 1 day due to the fact that it's usually a heavy operation based on the number of points.
- Reverse-geocoding points is now being performed in batches of 1,000 points to prevent memory exhaustion.
# 0.19.2 - 2024-12-04
## The Telemetry release

View file

@ -13,7 +13,11 @@ class StatsController < ApplicationController
end
def update
Stats::CalculatingJob.perform_later(current_user.id)
current_user.years_tracked.each do |year|
(1..12).each do |month|
Stats::CalculatingJob.perform_later(current_user.id, year, month)
end
end
redirect_to stats_path, notice: 'Stats are being updated', status: :see_other
end

View file

@ -7,7 +7,7 @@ class BulkStatsCalculatingJob < ApplicationJob
user_ids = User.pluck(:id)
user_ids.each do |user_id|
Stats::CalculatingJob.perform_later(user_id)
Stats::BulkCalculator.new(user_id).call
end
end
end

11
app/jobs/cache/preheating_job.rb vendored Normal file
View file

@ -0,0 +1,11 @@
# frozen_string_literal: true
class Cache::PreheatingJob < ApplicationJob
queue_as :default
def perform
User.find_each do |user|
Rails.cache.write("dawarich/user_#{user.id}_years_tracked", user.years_tracked, expires_in: 1.day)
end
end
end

View file

@ -3,8 +3,8 @@
class Stats::CalculatingJob < ApplicationJob
queue_as :stats
def perform(user_id, start_at: nil, end_at: nil)
Stats::Calculate.new(user_id, start_at:, end_at:).call
def perform(user_id, year, month)
Stats::CalculateMonth.new(user_id, year, month).call
create_stats_updated_notification(user_id)
rescue StandardError => e

View file

@ -16,4 +16,10 @@ class Import < ApplicationRecord
def process!
Imports::Create.new(user, self).call
end
def years_and_months_tracked
points.order(:timestamp).map do |point|
[Time.zone.at(point.timestamp).year, Time.zone.at(point.timestamp).month]
end.uniq
end
end

View file

@ -6,39 +6,16 @@ class Stat < ApplicationRecord
belongs_to :user
def distance_by_day
timespan.to_a.map.with_index(1) do |day, index|
beginning_of_day = day.beginning_of_day.to_i
end_of_day = day.end_of_day.to_i
# We have to filter by user as well
points = user
.tracked_points
.without_raw_data
.order(timestamp: :asc)
.where(timestamp: beginning_of_day..end_of_day)
data = { day: index, distance: 0 }
points.each_cons(2) do |point1, point2|
distance = Geocoder::Calculations.distance_between(
point1.to_coordinates, point2.to_coordinates, units: ::DISTANCE_UNIT
)
data[:distance] += distance
end
[data[:day], data[:distance].round(2)]
end
monthly_points = points
calculate_daily_distances(monthly_points)
end
def self.year_distance(year, user)
stats = where(year:, user:).order(:month)
(1..12).to_a.map do |month|
month_stat = stats.select { |stat| stat.month == month }.first
stats_by_month = where(year:, user:).order(:month).index_by(&:month)
(1..12).map do |month|
month_name = Date::MONTHNAMES[month]
distance = month_stat&.distance || 0
distance = stats_by_month[month]&.distance || 0
[month_name, distance]
end
@ -58,10 +35,11 @@ class Stat < ApplicationRecord
}
end
def self.years
starting_year = select(:year).min&.year || Time.current.year
(starting_year..Time.current.year).to_a.reverse
def points
user.tracked_points
.without_raw_data
.where(timestamp: timespan)
.order(timestamp: :asc)
end
private
@ -69,4 +47,25 @@ class Stat < ApplicationRecord
def timespan
DateTime.new(year, month).beginning_of_month..DateTime.new(year, month).end_of_month
end
def calculate_daily_distances(monthly_points)
timespan.to_a.map.with_index(1) do |day, index|
daily_points = filter_points_for_day(monthly_points, day)
distance = calculate_distance(daily_points)
[index, distance.round(2)]
end
end
def filter_points_for_day(points, day)
beginning_of_day = day.beginning_of_day.to_i
end_of_day = day.end_of_day.to_i
points.select { |p| p.timestamp.between?(beginning_of_day, end_of_day) }
end
def calculate_distance(points)
points.each_cons(2).sum do |point1, point2|
DistanceCalculator.new(point1, point2).call
end
end
end

View file

@ -62,6 +62,17 @@ class User < ApplicationRecord
settings['photoprism_url'].present? && settings['photoprism_api_key'].present?
end
def years_tracked
Rails.cache.fetch("dawarich/user_#{id}_years_tracked", expires_in: 1.day) do
tracked_points
.pluck(:timestamp)
.map { |ts| Time.zone.at(ts).year }
.uniq
.sort
.reverse
end
end
private
def create_api_key

View file

@ -0,0 +1,18 @@
# frozen_string_literal: true
class DistanceCalculator
def initialize(point1, point2)
@point1 = point1
@point2 = point2
end
def call
Geocoder::Calculations.distance_between(
point1.to_coordinates, point2.to_coordinates, units: ::DISTANCE_UNIT
)
end
private
attr_reader :point1, :point2
end

View file

@ -34,10 +34,9 @@ class Imports::Create
end
def schedule_stats_creating(user_id)
start_at = import.points.order(:timestamp).first.recorded_at
end_at = import.points.order(:timestamp).last.recorded_at
Stats::CalculatingJob.perform_later(user_id, start_at:, end_at:)
import.years_and_months_tracked.each do |year, month|
Stats::CalculatingJob.perform_later(user_id, year, month)
end
end
def schedule_visit_suggesting(user_id, import)

View file

@ -21,6 +21,6 @@ class Jobs::Create
raise InvalidJobName, 'Invalid job name'
end
points.each(&:async_reverse_geocode)
points.find_each(batch_size: 1_000, &:async_reverse_geocode)
end
end

View file

@ -0,0 +1,40 @@
# frozen_string_literal: true
module Stats
class BulkCalculator
def initialize(user_id)
@user_id = user_id
end
def call
months = extract_months(fetch_timestamps)
schedule_calculations(months)
end
private
attr_reader :user_id
def fetch_timestamps
last_calculated_at = Stat.where(user_id:).maximum(:updated_at)
last_calculated_at ||= DateTime.new(1970, 1, 1)
time_diff = last_calculated_at.to_i..Time.current.to_i
Point.where(user_id:, timestamp: time_diff).pluck(:timestamp)
end
def extract_months(timestamps)
timestamps.group_by do |timestamp|
time = Time.zone.at(timestamp)
[time.year, time.month]
end.keys
end
def schedule_calculations(months)
months.each do |year, month|
Stats::CalculatingJob.perform_later(user_id, year, month)
end
end
end
end

View file

@ -1,69 +0,0 @@
# frozen_string_literal: true
class Stats::Calculate
def initialize(user_id, start_at: nil, end_at: nil)
@user = User.find(user_id)
@start_at = start_at || DateTime.new(1970, 1, 1)
@end_at = end_at || Time.current
end
def call
points = points(start_timestamp, end_timestamp)
points_by_month = points.group_by_month(&:recorded_at)
points_by_month.each do |month, month_points|
update_month_stats(month_points, month.year, month.month)
end
rescue StandardError => e
create_stats_update_failed_notification(user, e)
end
private
attr_reader :user, :start_at, :end_at
def start_timestamp = start_at.to_i
def end_timestamp = end_at.to_i
def update_month_stats(month_points, year, month)
return if month_points.empty?
stat = current_stat(year, month)
distance_by_day = stat.distance_by_day
stat.daily_distance = distance_by_day
stat.distance = distance(distance_by_day)
stat.toponyms = toponyms(month_points)
stat.save
end
def points(start_at, end_at)
user
.tracked_points
.without_raw_data
.where(timestamp: start_at..end_at)
.order(:timestamp)
.select(:latitude, :longitude, :timestamp, :city, :country)
end
def distance(distance_by_day)
distance_by_day.sum { |day| day[1] }
end
def toponyms(points)
CountriesAndCities.new(points).call
end
def current_stat(year, month)
Stat.find_or_initialize_by(year:, month:, user:)
end
def create_stats_update_failed_notification(user, error)
Notifications::Create.new(
user:,
kind: :error,
title: 'Stats update failed',
content: "#{error.message}, stacktrace: #{error.backtrace.join("\n")}"
).call
end
end

View file

@ -0,0 +1,69 @@
# frozen_string_literal: true
class Stats::CalculateMonth
def initialize(user_id, year, month)
@user = User.find(user_id)
@year = year
@month = month
end
def call
return if points.empty?
update_month_stats(year, month)
rescue StandardError => e
create_stats_update_failed_notification(user, e)
end
private
attr_reader :user, :year, :month
def start_timestamp = DateTime.new(year, month, 1).to_i
def end_timestamp
DateTime.new(year, month, -1).to_i # -1 returns last day of month
end
def update_month_stats(year, month)
Stat.transaction do
stat = Stat.find_or_initialize_by(year:, month:, user:)
distance_by_day = stat.distance_by_day
stat.assign_attributes(
daily_distance: distance_by_day,
distance: distance(distance_by_day),
toponyms: toponyms
)
stat.save
end
end
def points
return @points if defined?(@points)
@points = user
.tracked_points
.without_raw_data
.where(timestamp: start_timestamp..end_timestamp)
.select(:latitude, :longitude, :timestamp, :city, :country)
.order(timestamp: :asc)
end
def distance(distance_by_day)
distance_by_day.sum { |day| day[1] }
end
def toponyms
CountriesAndCities.new(points).call
end
def create_stats_update_failed_notification(user, error)
Notifications::Create.new(
user:,
kind: :error,
title: 'Stats update failed',
content: "#{error.message}, stacktrace: #{error.backtrace.join("\n")}"
).call
end
end

View file

@ -4,7 +4,7 @@
<div class="dropdown">
<div tabindex="0" role="button" class="btn">Select year</div>
<ul tabindex="0" class="dropdown-content z-[1] menu p-2 shadow bg-base-100 rounded-box w-52">
<% current_user.stats.years.each do |year| %>
<% current_user.years_tracked.each do |year| %>
<li><%= link_to year, map_url(year_timespan(year).merge(year: year, import_id: params[:import_id])) %></li>
<% end %>
</ul>

View file

@ -1,7 +1,7 @@
# config/schedule.yml
bulk_stats_calculating_job:
cron: "0 */6 * * *" # every 6 hour
cron: "0 */1 * * *" # every 1 hour
class: "BulkStatsCalculatingJob"
queue: stats
@ -30,3 +30,8 @@ telemetry_sending_job:
cron: "0 */1 * * *" # every 1 hour
class: "TelemetrySendingJob"
queue: default
cache_preheating_job:
cron: "0 0 * * *" # every day at 0:00
class: "Cache::PreheatingJob"
queue: default

View file

@ -0,0 +1,9 @@
# frozen_string_literal: true
class AddIndexToPointsTimestamp < ActiveRecord::Migration[7.2]
disable_ddl_transaction!
def change
add_index :points, %i[user_id timestamp], algorithm: :concurrently
end
end

View file

@ -4,14 +4,17 @@ require 'rails_helper'
RSpec.describe BulkStatsCalculatingJob, type: :job do
describe '#perform' do
it 'enqueues Stats::CalculatingJob for each user' do
user1 = create(:user)
user2 = create(:user)
user3 = create(:user)
let(:user1) { create(:user) }
let(:user2) { create(:user) }
expect(Stats::CalculatingJob).to receive(:perform_later).with(user1.id)
expect(Stats::CalculatingJob).to receive(:perform_later).with(user2.id)
expect(Stats::CalculatingJob).to receive(:perform_later).with(user3.id)
let(:timestamp) { DateTime.new(2024, 1, 1).to_i }
let!(:points1) { create_list(:point, 10, user_id: user1.id, timestamp:) }
let!(:points2) { create_list(:point, 10, user_id: user2.id, timestamp:) }
it 'enqueues Stats::CalculatingJob for each user' do
expect(Stats::CalculatingJob).to receive(:perform_later).with(user1.id, 2024, 1)
expect(Stats::CalculatingJob).to receive(:perform_later).with(user2.id, 2024, 1)
BulkStatsCalculatingJob.perform_now
end

View file

@ -8,16 +8,14 @@ RSpec.describe ImportJob, type: :job do
let(:user) { create(:user) }
let!(:import) { create(:import, user:, name: 'owntracks_export.json') }
let!(:import_points) { create_list(:point, 9, import: import) }
let(:start_at) { Time.zone.at(1_709_283_789) } # Timestamp of the first point in the "2024-03.rec" fixture file
let(:end_at) { import.points.reload.order(:timestamp).last.recorded_at }
it 'creates points' do
expect { perform }.to change { Point.count }.by(9)
end
it 'calls Stats::CalculatingJob' do
expect(Stats::CalculatingJob).to receive(:perform_later).with(user.id, start_at:, end_at:)
# Timestamp of the first point in the "2024-03.rec" fixture file
expect(Stats::CalculatingJob).to receive(:perform_later).with(user.id, 2024, 3)
perform
end

View file

@ -5,24 +5,36 @@ require 'rails_helper'
RSpec.describe Stats::CalculatingJob, type: :job do
describe '#perform' do
let!(:user) { create(:user) }
let(:start_at) { nil }
let(:end_at) { nil }
subject { described_class.perform_now(user.id) }
subject { described_class.perform_now(user.id, 2024, 1) }
before do
allow(Stats::Calculate).to receive(:new).and_call_original
allow_any_instance_of(Stats::Calculate).to receive(:call)
allow(Stats::CalculateMonth).to receive(:new).and_call_original
allow_any_instance_of(Stats::CalculateMonth).to receive(:call)
end
it 'calls Stats::Calculate service' do
it 'calls Stats::CalculateMonth service' do
subject
expect(Stats::Calculate).to have_received(:new).with(user.id, { start_at:, end_at: })
expect(Stats::CalculateMonth).to have_received(:new).with(user.id, 2024, 1)
end
it 'created notifications' do
context 'when Stats::CalculateMonth raises an error' do
before do
allow_any_instance_of(Stats::CalculateMonth).to receive(:call).and_raise(StandardError)
end
it 'creates an error notification' do
expect { subject }.to change { Notification.count }.by(1)
expect(Notification.last.kind).to eq('error')
end
end
context 'when Stats::CalculateMonth does not raise an error' do
it 'creates an info notification' do
expect { subject }.to change { Notification.count }.by(1)
expect(Notification.last.kind).to eq('info')
end
end
end
end

View file

@ -22,4 +22,14 @@ RSpec.describe Import, type: :model do
)
end
end
describe '#years_and_months_tracked' do
let(:import) { create(:import) }
let(:timestamp) { Time.zone.local(2024, 11, 1) }
let!(:points) { create_list(:point, 3, import:, timestamp:) }
it 'returns years and months tracked' do
expect(import.years_and_months_tracked).to eq([[2024, 11]])
end
end
end

View file

@ -51,30 +51,6 @@ RSpec.describe Stat, type: :model do
end
end
describe '.years' do
subject { described_class.years }
context 'when there are no stats' do
it 'returns years' do
expect(subject).to eq([Time.current.year])
end
end
context 'when there are stats' do
let(:user) { create(:user) }
let(:expected_years) { (year..Time.current.year).to_a.reverse }
before do
create(:stat, year: 2021, user:)
create(:stat, year: 2020, user:)
end
it 'returns years' do
expect(subject).to eq(expected_years)
end
end
end
describe '#distance_by_day' do
subject { stat.distance_by_day }
@ -146,5 +122,17 @@ RSpec.describe Stat, type: :model do
end
end
end
describe '#points' do
subject { stat.points.to_a }
let(:stat) { create(:stat, year:, month: 1, user:) }
let(:timestamp) { DateTime.new(year, 1, 1, 5, 0, 0) }
let!(:points) { create_list(:point, 3, user:, timestamp:) }
it 'returns points' do
expect(subject).to eq(points)
end
end
end
end

View file

@ -104,5 +104,15 @@ RSpec.describe User, type: :model do
expect(subject).to eq(1)
end
end
describe '#years_tracked' do
let!(:points) { create_list(:point, 3, user:, timestamp: DateTime.new(2024, 1, 1, 5, 0, 0)) }
subject { user.years_tracked }
it 'returns years tracked' do
expect(subject).to eq([2024])
end
end
end
end

View file

@ -54,8 +54,14 @@ RSpec.describe '/stats', type: :request do
describe 'POST /update' do
let(:stat) { create(:stat, user:, year: 2024) }
it 'enqueues Stats::CalculatingJob' do
expect { post stats_url(stat.year) }.to have_enqueued_job(Stats::CalculatingJob)
it 'enqueues Stats::CalculatingJob for each tracked year and month' do
allow(user).to receive(:years_tracked).and_return([2024])
post stats_url
(1..12).each do |month|
expect(Stats::CalculatingJob).to have_been_enqueued.with(user.id, 2024, month)
end
end
end
end

View file

@ -11,7 +11,8 @@ RSpec.describe Imports::Create do
let(:import) { create(:import, source: 'google_semantic_history') }
it 'calls the GoogleMaps::SemanticHistoryParser' do
expect(GoogleMaps::SemanticHistoryParser).to receive(:new).with(import, user.id).and_return(double(call: true))
expect(GoogleMaps::SemanticHistoryParser).to \
receive(:new).with(import, user.id).and_return(double(call: true))
service.call
end
end
@ -20,7 +21,8 @@ RSpec.describe Imports::Create do
let(:import) { create(:import, source: 'google_phone_takeout') }
it 'calls the GoogleMaps::PhoneTakeoutParser' do
expect(GoogleMaps::PhoneTakeoutParser).to receive(:new).with(import, user.id).and_return(double(call: true))
expect(GoogleMaps::PhoneTakeoutParser).to \
receive(:new).with(import, user.id).and_return(double(call: true))
service.call
end
end
@ -29,7 +31,8 @@ RSpec.describe Imports::Create do
let(:import) { create(:import, source: 'owntracks') }
it 'calls the OwnTracks::ExportParser' do
expect(OwnTracks::ExportParser).to receive(:new).with(import, user.id).and_return(double(call: true))
expect(OwnTracks::ExportParser).to \
receive(:new).with(import, user.id).and_return(double(call: true))
service.call
end
@ -42,7 +45,8 @@ RSpec.describe Imports::Create do
it 'schedules stats creating' do
Sidekiq::Testing.inline! do
expect { service.call }.to have_enqueued_job(Stats::CalculatingJob)
expect { service.call }.to \
have_enqueued_job(Stats::CalculatingJob).with(user.id, 2024, 3)
end
end
@ -70,7 +74,8 @@ RSpec.describe Imports::Create do
let(:import) { create(:import, source: 'gpx') }
it 'calls the Gpx::TrackParser' do
expect(Gpx::TrackParser).to receive(:new).with(import, user.id).and_return(double(call: true))
expect(Gpx::TrackParser).to \
receive(:new).with(import, user.id).and_return(double(call: true))
service.call
end
end
@ -79,7 +84,8 @@ RSpec.describe Imports::Create do
let(:import) { create(:import, source: 'geojson') }
it 'calls the Geojson::ImportParser' do
expect(Geojson::ImportParser).to receive(:new).with(import, user.id).and_return(double(call: true))
expect(Geojson::ImportParser).to \
receive(:new).with(import, user.id).and_return(double(call: true))
service.call
end
end
@ -88,7 +94,8 @@ RSpec.describe Imports::Create do
let(:import) { create(:import, source: 'immich_api') }
it 'calls the Photos::ImportParser' do
expect(Photos::ImportParser).to receive(:new).with(import, user.id).and_return(double(call: true))
expect(Photos::ImportParser).to \
receive(:new).with(import, user.id).and_return(double(call: true))
service.call
end
end
@ -97,7 +104,8 @@ RSpec.describe Imports::Create do
let(:import) { create(:import, source: 'photoprism_api') }
it 'calls the Photos::ImportParser' do
expect(Photos::ImportParser).to receive(:new).with(import, user.id).and_return(double(call: true))
expect(Photos::ImportParser).to \
receive(:new).with(import, user.id).and_return(double(call: true))
service.call
end
end

View file

@ -2,11 +2,13 @@
require 'rails_helper'
RSpec.describe Stats::Calculate do
RSpec.describe Stats::CalculateMonth do
describe '#call' do
subject(:calculate_stats) { described_class.new(user.id).call }
subject(:calculate_stats) { described_class.new(user.id, year, month).call }
let(:user) { create(:user) }
let(:year) { 2021 }
let(:month) { 1 }
context 'when there are no points' do
it 'does not create stats' do
@ -15,9 +17,9 @@ RSpec.describe Stats::Calculate do
end
context 'when there are points' do
let(:timestamp1) { DateTime.new(2021, 1, 1, 12).to_i }
let(:timestamp2) { DateTime.new(2021, 1, 1, 13).to_i }
let(:timestamp3) { DateTime.new(2021, 1, 1, 14).to_i }
let(:timestamp1) { DateTime.new(year, month, 1, 12).to_i }
let(:timestamp2) { DateTime.new(year, month, 1, 13).to_i }
let(:timestamp3) { DateTime.new(year, month, 1, 14).to_i }
let!(:import) { create(:import, user:) }
let!(:point1) do
create(:point,