From 3336ac98642c694e20efcca9be98019b1dfbf4d5 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Thu, 22 Aug 2024 22:40:27 +0200 Subject: [PATCH] Add counter cache for imports --- .app_version | 2 +- CHANGELOG.md | 10 +++++ app/jobs/import_job.rb | 32 ++++++++++------ app/models/point.rb | 4 +- app/services/exports/create.rb | 37 +++++++++++-------- app/views/imports/index.html.erb | 2 +- config/schedule.yml | 2 +- ...0822094532_add_counter_cache_to_imports.rb | 13 +++++++ ...40822092405_add_points_count_to_imports.rb | 7 ++++ db/schema.rb | 34 ++++++++++++++++- 10 files changed, 109 insertions(+), 34 deletions(-) create mode 100644 db/data/20240822094532_add_counter_cache_to_imports.rb create mode 100644 db/migrate/20240822092405_add_points_count_to_imports.rb diff --git a/.app_version b/.app_version index af88ba82..bc859cbd 100644 --- a/.app_version +++ b/.app_version @@ -1 +1 @@ -0.11.1 +0.11.2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 28c6d3de..fd3f579c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [0.11.2] — 2024-08-22 + +### Changed + +### Fixed + +- Dawarich export was failing when attempted to be imported back to Dawarich. +- Imports page with a lot of imports should now load faster. + + ## [0.11.1] — 2024-08-21 ### Changed diff --git a/app/jobs/import_job.rb b/app/jobs/import_job.rb index 2e72f684..a8b449ce 100644 --- a/app/jobs/import_job.rb +++ b/app/jobs/import_job.rb @@ -13,21 +13,11 @@ class ImportJob < ApplicationJob raw_points: result[:raw_points], doubles: result[:doubles], processed: result[:processed] ) - Notifications::Create.new( - user:, - kind: :info, - title: 'Import finished', - content: "Import \"#{import.name}\" successfully finished." - ).call + create_import_finished_notification(import, user) StatCreatingJob.perform_later(user_id) rescue StandardError => e - Notifications::Create.new( - user:, - kind: :error, - title: 'Import failed', - content: "Import \"#{import.name}\" failed: #{e.message}, stacktrace: #{e.backtrace.join("\n")}" - ).call + create_import_failed_notification(import, user, e) end private @@ -43,4 +33,22 @@ class ImportJob < ApplicationJob when 'immich_api' then Immich::ImportParser end end + + def create_import_finished_notification(import, user) + Notifications::Create.new( + user:, + kind: :info, + title: 'Import finished', + content: "Import \"#{import.name}\" successfully finished." + ).call + end + + def create_import_failed_notification(import, user, error) + Notifications::Create.new( + user:, + kind: :error, + title: 'Import failed', + content: "Import \"#{import.name}\" failed: #{error.message}, stacktrace: #{error.backtrace.join("\n")}" + ).call + end end diff --git a/app/models/point.rb b/app/models/point.rb index 64143f18..d048808e 100644 --- a/app/models/point.rb +++ b/app/models/point.rb @@ -3,7 +3,7 @@ class Point < ApplicationRecord reverse_geocoded_by :latitude, :longitude - belongs_to :import, optional: true + belongs_to :import, optional: true, counter_cache: true belongs_to :visit, optional: true belongs_to :user @@ -15,7 +15,7 @@ class Point < ApplicationRecord report_location_message_event: 4, manual_event: 5, timer_based_event: 6, settings_monitoring_event: 7 }, _suffix: true - enum connection: { mobile: 0, wifi: 1, offline: 2 }, _suffix: true + enum connection: { mobile: 0, wifi: 1, offline: 2, unknown: 4 }, _suffix: true scope :reverse_geocoded, -> { where.not(geodata: {}) } scope :not_reverse_geocoded, -> { where(geodata: {}) } diff --git a/app/services/exports/create.rb b/app/services/exports/create.rb index f835116b..0024aa6c 100644 --- a/app/services/exports/create.rb +++ b/app/services/exports/create.rb @@ -11,11 +11,11 @@ class Exports::Create def call export.update!(status: :processing) - pp "====Exporting data for #{user.email} from #{start_at} to #{end_at}" + Rails.logger.debug "====Exporting data for #{user.email} from #{start_at} to #{end_at}" points = time_framed_points - pp "====Exporting #{points.size} points" + Rails.logger.debug "====Exporting #{points.size} points" data = ::ExportSerializer.new(points, user.email).call file_path = Rails.root.join('public', 'exports', "#{export.name}.json") @@ -24,21 +24,11 @@ class Exports::Create export.update!(status: :completed, url: "exports/#{export.name}.json") - Notifications::Create.new( - user:, - kind: :info, - title: 'Export finished', - content: "Export \"#{export.name}\" successfully finished." - ).call + create_export_finished_notification rescue StandardError => e Rails.logger.error("====Export failed to create: #{e.message}") - Notifications::Create.new( - user:, - kind: :error, - title: 'Export failed', - content: "Export \"#{export.name}\" failed: #{e.message}, stacktrace: #{e.backtrace.join("\n")}" - ).call + create_failed_export_notification(e) export.update!(status: :failed) end @@ -50,7 +40,24 @@ class Exports::Create def time_framed_points user .tracked_points - .without_raw_data .where('timestamp >= ? AND timestamp <= ?', start_at.to_i, end_at.to_i) end + + def create_export_finished_notification + Notifications::Create.new( + user:, + kind: :info, + title: 'Export finished', + content: "Export \"#{export.name}\" successfully finished." + ).call + end + + def create_failed_export_notification(error) + Notifications::Create.new( + user:, + kind: :error, + title: 'Export failed', + content: "Export \"#{export.name}\" failed: #{error.message}, stacktrace: #{error.backtrace.join("\n")}" + ).call + end end diff --git a/app/views/imports/index.html.erb b/app/views/imports/index.html.erb index c0bee84e..69af4ac8 100644 --- a/app/views/imports/index.html.erb +++ b/app/views/imports/index.html.erb @@ -46,7 +46,7 @@ <%= link_to import.name, import, class: 'underline hover:no-underline' %> (<%= import.source %>) - <%= "#{number_with_delimiter import.points.size}" %> + <%= "#{number_with_delimiter import.points_count}" %> <%= import.created_at.strftime("%d.%m.%Y, %H:%M") %> diff --git a/config/schedule.yml b/config/schedule.yml index d586b7fd..d2398095 100644 --- a/config/schedule.yml +++ b/config/schedule.yml @@ -1,7 +1,7 @@ # config/schedule.yml stat_creating_job: - cron: "0 */6 * * *" + cron: "0 */6 * * *" # every 6 hours class: "StatCreatingJob" queue: default diff --git a/db/data/20240822094532_add_counter_cache_to_imports.rb b/db/data/20240822094532_add_counter_cache_to_imports.rb new file mode 100644 index 00000000..d2edd125 --- /dev/null +++ b/db/data/20240822094532_add_counter_cache_to_imports.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class AddCounterCacheToImports < ActiveRecord::Migration[7.1] + def up + Import.find_each do |import| + Import.reset_counters(import.id, :points) + end + end + + def down + raise ActiveRecord::IrreversibleMigration + end +end diff --git a/db/migrate/20240822092405_add_points_count_to_imports.rb b/db/migrate/20240822092405_add_points_count_to_imports.rb new file mode 100644 index 00000000..bd0eec33 --- /dev/null +++ b/db/migrate/20240822092405_add_points_count_to_imports.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class AddPointsCountToImports < ActiveRecord::Migration[7.1] + def change + add_column :imports, :points_count, :integer, default: 0 + end +end diff --git a/db/schema.rb b/db/schema.rb index 7bf01c80..173d598e 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2024_08_08_121027) do +ActiveRecord::Schema[7.1].define(version: 2024_08_22_092405) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -77,6 +77,7 @@ ActiveRecord::Schema[7.1].define(version: 2024_08_08_121027) do t.integer "doubles", default: 0 t.integer "processed", default: 0 t.jsonb "raw_data" + t.integer "points_count", default: 0 t.index ["source"], name: "index_imports_on_source" t.index ["user_id"], name: "index_imports_on_user_id" end @@ -93,6 +94,28 @@ ActiveRecord::Schema[7.1].define(version: 2024_08_08_121027) do t.index ["user_id"], name: "index_notifications_on_user_id" end + create_table "place_visits", force: :cascade do |t| + t.bigint "place_id", null: false + t.bigint "visit_id", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["place_id"], name: "index_place_visits_on_place_id" + t.index ["visit_id"], name: "index_place_visits_on_visit_id" + end + + create_table "places", force: :cascade do |t| + t.string "name", null: false + t.decimal "longitude", precision: 10, scale: 6, null: false + t.decimal "latitude", precision: 10, scale: 6, null: false + t.string "city" + t.string "country" + t.integer "source", default: 0 + t.jsonb "geodata", default: {}, null: false + t.datetime "reverse_geocoded_at" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + create_table "points", force: :cascade do |t| t.integer "battery_status" t.string "ping" @@ -164,12 +187,14 @@ ActiveRecord::Schema[7.1].define(version: 2024_08_08_121027) do t.string "theme", default: "dark", null: false t.jsonb "settings", default: {"fog_of_war_meters"=>"100", "meters_between_routes"=>"1000", "minutes_between_routes"=>"60"} t.boolean "admin", default: false + t.string "provider" + t.string "uid" t.index ["email"], name: "index_users_on_email", unique: true t.index ["reset_password_token"], name: "index_users_on_reset_password_token", unique: true end create_table "visits", force: :cascade do |t| - t.bigint "area_id", null: false + t.bigint "area_id" t.bigint "user_id", null: false t.datetime "started_at", null: false t.datetime "ended_at", null: false @@ -178,7 +203,9 @@ ActiveRecord::Schema[7.1].define(version: 2024_08_08_121027) do t.integer "status", default: 0, null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false + t.bigint "place_id" t.index ["area_id"], name: "index_visits_on_area_id" + t.index ["place_id"], name: "index_visits_on_place_id" t.index ["user_id"], name: "index_visits_on_user_id" end @@ -186,9 +213,12 @@ ActiveRecord::Schema[7.1].define(version: 2024_08_08_121027) do add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id" add_foreign_key "areas", "users" add_foreign_key "notifications", "users" + add_foreign_key "place_visits", "places" + add_foreign_key "place_visits", "visits" add_foreign_key "points", "users" add_foreign_key "points", "visits" add_foreign_key "stats", "users" add_foreign_key "visits", "areas" + add_foreign_key "visits", "places" add_foreign_key "visits", "users" end