From 6ed6a4fd89e07aee5ba671cbfb4d824f9aa899bf Mon Sep 17 00:00:00 2001 From: Evgenii Burmakin Date: Tue, 30 Dec 2025 19:06:10 +0100 Subject: [PATCH] 0.37.1 (#2092) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: move foreman to global gems to fix startup crash (#1971) * Update exporting code to stream points data to file in batches to red… (#1980) * Update exporting code to stream points data to file in batches to reduce memory usage * Update changelog * Update changelog * Feature/maplibre frontend (#1953) * Add a plan to use MapLibre GL JS for the frontend map rendering, replacing Leaflet * Implement phase 1 * Phases 1-3 + part of 4 * Fix e2e tests * Phase 6 * Implement fog of war * Phase 7 * Next step: fix specs, phase 7 done * Use our own map tiles * Extract v2 map logic to separate manager classes * Update settings panel on v2 map * Update v2 e2e tests structure * Reimplement location search in maps v2 * Update speed routes * Implement visits and places creation in v2 * Fix last failing test * Implement visits merging * Fix a routes e2e test and simplify the routes layer styling. * Extract js to modules from maps_v2_controller.js * Implement area creation * Fix spec problem * Fix some e2e tests * Implement live mode in v2 map * Update icons and panel * Extract some styles * Remove unused file * Start adding dark theme to popups on MapLibre maps * Make popups respect dark theme * Move v2 maps to maplibre namespace * Update v2 references to maplibre * Put place, area and visit info into side panel * Update API to use safe settings config method * Fix specs * Fix method name to config in SafeSettings and update usages accordingly * Add missing public files * Add handling for real time points * Fix remembering enabled/disabled layers of the v2 map * Fix lots of e2e tests * Add settings to select map version * Use maps/v2 as main path for MapLibre maps * Update routing * Update live mode * Update maplibre controller * Update changelog * Remove some console.log statements * Pull only necessary data for map v2 points * Feature/raw data archive (#2009) * 0.36.2 (#2007) * fix: move foreman to global gems to fix startup crash (#1971) * Update exporting code to stream points data to file in batches to red… (#1980) * Update exporting code to stream points data to file in batches to reduce memory usage * Update changelog * Update changelog * Feature/maplibre frontend (#1953) * Add a plan to use MapLibre GL JS for the frontend map rendering, replacing Leaflet * Implement phase 1 * Phases 1-3 + part of 4 * Fix e2e tests * Phase 6 * Implement fog of war * Phase 7 * Next step: fix specs, phase 7 done * Use our own map tiles * Extract v2 map logic to separate manager classes * Update settings panel on v2 map * Update v2 e2e tests structure * Reimplement location search in maps v2 * Update speed routes * Implement visits and places creation in v2 * Fix last failing test * Implement visits merging * Fix a routes e2e test and simplify the routes layer styling. * Extract js to modules from maps_v2_controller.js * Implement area creation * Fix spec problem * Fix some e2e tests * Implement live mode in v2 map * Update icons and panel * Extract some styles * Remove unused file * Start adding dark theme to popups on MapLibre maps * Make popups respect dark theme * Move v2 maps to maplibre namespace * Update v2 references to maplibre * Put place, area and visit info into side panel * Update API to use safe settings config method * Fix specs * Fix method name to config in SafeSettings and update usages accordingly * Add missing public files * Add handling for real time points * Fix remembering enabled/disabled layers of the v2 map * Fix lots of e2e tests * Add settings to select map version * Use maps/v2 as main path for MapLibre maps * Update routing * Update live mode * Update maplibre controller * Update changelog * Remove some console.log statements --------- Co-authored-by: Robin Tuszik * Remove esbuild scripts from package.json * Remove sideEffects field from package.json * Raw data archivation * Add tests * Fix tests * Fix tests * Update ExceptionReporter * Add schedule to run raw data archival job monthly * Change file structure for raw data archival feature * Update changelog and version for raw data archival feature --------- Co-authored-by: Robin Tuszik * Set raw_data to an empty hash instead of nil when archiving * Fix storage configuration and file extraction * Consider MIN_MINUTES_SPENT_IN_CITY during stats calculation (#2018) * Consider MIN_MINUTES_SPENT_IN_CITY during stats calculation * Remove raw data from visited cities api endpoint * Use user timezone to show dates on maps (#2020) * Fix/pre epoch time (#2019) * Use user timezone to show dates on maps * Limit timestamps to valid range to prevent database errors when users enter pre-epoch dates. * Limit timestamps to valid range to prevent database errors when users enter pre-epoch dates. * Fix tests failing due to new index on stats table * Fix failing specs * Update redis client configuration to support unix socket connection * Update changelog * Fix kml kmz import issues (#2023) * Fix kml kmz import issues * Refactor KML importer to improve readability and maintainability * Implement moving points in map v2 and fix route rendering logic to ma… (#2027) * Implement moving points in map v2 and fix route rendering logic to match map v1. * Fix route spec * fix(maplibre): update date format to ISO 8601 (#2029) * Add verification step to raw data archival process (#2028) * Add verification step to raw data archival process * Add actual verification of raw data archives after creation, and only clear raw_data for verified archives. * Fix failing specs * Eliminate zip-bomb risk * Fix potential memory leak in js * Return .keep files * Use Toast instead of alert for notifications * Add help section to navbar dropdown * Update changelog * Remove raw_data_archival_job * Ensure file is being closed properly after reading in Archivable concern * Add composite index to stats table if not exists * Update changelog * Update entrypoint to always sync static assets (not only new ones) * Add family layer to MapLibre maps (#2055) * Add family layer to MapLibre maps * Update migration * Don't show family toggle if feature is disabled * Update changelog * Return changelog * Update changelog * Update tailwind file * Bump sentry-rails from 6.0.0 to 6.1.0 (#1945) Bumps [sentry-rails](https://github.com/getsentry/sentry-ruby) from 6.0.0 to 6.1.0. - [Release notes](https://github.com/getsentry/sentry-ruby/releases) - [Changelog](https://github.com/getsentry/sentry-ruby/blob/master/CHANGELOG.md) - [Commits](https://github.com/getsentry/sentry-ruby/compare/6.0.0...6.1.0) --- updated-dependencies: - dependency-name: sentry-rails dependency-version: 6.1.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump turbo-rails from 2.0.17 to 2.0.20 (#1944) Bumps [turbo-rails](https://github.com/hotwired/turbo-rails) from 2.0.17 to 2.0.20. - [Release notes](https://github.com/hotwired/turbo-rails/releases) - [Commits](https://github.com/hotwired/turbo-rails/compare/v2.0.17...v2.0.20) --- updated-dependencies: - dependency-name: turbo-rails dependency-version: 2.0.20 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Evgenii Burmakin * Bump webmock from 3.25.1 to 3.26.1 (#1943) Bumps [webmock](https://github.com/bblimke/webmock) from 3.25.1 to 3.26.1. - [Release notes](https://github.com/bblimke/webmock/releases) - [Changelog](https://github.com/bblimke/webmock/blob/master/CHANGELOG.md) - [Commits](https://github.com/bblimke/webmock/compare/v3.25.1...v3.26.1) --- updated-dependencies: - dependency-name: webmock dependency-version: 3.26.1 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Evgenii Burmakin * Bump brakeman from 7.1.0 to 7.1.1 (#1942) Bumps [brakeman](https://github.com/presidentbeef/brakeman) from 7.1.0 to 7.1.1. - [Release notes](https://github.com/presidentbeef/brakeman/releases) - [Changelog](https://github.com/presidentbeef/brakeman/blob/main/CHANGES.md) - [Commits](https://github.com/presidentbeef/brakeman/compare/v7.1.0...v7.1.1) --- updated-dependencies: - dependency-name: brakeman dependency-version: 7.1.1 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump redis from 5.4.0 to 5.4.1 (#1941) Bumps [redis](https://github.com/redis/redis-rb) from 5.4.0 to 5.4.1. - [Changelog](https://github.com/redis/redis-rb/blob/master/CHANGELOG.md) - [Commits](https://github.com/redis/redis-rb/compare/v5.4.0...v5.4.1) --- updated-dependencies: - dependency-name: redis dependency-version: 5.4.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Put import deletion into background job (#2045) * Put import deletion into background job * Update changelog * fix null type error and update heatmap styling (#2037) * fix: use constant weight for maplibre heatmap layer * fix null type, update heatmap styling * improve heatmap styling * fix typo * Fix stats calculation to recursively reduce H3 resolution when too ma… (#2065) * Fix stats calculation to recursively reduce H3 resolution when too many hexagons are generated * Update CHANGELOG.md * Validate trip start and end dates (#2066) * Validate trip start and end dates * Update changelog * Update migration to clean up duplicate stats before adding unique index * Fix fog of war radius setting being ignored and applying settings causing errors (#2068) * Update changelog * Add Rack::Deflater middleware to config/application.rb to enable gzip compression for responses. * Add composite index to points on user_id and timestamp * Deduplicte points based on timestamp brought to unix time * Fix/stats cache invalidation (#2072) * Fix family layer toggle in Map v2 settings for non-selfhosted env * Invalidate cache * Remove comments * Remove comment * Add new indicies to improve performance and remove unused ones to opt… (#2078) * Add new indicies to improve performance and remove unused ones to optimize database. * Remove comments * Update map search suggestions panel styling * Add yearly digest (#2073) * Add yearly digest * Rename YearlyDigests to Users::Digests * Minor changes * Update yearly digest layout and styles * Add flags and chart to email * Update colors * Fix layout of stats in yearly digest view * Remove cron job for yearly digest scheduling * Update CHANGELOG.md * Update digest email setting handling * Allow sharing digest for 1 week or 1 month * Change Digests Distance to Bigint * Fix settings page * Update changelog * Add RailsPulse (#2079) * Add RailsPulse * Add RailsPulse monitoring tool with basic HTTP authentication * Bring points_count to integer * Update migration and version * Update rubocop issues * Fix migrations and data verification to remove safety_assured blocks and handle missing points gracefully. * Update version --------- Signed-off-by: dependabot[bot] Co-authored-by: Robin Tuszik Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .app_version | 2 +- CHANGELOG.md | 7 +++ app/services/points/raw_data/verifier.rb | 45 ++++++++++--------- ...13164521_add_visited_countries_to_trips.rb | 6 +-- .../20250918215512_add_h3_hex_ids_to_stats.rb | 8 ++-- ...23614_change_digests_distance_to_bigint.rb | 4 +- 6 files changed, 40 insertions(+), 32 deletions(-) diff --git a/.app_version b/.app_version index 0f1a7dfc..9b1bb851 100644 --- a/.app_version +++ b/.app_version @@ -1 +1 @@ -0.37.0 +0.37.1 diff --git a/CHANGELOG.md b/CHANGELOG.md index fe29bbb3..9e43696c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +# [0.37.1] - 2025-12-30 + +## Fixed + +- The db migration preventing the app from starting. +- Raw data archive verifier now allows having points deleted from the db after archiving. + # [0.37.0] - 2025-12-30 ## Added diff --git a/app/services/points/raw_data/verifier.rb b/app/services/points/raw_data/verifier.rb index de42229f..2da7dfc2 100644 --- a/app/services/points/raw_data/verifier.rb +++ b/app/services/points/raw_data/verifier.rb @@ -110,18 +110,24 @@ module Points return { success: false, error: 'Point IDs checksum mismatch' } end - # 8. Verify all points still exist in database + # 8. Check which points still exist in database (informational only) existing_count = Point.where(id: point_ids).count if existing_count != point_ids.count - return { - success: false, - error: "Missing points in database: expected #{point_ids.count}, found #{existing_count}" - } + Rails.logger.info( + "Archive #{archive.id}: #{point_ids.count - existing_count} points no longer in database " \ + "(#{existing_count}/#{point_ids.count} remaining). This is OK if user deleted their data." + ) end - # 9. Verify archived raw_data matches current database raw_data - verification_result = verify_raw_data_matches(archived_data) - return verification_result unless verification_result[:success] + # 9. Verify archived raw_data matches current database raw_data (only for existing points) + if existing_count.positive? + verification_result = verify_raw_data_matches(archived_data) + return verification_result unless verification_result[:success] + else + Rails.logger.info( + "Archive #{archive.id}: Skipping raw_data verification - no points remain in database" + ) + end { success: true } end @@ -149,11 +155,18 @@ module Points point_ids_to_check = archived_data.keys.sample(100) end - mismatches = [] - found_points = 0 + # Filter to only check points that still exist in the database + existing_point_ids = Point.where(id: point_ids_to_check).pluck(:id) + + if existing_point_ids.empty? + # No points remain to verify, but that's OK + Rails.logger.info("No points remaining to verify raw_data matches") + return { success: true } + end - Point.where(id: point_ids_to_check).find_each do |point| - found_points += 1 + mismatches = [] + + Point.where(id: existing_point_ids).find_each do |point| archived_raw_data = archived_data[point.id] current_raw_data = point.raw_data @@ -167,14 +180,6 @@ module Points end end - # Check if we found all the points we were looking for - if found_points != point_ids_to_check.size - return { - success: false, - error: "Missing points during data verification: expected #{point_ids_to_check.size}, found #{found_points}" - } - end - if mismatches.any? return { success: false, diff --git a/db/migrate/20250513164521_add_visited_countries_to_trips.rb b/db/migrate/20250513164521_add_visited_countries_to_trips.rb index 27797428..d40359d5 100644 --- a/db/migrate/20250513164521_add_visited_countries_to_trips.rb +++ b/db/migrate/20250513164521_add_visited_countries_to_trips.rb @@ -2,10 +2,8 @@ class AddVisitedCountriesToTrips < ActiveRecord::Migration[8.0] def change - # safety_assured do - execute <<-SQL + execute <<-SQL ALTER TABLE trips ADD COLUMN visited_countries JSONB DEFAULT '{}'::jsonb NOT NULL; - SQL - # end + SQL end end diff --git a/db/migrate/20250918215512_add_h3_hex_ids_to_stats.rb b/db/migrate/20250918215512_add_h3_hex_ids_to_stats.rb index fbb19073..18d56394 100644 --- a/db/migrate/20250918215512_add_h3_hex_ids_to_stats.rb +++ b/db/migrate/20250918215512_add_h3_hex_ids_to_stats.rb @@ -5,10 +5,8 @@ class AddH3HexIdsToStats < ActiveRecord::Migration[8.0] def change add_column :stats, :h3_hex_ids, :jsonb, default: {}, if_not_exists: true - # safety_assured do - add_index :stats, :h3_hex_ids, using: :gin, - where: "(h3_hex_ids IS NOT NULL AND h3_hex_ids != '{}'::jsonb)", - algorithm: :concurrently, if_not_exists: true - # end + add_index :stats, :h3_hex_ids, using: :gin, + where: "(h3_hex_ids IS NOT NULL AND h3_hex_ids != '{}'::jsonb)", + algorithm: :concurrently, if_not_exists: true end end diff --git a/db/migrate/20251227223614_change_digests_distance_to_bigint.rb b/db/migrate/20251227223614_change_digests_distance_to_bigint.rb index 9467fa39..a504614e 100644 --- a/db/migrate/20251227223614_change_digests_distance_to_bigint.rb +++ b/db/migrate/20251227223614_change_digests_distance_to_bigint.rb @@ -4,10 +4,10 @@ class ChangeDigestsDistanceToBigint < ActiveRecord::Migration[8.0] disable_ddl_transaction! def up - safety_assured { change_column :digests, :distance, :bigint, null: false, default: 0 } + change_column :digests, :distance, :bigint, null: false, default: 0 end def down - safety_assured { change_column :digests, :distance, :integer, null: false, default: 0 } + change_column :digests, :distance, :integer, null: false, default: 0 end end