From 3450ca35b074b692002a4e5ca819a2e02809dc21 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Sat, 20 Sep 2025 12:57:30 +0200 Subject: [PATCH] Extract hexagon calculation to its own service --- app/services/stats/calculate_month.rb | 122 +-------------- app/services/stats/hexagon_calculator.rb | 139 ++++++++++++++++++ spec/services/stats/calculate_month_spec.rb | 111 -------------- .../services/stats/hexagon_calculator_spec.rb | 123 ++++++++++++++++ 4 files changed, 263 insertions(+), 232 deletions(-) create mode 100644 app/services/stats/hexagon_calculator.rb create mode 100644 spec/services/stats/hexagon_calculator_spec.rb diff --git a/app/services/stats/calculate_month.rb b/app/services/stats/calculate_month.rb index 42986c70..35a5cfee 100644 --- a/app/services/stats/calculate_month.rb +++ b/app/services/stats/calculate_month.rb @@ -3,12 +3,6 @@ class Stats::CalculateMonth include ActiveModel::Validations - # H3 Configuration - DEFAULT_H3_RESOLUTION = 8 # Small hexagons for good detail - MAX_HEXAGONS = 10_000 # Maximum number of hexagons to prevent memory issues - - class PostGISError < StandardError; end - def initialize(user_id, year, month) @user = User.find(user_id) @year = year.to_i @@ -27,44 +21,6 @@ class Stats::CalculateMonth create_stats_update_failed_notification(user, e) end - # Public method for calculating H3 hexagon centers with custom parameters - def calculate_h3_hexagon_centers - points = fetch_user_points_for_period - - return [] if points.empty? - - h3_indexes_with_counts = calculate_h3_indexes(points, h3_resolution) - - if h3_indexes_with_counts.size > MAX_HEXAGONS - Rails.logger.warn "Too many hexagons (#{h3_indexes_with_counts.size}), using lower resolution" - # Try with lower resolution (larger hexagons) - lower_resolution = [h3_resolution - 2, 0].max - Rails.logger.info "Recalculating with lower H3 resolution: #{lower_resolution}" - return calculate_h3_hexagon_centers( - user_id: user.id, - start_date: start_date_iso8601, - end_date: end_date_iso8601, - h3_resolution: lower_resolution - ) - end - - Rails.logger.info "Generated #{h3_indexes_with_counts.size} H3 hexagons at resolution #{h3_resolution} for user #{user.id}" - - # Convert to format: [h3_index_string, point_count, earliest_timestamp, latest_timestamp] - h3_indexes_with_counts.map do |h3_index, data| - [ - h3_index.to_s(16), # Store as hex string - data[:count], - data[:earliest], - data[:latest] - ] - end - rescue StandardError => e - message = "Failed to calculate H3 hexagon centers: #{e.message}" - ExceptionReporter.call(e, message) if defined?(ExceptionReporter) - raise PostGISError, message - end - private attr_reader :user, :year, :month @@ -131,82 +87,6 @@ class Stats::CalculateMonth end def calculate_h3_hex_ids - return {} if points.empty? - - begin - result = calculate_h3_hexagon_centers( - user_id: user.id, h3_resolution: DEFAULT_H3_RESOLUTION, - start_date: start_date_iso8601, end_date: end_date_iso8601 - ) - - if result.empty? - Rails.logger.info "No H3 hex IDs calculated for user #{user.id}, #{year}-#{month} (no data)" - return {} - end - - # Convert array format to hash format: { h3_index => [count, earliest, latest] } - hex_hash = result.each_with_object({}) do |hex_data, hash| - h3_index, count, earliest, latest = hex_data - hash[h3_index] = [count, earliest, latest] - end - - Rails.logger.info "Pre-calculated #{hex_hash.size} H3 hex IDs for user #{user.id}, #{year}-#{month}" - hex_hash - rescue PostGISError => e - Rails.logger.warn "H3 hex IDs calculation failed for user #{user.id}, #{year}-#{month}: #{e.message}" - {} - end - end - - def start_date_iso8601 - @start_date_iso8601 ||= DateTime.new(year, month, 1).beginning_of_day.iso8601 - end - - def end_date_iso8601 - @end_date_iso8601 ||= DateTime.new(year, month, -1).end_of_day.iso8601 - end - - def fetch_user_points_for_period - start_timestamp = start_date_iso8601.to_i - end_timestamp = end_date_iso8601.to_i - - Point.where(user_id: user.id) - .where(timestamp: start_timestamp..end_timestamp) - .where.not(lonlat: nil) - .select(:id, :lonlat, :timestamp) - end - - def calculate_h3_indexes(points, h3_resolution) - h3_data = Hash.new { |h, k| h[k] = { count: 0, earliest: nil, latest: nil } } - - points.find_each do |point| - # Extract lat/lng from PostGIS point - coordinates = [point.lonlat.y, point.lonlat.x] # [lat, lng] for H3 - - # Get H3 index for this point - h3_index = H3.from_geo_coordinates(coordinates, h3_resolution.clamp(0, 15)) - - # Aggregate data for this hexagon - data = h3_data[h3_index] - data[:count] += 1 - data[:earliest] = [data[:earliest], point.timestamp].compact.min - data[:latest] = [data[:latest], point.timestamp].compact.max - end - - h3_data - end - - def parse_date_parameter(param) - case param - when String - param.match?(/^\d+$/) ? param.to_i : Time.zone.parse(param).to_i - when Integer - param - else - param.to_i - end - rescue ArgumentError => e - Rails.logger.error "Invalid date format: #{param} - #{e.message}" - raise ArgumentError, "Invalid date format: #{param}" + Stats::HexagonCalculator.new(user.id, year, month).calculate_h3_hex_ids end end diff --git a/app/services/stats/hexagon_calculator.rb b/app/services/stats/hexagon_calculator.rb new file mode 100644 index 00000000..f76b65de --- /dev/null +++ b/app/services/stats/hexagon_calculator.rb @@ -0,0 +1,139 @@ +# frozen_string_literal: true + +class Stats::HexagonCalculator + # H3 Configuration + DEFAULT_H3_RESOLUTION = 8 # Small hexagons for good detail + MAX_HEXAGONS = 10_000 # Maximum number of hexagons to prevent memory issues + + class PostGISError < StandardError; end + + def initialize(user_id, year, month) + @user = User.find(user_id) + @year = year.to_i + @month = month.to_i + end + + def call(h3_resolution: DEFAULT_H3_RESOLUTION) + calculate_h3_hexagon_centers_with_resolution(h3_resolution) + end + + def calculate_h3_hex_ids + return {} if points.empty? + + begin + result = call + + if result.empty? + Rails.logger.info "No H3 hex IDs calculated for user #{user.id}, #{year}-#{month} (no data)" + return {} + end + + # Convert array format to hash format: { h3_index => [count, earliest, latest] } + hex_hash = result.each_with_object({}) do |hex_data, hash| + h3_index, count, earliest, latest = hex_data + hash[h3_index] = [count, earliest, latest] + end + + Rails.logger.info "Pre-calculated #{hex_hash.size} H3 hex IDs for user #{user.id}, #{year}-#{month}" + hex_hash + rescue PostGISError => e + Rails.logger.warn "H3 hex IDs calculation failed for user #{user.id}, #{year}-#{month}: #{e.message}" + {} + end + end + + private + + attr_reader :user, :year, :month + + def calculate_h3_hexagon_centers_with_resolution(h3_resolution) + points = fetch_user_points_for_period + + return [] if points.empty? + + h3_indexes_with_counts = calculate_h3_indexes(points, h3_resolution) + + if h3_indexes_with_counts.size > MAX_HEXAGONS + Rails.logger.warn "Too many hexagons (#{h3_indexes_with_counts.size}), using lower resolution" + # Try with lower resolution (larger hexagons) + lower_resolution = [h3_resolution - 2, 0].max + Rails.logger.info "Recalculating with lower H3 resolution: #{lower_resolution}" + # Create a new instance with lower resolution for recursion + return self.class.new(user.id, year, month) + .calculate_h3_hexagon_centers_with_resolution(lower_resolution) + end + + Rails.logger.info "Generated #{h3_indexes_with_counts.size} H3 hexagons at resolution #{h3_resolution} for user #{user.id}" + + # Convert to format: [h3_index_string, point_count, earliest_timestamp, latest_timestamp] + h3_indexes_with_counts.map do |h3_index, data| + [ + h3_index.to_s(16), # Store as hex string + data[:count], + data[:earliest], + data[:latest] + ] + end + rescue StandardError => e + message = "Failed to calculate H3 hexagon centers: #{e.message}" + ExceptionReporter.call(e, message) if defined?(ExceptionReporter) + raise PostGISError, message + end + + def start_timestamp + DateTime.new(year, month, 1).to_i + end + + def end_timestamp + DateTime.new(year, month, -1).to_i # -1 returns last day of month + end + + def points + return @points if defined?(@points) + + @points = user + .points + .without_raw_data + .where(timestamp: start_timestamp..end_timestamp) + .select(:lonlat, :timestamp) + .order(timestamp: :asc) + end + + def start_date_iso8601 + @start_date_iso8601 ||= DateTime.new(year, month, 1).beginning_of_day.iso8601 + end + + def end_date_iso8601 + @end_date_iso8601 ||= DateTime.new(year, month, -1).end_of_day.iso8601 + end + + def fetch_user_points_for_period + start_timestamp = DateTime.parse(start_date_iso8601).to_i + end_timestamp = DateTime.parse(end_date_iso8601).to_i + + Point.where(user_id: user.id) + .where(timestamp: start_timestamp..end_timestamp) + .where.not(lonlat: nil) + .select(:id, :lonlat, :timestamp) + end + + def calculate_h3_indexes(points, h3_resolution) + h3_data = Hash.new { |h, k| h[k] = { count: 0, earliest: nil, latest: nil } } + + points.find_each do |point| + # Extract lat/lng from PostGIS point + coordinates = [point.lonlat.y, point.lonlat.x] # [lat, lng] for H3 + + # Get H3 index for this point + h3_index = H3.from_geo_coordinates(coordinates, h3_resolution.clamp(0, 15)) + + # Aggregate data for this hexagon + data = h3_data[h3_index] + data[:count] += 1 + data[:earliest] = [data[:earliest], point.timestamp].compact.min + data[:latest] = [data[:latest], point.timestamp].compact.max + end + + h3_data + end +end \ No newline at end of file diff --git a/spec/services/stats/calculate_month_spec.rb b/spec/services/stats/calculate_month_spec.rb index 1045f9c6..275c46a9 100644 --- a/spec/services/stats/calculate_month_spec.rb +++ b/spec/services/stats/calculate_month_spec.rb @@ -95,115 +95,4 @@ RSpec.describe Stats::CalculateMonth do end end end - - describe '#calculate_h3_hexagon_centers' do - subject(:calculate_hexagons) do - described_class.new(user.id, year, month).calculate_h3_hexagon_centers( - user_id: user.id, - start_date: start_date, - end_date: end_date, - h3_resolution: h3_resolution - ) - end - - let(:user) { create(:user) } - let(:year) { 2024 } - let(:month) { 1 } - let(:start_date) { DateTime.new(year, month, 1).beginning_of_day.iso8601 } - let(:end_date) { DateTime.new(year, month, 1).end_of_month.end_of_day.iso8601 } - let(:h3_resolution) { 8 } - - context 'when there are no points' do - it 'returns empty array' do - expect(calculate_hexagons).to eq([]) - end - end - - context 'when there are points' do - let(:timestamp1) { DateTime.new(year, month, 1, 12).to_i } - let(:timestamp2) { DateTime.new(year, month, 1, 13).to_i } - let!(:import) { create(:import, user:) } - let!(:point1) do - create(:point, - user:, - import:, - timestamp: timestamp1, - lonlat: 'POINT(14.452712811406352 52.107902115161316)') - end - let!(:point2) do - create(:point, - user:, - import:, - timestamp: timestamp2, - lonlat: 'POINT(14.453712811406352 52.108902115161316)') - end - - it 'returns H3 hexagon data' do - result = calculate_hexagons - - expect(result).to be_an(Array) - expect(result).not_to be_empty - - # Each record should have: [h3_index_string, point_count, earliest_timestamp, latest_timestamp] - result.each do |record| - expect(record).to be_an(Array) - expect(record.size).to eq(4) - expect(record[0]).to be_a(String) # H3 index as hex string - expect(record[1]).to be_a(Integer) # Point count - expect(record[2]).to be_a(Integer) # Earliest timestamp - expect(record[3]).to be_a(Integer) # Latest timestamp - end - end - - it 'aggregates points correctly' do - result = calculate_hexagons - - total_points = result.sum { |record| record[1] } - expect(total_points).to eq(2) - end - - context 'when H3 raises an error' do - before do - allow(H3).to receive(:from_geo_coordinates).and_raise(StandardError, 'H3 error') - end - - it 'raises PostGISError' do - expect do - calculate_hexagons - end.to raise_error(Stats::CalculateMonth::PostGISError, /Failed to calculate H3 hexagon centers/) - end - - it 'reports the exception' do - expect(ExceptionReporter).to receive(:call) if defined?(ExceptionReporter) - - expect { calculate_hexagons }.to raise_error(Stats::CalculateMonth::PostGISError) - end - end - end - - describe 'date parameter parsing' do - let(:service) { described_class.new(user.id, year, month) } - - it 'handles string timestamps' do - result = service.send(:parse_date_parameter, '1640995200') - expect(result).to eq(1_640_995_200) - end - - it 'handles ISO date strings' do - result = service.send(:parse_date_parameter, '2024-01-01T00:00:00Z') - expect(result).to be_a(Integer) - end - - it 'handles integer timestamps' do - result = service.send(:parse_date_parameter, 1_640_995_200) - expect(result).to eq(1_640_995_200) - end - - it 'handles edge case gracefully' do - # Time.zone.parse is very lenient, so we'll test a different edge case - result = service.send(:parse_date_parameter, nil) - expect(result).to eq(0) - end - end - end end diff --git a/spec/services/stats/hexagon_calculator_spec.rb b/spec/services/stats/hexagon_calculator_spec.rb new file mode 100644 index 00000000..25c8f83e --- /dev/null +++ b/spec/services/stats/hexagon_calculator_spec.rb @@ -0,0 +1,123 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe Stats::HexagonCalculator do + describe '#call' do + subject(:calculate_hexagons) do + described_class.new(user.id, year, month).call(h3_resolution: h3_resolution) + end + + let(:user) { create(:user) } + let(:year) { 2024 } + let(:month) { 1 } + let(:h3_resolution) { 8 } + + context 'when there are no points' do + it 'returns empty array' do + expect(calculate_hexagons).to eq([]) + end + end + + context 'when there are points' do + let(:timestamp1) { DateTime.new(year, month, 1, 12).to_i } + let(:timestamp2) { DateTime.new(year, month, 1, 13).to_i } + let!(:import) { create(:import, user:) } + let!(:point1) do + create(:point, + user:, + import:, + timestamp: timestamp1, + lonlat: 'POINT(14.452712811406352 52.107902115161316)') + end + let!(:point2) do + create(:point, + user:, + import:, + timestamp: timestamp2, + lonlat: 'POINT(14.453712811406352 52.108902115161316)') + end + + it 'returns H3 hexagon data' do + result = calculate_hexagons + + expect(result).to be_an(Array) + expect(result).not_to be_empty + + # Each record should have: [h3_index_string, point_count, earliest_timestamp, latest_timestamp] + result.each do |record| + expect(record).to be_an(Array) + expect(record.size).to eq(4) + expect(record[0]).to be_a(String) # H3 index as hex string + expect(record[1]).to be_a(Integer) # Point count + expect(record[2]).to be_a(Integer) # Earliest timestamp + expect(record[3]).to be_a(Integer) # Latest timestamp + end + end + + it 'aggregates points correctly' do + result = calculate_hexagons + + total_points = result.sum { |record| record[1] } + expect(total_points).to eq(2) + end + + context 'when H3 raises an error' do + before do + allow(H3).to receive(:from_geo_coordinates).and_raise(StandardError, 'H3 error') + end + + it 'raises PostGISError' do + expect do + calculate_hexagons + end.to raise_error(Stats::HexagonCalculator::PostGISError, /Failed to calculate H3 hexagon centers/) + end + + it 'reports the exception' do + expect(ExceptionReporter).to receive(:call) if defined?(ExceptionReporter) + + expect { calculate_hexagons }.to raise_error(Stats::HexagonCalculator::PostGISError) + end + end + end + end + + describe '#calculate_h3_hex_ids' do + subject(:calculate_hex_ids) { described_class.new(user.id, year, month).calculate_h3_hex_ids } + + let(:user) { create(:user) } + let(:year) { 2024 } + let(:month) { 1 } + + context 'when there are no points' do + it 'returns empty hash' do + expect(calculate_hex_ids).to eq({}) + end + end + + context 'when there are points' do + let(:timestamp1) { DateTime.new(year, month, 1, 12).to_i } + let!(:import) { create(:import, user:) } + let!(:point1) do + create(:point, + user:, + import:, + timestamp: timestamp1, + lonlat: 'POINT(14.452712811406352 52.107902115161316)') + end + + it 'returns hash with H3 hex IDs' do + result = calculate_hex_ids + + expect(result).to be_a(Hash) + expect(result).not_to be_empty + + result.each do |h3_index, data| + expect(h3_index).to be_a(String) + expect(data).to be_an(Array) + expect(data.size).to eq(3) # [count, earliest, latest] + end + end + end + end +end \ No newline at end of file