diff --git a/Gemfile b/Gemfile index c172b2c1..d9bd57d7 100644 --- a/Gemfile +++ b/Gemfile @@ -25,7 +25,6 @@ gem 'kaminari' gem 'lograge' gem 'oj' gem 'parallel' -gem 'yajl-ruby', '~> 1.4' gem 'pg' gem 'prometheus_exporter' gem 'puma' diff --git a/Gemfile.lock b/Gemfile.lock index 969d6afa..618a8454 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -521,7 +521,6 @@ GEM websocket-extensions (0.1.5) xpath (3.2.0) nokogiri (~> 1.8) - yajl-ruby (1.4.3) zeitwerk (2.7.3) PLATFORMS @@ -599,7 +598,6 @@ DEPENDENCIES turbo-rails tzinfo-data webmock - yajl-ruby (~> 1.4) RUBY VERSION ruby 3.4.6p54 diff --git a/app/services/users/import_data/json_streamer.rb b/app/services/users/import_data/json_streamer.rb deleted file mode 100644 index edaab193..00000000 --- a/app/services/users/import_data/json_streamer.rb +++ /dev/null @@ -1,55 +0,0 @@ -# frozen_string_literal: true - -require 'yajl' - -class Users::ImportData::JsonStreamer - def initialize(zip_entry) - @zip_entry = zip_entry - @memory_tracker = Users::ImportData::MemoryTracker.new - end - - def stream_parse - Rails.logger.info "Starting JSON streaming for #{@zip_entry.name} (#{@zip_entry.size} bytes)" - - @memory_tracker.log("before_streaming") - - data = {} - - @zip_entry.get_input_stream do |input_stream| - parser = Yajl::Parser.new(symbolize_keys: false) - - # Set up the parser to handle objects - parser.on_parse_complete = proc do |parsed_data| - Rails.logger.info "JSON streaming completed, parsed #{parsed_data.keys.size} entity types" - - # Process each entity type - parsed_data.each do |entity_type, entity_data| - if entity_data.is_a?(Array) - Rails.logger.info "Streamed #{entity_type}: #{entity_data.size} items" - end - end - - data = parsed_data - @memory_tracker.log("after_parsing") - end - - # Stream parse the JSON - begin - parser.parse(input_stream) - rescue Yajl::ParseError => e - raise StandardError, "Invalid JSON format in data file: #{e.message}" - end - end - - @memory_tracker.log("streaming_completed") - - data - rescue StandardError => e - Rails.logger.error "JSON streaming failed: #{e.message}" - raise e - end - - private - - attr_reader :zip_entry -end \ No newline at end of file diff --git a/app/services/users/import_data/memory_tracker.rb b/app/services/users/import_data/memory_tracker.rb deleted file mode 100644 index 41b0d221..00000000 --- a/app/services/users/import_data/memory_tracker.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -class Users::ImportData::MemoryTracker - def initialize - @process_id = Process.pid - @start_time = Time.current - end - - def log(stage) - memory_mb = current_memory_usage - elapsed = elapsed_time - - Rails.logger.info "Memory usage at #{stage}: #{memory_mb} MB (elapsed: #{elapsed}s)" - - # Log a warning if memory usage is high - if memory_mb > 1000 # 1GB - Rails.logger.warn "High memory usage detected: #{memory_mb} MB at stage #{stage}" - end - - { memory_mb: memory_mb, elapsed: elapsed, stage: stage } - end - - private - - def current_memory_usage - # Get memory usage from /proc/PID/status on Linux or fallback to ps - if File.exist?("/proc/#{@process_id}/status") - status = File.read("/proc/#{@process_id}/status") - match = status.match(/VmRSS:\s+(\d+)\s+kB/) - return match[1].to_i / 1024.0 if match # Convert KB to MB - end - - # Fallback to ps command (works on macOS and Linux) - memory_kb = `ps -o rss= -p #{@process_id}`.strip.to_i - memory_kb / 1024.0 # Convert KB to MB - rescue StandardError => e - Rails.logger.warn "Failed to get memory usage: #{e.message}" - 0.0 - end - - def elapsed_time - (Time.current - @start_time).round(2) - end -end \ No newline at end of file diff --git a/config/initializers/03_dawarich_settings.rb b/config/initializers/03_dawarich_settings.rb index ebb51d80..08cb0785 100644 --- a/config/initializers/03_dawarich_settings.rb +++ b/config/initializers/03_dawarich_settings.rb @@ -4,7 +4,6 @@ class DawarichSettings BASIC_PAID_PLAN_LIMIT = 10_000_000 # 10 million points class << self - def reverse_geocoding_enabled? @reverse_geocoding_enabled ||= photon_enabled? || geoapify_enabled? || nominatim_enabled? end @@ -40,10 +39,6 @@ class DawarichSettings @store_geodata ||= STORE_GEODATA end - def import_batch_size - @import_batch_size ||= (ENV['IMPORT_BATCH_SIZE'].presence || 2500).to_i - end - def features @features ||= { reverse_geocoding: reverse_geocoding_enabled?