mirror of
https://github.com/Freika/dawarich.git
synced 2026-01-10 01:01:39 -05:00
Remove unused code
This commit is contained in:
parent
1506633441
commit
15e27c87d5
5 changed files with 0 additions and 107 deletions
1
Gemfile
1
Gemfile
|
|
@ -25,7 +25,6 @@ gem 'kaminari'
|
|||
gem 'lograge'
|
||||
gem 'oj'
|
||||
gem 'parallel'
|
||||
gem 'yajl-ruby', '~> 1.4'
|
||||
gem 'pg'
|
||||
gem 'prometheus_exporter'
|
||||
gem 'puma'
|
||||
|
|
|
|||
|
|
@ -521,7 +521,6 @@ GEM
|
|||
websocket-extensions (0.1.5)
|
||||
xpath (3.2.0)
|
||||
nokogiri (~> 1.8)
|
||||
yajl-ruby (1.4.3)
|
||||
zeitwerk (2.7.3)
|
||||
|
||||
PLATFORMS
|
||||
|
|
@ -599,7 +598,6 @@ DEPENDENCIES
|
|||
turbo-rails
|
||||
tzinfo-data
|
||||
webmock
|
||||
yajl-ruby (~> 1.4)
|
||||
|
||||
RUBY VERSION
|
||||
ruby 3.4.6p54
|
||||
|
|
|
|||
|
|
@ -1,55 +0,0 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require 'yajl'
|
||||
|
||||
class Users::ImportData::JsonStreamer
|
||||
def initialize(zip_entry)
|
||||
@zip_entry = zip_entry
|
||||
@memory_tracker = Users::ImportData::MemoryTracker.new
|
||||
end
|
||||
|
||||
def stream_parse
|
||||
Rails.logger.info "Starting JSON streaming for #{@zip_entry.name} (#{@zip_entry.size} bytes)"
|
||||
|
||||
@memory_tracker.log("before_streaming")
|
||||
|
||||
data = {}
|
||||
|
||||
@zip_entry.get_input_stream do |input_stream|
|
||||
parser = Yajl::Parser.new(symbolize_keys: false)
|
||||
|
||||
# Set up the parser to handle objects
|
||||
parser.on_parse_complete = proc do |parsed_data|
|
||||
Rails.logger.info "JSON streaming completed, parsed #{parsed_data.keys.size} entity types"
|
||||
|
||||
# Process each entity type
|
||||
parsed_data.each do |entity_type, entity_data|
|
||||
if entity_data.is_a?(Array)
|
||||
Rails.logger.info "Streamed #{entity_type}: #{entity_data.size} items"
|
||||
end
|
||||
end
|
||||
|
||||
data = parsed_data
|
||||
@memory_tracker.log("after_parsing")
|
||||
end
|
||||
|
||||
# Stream parse the JSON
|
||||
begin
|
||||
parser.parse(input_stream)
|
||||
rescue Yajl::ParseError => e
|
||||
raise StandardError, "Invalid JSON format in data file: #{e.message}"
|
||||
end
|
||||
end
|
||||
|
||||
@memory_tracker.log("streaming_completed")
|
||||
|
||||
data
|
||||
rescue StandardError => e
|
||||
Rails.logger.error "JSON streaming failed: #{e.message}"
|
||||
raise e
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
attr_reader :zip_entry
|
||||
end
|
||||
|
|
@ -1,44 +0,0 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class Users::ImportData::MemoryTracker
|
||||
def initialize
|
||||
@process_id = Process.pid
|
||||
@start_time = Time.current
|
||||
end
|
||||
|
||||
def log(stage)
|
||||
memory_mb = current_memory_usage
|
||||
elapsed = elapsed_time
|
||||
|
||||
Rails.logger.info "Memory usage at #{stage}: #{memory_mb} MB (elapsed: #{elapsed}s)"
|
||||
|
||||
# Log a warning if memory usage is high
|
||||
if memory_mb > 1000 # 1GB
|
||||
Rails.logger.warn "High memory usage detected: #{memory_mb} MB at stage #{stage}"
|
||||
end
|
||||
|
||||
{ memory_mb: memory_mb, elapsed: elapsed, stage: stage }
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def current_memory_usage
|
||||
# Get memory usage from /proc/PID/status on Linux or fallback to ps
|
||||
if File.exist?("/proc/#{@process_id}/status")
|
||||
status = File.read("/proc/#{@process_id}/status")
|
||||
match = status.match(/VmRSS:\s+(\d+)\s+kB/)
|
||||
return match[1].to_i / 1024.0 if match # Convert KB to MB
|
||||
end
|
||||
|
||||
# Fallback to ps command (works on macOS and Linux)
|
||||
memory_kb = `ps -o rss= -p #{@process_id}`.strip.to_i
|
||||
memory_kb / 1024.0 # Convert KB to MB
|
||||
rescue StandardError => e
|
||||
Rails.logger.warn "Failed to get memory usage: #{e.message}"
|
||||
0.0
|
||||
end
|
||||
|
||||
def elapsed_time
|
||||
(Time.current - @start_time).round(2)
|
||||
end
|
||||
end
|
||||
|
|
@ -4,7 +4,6 @@ class DawarichSettings
|
|||
BASIC_PAID_PLAN_LIMIT = 10_000_000 # 10 million points
|
||||
|
||||
class << self
|
||||
|
||||
def reverse_geocoding_enabled?
|
||||
@reverse_geocoding_enabled ||= photon_enabled? || geoapify_enabled? || nominatim_enabled?
|
||||
end
|
||||
|
|
@ -40,10 +39,6 @@ class DawarichSettings
|
|||
@store_geodata ||= STORE_GEODATA
|
||||
end
|
||||
|
||||
def import_batch_size
|
||||
@import_batch_size ||= (ENV['IMPORT_BATCH_SIZE'].presence || 2500).to_i
|
||||
end
|
||||
|
||||
def features
|
||||
@features ||= {
|
||||
reverse_geocoding: reverse_geocoding_enabled?
|
||||
|
|
|
|||
Loading…
Reference in a new issue