Rework importing user data archive

CHANGELOG.md

@@ -19,6 +19,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 - Onboarding modal window now features a link to the App Store and a QR code to configure the Dawarich iOS app.
 - A permanent option was removed from stats sharing options. Now, stats can be shared for 1, 12 or 24 hours only.
+- User data archive importing now uploads the file directly to the storage service instead of uploading it to the app first.
+- Importing progress bars are now looking nice.
 
 ## Added
 

Gemfile

@@ -25,6 +25,7 @@ gem 'kaminari'
 gem 'lograge'
 gem 'oj'
 gem 'parallel'
+gem 'yajl-ruby', '~> 1.4'
 gem 'pg'
 gem 'prometheus_exporter'
 gem 'puma'

Gemfile.lock

@@ -521,6 +521,7 @@ GEM
     websocket-extensions (0.1.5)
     xpath (3.2.0)
       nokogiri (~> 1.8)
+    yajl-ruby (1.4.3)
     zeitwerk (2.7.3)
 
 PLATFORMS
@@ -598,6 +599,7 @@ DEPENDENCIES
   turbo-rails
   tzinfo-data
   webmock
+  yajl-ruby (~> 1.4)
 
 RUBY VERSION
    ruby 3.4.1p0

app/controllers/settings/users_controller.rb

@@ -54,21 +54,28 @@ class Settings::UsersController < ApplicationController
   end
 
   def import
-    unless params[:archive].present?
+    if params[:archive].blank?
       redirect_to edit_user_registration_path, alert: 'Please select a ZIP archive to import.'
       return
     end
 
-    archive_file = params[:archive]
+    archive_param = params[:archive]
 
-    validate_archive_file(archive_file)
+    # Handle both direct upload (signed_id) and traditional upload (file)
+    if archive_param.is_a?(String)
+      # Direct upload: archive_param is a signed blob ID
+      import = create_import_from_signed_archive_id(archive_param)
+    else
+      # Traditional upload: archive_param is an uploaded file
+      validate_archive_file(archive_param)
 
-    import = current_user.imports.build(
-      name: archive_file.original_filename,
-      source: :user_data_archive
-    )
+      import = current_user.imports.build(
+        name: archive_param.original_filename,
+        source: :user_data_archive
+      )
 
-    import.file.attach(archive_file)
+      import.file.attach(archive_param)
+    end
 
     if import.save
       redirect_to edit_user_registration_path,
@@ -89,6 +96,36 @@ class Settings::UsersController < ApplicationController
     params.require(:user).permit(:email, :password)
   end
 
+  def create_import_from_signed_archive_id(signed_id)
+    Rails.logger.debug "Creating archive import from signed ID: #{signed_id[0..20]}..."
+
+    blob = ActiveStorage::Blob.find_signed(signed_id)
+
+    # Validate that it's a ZIP file
+    validate_blob_file_type(blob)
+
+    import_name = generate_unique_import_name(blob.filename.to_s)
+    import = current_user.imports.build(
+      name: import_name,
+      source: :user_data_archive
+    )
+    import.file.attach(blob)
+
+    import
+  end
+
+  def generate_unique_import_name(original_name)
+    return original_name unless current_user.imports.exists?(name: original_name)
+
+    # Extract filename and extension
+    basename = File.basename(original_name, File.extname(original_name))
+    extension = File.extname(original_name)
+
+    # Add current datetime
+    timestamp = Time.current.strftime('%Y%m%d_%H%M%S')
+    "#{basename}_#{timestamp}#{extension}"
+  end
+
   def validate_archive_file(archive_file)
     unless ['application/zip', 'application/x-zip-compressed'].include?(archive_file.content_type) ||
            File.extname(archive_file.original_filename).downcase == '.zip'
@@ -96,4 +133,12 @@ class Settings::UsersController < ApplicationController
       redirect_to edit_user_registration_path, alert: 'Please upload a valid ZIP file.' and return
     end
   end
+
+  def validate_blob_file_type(blob)
+    unless ['application/zip', 'application/x-zip-compressed'].include?(blob.content_type) ||
+           File.extname(blob.filename.to_s).downcase == '.zip'
+
+      raise StandardError, 'Please upload a valid ZIP file.'
+    end
+  end
 end

app/javascript/controllers/direct_upload_controller.js

@@ -82,31 +82,33 @@ export default class extends Controller {
       this.progressTarget.remove()
     }
 
-    // Create a wrapper div for better positioning and visibility
+    // Create a wrapper div with better DaisyUI styling
     const progressWrapper = document.createElement("div")
-    progressWrapper.className = "mt-4 mb-6 border p-4 rounded-lg bg-gray-50"
+    progressWrapper.className = "w-full mt-4 mb-4"
 
-    // Add a label
+    // Add a label with better typography
     const progressLabel = document.createElement("div")
-    progressLabel.className = "font-medium mb-2 text-gray-700"
-    progressLabel.textContent = "Upload Progress"
+    progressLabel.className = "text-sm font-medium text-base-content mb-2 flex justify-between items-center"
+    progressLabel.innerHTML = `
+      <span>Upload Progress</span>
+      <span class="text-xs text-base-content/70 progress-percentage">0%</span>
+    `
     progressWrapper.appendChild(progressLabel)
 
-    // Create a new progress container
-    const progressContainer = document.createElement("div")
+    // Create DaisyUI progress element
+    const progressContainer = document.createElement("progress")
     progressContainer.setAttribute("data-direct-upload-target", "progress")
-    progressContainer.className = "w-full bg-gray-200 rounded-full h-4"
+    progressContainer.className = "progress progress-primary w-full h-3"
+    progressContainer.value = 0
+    progressContainer.max = 100
 
-    // Create the progress bar fill element
+    // Create a hidden div for the progress bar target (for compatibility)
     const progressBarFill = document.createElement("div")
     progressBarFill.setAttribute("data-direct-upload-target", "progressBar")
-    progressBarFill.className = "bg-blue-600 h-4 rounded-full transition-all duration-300"
-    progressBarFill.style.width = "0%"
+    progressBarFill.style.display = "none"
 
-    // Add the fill element to the container
-    progressContainer.appendChild(progressBarFill)
     progressWrapper.appendChild(progressContainer)
-    progressBarFill.dataset.percentageDisplay = "true"
+    progressWrapper.appendChild(progressBarFill)
 
     // Add the progress wrapper AFTER the file input field but BEFORE the submit button
     this.submitTarget.parentNode.insertBefore(progressWrapper, this.submitTarget)
@@ -158,6 +160,19 @@ export default class extends Controller {
             showFlashMessage('error', 'No files were successfully uploaded. Please try again.')
           } else {
             showFlashMessage('notice', `${successfulUploads} file(s) uploaded successfully. Ready to submit.`)
+
+            // Add a completion animation to the progress bar
+            const percentageDisplay = this.element.querySelector('.progress-percentage')
+            if (percentageDisplay) {
+              percentageDisplay.textContent = '100%'
+              percentageDisplay.classList.add('text-success')
+            }
+
+            if (this.hasProgressTarget) {
+              this.progressTarget.value = 100
+              this.progressTarget.classList.add('progress-success')
+              this.progressTarget.classList.remove('progress-primary')
+            }
           }
           this.isUploading = false
           console.log("All uploads completed")
@@ -169,18 +184,20 @@ export default class extends Controller {
 
     directUploadWillStoreFileWithXHR(request) {
     request.upload.addEventListener("progress", event => {
-      if (!this.hasProgressBarTarget) {
-        console.warn("Progress bar target not found")
+      if (!this.hasProgressTarget) {
+        console.warn("Progress target not found")
         return
       }
 
       const progress = (event.loaded / event.total) * 100
       const progressPercentage = `${progress.toFixed(1)}%`
       console.log(`Upload progress: ${progressPercentage}`)
-      this.progressBarTarget.style.width = progressPercentage
 
-      // Update text percentage if exists
-      const percentageDisplay = this.element.querySelector('[data-percentage-display="true"]')
+      // Update the DaisyUI progress element
+      this.progressTarget.value = progress
+
+      // Update the percentage display
+      const percentageDisplay = this.element.querySelector('.progress-percentage')
       if (percentageDisplay) {
         percentageDisplay.textContent = progressPercentage
       }

app/javascript/controllers/user_data_archive_direct_upload_controller.js

@@ -0,0 +1,217 @@
+import { Controller } from "@hotwired/stimulus"
+import { DirectUpload } from "@rails/activestorage"
+import { showFlashMessage } from "../maps/helpers"
+
+export default class extends Controller {
+  static targets = ["input", "progress", "progressBar", "submit", "form"]
+  static values = {
+    url: String,
+    userTrial: Boolean
+  }
+
+  connect() {
+    this.inputTarget.addEventListener("change", this.upload.bind(this))
+
+    // Add form submission handler to disable the file input
+    if (this.hasFormTarget) {
+      this.formTarget.addEventListener("submit", this.onSubmit.bind(this))
+    }
+
+    // Initially disable submit button if no files are uploaded
+    if (this.hasSubmitTarget) {
+      const hasUploadedFiles = this.element.querySelectorAll('input[name="archive"][type="hidden"]').length > 0
+      this.submitTarget.disabled = !hasUploadedFiles
+    }
+  }
+
+  onSubmit(event) {
+    if (this.isUploading) {
+      // If still uploading, prevent submission
+      event.preventDefault()
+      console.log("Form submission prevented during upload")
+      return
+    }
+
+    // Disable the file input to prevent it from being submitted with the form
+    // This ensures only our hidden input with signed ID is submitted
+    this.inputTarget.disabled = true
+
+    // Check if we have a signed ID
+    const signedId = this.element.querySelector('input[name="archive"][type="hidden"]')
+    if (!signedId) {
+      event.preventDefault()
+      console.log("No file uploaded yet")
+      alert("Please select and upload a ZIP archive first")
+    } else {
+      console.log("Submitting form with uploaded archive")
+    }
+  }
+
+  upload() {
+    const files = this.inputTarget.files
+    if (files.length === 0) return
+
+    const file = files[0] // Only handle single file for archives
+
+    // Validate file type
+    if (!this.isValidZipFile(file)) {
+      showFlashMessage('error', 'Please select a valid ZIP file.')
+      this.inputTarget.value = ''
+      return
+    }
+
+    // Check file size limits for trial users
+    if (this.userTrialValue) {
+      const MAX_FILE_SIZE = 11 * 1024 * 1024 // 11MB in bytes
+
+      if (file.size > MAX_FILE_SIZE) {
+        const message = `File size limit exceeded. Trial users can only upload files up to 10MB. File size: ${(file.size / 1024 / 1024).toFixed(1)}MB`
+        showFlashMessage('error', message)
+
+        // Clear the file input
+        this.inputTarget.value = ''
+        return
+      }
+    }
+
+    console.log(`Uploading archive: ${file.name}`)
+    this.isUploading = true
+
+    // Disable submit button during upload
+    this.submitTarget.disabled = true
+    this.submitTarget.classList.add("opacity-50", "cursor-not-allowed")
+
+    // Show uploading message using flash
+    showFlashMessage('notice', `Uploading ${file.name}, please wait...`)
+
+    // Always remove any existing progress bar to ensure we create a fresh one
+    if (this.hasProgressTarget) {
+      this.progressTarget.remove()
+    }
+
+    // Create a wrapper div with better DaisyUI styling
+    const progressWrapper = document.createElement("div")
+    progressWrapper.className = "w-full mt-4 mb-4"
+
+    // Add a label with better typography
+    const progressLabel = document.createElement("div")
+    progressLabel.className = "text-sm font-medium text-base-content mb-2 flex justify-between items-center"
+    progressLabel.innerHTML = `
+      <span>Upload Progress</span>
+      <span class="text-xs text-base-content/70 progress-percentage">0%</span>
+    `
+    progressWrapper.appendChild(progressLabel)
+
+    // Create DaisyUI progress element
+    const progressContainer = document.createElement("progress")
+    progressContainer.setAttribute("data-user-data-archive-direct-upload-target", "progress")
+    progressContainer.className = "progress progress-primary w-full h-3"
+    progressContainer.value = 0
+    progressContainer.max = 100
+
+    // Create a hidden div for the progress bar target (for compatibility)
+    const progressBarFill = document.createElement("div")
+    progressBarFill.setAttribute("data-user-data-archive-direct-upload-target", "progressBar")
+    progressBarFill.style.display = "none"
+
+    progressWrapper.appendChild(progressContainer)
+    progressWrapper.appendChild(progressBarFill)
+
+    // Add the progress wrapper after the form-control div containing the file input
+    const formControl = this.inputTarget.closest('.form-control')
+    if (formControl) {
+      formControl.parentNode.insertBefore(progressWrapper, formControl.nextSibling)
+    } else {
+      // Fallback: insert before submit button
+      this.submitTarget.parentNode.insertBefore(progressWrapper, this.submitTarget)
+    }
+
+    console.log("Progress bar created and inserted after file input")
+
+    // Clear any existing hidden field for archive
+    const existingHiddenField = this.element.querySelector('input[name="archive"][type="hidden"]')
+    if (existingHiddenField) {
+      existingHiddenField.remove()
+    }
+
+    const upload = new DirectUpload(file, this.urlValue, this)
+    upload.create((error, blob) => {
+      if (error) {
+        console.error("Error uploading file:", error)
+        // Show error to user using flash
+        showFlashMessage('error', `Error uploading ${file.name}: ${error.message || 'Unknown error'}`)
+
+        // Re-enable submit button but keep it disabled since no file was uploaded
+        this.submitTarget.disabled = true
+        this.submitTarget.classList.add("opacity-50", "cursor-not-allowed")
+      } else {
+        console.log(`Successfully uploaded ${file.name} with ID: ${blob.signed_id}`)
+
+        // Create a hidden field with the correct name
+        const hiddenField = document.createElement("input")
+        hiddenField.setAttribute("type", "hidden")
+        hiddenField.setAttribute("name", "archive")
+        hiddenField.setAttribute("value", blob.signed_id)
+        this.element.appendChild(hiddenField)
+
+        console.log("Added hidden field with signed ID:", blob.signed_id)
+
+        // Enable submit button
+        this.submitTarget.disabled = false
+        this.submitTarget.classList.remove("opacity-50", "cursor-not-allowed")
+
+        showFlashMessage('notice', `Archive uploaded successfully. Ready to import.`)
+
+        // Add a completion animation to the progress bar
+        const percentageDisplay = this.element.querySelector('.progress-percentage')
+        if (percentageDisplay) {
+          percentageDisplay.textContent = '100%'
+          percentageDisplay.classList.add('text-success')
+        }
+
+        if (this.hasProgressTarget) {
+          this.progressTarget.value = 100
+          this.progressTarget.classList.add('progress-success')
+          this.progressTarget.classList.remove('progress-primary')
+        }
+      }
+
+      this.isUploading = false
+      console.log("Upload completed")
+    })
+  }
+
+  isValidZipFile(file) {
+    // Check MIME type
+    const validMimeTypes = ['application/zip', 'application/x-zip-compressed']
+    if (validMimeTypes.includes(file.type)) {
+      return true
+    }
+
+    // Check file extension as fallback
+    const filename = file.name.toLowerCase()
+    return filename.endsWith('.zip')
+  }
+
+  directUploadWillStoreFileWithXHR(request) {
+    request.upload.addEventListener("progress", event => {
+      if (!this.hasProgressTarget) {
+        console.warn("Progress target not found")
+        return
+      }
+
+      const progress = (event.loaded / event.total) * 100
+      const progressPercentage = `${progress.toFixed(1)}%`
+      console.log(`Upload progress: ${progressPercentage}`)
+
+      // Update the DaisyUI progress element
+      this.progressTarget.value = progress
+
+      // Update the percentage display
+      const percentageDisplay = this.element.querySelector('.progress-percentage')
+      if (percentageDisplay) {
+        percentageDisplay.textContent = progressPercentage
+      }
+    })
+  }
+}

app/services/users/import_data.rb

@@ -41,63 +41,164 @@ class Users::ImportData
   end
 
   def import
-    @import_directory = Rails.root.join('tmp', "import_#{user.email.gsub(/[^0-9A-Za-z._-]/, '_')}_#{Time.current.to_i}")
-    FileUtils.mkdir_p(@import_directory)
+    data = stream_and_parse_archive
 
-    ActiveRecord::Base.transaction do
-      extract_archive
-      data = load_json_data
+    import_in_segments(data)
 
-      import_in_correct_order(data)
+    create_success_notification
 
-      create_success_notification
-
-      @import_stats
-    end
+    @import_stats
   rescue StandardError => e
     ExceptionReporter.call(e, 'Data import failed')
     create_failure_notification(e)
     raise e
   ensure
-    cleanup_temporary_files(@import_directory) if @import_directory&.exist?
+    # Clean up any temporary files created during streaming
+    cleanup_temporary_files
   end
 
   private
 
   attr_reader :user, :archive_path, :import_stats
 
-  def extract_archive
-    Rails.logger.info "Extracting archive: #{archive_path}"
+  def stream_and_parse_archive
+    Rails.logger.info "Streaming archive: #{archive_path}"
+
+    @temp_files = {}
+    @memory_tracker = Users::ImportData::MemoryTracker.new
+    data_json = nil
+
+    @memory_tracker.log('before_zip_processing')
 
     Zip::File.open(archive_path) do |zip_file|
       zip_file.each do |entry|
-        extraction_path = @import_directory.join(entry.name)
+        if entry.name == 'data.json'
+          Rails.logger.info "Processing data.json (#{entry.size} bytes)"
 
-        FileUtils.mkdir_p(File.dirname(extraction_path))
+          # Use streaming JSON parser for all files to reduce memory usage
+          streamer = Users::ImportData::JsonStreamer.new(entry)
+          data_json = streamer.stream_parse
 
-        entry.extract(extraction_path)
+          @memory_tracker.log('after_json_parsing')
+        elsif entry.name.start_with?('files/')
+          # Only extract files that are needed for file attachments
+          temp_path = stream_file_to_temp(entry)
+          @temp_files[entry.name] = temp_path
+        end
+        # Skip extracting other files to save disk space
       end
     end
+
+    raise StandardError, 'Data file not found in archive: data.json' unless data_json
+
+    @memory_tracker.log('archive_processing_completed')
+    data_json
+  end
+
+  def stream_file_to_temp(zip_entry)
+    require 'tmpdir'
+
+    # Create a temporary file for this attachment
+    temp_file = Tempfile.new([File.basename(zip_entry.name, '.*'), File.extname(zip_entry.name)])
+    temp_file.binmode
+
+    zip_entry.get_input_stream do |input_stream|
+      IO.copy_stream(input_stream, temp_file)
+    end
+
+    temp_file.close
+    temp_file.path
+  end
+
+  def import_in_segments(data)
+    Rails.logger.info "Starting segmented data import for user: #{user.email}"
+
+    @memory_tracker&.log('before_core_segment')
+    # Segment 1: User settings and core data (small, fast transaction)
+    import_core_data_segment(data)
+
+    @memory_tracker&.log('before_independent_segment')
+    # Segment 2: Independent entities that can be imported in parallel
+    import_independent_entities_segment(data)
+
+    @memory_tracker&.log('before_dependent_segment')
+    # Segment 3: Dependent entities that require references
+    import_dependent_entities_segment(data)
+
+    # Final validation check
+    validate_import_completeness(data['counts']) if data['counts']
+
+    @memory_tracker&.log('import_completed')
+    Rails.logger.info "Segmented data import completed. Stats: #{@import_stats}"
+  end
+
+  def import_core_data_segment(data)
+    ActiveRecord::Base.transaction do
+      Rails.logger.info 'Importing core data segment'
+
+      import_settings(data['settings']) if data['settings']
+      import_areas(data['areas']) if data['areas']
+      import_places(data['places']) if data['places']
+
+      Rails.logger.info 'Core data segment completed'
+    end
   end
 
-  def load_json_data
-    json_path = @import_directory.join('data.json')
+  def import_independent_entities_segment(data)
+    # These entities don't depend on each other and can be imported in parallel
+    entity_types = %w[imports exports trips stats notifications].select { |type| data[type] }
 
-    unless File.exist?(json_path)
-      raise StandardError, "Data file not found in archive: data.json"
+    if entity_types.empty?
+      Rails.logger.info 'No independent entities to import'
+      return
     end
 
-    JSON.parse(File.read(json_path))
-  rescue JSON::ParserError => e
-    raise StandardError, "Invalid JSON format in data file: #{e.message}"
+    Rails.logger.info "Processing #{entity_types.size} independent entity types in parallel"
+
+    # Use parallel processing for independent entities
+    Parallel.each(entity_types, in_threads: [entity_types.size, 3].min) do |entity_type|
+      ActiveRecord::Base.connection_pool.with_connection do
+        ActiveRecord::Base.transaction do
+          case entity_type
+          when 'imports'
+            import_imports(data['imports'])
+          when 'exports'
+            import_exports(data['exports'])
+          when 'trips'
+            import_trips(data['trips'])
+          when 'stats'
+            import_stats(data['stats'])
+          when 'notifications'
+            import_notifications(data['notifications'])
+          end
+
+          Rails.logger.info "#{entity_type.capitalize} segment completed in parallel"
+        end
+      end
+    end
+
+    Rails.logger.info 'All independent entities processing completed'
+  end
+
+  def import_dependent_entities_segment(data)
+    ActiveRecord::Base.transaction do
+      Rails.logger.info 'Importing dependent entities segment'
+
+      # Import visits after places to ensure proper place resolution
+      visits_imported = import_visits(data['visits']) if data['visits']
+      Rails.logger.info "Visits import phase completed: #{visits_imported} visits imported"
+
+      # Points are imported in their own optimized batching system
+      import_points(data['points']) if data['points']
+
+      Rails.logger.info 'Dependent entities segment completed'
+    end
   end
 
   def import_in_correct_order(data)
     Rails.logger.info "Starting data import for user: #{user.email}"
 
-    if data['counts']
-      Rails.logger.info "Expected entity counts from export: #{data['counts']}"
-    end
+    Rails.logger.info "Expected entity counts from export: #{data['counts']}" if data['counts']
 
     Rails.logger.debug "Available data keys: #{data.keys.inspect}"
 
@@ -121,9 +222,7 @@ class Users::ImportData
     import_points(data['points']) if data['points']
 
     # Final validation check
-    if data['counts']
-      validate_import_completeness(data['counts'])
-    end
+    validate_import_completeness(data['counts']) if data['counts']
 
     Rails.logger.info "Data import completed. Stats: #{@import_stats}"
   end
@@ -149,14 +248,14 @@ class Users::ImportData
 
   def import_imports(imports_data)
     Rails.logger.debug "Importing #{imports_data&.size || 0} imports"
-    imports_created, files_restored = Users::ImportData::Imports.new(user, imports_data, @import_directory.join('files')).call
+    imports_created, files_restored = Users::ImportData::Imports.new(user, imports_data, @temp_files).call
     @import_stats[:imports_created] = imports_created
     @import_stats[:files_restored] += files_restored
   end
 
   def import_exports(exports_data)
     Rails.logger.debug "Importing #{exports_data&.size || 0} exports"
-    exports_created, files_restored = Users::ImportData::Exports.new(user, exports_data, @import_directory.join('files')).call
+    exports_created, files_restored = Users::ImportData::Exports.new(user, exports_data, @temp_files).call
     @import_stats[:exports_created] = exports_created
     @import_stats[:files_restored] += files_restored
   end
@@ -199,11 +298,18 @@ class Users::ImportData
     end
   end
 
-  def cleanup_temporary_files(import_directory)
-    return unless File.directory?(import_directory)
+  def cleanup_temporary_files
+    return unless @temp_files
 
-    Rails.logger.info "Cleaning up temporary import directory: #{import_directory}"
-    FileUtils.rm_rf(import_directory)
+    Rails.logger.info "Cleaning up #{@temp_files.size} temporary files"
+
+    @temp_files.each_value do |temp_path|
+      File.delete(temp_path) if File.exist?(temp_path)
+    rescue StandardError => e
+      Rails.logger.warn "Failed to delete temporary file #{temp_path}: #{e.message}"
+    end
+
+    @temp_files.clear
   rescue StandardError => e
     ExceptionReporter.call(e, 'Failed to cleanup temporary files')
   end
@@ -238,24 +344,24 @@ class Users::ImportData
   end
 
   def validate_import_completeness(expected_counts)
-    Rails.logger.info "Validating import completeness..."
+    Rails.logger.info 'Validating import completeness...'
 
     discrepancies = []
 
     expected_counts.each do |entity, expected_count|
       actual_count = @import_stats[:"#{entity}_created"] || 0
 
-      if actual_count < expected_count
-        discrepancy = "#{entity}: expected #{expected_count}, got #{actual_count} (#{expected_count - actual_count} missing)"
-        discrepancies << discrepancy
-        Rails.logger.warn "Import discrepancy - #{discrepancy}"
-      end
+      next unless actual_count < expected_count
+
+      discrepancy = "#{entity}: expected #{expected_count}, got #{actual_count} (#{expected_count - actual_count} missing)"
+      discrepancies << discrepancy
+      Rails.logger.warn "Import discrepancy - #{discrepancy}"
     end
 
     if discrepancies.any?
       Rails.logger.warn "Import completed with discrepancies: #{discrepancies.join(', ')}"
     else
-      Rails.logger.info "Import validation successful - all entities imported correctly"
+      Rails.logger.info 'Import validation successful - all entities imported correctly'
     end
   end
 end

app/services/users/import_data/areas.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 class Users::ImportData::Areas
-  BATCH_SIZE = 1000
 
   def initialize(user, areas_data)
     @user = user
@@ -36,6 +35,10 @@ class Users::ImportData::Areas
 
   attr_reader :user, :areas_data
 
+  def batch_size
+    @batch_size ||= DawarichSettings.import_batch_size
+  end
+
   def filter_and_prepare_areas
     valid_areas = []
     skipped_count = 0
@@ -100,7 +103,7 @@ class Users::ImportData::Areas
   def bulk_import_areas(areas)
     total_created = 0
 
-    areas.each_slice(BATCH_SIZE) do |batch|
+    areas.each_slice(batch_size) do |batch|
       begin
         result = Area.upsert_all(
           batch,

app/services/users/import_data/json_streamer.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require 'yajl'
+
+class Users::ImportData::JsonStreamer
+  def initialize(zip_entry)
+    @zip_entry = zip_entry
+    @memory_tracker = Users::ImportData::MemoryTracker.new
+  end
+
+  def stream_parse
+    Rails.logger.info "Starting JSON streaming for #{@zip_entry.name} (#{@zip_entry.size} bytes)"
+
+    @memory_tracker.log("before_streaming")
+
+    data = {}
+
+    @zip_entry.get_input_stream do |input_stream|
+      parser = Yajl::Parser.new(symbolize_keys: false)
+
+      # Set up the parser to handle objects
+      parser.on_parse_complete = proc do |parsed_data|
+        Rails.logger.info "JSON streaming completed, parsed #{parsed_data.keys.size} entity types"
+
+        # Process each entity type
+        parsed_data.each do |entity_type, entity_data|
+          if entity_data.is_a?(Array)
+            Rails.logger.info "Streamed #{entity_type}: #{entity_data.size} items"
+          end
+        end
+
+        data = parsed_data
+        @memory_tracker.log("after_parsing")
+      end
+
+      # Stream parse the JSON
+      begin
+        parser.parse(input_stream)
+      rescue Yajl::ParseError => e
+        raise StandardError, "Invalid JSON format in data file: #{e.message}"
+      end
+    end
+
+    @memory_tracker.log("streaming_completed")
+
+    data
+  rescue StandardError => e
+    Rails.logger.error "JSON streaming failed: #{e.message}"
+    raise e
+  end
+
+  private
+
+  attr_reader :zip_entry
+end

app/services/users/import_data/memory_tracker.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+class Users::ImportData::MemoryTracker
+  def initialize
+    @process_id = Process.pid
+    @start_time = Time.current
+  end
+
+  def log(stage)
+    memory_mb = current_memory_usage
+    elapsed = elapsed_time
+
+    Rails.logger.info "Memory usage at #{stage}: #{memory_mb} MB (elapsed: #{elapsed}s)"
+
+    # Log a warning if memory usage is high
+    if memory_mb > 1000 # 1GB
+      Rails.logger.warn "High memory usage detected: #{memory_mb} MB at stage #{stage}"
+    end
+
+    { memory_mb: memory_mb, elapsed: elapsed, stage: stage }
+  end
+
+  private
+
+  def current_memory_usage
+    # Get memory usage from /proc/PID/status on Linux or fallback to ps
+    if File.exist?("/proc/#{@process_id}/status")
+      status = File.read("/proc/#{@process_id}/status")
+      match = status.match(/VmRSS:\s+(\d+)\s+kB/)
+      return match[1].to_i / 1024.0 if match # Convert KB to MB
+    end
+
+    # Fallback to ps command (works on macOS and Linux)
+    memory_kb = `ps -o rss= -p #{@process_id}`.strip.to_i
+    memory_kb / 1024.0 # Convert KB to MB
+  rescue StandardError => e
+    Rails.logger.warn "Failed to get memory usage: #{e.message}"
+    0.0
+  end
+
+  def elapsed_time
+    (Time.current - @start_time).round(2)
+  end
+end

app/services/users/import_data/notifications.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 class Users::ImportData::Notifications
-  BATCH_SIZE = 1000
 
   def initialize(user, notifications_data)
     @user = user
@@ -36,6 +35,10 @@ class Users::ImportData::Notifications
 
   attr_reader :user, :notifications_data
 
+  def batch_size
+    @batch_size ||= DawarichSettings.import_batch_size
+  end
+
   def filter_and_prepare_notifications
     valid_notifications = []
     skipped_count = 0
@@ -123,7 +126,7 @@ class Users::ImportData::Notifications
   def bulk_import_notifications(notifications)
     total_created = 0
 
-    notifications.each_slice(BATCH_SIZE) do |batch|
+    notifications.each_slice(batch_size) do |batch|
       begin
         result = Notification.upsert_all(
           batch,

app/services/users/import_data/points.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 class Users::ImportData::Points
-  BATCH_SIZE = 1000
 
   def initialize(user, points_data)
     @user = user
@@ -38,6 +37,10 @@ class Users::ImportData::Points
 
   attr_reader :user, :points_data, :imports_lookup, :countries_lookup, :visits_lookup
 
+  def batch_size
+    @batch_size ||= DawarichSettings.import_batch_size
+  end
+
   def preload_reference_data
     @imports_lookup = {}
     user.imports.each do |import|
@@ -71,14 +74,12 @@ class Users::ImportData::Points
 
       unless valid_point_data?(point_data)
         skipped_count += 1
-        Rails.logger.debug "Skipped point #{index}: invalid data - #{point_data.slice('timestamp', 'longitude', 'latitude', 'lonlat')}"
         next
       end
 
       prepared_attributes = prepare_point_attributes(point_data)
       unless prepared_attributes
         skipped_count += 1
-        Rails.logger.debug "Skipped point #{index}: failed to prepare attributes"
         next
       end
 
@@ -116,10 +117,7 @@ class Users::ImportData::Points
     resolve_country_reference(attributes, point_data['country_info'])
     resolve_visit_reference(attributes, point_data['visit_reference'])
 
-    result = attributes.symbolize_keys
-
-    Rails.logger.debug "Prepared point attributes: #{result.slice(:lonlat, :timestamp, :import_id, :country_id, :visit_id)}"
-    result
+    attributes.symbolize_keys
   rescue StandardError => e
     ExceptionReporter.call(e, 'Failed to prepare point attributes')
 
@@ -194,25 +192,20 @@ class Users::ImportData::Points
   end
 
   def normalize_point_keys(points)
-    all_keys = points.flat_map(&:keys).uniq
-
-    # Normalize each point to have all keys (with nil for missing ones)
-    points.map do |point|
-      normalized = {}
-      all_keys.each do |key|
-        normalized[key] = point[key]
-      end
-      normalized
-    end
+    # Return points as-is since upsert_all can handle inconsistent keys
+    # This eliminates the expensive hash reconstruction overhead
+    points
   end
 
   def bulk_import_points(points)
     total_created = 0
 
-    points.each_slice(BATCH_SIZE) do |batch|
+    points.each_slice(batch_size) do |batch|
       begin
-        Rails.logger.debug "Processing batch of #{batch.size} points"
-        Rails.logger.debug "First point in batch: #{batch.first.inspect}"
+        # Only log every 10th batch to reduce noise
+        if (total_created / batch_size) % 10 == 0
+          Rails.logger.info "Processed #{total_created} points so far, current batch: #{batch.size}"
+        end
 
         normalized_batch = normalize_point_keys(batch)
 
@@ -226,8 +219,6 @@ class Users::ImportData::Points
         batch_created = result.count
         total_created += batch_created
 
-        Rails.logger.debug "Processed batch of #{batch.size} points, created #{batch_created}, total created: #{total_created}"
-
       rescue StandardError => e
         Rails.logger.error "Failed to process point batch: #{e.message}"
         Rails.logger.error "Batch size: #{batch.size}"

app/services/users/import_data/stats.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 class Users::ImportData::Stats
-  BATCH_SIZE = 1000
 
   def initialize(user, stats_data)
     @user = user
@@ -36,6 +35,10 @@ class Users::ImportData::Stats
 
   attr_reader :user, :stats_data
 
+  def batch_size
+    @batch_size ||= DawarichSettings.import_batch_size
+  end
+
   def filter_and_prepare_stats
     valid_stats = []
     skipped_count = 0
@@ -99,7 +102,7 @@ class Users::ImportData::Stats
   def bulk_import_stats(stats)
     total_created = 0
 
-    stats.each_slice(BATCH_SIZE) do |batch|
+    stats.each_slice(batch_size) do |batch|
       begin
         result = Stat.upsert_all(
           batch,

app/services/users/import_data/trips.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 class Users::ImportData::Trips
-  BATCH_SIZE = 1000
 
   def initialize(user, trips_data)
     @user = user
@@ -36,6 +35,10 @@ class Users::ImportData::Trips
 
   attr_reader :user, :trips_data
 
+  def batch_size
+    @batch_size ||= DawarichSettings.import_batch_size
+  end
+
   def filter_and_prepare_trips
     valid_trips = []
     skipped_count = 0
@@ -111,7 +114,7 @@ class Users::ImportData::Trips
   def bulk_import_trips(trips)
     total_created = 0
 
-    trips.each_slice(BATCH_SIZE) do |batch|
+    trips.each_slice(batch_size) do |batch|
       begin
         result = Trip.upsert_all(
           batch,

app/views/devise/registrations/edit.html.erb

@@ -82,16 +82,35 @@
           <h3 class="font-bold text-lg mb-4">Import your data</h3>
           <p class="mb-4 text-sm text-gray-600">Upload a ZIP file containing your exported Dawarich data to restore your points, trips, and settings.</p>
 
-          <%= form_with url: import_settings_users_path, method: :post, multipart: true, class: 'space-y-4', data: { turbo: false } do |f| %>
+          <%= form_with url: import_settings_users_path, method: :post, multipart: true, class: 'space-y-4', data: {
+            turbo: false,
+            controller: "user-data-archive-direct-upload",
+            user_data_archive_direct_upload_url_value: rails_direct_uploads_url,
+            user_data_archive_direct_upload_user_trial_value: current_user.trial?,
+            user_data_archive_direct_upload_target: "form"
+          } do |f| %>
             <div class="form-control">
               <%= f.label :archive, class: 'label' do %>
                 <span class="label-text">Select ZIP archive</span>
               <% end %>
-              <%= f.file_field :archive, accept: '.zip', required: true, class: 'file-input file-input-bordered w-full' %>
+              <%= f.file_field :archive,
+                  accept: '.zip',
+                  required: true,
+                  direct_upload: true,
+                  class: 'file-input file-input-bordered w-full',
+                  data: { user_data_archive_direct_upload_target: "input" } %>
+              <div class="text-sm text-gray-500 mt-2">
+                File will be uploaded directly to storage. Please be patient during upload.
+              </div>
             </div>
 
             <div class="modal-action">
-              <%= f.submit "Import Data", class: 'btn btn-primary', data: { disable_with: 'Importing...' } %>
+              <%= f.submit "Import Data",
+                  class: 'btn btn-primary',
+                  data: {
+                    disable_with: 'Importing...',
+                    user_data_archive_direct_upload_target: "submit"
+                  } %>
               <button type="button" class="btn" onclick="import_modal.close()">Cancel</button>
             </div>
           <% end %>

config/initializers/03_dawarich_settings.rb

@@ -40,6 +40,10 @@ class DawarichSettings
       @store_geodata ||= STORE_GEODATA
     end
 
+    def import_batch_size
+      @import_batch_size ||= (ENV['IMPORT_BATCH_SIZE'].presence || 2500).to_i
+    end
+
     def features
       @features ||= {
         reverse_geocoding: reverse_geocoding_enabled?