mirror of
https://github.com/Freika/dawarich.git
synced 2026-01-10 17:21:38 -05:00
Rework importing user data archive
This commit is contained in:
Eugene Burmakin
parent
a84fde553e
commit
031104cdaa
17 changed files with 618 additions and 103 deletions
|
|
@ -19,6 +19,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
|
|||
|
||||
- Onboarding modal window now features a link to the App Store and a QR code to configure the Dawarich iOS app.
|
||||
- A permanent option was removed from stats sharing options. Now, stats can be shared for 1, 12 or 24 hours only.
|
||||
- User data archive importing now uploads the file directly to the storage service instead of uploading it to the app first.
|
||||
- Importing progress bars are now looking nice.
|
||||
|
||||
## Added
|
||||
|
||||
|
|
|
|||
1
Gemfile
1
Gemfile
|
|
@ -25,6 +25,7 @@ gem 'kaminari'
|
|||
gem 'lograge'
|
||||
gem 'oj'
|
||||
gem 'parallel'
|
||||
gem 'yajl-ruby', '~> 1.4'
|
||||
gem 'pg'
|
||||
gem 'prometheus_exporter'
|
||||
gem 'puma'
|
||||
|
|
|
|||
|
|
@ -521,6 +521,7 @@ GEM
|
|||
websocket-extensions (0.1.5)
|
||||
xpath (3.2.0)
|
||||
nokogiri (~> 1.8)
|
||||
yajl-ruby (1.4.3)
|
||||
zeitwerk (2.7.3)
|
||||
|
||||
PLATFORMS
|
||||
|
|
@ -598,6 +599,7 @@ DEPENDENCIES
|
|||
turbo-rails
|
||||
tzinfo-data
|
||||
webmock
|
||||
yajl-ruby (~> 1.4)
|
||||
|
||||
RUBY VERSION
|
||||
ruby 3.4.1p0
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -54,21 +54,28 @@ class Settings::UsersController < ApplicationController
|
|||
end
|
||||
|
||||
def import
|
||||
unless params[:archive].present?
|
||||
if params[:archive].blank?
|
||||
redirect_to edit_user_registration_path, alert: 'Please select a ZIP archive to import.'
|
||||
return
|
||||
end
|
||||
|
||||
archive_file = params[:archive]
|
||||
archive_param = params[:archive]
|
||||
|
||||
validate_archive_file(archive_file)
|
||||
# Handle both direct upload (signed_id) and traditional upload (file)
|
||||
if archive_param.is_a?(String)
|
||||
# Direct upload: archive_param is a signed blob ID
|
||||
import = create_import_from_signed_archive_id(archive_param)
|
||||
else
|
||||
# Traditional upload: archive_param is an uploaded file
|
||||
validate_archive_file(archive_param)
|
||||
|
||||
import = current_user.imports.build(
|
||||
name: archive_file.original_filename,
|
||||
source: :user_data_archive
|
||||
)
|
||||
import = current_user.imports.build(
|
||||
name: archive_param.original_filename,
|
||||
source: :user_data_archive
|
||||
)
|
||||
|
||||
import.file.attach(archive_file)
|
||||
import.file.attach(archive_param)
|
||||
end
|
||||
|
||||
if import.save
|
||||
redirect_to edit_user_registration_path,
|
||||
|
|
@ -89,6 +96,36 @@ class Settings::UsersController < ApplicationController
|
|||
params.require(:user).permit(:email, :password)
|
||||
end
|
||||
|
||||
def create_import_from_signed_archive_id(signed_id)
|
||||
Rails.logger.debug "Creating archive import from signed ID: #{signed_id[0..20]}..."
|
||||
|
||||
blob = ActiveStorage::Blob.find_signed(signed_id)
|
||||
|
||||
# Validate that it's a ZIP file
|
||||
validate_blob_file_type(blob)
|
||||
|
||||
import_name = generate_unique_import_name(blob.filename.to_s)
|
||||
import = current_user.imports.build(
|
||||
name: import_name,
|
||||
source: :user_data_archive
|
||||
)
|
||||
import.file.attach(blob)
|
||||
|
||||
import
|
||||
end
|
||||
|
||||
def generate_unique_import_name(original_name)
|
||||
return original_name unless current_user.imports.exists?(name: original_name)
|
||||
|
||||
# Extract filename and extension
|
||||
basename = File.basename(original_name, File.extname(original_name))
|
||||
extension = File.extname(original_name)
|
||||
|
||||
# Add current datetime
|
||||
timestamp = Time.current.strftime('%Y%m%d_%H%M%S')
|
||||
"#{basename}_#{timestamp}#{extension}"
|
||||
end
|
||||
|
||||
def validate_archive_file(archive_file)
|
||||
unless ['application/zip', 'application/x-zip-compressed'].include?(archive_file.content_type) ||
|
||||
File.extname(archive_file.original_filename).downcase == '.zip'
|
||||
|
|
@ -96,4 +133,12 @@ class Settings::UsersController < ApplicationController
|
|||
redirect_to edit_user_registration_path, alert: 'Please upload a valid ZIP file.' and return
|
||||
end
|
||||
end
|
||||
|
||||
def validate_blob_file_type(blob)
|
||||
unless ['application/zip', 'application/x-zip-compressed'].include?(blob.content_type) ||
|
||||
File.extname(blob.filename.to_s).downcase == '.zip'
|
||||
|
||||
raise StandardError, 'Please upload a valid ZIP file.'
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -82,31 +82,33 @@ export default class extends Controller {
|
|||
this.progressTarget.remove()
|
||||
}
|
||||
|
||||
// Create a wrapper div for better positioning and visibility
|
||||
// Create a wrapper div with better DaisyUI styling
|
||||
const progressWrapper = document.createElement("div")
|
||||
progressWrapper.className = "mt-4 mb-6 border p-4 rounded-lg bg-gray-50"
|
||||
progressWrapper.className = "w-full mt-4 mb-4"
|
||||
|
||||
// Add a label
|
||||
// Add a label with better typography
|
||||
const progressLabel = document.createElement("div")
|
||||
progressLabel.className = "font-medium mb-2 text-gray-700"
|
||||
progressLabel.textContent = "Upload Progress"
|
||||
progressLabel.className = "text-sm font-medium text-base-content mb-2 flex justify-between items-center"
|
||||
progressLabel.innerHTML = `
|
||||
<span>Upload Progress</span>
|
||||
<span class="text-xs text-base-content/70 progress-percentage">0%</span>
|
||||
`
|
||||
progressWrapper.appendChild(progressLabel)
|
||||
|
||||
// Create a new progress container
|
||||
const progressContainer = document.createElement("div")
|
||||
// Create DaisyUI progress element
|
||||
const progressContainer = document.createElement("progress")
|
||||
progressContainer.setAttribute("data-direct-upload-target", "progress")
|
||||
progressContainer.className = "w-full bg-gray-200 rounded-full h-4"
|
||||
progressContainer.className = "progress progress-primary w-full h-3"
|
||||
progressContainer.value = 0
|
||||
progressContainer.max = 100
|
||||
|
||||
// Create the progress bar fill element
|
||||
// Create a hidden div for the progress bar target (for compatibility)
|
||||
const progressBarFill = document.createElement("div")
|
||||
progressBarFill.setAttribute("data-direct-upload-target", "progressBar")
|
||||
progressBarFill.className = "bg-blue-600 h-4 rounded-full transition-all duration-300"
|
||||
progressBarFill.style.width = "0%"
|
||||
progressBarFill.style.display = "none"
|
||||
|
||||
// Add the fill element to the container
|
||||
progressContainer.appendChild(progressBarFill)
|
||||
progressWrapper.appendChild(progressContainer)
|
||||
progressBarFill.dataset.percentageDisplay = "true"
|
||||
progressWrapper.appendChild(progressBarFill)
|
||||
|
||||
// Add the progress wrapper AFTER the file input field but BEFORE the submit button
|
||||
this.submitTarget.parentNode.insertBefore(progressWrapper, this.submitTarget)
|
||||
|
|
@ -158,6 +160,19 @@ export default class extends Controller {
|
|||
showFlashMessage('error', 'No files were successfully uploaded. Please try again.')
|
||||
} else {
|
||||
showFlashMessage('notice', `${successfulUploads} file(s) uploaded successfully. Ready to submit.`)
|
||||
|
||||
// Add a completion animation to the progress bar
|
||||
const percentageDisplay = this.element.querySelector('.progress-percentage')
|
||||
if (percentageDisplay) {
|
||||
percentageDisplay.textContent = '100%'
|
||||
percentageDisplay.classList.add('text-success')
|
||||
}
|
||||
|
||||
if (this.hasProgressTarget) {
|
||||
this.progressTarget.value = 100
|
||||
this.progressTarget.classList.add('progress-success')
|
||||
this.progressTarget.classList.remove('progress-primary')
|
||||
}
|
||||
}
|
||||
this.isUploading = false
|
||||
console.log("All uploads completed")
|
||||
|
|
@ -169,18 +184,20 @@ export default class extends Controller {
|
|||
|
||||
directUploadWillStoreFileWithXHR(request) {
|
||||
request.upload.addEventListener("progress", event => {
|
||||
if (!this.hasProgressBarTarget) {
|
||||
console.warn("Progress bar target not found")
|
||||
if (!this.hasProgressTarget) {
|
||||
console.warn("Progress target not found")
|
||||
return
|
||||
}
|
||||
|
||||
const progress = (event.loaded / event.total) * 100
|
||||
const progressPercentage = `${progress.toFixed(1)}%`
|
||||
console.log(`Upload progress: ${progressPercentage}`)
|
||||
this.progressBarTarget.style.width = progressPercentage
|
||||
|
||||
// Update text percentage if exists
|
||||
const percentageDisplay = this.element.querySelector('[data-percentage-display="true"]')
|
||||
// Update the DaisyUI progress element
|
||||
this.progressTarget.value = progress
|
||||
|
||||
// Update the percentage display
|
||||
const percentageDisplay = this.element.querySelector('.progress-percentage')
|
||||
if (percentageDisplay) {
|
||||
percentageDisplay.textContent = progressPercentage
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,217 @@
|
|||
import { Controller } from "@hotwired/stimulus"
|
||||
import { DirectUpload } from "@rails/activestorage"
|
||||
import { showFlashMessage } from "../maps/helpers"
|
||||
|
||||
export default class extends Controller {
|
||||
static targets = ["input", "progress", "progressBar", "submit", "form"]
|
||||
static values = {
|
||||
url: String,
|
||||
userTrial: Boolean
|
||||
}
|
||||
|
||||
connect() {
|
||||
this.inputTarget.addEventListener("change", this.upload.bind(this))
|
||||
|
||||
// Add form submission handler to disable the file input
|
||||
if (this.hasFormTarget) {
|
||||
this.formTarget.addEventListener("submit", this.onSubmit.bind(this))
|
||||
}
|
||||
|
||||
// Initially disable submit button if no files are uploaded
|
||||
if (this.hasSubmitTarget) {
|
||||
const hasUploadedFiles = this.element.querySelectorAll('input[name="archive"][type="hidden"]').length > 0
|
||||
this.submitTarget.disabled = !hasUploadedFiles
|
||||
}
|
||||
}
|
||||
|
||||
onSubmit(event) {
|
||||
if (this.isUploading) {
|
||||
// If still uploading, prevent submission
|
||||
event.preventDefault()
|
||||
console.log("Form submission prevented during upload")
|
||||
return
|
||||
}
|
||||
|
||||
// Disable the file input to prevent it from being submitted with the form
|
||||
// This ensures only our hidden input with signed ID is submitted
|
||||
this.inputTarget.disabled = true
|
||||
|
||||
// Check if we have a signed ID
|
||||
const signedId = this.element.querySelector('input[name="archive"][type="hidden"]')
|
||||
if (!signedId) {
|
||||
event.preventDefault()
|
||||
console.log("No file uploaded yet")
|
||||
alert("Please select and upload a ZIP archive first")
|
||||
} else {
|
||||
console.log("Submitting form with uploaded archive")
|
||||
}
|
||||
}
|
||||
|
||||
upload() {
|
||||
const files = this.inputTarget.files
|
||||
if (files.length === 0) return
|
||||
|
||||
const file = files[0] // Only handle single file for archives
|
||||
|
||||
// Validate file type
|
||||
if (!this.isValidZipFile(file)) {
|
||||
showFlashMessage('error', 'Please select a valid ZIP file.')
|
||||
this.inputTarget.value = ''
|
||||
return
|
||||
}
|
||||
|
||||
// Check file size limits for trial users
|
||||
if (this.userTrialValue) {
|
||||
const MAX_FILE_SIZE = 11 * 1024 * 1024 // 11MB in bytes
|
||||
|
||||
if (file.size > MAX_FILE_SIZE) {
|
||||
const message = `File size limit exceeded. Trial users can only upload files up to 10MB. File size: ${(file.size / 1024 / 1024).toFixed(1)}MB`
|
||||
showFlashMessage('error', message)
|
||||
|
||||
// Clear the file input
|
||||
this.inputTarget.value = ''
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Uploading archive: ${file.name}`)
|
||||
this.isUploading = true
|
||||
|
||||
// Disable submit button during upload
|
||||
this.submitTarget.disabled = true
|
||||
this.submitTarget.classList.add("opacity-50", "cursor-not-allowed")
|
||||
|
||||
// Show uploading message using flash
|
||||
showFlashMessage('notice', `Uploading ${file.name}, please wait...`)
|
||||
|
||||
// Always remove any existing progress bar to ensure we create a fresh one
|
||||
if (this.hasProgressTarget) {
|
||||
this.progressTarget.remove()
|
||||
}
|
||||
|
||||
// Create a wrapper div with better DaisyUI styling
|
||||
const progressWrapper = document.createElement("div")
|
||||
progressWrapper.className = "w-full mt-4 mb-4"
|
||||
|
||||
// Add a label with better typography
|
||||
const progressLabel = document.createElement("div")
|
||||
progressLabel.className = "text-sm font-medium text-base-content mb-2 flex justify-between items-center"
|
||||
progressLabel.innerHTML = `
|
||||
<span>Upload Progress</span>
|
||||
<span class="text-xs text-base-content/70 progress-percentage">0%</span>
|
||||
`
|
||||
progressWrapper.appendChild(progressLabel)
|
||||
|
||||
// Create DaisyUI progress element
|
||||
const progressContainer = document.createElement("progress")
|
||||
progressContainer.setAttribute("data-user-data-archive-direct-upload-target", "progress")
|
||||
progressContainer.className = "progress progress-primary w-full h-3"
|
||||
progressContainer.value = 0
|
||||
progressContainer.max = 100
|
||||
|
||||
// Create a hidden div for the progress bar target (for compatibility)
|
||||
const progressBarFill = document.createElement("div")
|
||||
progressBarFill.setAttribute("data-user-data-archive-direct-upload-target", "progressBar")
|
||||
progressBarFill.style.display = "none"
|
||||
|
||||
progressWrapper.appendChild(progressContainer)
|
||||
progressWrapper.appendChild(progressBarFill)
|
||||
|
||||
// Add the progress wrapper after the form-control div containing the file input
|
||||
const formControl = this.inputTarget.closest('.form-control')
|
||||
if (formControl) {
|
||||
formControl.parentNode.insertBefore(progressWrapper, formControl.nextSibling)
|
||||
} else {
|
||||
// Fallback: insert before submit button
|
||||
this.submitTarget.parentNode.insertBefore(progressWrapper, this.submitTarget)
|
||||
}
|
||||
|
||||
console.log("Progress bar created and inserted after file input")
|
||||
|
||||
// Clear any existing hidden field for archive
|
||||
const existingHiddenField = this.element.querySelector('input[name="archive"][type="hidden"]')
|
||||
if (existingHiddenField) {
|
||||
existingHiddenField.remove()
|
||||
}
|
||||
|
||||
const upload = new DirectUpload(file, this.urlValue, this)
|
||||
upload.create((error, blob) => {
|
||||
if (error) {
|
||||
console.error("Error uploading file:", error)
|
||||
// Show error to user using flash
|
||||
showFlashMessage('error', `Error uploading ${file.name}: ${error.message || 'Unknown error'}`)
|
||||
|
||||
// Re-enable submit button but keep it disabled since no file was uploaded
|
||||
this.submitTarget.disabled = true
|
||||
this.submitTarget.classList.add("opacity-50", "cursor-not-allowed")
|
||||
} else {
|
||||
console.log(`Successfully uploaded ${file.name} with ID: ${blob.signed_id}`)
|
||||
|
||||
// Create a hidden field with the correct name
|
||||
const hiddenField = document.createElement("input")
|
||||
hiddenField.setAttribute("type", "hidden")
|
||||
hiddenField.setAttribute("name", "archive")
|
||||
hiddenField.setAttribute("value", blob.signed_id)
|
||||
this.element.appendChild(hiddenField)
|
||||
|
||||
console.log("Added hidden field with signed ID:", blob.signed_id)
|
||||
|
||||
// Enable submit button
|
||||
this.submitTarget.disabled = false
|
||||
this.submitTarget.classList.remove("opacity-50", "cursor-not-allowed")
|
||||
|
||||
showFlashMessage('notice', `Archive uploaded successfully. Ready to import.`)
|
||||
|
||||
// Add a completion animation to the progress bar
|
||||
const percentageDisplay = this.element.querySelector('.progress-percentage')
|
||||
if (percentageDisplay) {
|
||||
percentageDisplay.textContent = '100%'
|
||||
percentageDisplay.classList.add('text-success')
|
||||
}
|
||||
|
||||
if (this.hasProgressTarget) {
|
||||
this.progressTarget.value = 100
|
||||
this.progressTarget.classList.add('progress-success')
|
||||
this.progressTarget.classList.remove('progress-primary')
|
||||
}
|
||||
}
|
||||
|
||||
this.isUploading = false
|
||||
console.log("Upload completed")
|
||||
})
|
||||
}
|
||||
|
||||
isValidZipFile(file) {
|
||||
// Check MIME type
|
||||
const validMimeTypes = ['application/zip', 'application/x-zip-compressed']
|
||||
if (validMimeTypes.includes(file.type)) {
|
||||
return true
|
||||
}
|
||||
|
||||
// Check file extension as fallback
|
||||
const filename = file.name.toLowerCase()
|
||||
return filename.endsWith('.zip')
|
||||
}
|
||||
|
||||
directUploadWillStoreFileWithXHR(request) {
|
||||
request.upload.addEventListener("progress", event => {
|
||||
if (!this.hasProgressTarget) {
|
||||
console.warn("Progress target not found")
|
||||
return
|
||||
}
|
||||
|
||||
const progress = (event.loaded / event.total) * 100
|
||||
const progressPercentage = `${progress.toFixed(1)}%`
|
||||
console.log(`Upload progress: ${progressPercentage}`)
|
||||
|
||||
// Update the DaisyUI progress element
|
||||
this.progressTarget.value = progress
|
||||
|
||||
// Update the percentage display
|
||||
const percentageDisplay = this.element.querySelector('.progress-percentage')
|
||||
if (percentageDisplay) {
|
||||
percentageDisplay.textContent = progressPercentage
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -41,63 +41,164 @@ class Users::ImportData
|
|||
end
|
||||
|
||||
def import
|
||||
@import_directory = Rails.root.join('tmp', "import_#{user.email.gsub(/[^0-9A-Za-z._-]/, '_')}_#{Time.current.to_i}")
|
||||
FileUtils.mkdir_p(@import_directory)
|
||||
data = stream_and_parse_archive
|
||||
|
||||
ActiveRecord::Base.transaction do
|
||||
extract_archive
|
||||
data = load_json_data
|
||||
import_in_segments(data)
|
||||
|
||||
import_in_correct_order(data)
|
||||
create_success_notification
|
||||
|
||||
create_success_notification
|
||||
|
||||
@import_stats
|
||||
end
|
||||
@import_stats
|
||||
rescue StandardError => e
|
||||
ExceptionReporter.call(e, 'Data import failed')
|
||||
create_failure_notification(e)
|
||||
raise e
|
||||
ensure
|
||||
cleanup_temporary_files(@import_directory) if @import_directory&.exist?
|
||||
# Clean up any temporary files created during streaming
|
||||
cleanup_temporary_files
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
attr_reader :user, :archive_path, :import_stats
|
||||
|
||||
def extract_archive
|
||||
Rails.logger.info "Extracting archive: #{archive_path}"
|
||||
def stream_and_parse_archive
|
||||
Rails.logger.info "Streaming archive: #{archive_path}"
|
||||
|
||||
@temp_files = {}
|
||||
@memory_tracker = Users::ImportData::MemoryTracker.new
|
||||
data_json = nil
|
||||
|
||||
@memory_tracker.log('before_zip_processing')
|
||||
|
||||
Zip::File.open(archive_path) do |zip_file|
|
||||
zip_file.each do |entry|
|
||||
extraction_path = @import_directory.join(entry.name)
|
||||
if entry.name == 'data.json'
|
||||
Rails.logger.info "Processing data.json (#{entry.size} bytes)"
|
||||
|
||||
FileUtils.mkdir_p(File.dirname(extraction_path))
|
||||
# Use streaming JSON parser for all files to reduce memory usage
|
||||
streamer = Users::ImportData::JsonStreamer.new(entry)
|
||||
data_json = streamer.stream_parse
|
||||
|
||||
entry.extract(extraction_path)
|
||||
@memory_tracker.log('after_json_parsing')
|
||||
elsif entry.name.start_with?('files/')
|
||||
# Only extract files that are needed for file attachments
|
||||
temp_path = stream_file_to_temp(entry)
|
||||
@temp_files[entry.name] = temp_path
|
||||
end
|
||||
# Skip extracting other files to save disk space
|
||||
end
|
||||
end
|
||||
|
||||
raise StandardError, 'Data file not found in archive: data.json' unless data_json
|
||||
|
||||
@memory_tracker.log('archive_processing_completed')
|
||||
data_json
|
||||
end
|
||||
|
||||
def stream_file_to_temp(zip_entry)
|
||||
require 'tmpdir'
|
||||
|
||||
# Create a temporary file for this attachment
|
||||
temp_file = Tempfile.new([File.basename(zip_entry.name, '.*'), File.extname(zip_entry.name)])
|
||||
temp_file.binmode
|
||||
|
||||
zip_entry.get_input_stream do |input_stream|
|
||||
IO.copy_stream(input_stream, temp_file)
|
||||
end
|
||||
|
||||
temp_file.close
|
||||
temp_file.path
|
||||
end
|
||||
|
||||
def import_in_segments(data)
|
||||
Rails.logger.info "Starting segmented data import for user: #{user.email}"
|
||||
|
||||
@memory_tracker&.log('before_core_segment')
|
||||
# Segment 1: User settings and core data (small, fast transaction)
|
||||
import_core_data_segment(data)
|
||||
|
||||
@memory_tracker&.log('before_independent_segment')
|
||||
# Segment 2: Independent entities that can be imported in parallel
|
||||
import_independent_entities_segment(data)
|
||||
|
||||
@memory_tracker&.log('before_dependent_segment')
|
||||
# Segment 3: Dependent entities that require references
|
||||
import_dependent_entities_segment(data)
|
||||
|
||||
# Final validation check
|
||||
validate_import_completeness(data['counts']) if data['counts']
|
||||
|
||||
@memory_tracker&.log('import_completed')
|
||||
Rails.logger.info "Segmented data import completed. Stats: #{@import_stats}"
|
||||
end
|
||||
|
||||
def import_core_data_segment(data)
|
||||
ActiveRecord::Base.transaction do
|
||||
Rails.logger.info 'Importing core data segment'
|
||||
|
||||
import_settings(data['settings']) if data['settings']
|
||||
import_areas(data['areas']) if data['areas']
|
||||
import_places(data['places']) if data['places']
|
||||
|
||||
Rails.logger.info 'Core data segment completed'
|
||||
end
|
||||
end
|
||||
|
||||
def load_json_data
|
||||
json_path = @import_directory.join('data.json')
|
||||
def import_independent_entities_segment(data)
|
||||
# These entities don't depend on each other and can be imported in parallel
|
||||
entity_types = %w[imports exports trips stats notifications].select { |type| data[type] }
|
||||
|
||||
unless File.exist?(json_path)
|
||||
raise StandardError, "Data file not found in archive: data.json"
|
||||
if entity_types.empty?
|
||||
Rails.logger.info 'No independent entities to import'
|
||||
return
|
||||
end
|
||||
|
||||
JSON.parse(File.read(json_path))
|
||||
rescue JSON::ParserError => e
|
||||
raise StandardError, "Invalid JSON format in data file: #{e.message}"
|
||||
Rails.logger.info "Processing #{entity_types.size} independent entity types in parallel"
|
||||
|
||||
# Use parallel processing for independent entities
|
||||
Parallel.each(entity_types, in_threads: [entity_types.size, 3].min) do |entity_type|
|
||||
ActiveRecord::Base.connection_pool.with_connection do
|
||||
ActiveRecord::Base.transaction do
|
||||
case entity_type
|
||||
when 'imports'
|
||||
import_imports(data['imports'])
|
||||
when 'exports'
|
||||
import_exports(data['exports'])
|
||||
when 'trips'
|
||||
import_trips(data['trips'])
|
||||
when 'stats'
|
||||
import_stats(data['stats'])
|
||||
when 'notifications'
|
||||
import_notifications(data['notifications'])
|
||||
end
|
||||
|
||||
Rails.logger.info "#{entity_type.capitalize} segment completed in parallel"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Rails.logger.info 'All independent entities processing completed'
|
||||
end
|
||||
|
||||
def import_dependent_entities_segment(data)
|
||||
ActiveRecord::Base.transaction do
|
||||
Rails.logger.info 'Importing dependent entities segment'
|
||||
|
||||
# Import visits after places to ensure proper place resolution
|
||||
visits_imported = import_visits(data['visits']) if data['visits']
|
||||
Rails.logger.info "Visits import phase completed: #{visits_imported} visits imported"
|
||||
|
||||
# Points are imported in their own optimized batching system
|
||||
import_points(data['points']) if data['points']
|
||||
|
||||
Rails.logger.info 'Dependent entities segment completed'
|
||||
end
|
||||
end
|
||||
|
||||
def import_in_correct_order(data)
|
||||
Rails.logger.info "Starting data import for user: #{user.email}"
|
||||
|
||||
if data['counts']
|
||||
Rails.logger.info "Expected entity counts from export: #{data['counts']}"
|
||||
end
|
||||
Rails.logger.info "Expected entity counts from export: #{data['counts']}" if data['counts']
|
||||
|
||||
Rails.logger.debug "Available data keys: #{data.keys.inspect}"
|
||||
|
||||
|
|
@ -121,9 +222,7 @@ class Users::ImportData
|
|||
import_points(data['points']) if data['points']
|
||||
|
||||
# Final validation check
|
||||
if data['counts']
|
||||
validate_import_completeness(data['counts'])
|
||||
end
|
||||
validate_import_completeness(data['counts']) if data['counts']
|
||||
|
||||
Rails.logger.info "Data import completed. Stats: #{@import_stats}"
|
||||
end
|
||||
|
|
@ -149,14 +248,14 @@ class Users::ImportData
|
|||
|
||||
def import_imports(imports_data)
|
||||
Rails.logger.debug "Importing #{imports_data&.size || 0} imports"
|
||||
imports_created, files_restored = Users::ImportData::Imports.new(user, imports_data, @import_directory.join('files')).call
|
||||
imports_created, files_restored = Users::ImportData::Imports.new(user, imports_data, @temp_files).call
|
||||
@import_stats[:imports_created] = imports_created
|
||||
@import_stats[:files_restored] += files_restored
|
||||
end
|
||||
|
||||
def import_exports(exports_data)
|
||||
Rails.logger.debug "Importing #{exports_data&.size || 0} exports"
|
||||
exports_created, files_restored = Users::ImportData::Exports.new(user, exports_data, @import_directory.join('files')).call
|
||||
exports_created, files_restored = Users::ImportData::Exports.new(user, exports_data, @temp_files).call
|
||||
@import_stats[:exports_created] = exports_created
|
||||
@import_stats[:files_restored] += files_restored
|
||||
end
|
||||
|
|
@ -199,11 +298,18 @@ class Users::ImportData
|
|||
end
|
||||
end
|
||||
|
||||
def cleanup_temporary_files(import_directory)
|
||||
return unless File.directory?(import_directory)
|
||||
def cleanup_temporary_files
|
||||
return unless @temp_files
|
||||
|
||||
Rails.logger.info "Cleaning up temporary import directory: #{import_directory}"
|
||||
FileUtils.rm_rf(import_directory)
|
||||
Rails.logger.info "Cleaning up #{@temp_files.size} temporary files"
|
||||
|
||||
@temp_files.each_value do |temp_path|
|
||||
File.delete(temp_path) if File.exist?(temp_path)
|
||||
rescue StandardError => e
|
||||
Rails.logger.warn "Failed to delete temporary file #{temp_path}: #{e.message}"
|
||||
end
|
||||
|
||||
@temp_files.clear
|
||||
rescue StandardError => e
|
||||
ExceptionReporter.call(e, 'Failed to cleanup temporary files')
|
||||
end
|
||||
|
|
@ -238,24 +344,24 @@ class Users::ImportData
|
|||
end
|
||||
|
||||
def validate_import_completeness(expected_counts)
|
||||
Rails.logger.info "Validating import completeness..."
|
||||
Rails.logger.info 'Validating import completeness...'
|
||||
|
||||
discrepancies = []
|
||||
|
||||
expected_counts.each do |entity, expected_count|
|
||||
actual_count = @import_stats[:"#{entity}_created"] || 0
|
||||
|
||||
if actual_count < expected_count
|
||||
discrepancy = "#{entity}: expected #{expected_count}, got #{actual_count} (#{expected_count - actual_count} missing)"
|
||||
discrepancies << discrepancy
|
||||
Rails.logger.warn "Import discrepancy - #{discrepancy}"
|
||||
end
|
||||
next unless actual_count < expected_count
|
||||
|
||||
discrepancy = "#{entity}: expected #{expected_count}, got #{actual_count} (#{expected_count - actual_count} missing)"
|
||||
discrepancies << discrepancy
|
||||
Rails.logger.warn "Import discrepancy - #{discrepancy}"
|
||||
end
|
||||
|
||||
if discrepancies.any?
|
||||
Rails.logger.warn "Import completed with discrepancies: #{discrepancies.join(', ')}"
|
||||
else
|
||||
Rails.logger.info "Import validation successful - all entities imported correctly"
|
||||
Rails.logger.info 'Import validation successful - all entities imported correctly'
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class Users::ImportData::Areas
|
||||
BATCH_SIZE = 1000
|
||||
|
||||
def initialize(user, areas_data)
|
||||
@user = user
|
||||
|
|
@ -36,6 +35,10 @@ class Users::ImportData::Areas
|
|||
|
||||
attr_reader :user, :areas_data
|
||||
|
||||
def batch_size
|
||||
@batch_size ||= DawarichSettings.import_batch_size
|
||||
end
|
||||
|
||||
def filter_and_prepare_areas
|
||||
valid_areas = []
|
||||
skipped_count = 0
|
||||
|
|
@ -100,7 +103,7 @@ class Users::ImportData::Areas
|
|||
def bulk_import_areas(areas)
|
||||
total_created = 0
|
||||
|
||||
areas.each_slice(BATCH_SIZE) do |batch|
|
||||
areas.each_slice(batch_size) do |batch|
|
||||
begin
|
||||
result = Area.upsert_all(
|
||||
batch,
|
||||
|
|
|
|||
55
app/services/users/import_data/json_streamer.rb
Normal file
55
app/services/users/import_data/json_streamer.rb
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require 'yajl'
|
||||
|
||||
class Users::ImportData::JsonStreamer
|
||||
def initialize(zip_entry)
|
||||
@zip_entry = zip_entry
|
||||
@memory_tracker = Users::ImportData::MemoryTracker.new
|
||||
end
|
||||
|
||||
def stream_parse
|
||||
Rails.logger.info "Starting JSON streaming for #{@zip_entry.name} (#{@zip_entry.size} bytes)"
|
||||
|
||||
@memory_tracker.log("before_streaming")
|
||||
|
||||
data = {}
|
||||
|
||||
@zip_entry.get_input_stream do |input_stream|
|
||||
parser = Yajl::Parser.new(symbolize_keys: false)
|
||||
|
||||
# Set up the parser to handle objects
|
||||
parser.on_parse_complete = proc do |parsed_data|
|
||||
Rails.logger.info "JSON streaming completed, parsed #{parsed_data.keys.size} entity types"
|
||||
|
||||
# Process each entity type
|
||||
parsed_data.each do |entity_type, entity_data|
|
||||
if entity_data.is_a?(Array)
|
||||
Rails.logger.info "Streamed #{entity_type}: #{entity_data.size} items"
|
||||
end
|
||||
end
|
||||
|
||||
data = parsed_data
|
||||
@memory_tracker.log("after_parsing")
|
||||
end
|
||||
|
||||
# Stream parse the JSON
|
||||
begin
|
||||
parser.parse(input_stream)
|
||||
rescue Yajl::ParseError => e
|
||||
raise StandardError, "Invalid JSON format in data file: #{e.message}"
|
||||
end
|
||||
end
|
||||
|
||||
@memory_tracker.log("streaming_completed")
|
||||
|
||||
data
|
||||
rescue StandardError => e
|
||||
Rails.logger.error "JSON streaming failed: #{e.message}"
|
||||
raise e
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
attr_reader :zip_entry
|
||||
end
|
||||
44
app/services/users/import_data/memory_tracker.rb
Normal file
44
app/services/users/import_data/memory_tracker.rb
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class Users::ImportData::MemoryTracker
|
||||
def initialize
|
||||
@process_id = Process.pid
|
||||
@start_time = Time.current
|
||||
end
|
||||
|
||||
def log(stage)
|
||||
memory_mb = current_memory_usage
|
||||
elapsed = elapsed_time
|
||||
|
||||
Rails.logger.info "Memory usage at #{stage}: #{memory_mb} MB (elapsed: #{elapsed}s)"
|
||||
|
||||
# Log a warning if memory usage is high
|
||||
if memory_mb > 1000 # 1GB
|
||||
Rails.logger.warn "High memory usage detected: #{memory_mb} MB at stage #{stage}"
|
||||
end
|
||||
|
||||
{ memory_mb: memory_mb, elapsed: elapsed, stage: stage }
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def current_memory_usage
|
||||
# Get memory usage from /proc/PID/status on Linux or fallback to ps
|
||||
if File.exist?("/proc/#{@process_id}/status")
|
||||
status = File.read("/proc/#{@process_id}/status")
|
||||
match = status.match(/VmRSS:\s+(\d+)\s+kB/)
|
||||
return match[1].to_i / 1024.0 if match # Convert KB to MB
|
||||
end
|
||||
|
||||
# Fallback to ps command (works on macOS and Linux)
|
||||
memory_kb = `ps -o rss= -p #{@process_id}`.strip.to_i
|
||||
memory_kb / 1024.0 # Convert KB to MB
|
||||
rescue StandardError => e
|
||||
Rails.logger.warn "Failed to get memory usage: #{e.message}"
|
||||
0.0
|
||||
end
|
||||
|
||||
def elapsed_time
|
||||
(Time.current - @start_time).round(2)
|
||||
end
|
||||
end
|
||||
|
|
@ -1,7 +1,6 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class Users::ImportData::Notifications
|
||||
BATCH_SIZE = 1000
|
||||
|
||||
def initialize(user, notifications_data)
|
||||
@user = user
|
||||
|
|
@ -36,6 +35,10 @@ class Users::ImportData::Notifications
|
|||
|
||||
attr_reader :user, :notifications_data
|
||||
|
||||
def batch_size
|
||||
@batch_size ||= DawarichSettings.import_batch_size
|
||||
end
|
||||
|
||||
def filter_and_prepare_notifications
|
||||
valid_notifications = []
|
||||
skipped_count = 0
|
||||
|
|
@ -123,7 +126,7 @@ class Users::ImportData::Notifications
|
|||
def bulk_import_notifications(notifications)
|
||||
total_created = 0
|
||||
|
||||
notifications.each_slice(BATCH_SIZE) do |batch|
|
||||
notifications.each_slice(batch_size) do |batch|
|
||||
begin
|
||||
result = Notification.upsert_all(
|
||||
batch,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class Users::ImportData::Points
|
||||
BATCH_SIZE = 1000
|
||||
|
||||
def initialize(user, points_data)
|
||||
@user = user
|
||||
|
|
@ -38,6 +37,10 @@ class Users::ImportData::Points
|
|||
|
||||
attr_reader :user, :points_data, :imports_lookup, :countries_lookup, :visits_lookup
|
||||
|
||||
def batch_size
|
||||
@batch_size ||= DawarichSettings.import_batch_size
|
||||
end
|
||||
|
||||
def preload_reference_data
|
||||
@imports_lookup = {}
|
||||
user.imports.each do |import|
|
||||
|
|
@ -71,14 +74,12 @@ class Users::ImportData::Points
|
|||
|
||||
unless valid_point_data?(point_data)
|
||||
skipped_count += 1
|
||||
Rails.logger.debug "Skipped point #{index}: invalid data - #{point_data.slice('timestamp', 'longitude', 'latitude', 'lonlat')}"
|
||||
next
|
||||
end
|
||||
|
||||
prepared_attributes = prepare_point_attributes(point_data)
|
||||
unless prepared_attributes
|
||||
skipped_count += 1
|
||||
Rails.logger.debug "Skipped point #{index}: failed to prepare attributes"
|
||||
next
|
||||
end
|
||||
|
||||
|
|
@ -116,10 +117,7 @@ class Users::ImportData::Points
|
|||
resolve_country_reference(attributes, point_data['country_info'])
|
||||
resolve_visit_reference(attributes, point_data['visit_reference'])
|
||||
|
||||
result = attributes.symbolize_keys
|
||||
|
||||
Rails.logger.debug "Prepared point attributes: #{result.slice(:lonlat, :timestamp, :import_id, :country_id, :visit_id)}"
|
||||
result
|
||||
attributes.symbolize_keys
|
||||
rescue StandardError => e
|
||||
ExceptionReporter.call(e, 'Failed to prepare point attributes')
|
||||
|
||||
|
|
@ -194,25 +192,20 @@ class Users::ImportData::Points
|
|||
end
|
||||
|
||||
def normalize_point_keys(points)
|
||||
all_keys = points.flat_map(&:keys).uniq
|
||||
|
||||
# Normalize each point to have all keys (with nil for missing ones)
|
||||
points.map do |point|
|
||||
normalized = {}
|
||||
all_keys.each do |key|
|
||||
normalized[key] = point[key]
|
||||
end
|
||||
normalized
|
||||
end
|
||||
# Return points as-is since upsert_all can handle inconsistent keys
|
||||
# This eliminates the expensive hash reconstruction overhead
|
||||
points
|
||||
end
|
||||
|
||||
def bulk_import_points(points)
|
||||
total_created = 0
|
||||
|
||||
points.each_slice(BATCH_SIZE) do |batch|
|
||||
points.each_slice(batch_size) do |batch|
|
||||
begin
|
||||
Rails.logger.debug "Processing batch of #{batch.size} points"
|
||||
Rails.logger.debug "First point in batch: #{batch.first.inspect}"
|
||||
# Only log every 10th batch to reduce noise
|
||||
if (total_created / batch_size) % 10 == 0
|
||||
Rails.logger.info "Processed #{total_created} points so far, current batch: #{batch.size}"
|
||||
end
|
||||
|
||||
normalized_batch = normalize_point_keys(batch)
|
||||
|
||||
|
|
@ -226,8 +219,6 @@ class Users::ImportData::Points
|
|||
batch_created = result.count
|
||||
total_created += batch_created
|
||||
|
||||
Rails.logger.debug "Processed batch of #{batch.size} points, created #{batch_created}, total created: #{total_created}"
|
||||
|
||||
rescue StandardError => e
|
||||
Rails.logger.error "Failed to process point batch: #{e.message}"
|
||||
Rails.logger.error "Batch size: #{batch.size}"
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class Users::ImportData::Stats
|
||||
BATCH_SIZE = 1000
|
||||
|
||||
def initialize(user, stats_data)
|
||||
@user = user
|
||||
|
|
@ -36,6 +35,10 @@ class Users::ImportData::Stats
|
|||
|
||||
attr_reader :user, :stats_data
|
||||
|
||||
def batch_size
|
||||
@batch_size ||= DawarichSettings.import_batch_size
|
||||
end
|
||||
|
||||
def filter_and_prepare_stats
|
||||
valid_stats = []
|
||||
skipped_count = 0
|
||||
|
|
@ -99,7 +102,7 @@ class Users::ImportData::Stats
|
|||
def bulk_import_stats(stats)
|
||||
total_created = 0
|
||||
|
||||
stats.each_slice(BATCH_SIZE) do |batch|
|
||||
stats.each_slice(batch_size) do |batch|
|
||||
begin
|
||||
result = Stat.upsert_all(
|
||||
batch,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class Users::ImportData::Trips
|
||||
BATCH_SIZE = 1000
|
||||
|
||||
def initialize(user, trips_data)
|
||||
@user = user
|
||||
|
|
@ -36,6 +35,10 @@ class Users::ImportData::Trips
|
|||
|
||||
attr_reader :user, :trips_data
|
||||
|
||||
def batch_size
|
||||
@batch_size ||= DawarichSettings.import_batch_size
|
||||
end
|
||||
|
||||
def filter_and_prepare_trips
|
||||
valid_trips = []
|
||||
skipped_count = 0
|
||||
|
|
@ -111,7 +114,7 @@ class Users::ImportData::Trips
|
|||
def bulk_import_trips(trips)
|
||||
total_created = 0
|
||||
|
||||
trips.each_slice(BATCH_SIZE) do |batch|
|
||||
trips.each_slice(batch_size) do |batch|
|
||||
begin
|
||||
result = Trip.upsert_all(
|
||||
batch,
|
||||
|
|
|
|||
|
|
@ -82,16 +82,35 @@
|
|||
<h3 class="font-bold text-lg mb-4">Import your data</h3>
|
||||
<p class="mb-4 text-sm text-gray-600">Upload a ZIP file containing your exported Dawarich data to restore your points, trips, and settings.</p>
|
||||
|
||||
<%= form_with url: import_settings_users_path, method: :post, multipart: true, class: 'space-y-4', data: { turbo: false } do |f| %>
|
||||
<%= form_with url: import_settings_users_path, method: :post, multipart: true, class: 'space-y-4', data: {
|
||||
turbo: false,
|
||||
controller: "user-data-archive-direct-upload",
|
||||
user_data_archive_direct_upload_url_value: rails_direct_uploads_url,
|
||||
user_data_archive_direct_upload_user_trial_value: current_user.trial?,
|
||||
user_data_archive_direct_upload_target: "form"
|
||||
} do |f| %>
|
||||
<div class="form-control">
|
||||
<%= f.label :archive, class: 'label' do %>
|
||||
<span class="label-text">Select ZIP archive</span>
|
||||
<% end %>
|
||||
<%= f.file_field :archive, accept: '.zip', required: true, class: 'file-input file-input-bordered w-full' %>
|
||||
<%= f.file_field :archive,
|
||||
accept: '.zip',
|
||||
required: true,
|
||||
direct_upload: true,
|
||||
class: 'file-input file-input-bordered w-full',
|
||||
data: { user_data_archive_direct_upload_target: "input" } %>
|
||||
<div class="text-sm text-gray-500 mt-2">
|
||||
File will be uploaded directly to storage. Please be patient during upload.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="modal-action">
|
||||
<%= f.submit "Import Data", class: 'btn btn-primary', data: { disable_with: 'Importing...' } %>
|
||||
<%= f.submit "Import Data",
|
||||
class: 'btn btn-primary',
|
||||
data: {
|
||||
disable_with: 'Importing...',
|
||||
user_data_archive_direct_upload_target: "submit"
|
||||
} %>
|
||||
<button type="button" class="btn" onclick="import_modal.close()">Cancel</button>
|
||||
</div>
|
||||
<% end %>
|
||||
|
|
|
|||
|
|
@ -40,6 +40,10 @@ class DawarichSettings
|
|||
@store_geodata ||= STORE_GEODATA
|
||||
end
|
||||
|
||||
def import_batch_size
|
||||
@import_batch_size ||= (ENV['IMPORT_BATCH_SIZE'].presence || 2500).to_i
|
||||
end
|
||||
|
||||
def features
|
||||
@features ||= {
|
||||
reverse_geocoding: reverse_geocoding_enabled?
|
||||
|
|
|
|||
Loading…
Reference in a new issue