Merge pull request #1790 from Freika/refactoring/user-data-import

Rework importing user data archive

CHANGELOG.md

@@ -22,6 +22,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 - Onboarding modal window now features a link to the App Store and a QR code to configure the Dawarich iOS app.
 - A permanent option was removed from stats sharing options. Now, stats can be shared for 1, 12 or 24 hours only.
+- User data archive importing now uploads the file directly to the storage service instead of uploading it to the app first.
+- Importing progress bars are now looking nice.
 - Ruby version was updated to 3.4.6.
 
 ## Added

app/controllers/settings/users_controller.rb

@@ -54,21 +54,28 @@ class Settings::UsersController < ApplicationController
   end
 
   def import
-    unless params[:archive].present?
+    if params[:archive].blank?
       redirect_to edit_user_registration_path, alert: 'Please select a ZIP archive to import.'
       return
     end
 
-    archive_file = params[:archive]
+    archive_param = params[:archive]
 
-    validate_archive_file(archive_file)
+    # Handle both direct upload (signed_id) and traditional upload (file)
+    if archive_param.is_a?(String)
+      # Direct upload: archive_param is a signed blob ID
+      import = create_import_from_signed_archive_id(archive_param)
+    else
+      # Traditional upload: archive_param is an uploaded file
+      validate_archive_file(archive_param)
 
-    import = current_user.imports.build(
-      name: archive_file.original_filename,
-      source: :user_data_archive
-    )
+      import = current_user.imports.build(
+        name: archive_param.original_filename,
+        source: :user_data_archive
+      )
 
-    import.file.attach(archive_file)
+      import.file.attach(archive_param)
+    end
 
     if import.save
       redirect_to edit_user_registration_path,
@@ -89,6 +96,36 @@ class Settings::UsersController < ApplicationController
     params.require(:user).permit(:email, :password)
   end
 
+  def create_import_from_signed_archive_id(signed_id)
+    Rails.logger.debug "Creating archive import from signed ID: #{signed_id[0..20]}..."
+
+    blob = ActiveStorage::Blob.find_signed(signed_id)
+
+    # Validate that it's a ZIP file
+    validate_blob_file_type(blob)
+
+    import_name = generate_unique_import_name(blob.filename.to_s)
+    import = current_user.imports.build(
+      name: import_name,
+      source: :user_data_archive
+    )
+    import.file.attach(blob)
+
+    import
+  end
+
+  def generate_unique_import_name(original_name)
+    return original_name unless current_user.imports.exists?(name: original_name)
+
+    # Extract filename and extension
+    basename = File.basename(original_name, File.extname(original_name))
+    extension = File.extname(original_name)
+
+    # Add current datetime
+    timestamp = Time.current.strftime('%Y%m%d_%H%M%S')
+    "#{basename}_#{timestamp}#{extension}"
+  end
+
   def validate_archive_file(archive_file)
     unless ['application/zip', 'application/x-zip-compressed'].include?(archive_file.content_type) ||
            File.extname(archive_file.original_filename).downcase == '.zip'
@@ -96,4 +133,12 @@ class Settings::UsersController < ApplicationController
       redirect_to edit_user_registration_path, alert: 'Please upload a valid ZIP file.' and return
     end
   end
+
+  def validate_blob_file_type(blob)
+    unless ['application/zip', 'application/x-zip-compressed'].include?(blob.content_type) ||
+           File.extname(blob.filename.to_s).downcase == '.zip'
+
+      raise StandardError, 'Please upload a valid ZIP file.'
+    end
+  end
 end

app/javascript/controllers/direct_upload_controller.js

@@ -93,31 +93,33 @@ export default class extends Controller {
       this.progressTarget.remove()
     }
 
-    // Create a wrapper div for better positioning and visibility
+    // Create a wrapper div with better DaisyUI styling
     const progressWrapper = document.createElement("div")
-    progressWrapper.className = "mt-4 mb-6 border p-4 rounded-lg bg-gray-50"
+    progressWrapper.className = "w-full mt-4 mb-4"
 
-    // Add a label
+    // Add a label with better typography
     const progressLabel = document.createElement("div")
-    progressLabel.className = "font-medium mb-2 text-gray-700"
-    progressLabel.textContent = "Upload Progress"
+    progressLabel.className = "text-sm font-medium text-base-content mb-2 flex justify-between items-center"
+    progressLabel.innerHTML = `
+      <span>Upload Progress</span>
+      <span class="text-xs text-base-content/70 progress-percentage">0%</span>
+    `
     progressWrapper.appendChild(progressLabel)
 
-    // Create a new progress container
-    const progressContainer = document.createElement("div")
+    // Create DaisyUI progress element
+    const progressContainer = document.createElement("progress")
     progressContainer.setAttribute("data-direct-upload-target", "progress")
-    progressContainer.className = "w-full bg-gray-200 rounded-full h-4"
+    progressContainer.className = "progress progress-primary w-full h-3"
+    progressContainer.value = 0
+    progressContainer.max = 100
 
-    // Create the progress bar fill element
+    // Create a hidden div for the progress bar target (for compatibility)
     const progressBarFill = document.createElement("div")
     progressBarFill.setAttribute("data-direct-upload-target", "progressBar")
-    progressBarFill.className = "bg-blue-600 h-4 rounded-full transition-all duration-300"
-    progressBarFill.style.width = "0%"
+    progressBarFill.style.display = "none"
 
-    // Add the fill element to the container
-    progressContainer.appendChild(progressBarFill)
     progressWrapper.appendChild(progressContainer)
-    progressBarFill.dataset.percentageDisplay = "true"
+    progressWrapper.appendChild(progressBarFill)
 
     // Add the progress wrapper AFTER the file input field but BEFORE the submit button
     this.submitTarget.parentNode.insertBefore(progressWrapper, this.submitTarget)
@@ -169,6 +171,19 @@ export default class extends Controller {
             showFlashMessage('error', 'No files were successfully uploaded. Please try again.')
           } else {
             showFlashMessage('notice', `${successfulUploads} file(s) uploaded successfully. Ready to submit.`)
+
+            // Add a completion animation to the progress bar
+            const percentageDisplay = this.element.querySelector('.progress-percentage')
+            if (percentageDisplay) {
+              percentageDisplay.textContent = '100%'
+              percentageDisplay.classList.add('text-success')
+            }
+
+            if (this.hasProgressTarget) {
+              this.progressTarget.value = 100
+              this.progressTarget.classList.add('progress-success')
+              this.progressTarget.classList.remove('progress-primary')
+            }
           }
           this.isUploading = false
           console.log("All uploads completed")
@@ -180,18 +195,20 @@ export default class extends Controller {
 
     directUploadWillStoreFileWithXHR(request) {
     request.upload.addEventListener("progress", event => {
-      if (!this.hasProgressBarTarget) {
-        console.warn("Progress bar target not found")
+      if (!this.hasProgressTarget) {
+        console.warn("Progress target not found")
         return
       }
 
       const progress = (event.loaded / event.total) * 100
       const progressPercentage = `${progress.toFixed(1)}%`
       console.log(`Upload progress: ${progressPercentage}`)
-      this.progressBarTarget.style.width = progressPercentage
 
-      // Update text percentage if exists
-      const percentageDisplay = this.element.querySelector('[data-percentage-display="true"]')
+      // Update the DaisyUI progress element
+      this.progressTarget.value = progress
+
+      // Update the percentage display
+      const percentageDisplay = this.element.querySelector('.progress-percentage')
       if (percentageDisplay) {
         percentageDisplay.textContent = progressPercentage
       }

app/javascript/controllers/user_data_archive_direct_upload_controller.js

@@ -0,0 +1,217 @@
+import { Controller } from "@hotwired/stimulus"
+import { DirectUpload } from "@rails/activestorage"
+import { showFlashMessage } from "../maps/helpers"
+
+export default class extends Controller {
+  static targets = ["input", "progress", "progressBar", "submit", "form"]
+  static values = {
+    url: String,
+    userTrial: Boolean
+  }
+
+  connect() {
+    this.inputTarget.addEventListener("change", this.upload.bind(this))
+
+    // Add form submission handler to disable the file input
+    if (this.hasFormTarget) {
+      this.formTarget.addEventListener("submit", this.onSubmit.bind(this))
+    }
+
+    // Initially disable submit button if no files are uploaded
+    if (this.hasSubmitTarget) {
+      const hasUploadedFiles = this.element.querySelectorAll('input[name="archive"][type="hidden"]').length > 0
+      this.submitTarget.disabled = !hasUploadedFiles
+    }
+  }
+
+  onSubmit(event) {
+    if (this.isUploading) {
+      // If still uploading, prevent submission
+      event.preventDefault()
+      console.log("Form submission prevented during upload")
+      return
+    }
+
+    // Disable the file input to prevent it from being submitted with the form
+    // This ensures only our hidden input with signed ID is submitted
+    this.inputTarget.disabled = true
+
+    // Check if we have a signed ID
+    const signedId = this.element.querySelector('input[name="archive"][type="hidden"]')
+    if (!signedId) {
+      event.preventDefault()
+      console.log("No file uploaded yet")
+      alert("Please select and upload a ZIP archive first")
+    } else {
+      console.log("Submitting form with uploaded archive")
+    }
+  }
+
+  upload() {
+    const files = this.inputTarget.files
+    if (files.length === 0) return
+
+    const file = files[0] // Only handle single file for archives
+
+    // Validate file type
+    if (!this.isValidZipFile(file)) {
+      showFlashMessage('error', 'Please select a valid ZIP file.')
+      this.inputTarget.value = ''
+      return
+    }
+
+    // Check file size limits for trial users
+    if (this.userTrialValue) {
+      const MAX_FILE_SIZE = 11 * 1024 * 1024 // 11MB in bytes
+
+      if (file.size > MAX_FILE_SIZE) {
+        const message = `File size limit exceeded. Trial users can only upload files up to 10MB. File size: ${(file.size / 1024 / 1024).toFixed(1)}MB`
+        showFlashMessage('error', message)
+
+        // Clear the file input
+        this.inputTarget.value = ''
+        return
+      }
+    }
+
+    console.log(`Uploading archive: ${file.name}`)
+    this.isUploading = true
+
+    // Disable submit button during upload
+    this.submitTarget.disabled = true
+    this.submitTarget.classList.add("opacity-50", "cursor-not-allowed")
+
+    // Show uploading message using flash
+    showFlashMessage('notice', `Uploading ${file.name}, please wait...`)
+
+    // Always remove any existing progress bar to ensure we create a fresh one
+    if (this.hasProgressTarget) {
+      this.progressTarget.remove()
+    }
+
+    // Create a wrapper div with better DaisyUI styling
+    const progressWrapper = document.createElement("div")
+    progressWrapper.className = "w-full mt-4 mb-4"
+
+    // Add a label with better typography
+    const progressLabel = document.createElement("div")
+    progressLabel.className = "text-sm font-medium text-base-content mb-2 flex justify-between items-center"
+    progressLabel.innerHTML = `
+      <span>Upload Progress</span>
+      <span class="text-xs text-base-content/70 progress-percentage">0%</span>
+    `
+    progressWrapper.appendChild(progressLabel)
+
+    // Create DaisyUI progress element
+    const progressContainer = document.createElement("progress")
+    progressContainer.setAttribute("data-user-data-archive-direct-upload-target", "progress")
+    progressContainer.className = "progress progress-primary w-full h-3"
+    progressContainer.value = 0
+    progressContainer.max = 100
+
+    // Create a hidden div for the progress bar target (for compatibility)
+    const progressBarFill = document.createElement("div")
+    progressBarFill.setAttribute("data-user-data-archive-direct-upload-target", "progressBar")
+    progressBarFill.style.display = "none"
+
+    progressWrapper.appendChild(progressContainer)
+    progressWrapper.appendChild(progressBarFill)
+
+    // Add the progress wrapper after the form-control div containing the file input
+    const formControl = this.inputTarget.closest('.form-control')
+    if (formControl) {
+      formControl.parentNode.insertBefore(progressWrapper, formControl.nextSibling)
+    } else {
+      // Fallback: insert before submit button
+      this.submitTarget.parentNode.insertBefore(progressWrapper, this.submitTarget)
+    }
+
+    console.log("Progress bar created and inserted after file input")
+
+    // Clear any existing hidden field for archive
+    const existingHiddenField = this.element.querySelector('input[name="archive"][type="hidden"]')
+    if (existingHiddenField) {
+      existingHiddenField.remove()
+    }
+
+    const upload = new DirectUpload(file, this.urlValue, this)
+    upload.create((error, blob) => {
+      if (error) {
+        console.error("Error uploading file:", error)
+        // Show error to user using flash
+        showFlashMessage('error', `Error uploading ${file.name}: ${error.message || 'Unknown error'}`)
+
+        // Re-enable submit button but keep it disabled since no file was uploaded
+        this.submitTarget.disabled = true
+        this.submitTarget.classList.add("opacity-50", "cursor-not-allowed")
+      } else {
+        console.log(`Successfully uploaded ${file.name} with ID: ${blob.signed_id}`)
+
+        // Create a hidden field with the correct name
+        const hiddenField = document.createElement("input")
+        hiddenField.setAttribute("type", "hidden")
+        hiddenField.setAttribute("name", "archive")
+        hiddenField.setAttribute("value", blob.signed_id)
+        this.element.appendChild(hiddenField)
+
+        console.log("Added hidden field with signed ID:", blob.signed_id)
+
+        // Enable submit button
+        this.submitTarget.disabled = false
+        this.submitTarget.classList.remove("opacity-50", "cursor-not-allowed")
+
+        showFlashMessage('notice', `Archive uploaded successfully. Ready to import.`)
+
+        // Add a completion animation to the progress bar
+        const percentageDisplay = this.element.querySelector('.progress-percentage')
+        if (percentageDisplay) {
+          percentageDisplay.textContent = '100%'
+          percentageDisplay.classList.add('text-success')
+        }
+
+        if (this.hasProgressTarget) {
+          this.progressTarget.value = 100
+          this.progressTarget.classList.add('progress-success')
+          this.progressTarget.classList.remove('progress-primary')
+        }
+      }
+
+      this.isUploading = false
+      console.log("Upload completed")
+    })
+  }
+
+  isValidZipFile(file) {
+    // Check MIME type
+    const validMimeTypes = ['application/zip', 'application/x-zip-compressed']
+    if (validMimeTypes.includes(file.type)) {
+      return true
+    }
+
+    // Check file extension as fallback
+    const filename = file.name.toLowerCase()
+    return filename.endsWith('.zip')
+  }
+
+  directUploadWillStoreFileWithXHR(request) {
+    request.upload.addEventListener("progress", event => {
+      if (!this.hasProgressTarget) {
+        console.warn("Progress target not found")
+        return
+      }
+
+      const progress = (event.loaded / event.total) * 100
+      const progressPercentage = `${progress.toFixed(1)}%`
+      console.log(`Upload progress: ${progressPercentage}`)
+
+      // Update the DaisyUI progress element
+      this.progressTarget.value = progress
+
+      // Update the percentage display
+      const percentageDisplay = this.element.querySelector('.progress-percentage')
+      if (percentageDisplay) {
+        percentageDisplay.textContent = progressPercentage
+      }
+    })
+  }
+}

app/views/devise/registrations/edit.html.erb

@@ -82,16 +82,35 @@
           <h3 class="font-bold text-lg mb-4">Import your data</h3>
           <p class="mb-4 text-sm text-gray-600">Upload a ZIP file containing your exported Dawarich data to restore your points, trips, and settings.</p>
 
-          <%= form_with url: import_settings_users_path, method: :post, multipart: true, class: 'space-y-4', data: { turbo: false } do |f| %>
+          <%= form_with url: import_settings_users_path, method: :post, multipart: true, class: 'space-y-4', data: {
+            turbo: false,
+            controller: "user-data-archive-direct-upload",
+            user_data_archive_direct_upload_url_value: rails_direct_uploads_url,
+            user_data_archive_direct_upload_user_trial_value: current_user.trial?,
+            user_data_archive_direct_upload_target: "form"
+          } do |f| %>
             <div class="form-control">
               <%= f.label :archive, class: 'label' do %>
                 <span class="label-text">Select ZIP archive</span>
               <% end %>
-              <%= f.file_field :archive, accept: '.zip', required: true, class: 'file-input file-input-bordered w-full' %>
+              <%= f.file_field :archive,
+                  accept: '.zip',
+                  required: true,
+                  direct_upload: true,
+                  class: 'file-input file-input-bordered w-full',
+                  data: { user_data_archive_direct_upload_target: "input" } %>
+              <div class="text-sm text-gray-500 mt-2">
+                File will be uploaded directly to storage. Please be patient during upload.
+              </div>
             </div>
 
             <div class="modal-action">
-              <%= f.submit "Import Data", class: 'btn btn-primary', data: { disable_with: 'Importing...' } %>
+              <%= f.submit "Import Data",
+                  class: 'btn btn-primary',
+                  data: {
+                    disable_with: 'Importing...',
+                    user_data_archive_direct_upload_target: "submit"
+                  } %>
               <button type="button" class="btn" onclick="import_modal.close()">Cancel</button>
             </div>
           <% end %>

config/initializers/03_dawarich_settings.rb

@@ -4,7 +4,6 @@ class DawarichSettings
   BASIC_PAID_PLAN_LIMIT = 10_000_000 # 10 million points
 
   class << self
-
     def reverse_geocoding_enabled?
       @reverse_geocoding_enabled ||= photon_enabled? || geoapify_enabled? || nominatim_enabled?
     end