User export: exporting exports and imports data with files

CHANGELOG.md

@@ -4,6 +4,27 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+# [UNRELEASED]
+
+## Added
+
+- [x] In the User Settings, you can now export your user data as a zip file. It will contain the following:
+  - [ ] All your points
+  - [ ] All your areas
+  - [ ] All your visits
+  - [x] All your imports with files
+  - [ ] All your exports with files
+  - [ ] All your trips
+  - [ ] All your places
+  - [ ] All your notifications
+  - [ ] All your stats
+
+- [ ] In the User Settings, you can now import your user data from a zip file. It will import all the data from the zip file, listed above. It will also start stats recalculation.
+
+## Changed
+
+- Oj is now being used for JSON serialization.
+
 # 0.28.1 - 2025-06-11
 
 ## Fixed

Gemfile

@@ -76,3 +76,5 @@ group :development do
   gem 'foreman'
   gem 'rubocop-rails', require: false
 end
+
+gem "rubyzip", "~> 2.4"

Gemfile.lock

@@ -557,6 +557,7 @@ DEPENDENCIES
   rswag-specs
   rswag-ui
   rubocop-rails
+  rubyzip (~> 2.4)
   selenium-webdriver
   sentry-rails
   sentry-ruby

app/controllers/settings/users_controller.rb

@@ -46,6 +46,15 @@ class Settings::UsersController < ApplicationController
     end
   end
 
+  def export
+    current_user.export_data
+
+    redirect_to exports_path, notice: 'Your data is being exported. You will receive a notification when it is ready.'
+  end
+
+  def import
+    @user = User.find(params[:id])
+
   private
 
   def user_params

app/jobs/users/export_data_job.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class Users::ExportDataJob < ApplicationJob
+  queue_as :exports
+
+  def perform(user_id)
+    user = User.find(user_id)
+
+    Users::ExportData.new(user).export
+  end
+end

app/models/export.rb

@@ -4,7 +4,8 @@ class Export < ApplicationRecord
   belongs_to :user
 
   enum :status, { created: 0, processing: 1, completed: 2, failed: 3 }
-  enum :file_format, { json: 0, gpx: 1 }
+  enum :file_format, { json: 0, gpx: 1, archive: 2 }
+  enum :file_type, { points: 0, user_data: 1 }
 
   validates :name, presence: true
 

app/models/import.rb

@@ -9,6 +9,8 @@ class Import < ApplicationRecord
   after_commit -> { Import::ProcessJob.perform_later(id) }, on: :create
   after_commit :remove_attached_file, on: :destroy
 
+  validates :name, presence: true, uniqueness: { scope: :user_id }
+
   enum :source, {
     google_semantic_history: 0, owntracks: 1, google_records: 2,
     google_phone_takeout: 3, gpx: 4, immich_api: 5, geojson: 6, photoprism_api: 7

app/models/user.rb

@@ -115,6 +115,10 @@ class User < ApplicationRecord
     JWT.encode(payload, secret_key, 'HS256')
   end
 
+  def export_data
+    Users::ExportDataJob.perform_later(id)
+  end
+
   private
 
   def create_api_key

app/services/users/export_data.rb

@@ -0,0 +1,213 @@
+# frozen_string_literal: true
+
+require 'zip'
+
+class Users::ExportData
+  def initialize(user)
+    @user = user
+    @export_directory = export_directory
+    @files_directory = files_directory
+  end
+
+  def export
+    # TODO: Implement
+    # 1. Export user settings
+    # 2. Export user points
+    # 3. Export user areas
+    # 4. Export user visits
+    # 7. Export user trips
+    # 8. Export user places
+    # 9. Export user notifications
+    # 10. Export user stats
+
+    # 11. Zip all the files
+
+    FileUtils.mkdir_p(files_directory)
+
+    begin
+      data = {}
+
+      data[:settings] = user.safe_settings.settings
+      data[:points] = nil
+      data[:areas] = nil
+      data[:visits] = nil
+      data[:imports] = serialized_imports
+      data[:exports] = serialized_exports
+      data[:trips] = nil
+      data[:places] = nil
+
+      json_file_path = export_directory.join('data.json')
+      File.write(json_file_path, data.to_json)
+
+      zip_file_path = export_directory.join('export.zip')
+      create_zip_archive(zip_file_path)
+
+      # Move the zip file to a final location (e.g., tmp root) before cleanup
+      final_zip_path = Rails.root.join('tmp', "#{user.email}_export_#{Time.current.strftime('%Y%m%d_%H%M%S')}.zip")
+      FileUtils.mv(zip_file_path, final_zip_path)
+
+      final_zip_path
+    ensure
+      cleanup_temporary_files
+    end
+  end
+
+  private
+
+  attr_reader :user
+
+  def export_directory
+    @export_directory ||= Rails.root.join('tmp', "#{user.email}_#{Time.current.strftime('%Y%m%d_%H%M%S')}")
+  end
+
+  def files_directory
+    @files_directory ||= export_directory.join('files')
+  end
+
+  def serialized_exports
+    exports_data = user.exports.includes(:file_attachment).map do |export|
+      process_export(export)
+    end
+
+    {
+      exports: exports_data,
+      export_directory: export_directory.to_s,
+      files_directory: files_directory.to_s
+    }
+  end
+
+  def process_export(export)
+    Rails.logger.info "Processing export #{export.name}"
+
+    # Only include essential attributes, exclude any potentially large fields
+    export_hash = export.as_json(except: %w[user_id])
+
+    if export.file.attached?
+      add_file_data_to_export(export, export_hash)
+    else
+      add_empty_file_data_to_export(export_hash)
+    end
+
+    Rails.logger.info "Export #{export.name} processed"
+
+    export_hash
+  end
+
+  def add_file_data_to_export(export, export_hash)
+    sanitized_filename = generate_sanitized_export_filename(export)
+    file_path = files_directory.join(sanitized_filename)
+
+    begin
+      download_and_save_export_file(export, file_path)
+      add_file_metadata_to_export(export, export_hash, sanitized_filename)
+    rescue StandardError => e
+      Rails.logger.error "Failed to download export file #{export.id}: #{e.message}"
+      export_hash['file_error'] = "Failed to download: #{e.message}"
+    end
+  end
+
+  def add_empty_file_data_to_export(export_hash)
+    export_hash['file_name'] = nil
+    export_hash['original_filename'] = nil
+  end
+
+  def generate_sanitized_export_filename(export)
+    "export_#{export.id}_#{export.file.blob.filename}".gsub(/[^0-9A-Za-z._-]/, '_')
+  end
+
+  def download_and_save_export_file(export, file_path)
+    file_content = Imports::SecureFileDownloader.new(export.file).download_with_verification
+    File.write(file_path, file_content, mode: 'wb')
+  end
+
+  def add_file_metadata_to_export(export, export_hash, sanitized_filename)
+    export_hash['file_name'] = sanitized_filename
+    export_hash['original_filename'] = export.file.blob.filename.to_s
+    export_hash['file_size'] = export.file.blob.byte_size
+    export_hash['content_type'] = export.file.blob.content_type
+  end
+
+  def serialized_imports
+    imports_data = user.imports.includes(:file_attachment).map do |import|
+      process_import(import)
+    end
+
+    {
+      imports: imports_data,
+      export_directory: export_directory.to_s,
+      files_directory: files_directory.to_s
+    }
+  end
+
+  def process_import(import)
+    Rails.logger.info "Processing import #{import.name}"
+
+    # Only include essential attributes, exclude large fields like raw_data
+    import_hash = import.as_json(except: %w[user_id raw_data])
+
+    if import.file.attached?
+      add_file_data_to_import(import, import_hash)
+    else
+      add_empty_file_data_to_import(import_hash)
+    end
+
+    Rails.logger.info "Import #{import.name} processed"
+
+    import_hash
+  end
+
+  def add_file_data_to_import(import, import_hash)
+    sanitized_filename = generate_sanitized_filename(import)
+    file_path = files_directory.join(sanitized_filename)
+
+    begin
+      download_and_save_import_file(import, file_path)
+      add_file_metadata_to_import(import, import_hash, sanitized_filename)
+    rescue StandardError => e
+      Rails.logger.error "Failed to download import file #{import.id}: #{e.message}"
+      import_hash['file_error'] = "Failed to download: #{e.message}"
+    end
+  end
+
+  def add_empty_file_data_to_import(import_hash)
+    import_hash['file_name'] = nil
+    import_hash['original_filename'] = nil
+  end
+
+  def generate_sanitized_filename(import)
+    "import_#{import.id}_#{import.file.blob.filename}".gsub(/[^0-9A-Za-z._-]/, '_')
+  end
+
+  def download_and_save_import_file(import, file_path)
+    file_content = Imports::SecureFileDownloader.new(import.file).download_with_verification
+    File.write(file_path, file_content, mode: 'wb')
+  end
+
+  def add_file_metadata_to_import(import, import_hash, sanitized_filename)
+    import_hash['file_name'] = sanitized_filename
+    import_hash['original_filename'] = import.file.blob.filename.to_s
+    import_hash['file_size'] = import.file.blob.byte_size
+    import_hash['content_type'] = import.file.blob.content_type
+  end
+
+  def create_zip_archive(zip_file_path)
+    Zip::File.open(zip_file_path, Zip::File::CREATE) do |zipfile|
+      Dir.glob(export_directory.join('**', '*')).each do |file|
+        next if File.directory?(file) || file == zip_file_path.to_s
+
+        relative_path = file.sub(export_directory.to_s + '/', '')
+        zipfile.add(relative_path, file)
+      end
+    end
+  end
+
+  def cleanup_temporary_files
+    return unless File.directory?(export_directory)
+
+    Rails.logger.info "Cleaning up temporary export directory: #{export_directory}"
+    FileUtils.rm_rf(export_directory)
+  rescue StandardError => e
+    Rails.logger.error "Failed to cleanup temporary files: #{e.message}"
+    # Don't re-raise the error as cleanup failure shouldn't break the export
+  end
+end

app/views/devise/registrations/edit.html.erb

@@ -61,6 +61,11 @@
       <% end %>
 
       <p class='mt-3'>Unhappy? <%= link_to "Cancel my account", registration_path(resource_name), data: { confirm: "Are you sure?", turbo_confirm: "Are you sure?", turbo_method: :delete }, method: :delete, class: 'btn' %></p>
+      <div class="divider"></div>
+      <p class='mt-3 flex flex-col gap-2'>
+        <%= link_to "Export my data", export_settings_users_path, class: 'btn btn-primary' %>
+        <%= link_to "Import my data", import_settings_users_path, class: 'btn btn-primary' %>
+      </p>
     </div>
   </div>
 </div>

config/initializers/oj.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+Oj.optimize_rails

config/routes.rb

@@ -36,7 +36,13 @@ Rails.application.routes.draw do
   resources :settings, only: :index
   namespace :settings do
     resources :background_jobs, only: %i[index create]
-    resources :users, only: %i[index create destroy edit update]
+    resources :users, only: %i[index create destroy edit update] do
+      collection do
+        get 'export'
+        post 'import'
+      end
+    end
+
     resources :maps, only: %i[index]
     patch 'maps', to: 'maps#update'
   end

db/migrate/20250625185030_add_file_type_to_exports.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class AddFileTypeToExports < ActiveRecord::Migration[8.0]
+  disable_ddl_transaction!
+
+  def up
+    add_column :exports, :file_type, :integer, default: 0, null: false
+    add_index :exports, :file_type, algorithm: :concurrently
+  end
+
+  def down
+    remove_index :exports, :file_type, algorithm: :concurrently
+    remove_column :exports, :file_type
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[8.0].define(version: 2025_05_15_192211) do
+ActiveRecord::Schema[8.0].define(version: 2025_06_25_185030) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_catalog.plpgsql"
   enable_extension "postgis"
@@ -90,6 +90,8 @@ ActiveRecord::Schema[8.0].define(version: 2025_05_15_192211) do
     t.integer "file_format", default: 0
     t.datetime "start_at"
     t.datetime "end_at"
+    t.integer "file_type", default: 0, null: false
+    t.index ["file_type"], name: "index_exports_on_file_type"
     t.index ["status"], name: "index_exports_on_status"
     t.index ["user_id"], name: "index_exports_on_user_id"
   end

spec/jobs/users/export_data_job_spec.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe Users::ExportDataJob, type: :job do
+  let(:user) { create(:user) }
+  let(:export_data) { Users::ExportData.new(user) }
+
+  it 'exports the user data' do
+    expect(Users::ExportData).to receive(:new).with(user).and_return(export_data)
+    expect(export_data).to receive(:export)
+
+    Users::ExportDataJob.perform_now(user.id)
+  end
+end

spec/models/import_spec.rb

@@ -8,6 +8,11 @@ RSpec.describe Import, type: :model do
     it { is_expected.to belong_to(:user) }
   end
 
+  describe 'validations' do
+    it { is_expected.to validate_presence_of(:name) }
+    it { is_expected.to validate_uniqueness_of(:name).scoped_to(:user_id) }
+  end
+
   describe 'enums' do
     it do
       is_expected.to define_enum_for(:source).with_values(

spec/models/user_spec.rb

@@ -204,5 +204,11 @@ RSpec.describe User, type: :model do
         end
       end
     end
+
+    describe '#export_data' do
+      it 'enqueues the export data job' do
+        expect { user.export_data }.to have_enqueued_job(Users::ExportDataJob).with(user.id)
+      end
+    end
   end
 end