diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d28189c..2540fc88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## Added - [x] In the User Settings, you can now export your user data as a zip file. It will contain the following: - - [ ] All your points - - [ ] All your places - - [ ] All your visits + - [x] All your points + - [x] All your places + - [x] All your visits - [x] All your areas - [x] All your imports with files - [x] All your exports with files diff --git a/app/models/country.rb b/app/models/country.rb index 2cc5d4b7..9ef64687 100644 --- a/app/models/country.rb +++ b/app/models/country.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class Country < ApplicationRecord + has_many :points, dependent: :nullify + validates :name, :iso_a2, :iso_a3, :geom, presence: true def self.containing_point(lon, lat) diff --git a/app/models/point.rb b/app/models/point.rb index dab71d5f..6620dc14 100644 --- a/app/models/point.rb +++ b/app/models/point.rb @@ -7,6 +7,7 @@ class Point < ApplicationRecord belongs_to :import, optional: true, counter_cache: true belongs_to :visit, optional: true belongs_to :user + belongs_to :country, optional: true validates :timestamp, :lonlat, presence: true validates :lonlat, uniqueness: { diff --git a/app/services/users/export_data.rb b/app/services/users/export_data.rb index 7f16b0e9..fbc2b308 100644 --- a/app/services/users/export_data.rb +++ b/app/services/users/export_data.rb @@ -173,15 +173,15 @@ class Users::ExportData data = {} data[:settings] = user.safe_settings.settings - data[:areas] = serialized_areas - data[:imports] = serialized_imports - data[:exports] = serialized_exports - data[:trips] = serialized_trips - data[:stats] = serialized_stats - data[:notifications] = serialized_notifications - data[:points] = serialized_points - data[:visits] = serialized_visits - data[:places] = serialized_places + data[:areas] = Users::ExportData::Areas.new(user).call + data[:imports] = Users::ExportData::Imports.new(user, files_directory).call + data[:exports] = Users::ExportData::Exports.new(user, files_directory).call + data[:trips] = Users::ExportData::Trips.new(user).call + data[:stats] = Users::ExportData::Stats.new(user).call + data[:notifications] = Users::ExportData::Notifications.new(user).call + data[:points] = Users::ExportData::Points.new(user).call + data[:visits] = Users::ExportData::Visits.new(user).call + data[:places] = Users::ExportData::Places.new(user).call json_file_path = export_directory.join('data.json') File.write(json_file_path, data.to_json) @@ -211,124 +211,6 @@ class Users::ExportData @files_directory ||= export_directory.join('files') end - def serialized_exports - exports_data = user.exports.includes(:file_attachment).map do |export| - process_export(export) - end - - exports_data - end - - def process_export(export) - Rails.logger.info "Processing export #{export.name}" - - # Only include essential attributes, exclude any potentially large fields - export_hash = export.as_json(except: %w[user_id]) - - if export.file.attached? - add_file_data_to_export(export, export_hash) - else - add_empty_file_data_to_export(export_hash) - end - - Rails.logger.info "Export #{export.name} processed" - - export_hash - end - - def add_file_data_to_export(export, export_hash) - sanitized_filename = generate_sanitized_export_filename(export) - file_path = files_directory.join(sanitized_filename) - - begin - download_and_save_export_file(export, file_path) - add_file_metadata_to_export(export, export_hash, sanitized_filename) - rescue StandardError => e - Rails.logger.error "Failed to download export file #{export.id}: #{e.message}" - export_hash['file_error'] = "Failed to download: #{e.message}" - end - end - - def add_empty_file_data_to_export(export_hash) - export_hash['file_name'] = nil - export_hash['original_filename'] = nil - end - - def generate_sanitized_export_filename(export) - "export_#{export.id}_#{export.file.blob.filename}".gsub(/[^0-9A-Za-z._-]/, '_') - end - - def download_and_save_export_file(export, file_path) - file_content = Imports::SecureFileDownloader.new(export.file).download_with_verification - File.write(file_path, file_content, mode: 'wb') - end - - def add_file_metadata_to_export(export, export_hash, sanitized_filename) - export_hash['file_name'] = sanitized_filename - export_hash['original_filename'] = export.file.blob.filename.to_s - export_hash['file_size'] = export.file.blob.byte_size - export_hash['content_type'] = export.file.blob.content_type - end - - def serialized_imports - imports_data = user.imports.includes(:file_attachment).map do |import| - process_import(import) - end - - imports_data - end - - def process_import(import) - Rails.logger.info "Processing import #{import.name}" - - # Only include essential attributes, exclude large fields like raw_data - import_hash = import.as_json(except: %w[user_id raw_data]) - - if import.file.attached? - add_file_data_to_import(import, import_hash) - else - add_empty_file_data_to_import(import_hash) - end - - Rails.logger.info "Import #{import.name} processed" - - import_hash - end - - def add_file_data_to_import(import, import_hash) - sanitized_filename = generate_sanitized_filename(import) - file_path = files_directory.join(sanitized_filename) - - begin - download_and_save_import_file(import, file_path) - add_file_metadata_to_import(import, import_hash, sanitized_filename) - rescue StandardError => e - Rails.logger.error "Failed to download import file #{import.id}: #{e.message}" - import_hash['file_error'] = "Failed to download: #{e.message}" - end - end - - def add_empty_file_data_to_import(import_hash) - import_hash['file_name'] = nil - import_hash['original_filename'] = nil - end - - def generate_sanitized_filename(import) - "import_#{import.id}_#{import.file.blob.filename}".gsub(/[^0-9A-Za-z._-]/, '_') - end - - def download_and_save_import_file(import, file_path) - file_content = Imports::SecureFileDownloader.new(import.file).download_with_verification - File.write(file_path, file_content, mode: 'wb') - end - - def add_file_metadata_to_import(import, import_hash, sanitized_filename) - import_hash['file_name'] = sanitized_filename - import_hash['original_filename'] = import.file.blob.filename.to_s - import_hash['file_size'] = import.file.blob.byte_size - import_hash['content_type'] = import.file.blob.content_type - end - def create_zip_archive(zip_file_path) Zip::File.open(zip_file_path, Zip::File::CREATE) do |zipfile| Dir.glob(export_directory.join('**', '*')).each do |file| @@ -349,86 +231,4 @@ class Users::ExportData Rails.logger.error "Failed to cleanup temporary files: #{e.message}" # Don't re-raise the error as cleanup failure shouldn't break the export end - - def serialized_trips - user.trips.as_json(except: %w[user_id id]) - end - - def serialized_areas - user.areas.as_json(except: %w[user_id id]) - end - - def serialized_stats - user.stats.as_json(except: %w[user_id id]) - end - - def serialized_notifications - user.notifications.as_json(except: %w[user_id id]) - end - - def serialized_points - # Include relationship with country to avoid N+1 queries - user.tracked_points.includes(:country, :import, :visit).find_each(batch_size: 1000).map do |point| - point_hash = point.as_json(except: %w[user_id import_id country_id visit_id id]) - - # Replace import_id with import natural key - if point.import - point_hash['import_reference'] = { - 'name' => point.import.name, - 'source' => point.import.source, - 'created_at' => point.import.created_at.iso8601 - } - else - point_hash['import_reference'] = nil - end - - # Replace country_id with country information - if point.country - point_hash['country_info'] = { - 'name' => point.country.name, - 'iso_a2' => point.country.iso_a2, - 'iso_a3' => point.country.iso_a3 - } - else - point_hash['country_info'] = nil - end - - # Replace visit_id with visit natural key - if point.visit - point_hash['visit_reference'] = { - 'name' => point.visit.name, - 'started_at' => point.visit.started_at&.iso8601, - 'ended_at' => point.visit.ended_at&.iso8601 - } - else - point_hash['visit_reference'] = nil - end - - point_hash - end - end - - def serialized_visits - user.visits.includes(:place).map do |visit| - visit_hash = visit.as_json(except: %w[user_id place_id id]) - - # Replace place_id with place natural key - if visit.place - visit_hash['place_reference'] = { - 'name' => visit.place.name, - 'latitude' => visit.place.lat.to_s, - 'longitude' => visit.place.lon.to_s, - 'source' => visit.place.source - } - else - visit_hash['place_reference'] = nil - end - - visit_hash - end - end - - def serialized_places - user.places.as_json(except: %w[user_id id]) - end end diff --git a/app/services/users/export_data/areas.rb b/app/services/users/export_data/areas.rb new file mode 100644 index 00000000..dd383a65 --- /dev/null +++ b/app/services/users/export_data/areas.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class Users::ExportData::Areas + def initialize(user) + @user = user + end + + def call + user.areas.as_json(except: %w[user_id id]) + end + + private + + attr_reader :user +end diff --git a/app/services/users/export_data/exports.rb b/app/services/users/export_data/exports.rb new file mode 100644 index 00000000..77480983 --- /dev/null +++ b/app/services/users/export_data/exports.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +class Users::ExportData::Exports + def initialize(user, files_directory) + @user = user + @files_directory = files_directory + end + + def call + user.exports.includes(:file_attachment).map do |export| + process_export(export) + end + end + + private + + attr_reader :user, :files_directory + + def process_export(export) + Rails.logger.info "Processing export #{export.name}" + + export_hash = export.as_json(except: %w[user_id id]) + + if export.file.attached? + add_file_data_to_export(export, export_hash) + else + add_empty_file_data_to_export(export_hash) + end + + Rails.logger.info "Export #{export.name} processed" + + export_hash + end + + def add_file_data_to_export(export, export_hash) + sanitized_filename = generate_sanitized_export_filename(export) + file_path = files_directory.join(sanitized_filename) + + begin + download_and_save_export_file(export, file_path) + add_file_metadata_to_export(export, export_hash, sanitized_filename) + rescue StandardError => e + Rails.logger.error "Failed to download export file #{export.id}: #{e.message}" + export_hash['file_error'] = "Failed to download: #{e.message}" + end + end + + def add_empty_file_data_to_export(export_hash) + export_hash['file_name'] = nil + export_hash['original_filename'] = nil + end + + def generate_sanitized_export_filename(export) + "export_#{export.id}_#{export.file.blob.filename}".gsub(/[^0-9A-Za-z._-]/, '_') + end + + def download_and_save_export_file(export, file_path) + file_content = Imports::SecureFileDownloader.new(export.file).download_with_verification + File.write(file_path, file_content, mode: 'wb') + end + + def add_file_metadata_to_export(export, export_hash, sanitized_filename) + export_hash['file_name'] = sanitized_filename + export_hash['original_filename'] = export.file.blob.filename.to_s + export_hash['file_size'] = export.file.blob.byte_size + export_hash['content_type'] = export.file.blob.content_type + end +end diff --git a/app/services/users/export_data/imports.rb b/app/services/users/export_data/imports.rb new file mode 100644 index 00000000..3e2336cb --- /dev/null +++ b/app/services/users/export_data/imports.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +class Users::ExportData::Imports + def initialize(user, files_directory) + @user = user + @files_directory = files_directory + end + + def call + user.imports.includes(:file_attachment).map do |import| + process_import(import) + end + end + + private + + attr_reader :user, :files_directory + + def process_import(import) + Rails.logger.info "Processing import #{import.name}" + + import_hash = import.as_json(except: %w[user_id raw_data id]) + + if import.file.attached? + add_file_data_to_import(import, import_hash) + else + add_empty_file_data_to_import(import_hash) + end + + Rails.logger.info "Import #{import.name} processed" + + import_hash + end + + def add_file_data_to_import(import, import_hash) + sanitized_filename = generate_sanitized_filename(import) + file_path = files_directory.join(sanitized_filename) + + begin + download_and_save_import_file(import, file_path) + add_file_metadata_to_import(import, import_hash, sanitized_filename) + rescue StandardError => e + Rails.logger.error "Failed to download import file #{import.id}: #{e.message}" + import_hash['file_error'] = "Failed to download: #{e.message}" + end + end + + def add_empty_file_data_to_import(import_hash) + import_hash['file_name'] = nil + import_hash['original_filename'] = nil + end + + def generate_sanitized_filename(import) + "import_#{import.id}_#{import.file.blob.filename}".gsub(/[^0-9A-Za-z._-]/, '_') + end + + def download_and_save_import_file(import, file_path) + file_content = Imports::SecureFileDownloader.new(import.file).download_with_verification + File.write(file_path, file_content, mode: 'wb') + end + + def add_file_metadata_to_import(import, import_hash, sanitized_filename) + import_hash['file_name'] = sanitized_filename + import_hash['original_filename'] = import.file.blob.filename.to_s + import_hash['file_size'] = import.file.blob.byte_size + import_hash['content_type'] = import.file.blob.content_type + end +end diff --git a/app/services/users/export_data/notifications.rb b/app/services/users/export_data/notifications.rb new file mode 100644 index 00000000..361f1d37 --- /dev/null +++ b/app/services/users/export_data/notifications.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class Users::ExportData::Notifications + def initialize(user) + @user = user + end + + def call + user.notifications.as_json(except: %w[user_id id]) + end + + private + + attr_reader :user +end diff --git a/app/services/users/export_data/places.rb b/app/services/users/export_data/places.rb new file mode 100644 index 00000000..d7ce61c3 --- /dev/null +++ b/app/services/users/export_data/places.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class Users::ExportData::Places + def initialize(user) + @user = user + end + + def call + user.places.as_json(except: %w[user_id id]) + end + + private + + attr_reader :user +end diff --git a/app/services/users/export_data/points.rb b/app/services/users/export_data/points.rb new file mode 100644 index 00000000..bf923e68 --- /dev/null +++ b/app/services/users/export_data/points.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +class Users::ExportData::Points + def initialize(user) + @user = user + end + + def call + points_data = Point.where(user_id: user.id).order(id: :asc) + + return [] if points_data.empty? + + # Get unique IDs for batch loading + import_ids = points_data.filter_map { |row| row['import_id'] }.uniq + country_ids = points_data.filter_map { |row| row['country_id'] }.uniq + visit_ids = points_data.filter_map { |row| row['visit_id'] }.uniq + + # Load all imports in one query + imports_map = {} + if import_ids.any? + Import.where(id: import_ids).find_each do |import| + imports_map[import.id] = { + 'name' => import.name, + 'source' => import.source, + 'created_at' => import.created_at.iso8601 + } + end + end + + # Load all countries in one query + countries_map = {} + if country_ids.any? + Country.where(id: country_ids).find_each do |country| + countries_map[country.id] = { + 'name' => country.name, + 'iso_a2' => country.iso_a2, + 'iso_a3' => country.iso_a3 + } + end + end + + # Load all visits in one query + visits_map = {} + if visit_ids.any? + Visit.where(id: visit_ids).find_each do |visit| + visits_map[visit.id] = { + 'name' => visit.name, + 'started_at' => visit.started_at&.iso8601, + 'ended_at' => visit.ended_at&.iso8601 + } + end + end + + # Build the final result + points_data.map do |row| + point_hash = row.except('import_id', 'country_id', 'visit_id', 'id').to_h + + # Add relationship references + point_hash['import_reference'] = imports_map[row['import_id']] + point_hash['country_info'] = countries_map[row['country_id']] + point_hash['visit_reference'] = visits_map[row['visit_id']] + + point_hash + end + end + + private + + attr_reader :user +end diff --git a/app/services/users/export_data/stats.rb b/app/services/users/export_data/stats.rb new file mode 100644 index 00000000..a40d2ff3 --- /dev/null +++ b/app/services/users/export_data/stats.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class Users::ExportData::Stats + def initialize(user) + @user = user + end + + def call + user.stats.as_json(except: %w[user_id id]) + end + + private + + attr_reader :user +end diff --git a/app/services/users/export_data/trips.rb b/app/services/users/export_data/trips.rb new file mode 100644 index 00000000..e20bc7c6 --- /dev/null +++ b/app/services/users/export_data/trips.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class Users::ExportData::Trips + def initialize(user) + @user = user + end + + def call + user.trips.as_json(except: %w[user_id id]) + end + + private + + attr_reader :user +end diff --git a/app/services/users/export_data/visits.rb b/app/services/users/export_data/visits.rb new file mode 100644 index 00000000..7d60d22d --- /dev/null +++ b/app/services/users/export_data/visits.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class Users::ExportData::Visits + def initialize(user) + @user = user + end + + def call + user.visits.includes(:place).map do |visit| + visit_hash = visit.as_json(except: %w[user_id place_id id]) + + if visit.place + visit_hash['place_reference'] = { + 'name' => visit.place.name, + 'latitude' => visit.place.lat.to_s, + 'longitude' => visit.place.lon.to_s, + 'source' => visit.place.source + } + else + visit_hash['place_reference'] = nil + end + + visit_hash + end + end + + private + + attr_reader :user +end