diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fdabb84..f951e4a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,12 +4,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -# Unreleased +# 0.25.4 - 2025-04-02 -## TODO: +In this release we're changing the way import files are being stored. Previously, they were being stored in the `raw_data` column of the `imports` table. Now, they are being attached to the import record. All new imports will be using the new storage, to migrate existing imports, you can use the `rake imports:migrate_to_new_storage` task. Run it in the container shell. -- Migrate existing imports from `raw_data` to the new file storage. -- Stream import files for parsing instead of downloading them. +This is an optional task, that will not affect your points or other data. +Big imports might take a while to migrate, so be patient. + +If your hardware doesn't have enough memory to migrate the imports, you can delete your imports and re-import them. ## Changed @@ -18,6 +20,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Export files are now being attached to the export record instead of being stored in the file system. - Export files can now be stored in S3-compatible storage. + # 0.25.3 - 2025-03-22 ## Fixed diff --git a/app/models/import.rb b/app/models/import.rb index 1e490e56..3965c219 100644 --- a/app/models/import.rb +++ b/app/models/import.rb @@ -30,4 +30,12 @@ class Import < ApplicationRecord [time.year, time.month] end.uniq end + + def migrate_to_new_storage + return if file.attached? + + raw_file = File.new(raw_data) + + file.attach(io: raw_file, filename: name, content_type: 'application/json') + end end diff --git a/config/credentials/production.key b/config/credentials/production.key new file mode 100644 index 00000000..4c969005 --- /dev/null +++ b/config/credentials/production.key @@ -0,0 +1 @@ +41976cfff86107bc1bb52cec7d8107b0 \ No newline at end of file diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 72f7d1ba..14f30548 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -1,8 +1,7 @@ # frozen_string_literal: true -# Usage: rake import:big_file['/path/to/file.json','user@email.com'] - namespace :import do + # Usage: rake import:big_file['/path/to/file.json','user@email.com'] desc 'Accepts a file path and user email and imports the data into the database' task :big_file, %i[file_path user_email] => :environment do |_, args| diff --git a/lib/tasks/imports.rake b/lib/tasks/imports.rake new file mode 100644 index 00000000..ffd49fbe --- /dev/null +++ b/lib/tasks/imports.rake @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +namespace :imports do + desc 'Migrate existing imports from `raw_data` to the new file storage' + + task migrate_to_new_storage: :environment do + Import.find_each do |import| + import.migrate_to_new_storage + rescue StandardError => e + puts "Error migrating import #{import.id}: #{e.message}" + end + end +end diff --git a/spec/models/import_spec.rb b/spec/models/import_spec.rb index 8b682409..07844e33 100644 --- a/spec/models/import_spec.rb +++ b/spec/models/import_spec.rb @@ -36,4 +36,23 @@ RSpec.describe Import, type: :model do expect(import.years_and_months_tracked).to eq([[2024, 11]]) end end + + describe '#migrate_to_new_storage' do + let(:raw_data) { Rails.root.join('spec/fixtures/files/geojson/export.json') } + let(:import) { create(:import, source: 'geojson', raw_data:) } + + it 'attaches the file to the import' do + import.migrate_to_new_storage + + expect(import.file.attached?).to be_truthy + end + + context 'when file is attached' do + it 'is a importable file' do + import.migrate_to_new_storage + + expect { import.process! }.to change(Point, :count).by(10) + end + end + end end