From 973a4aae849c5678b76200a7c3ec34d90ee4d7cb Mon Sep 17 00:00:00 2001
From: Eugene Burmakin <iamfrey@gmail.com>
Date: Sun, 7 Dec 2025 14:29:39 +0100
Subject: [PATCH] Change file structure for raw data archival feature

---
 app/models/points/raw_data_archive.rb         |  2 +-
 app/services/points/raw_data/archiver.rb      |  9 +++++----
 spec/jobs/points/raw_data/archive_job_spec.rb | 19 +++++++++++++++++--
 spec/models/points/raw_data_archive_spec.rb   |  4 ++--
 .../services/points/raw_data/archiver_spec.rb |  2 +-
 5 files changed, 26 insertions(+), 10 deletions(-)

diff --git a/app/models/points/raw_data_archive.rb b/app/models/points/raw_data_archive.rb
index 0f7968eb..4657e091 100644
--- a/app/models/points/raw_data_archive.rb
+++ b/app/models/points/raw_data_archive.rb
@@ -28,7 +28,7 @@ module Points
     end
 
     def filename
-      "raw_data_#{user_id}_#{year}_#{format('%02d', month)}_chunk#{format('%03d', chunk_number)}.jsonl.gz"
+      "raw_data_archives/#{user_id}/#{year}/#{format('%02d', month)}/#{format('%03d', chunk_number)}.jsonl.gz"
     end
 
     def size_mb
diff --git a/app/services/points/raw_data/archiver.rb b/app/services/points/raw_data/archiver.rb
index 76a13feb..717275bf 100644
--- a/app/services/points/raw_data/archiver.rb
+++ b/app/services/points/raw_data/archiver.rb
@@ -164,12 +164,13 @@ module Points
         )
 
         # Attach compressed file via ActiveStorage
-        filename = "raw_data_#{user_id}_#{year}_#{format('%02d', month)}_chunk#{format('%03d', chunk_number)}.jsonl.gz"
-
+        # Uses directory structure: raw_data_archives/:user_id/:year/:month/:chunk.jsonl.gz
+        # The key parameter controls the actual storage path
         archive.file.attach(
           io: StringIO.new(compressed_data),
-          filename: filename,
-          content_type: 'application/gzip'
+          filename: "#{format('%03d', chunk_number)}.jsonl.gz",
+          content_type: 'application/gzip',
+          key: "raw_data_archives/#{user_id}/#{year}/#{format('%02d', month)}/#{format('%03d', chunk_number)}.jsonl.gz"
         )
 
         archive
diff --git a/spec/jobs/points/raw_data/archive_job_spec.rb b/spec/jobs/points/raw_data/archive_job_spec.rb
index 26213cb0..70e1eb7b 100644
--- a/spec/jobs/points/raw_data/archive_job_spec.rb
+++ b/spec/jobs/points/raw_data/archive_job_spec.rb
@@ -7,18 +7,25 @@ RSpec.describe Points::RawData::ArchiveJob, type: :job do
     let(:archiver) { instance_double(Points::RawData::Archiver) }
 
     before do
+      # Enable archival for tests
+      allow(ENV).to receive(:[]).and_call_original
+      allow(ENV).to receive(:[]).with('ARCHIVE_RAW_DATA').and_return('true')
+
       allow(Points::RawData::Archiver).to receive(:new).and_return(archiver)
+      allow(archiver).to receive(:call).and_return({ processed: 5, archived: 100, failed: 0 })
     end
 
     it 'calls the archiver service' do
-      expect(archiver).to receive(:call).and_return({ processed: 5, archived: 100, failed: 0 })
+      expect(archiver).to receive(:call)
 
       described_class.perform_now
     end
 
     context 'when archiver raises an error' do
+      let(:error) { StandardError.new('Archive failed') }
+
       before do
-        allow(archiver).to receive(:call).and_raise(StandardError, 'Archive failed')
+        allow(archiver).to receive(:call).and_raise(error)
       end
 
       it 're-raises the error' do
@@ -26,6 +33,14 @@ RSpec.describe Points::RawData::ArchiveJob, type: :job do
           described_class.perform_now
         end.to raise_error(StandardError, 'Archive failed')
       end
+
+      it 'reports the error before re-raising' do
+        expect(ExceptionReporter).to receive(:call).with(error, 'Points raw data archival job failed')
+
+        expect do
+          described_class.perform_now
+        end.to raise_error(StandardError)
+      end
     end
   end
 end
diff --git a/spec/models/points/raw_data_archive_spec.rb b/spec/models/points/raw_data_archive_spec.rb
index 0ba80a03..0ffa54d4 100644
--- a/spec/models/points/raw_data_archive_spec.rb
+++ b/spec/models/points/raw_data_archive_spec.rb
@@ -64,9 +64,9 @@ RSpec.describe Points::RawDataArchive, type: :model do
   end
 
   describe '#filename' do
-    it 'generates correct filename' do
+    it 'generates correct filename with directory structure' do
       archive = build(:points_raw_data_archive, user_id: 123, year: 2024, month: 6, chunk_number: 5)
-      expect(archive.filename).to eq('raw_data_123_2024_06_chunk005.jsonl.gz')
+      expect(archive.filename).to eq('raw_data_archives/123/2024/06/005.jsonl.gz')
     end
   end
 
diff --git a/spec/services/points/raw_data/archiver_spec.rb b/spec/services/points/raw_data/archiver_spec.rb
index c0bc71e6..ae350f06 100644
--- a/spec/services/points/raw_data/archiver_spec.rb
+++ b/spec/services/points/raw_data/archiver_spec.rb
@@ -131,7 +131,7 @@ RSpec.describe Points::RawData::Archiver do
 
       archive = user.raw_data_archives.last
       expect(archive.file).to be_attached
-      expect(archive.file.filename.to_s).to match(/raw_data_.*_chunk001\.jsonl\.gz/)
+      expect(archive.file.key).to match(%r{raw_data_archives/\d+/\d{4}/\d{2}/001\.jsonl\.gz})
     end
   end