diff --git a/CHANGELOG.md b/CHANGELOG.md index 35d088d0..68c1849a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## TODO: - Migrate existing imports from `raw_data` to the new file storage. -- Delete import files when import is deleted. - Stream import files for parsing instead of downloading them. +- Add randomized name to the import files before attaching them to the import record. ## Changed diff --git a/Gemfile b/Gemfile index 564515c4..ed3e9e9d 100644 --- a/Gemfile +++ b/Gemfile @@ -5,6 +5,10 @@ git_source(:github) { |repo| "https://github.com/#{repo}.git" } ruby File.read('.ruby-version').strip +# https://meta.discourse.org/t/cant-rebuild-due-to-aws-sdk-gem-bump-and-new-aws-data-integrity-protections/354217/40 +gem 'aws-sdk-s3', '~> 1.177.0', require: false +gem 'aws-sdk-core', '~> 3.215.1', require: false +gem 'aws-sdk-kms', '~> 1.96.0', require: false gem 'bootsnap', require: false gem 'chartkick' gem 'data_migrate' diff --git a/Gemfile.lock b/Gemfile.lock index adff6149..35bfd90a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -79,6 +79,22 @@ GEM public_suffix (>= 2.0.2, < 7.0) ast (2.4.2) attr_extras (7.1.0) + aws-eventstream (1.3.2) + aws-partitions (1.1072.0) + aws-sdk-core (3.215.1) + aws-eventstream (~> 1, >= 1.3.0) + aws-partitions (~> 1, >= 1.992.0) + aws-sigv4 (~> 1.9) + jmespath (~> 1, >= 1.6.1) + aws-sdk-kms (1.96.0) + aws-sdk-core (~> 3, >= 3.210.0) + aws-sigv4 (~> 1.5) + aws-sdk-s3 (1.177.0) + aws-sdk-core (~> 3, >= 3.210.0) + aws-sdk-kms (~> 1) + aws-sigv4 (~> 1.5) + aws-sigv4 (1.11.0) + aws-eventstream (~> 1, >= 1.0.2) base64 (0.2.0) bcrypt (3.1.20) benchmark (0.4.0) @@ -91,7 +107,6 @@ GEM coderay (1.1.3) concurrent-ruby (1.3.5) connection_pool (2.5.0) - content_disposition (1.0.0) crack (1.0.0) bigdecimal rexml @@ -121,8 +136,6 @@ GEM dotenv-rails (3.1.7) dotenv (= 3.1.7) railties (>= 6.1) - down (5.4.2) - addressable (~> 2.8) drb (2.2.1) erubi (1.13.1) et-orbi (1.2.11) @@ -164,6 +177,7 @@ GEM pp (>= 0.6.0) rdoc (>= 4.0.0) reline (>= 0.4.2) + jmespath (1.6.2) json (2.10.1) json-schema (5.0.1) addressable (~> 2.8) @@ -371,9 +385,6 @@ GEM securerandom (0.4.1) shoulda-matchers (6.4.0) activesupport (>= 5.2.0) - shrine (3.6.0) - content_disposition (~> 1.0) - down (~> 5.1) sidekiq (7.3.9) base64 connection_pool (>= 2.3.0) @@ -453,6 +464,9 @@ PLATFORMS DEPENDENCIES activerecord-postgis-adapter + aws-sdk-core (~> 3.215.1) + aws-sdk-kms (~> 1.96.0) + aws-sdk-s3 (~> 1.177.0) bootsnap chartkick data_migrate @@ -488,7 +502,6 @@ DEPENDENCIES rswag-ui rubocop-rails shoulda-matchers - shrine (~> 3.6) sidekiq sidekiq-cron sidekiq-limit_fetch diff --git a/app/models/import.rb b/app/models/import.rb index 89a2387f..1e490e56 100644 --- a/app/models/import.rb +++ b/app/models/import.rb @@ -9,6 +9,7 @@ class Import < ApplicationRecord has_one_attached :file after_commit -> { Import::ProcessJob.perform_later(id) }, on: :create + after_commit -> { file.purge }, on: :destroy enum :source, { google_semantic_history: 0, owntracks: 1, google_records: 2, diff --git a/app/services/google_maps/semantic_history_parser.rb b/app/services/google_maps/semantic_history_parser.rb index 77984c09..e4c2f046 100644 --- a/app/services/google_maps/semantic_history_parser.rb +++ b/app/services/google_maps/semantic_history_parser.rb @@ -63,9 +63,13 @@ class GoogleMaps::SemanticHistoryParser end def parse_json - import.raw_data['timelineObjects'].flat_map do |timeline_object| - parse_timeline_object(timeline_object) - end.compact + import.file.download do |f| + json = Oj.load(f) + + json['timelineObjects'].flat_map do |timeline_object| + parse_timeline_object(timeline_object) + end.compact + end end def parse_timeline_object(timeline_object) diff --git a/config/environments/development.rb b/config/environments/development.rb index 29b9a038..3edfc64e 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -98,4 +98,6 @@ Rails.application.configure do config.logger = Logger.new($stdout) config.lograge.enabled = true config.lograge.formatter = Lograge::Formatters::Json.new + + config.active_storage.service = ENV['SELF_HOSTED'] == 'true' ? :local : :s3 end diff --git a/config/initializers/aws.rb b/config/initializers/aws.rb new file mode 100644 index 00000000..8be964d9 --- /dev/null +++ b/config/initializers/aws.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require 'aws-sdk-core' + +Aws.config.update( + { + region: ENV['AWS_REGION'], + endpoint: ENV['AWS_ENDPOINT'], + credentials: Aws::Credentials.new(ENV['AWS_ACCESS_KEY_ID'], ENV['AWS_SECRET_ACCESS_KEY']) + } +) diff --git a/config/storage.yml b/config/storage.yml index 4942ab66..02112314 100644 --- a/config/storage.yml +++ b/config/storage.yml @@ -6,13 +6,12 @@ local: service: Disk root: <%= Rails.root.join("storage") %> -# Use bin/rails credentials:edit to set the AWS secrets (as aws:access_key_id|secret_access_key) -# amazon: -# service: S3 -# access_key_id: <%= Rails.application.credentials.dig(:aws, :access_key_id) %> -# secret_access_key: <%= Rails.application.credentials.dig(:aws, :secret_access_key) %> -# region: us-east-1 -# bucket: your_own_bucket-<%= Rails.env %> +s3: + service: S3 + access_key_id: <%= ENV.fetch("AWS_ACCESS_KEY_ID") %> + secret_access_key: <%= ENV.fetch("AWS_SECRET_ACCESS_KEY") %> + region: <%= ENV.fetch("AWS_REGION") %> + bucket: <%= ENV.fetch("AWS_BUCKET") %> # Remember not to checkin your GCS keyfile to a repository # google: