Merge pull request #209 from Freika/feature/gpx-export

Export to GPX

.app_version

@@ -1 +1 @@
-0.12.3
+0.13.0

.dockerignore

@@ -1,2 +1,8 @@
 /log
 /tmp
+
+# We need directories for import and export files, but not the files themselves.
+/public/exports/*
+!/public/exports/.keep
+/public/imports/*
+!/public/imports/.keep

CHANGELOG.md

@@ -5,6 +5,42 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [0.13.0] — 2024-09-03
+
+⚠️ BREAKING CHANGES: ⚠️
+
+Default exporting format is now GeoJSON instead of Owntracks-like JSON. This will allow you to use the exported data in other applications that support GeoJSON format. It's also important to highlight, that GeoJSON format does not describe a way to store any time-related data. Dawarich relies on the `timestamp` field in the GeoJSON format to determine the time of the point. The value of the `timestamp` field should be a Unix timestamp in seconds. If you import GeoJSON data that does not have a `timestamp` field, the point will not be imported.
+
+Example of a valid point in GeoJSON format:
+
+```json
+{
+  "type": "Feature",
+  "geometry": {
+    "type": "Point",
+    "coordinates": [13.350110811262352, 52.51450815]
+  },
+  "properties": {
+    "timestamp": 1725310036
+  }
+}
+```
+
+### Added
+
+- GeoJSON format is now available for exporting data.
+- GPX format is now available for exporting data.
+- Importing GeoJSON is now available.
+
+### Changed
+
+- Default exporting format is now GeoJSON instead of Owntracks-like JSON. This will allow you to use the exported data in other applications that support GeoJSON format.
+
+### Fixed
+
+- Fixed a bug where the confirmation alert was shown more than once when deleting a point.
+
+
 ## [0.12.3] — 2024-09-02
 
 ### Added

Gemfile

@@ -9,6 +9,7 @@ gem 'chartkick'
 gem 'data_migrate'
 gem 'devise'
 gem 'geocoder'
+gem 'gpx'
 gem 'httparty'
 gem 'importmap-rails'
 gem 'kaminari'

Gemfile.lock

@@ -137,6 +137,9 @@ GEM
       csv (>= 3.0.0)
     globalid (1.2.1)
       activesupport (>= 6.1)
+    gpx (1.1.1)
+      nokogiri (~> 1.7)
+      rake
     hashdiff (1.1.0)
     httparty (0.22.0)
       csv
@@ -433,6 +436,7 @@ DEPENDENCIES
   ffaker
   foreman
   geocoder
+  gpx
   httparty
   importmap-rails
   kaminari

app/controllers/exports_controller.rb

@@ -12,7 +12,7 @@ class ExportsController < ApplicationController
     export_name = "export_from_#{params[:start_at].to_date}_to_#{params[:end_at].to_date}"
     export = current_user.exports.create(name: export_name, status: :created)
 
-    ExportJob.perform_later(export.id, params[:start_at], params[:end_at])
+    ExportJob.perform_later(export.id, params[:start_at], params[:end_at], file_format: params[:file_format])
 
     redirect_to exports_url, notice: 'Export was successfully initiated. Please wait until it\'s finished.'
   rescue StandardError => e

app/javascript/controllers/maps_controller.js

@@ -145,8 +145,12 @@ export default class extends Controller {
     `;
   }
 
+  removeEventListeners() {
+    document.removeEventListener('click', this.handleDeleteClick);
+  }
+
   addEventListeners() {
-    document.addEventListener('click', (event) => {
+    this.handleDeleteClick = (event) => {
       if (event.target && event.target.classList.contains('delete-point')) {
         event.preventDefault();
         const pointId = event.target.getAttribute('data-id');
@@ -155,7 +159,11 @@ export default class extends Controller {
           this.deletePoint(pointId, this.apiKey);
         }
       }
-    });
+    };
+
+    // Ensure only one listener is attached by removing any existing ones first
+    this.removeEventListeners();
+    document.addEventListener('click', this.handleDeleteClick);
   }
 
   deletePoint(id, apiKey) {

app/jobs/export_job.rb

@@ -3,9 +3,9 @@
 class ExportJob < ApplicationJob
   queue_as :exports
 
-  def perform(export_id, start_at, end_at)
+  def perform(export_id, start_at, end_at, file_format: :json)
     export = Export.find(export_id)
 
-    Exports::Create.new(export:, start_at:, end_at:).call
+    Exports::Create.new(export:, start_at:, end_at:, file_format:).call
   end
 end

app/jobs/import_job.rb

@@ -32,6 +32,7 @@ class ImportJob < ApplicationJob
     when 'owntracks'                then OwnTracks::ExportParser
     when 'gpx'                      then Gpx::TrackParser
     when 'immich_api'               then Immich::ImportParser
+    when 'geojson'                  then Geojson::ImportParser
     end
   end
 

app/models/import.rb

@@ -10,6 +10,6 @@ class Import < ApplicationRecord
 
   enum source: {
     google_semantic_history: 0, owntracks: 1, google_records: 2,
-    google_phone_takeout: 3, gpx: 4, immich_api: 5
+    google_phone_takeout: 3, gpx: 4, immich_api: 5, geojson: 6
   }
 end

app/serializers/point_serializer.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class PointSerializer
+  EXCLUDED_ATTRIBUTES = %w[created_at updated_at visit_id id import_id user_id].freeze
+
+  def initialize(point)
+    @point = point
+  end
+
+  def call
+    point.attributes.except(*EXCLUDED_ATTRIBUTES)
+  end
+
+  private
+
+  attr_reader :point
+end

app/serializers/points/geojson_serializer.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+class Points::GeojsonSerializer
+  def initialize(points)
+    @points = points
+  end
+
+  # rubocop:disable Metrics/MethodLength
+  def call
+    {
+      type: 'FeatureCollection',
+      features: points.map do |point|
+        {
+          type: 'Feature',
+          geometry: {
+            type: 'Point',
+            coordinates: [point.longitude, point.latitude]
+          },
+          properties: PointSerializer.new(point).call
+        }
+      end
+    }.to_json
+  end
+  # rubocop:enable Metrics/MethodLength
+
+  private
+
+  attr_reader :points
+end

app/serializers/points/gpx_serializer.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class Points::GpxSerializer
+  def initialize(points)
+    @points = points
+  end
+
+  def call
+    geojson_data = Points::GeojsonSerializer.new(points).call
+
+    GPX::GeoJSON.convert_to_gpx(geojson_data:)
+  end
+
+  private
+
+  attr_reader :points
+end

app/services/exports/create.rb

@@ -1,33 +1,27 @@
 # frozen_string_literal: true
 
 class Exports::Create
-  def initialize(export:, start_at:, end_at:)
-    @export = export
-    @user = export.user
-    @start_at = start_at.to_datetime
-    @end_at = end_at.to_datetime
+  def initialize(export:, start_at:, end_at:, file_format: :json)
+    @export       = export
+    @user         = export.user
+    @start_at     = start_at.to_datetime
+    @end_at       = end_at.to_datetime
+    @file_format  = file_format
   end
 
   def call
     export.update!(status: :processing)
 
-    Rails.logger.debug "====Exporting data for #{user.email} from #{start_at} to #{end_at}"
-
     points = time_framed_points
 
-    Rails.logger.debug "====Exporting #{points.size} points"
+    data = points_data(points)
 
-    data      = ::ExportSerializer.new(points, user.email).call
-    file_path = Rails.root.join('public', 'exports', "#{export.name}.json")
+    create_export_file(data)
 
-    File.open(file_path, 'w') { |file| file.write(data) }
-
-    export.update!(status: :completed, url: "exports/#{export.name}.json")
+    export.update!(status: :completed, url: "exports/#{export.name}.#{file_format}")
 
     create_export_finished_notification
   rescue StandardError => e
-    Rails.logger.error("====Export failed to create: #{e.message}")
-
     create_failed_export_notification(e)
 
     export.update!(status: :failed)
@@ -35,7 +29,7 @@ class Exports::Create
 
   private
 
-  attr_reader :user, :export, :start_at, :end_at
+  attr_reader :user, :export, :start_at, :end_at, :file_format
 
   def time_framed_points
     user
@@ -60,4 +54,26 @@ class Exports::Create
       content: "Export \"#{export.name}\" failed: #{error.message}, stacktrace: #{error.backtrace.join("\n")}"
     ).call
   end
+
+  def points_data(points)
+    case file_format.to_sym
+    when :json then process_geojson_export(points)
+    when :gpx then process_gpx_export(points)
+    else raise ArgumentError, "Unsupported file format: #{file_format}"
+    end
+  end
+
+  def process_geojson_export(points)
+    Points::GeojsonSerializer.new(points).call
+  end
+
+  def process_gpx_export(points)
+    Points::GpxSerializer.new(points).call
+  end
+
+  def create_export_file(data)
+    file_path = Rails.root.join('public', 'exports', "#{export.name}.#{file_format}")
+
+    File.open(file_path, 'w') { |file| file.write(data) }
+  end
 end

app/services/geojson/import_parser.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+class Geojson::ImportParser
+  attr_reader :import, :json, :user_id
+
+  def initialize(import, user_id)
+    @import  = import
+    @json    = import.raw_data
+    @user_id = user_id
+  end
+
+  def call
+    data = Geojson::Params.new(json).call
+
+    data.each do |point|
+      next if point_exists?(point, user_id)
+
+      Point.create!(point.merge(user_id:, import_id: import.id))
+    end
+  end
+
+  private
+
+  def point_exists?(params, user_id)
+    Point.exists?(
+      latitude:  params[:latitude],
+      longitude: params[:longitude],
+      timestamp: params[:timestamp],
+      user_id:
+    )
+  end
+end

app/services/geojson/params.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+class Geojson::Params
+  attr_reader :json
+
+  def initialize(json)
+    @json = json.with_indifferent_access
+  end
+
+  def call
+    json['features'].map do |point|
+      next if point[:geometry].nil? || point.dig(:properties, :timestamp).nil?
+
+      {
+        latitude:           point[:geometry][:coordinates][1],
+        longitude:          point[:geometry][:coordinates][0],
+        battery_status:     point[:properties][:battery_state],
+        battery:            battery_level(point[:properties][:battery_level]),
+        timestamp:          Time.zone.at(point[:properties][:timestamp]),
+        altitude:           point[:properties][:altitude],
+        velocity:           point[:properties][:speed],
+        tracker_id:         point[:properties][:device_id],
+        ssid:               point[:properties][:wifi],
+        accuracy:           point[:properties][:horizontal_accuracy],
+        vertical_accuracy:  point[:properties][:vertical_accuracy],
+        raw_data:           point
+      }
+    end.compact
+  end
+
+  private
+
+  def battery_level(level)
+    value = (level.to_f * 100).to_i
+
+    value.positive? ? value : nil
+  end
+end

app/views/imports/_form.html.erb

@@ -22,6 +22,15 @@
           <p class="text-sm mt-2">A JSON file you exported by pressing Download button in top right corner of OwnTracks web interface</p>
         </div>
       </div>
+      <div class="card bordered shadow-lg p-3 hover:shadow-blue-500/50">
+        <div class="form-control">
+          <label class="label cursor-pointer space-x-3">
+            <%= form.radio_button :source, :geojson, class: "radio radio-primary" %>
+            <span class="label-text">GeoJSON</span>
+          </label>
+          <p class="text-sm mt-2">A valid GeoJSON file. For example, a file, exported from a Dawarich instance</p>
+        </div>
+      </div>
       <div class="card bordered shadow-lg p-3 hover:shadow-blue-500/50">
         <div class="form-control">
           <label class="label cursor-pointer space-x-3">

app/views/points/index.html.erb

@@ -3,26 +3,31 @@
 <div class="w-full">
   <%= form_with url: points_path, method: :get do |f| %>
     <div class="flex flex-col md:flex-row md:space-x-4 md:items-end">
-      <div class="w-full md:w-2/6">
+      <div class="w-full md:w-3/12">
         <div class="flex flex-col space-y-2">
           <%= f.label :start_at, class: "text-sm font-semibold" %>
           <%= f.datetime_local_field :start_at, class: "rounded-md w-full", value: @start_at %>
         </div>
       </div>
-      <div class="w-full md:w-2/6">
+      <div class="w-full md:w-3/12">
         <div class="flex flex-col space-y-2">
           <%= f.label :end_at, class: "text-sm font-semibold" %>
           <%= f.datetime_local_field :end_at, class: "rounded-md w-full", value: @end_at %>
         </div>
       </div>
-      <div class="w-full md:w-2/6">
+      <div class="w-full md:w-1/12">
         <div class="flex flex-col space-y-2">
           <%= f.submit "Search", class: "px-4 py-2 bg-blue-500 text-white rounded-md" %>
         </div>
       </div>
-      <div class="w-full md:w-2/6">
+      <div class="w-full md:w-2/12">
         <div class="flex flex-col space-y-2 text-center">
-          <%= link_to 'Export points', exports_path(start_at: @start_at, end_at: @end_at), data: { confirm: "Are you sure?", turbo_confirm: "Are you sure? This will start background process of exporting points withing timeframe, selected between #{@start_at} and #{@end_at}", turbo_method: :post }, class: "px-4 py-2 bg-green-500 text-white rounded-md" %>
+          <%= link_to 'Export as GeoJSON', exports_path(start_at: @start_at, end_at: @end_at, file_format: :json), data: { confirm: "Are you sure?", turbo_confirm: "Are you sure? This will start background process of exporting points withing timeframe, selected between #{@start_at} and #{@end_at}", turbo_method: :post }, class: "px-4 py-2 bg-green-500 text-white rounded-md join-item" %>
+        </div>
+      </div>
+      <div class="w-full md:w-2/12">
+        <div class="flex flex-col space-y-2 text-center">
+          <%= link_to 'Export as GPX', exports_path(start_at: @start_at, end_at: @end_at, file_format: :gpx), data: { confirm: "Are you sure?", turbo_confirm: "Are you sure? This will start background process of exporting points withing timeframe, selected between #{@start_at} and #{@end_at}", turbo_method: :post }, class: "px-4 py-2 bg-green-500 text-white rounded-md join-item" %>
         </div>
       </div>
     </div>

spec/jobs/export_job_spec.rb

@@ -8,7 +8,7 @@ RSpec.describe ExportJob, type: :job do
   let(:end_at) { Time.zone.now }
 
   it 'calls the Exports::Create service class' do
-    expect(Exports::Create).to receive(:new).with(export:, start_at:, end_at:).and_call_original
+    expect(Exports::Create).to receive(:new).with(export:, start_at:, end_at:, file_format: :json).and_call_original
 
     described_class.perform_now(export.id, start_at, end_at)
   end

spec/models/import_spec.rb

@@ -16,7 +16,8 @@ RSpec.describe Import, type: :model do
         google_records: 2,
         google_phone_takeout: 3,
         gpx: 4,
-        immich_api: 5
+        immich_api: 5,
+        geojson: 6
       )
     end
   end

spec/serializers/point_serializer_spec.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe PointSerializer do
+  describe '#call' do
+    subject(:serializer) { described_class.new(point).call }
+
+    let(:point) { create(:point) }
+    let(:expected_json) do
+      point.attributes.except(*PointSerializer::EXCLUDED_ATTRIBUTES)
+    end
+
+    it 'returns JSON' do
+      expect(serializer.to_json).to eq(expected_json.to_json)
+    end
+
+    it 'does not include excluded attributes' do
+      expect(serializer).not_to include(*PointSerializer::EXCLUDED_ATTRIBUTES)
+    end
+  end
+end

spec/serializers/points/geojson_serializer_spec.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe Points::GeojsonSerializer do
+  describe '#call' do
+    subject(:serializer) { described_class.new(points).call }
+
+    let(:points) { create_list(:point, 3) }
+    let(:expected_json) do
+      {
+        type: 'FeatureCollection',
+        features: points.map do |point|
+          {
+            type: 'Feature',
+            geometry: {
+              type: 'Point',
+              coordinates: [point.longitude, point.latitude]
+            },
+            properties: PointSerializer.new(point).call
+          }
+        end
+      }
+    end
+
+    it 'returns JSON' do
+      expect(serializer).to eq(expected_json.to_json)
+    end
+  end
+end

spec/serializers/points/gpx_serializer.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe Points::GpxSerializer do
+  describe '#call' do
+    subject(:serializer) { described_class.new(points).call }
+
+    let(:points) { create_list(:point, 3) }
+    let(:geojson_data) { Points::GeojsonSerializer.new(points).call }
+    let(:gpx) { GPX::GeoJSON.convert_to_gpx(geojson_data:) }
+
+    it 'returns JSON' do
+      expect(serializer).to be_a(GPX::GPXFile)
+    end
+  end
+end

spec/services/exports/create_spec.rb

@@ -4,20 +4,21 @@ require 'rails_helper'
 
 RSpec.describe Exports::Create do
   describe '#call' do
-    subject(:create_export) { described_class.new(export:, start_at:, end_at:).call }
+    subject(:create_export) { described_class.new(export:, start_at:, end_at:, file_format:).call }
 
-    let(:user) { create(:user) }
-    let(:start_at) { DateTime.new(2021, 1, 1).to_s }
-    let(:end_at) { DateTime.new(2021, 1, 2).to_s }
-    let(:export_name) { "#{start_at.to_date}_#{end_at.to_date}" }
-    let(:export) { create(:export, user:, name: export_name, status: :created) }
-    let(:export_content) { ExportSerializer.new(points, user.email).call }
-    let!(:points) { create_list(:point, 10, user:, timestamp: start_at.to_datetime.to_i) }
+    let(:file_format)     { :json }
+    let(:user)            { create(:user) }
+    let(:start_at)        { DateTime.new(2021, 1, 1).to_s }
+    let(:end_at)          { DateTime.new(2021, 1, 2).to_s }
+    let(:export_name)     { "#{start_at.to_date}_#{end_at.to_date}" }
+    let(:export)          { create(:export, user:, name: export_name, status: :created) }
+    let(:export_content)  { Points::GeojsonSerializer.new(points).call }
+    let!(:points)         { create_list(:point, 10, user:, timestamp: start_at.to_datetime.to_i) }
 
     it 'writes the data to a file' do
       create_export
 
-      file_path = Rails.root.join('public', 'exports', "#{export_name}.json")
+      file_path = Rails.root.join('spec/fixtures/files/geojson/export_same_points.json')
 
       expect(File.read(file_path)).to eq(export_content)
     end
@@ -49,12 +50,6 @@ RSpec.describe Exports::Create do
         expect(export.reload.failed?).to be_truthy
       end
 
-      it 'logs the error' do
-        expect(Rails.logger).to receive(:error).with('====Export failed to create: StandardError')
-
-        create_export
-      end
-
       it 'creates a notification' do
         expect { create_export }.to change { Notification.count }.by(1)
       end

spec/services/geojson/import_parser_spec.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe Geojson::ImportParser do
+  describe '#call' do
+    subject(:service) { described_class.new(import, user.id).call }
+
+    let(:user) { create(:user) }
+
+    let(:user) { create(:user) }
+
+    context 'when file content is an object' do
+      let(:file_path) { Rails.root.join('spec/fixtures/files/geojson/export.json') }
+      let(:raw_data) { JSON.parse(File.read(file_path)) }
+      let(:import) { create(:import, user:, name: 'geojson.json', raw_data:) }
+
+      it 'creates new points' do
+        expect { service }.to change { Point.count }.by(10)
+      end
+    end
+  end
+end