Merge pull request #1075 from Freika/dev

0.25.5

.app_version

@@ -1 +1 @@
-0.25.4
+0.25.5

.devcontainer/docker-compose.yml

@@ -32,7 +32,6 @@ services:
       PROMETHEUS_EXPORTER_ENABLED: false
       PROMETHEUS_EXPORTER_HOST: 0.0.0.0
       PROMETHEUS_EXPORTER_PORT: 9394
-      ENABLE_TELEMETRY: false # More on telemetry: https://dawarich.app/docs/tutorials/telemetry
   dawarich_redis:
     image: redis:7.4-alpine
     container_name: dawarich_redis

.gitignore

@@ -65,3 +65,8 @@
 .dotnet/
 .cursorrules
 .cursormemory.md
+
+/config/credentials/production.key
+/config/credentials/production.yml.enc
+
+Makefile

CHANGELOG.md

@@ -5,6 +5,37 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
 
+# 0.25.5 - 2025-04-18
+
+This release introduces a new way to send transactional emails using SMTP. Example may include password reset, email confirmation, etc.
+
+To enable SMTP mailing, you need to set the following environment variables:
+
+- `SMTP_SERVER` - SMTP server address.
+- `SMTP_PORT` - SMTP server port.
+- `SMTP_DOMAIN` - SMTP server domain.
+- `SMTP_USERNAME` - SMTP server username.
+- `SMTP_PASSWORD` - SMTP server password.
+- `SMTP_FROM` - Email address to send emails from.
+
+This is optional feature and is not required for the app to work.
+
+## Removed
+
+- Optional telemetry was removed from the app.
+- Sidekiq Web UI is now protected by basic auth in non-self-hosted mode.
+
+## Changed
+
+- `rake points:migrate_to_lonlat` task now also tries to extract latitude and longitude from `raw_data` column before using `longitude` and `latitude` columns to fill `lonlat` column.
+- Docker entrypoints are now using `DATABASE_NAME` environment variable to check if Postgres is existing/available.
+
+## Added
+
+- You can now provide SMTP settings in ENV vars to send emails.
+- You can now edit imports. #1044 #623
+
+
 # 0.25.4 - 2025-04-02
 
 ⚠️ This release includes a breaking change. ⚠️

Gemfile

@@ -27,6 +27,7 @@ gem 'activerecord-postgis-adapter'
 gem 'puma'
 gem 'pundit'
 gem 'rails', '~> 8.0'
+gem 'rexml'
 gem 'rgeo'
 gem 'rgeo-activerecord'
 gem 'rswag-api'

Gemfile.lock

@@ -77,7 +77,7 @@ GEM
       uri (>= 0.13.1)
     addressable (2.8.7)
       public_suffix (>= 2.0.2, < 7.0)
-    ast (2.4.2)
+    ast (2.4.3)
     attr_extras (7.1.0)
     aws-eventstream (1.3.2)
     aws-partitions (1.1072.0)
@@ -105,7 +105,7 @@ GEM
       racc
     builder (3.3.0)
     byebug (11.1.3)
-    chartkick (5.1.3)
+    chartkick (5.1.4)
     coderay (1.1.3)
     concurrent-ruby (1.3.5)
     connection_pool (2.5.0)
@@ -180,7 +180,7 @@ GEM
       rdoc (>= 4.0.0)
       reline (>= 0.4.2)
     jmespath (1.6.2)
-    json (2.10.1)
+    json (2.10.2)
     json-schema (5.0.1)
       addressable (~> 2.8)
     jwt (2.10.1)
@@ -199,7 +199,7 @@ GEM
     kaminari-core (1.2.2)
     language_server-protocol (3.17.0.4)
     lint_roller (1.1.0)
-    logger (1.6.6)
+    logger (1.7.0)
     lograge (0.14.0)
       actionpack (>= 4)
       activesupport (>= 4)
@@ -217,7 +217,7 @@ GEM
     method_source (1.1.0)
     mini_mime (1.1.5)
     mini_portile2 (2.8.8)
-    minitest (5.25.4)
+    minitest (5.25.5)
     msgpack (1.7.3)
     multi_xml (0.7.1)
       bigdecimal (~> 3.1)
@@ -251,7 +251,7 @@ GEM
     orm_adapter (0.5.0)
     ostruct (0.6.1)
     parallel (1.26.3)
-    parser (3.3.7.1)
+    parser (3.3.7.4)
       ast (~> 2.4.1)
       racc
     patience_diff (1.2.0)
@@ -260,6 +260,7 @@ GEM
     pp (0.6.2)
       prettyprint
     prettyprint (0.2.0)
+    prism (1.4.0)
     prometheus_exporter (2.2.0)
       webrick
     pry (0.14.2)
@@ -276,11 +277,11 @@ GEM
     public_suffix (6.0.1)
     puma (6.6.0)
       nio4r (~> 2.0)
-    pundit (2.4.0)
+    pundit (2.5.0)
       activesupport (>= 3.0.0)
     raabro (1.4.0)
     racc (1.8.1)
-    rack (3.1.10)
+    rack (3.1.12)
     rack-session (2.1.0)
       base64 (>= 0.1.0)
       rack (>= 3.0.0)
@@ -321,9 +322,9 @@ GEM
     rake (13.2.1)
     rdoc (6.12.0)
       psych (>= 4.0.0)
-    redis (5.3.0)
+    redis (5.4.0)
       redis-client (>= 0.22.0)
-    redis-client (0.23.2)
+    redis-client (0.24.0)
       connection_pool
     regexp_parser (2.10.0)
     reline (0.6.0)
@@ -366,7 +367,7 @@ GEM
     rswag-ui (2.16.0)
       actionpack (>= 5.2, < 8.1)
       railties (>= 5.2, < 8.1)
-    rubocop (1.72.1)
+    rubocop (1.75.2)
       json (~> 2.3)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.1.0)
@@ -374,16 +375,17 @@ GEM
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 2.9.3, < 3.0)
-      rubocop-ast (>= 1.38.0, < 2.0)
+      rubocop-ast (>= 1.44.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 4.0)
-    rubocop-ast (1.38.0)
-      parser (>= 3.3.1.0)
-    rubocop-rails (2.30.1)
+    rubocop-ast (1.44.0)
+      parser (>= 3.3.7.2)
+      prism (~> 1.4)
+    rubocop-rails (2.31.0)
       activesupport (>= 4.2.0)
       lint_roller (~> 1.1)
       rack (>= 1.1)
-      rubocop (>= 1.72.1, < 2.0)
+      rubocop (>= 1.75.0, < 2.0)
       rubocop-ast (>= 1.38.0, < 2.0)
     ruby-progressbar (1.13.0)
     securerandom (0.4.1)
@@ -450,7 +452,7 @@ GEM
     unicode-display_width (3.1.4)
       unicode-emoji (~> 4.0, >= 4.0.4)
     unicode-emoji (4.0.4)
-    uri (1.0.2)
+    uri (1.0.3)
     useragent (0.16.11)
     warden (1.2.9)
       rack (>= 2.0.9)
@@ -506,6 +508,7 @@ DEPENDENCIES
   pundit
   rails (~> 8.0)
   redis
+  rexml
   rgeo
   rgeo-activerecord
   rspec-rails

Makefile

@@ -1,6 +0,0 @@
-build_and_push:
-	git tag -a "$(version)" -f -m "$(version)"
-	docker build . -t dawarich:$(version) --platform=linux/amd64
-	docker tag dawarich:$(version) registry.chibi.rodeo/dawarich:$(version)
-	docker tag registry.chibi.rodeo/dawarich:$(version) registry.chibi.rodeo/dawarich:latest
-	docker push registry.chibi.rodeo/dawarich:$(version)

app/controllers/home_controller.rb

@@ -2,6 +2,8 @@
 
 class HomeController < ApplicationController
   def index
+    # redirect_to 'https://dawarich.app', allow_other_host: true and return unless SELF_HOSTED
+
     redirect_to map_url if current_user
 
     @points = current_user.tracked_points.without_raw_data if current_user

app/controllers/imports_controller.rb

@@ -5,7 +5,7 @@ class ImportsController < ApplicationController
 
   before_action :authenticate_user!
   before_action :authenticate_active_user!, only: %i[new create]
-  before_action :set_import, only: %i[show destroy]
+  before_action :set_import, only: %i[show edit update destroy]
 
   def index
     @imports =
@@ -18,10 +18,18 @@ class ImportsController < ApplicationController
 
   def show; end
 
+  def edit; end
+
   def new
     @import = Import.new
   end
 
+  def update
+    @import.update(import_params)
+
+    redirect_to imports_url, notice: 'Import was successfully updated.', status: :see_other
+  end
+
   def create
     files = import_params[:files].reject(&:blank?)
 

app/jobs/telemetry_sending_job.rb

@@ -1,14 +0,0 @@
-# frozen_string_literal: true
-
-class TelemetrySendingJob < ApplicationJob
-  queue_as :default
-
-  def perform
-    return unless ENV['ENABLE_TELEMETRY'] == 'true'
-
-    data = Telemetry::Gather.new.call
-    Rails.logger.info("Telemetry data: #{data}")
-
-    Telemetry::Send.new(data).call
-  end
-end

app/mailers/application_mailer.rb

@@ -1,4 +1,6 @@
+# frozen_string_literal: true
+
 class ApplicationMailer < ActionMailer::Base
-  default from: "from@example.com"
-  layout "mailer"
+  default from: ENV['SMTP_FROM']
+  layout 'mailer'
 end

app/models/user.rb

@@ -101,7 +101,7 @@ class User < ApplicationRecord
   end
 
   def can_subscribe?
-    active_until&.past? && !DawarichSettings.self_hosted?
+    (active_until.nil? || active_until&.past?) && !DawarichSettings.self_hosted?
   end
 
   def generate_subscription_token

app/services/gpx/track_importer.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'rexml/document'
+
 class Gpx::TrackImporter
   include Imports::Broadcaster
 

app/services/photos/import_parser.rb

@@ -3,16 +3,19 @@
 class Photos::ImportParser
   include Imports::Broadcaster
   include PointValidation
-  attr_reader :import, :json, :user_id
+  attr_reader :import, :user_id
 
   def initialize(import, user_id)
     @import = import
-    @json = import.raw_data
     @user_id = user_id
   end
 
   def call
-    json.each.with_index(1) { |point, index| create_point(point, index) }
+    import.file.download do |file|
+      json = Oj.load(file)
+
+      json.each.with_index(1) { |point, index| create_point(point, index) }
+    end
   end
 
   def create_point(point, index)

app/services/points/raw_data_lonlat_extractor.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+class Points::RawDataLonlatExtractor
+  def initialize(point)
+    @point = point
+  end
+
+  def call
+    lonlat = extract_lonlat(@point)
+
+    @point.update(
+      longitude: lonlat[0],
+      latitude: lonlat[1]
+    )
+  end
+
+  private
+
+  # rubocop:disable Metrics/MethodLength
+  def extract_lonlat(point)
+    if point.raw_data.dig('activitySegment', 'waypointPath', 'waypoints', 0)
+      # google_semantic_history_parser
+      [
+        point.raw_data['activitySegment']['waypointPath']['waypoints'][0]['lngE7'].to_f / 10**7,
+        point.raw_data['activitySegment']['waypointPath']['waypoints'][0]['latE7'].to_f / 10**7
+      ]
+    elsif point.raw_data['longitudeE7'] && point.raw_data['latitudeE7']
+      # google records
+      [
+        point.raw_data['longitudeE7'].to_f / 10**7,
+        point.raw_data['latitudeE7'].to_f / 10**7
+      ]
+    elsif point.raw_data.dig('position', 'LatLng')
+      # google phone export
+      raw_coordinates = point.raw_data['position']['LatLng']
+      if raw_coordinates.include?('°')
+        raw_coordinates.split(', ').map { _1.chomp('°') }
+      else
+        raw_coordinates.delete('geo:').split(',')
+      end
+    elsif point.raw_data['lon'] && point.raw_data['lat']
+      # gpx_track_importer, owntracks
+      [point.raw_data['lon'], point.raw_data['lat']]
+    elsif point.raw_data.dig('geometry', 'coordinates', 0) && point.raw_data.dig('geometry', 'coordinates', 1)
+      # geojson
+      [
+        point.raw_data['geometry']['coordinates'][0],
+        point.raw_data['geometry']['coordinates'][1]
+      ]
+    elsif point.raw_data['longitude'] && point.raw_data['latitude']
+      # immich_api, photoprism_api
+      [point.raw_data['longitude'], point.raw_data['latitude']]
+    end
+  end
+  # rubocop:enable Metrics/MethodLength
+end

app/services/telemetry/gather.rb

@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-
-class Telemetry::Gather
-  def initialize(measurement: 'dawarich_usage_metrics')
-    @measurement = measurement
-  end
-
-  def call
-    {
-      measurement:,
-      timestamp: Time.current.to_i,
-      tags: { instance_id: },
-      fields: { dau:, app_version: }
-    }
-  end
-
-  private
-
-  attr_reader :measurement
-
-  def instance_id
-    @instance_id ||= Digest::SHA2.hexdigest(User.first.api_key)
-  end
-
-  def app_version
-    "\"#{APP_VERSION}\""
-  end
-
-  def dau
-    User.where(last_sign_in_at: Time.zone.today.beginning_of_day..Time.zone.today.end_of_day).count
-  end
-end

app/services/telemetry/send.rb

@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-
-class Telemetry::Send
-  BUCKET = 'dawarich_metrics'
-  ORG = 'monitoring'
-
-  def initialize(payload)
-    @payload = payload
-  end
-
-  def call
-    return unless ENV['ENABLE_TELEMETRY'] == 'true'
-
-    line_protocol = build_line_protocol
-    response = send_request(line_protocol)
-    handle_response(response)
-  end
-
-  private
-
-  attr_reader :payload
-
-  def build_line_protocol
-    tag_string = payload[:tags].map { |k, v| "#{k}=#{v}" }.join(',')
-    field_string = payload[:fields].map { |k, v| "#{k}=#{v}" }.join(',')
-
-    "#{payload[:measurement]},#{tag_string} #{field_string} #{payload[:timestamp].to_i}"
-  end
-
-  def send_request(line_protocol)
-    HTTParty.post(
-      "#{TELEMETRY_URL}?org=#{ORG}&bucket=#{BUCKET}&precision=s",
-      body: line_protocol,
-      headers: {
-        'Authorization' => "Token #{Base64.decode64(TELEMETRY_STRING)}",
-        'Content-Type' => 'text/plain'
-      }
-    )
-  end
-
-  def handle_response(response)
-    Rails.logger.error("InfluxDB write failed: #{response.body}") unless response.success?
-
-    response
-  end
-end

app/views/devise/shared/_links.html.erb

@@ -5,6 +5,12 @@
     </div>
   <% end %>
 
+  <% if !SELF_HOSTED && defined?(devise_mapping) && devise_mapping&.registerable? && controller_name != 'registrations' %>
+    <div class='my-2'>
+      <%= link_to "Register", new_registration_path(resource_name) %>
+    </div>
+  <% end %>
+
   <% if devise_mapping.recoverable? && controller_name != 'passwords' && controller_name != 'registrations' %>
     <div class='my-2'>
       <%= link_to "Forgot your password?", new_password_path(resource_name) %>

app/views/layouts/application.html.erb

@@ -7,7 +7,6 @@
     <%= csrf_meta_tags %>
     <%= csp_meta_tag %>
 
-    <link href="https://cdn.jsdelivr.net/npm/daisyui@4.12.14/dist/full.css" rel="stylesheet" type="text/css">
     <link rel="stylesheet" href="https://unpkg.com/leaflet@1.9.4/dist/leaflet.css" integrity="sha256-p4NxAoJBhIIN+hmNHrzRCf9tD/miZyoHS5obTRR9BMY=" crossorigin="" />
     <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/leaflet.draw/1.0.4/leaflet.draw.css"/>
 
@@ -25,7 +24,7 @@
       <div class="flex flex-row gap-5 w-full px-5">
         <%= yield %>
       </div>
-      <%= render 'shared/footer' %>
+      <%= render SELF_HOSTED ? 'shared/footer' : 'shared/legal_footer' %>
     </div>
   </body>
 </html>

app/views/settings/subscriptions/index.html.erb

@@ -7,7 +7,7 @@
     <div class="hero-content text-center">
       <div class="max-w-md">
         <h1 class="text-5xl font-bold">Hello there!</h1>
-        <% if current_user.active_until.future? %>
+        <% if current_user.active_until&.future? %>
           <p class="py-6">
             You are currently subscribed to Dawarich, hurray!
           </p>
@@ -16,13 +16,13 @@
             Your subscription will be valid for the next <span class="text-accent"><%= days_left(current_user.active_until) %></span>.
           </p>
 
-          <%= link_to 'Manage subscription', "#{ENV['SUBSCRIPTION_URL']}/auth/dawarich?token=#{current_user.generate_subscription_token}", class: 'btn btn-primary my-4' %>
+          <%= link_to 'Manage subscription', "#{MANAGER_URL}/auth/dawarich?token=#{current_user.generate_subscription_token}", class: 'btn btn-primary my-4' %>
         <% else %>
           <p class="py-6">
             You are currently not subscribed to Dawarich. How about we fix that?
           </p>
 
-          <%= link_to 'Manage subscription', "#{ENV['SUBSCRIPTION_URL']}/auth/dawarich?token=#{current_user.generate_subscription_token}", class: 'btn btn-primary my-4' %>
+          <%= link_to 'Manage subscription', "#{MANAGER_URL}/auth/dawarich?token=#{current_user.generate_subscription_token}", class: 'btn btn-primary my-4' %>
         <% end %>
       </div>
     </div>

app/views/shared/_footer.html.erb

@@ -1,4 +1,4 @@
-<footer class="footer footer-center p-4 bg-base-300 text-base-content">
+<footer class="footer bg-base-200 text-content-neutral p-4">
   <aside>
     <p><a href="https://dawarich.app/" class="link hover:no-underline" target="_blank">Dawarich</a> 2023-<%=Time.zone.now.year %></p>
   </aside>

app/views/shared/_legal_footer.html.erb

@@ -0,0 +1,34 @@
+<footer class="footer bg-base-200 text-content-neutral p-4">
+  <nav>
+    <h6 class="footer-title"><strong>Dawarich</strong></h6>
+    <p>
+      Made and hosted in 🇪🇺 Europe
+    </p>
+    <p>
+      Copyright © <%= Time.zone.now.year %> ZeitFlow UG
+    </p>
+  </nav>
+  <nav>
+    <h6 class="footer-title"><strong>Community</strong></h6>
+    <a class="hover:underline" href="https://discord.gg/pHsBjpt5J8" target="_blank">Discord</a>
+    <a class="hover:underline" href="https://x.com/freymakesstuff" target="_blank">X</a>
+    <a class="hover:underline" href="https://github.com/Freika/dawarich" target="_blank">Github</a>
+    <a class="hover:underline" href="https://mastodon.social/@dawarich" target="_blank">Mastodon</a>
+  </nav>
+  <nav>
+    <h6 class="footer-title"><strong>Docs</strong></h6>
+    <a class="hover:underline" href="https://dawarich.app/docs/intro" target="_blank">Tutorial</a>
+    <a class="hover:underline" href="https://dawarich.app/docs/tutorials/import-existing-data" target="_blank">Import existing data</a>
+    <a class="hover:underline" href="https://dawarich.app/docs/tutorials/export-your-data" target="_blank">Exporting data</a>
+    <a class="hover:underline" href="https://dawarich.app/docs/FAQ" target="_blank">FAQ</a>
+    <a class="hover:underline" href="https://dawarich.app/contact" target="_blank">Contact</a>
+  </nav>
+  <nav>
+    <h6 class="footer-title"><strong>More</strong></h6>
+    <a class="hover:underline" href="https://dawarich.app/privacy-policy" target="_blank">Privacy policy</a>
+    <a class="hover:underline" href="https://dawarich.app/terms-and-conditions" target="_blank">Terms and Conditions</a>
+    <a class="hover:underline" href="https://dawarich.app/refund-policy" target="_blank">Refund policy</a>
+    <a class="hover:underline" href="https://dawarich.app/impressum" target="_blank">Impressum</a>
+    <a class="hover:underline" href="https://dawarich.app/blog" target="_blank">Blog</a>
+  </nav>
+</footer>

app/views/shared/_navbar.html.erb

@@ -20,7 +20,7 @@
           </details>
         </li>
         <% if user_signed_in? && current_user.can_subscribe? %>
-          <li><%= link_to 'Subscribe', "#{ENV['SUBSCRIPTION_URL']}/auth/dawarich?token=#{current_user.generate_subscription_token}", class: 'btn btn-sm btn-success' %></li>
+          <li><%= link_to 'Subscribe', "#{MANAGER_URL}/auth/dawarich?token=#{current_user.generate_subscription_token}", class: 'btn btn-sm btn-success' %></li>
         <% end %>
       </ul>
     </div>
@@ -71,7 +71,7 @@
     <ul class="menu menu-horizontal bg-base-100 rounded-box px-1">
       <% if user_signed_in? %>
         <% if current_user.can_subscribe? %>
-          <li><%= link_to 'Subscribe', "#{ENV['SUBSCRIPTION_URL']}/auth/dawarich?token=#{current_user.generate_subscription_token}", class: 'btn btn-sm btn-success' %></li>
+          <li><%= link_to 'Subscribe', "#{MANAGER_URL}/auth/dawarich?token=#{current_user.generate_subscription_token}", class: 'btn btn-sm btn-success' %></li>
         <% end %>
 
         <div class="dropdown dropdown-end dropdown-bottom dropdown-hover"
@@ -98,8 +98,8 @@
           </div>
           <ul tabindex="0" class="dropdown-content z-[5000] menu p-2 shadow-lg bg-base-100 rounded-box min-w-52" data-notifications-target="list">
             <li><%= link_to 'See all', notifications_path %></li>
-            <div class="divider p-0 m-0"></div>
             <% @unread_notifications.first(10).each do |notification| %>
+              <div class="divider p-0 m-0"></div>
               <li class='notification-item'>
                 <%= link_to notification do %>
                   <%= notification.title %>
@@ -126,6 +126,9 @@
         </li>
       <% else %>
         <li><%= link_to 'Login', new_user_session_path %></li>
+        <% if !SELF_HOSTED && defined?(devise_mapping) && devise_mapping&.registerable? %>
+          <li><%= link_to 'Register', new_user_registration_path %></li>
+        <% end %>
       <% end %>
     </ul>
   </div>

config/environments/production.rb

@@ -107,4 +107,17 @@ Rails.application.configure do
 
   config.action_mailer.default_url_options = { host: hosts.first, port: 3000 }
   config.hosts.concat(hosts) if hosts.present?
+
+  config.action_mailer.delivery_method = :smtp
+  config.action_mailer.smtp_settings = {
+    address:         ENV['SMTP_SERVER'],
+    port:            ENV['SMTP_PORT'],
+    domain:          ENV['SMTP_DOMAIN'],
+    user_name:       ENV['SMTP_USERNAME'],
+    password:        ENV['SMTP_PASSWORD'],
+    authentication:  'plain',
+    enable_starttls: true,
+    open_timeout:    5,
+    read_timeout:    5
+  }
 end

config/initializers/01_constants.rb

@@ -7,9 +7,6 @@ DISTANCE_UNIT = ENV.fetch('DISTANCE_UNIT', 'km').to_sym
 
 APP_VERSION = File.read('.app_version').strip
 
-TELEMETRY_STRING = Base64.encode64('IjVFvb8j3P9-ArqhSGav9j8YcJaQiuNIzkfOPKQDk2lvKXqb8t1NSRv50oBkaKtlrB_ZRzO9NdurpMtncV_HYQ==')
-TELEMETRY_URL = 'https://influxdb2.frey.today/api/v2/write'
-
 # Reverse geocoding settings
 PHOTON_API_HOST = ENV.fetch('PHOTON_API_HOST', nil)
 PHOTON_API_KEY = ENV.fetch('PHOTON_API_KEY', nil)
@@ -23,3 +20,4 @@ GEOAPIFY_API_KEY = ENV.fetch('GEOAPIFY_API_KEY', nil)
 # /Reverse geocoding settings
 
 SENTRY_DSN = ENV.fetch('SENTRY_DSN', nil)
+MANAGER_URL = SELF_HOSTED ? nil : ENV.fetch('MANAGER_URL', nil)

config/initializers/devise.rb

@@ -24,7 +24,7 @@ Devise.setup do |config|
   # Configure the e-mail address which will be shown in Devise::Mailer,
   # note that it will be overwritten if you use your own mailer class
   # with default "from" parameter.
-  config.mailer_sender = 'please-change-me-at-config-initializers-devise@example.com'
+  config.mailer_sender = ENV['SMTP_FROM']
 
   # Configure the class responsible to send e-mails.
   # config.mailer = 'Devise::Mailer'

config/routes.rb

@@ -6,7 +6,22 @@ Rails.application.routes.draw do
   mount ActionCable.server => '/cable'
   mount Rswag::Api::Engine => '/api-docs'
   mount Rswag::Ui::Engine => '/api-docs'
-  authenticate :user, ->(u) { u.admin? && DawarichSettings.self_hosted? } do
+
+  Sidekiq::Web.use(Rack::Auth::Basic) do |username, password|
+    ActiveSupport::SecurityUtils.secure_compare(
+      ::Digest::SHA256.hexdigest(username),
+      ::Digest::SHA256.hexdigest(ENV['SIDEKIQ_USERNAME'])
+    ) &
+      ActiveSupport::SecurityUtils.secure_compare(
+        ::Digest::SHA256.hexdigest(password),
+        ::Digest::SHA256.hexdigest(ENV['SIDEKIQ_PASSWORD'])
+      )
+  end
+
+  authenticate :user, lambda { |u|
+    (u.admin? && DawarichSettings.self_hosted?) ||
+      (u.admin? && ENV['SIDEKIQ_USERNAME'].present? && ENV['SIDEKIQ_PASSWORD'].present?)
+  } do
     mount Sidekiq::Web => '/sidekiq'
   end
 

db/data/20240808133112_run_initial_visit_suggestion.rb

@@ -5,7 +5,9 @@ class RunInitialVisitSuggestion < ActiveRecord::Migration[7.1]
     start_at = 30.years.ago
     end_at = Time.current
 
-    VisitSuggestingJob.perform_later(start_at:, end_at:)
+    User.find_each do |user|
+      VisitSuggestingJob.perform_later(user_id: user.id, start_at:, end_at:)
+    end
   end
 
   def down

db/data/20241206163450_create_telemetry_notification.rb

@@ -1,44 +1,7 @@
 # frozen_string_literal: true
 
 class CreateTelemetryNotification < ActiveRecord::Migration[7.2]
-  def up
-    # TODO: Remove
-    # User.find_each do |user|
-    #   Notifications::Create.new(
-    #     user:, kind: :info, title: 'Telemetry enabled', content: notification_content
-    #   ).call
-    # end
-  end
+  def up; end
 
-  def down
-    raise ActiveRecord::IrreversibleMigration
-  end
-
-  private
-
-  def notification_content
-    <<~CONTENT
-      <p>With the release 0.19.2, Dawarich now can collect usage some metrics and send them to InfluxDB.</p>
-      <br>
-      <p>Before this release, the only metrics that could be somehow tracked by developers (only <a href="https://github.com/Freika" class="underline">Freika</a>, as of now) were the number of stars on GitHub and the overall number of docker images being pulled, across all versions of Dawarich, non-splittable by version. New in-app telemetry will allow us to track more granular metrics, allowing me to make decisions based on facts, not just guesses.</p>
-      <br>
-      <p>I'm aware about the privacy concerns, so I want to be very transparent about what data is being sent and how it's used.</p>
-      <br>
-      <p>Data being sent:</p>
-      <br>
-      <ul class="list-disc">
-        <li>Number of DAU (Daily Active Users)</li>
-        <li>App version</li>
-        <li>Instance ID (unique identifier of the Dawarich instance built by hashing the api key of the first user in the database)</li>
-      </ul>
-      <br>
-      <p>The data is being sent to a InfluxDB instance hosted by me and won't be shared with anyone.</p>
-      <br>
-      <p>Basically this set of metrics allows me to see how many people are using Dawarich and what versions they are using. No other data is being sent, nor it gives me any knowledge about individual users or their data or activity.</p>
-      <br>
-      <p>The telemetry is enabled by default, but it <strong class="text-info underline">can be disabled</strong> by setting <code>DISABLE_TELEMETRY</code> env var to <code>true</code>. The dataset might change in the future, but any changes will be documented here in the changelog and in every release as well as on the <a href="https://dawarich.app/docs/tutorials/telemetry" class="underline">telemetry page</a> of the website docs.</p>
-      <br>
-      <p>You can read more about it in the <a href="https://github.com/Freika/dawarich/releases/tag/0.19.2" class="underline">release page</a>.</p>
-    CONTENT
-  end
+  def down; end
 end

docker/Dockerfile.dev

@@ -6,6 +6,7 @@ ENV BUNDLE_PATH=/usr/local/bundle/gems
 ENV RAILS_LOG_TO_STDOUT=true
 ENV RAILS_PORT=3000
 ENV RAILS_ENV=development
+ENV SELF_HOSTED=true
 
 # Install dependencies for application
 RUN apk -U add --no-cache \

docker/docker-compose.yml

@@ -69,8 +69,8 @@ services:
       PROMETHEUS_EXPORTER_ENABLED: false
       PROMETHEUS_EXPORTER_HOST: 0.0.0.0
       PROMETHEUS_EXPORTER_PORT: 9394
-      ENABLE_TELEMETRY: false # More on telemetry: https://dawarich.app/docs/tutorials/telemetry
       SELF_HOSTED: "true"
+      RAILS_MASTER_KEY: ${RAILS_MASTER_KEY}
     logging:
       driver: "json-file"
       options:
@@ -122,8 +122,8 @@ services:
       PROMETHEUS_EXPORTER_ENABLED: false
       PROMETHEUS_EXPORTER_HOST: dawarich_app
       PROMETHEUS_EXPORTER_PORT: 9394
-      ENABLE_TELEMETRY: false # More on telemetry: https://dawarich.app/docs/tutorials/telemetry
       SELF_HOSTED: "true"
+      RAILS_MASTER_KEY: ${RAILS_MASTER_KEY}
     logging:
       driver: "json-file"
       options:

docker/sidekiq-entrypoint.sh

@@ -14,17 +14,19 @@ if [ -n "$DATABASE_URL" ]; then
   DATABASE_PORT=$(echo $DATABASE_URL | awk -F[@/:] '{print $5}')
   DATABASE_USERNAME=$(echo $DATABASE_URL | awk -F[:/@] '{print $4}')
   DATABASE_PASSWORD=$(echo $DATABASE_URL | awk -F[:/@] '{print $5}')
+  DATABASE_NAME=$(echo $DATABASE_URL | awk -F[@/] '{print $5}')
 else
   # Use existing environment variables
   DATABASE_HOST=${DATABASE_HOST}
   DATABASE_PORT=${DATABASE_PORT}
   DATABASE_USERNAME=${DATABASE_USERNAME}
   DATABASE_PASSWORD=${DATABASE_PASSWORD}
+  DATABASE_NAME=${DATABASE_NAME}
 fi
 
 # Wait for the database to become available
 echo "⏳ Waiting for database to be ready..."
-until PGPASSWORD=$DATABASE_PASSWORD psql -h "$DATABASE_HOST" -p "$DATABASE_PORT" -U "$DATABASE_USERNAME" -c '\q'; do
+until PGPASSWORD=$DATABASE_PASSWORD psql -h "$DATABASE_HOST" -p "$DATABASE_PORT" -U "$DATABASE_USERNAME" -d "$DATABASE_NAME" -c '\q'; do
   >&2 echo "Postgres is unavailable - retrying..."
   sleep 2
 done

docker/web-entrypoint.sh

@@ -29,16 +29,16 @@ rm -f $APP_PATH/tmp/pids/server.pid
 
 # Wait for the database to become available
 echo "⏳ Waiting for database to be ready..."
-until PGPASSWORD=$DATABASE_PASSWORD psql -h "$DATABASE_HOST" -p "$DATABASE_PORT" -U "$DATABASE_USERNAME" -c '\q'; do
+until PGPASSWORD=$DATABASE_PASSWORD psql -h "$DATABASE_HOST" -p "$DATABASE_PORT" -U "$DATABASE_USERNAME" -d "$DATABASE_NAME" -c '\q'; do
   >&2 echo "Postgres is unavailable - retrying..."
   sleep 2
 done
 echo "✅ PostgreSQL is ready!"
 
 # Create database if it doesn't exist
-if ! PGPASSWORD=$DATABASE_PASSWORD psql -h "$DATABASE_HOST" -p "$DATABASE_PORT" -U "$DATABASE_USERNAME" -c "SELECT 1 FROM pg_database WHERE datname='$DATABASE_NAME'" | grep -q 1; then
+if ! PGPASSWORD=$DATABASE_PASSWORD psql -h "$DATABASE_HOST" -p "$DATABASE_PORT" -U "$DATABASE_USERNAME" -d "$DATABASE_NAME" -c "SELECT 1 FROM pg_database WHERE datname='$DATABASE_NAME'" | grep -q 1; then
   echo "Creating database $DATABASE_NAME..."
-  bundle exec rails db:create
+  PGPASSWORD=$DATABASE_PASSWORD createdb -h "$DATABASE_HOST" -p "$DATABASE_PORT" -U "$DATABASE_USERNAME" "$DATABASE_NAME"
 fi
 
 # Run database migrations

lib/tasks/points.rake

@@ -5,6 +5,12 @@ namespace :points do
   task migrate_to_lonlat: :environment do
     puts 'Updating points to use lonlat...'
 
+    points = Point.where(longitude: nil, latitude: nil).select(:id, :longitude, :latitude, :raw_data)
+
+    points.find_each do |point|
+      Points::RawDataLonlatExtractor.new(point).call
+    end
+
     ActiveRecord::Base.connection.execute('REINDEX TABLE points;')
 
     ActiveRecord::Base.transaction do

spec/jobs/telemetry_sending_job_spec.rb

@@ -1,39 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-RSpec.describe TelemetrySendingJob, type: :job do
-  describe '#perform' do
-    let(:gather_service) { instance_double(Telemetry::Gather) }
-    let(:send_service) { instance_double(Telemetry::Send) }
-    let(:telemetry_data) { { some: 'data' } }
-
-    before do
-      allow(Telemetry::Gather).to receive(:new).and_return(gather_service)
-      allow(gather_service).to receive(:call).and_return(telemetry_data)
-      allow(Telemetry::Send).to receive(:new).with(telemetry_data).and_return(send_service)
-      allow(send_service).to receive(:call)
-    end
-
-    context 'with default env' do
-      it 'does not send telemetry data' do
-        described_class.perform_now
-
-        expect(send_service).not_to have_received(:call)
-      end
-    end
-
-    context 'when ENABLE_TELEMETRY is set to true' do
-      before do
-        stub_const('ENV', ENV.to_h.merge('ENABLE_TELEMETRY' => 'true'))
-      end
-
-        it 'gathers telemetry data and sends it' do
-        described_class.perform_now
-
-        expect(gather_service).to have_received(:call)
-        expect(send_service).to have_received(:call)
-      end
-    end
-  end
-end

spec/requests/imports_spec.rb

@@ -77,4 +77,65 @@ RSpec.describe 'Imports', type: :request do
       end
     end
   end
+
+  describe 'GET /imports/new' do
+    context 'when user is logged in' do
+      let(:user) { create(:user) }
+
+      before { sign_in user }
+
+      it 'returns http success' do
+        get new_import_path
+
+        expect(response).to have_http_status(200)
+      end
+    end
+  end
+
+  describe 'DELETE /imports/:id' do
+    context 'when user is logged in' do
+      let(:user) { create(:user) }
+      let!(:import) { create(:import, user:) }
+
+      before { sign_in user }
+
+      it 'deletes the import' do
+        expect do
+          delete import_path(import)
+        end.to change(user.imports, :count).by(-1)
+
+        expect(response).to redirect_to(imports_path)
+      end
+    end
+  end
+
+  describe 'GET /imports/:id/edit' do
+    context 'when user is logged in' do
+      let(:user) { create(:user) }
+      let(:import) { create(:import, user:) }
+
+      before { sign_in user }
+
+      it 'returns http success' do
+        get edit_import_path(import)
+
+        expect(response).to have_http_status(200)
+      end
+    end
+  end
+
+  describe 'PATCH /imports/:id' do
+    context 'when user is logged in' do
+      let(:user) { create(:user) }
+      let(:import) { create(:import, user:) }
+
+      before { sign_in user }
+
+      it 'updates the import' do
+        patch import_path(import), params: { import: { name: 'New Name' } }
+
+        expect(response).to redirect_to(imports_path)
+      end
+    end
+  end
 end

spec/requests/sidekiq_spec.rb

@@ -1,11 +1,24 @@
 # frozen_string_literal: true
 
 require 'rails_helper'
+require 'sidekiq/web'
 
 RSpec.describe '/sidekiq', type: :request do
+  before do
+    # Allow any ENV key to be accessed and return nil by default
+    allow(ENV).to receive(:[]).and_return(nil)
+
+    # Stub Sidekiq::Web with a simple Rack app for testing
+    allow(Sidekiq::Web).to receive(:call) do |_env|
+      [200, { 'Content-Type' => 'text/html' }, ['Sidekiq Web UI']]
+    end
+  end
+
   context 'when Dawarich is in self-hosted mode' do
     before do
       allow(DawarichSettings).to receive(:self_hosted?).and_return(true)
+      allow(ENV).to receive(:[]).with('SIDEKIQ_USERNAME').and_return(nil)
+      allow(ENV).to receive(:[]).with('SIDEKIQ_PASSWORD').and_return(nil)
     end
 
     context 'when user is not authenticated' do
@@ -48,6 +61,8 @@ RSpec.describe '/sidekiq', type: :request do
   context 'when Dawarich is not in self-hosted mode' do
     before do
       allow(DawarichSettings).to receive(:self_hosted?).and_return(false)
+      allow(ENV).to receive(:[]).with('SIDEKIQ_USERNAME').and_return(nil)
+      allow(ENV).to receive(:[]).with('SIDEKIQ_PASSWORD').and_return(nil)
       Rails.application.reload_routes!
     end
 
@@ -70,4 +85,41 @@ RSpec.describe '/sidekiq', type: :request do
       end
     end
   end
+
+  context 'when SIDEKIQ_USERNAME and SIDEKIQ_PASSWORD are set' do
+    before do
+      allow(DawarichSettings).to receive(:self_hosted?).and_return(false)
+      allow(ENV).to receive(:[]).with('SIDEKIQ_USERNAME').and_return('admin')
+      allow(ENV).to receive(:[]).with('SIDEKIQ_PASSWORD').and_return('password')
+    end
+
+    context 'when user is not authenticated' do
+      it 'redirects to sign in page' do
+        get sidekiq_url
+
+        expect(response).to redirect_to('/users/sign_in')
+      end
+    end
+
+    context 'when user is not admin' do
+      before { sign_in create(:user) }
+
+      it 'redirects to root page' do
+        get sidekiq_url
+
+        expect(response).to redirect_to(root_url)
+        expect(flash[:error]).to eq('You are not authorized to perform this action.')
+      end
+    end
+
+    context 'when user is admin' do
+      before { sign_in create(:user, :admin) }
+
+      it 'renders a successful response' do
+        get sidekiq_url
+
+        expect(response).to be_successful
+      end
+    end
+  end
 end

spec/services/photos/import_parser_spec.rb

@@ -13,7 +13,14 @@ RSpec.describe Photos::ImportParser do
     let(:immich_data) do
       JSON.parse(File.read(Rails.root.join('spec/fixtures/files/immich/geodata.json')))
     end
-    let(:import) { create(:import, user:, raw_data: immich_data) }
+    let(:import) { create(:import, user:) }
+
+    let(:file_path) { Rails.root.join('spec/fixtures/files/immich/geodata.json') }
+    let(:file) { Rack::Test::UploadedFile.new(file_path, 'text/plain') }
+
+    before do
+      import.file.attach(io: File.open(file_path), filename: 'immich_geodata.json', content_type: 'application/json')
+    end
 
     context 'when there are no points' do
       it 'creates new points' do

spec/services/telemetry/gather_spec.rb

@@ -1,45 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-RSpec.describe Telemetry::Gather do
-  let!(:user) { create(:user, last_sign_in_at: Time.zone.today) }
-
-  describe '#call' do
-    subject(:gather) { described_class.new.call }
-
-    it 'returns a hash with measurement, timestamp, tags, and fields' do
-      expect(gather).to include(:measurement, :timestamp, :tags, :fields)
-    end
-
-    it 'includes the correct measurement' do
-      expect(gather[:measurement]).to eq('dawarich_usage_metrics')
-    end
-
-    it 'includes the current timestamp' do
-      expect(gather[:timestamp]).to be_within(1).of(Time.current.to_i)
-    end
-
-    it 'includes the correct instance_id in tags' do
-      expect(gather[:tags][:instance_id]).to eq(Digest::SHA2.hexdigest(user.api_key))
-    end
-
-    it 'includes the correct app_version in fields' do
-      expect(gather[:fields][:app_version]).to eq("\"#{APP_VERSION}\"")
-    end
-
-    it 'includes the correct dau in fields' do
-      expect(gather[:fields][:dau]).to eq(1)
-    end
-
-    context 'with a custom measurement' do
-      let(:measurement) { 'custom_measurement' }
-
-      subject(:gather) { described_class.new(measurement:).call }
-
-      it 'includes the correct measurement' do
-        expect(gather[:measurement]).to eq('custom_measurement')
-      end
-    end
-  end
-end