Return sidekiq and redis to Dawarich

.circleci/config.yml

@@ -16,16 +16,12 @@ jobs:
           DATABASE_USERNAME: postgres
           DATABASE_PASSWORD: mysecretpassword
           DATABASE_PORT: 5432
-          QUEUE_DATABASE_HOST: localhost
-          QUEUE_DATABASE_NAME: dawarich_test_queue
-          QUEUE_DATABASE_USERNAME: postgres
-          QUEUE_DATABASE_PASSWORD: mysecretpassword
-          QUEUE_DATABASE_PORT: 5432
       - image: cimg/postgres:13.3-postgis
         environment:
           POSTGRES_USER: postgres
           POSTGRES_DB: dawarich_test
           POSTGRES_PASSWORD: mysecretpassword
+      - image: redis:7.0
       - image: selenium/standalone-chrome:latest
         name: chrome
         environment:

.devcontainer/docker-compose.yml

@@ -20,6 +20,7 @@ services:
     tty: true
     environment:
       RAILS_ENV: development
+      REDIS_URL: redis://dawarich_redis:6379/0
       DATABASE_HOST: dawarich_db
       DATABASE_USERNAME: postgres
       DATABASE_PASSWORD: password
@@ -40,6 +41,21 @@ services:
       PROMETHEUS_EXPORTER_ENABLED: false
       PROMETHEUS_EXPORTER_HOST: 0.0.0.0
       PROMETHEUS_EXPORTER_PORT: 9394
+  dawarich_redis:
+    image: redis:7.4-alpine
+    container_name: dawarich_redis
+    command: redis-server
+    networks:
+      - dawarich
+    volumes:
+      - dawarich_shared:/data
+    restart: always
+    healthcheck:
+      test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ]
+      interval: 10s
+      retries: 5
+      start_period: 30s
+      timeout: 10s
   dawarich_db:
     image: postgis/postgis:17-3.5-alpine
     container_name: dawarich_db

.env.development

@@ -3,3 +3,4 @@ DATABASE_USERNAME=postgres
 DATABASE_PASSWORD=password
 DATABASE_NAME=dawarich_development
 DATABASE_PORT=5432
+REDIS_URL=redis://localhost:6379/1

.env.test

@@ -3,3 +3,4 @@ DATABASE_USERNAME=postgres
 DATABASE_PASSWORD=password
 DATABASE_NAME=dawarich_test
 DATABASE_PORT=5432
+REDIS_URL=redis://localhost:6379/1

.github/workflows/ci.yml

@@ -19,6 +19,10 @@ jobs:
         ports:
           - 5432:5432
         options: --health-cmd="pg_isready" --health-interval=10s --health-timeout=5s --health-retries=3
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
 
     steps:
       - name: Install packages
@@ -53,12 +57,14 @@ jobs:
         env:
           RAILS_ENV: test
           DATABASE_URL: postgres://postgres:postgres@localhost:5432
+          REDIS_URL: redis://localhost:6379/1
         run: bin/rails db:setup
 
       - name: Run main tests (excluding system tests)
         env:
           RAILS_ENV: test
           DATABASE_URL: postgres://postgres:postgres@localhost:5432
+          REDIS_URL: redis://localhost:6379/1
         run: |
           bundle exec rspec --exclude-pattern "spec/system/**/*_spec.rb" || (cat log/test.log && exit 1)
 
@@ -66,6 +72,7 @@ jobs:
         env:
           RAILS_ENV: test
           DATABASE_URL: postgres://postgres:postgres@localhost:5432
+          REDIS_URL: redis://localhost:6379/1
         run: |
           bundle exec rspec spec/system/ || (cat log/test.log && exit 1)
 

DEVELOPMENT.md

@@ -7,6 +7,12 @@ Now you can create/prepare the Database (this need to be done once):
 bundle exec rails db:prepare
 ```
 
+Afterwards you can run sidekiq:
+```bash
+bundle exec sidekiq
+
+```
+
 And in a second terminal the dawarich-app:
 ```bash
 bundle exec bin/dev

Gemfile

@@ -28,6 +28,7 @@ gem 'activerecord-postgis-adapter'
 gem 'puma'
 gem 'pundit'
 gem 'rails', '~> 8.0'
+gem 'redis'
 gem 'rexml'
 gem 'rgeo'
 gem 'rgeo-activerecord'
@@ -38,6 +39,9 @@ gem 'sentry-ruby'
 gem 'sentry-rails'
 gem 'sqlite3', '~> 2.6'
 gem 'stackprof'
+gem 'sidekiq'
+gem 'sidekiq-cron'
+gem 'sidekiq-limit_fetch'
 gem 'sprockets-rails'
 gem 'stimulus-rails'
 gem 'strong_migrations'
@@ -64,6 +68,7 @@ end
 
 group :test do
   gem 'capybara'
+  gem 'fakeredis'
   gem 'selenium-webdriver'
   gem 'shoulda-matchers'
   gem 'simplecov', require: false

Gemfile.lock

@@ -134,6 +134,9 @@ GEM
       bigdecimal
       rexml
     crass (1.0.6)
+    cronex (0.15.0)
+      tzinfo
+      unicode (>= 0.4.4.5)
     csv (3.3.4)
     data_migrate (11.3.0)
       activerecord (>= 6.1)
@@ -166,6 +169,7 @@ GEM
     factory_bot_rails (6.4.4)
       factory_bot (~> 6.5)
       railties (>= 5.0.0)
+    fakeredis (0.1.4)
     ffaker (2.24.0)
     foreman (0.88.1)
     fugit (1.11.1)
@@ -350,6 +354,10 @@ GEM
     rdoc (6.14.0)
       erb
       psych (>= 4.0.0)
+    redis (5.4.0)
+      redis-client (>= 0.22.0)
+    redis-client (0.24.0)
+      connection_pool
     regexp_parser (2.10.0)
     reline (0.6.1)
       io-console (~> 0.5)
@@ -431,6 +439,19 @@ GEM
       concurrent-ruby (~> 1.0, >= 1.0.2)
     shoulda-matchers (6.5.0)
       activesupport (>= 5.2.0)
+    sidekiq (8.0.4)
+      connection_pool (>= 2.5.0)
+      json (>= 2.9.0)
+      logger (>= 1.6.2)
+      rack (>= 3.1.0)
+      redis-client (>= 0.23.2)
+    sidekiq-cron (2.3.0)
+      cronex (>= 0.13.0)
+      fugit (~> 1.8, >= 1.11.1)
+      globalid (>= 1.0.1)
+      sidekiq (>= 6.5.0)
+    sidekiq-limit_fetch (4.4.1)
+      sidekiq (>= 6)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -492,6 +513,7 @@ GEM
       railties (>= 7.1.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
+    unicode (0.4.4.5)
     unicode-display_width (3.1.4)
       unicode-emoji (~> 4.0, >= 4.0.4)
     unicode-emoji (4.0.4)
@@ -537,6 +559,7 @@ DEPENDENCIES
   devise
   dotenv-rails
   factory_bot_rails
+  fakeredis
   ffaker
   foreman
   geocoder!
@@ -556,6 +579,7 @@ DEPENDENCIES
   puma
   pundit
   rails (~> 8.0)
+  redis
   rexml
   rgeo
   rgeo-activerecord
@@ -569,6 +593,9 @@ DEPENDENCIES
   sentry-rails
   sentry-ruby
   shoulda-matchers
+  sidekiq
+  sidekiq-cron
+  sidekiq-limit_fetch
   simplecov
   solid_cable (~> 3.0)
   solid_cache (= 1.0.7)

app/jobs/app_version_checking_job.rb

@@ -2,6 +2,7 @@
 
 class AppVersionCheckingJob < ApplicationJob
   queue_as :default
+  sidekiq_options retry: false
 
   def perform
     Rails.cache.delete(CheckAppVersion::VERSION_CACHE_KEY)

app/jobs/area_visits_calculating_job.rb

@@ -2,6 +2,7 @@
 
 class AreaVisitsCalculatingJob < ApplicationJob
   queue_as :default
+  sidekiq_options retry: false
 
   def perform(user_id)
     user = User.find(user_id)

app/jobs/area_visits_calculation_scheduling_job.rb

@@ -2,6 +2,7 @@
 
 class AreaVisitsCalculationSchedulingJob < ApplicationJob
   queue_as :default
+  sidekiq_options retry: false
 
   def perform
     User.find_each { AreaVisitsCalculatingJob.perform_later(_1.id) }

app/jobs/bulk_visits_suggesting_job.rb

@@ -4,6 +4,7 @@
 # with the default timespan of 1 day.
 class BulkVisitsSuggestingJob < ApplicationJob
   queue_as :visit_suggesting
+  sidekiq_options retry: false
 
   # Passing timespan of more than 3 years somehow results in duplicated Places
   def perform(start_at: 1.day.ago.beginning_of_day, end_at: 1.day.ago.end_of_day, user_ids: [])

app/jobs/import/google_takeout_job.rb

@@ -2,6 +2,7 @@
 
 class Import::GoogleTakeoutJob < ApplicationJob
   queue_as :imports
+  sidekiq_options retry: false
 
   def perform(import_id, locations, current_index)
     locations_batch = Oj.load(locations)

app/jobs/import/photoprism_geodata_job.rb

@@ -2,6 +2,7 @@
 
 class Import::PhotoprismGeodataJob < ApplicationJob
   queue_as :imports
+  sidekiq_options retry: false
 
   def perform(user_id)
     user = User.find(user_id)

app/jobs/import/watcher_job.rb

@@ -2,6 +2,7 @@
 
 class Import::WatcherJob < ApplicationJob
   queue_as :imports
+  sidekiq_options retry: false
 
   def perform
     return unless DawarichSettings.self_hosted?

app/jobs/visit_suggesting_job.rb

@@ -2,6 +2,7 @@
 
 class VisitSuggestingJob < ApplicationJob
   queue_as :visit_suggesting
+  sidekiq_options retry: false
 
   # Passing timespan of more than 3 years somehow results in duplicated Places
   def perform(user_id:, start_at:, end_at:)

app/services/tasks/imports/google_records.rb

@@ -54,6 +54,6 @@ class Tasks::Imports::GoogleRecords
   end
 
   def log_success
-    Rails.logger.info("Imported #{@file_path} for #{@user.email} successfully! Wait for the processing to finish. You can check the status of the import in the Jobs UI (http://<your-dawarich-url>/jobs).")
+    Rails.logger.info("Imported #{@file_path} for #{@user.email} successfully! Wait for the processing to finish. You can check the status of the import in the Sidekiq UI (http://<your-dawarich-url>/sidekiq).")
   end
 end

app/views/notifications/_notification.html.erb

@@ -11,7 +11,7 @@
 
         <% if notification.error? %>
           <div class="mt-2">
-            Please, when reporting a bug to <a href="https://github.com/Freika/dawarich/issues" class="link hover:no-underline text-blue-600">Github Issues</a>, don't forget to include logs from <code>dawarich_app</code> docker container. Thank you!
+            Please, when reporting a bug to <a href="https://github.com/Freika/dawarich/issues" class="link hover:no-underline text-blue-600">Github Issues</a>, don't forget to include logs from <code>dawarich_app</code> and <code>dawarich_sidekiq</code> docker containers. Thank you!
           </div>
         <% end %>
       </div>

app/views/settings/background_jobs/index.html.erb

@@ -45,7 +45,7 @@
         <h2 class="card-title">Background Jobs Dashboard</h2>
         <p>This will open the background jobs dashboard in a new tab.</p>
         <div class="card-actions justify-end">
-          <%= link_to 'Open Dashboard', mission_control_jobs_url, target: '_blank', class: 'btn btn-primary' %>
+          <%= link_to 'Open Dashboard', '/sidekiq', target: '_blank', class: 'btn btn-primary' %>
         </div>
       </div>
     </div>

config/application.rb

@@ -34,5 +34,7 @@ module Dawarich
       g.routing_specs false
       g.helper_specs false
     end
+
+    config.active_job.queue_adapter = :sidekiq
   end
 end

config/initializers/geocoder.rb

@@ -4,7 +4,7 @@ settings = {
   debug_mode: true,
   timeout: 5,
   units: :km,
-  cache: Geocoder::CacheStore::Generic.new(Rails.cache, {}),
+  cache: cache: Redis.new,
   always_raise: :all,
   http_headers: {
     'User-Agent' => "Dawarich #{APP_VERSION} (https://dawarich.app)"

config/initializers/sidekiq.rb

@@ -1,24 +1,30 @@
 # frozen_string_literal: true
 
-# Sidekiq.configure_server do |config|
+Sidekiq.configure_server do |config|
-#   if ENV['PROMETHEUS_EXPORTER_ENABLED'].to_s == 'true'
+  config.redis = { url: ENV['REDIS_URL'] }
-#     require 'prometheus_exporter/instrumentation'
+  config.logger = Sidekiq::Logger.new($stdout)
 
-#     # Add middleware for collecting job-level metrics
+  if ENV['PROMETHEUS_EXPORTER_ENABLED'].to_s == 'true'
-#     config.server_middleware do |chain|
+    require 'prometheus_exporter/instrumentation'
-#       chain.add PrometheusExporter::Instrumentation::Sidekiq
+    # Add middleware for collecting job-level metrics
-#     end
+    config.server_middleware do |chain|
+      chain.add PrometheusExporter::Instrumentation::Sidekiq
+    end
 
-#     # Capture metrics for failed jobs
+    # Capture metrics for failed jobs
-#     config.death_handlers << PrometheusExporter::Instrumentation::Sidekiq.death_handler
+    config.death_handlers << PrometheusExporter::Instrumentation::Sidekiq.death_handler
 
-#     # Start Prometheus instrumentation
+    # Start Prometheus instrumentation
-#     config.on :startup do
+    config.on :startup do
-#       PrometheusExporter::Instrumentation::SidekiqProcess.start
+      PrometheusExporter::Instrumentation::SidekiqProcess.start
-#       PrometheusExporter::Instrumentation::SidekiqQueue.start
+      PrometheusExporter::Instrumentation::SidekiqQueue.start
-#       PrometheusExporter::Instrumentation::SidekiqStats.start
+      PrometheusExporter::Instrumentation::SidekiqStats.start
-#     end
+    end
-#   end
+  end
-# end
+end
 
-# Sidekiq::Queue['reverse_geocoding'].limit = 1 if Sidekiq.server? && DawarichSettings.photon_uses_komoot_io?
+Sidekiq.configure_client do |config|
+  config.redis = { url: ENV['REDIS_URL'] }
+end
+
+Sidekiq::Queue['reverse_geocoding'].limit = 1 if Sidekiq.server? && DawarichSettings.photon_uses_komoot_io?

config/routes.rb

@@ -1,19 +1,34 @@
 # frozen_string_literal: true
 
+require 'sidekiq/web'
+
 Rails.application.routes.draw do
   mount ActionCable.server => '/cable'
   mount Rswag::Api::Engine => '/api-docs'
   mount Rswag::Ui::Engine => '/api-docs'
 
+  unless DawarichSettings.self_hosted?
+    Sidekiq::Web.use(Rack::Auth::Basic) do |username, password|
+      ActiveSupport::SecurityUtils.secure_compare(
+        ::Digest::SHA256.hexdigest(username),
+        ::Digest::SHA256.hexdigest(ENV['SIDEKIQ_USERNAME'])
+      ) &
+        ActiveSupport::SecurityUtils.secure_compare(
+          ::Digest::SHA256.hexdigest(password),
+          ::Digest::SHA256.hexdigest(ENV['SIDEKIQ_PASSWORD'])
+        )
+    end
+  end
+
   authenticate :user, lambda { |u|
     (u.admin? && DawarichSettings.self_hosted?) ||
       (u.admin? && ENV['SIDEKIQ_USERNAME'].present? && ENV['SIDEKIQ_PASSWORD'].present?)
   } do
-    mount MissionControl::Jobs::Engine, at: '/jobs'
+    mount Sidekiq::Web => '/sidekiq'
   end
 
-  # We want to return a nice error message if the user is not authorized to access Jobs
+  # We want to return a nice error message if the user is not authorized to access Sidekiq
-  match '/jobs' => redirect { |_, request|
+  match '/sidekiq' => redirect { |_, request|
                         request.flash[:error] = 'You are not authorized to perform this action.'
                         '/'
                       }, via: :get

config/sidekiq.yml

@@ -0,0 +1,10 @@
+---
+:concurrency: <%= ENV.fetch("BACKGROUND_PROCESSING_CONCURRENCY", 10) %>
+:queues:
+  - points
+  - default
+  - imports
+  - exports
+  - stats
+  - reverse_geocoding
+  - visit_suggesting

docker/Dockerfile.dev

@@ -62,6 +62,9 @@ RUN mkdir -p $APP_PATH/tmp && touch $APP_PATH/tmp/caching-dev.txt
 COPY ./docker/web-entrypoint.sh /usr/local/bin/web-entrypoint.sh
 RUN chmod +x /usr/local/bin/web-entrypoint.sh
 
+COPY ./docker/sidekiq-entrypoint.sh /usr/local/bin/sidekiq-entrypoint.sh
+RUN chmod +x /usr/local/bin/sidekiq-entrypoint.sh
+
 EXPOSE $RAILS_PORT
 
 ENTRYPOINT ["bundle", "exec"]

docker/Dockerfile.prod

@@ -61,6 +61,9 @@ RUN SECRET_KEY_BASE_DUMMY=1 bundle exec rake assets:precompile \
 COPY ./docker/web-entrypoint.sh /usr/local/bin/web-entrypoint.sh
 RUN chmod +x /usr/local/bin/web-entrypoint.sh
 
+COPY ./docker/sidekiq-entrypoint.sh /usr/local/bin/sidekiq-entrypoint.sh
+RUN chmod +x /usr/local/bin/sidekiq-entrypoint.sh
+
 EXPOSE $RAILS_PORT
 
 ENTRYPOINT [ "bundle", "exec" ]

docker/docker-compose.production.yml

@@ -1,6 +1,21 @@
 networks:
   dawarich:
 services:
+  dawarich_redis:
+    image: redis:7.4-alpine
+    container_name: dawarich_redis
+    command: redis-server
+    networks:
+      - dawarich
+    volumes:
+      - dawarich_redis_data:/var/shared/redis
+    restart: always
+    healthcheck:
+      test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ]
+      interval: 10s
+      retries: 5
+      start_period: 30s
+      timeout: 10s
   dawarich_db:
     image: postgis/postgis:17-3.5-alpine
     shm_size: 1G
@@ -41,6 +56,7 @@ services:
     restart: on-failure
     environment:
       RAILS_ENV: production
+      REDIS_URL: redis://dawarich_redis:6379/0
       DATABASE_HOST: dawarich_db
       DATABASE_PORT: 5432
       DATABASE_USERNAME: postgres
@@ -80,14 +96,69 @@ services:
       dawarich_db:
         condition: service_healthy
         restart: true
+    dawarich_redis:
+        condition: service_healthy
+        restart: true
     deploy:
       resources:
         limits:
           cpus: '0.50'    # Limit CPU usage to 50% of one core
           memory: '4G'    # Limit memory usage to 2GB
-
+  dawarich_sidekiq:
+    image: dawarich:prod
+    container_name: dawarich_sidekiq
+    volumes:
+      - dawarich_public:/var/app/public
+      - dawarich_watched:/var/app/tmp/imports/watched
+      - dawarich_storage:/var/app/storage
+    networks:
+      - dawarich
+    stdin_open: true
+    tty: true
+    entrypoint: sidekiq-entrypoint.sh
+    command: ['bundle', 'exec', 'sidekiq']
+    restart: on-failure
+    environment:
+      RAILS_ENV: production
+      REDIS_URL: redis://dawarich_redis:6379/0
+      DATABASE_HOST: dawarich_db
+      DATABASE_PORT: 5432
+      DATABASE_USERNAME: postgres
+      DATABASE_PASSWORD: password
+      DATABASE_NAME: dawarich_production
+      APPLICATION_HOSTS: localhost,::1,127.0.0.1
+      BACKGROUND_PROCESSING_CONCURRENCY: 10
+      APPLICATION_PROTOCOL: http
+      PROMETHEUS_EXPORTER_ENABLED: false
+      PROMETHEUS_EXPORTER_HOST: dawarich_app
+      PROMETHEUS_EXPORTER_PORT: 9394
+      SECRET_KEY_BASE: 1234567890
+      RAILS_LOG_TO_STDOUT: "true"
+      STORE_GEODATA: "true"
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "100m"
+        max-file: "5"
+    healthcheck:
+      test: [ "CMD-SHELL", "bundle exec sidekiqmon processes | grep $${HOSTNAME}" ]
+      interval: 10s
+      retries: 30
+      start_period: 30s
+      timeout: 10s
+    depends_on:
+      dawarich_db:
+        condition: service_healthy
+        restart: true
+      dawarich_redis:
+        condition: service_healthy
+        restart: true
+      dawarich_app:
+        condition: service_healthy
+        restart: true
 volumes:
   dawarich_db_data:
+  dawarich_redis_data:
   dawarich_public:
   dawarich_watched:
   dawarich_storage:

docker/docker-compose.yml

@@ -1,6 +1,21 @@
 networks:
   dawarich:
 services:
+  dawarich_redis:
+    image: redis:7.4-alpine
+    container_name: dawarich_redis
+    command: redis-server
+    networks:
+      - dawarich
+    volumes:
+      - dawarich_shared:/data
+    restart: always
+    healthcheck:
+      test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ]
+      interval: 10s
+      retries: 5
+      start_period: 30s
+      timeout: 10s
   dawarich_db:
     image: postgis/postgis:17-3.5-alpine
     shm_size: 1G
@@ -44,6 +59,7 @@ services:
     restart: on-failure
     environment:
       RAILS_ENV: development
+      REDIS_URL: redis://dawarich_redis:6379/0
       DATABASE_HOST: dawarich_db
       DATABASE_USERNAME: postgres
       DATABASE_PASSWORD: password
@@ -81,12 +97,64 @@ services:
       dawarich_db:
         condition: service_healthy
         restart: true
+    dawarich_redis:
+        condition: service_healthy
+        restart: true
     deploy:
       resources:
         limits:
           cpus: '0.50'    # Limit CPU usage to 50% of one core
           memory: '4G'    # Limit memory usage to 4GB
-
+  dawarich_sidekiq:
+    image: freikin/dawarich:latest
+    container_name: dawarich_sidekiq
+    volumes:
+      - dawarich_public:/var/app/public
+      - dawarich_watched:/var/app/tmp/imports/watched
+      - dawarich_storage:/var/app/storage
+    networks:
+      - dawarich
+    stdin_open: true
+    tty: true
+    entrypoint: sidekiq-entrypoint.sh
+    command: ['sidekiq']
+    restart: on-failure
+    environment:
+      RAILS_ENV: development
+      REDIS_URL: redis://dawarich_redis:6379/0
+      DATABASE_HOST: dawarich_db
+      DATABASE_USERNAME: postgres
+      DATABASE_PASSWORD: password
+      DATABASE_NAME: dawarich_development
+      APPLICATION_HOSTS: localhost
+      BACKGROUND_PROCESSING_CONCURRENCY: 10
+      APPLICATION_PROTOCOL: http
+      PROMETHEUS_EXPORTER_ENABLED: false
+      PROMETHEUS_EXPORTER_HOST: dawarich_app
+      PROMETHEUS_EXPORTER_PORT: 9394
+      SELF_HOSTED: "true"
+      STORE_GEODATA: "true"
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "100m"
+        max-file: "5"
+    healthcheck:
+      test: [ "CMD-SHELL", "bundle exec sidekiqmon processes | grep $${HOSTNAME}" ]
+      interval: 10s
+      retries: 30
+      start_period: 30s
+      timeout: 10s
+    depends_on:
+      dawarich_db:
+        condition: service_healthy
+        restart: true
+      dawarich_redis:
+        condition: service_healthy
+        restart: true
+      dawarich_app:
+        condition: service_healthy
+        restart: true
 volumes:
   dawarich_db_data:
   dawarich_sqlite_data:

docker/sidekiq-entrypoint.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+
+unset BUNDLE_PATH
+unset BUNDLE_BIN
+
+set -e
+
+echo "⚠️ Starting Sidekiq in $RAILS_ENV environment ⚠️"
+
+# Parse DATABASE_URL if present, otherwise use individual variables
+if [ -n "$DATABASE_URL" ]; then
+  # Extract components from DATABASE_URL
+  DATABASE_HOST=$(echo $DATABASE_URL | awk -F[@/] '{print $4}')
+  DATABASE_PORT=$(echo $DATABASE_URL | awk -F[@/:] '{print $5}')
+  DATABASE_USERNAME=$(echo $DATABASE_URL | awk -F[:/@] '{print $4}')
+  DATABASE_PASSWORD=$(echo $DATABASE_URL | awk -F[:/@] '{print $5}')
+  DATABASE_NAME=$(echo $DATABASE_URL | awk -F[@/] '{print $5}')
+else
+  # Use existing environment variables
+  DATABASE_HOST=${DATABASE_HOST}
+  DATABASE_PORT=${DATABASE_PORT}
+  DATABASE_USERNAME=${DATABASE_USERNAME}
+  DATABASE_PASSWORD=${DATABASE_PASSWORD}
+  DATABASE_NAME=${DATABASE_NAME}
+fi
+
+# Wait for the database to become available
+echo "⏳ Waiting for database to be ready..."
+until PGPASSWORD=$DATABASE_PASSWORD psql -h "$DATABASE_HOST" -p "$DATABASE_PORT" -U "$DATABASE_USERNAME" -d "$DATABASE_NAME" -c '\q'; do
+  >&2 echo "Postgres is unavailable - retrying..."
+  sleep 2
+done
+echo "✅ PostgreSQL is ready!"
+
+# run sidekiq
+bundle exec sidekiq

docs/How_to_install_Dawarich_in_k8s.md

@@ -6,7 +6,7 @@
 
 - Kubernetes cluster and basic kubectl knowledge.
 - Some persistent storage class prepared, in this example, Longhorn.
-- Working Postgres instance. In this example Postgres lives in 'db' namespace.
+- Working Postgres and Redis instances. In this example Postgres lives in 'db' namespace and Redis in 'redis' namespace.
 - Ngingx ingress controller with Letsencrypt integeation.
 - This example uses 'example.com' as a domain name, you want to change it to your own.
 - This will work on IPv4 and IPv6 Single Stack clusters, as well as Dual Stack deployments.
@@ -80,6 +80,8 @@ spec:
               value: "Europe/Prague"
             - name: RAILS_ENV
               value: development
+            - name: REDIS_URL
+              value: redis://redis-master.redis.svc.cluster.local:6379/10
             - name: DATABASE_HOST
               value: postgres-postgresql.db.svc.cluster.local
             - name: DATABASE_PORT
@@ -126,10 +128,73 @@ spec:
               cpu: "2000m"
           ports:
           - containerPort: 3000
+        - name: dawarich-sidekiq
+          env:
+            - name: RAILS_ENV
+              value: development
+            - name: REDIS_URL
+              value: redis://redis-master.redis.svc.cluster.local:6379/10
+            - name: DATABASE_HOST
+              value: postgres-postgresql.db.svc.cluster.local
+            - name: DATABASE_PORT
+              value: "5432"
+            - name: DATABASE_USERNAME
+              value: postgres
+            - name: DATABASE_PASSWORD
+              value: Password123!
+            - name: DATABASE_NAME
+              value: dawarich_development
+            - name: RAILS_MIN_THREADS
+              value: "5"
+            - name: RAILS_MAX_THREADS
+              value: "10"
+            - name: BACKGROUND_PROCESSING_CONCURRENCY
+              value: "20"
+            - name: APPLICATION_HOST
+              value: localhost
+            - name: APPLICATION_HOSTS
+              value: "dawarich.example.com, localhost"
+            - name: APPLICATION_PROTOCOL
+              value: http
+            - name: PHOTON_API_HOST
+              value: photon.komoot.io
+            - name: PHOTON_API_USE_HTTPS
+              value: "true"
+          image: freikin/dawarich:latest
+          imagePullPolicy: Always
+          volumeMounts:
+            - mountPath: /var/app/public
+              name: public
+            - mountPath: /var/app/tmp/imports/watched
+              name: watched
+          command:
+            - "sidekiq-entrypoint.sh"
+          args:
+            - "bundle exec sidekiq"
+          resources:
+            requests:
+              memory: "1Gi"
+              cpu: "250m"
+            limits:
+              memory: "3Gi"
+              cpu: "1500m"
+          livenessProbe:
+            httpGet:
+              path: /api/v1/health
+              port: 3000
+            initialDelaySeconds: 60
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 3000
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            timeoutSeconds: 3
+            failureThreshold: 3
       volumes:
-        - name: gem-cache
-          persistentVolumeClaim:
-            claimName: gem-cache
         - name: public
           persistentVolumeClaim:
             claimName: public

docs/How_to_install_Dawarich_on_Synology.md

@@ -29,7 +29,7 @@ If you don't want to use dedicated share for projects installed by docker skip i
 ### Dawarich root folder
 1. Open your [Docker root folder](#docker-root-share) in **File station**.
 2. Create new folder **dawarich** and open it.
-3. Create folders **db_data**, **db_shared** and **public** in **dawarich** folder.
+3. Create folders **redis**, **db_data**, **db_shared** and **public** in **dawarich** folder.
 4. Copy [docker compose](synology/docker-compose.yml) and [.env](synology/.env) files form **synology** repo folder into **dawarich** folder on your synology.
 
 # Installation

docs/how_to_setup_reverse_proxy.md

@@ -17,6 +17,17 @@ dawarich_app:
       APPLICATION_HOSTS: "yourhost.com,www.yourhost.com,127.0.0.1" <-- Edit this
 ```
 
+```yaml
+dawarich_sidekiq:
+    image: freikin/dawarich:latest
+    container_name: dawarich_sidekiq
+    ...
+    environment:
+      ...
+      APPLICATION_HOSTS: "yourhost.com,www.yourhost.com,127.0.0.1" <-- Edit this
+      ...
+```
+
 For a Synology install, refer to **[Synology Install Tutorial](How_to_install_Dawarich_on_Synology.md)**. In this page, it is explained how to set the APPLICATION_HOSTS environment variable.
 
 ### Virtual Host

docs/synology/docker-compose.yml

@@ -1,6 +1,13 @@
 version: '3'
 
 services:
+  dawarich_redis:
+    image: redis:7.4-alpine
+    container_name: dawarich_redis
+    command: redis-server
+    restart: unless-stopped
+    volumes:
+      - ./redis:/var/shared/redis
   dawarich_db:
     image: postgis/postgis:17-3.5-alpine
     container_name: dawarich_db
@@ -17,6 +24,7 @@ services:
     container_name: dawarich_app
     depends_on:
       - dawarich_db
+      - dawarich_redis
     stdin_open: true
     tty: true
     entrypoint: web-entrypoint.sh
@@ -29,3 +37,19 @@ services:
       - ./app_storage:/var/app/storage
     ports:
       - 32568:3000
+
+  dawarich_sidekiq:
+    image: freikin/dawarich:latest
+    container_name: dawarich_sidekiq
+    depends_on:
+      - dawarich_db
+      - dawarich_redis
+      - dawarich_app
+    entrypoint: sidekiq-entrypoint.sh
+    command: ['sidekiq']
+    restart: unless-stopped
+    env_file:
+      - .env
+    volumes:
+      - ./public:/var/app/public
+      - ./app_storage:/var/app/storage

spec/rails_helper.rb

@@ -7,6 +7,7 @@ require_relative '../config/environment'
 abort('The Rails environment is running in production mode!') if Rails.env.production?
 require 'rspec/rails'
 require 'rswag/specs'
+require 'sidekiq/testing'
 require 'super_diff/rspec-rails'
 
 require 'rake'

spec/requests/sidekiq_spec.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+require 'sidekiq/web'
+
+RSpec.describe '/sidekiq', type: :request do
+  before do
+    # Allow any ENV key to be accessed and return nil by default
+    allow(ENV).to receive(:[]).and_return(nil)
+
+    # Stub Sidekiq::Web with a simple Rack app for testing
+    allow(Sidekiq::Web).to receive(:call) do |_env|
+      [200, { 'Content-Type' => 'text/html' }, ['Sidekiq Web UI']]
+    end
+  end
+
+  context 'when Dawarich is in self-hosted mode' do
+    before do
+      allow(DawarichSettings).to receive(:self_hosted?).and_return(true)
+      allow(ENV).to receive(:[]).with('SIDEKIQ_USERNAME').and_return(nil)
+      allow(ENV).to receive(:[]).with('SIDEKIQ_PASSWORD').and_return(nil)
+    end
+
+    context 'when user is not authenticated' do
+      it 'redirects to sign in page' do
+        get sidekiq_url
+
+        expect(response).to redirect_to('/users/sign_in')
+      end
+    end
+
+    context 'when user is authenticated' do
+      context 'when user is not admin' do
+        before { sign_in create(:user) }
+
+        it 'redirects to root page' do
+          get sidekiq_url
+
+          expect(response).to redirect_to(root_url)
+        end
+
+        it 'shows flash message' do
+          get sidekiq_url
+
+          expect(flash[:error]).to eq('You are not authorized to perform this action.')
+        end
+      end
+
+      context 'when user is admin' do
+        before { sign_in create(:user, :admin) }
+
+        it 'renders a successful response' do
+          get sidekiq_url
+
+          expect(response).to be_successful
+        end
+      end
+    end
+  end
+
+  context 'when Dawarich is not in self-hosted mode' do
+    before do
+      allow(DawarichSettings).to receive(:self_hosted?).and_return(false)
+      allow(ENV).to receive(:[]).with('SIDEKIQ_USERNAME').and_return(nil)
+      allow(ENV).to receive(:[]).with('SIDEKIQ_PASSWORD').and_return(nil)
+      Rails.application.reload_routes!
+    end
+
+    context 'when user is not authenticated' do
+      it 'redirects to sign in page' do
+        get sidekiq_url
+
+        expect(response).to redirect_to('/users/sign_in')
+      end
+    end
+
+    context 'when user is authenticated' do
+      before { sign_in create(:user, :admin) }
+
+      it 'redirects to root page' do
+        get sidekiq_url
+
+        expect(response).to redirect_to(root_url)
+        expect(flash[:error]).to eq('You are not authorized to perform this action.')
+      end
+    end
+  end
+
+  context 'when SIDEKIQ_USERNAME and SIDEKIQ_PASSWORD are set' do
+    before do
+      allow(DawarichSettings).to receive(:self_hosted?).and_return(false)
+      allow(ENV).to receive(:[]).with('SIDEKIQ_USERNAME').and_return('admin')
+      allow(ENV).to receive(:[]).with('SIDEKIQ_PASSWORD').and_return('password')
+    end
+
+    context 'when user is not authenticated' do
+      it 'redirects to sign in page' do
+        get sidekiq_url
+
+        expect(response).to redirect_to('/users/sign_in')
+      end
+    end
+
+    context 'when user is not admin' do
+      before { sign_in create(:user) }
+
+      it 'redirects to root page' do
+        get sidekiq_url
+
+        expect(response).to redirect_to(root_url)
+        expect(flash[:error]).to eq('You are not authorized to perform this action.')
+      end
+    end
+
+    context 'when user is admin' do
+      before { sign_in create(:user, :admin) }
+
+      it 'renders a successful response' do
+        get sidekiq_url
+
+        expect(response).to be_successful
+      end
+    end
+  end
+end

spec/services/imports/watcher_spec.rb

@@ -12,6 +12,8 @@ RSpec.describe Imports::Watcher do
       stub_const('Imports::Watcher::WATCHED_DIR_PATH', watched_dir_path)
     end
 
+    after { Sidekiq::Testing.fake! }
+
     context 'when user exists' do
       let!(:user) { create(:user, email: 'user@domain.com') }