From b76602d9c8a80ce5d88494b86f7df802f7de02d0 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Mon, 9 Jun 2025 13:39:25 +0200 Subject: [PATCH] Return sidekiq and redis to Dawarich --- .circleci/config.yml | 6 +- .devcontainer/docker-compose.yml | 16 +++ .env.development | 1 + .env.test | 1 + .github/workflows/ci.yml | 7 + DEVELOPMENT.md | 6 + Gemfile | 5 + Gemfile.lock | 27 ++++ app/jobs/app_version_checking_job.rb | 1 + app/jobs/area_visits_calculating_job.rb | 1 + .../area_visits_calculation_scheduling_job.rb | 1 + app/jobs/bulk_visits_suggesting_job.rb | 1 + app/jobs/import/google_takeout_job.rb | 1 + app/jobs/import/photoprism_geodata_job.rb | 1 + app/jobs/import/watcher_job.rb | 1 + app/jobs/visit_suggesting_job.rb | 1 + app/services/tasks/imports/google_records.rb | 2 +- .../notifications/_notification.html.erb | 2 +- .../settings/background_jobs/index.html.erb | 2 +- config/application.rb | 2 + config/initializers/geocoder.rb | 2 +- config/initializers/sidekiq.rb | 42 +++--- config/routes.rb | 21 ++- config/sidekiq.yml | 10 ++ docker/Dockerfile.dev | 3 + docker/Dockerfile.prod | 3 + docker/docker-compose.production.yml | 73 +++++++++- docker/docker-compose.yml | 70 +++++++++- docker/sidekiq-entrypoint.sh | 36 +++++ docs/How_to_install_Dawarich_in_k8s.md | 73 +++++++++- docs/How_to_install_Dawarich_on_Synology.md | 2 +- docs/how_to_setup_reverse_proxy.md | 11 ++ docs/synology/docker-compose.yml | 24 ++++ spec/rails_helper.rb | 1 + spec/requests/sidekiq_spec.rb | 125 ++++++++++++++++++ spec/services/imports/watcher_spec.rb | 2 + 36 files changed, 546 insertions(+), 37 deletions(-) create mode 100644 config/sidekiq.yml create mode 100644 docker/sidekiq-entrypoint.sh create mode 100644 spec/requests/sidekiq_spec.rb diff --git a/.circleci/config.yml b/.circleci/config.yml index e7410a62..13f89c17 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,16 +16,12 @@ jobs: DATABASE_USERNAME: postgres DATABASE_PASSWORD: mysecretpassword DATABASE_PORT: 5432 - QUEUE_DATABASE_HOST: localhost - QUEUE_DATABASE_NAME: dawarich_test_queue - QUEUE_DATABASE_USERNAME: postgres - QUEUE_DATABASE_PASSWORD: mysecretpassword - QUEUE_DATABASE_PORT: 5432 - image: cimg/postgres:13.3-postgis environment: POSTGRES_USER: postgres POSTGRES_DB: dawarich_test POSTGRES_PASSWORD: mysecretpassword + - image: redis:7.0 - image: selenium/standalone-chrome:latest name: chrome environment: diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index cf658aa5..eb632340 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -20,6 +20,7 @@ services: tty: true environment: RAILS_ENV: development + REDIS_URL: redis://dawarich_redis:6379/0 DATABASE_HOST: dawarich_db DATABASE_USERNAME: postgres DATABASE_PASSWORD: password @@ -40,6 +41,21 @@ services: PROMETHEUS_EXPORTER_ENABLED: false PROMETHEUS_EXPORTER_HOST: 0.0.0.0 PROMETHEUS_EXPORTER_PORT: 9394 + dawarich_redis: + image: redis:7.4-alpine + container_name: dawarich_redis + command: redis-server + networks: + - dawarich + volumes: + - dawarich_shared:/data + restart: always + healthcheck: + test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ] + interval: 10s + retries: 5 + start_period: 30s + timeout: 10s dawarich_db: image: postgis/postgis:17-3.5-alpine container_name: dawarich_db diff --git a/.env.development b/.env.development index 8e85d36c..8aeb3141 100644 --- a/.env.development +++ b/.env.development @@ -3,3 +3,4 @@ DATABASE_USERNAME=postgres DATABASE_PASSWORD=password DATABASE_NAME=dawarich_development DATABASE_PORT=5432 +REDIS_URL=redis://localhost:6379/1 diff --git a/.env.test b/.env.test index fcfeae00..fea48769 100644 --- a/.env.test +++ b/.env.test @@ -3,3 +3,4 @@ DATABASE_USERNAME=postgres DATABASE_PASSWORD=password DATABASE_NAME=dawarich_test DATABASE_PORT=5432 +REDIS_URL=redis://localhost:6379/1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0971ad40..bd322ea9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,10 @@ jobs: ports: - 5432:5432 options: --health-cmd="pg_isready" --health-interval=10s --health-timeout=5s --health-retries=3 + redis: + image: redis + ports: + - 6379:6379 steps: - name: Install packages @@ -53,12 +57,14 @@ jobs: env: RAILS_ENV: test DATABASE_URL: postgres://postgres:postgres@localhost:5432 + REDIS_URL: redis://localhost:6379/1 run: bin/rails db:setup - name: Run main tests (excluding system tests) env: RAILS_ENV: test DATABASE_URL: postgres://postgres:postgres@localhost:5432 + REDIS_URL: redis://localhost:6379/1 run: | bundle exec rspec --exclude-pattern "spec/system/**/*_spec.rb" || (cat log/test.log && exit 1) @@ -66,6 +72,7 @@ jobs: env: RAILS_ENV: test DATABASE_URL: postgres://postgres:postgres@localhost:5432 + REDIS_URL: redis://localhost:6379/1 run: | bundle exec rspec spec/system/ || (cat log/test.log && exit 1) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index f5551a76..8b1b6a97 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -7,6 +7,12 @@ Now you can create/prepare the Database (this need to be done once): bundle exec rails db:prepare ``` +Afterwards you can run sidekiq: +```bash +bundle exec sidekiq + +``` + And in a second terminal the dawarich-app: ```bash bundle exec bin/dev diff --git a/Gemfile b/Gemfile index 504b4811..688eb883 100644 --- a/Gemfile +++ b/Gemfile @@ -28,6 +28,7 @@ gem 'activerecord-postgis-adapter' gem 'puma' gem 'pundit' gem 'rails', '~> 8.0' +gem 'redis' gem 'rexml' gem 'rgeo' gem 'rgeo-activerecord' @@ -38,6 +39,9 @@ gem 'sentry-ruby' gem 'sentry-rails' gem 'sqlite3', '~> 2.6' gem 'stackprof' +gem 'sidekiq' +gem 'sidekiq-cron' +gem 'sidekiq-limit_fetch' gem 'sprockets-rails' gem 'stimulus-rails' gem 'strong_migrations' @@ -64,6 +68,7 @@ end group :test do gem 'capybara' + gem 'fakeredis' gem 'selenium-webdriver' gem 'shoulda-matchers' gem 'simplecov', require: false diff --git a/Gemfile.lock b/Gemfile.lock index 8fe529cf..8777986d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -134,6 +134,9 @@ GEM bigdecimal rexml crass (1.0.6) + cronex (0.15.0) + tzinfo + unicode (>= 0.4.4.5) csv (3.3.4) data_migrate (11.3.0) activerecord (>= 6.1) @@ -166,6 +169,7 @@ GEM factory_bot_rails (6.4.4) factory_bot (~> 6.5) railties (>= 5.0.0) + fakeredis (0.1.4) ffaker (2.24.0) foreman (0.88.1) fugit (1.11.1) @@ -350,6 +354,10 @@ GEM rdoc (6.14.0) erb psych (>= 4.0.0) + redis (5.4.0) + redis-client (>= 0.22.0) + redis-client (0.24.0) + connection_pool regexp_parser (2.10.0) reline (0.6.1) io-console (~> 0.5) @@ -431,6 +439,19 @@ GEM concurrent-ruby (~> 1.0, >= 1.0.2) shoulda-matchers (6.5.0) activesupport (>= 5.2.0) + sidekiq (8.0.4) + connection_pool (>= 2.5.0) + json (>= 2.9.0) + logger (>= 1.6.2) + rack (>= 3.1.0) + redis-client (>= 0.23.2) + sidekiq-cron (2.3.0) + cronex (>= 0.13.0) + fugit (~> 1.8, >= 1.11.1) + globalid (>= 1.0.1) + sidekiq (>= 6.5.0) + sidekiq-limit_fetch (4.4.1) + sidekiq (>= 6) simplecov (0.22.0) docile (~> 1.1) simplecov-html (~> 0.11) @@ -492,6 +513,7 @@ GEM railties (>= 7.1.0) tzinfo (2.0.6) concurrent-ruby (~> 1.0) + unicode (0.4.4.5) unicode-display_width (3.1.4) unicode-emoji (~> 4.0, >= 4.0.4) unicode-emoji (4.0.4) @@ -537,6 +559,7 @@ DEPENDENCIES devise dotenv-rails factory_bot_rails + fakeredis ffaker foreman geocoder! @@ -556,6 +579,7 @@ DEPENDENCIES puma pundit rails (~> 8.0) + redis rexml rgeo rgeo-activerecord @@ -569,6 +593,9 @@ DEPENDENCIES sentry-rails sentry-ruby shoulda-matchers + sidekiq + sidekiq-cron + sidekiq-limit_fetch simplecov solid_cable (~> 3.0) solid_cache (= 1.0.7) diff --git a/app/jobs/app_version_checking_job.rb b/app/jobs/app_version_checking_job.rb index a6fc2d9b..2463326d 100644 --- a/app/jobs/app_version_checking_job.rb +++ b/app/jobs/app_version_checking_job.rb @@ -2,6 +2,7 @@ class AppVersionCheckingJob < ApplicationJob queue_as :default + sidekiq_options retry: false def perform Rails.cache.delete(CheckAppVersion::VERSION_CACHE_KEY) diff --git a/app/jobs/area_visits_calculating_job.rb b/app/jobs/area_visits_calculating_job.rb index fe74ff9d..95850286 100644 --- a/app/jobs/area_visits_calculating_job.rb +++ b/app/jobs/area_visits_calculating_job.rb @@ -2,6 +2,7 @@ class AreaVisitsCalculatingJob < ApplicationJob queue_as :default + sidekiq_options retry: false def perform(user_id) user = User.find(user_id) diff --git a/app/jobs/area_visits_calculation_scheduling_job.rb b/app/jobs/area_visits_calculation_scheduling_job.rb index a1addc82..db4c5d3e 100644 --- a/app/jobs/area_visits_calculation_scheduling_job.rb +++ b/app/jobs/area_visits_calculation_scheduling_job.rb @@ -2,6 +2,7 @@ class AreaVisitsCalculationSchedulingJob < ApplicationJob queue_as :default + sidekiq_options retry: false def perform User.find_each { AreaVisitsCalculatingJob.perform_later(_1.id) } diff --git a/app/jobs/bulk_visits_suggesting_job.rb b/app/jobs/bulk_visits_suggesting_job.rb index a1ff2ae4..54174bca 100644 --- a/app/jobs/bulk_visits_suggesting_job.rb +++ b/app/jobs/bulk_visits_suggesting_job.rb @@ -4,6 +4,7 @@ # with the default timespan of 1 day. class BulkVisitsSuggestingJob < ApplicationJob queue_as :visit_suggesting + sidekiq_options retry: false # Passing timespan of more than 3 years somehow results in duplicated Places def perform(start_at: 1.day.ago.beginning_of_day, end_at: 1.day.ago.end_of_day, user_ids: []) diff --git a/app/jobs/import/google_takeout_job.rb b/app/jobs/import/google_takeout_job.rb index f52e61c3..02702cf7 100644 --- a/app/jobs/import/google_takeout_job.rb +++ b/app/jobs/import/google_takeout_job.rb @@ -2,6 +2,7 @@ class Import::GoogleTakeoutJob < ApplicationJob queue_as :imports + sidekiq_options retry: false def perform(import_id, locations, current_index) locations_batch = Oj.load(locations) diff --git a/app/jobs/import/photoprism_geodata_job.rb b/app/jobs/import/photoprism_geodata_job.rb index 161667d5..7aa2d27e 100644 --- a/app/jobs/import/photoprism_geodata_job.rb +++ b/app/jobs/import/photoprism_geodata_job.rb @@ -2,6 +2,7 @@ class Import::PhotoprismGeodataJob < ApplicationJob queue_as :imports + sidekiq_options retry: false def perform(user_id) user = User.find(user_id) diff --git a/app/jobs/import/watcher_job.rb b/app/jobs/import/watcher_job.rb index f25c95a8..a2f6676f 100644 --- a/app/jobs/import/watcher_job.rb +++ b/app/jobs/import/watcher_job.rb @@ -2,6 +2,7 @@ class Import::WatcherJob < ApplicationJob queue_as :imports + sidekiq_options retry: false def perform return unless DawarichSettings.self_hosted? diff --git a/app/jobs/visit_suggesting_job.rb b/app/jobs/visit_suggesting_job.rb index 87a2adc7..2659d2d3 100644 --- a/app/jobs/visit_suggesting_job.rb +++ b/app/jobs/visit_suggesting_job.rb @@ -2,6 +2,7 @@ class VisitSuggestingJob < ApplicationJob queue_as :visit_suggesting + sidekiq_options retry: false # Passing timespan of more than 3 years somehow results in duplicated Places def perform(user_id:, start_at:, end_at:) diff --git a/app/services/tasks/imports/google_records.rb b/app/services/tasks/imports/google_records.rb index 7f888bc2..70b5d389 100644 --- a/app/services/tasks/imports/google_records.rb +++ b/app/services/tasks/imports/google_records.rb @@ -54,6 +54,6 @@ class Tasks::Imports::GoogleRecords end def log_success - Rails.logger.info("Imported #{@file_path} for #{@user.email} successfully! Wait for the processing to finish. You can check the status of the import in the Jobs UI (http:///jobs).") + Rails.logger.info("Imported #{@file_path} for #{@user.email} successfully! Wait for the processing to finish. You can check the status of the import in the Sidekiq UI (http:///sidekiq).") end end diff --git a/app/views/notifications/_notification.html.erb b/app/views/notifications/_notification.html.erb index 1ce36922..62a32b81 100644 --- a/app/views/notifications/_notification.html.erb +++ b/app/views/notifications/_notification.html.erb @@ -11,7 +11,7 @@ <% if notification.error? %>
- Please, when reporting a bug to Github Issues, don't forget to include logs from dawarich_app docker container. Thank you! + Please, when reporting a bug to Github Issues, don't forget to include logs from dawarich_app and dawarich_sidekiq docker containers. Thank you!
<% end %> diff --git a/app/views/settings/background_jobs/index.html.erb b/app/views/settings/background_jobs/index.html.erb index f31335ae..ebdaaa2c 100644 --- a/app/views/settings/background_jobs/index.html.erb +++ b/app/views/settings/background_jobs/index.html.erb @@ -45,7 +45,7 @@

Background Jobs Dashboard

This will open the background jobs dashboard in a new tab.

- <%= link_to 'Open Dashboard', mission_control_jobs_url, target: '_blank', class: 'btn btn-primary' %> + <%= link_to 'Open Dashboard', '/sidekiq', target: '_blank', class: 'btn btn-primary' %>
diff --git a/config/application.rb b/config/application.rb index a76fdc15..3d2dd0be 100644 --- a/config/application.rb +++ b/config/application.rb @@ -34,5 +34,7 @@ module Dawarich g.routing_specs false g.helper_specs false end + + config.active_job.queue_adapter = :sidekiq end end diff --git a/config/initializers/geocoder.rb b/config/initializers/geocoder.rb index 60b61bed..e38248d0 100644 --- a/config/initializers/geocoder.rb +++ b/config/initializers/geocoder.rb @@ -4,7 +4,7 @@ settings = { debug_mode: true, timeout: 5, units: :km, - cache: Geocoder::CacheStore::Generic.new(Rails.cache, {}), + cache: cache: Redis.new, always_raise: :all, http_headers: { 'User-Agent' => "Dawarich #{APP_VERSION} (https://dawarich.app)" diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index 47d89146..6b262868 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -1,24 +1,30 @@ # frozen_string_literal: true -# Sidekiq.configure_server do |config| -# if ENV['PROMETHEUS_EXPORTER_ENABLED'].to_s == 'true' -# require 'prometheus_exporter/instrumentation' +Sidekiq.configure_server do |config| + config.redis = { url: ENV['REDIS_URL'] } + config.logger = Sidekiq::Logger.new($stdout) -# # Add middleware for collecting job-level metrics -# config.server_middleware do |chain| -# chain.add PrometheusExporter::Instrumentation::Sidekiq -# end + if ENV['PROMETHEUS_EXPORTER_ENABLED'].to_s == 'true' + require 'prometheus_exporter/instrumentation' + # Add middleware for collecting job-level metrics + config.server_middleware do |chain| + chain.add PrometheusExporter::Instrumentation::Sidekiq + end -# # Capture metrics for failed jobs -# config.death_handlers << PrometheusExporter::Instrumentation::Sidekiq.death_handler + # Capture metrics for failed jobs + config.death_handlers << PrometheusExporter::Instrumentation::Sidekiq.death_handler -# # Start Prometheus instrumentation -# config.on :startup do -# PrometheusExporter::Instrumentation::SidekiqProcess.start -# PrometheusExporter::Instrumentation::SidekiqQueue.start -# PrometheusExporter::Instrumentation::SidekiqStats.start -# end -# end -# end + # Start Prometheus instrumentation + config.on :startup do + PrometheusExporter::Instrumentation::SidekiqProcess.start + PrometheusExporter::Instrumentation::SidekiqQueue.start + PrometheusExporter::Instrumentation::SidekiqStats.start + end + end +end -# Sidekiq::Queue['reverse_geocoding'].limit = 1 if Sidekiq.server? && DawarichSettings.photon_uses_komoot_io? +Sidekiq.configure_client do |config| + config.redis = { url: ENV['REDIS_URL'] } +end + +Sidekiq::Queue['reverse_geocoding'].limit = 1 if Sidekiq.server? && DawarichSettings.photon_uses_komoot_io? diff --git a/config/routes.rb b/config/routes.rb index 011cd4e3..1a03af7a 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,19 +1,34 @@ # frozen_string_literal: true +require 'sidekiq/web' + Rails.application.routes.draw do mount ActionCable.server => '/cable' mount Rswag::Api::Engine => '/api-docs' mount Rswag::Ui::Engine => '/api-docs' + unless DawarichSettings.self_hosted? + Sidekiq::Web.use(Rack::Auth::Basic) do |username, password| + ActiveSupport::SecurityUtils.secure_compare( + ::Digest::SHA256.hexdigest(username), + ::Digest::SHA256.hexdigest(ENV['SIDEKIQ_USERNAME']) + ) & + ActiveSupport::SecurityUtils.secure_compare( + ::Digest::SHA256.hexdigest(password), + ::Digest::SHA256.hexdigest(ENV['SIDEKIQ_PASSWORD']) + ) + end + end + authenticate :user, lambda { |u| (u.admin? && DawarichSettings.self_hosted?) || (u.admin? && ENV['SIDEKIQ_USERNAME'].present? && ENV['SIDEKIQ_PASSWORD'].present?) } do - mount MissionControl::Jobs::Engine, at: '/jobs' + mount Sidekiq::Web => '/sidekiq' end - # We want to return a nice error message if the user is not authorized to access Jobs - match '/jobs' => redirect { |_, request| + # We want to return a nice error message if the user is not authorized to access Sidekiq + match '/sidekiq' => redirect { |_, request| request.flash[:error] = 'You are not authorized to perform this action.' '/' }, via: :get diff --git a/config/sidekiq.yml b/config/sidekiq.yml new file mode 100644 index 00000000..7bde1468 --- /dev/null +++ b/config/sidekiq.yml @@ -0,0 +1,10 @@ +--- +:concurrency: <%= ENV.fetch("BACKGROUND_PROCESSING_CONCURRENCY", 10) %> +:queues: + - points + - default + - imports + - exports + - stats + - reverse_geocoding + - visit_suggesting diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index d3c7f1cd..aed33719 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -62,6 +62,9 @@ RUN mkdir -p $APP_PATH/tmp && touch $APP_PATH/tmp/caching-dev.txt COPY ./docker/web-entrypoint.sh /usr/local/bin/web-entrypoint.sh RUN chmod +x /usr/local/bin/web-entrypoint.sh +COPY ./docker/sidekiq-entrypoint.sh /usr/local/bin/sidekiq-entrypoint.sh +RUN chmod +x /usr/local/bin/sidekiq-entrypoint.sh + EXPOSE $RAILS_PORT ENTRYPOINT ["bundle", "exec"] diff --git a/docker/Dockerfile.prod b/docker/Dockerfile.prod index 12db5c06..e5fd1d61 100644 --- a/docker/Dockerfile.prod +++ b/docker/Dockerfile.prod @@ -61,6 +61,9 @@ RUN SECRET_KEY_BASE_DUMMY=1 bundle exec rake assets:precompile \ COPY ./docker/web-entrypoint.sh /usr/local/bin/web-entrypoint.sh RUN chmod +x /usr/local/bin/web-entrypoint.sh +COPY ./docker/sidekiq-entrypoint.sh /usr/local/bin/sidekiq-entrypoint.sh +RUN chmod +x /usr/local/bin/sidekiq-entrypoint.sh + EXPOSE $RAILS_PORT ENTRYPOINT [ "bundle", "exec" ] diff --git a/docker/docker-compose.production.yml b/docker/docker-compose.production.yml index 40ce7c74..37aeb19a 100644 --- a/docker/docker-compose.production.yml +++ b/docker/docker-compose.production.yml @@ -1,6 +1,21 @@ networks: dawarich: services: + dawarich_redis: + image: redis:7.4-alpine + container_name: dawarich_redis + command: redis-server + networks: + - dawarich + volumes: + - dawarich_redis_data:/var/shared/redis + restart: always + healthcheck: + test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ] + interval: 10s + retries: 5 + start_period: 30s + timeout: 10s dawarich_db: image: postgis/postgis:17-3.5-alpine shm_size: 1G @@ -41,6 +56,7 @@ services: restart: on-failure environment: RAILS_ENV: production + REDIS_URL: redis://dawarich_redis:6379/0 DATABASE_HOST: dawarich_db DATABASE_PORT: 5432 DATABASE_USERNAME: postgres @@ -80,14 +96,69 @@ services: dawarich_db: condition: service_healthy restart: true + dawarich_redis: + condition: service_healthy + restart: true deploy: resources: limits: cpus: '0.50' # Limit CPU usage to 50% of one core memory: '4G' # Limit memory usage to 2GB - + dawarich_sidekiq: + image: dawarich:prod + container_name: dawarich_sidekiq + volumes: + - dawarich_public:/var/app/public + - dawarich_watched:/var/app/tmp/imports/watched + - dawarich_storage:/var/app/storage + networks: + - dawarich + stdin_open: true + tty: true + entrypoint: sidekiq-entrypoint.sh + command: ['bundle', 'exec', 'sidekiq'] + restart: on-failure + environment: + RAILS_ENV: production + REDIS_URL: redis://dawarich_redis:6379/0 + DATABASE_HOST: dawarich_db + DATABASE_PORT: 5432 + DATABASE_USERNAME: postgres + DATABASE_PASSWORD: password + DATABASE_NAME: dawarich_production + APPLICATION_HOSTS: localhost,::1,127.0.0.1 + BACKGROUND_PROCESSING_CONCURRENCY: 10 + APPLICATION_PROTOCOL: http + PROMETHEUS_EXPORTER_ENABLED: false + PROMETHEUS_EXPORTER_HOST: dawarich_app + PROMETHEUS_EXPORTER_PORT: 9394 + SECRET_KEY_BASE: 1234567890 + RAILS_LOG_TO_STDOUT: "true" + STORE_GEODATA: "true" + logging: + driver: "json-file" + options: + max-size: "100m" + max-file: "5" + healthcheck: + test: [ "CMD-SHELL", "bundle exec sidekiqmon processes | grep $${HOSTNAME}" ] + interval: 10s + retries: 30 + start_period: 30s + timeout: 10s + depends_on: + dawarich_db: + condition: service_healthy + restart: true + dawarich_redis: + condition: service_healthy + restart: true + dawarich_app: + condition: service_healthy + restart: true volumes: dawarich_db_data: + dawarich_redis_data: dawarich_public: dawarich_watched: dawarich_storage: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 38faea53..220719fc 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,6 +1,21 @@ networks: dawarich: services: + dawarich_redis: + image: redis:7.4-alpine + container_name: dawarich_redis + command: redis-server + networks: + - dawarich + volumes: + - dawarich_shared:/data + restart: always + healthcheck: + test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ] + interval: 10s + retries: 5 + start_period: 30s + timeout: 10s dawarich_db: image: postgis/postgis:17-3.5-alpine shm_size: 1G @@ -44,6 +59,7 @@ services: restart: on-failure environment: RAILS_ENV: development + REDIS_URL: redis://dawarich_redis:6379/0 DATABASE_HOST: dawarich_db DATABASE_USERNAME: postgres DATABASE_PASSWORD: password @@ -81,12 +97,64 @@ services: dawarich_db: condition: service_healthy restart: true + dawarich_redis: + condition: service_healthy + restart: true deploy: resources: limits: cpus: '0.50' # Limit CPU usage to 50% of one core memory: '4G' # Limit memory usage to 4GB - + dawarich_sidekiq: + image: freikin/dawarich:latest + container_name: dawarich_sidekiq + volumes: + - dawarich_public:/var/app/public + - dawarich_watched:/var/app/tmp/imports/watched + - dawarich_storage:/var/app/storage + networks: + - dawarich + stdin_open: true + tty: true + entrypoint: sidekiq-entrypoint.sh + command: ['sidekiq'] + restart: on-failure + environment: + RAILS_ENV: development + REDIS_URL: redis://dawarich_redis:6379/0 + DATABASE_HOST: dawarich_db + DATABASE_USERNAME: postgres + DATABASE_PASSWORD: password + DATABASE_NAME: dawarich_development + APPLICATION_HOSTS: localhost + BACKGROUND_PROCESSING_CONCURRENCY: 10 + APPLICATION_PROTOCOL: http + PROMETHEUS_EXPORTER_ENABLED: false + PROMETHEUS_EXPORTER_HOST: dawarich_app + PROMETHEUS_EXPORTER_PORT: 9394 + SELF_HOSTED: "true" + STORE_GEODATA: "true" + logging: + driver: "json-file" + options: + max-size: "100m" + max-file: "5" + healthcheck: + test: [ "CMD-SHELL", "bundle exec sidekiqmon processes | grep $${HOSTNAME}" ] + interval: 10s + retries: 30 + start_period: 30s + timeout: 10s + depends_on: + dawarich_db: + condition: service_healthy + restart: true + dawarich_redis: + condition: service_healthy + restart: true + dawarich_app: + condition: service_healthy + restart: true volumes: dawarich_db_data: dawarich_sqlite_data: diff --git a/docker/sidekiq-entrypoint.sh b/docker/sidekiq-entrypoint.sh new file mode 100644 index 00000000..b55f3ff0 --- /dev/null +++ b/docker/sidekiq-entrypoint.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +unset BUNDLE_PATH +unset BUNDLE_BIN + +set -e + +echo "⚠️ Starting Sidekiq in $RAILS_ENV environment ⚠️" + +# Parse DATABASE_URL if present, otherwise use individual variables +if [ -n "$DATABASE_URL" ]; then + # Extract components from DATABASE_URL + DATABASE_HOST=$(echo $DATABASE_URL | awk -F[@/] '{print $4}') + DATABASE_PORT=$(echo $DATABASE_URL | awk -F[@/:] '{print $5}') + DATABASE_USERNAME=$(echo $DATABASE_URL | awk -F[:/@] '{print $4}') + DATABASE_PASSWORD=$(echo $DATABASE_URL | awk -F[:/@] '{print $5}') + DATABASE_NAME=$(echo $DATABASE_URL | awk -F[@/] '{print $5}') +else + # Use existing environment variables + DATABASE_HOST=${DATABASE_HOST} + DATABASE_PORT=${DATABASE_PORT} + DATABASE_USERNAME=${DATABASE_USERNAME} + DATABASE_PASSWORD=${DATABASE_PASSWORD} + DATABASE_NAME=${DATABASE_NAME} +fi + +# Wait for the database to become available +echo "⏳ Waiting for database to be ready..." +until PGPASSWORD=$DATABASE_PASSWORD psql -h "$DATABASE_HOST" -p "$DATABASE_PORT" -U "$DATABASE_USERNAME" -d "$DATABASE_NAME" -c '\q'; do + >&2 echo "Postgres is unavailable - retrying..." + sleep 2 +done +echo "✅ PostgreSQL is ready!" + +# run sidekiq +bundle exec sidekiq diff --git a/docs/How_to_install_Dawarich_in_k8s.md b/docs/How_to_install_Dawarich_in_k8s.md index 18bd316b..fa108f15 100644 --- a/docs/How_to_install_Dawarich_in_k8s.md +++ b/docs/How_to_install_Dawarich_in_k8s.md @@ -6,7 +6,7 @@ - Kubernetes cluster and basic kubectl knowledge. - Some persistent storage class prepared, in this example, Longhorn. -- Working Postgres instance. In this example Postgres lives in 'db' namespace. +- Working Postgres and Redis instances. In this example Postgres lives in 'db' namespace and Redis in 'redis' namespace. - Ngingx ingress controller with Letsencrypt integeation. - This example uses 'example.com' as a domain name, you want to change it to your own. - This will work on IPv4 and IPv6 Single Stack clusters, as well as Dual Stack deployments. @@ -80,6 +80,8 @@ spec: value: "Europe/Prague" - name: RAILS_ENV value: development + - name: REDIS_URL + value: redis://redis-master.redis.svc.cluster.local:6379/10 - name: DATABASE_HOST value: postgres-postgresql.db.svc.cluster.local - name: DATABASE_PORT @@ -126,10 +128,73 @@ spec: cpu: "2000m" ports: - containerPort: 3000 + - name: dawarich-sidekiq + env: + - name: RAILS_ENV + value: development + - name: REDIS_URL + value: redis://redis-master.redis.svc.cluster.local:6379/10 + - name: DATABASE_HOST + value: postgres-postgresql.db.svc.cluster.local + - name: DATABASE_PORT + value: "5432" + - name: DATABASE_USERNAME + value: postgres + - name: DATABASE_PASSWORD + value: Password123! + - name: DATABASE_NAME + value: dawarich_development + - name: RAILS_MIN_THREADS + value: "5" + - name: RAILS_MAX_THREADS + value: "10" + - name: BACKGROUND_PROCESSING_CONCURRENCY + value: "20" + - name: APPLICATION_HOST + value: localhost + - name: APPLICATION_HOSTS + value: "dawarich.example.com, localhost" + - name: APPLICATION_PROTOCOL + value: http + - name: PHOTON_API_HOST + value: photon.komoot.io + - name: PHOTON_API_USE_HTTPS + value: "true" + image: freikin/dawarich:latest + imagePullPolicy: Always + volumeMounts: + - mountPath: /var/app/public + name: public + - mountPath: /var/app/tmp/imports/watched + name: watched + command: + - "sidekiq-entrypoint.sh" + args: + - "bundle exec sidekiq" + resources: + requests: + memory: "1Gi" + cpu: "250m" + limits: + memory: "3Gi" + cpu: "1500m" + livenessProbe: + httpGet: + path: /api/v1/health + port: 3000 + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: / + port: 3000 + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 volumes: - - name: gem-cache - persistentVolumeClaim: - claimName: gem-cache - name: public persistentVolumeClaim: claimName: public diff --git a/docs/How_to_install_Dawarich_on_Synology.md b/docs/How_to_install_Dawarich_on_Synology.md index db2d522f..ff17f1a8 100644 --- a/docs/How_to_install_Dawarich_on_Synology.md +++ b/docs/How_to_install_Dawarich_on_Synology.md @@ -29,7 +29,7 @@ If you don't want to use dedicated share for projects installed by docker skip i ### Dawarich root folder 1. Open your [Docker root folder](#docker-root-share) in **File station**. 2. Create new folder **dawarich** and open it. -3. Create folders **db_data**, **db_shared** and **public** in **dawarich** folder. +3. Create folders **redis**, **db_data**, **db_shared** and **public** in **dawarich** folder. 4. Copy [docker compose](synology/docker-compose.yml) and [.env](synology/.env) files form **synology** repo folder into **dawarich** folder on your synology. # Installation diff --git a/docs/how_to_setup_reverse_proxy.md b/docs/how_to_setup_reverse_proxy.md index 95706525..efaddd2d 100644 --- a/docs/how_to_setup_reverse_proxy.md +++ b/docs/how_to_setup_reverse_proxy.md @@ -17,6 +17,17 @@ dawarich_app: APPLICATION_HOSTS: "yourhost.com,www.yourhost.com,127.0.0.1" <-- Edit this ``` +```yaml +dawarich_sidekiq: + image: freikin/dawarich:latest + container_name: dawarich_sidekiq + ... + environment: + ... + APPLICATION_HOSTS: "yourhost.com,www.yourhost.com,127.0.0.1" <-- Edit this + ... +``` + For a Synology install, refer to **[Synology Install Tutorial](How_to_install_Dawarich_on_Synology.md)**. In this page, it is explained how to set the APPLICATION_HOSTS environment variable. ### Virtual Host diff --git a/docs/synology/docker-compose.yml b/docs/synology/docker-compose.yml index 534225c5..7822b7c7 100644 --- a/docs/synology/docker-compose.yml +++ b/docs/synology/docker-compose.yml @@ -1,6 +1,13 @@ version: '3' services: + dawarich_redis: + image: redis:7.4-alpine + container_name: dawarich_redis + command: redis-server + restart: unless-stopped + volumes: + - ./redis:/var/shared/redis dawarich_db: image: postgis/postgis:17-3.5-alpine container_name: dawarich_db @@ -17,6 +24,7 @@ services: container_name: dawarich_app depends_on: - dawarich_db + - dawarich_redis stdin_open: true tty: true entrypoint: web-entrypoint.sh @@ -29,3 +37,19 @@ services: - ./app_storage:/var/app/storage ports: - 32568:3000 + + dawarich_sidekiq: + image: freikin/dawarich:latest + container_name: dawarich_sidekiq + depends_on: + - dawarich_db + - dawarich_redis + - dawarich_app + entrypoint: sidekiq-entrypoint.sh + command: ['sidekiq'] + restart: unless-stopped + env_file: + - .env + volumes: + - ./public:/var/app/public + - ./app_storage:/var/app/storage diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index 8cd4b1f0..4e34b6af 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -7,6 +7,7 @@ require_relative '../config/environment' abort('The Rails environment is running in production mode!') if Rails.env.production? require 'rspec/rails' require 'rswag/specs' +require 'sidekiq/testing' require 'super_diff/rspec-rails' require 'rake' diff --git a/spec/requests/sidekiq_spec.rb b/spec/requests/sidekiq_spec.rb new file mode 100644 index 00000000..0fc2d1fe --- /dev/null +++ b/spec/requests/sidekiq_spec.rb @@ -0,0 +1,125 @@ +# frozen_string_literal: true + +require 'rails_helper' +require 'sidekiq/web' + +RSpec.describe '/sidekiq', type: :request do + before do + # Allow any ENV key to be accessed and return nil by default + allow(ENV).to receive(:[]).and_return(nil) + + # Stub Sidekiq::Web with a simple Rack app for testing + allow(Sidekiq::Web).to receive(:call) do |_env| + [200, { 'Content-Type' => 'text/html' }, ['Sidekiq Web UI']] + end + end + + context 'when Dawarich is in self-hosted mode' do + before do + allow(DawarichSettings).to receive(:self_hosted?).and_return(true) + allow(ENV).to receive(:[]).with('SIDEKIQ_USERNAME').and_return(nil) + allow(ENV).to receive(:[]).with('SIDEKIQ_PASSWORD').and_return(nil) + end + + context 'when user is not authenticated' do + it 'redirects to sign in page' do + get sidekiq_url + + expect(response).to redirect_to('/users/sign_in') + end + end + + context 'when user is authenticated' do + context 'when user is not admin' do + before { sign_in create(:user) } + + it 'redirects to root page' do + get sidekiq_url + + expect(response).to redirect_to(root_url) + end + + it 'shows flash message' do + get sidekiq_url + + expect(flash[:error]).to eq('You are not authorized to perform this action.') + end + end + + context 'when user is admin' do + before { sign_in create(:user, :admin) } + + it 'renders a successful response' do + get sidekiq_url + + expect(response).to be_successful + end + end + end + end + + context 'when Dawarich is not in self-hosted mode' do + before do + allow(DawarichSettings).to receive(:self_hosted?).and_return(false) + allow(ENV).to receive(:[]).with('SIDEKIQ_USERNAME').and_return(nil) + allow(ENV).to receive(:[]).with('SIDEKIQ_PASSWORD').and_return(nil) + Rails.application.reload_routes! + end + + context 'when user is not authenticated' do + it 'redirects to sign in page' do + get sidekiq_url + + expect(response).to redirect_to('/users/sign_in') + end + end + + context 'when user is authenticated' do + before { sign_in create(:user, :admin) } + + it 'redirects to root page' do + get sidekiq_url + + expect(response).to redirect_to(root_url) + expect(flash[:error]).to eq('You are not authorized to perform this action.') + end + end + end + + context 'when SIDEKIQ_USERNAME and SIDEKIQ_PASSWORD are set' do + before do + allow(DawarichSettings).to receive(:self_hosted?).and_return(false) + allow(ENV).to receive(:[]).with('SIDEKIQ_USERNAME').and_return('admin') + allow(ENV).to receive(:[]).with('SIDEKIQ_PASSWORD').and_return('password') + end + + context 'when user is not authenticated' do + it 'redirects to sign in page' do + get sidekiq_url + + expect(response).to redirect_to('/users/sign_in') + end + end + + context 'when user is not admin' do + before { sign_in create(:user) } + + it 'redirects to root page' do + get sidekiq_url + + expect(response).to redirect_to(root_url) + expect(flash[:error]).to eq('You are not authorized to perform this action.') + end + end + + context 'when user is admin' do + before { sign_in create(:user, :admin) } + + it 'renders a successful response' do + get sidekiq_url + + expect(response).to be_successful + end + end + end +end diff --git a/spec/services/imports/watcher_spec.rb b/spec/services/imports/watcher_spec.rb index 15c8791a..94c04053 100644 --- a/spec/services/imports/watcher_spec.rb +++ b/spec/services/imports/watcher_spec.rb @@ -12,6 +12,8 @@ RSpec.describe Imports::Watcher do stub_const('Imports::Watcher::WATCHED_DIR_PATH', watched_dir_path) end + after { Sidekiq::Testing.fake! } + context 'when user exists' do let!(:user) { create(:user, email: 'user@domain.com') }