diff --git a/.app_version b/.app_version index b38e1e76..9f222923 100644 --- a/.app_version +++ b/.app_version @@ -1 +1 @@ -0.27.3 +0.27.4 diff --git a/.circleci/config.yml b/.circleci/config.yml index 1dd09644..e7410a62 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,10 +11,20 @@ jobs: environment: RAILS_ENV: test CI: true + DATABASE_HOST: localhost + DATABASE_NAME: dawarich_test + DATABASE_USERNAME: postgres + DATABASE_PASSWORD: mysecretpassword + DATABASE_PORT: 5432 + QUEUE_DATABASE_HOST: localhost + QUEUE_DATABASE_NAME: dawarich_test_queue + QUEUE_DATABASE_USERNAME: postgres + QUEUE_DATABASE_PASSWORD: mysecretpassword + QUEUE_DATABASE_PORT: 5432 - image: cimg/postgres:13.3-postgis environment: POSTGRES_USER: postgres - POSTGRES_DB: test_database + POSTGRES_DB: dawarich_test POSTGRES_PASSWORD: mysecretpassword - image: selenium/standalone-chrome:latest name: chrome @@ -39,8 +49,10 @@ jobs: - run: name: Database Setup command: | - bundle exec rails db:create - bundle exec rails db:schema:load + bundle exec rails db:create RAILS_ENV=test + bundle exec rails db:schema:load RAILS_ENV=test + # Create the queue database manually if it doesn't exist + PGPASSWORD=mysecretpassword createdb -h localhost -U postgres dawarich_test_queue || true - run: name: Run RSpec tests command: bundle exec rspec diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 49649d71..cf658aa5 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -10,6 +10,7 @@ services: - dawarich_public:/var/app/public - dawarich_watched:/var/app/tmp/imports/watched - dawarich_storage:/var/app/storage + - dawarich_sqlite_data:/dawarich_sqlite_data networks: - dawarich ports: @@ -23,6 +24,15 @@ services: DATABASE_USERNAME: postgres DATABASE_PASSWORD: password DATABASE_NAME: dawarich_development + # PostgreSQL database name for solid_queue + QUEUE_DATABASE_HOST: dawarich_db + QUEUE_DATABASE_USERNAME: postgres + QUEUE_DATABASE_PASSWORD: password + QUEUE_DATABASE_PORT: 5432 + QUEUE_DATABASE_NAME: dawarich_development_queue + # SQLite database paths for cache and cable databases + CACHE_DATABASE_PATH: /dawarich_sqlite_data/dawarich_development_cache.sqlite3 + CABLE_DATABASE_PATH: /dawarich_sqlite_data/dawarich_development_cable.sqlite3 MIN_MINUTES_SPENT_IN_CITY: 60 APPLICATION_HOSTS: localhost TIME_ZONE: Europe/London @@ -54,3 +64,4 @@ volumes: dawarich_public: dawarich_watched: dawarich_storage: + dawarich_sqlite_data: diff --git a/CHANGELOG.md b/CHANGELOG.md index 234b872c..28337146 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,54 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +# 0.27.4 - 2025-06-06 + +⚠️ This release includes a breaking change. ⚠️ + +## Changed + +- SolidQueue is now using PostgreSQL instead of SQLite. Provide `QUEUE_DATABASE_NAME`, `QUEUE_DATABASE_PASSWORD`, `QUEUE_DATABASE_USERNAME`, `QUEUE_DATABASE_PORT` and `QUEUE_DATABASE_HOST` environment variables to configure it. #1331 +- SQLite databases are now being stored in the `dawarich_sqlite_data` volume. #1361 #1357 + +```diff +... + dawarich_app: + image: freikin/dawarich:latest + container_name: dawarich_app + volumes: + - dawarich_public:/var/app/public + - dawarich_watched:/var/app/tmp/imports/watched + - dawarich_storage:/var/app/storage + - dawarich_db_data:/dawarich_db_data ++ - dawarich_sqlite_data:/dawarich_sqlite_data + ... + restart: on-failure + environment: + ... + DATABASE_NAME: dawarich_development ++ # PostgreSQL database name for solid_queue ++ QUEUE_DATABASE_NAME: dawarich_development_queue ++ QUEUE_DATABASE_PASSWORD: password ++ QUEUE_DATABASE_USERNAME: postgres ++ QUEUE_DATABASE_PORT: 5432 ++ QUEUE_DATABASE_HOST: dawarich_db + # SQLite database paths for cache and cable databases +- QUEUE_DATABASE_PATH: /dawarich_db_data/dawarich_development_queue.sqlite3 +- CACHE_DATABASE_PATH: /dawarich_db_data/dawarich_development_cache.sqlite3 +- CABLE_DATABASE_PATH: /dawarich_db_data/dawarich_development_cable.sqlite3 ++ CACHE_DATABASE_PATH: /dawarich_sqlite_data/dawarich_development_cache.sqlite3 ++ CABLE_DATABASE_PATH: /dawarich_sqlite_data/dawarich_development_cable.sqlite3 + +volumes: + dawarich_db_data: ++ dawarich_sqlite_data: + dawarich_shared: + dawarich_public: + dawarich_watched: + dawarich_storage: +... +``` + # 0.27.3 - 2025-06-05 ## Changed diff --git a/config/database.yml b/config/database.yml index 26448dc5..f82b2d8a 100644 --- a/config/database.yml +++ b/config/database.yml @@ -12,16 +12,19 @@ default: &default sqlite_default: &sqlite_default adapter: sqlite3 pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 10 } %> - retries: 100 - default_transaction_mode: IMMEDIATE + timeout: 5000 development: primary: <<: *default database: <%= ENV['DATABASE_NAME'] || 'dawarich_development' %> queue: - <<: *sqlite_default - database: <%= ENV['QUEUE_DATABASE_PATH'] || 'db/queue.sqlite3' %> + <<: *default + database: <%= ENV['QUEUE_DATABASE_NAME'] || 'dawarich_development_queue' %> + password: <%= ENV['QUEUE_DATABASE_PASSWORD'] %> + username: <%= ENV['QUEUE_DATABASE_USERNAME'] %> + port: <%= ENV['QUEUE_DATABASE_PORT'] || '5432' %> + host: <%= ENV['QUEUE_DATABASE_HOST'] %> migrations_paths: db/queue_migrate cache: <<: *sqlite_default @@ -36,14 +39,27 @@ test: primary: <<: *default database: <%= ENV['DATABASE_NAME'] || 'dawarich_test' %> + password: <%= ENV['DATABASE_PASSWORD'] %> + queue: + <<: *default + database: <%= ENV['QUEUE_DATABASE_NAME'] || 'dawarich_test_queue' %> + password: <%= ENV['QUEUE_DATABASE_PASSWORD'] %> + username: <%= ENV['QUEUE_DATABASE_USERNAME'] %> + port: <%= ENV['QUEUE_DATABASE_PORT'] || '5432' %> + host: <%= ENV['QUEUE_DATABASE_HOST'] %> + migrations_paths: db/queue_migrate production: primary: <<: *default database: <%= ENV['DATABASE_NAME'] || 'dawarich_production' %> queue: - <<: *sqlite_default - database: <%= ENV['QUEUE_DATABASE_PATH'] || 'db/queue.sqlite3' %> + <<: *default + database: <%= ENV['QUEUE_DATABASE_NAME'] || 'dawarich_production_queue' %> + password: <%= ENV['QUEUE_DATABASE_PASSWORD'] %> + username: <%= ENV['QUEUE_DATABASE_USERNAME'] %> + port: <%= ENV['QUEUE_DATABASE_PORT'] || '5432' %> + host: <%= ENV['QUEUE_DATABASE_HOST'] %> migrations_paths: db/queue_migrate cable: <<: *sqlite_default @@ -59,14 +75,18 @@ staging: <<: *default database: <%= ENV['DATABASE_NAME'] || 'dawarich_staging' %> password: <%= ENV['DATABASE_PASSWORD'] %> + queue: + <<: *default + database: <%= ENV['QUEUE_DATABASE_NAME'] || 'dawarich_staging_queue' %> + password: <%= ENV['QUEUE_DATABASE_PASSWORD'] %> + username: <%= ENV['QUEUE_DATABASE_USERNAME'] %> + port: <%= ENV['QUEUE_DATABASE_PORT'] || '5432' %> + host: <%= ENV['QUEUE_DATABASE_HOST'] %> + migrations_paths: db/queue_migrate cache: <<: *sqlite_default database: <%= ENV['CACHE_DATABASE_PATH'] || 'db/cache.sqlite3' %> migrations_paths: db/cache_migrate - queue: - <<: *sqlite_default - database: <%= ENV['QUEUE_DATABASE_PATH'] || 'db/queue.sqlite3' %> - migrations_paths: db/queue_migrate cable: <<: *sqlite_default database: <%= ENV['CABLE_DATABASE_PATH'] || 'db/cable.sqlite3' %> diff --git a/db/queue_schema.rb b/db/queue_schema.rb index 4b2cdcdc..30f375a5 100644 --- a/db/queue_schema.rb +++ b/db/queue_schema.rb @@ -11,6 +11,8 @@ # It's strongly recommended that you check this file into your version control system. ActiveRecord::Schema[8.0].define(version: 1) do + enable_extension "pg_catalog.plpgsql" + create_table "solid_queue_blocked_executions", force: :cascade do |t| t.bigint "job_id", null: false t.string "queue_name", null: false diff --git a/docker/docker-compose.production.yml b/docker/docker-compose.production.yml index d52542a3..40ce7c74 100644 --- a/docker/docker-compose.production.yml +++ b/docker/docker-compose.production.yml @@ -27,6 +27,8 @@ services: - dawarich_public:/var/app/public - dawarich_watched:/var/app/tmp/imports/watched - dawarich_storage:/var/app/storage + - dawarich_db_data:/dawarich_db_data + - dawarich_sqlite_data:/dawarich_sqlite_data networks: - dawarich ports: @@ -44,6 +46,15 @@ services: DATABASE_USERNAME: postgres DATABASE_PASSWORD: password DATABASE_NAME: dawarich_production + # PostgreSQL database name for solid_queue + QUEUE_DATABASE_NAME: dawarich_production_queue + QUEUE_DATABASE_PASSWORD: password + QUEUE_DATABASE_USERNAME: postgres + QUEUE_DATABASE_HOST: dawarich_db + QUEUE_DATABASE_PORT: 5432 + # SQLite database paths for cache and cable databases + CACHE_DATABASE_PATH: /dawarich_sqlite_data/dawarich_production_cache.sqlite3 + CABLE_DATABASE_PATH: /dawarich_sqlite_data/dawarich_production_cable.sqlite3 MIN_MINUTES_SPENT_IN_CITY: 60 APPLICATION_HOSTS: localhost,::1,127.0.0.1 TIME_ZONE: Europe/London @@ -80,3 +91,4 @@ volumes: dawarich_public: dawarich_watched: dawarich_storage: + dawarich_sqlite_data: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index cf9e3ff8..38faea53 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -31,6 +31,7 @@ services: - dawarich_watched:/var/app/tmp/imports/watched - dawarich_storage:/var/app/storage - dawarich_db_data:/dawarich_db_data + - dawarich_sqlite_data:/dawarich_sqlite_data networks: - dawarich ports: @@ -47,10 +48,15 @@ services: DATABASE_USERNAME: postgres DATABASE_PASSWORD: password DATABASE_NAME: dawarich_development - # SQLite database paths for secondary databases - QUEUE_DATABASE_PATH: /dawarich_db_data/dawarich_development_queue.sqlite3 - CACHE_DATABASE_PATH: /dawarich_db_data/dawarich_development_cache.sqlite3 - CABLE_DATABASE_PATH: /dawarich_db_data/dawarich_development_cable.sqlite3 + # PostgreSQL database name for solid_queue + QUEUE_DATABASE_NAME: dawarich_development_queue + QUEUE_DATABASE_PASSWORD: password + QUEUE_DATABASE_USERNAME: postgres + QUEUE_DATABASE_HOST: dawarich_db + QUEUE_DATABASE_PORT: 5432 + # SQLite database paths for cache and cable databases + CACHE_DATABASE_PATH: /dawarich_sqlite_data/dawarich_development_cache.sqlite3 + CABLE_DATABASE_PATH: /dawarich_sqlite_data/dawarich_development_cable.sqlite3 MIN_MINUTES_SPENT_IN_CITY: 60 APPLICATION_HOSTS: localhost TIME_ZONE: Europe/London @@ -83,6 +89,7 @@ services: volumes: dawarich_db_data: + dawarich_sqlite_data: dawarich_shared: dawarich_public: dawarich_watched: diff --git a/docker/web-entrypoint.sh b/docker/web-entrypoint.sh index 8e882285..228d89b7 100644 --- a/docker/web-entrypoint.sh +++ b/docker/web-entrypoint.sh @@ -31,6 +31,19 @@ export DATABASE_USERNAME export DATABASE_PASSWORD export DATABASE_NAME +# Set queue database name and connection parameters with defaults +QUEUE_DATABASE_NAME=${QUEUE_DATABASE_NAME:-"${DATABASE_NAME}_queue"} +QUEUE_DATABASE_PASSWORD=${QUEUE_DATABASE_PASSWORD:-"$DATABASE_PASSWORD"} +QUEUE_DATABASE_USERNAME=${QUEUE_DATABASE_USERNAME:-"$DATABASE_USERNAME"} +QUEUE_DATABASE_PORT=${QUEUE_DATABASE_PORT:-"$DATABASE_PORT"} +QUEUE_DATABASE_HOST=${QUEUE_DATABASE_HOST:-"$DATABASE_HOST"} + +export QUEUE_DATABASE_NAME +export QUEUE_DATABASE_PASSWORD +export QUEUE_DATABASE_USERNAME +export QUEUE_DATABASE_PORT +export QUEUE_DATABASE_HOST + # Remove pre-existing puma/passenger server.pid rm -f $APP_PATH/tmp/pids/server.pid @@ -38,21 +51,24 @@ rm -f $APP_PATH/tmp/pids/server.pid create_database() { local db_name=$1 local db_password=$2 + local db_host=$3 + local db_port=$4 + local db_username=$5 echo "Attempting to create database $db_name if it doesn't exist..." - PGPASSWORD=$db_password createdb -h "$DATABASE_HOST" -p "$DATABASE_PORT" -U "$DATABASE_USERNAME" "$db_name" 2>/dev/null || echo "Note: Database $db_name may already exist or couldn't be created now" + PGPASSWORD=$db_password createdb -h "$db_host" -p "$db_port" -U "$db_username" "$db_name" 2>/dev/null || echo "Note: Database $db_name may already exist or couldn't be created now" # Wait for the database to become available echo "⏳ Waiting for database $db_name to be ready..." - until PGPASSWORD=$db_password psql -h "$DATABASE_HOST" -p "$DATABASE_PORT" -U "$DATABASE_USERNAME" -d "$db_name" -c '\q' 2>/dev/null; do + until PGPASSWORD=$db_password psql -h "$db_host" -p "$db_port" -U "$db_username" -d "$db_name" -c '\q' 2>/dev/null; do >&2 echo "Postgres database $db_name is unavailable - retrying..." sleep 2 done echo "✅ PostgreSQL database $db_name is ready!" } -# Set up SQLite database directory in the volume -SQLITE_DB_DIR="/dawarich_db_data" +# Set up SQLite database directory in the volume for cache and cable +SQLITE_DB_DIR="/dawarich_sqlite_data" mkdir -p $SQLITE_DB_DIR echo "Created SQLite database directory at $SQLITE_DB_DIR" @@ -60,14 +76,12 @@ echo "Created SQLite database directory at $SQLITE_DB_DIR" echo "Setting up all required databases..." # Create primary PostgreSQL database -create_database "$DATABASE_NAME" "$DATABASE_PASSWORD" +create_database "$DATABASE_NAME" "$DATABASE_PASSWORD" "$DATABASE_HOST" "$DATABASE_PORT" "$DATABASE_USERNAME" -# Setup SQLite databases based on environment +# Create PostgreSQL queue database for solid_queue +create_database "$QUEUE_DATABASE_NAME" "$QUEUE_DATABASE_PASSWORD" "$QUEUE_DATABASE_HOST" "$QUEUE_DATABASE_PORT" "$QUEUE_DATABASE_USERNAME" -# Setup Queue database with SQLite -QUEUE_DATABASE_PATH=${QUEUE_DATABASE_PATH:-"$SQLITE_DB_DIR/${DATABASE_NAME}_queue.sqlite3"} -export QUEUE_DATABASE_PATH -echo "✅ SQLite queue database configured at $QUEUE_DATABASE_PATH" +# Setup SQLite databases for cache and cable # Setup Cache database with SQLite CACHE_DATABASE_PATH=${CACHE_DATABASE_PATH:-"$SQLITE_DB_DIR/${DATABASE_NAME}_cache.sqlite3"} @@ -84,17 +98,18 @@ fi # Step 2: Run migrations for all databases echo "Running migrations for all databases..." -# Run primary database migrations first (needed before SQLite migrations) +# Run primary database migrations first (needed before other migrations) echo "Running primary database migrations..." bundle exec rails db:migrate +# Run PostgreSQL queue database migrations +echo "Running queue database migrations..." +bundle exec rails db:migrate:queue + # Run SQLite database migrations echo "Running cache database migrations..." bundle exec rails db:migrate:cache -echo "Running queue database migrations..." -bundle exec rails db:migrate:queue - # Run cable migrations for production/staging if [ "$RAILS_ENV" = "production" ] || [ "$RAILS_ENV" = "staging" ]; then echo "Running cable database migrations..." diff --git a/docs/synology/.env b/docs/synology/.env index 2eab0f4d..efa41fdf 100644 --- a/docs/synology/.env +++ b/docs/synology/.env @@ -17,9 +17,9 @@ DATABASE_HOST=dawarich_db DATABASE_USERNAME=postgres DATABASE_PASSWORD=password DATABASE_NAME=dawarich - -################################################################################### -# Redis -################################################################################### - -REDIS_URL=redis://dawarich_redis:6379/0 +# PostgreSQL database name for solid_queue +QUEUE_DATABASE_NAME=dawarich_development_queue +QUEUE_DATABASE_PASSWORD=password +# SQLite database paths for cache and cable databases +CACHE_DATABASE_PATH=/dawarich_sqlite_data/dawarich_development_cache.sqlite3 +CABLE_DATABASE_PATH=/dawarich_sqlite_data/dawarich_development_cable.sqlite3