diff --git a/.gitignore b/.gitignore index c5df6beb..71be6ec5 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,5 @@ /config/credentials/staging.yml.enc Makefile + +/db/*.sqlite3 diff --git a/CHANGELOG.md b/CHANGELOG.md index a807057a..13d384f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,14 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). This release introduces a new way to run background jobs and cache data. Before updating, make sure your Sidekiq queues (https://your_dawarich_app/sidekiq) are empty. -Moving to SolidQueue and SolidCache will require creating new databases, which will be created automatically when you start the app. If that didn't happen, you can create them manually and set the following environment variables: - -- `QUEUE_DATABASE_NAME` - name of the queue database (default is `dawarich_development_queue`) -- `QUEUE_DATABASE_PASSWORD` - password for the queue database -- `CACHE_DATABASE_NAME` - name of the cache database (default is `dawarich_development_cache`) -- `CACHE_DATABASE_PASSWORD` - password for the cache database -- `CABLE_DATABASE_NAME` - name of the cable database (default is `dawarich_development_cable`) -- `CABLE_DATABASE_PASSWORD` - password for the cable database +Moving to SolidQueue and SolidCache will require creating new SQLite databases, which will be created automatically when you start the app. They will be stored in the `dawarich_db_data` volume. ## Fixed diff --git a/Gemfile b/Gemfile index ea404a18..7022ab66 100644 --- a/Gemfile +++ b/Gemfile @@ -23,6 +23,7 @@ gem 'lograge' gem 'mission_control-jobs' gem 'oj' gem 'pg' +gem 'sqlite3', '~> 2.6' gem 'prometheus_exporter' gem 'activerecord-postgis-adapter' gem 'puma' diff --git a/Gemfile.lock b/Gemfile.lock index 091acecb..a9735431 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -475,6 +475,12 @@ GEM actionpack (>= 6.1) activesupport (>= 6.1) sprockets (>= 3.0.0) + sqlite3 (2.6.0-aarch64-linux-gnu) + sqlite3 (2.6.0-arm-linux-gnu) + sqlite3 (2.6.0-arm64-darwin) + sqlite3 (2.6.0-x86-linux-gnu) + sqlite3 (2.6.0-x86_64-darwin) + sqlite3 (2.6.0-x86_64-linux-gnu) stackprof (0.2.27) stimulus-rails (1.3.4) railties (>= 6.0.0) @@ -589,6 +595,7 @@ DEPENDENCIES solid_cache (= 1.0.7) solid_queue (~> 1.1) sprockets-rails + sqlite3 (~> 2.6) stackprof stimulus-rails strong_migrations diff --git a/config/database.yml b/config/database.yml index d80f85ef..5f4dff65 100644 --- a/config/database.yml +++ b/config/database.yml @@ -9,19 +9,22 @@ default: &default pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 10 } %> timeout: 5000 +sqlite_default: &sqlite_default + adapter: sqlite3 + pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 10 } %> + timeout: 5000 + development: primary: <<: *default database: <%= ENV['DATABASE_NAME'] || 'dawarich_development' %> queue: - <<: *default - database: <%= ENV['QUEUE_DATABASE_NAME'] || 'dawarich_development_queue' %> - password: <%= ENV['QUEUE_DATABASE_PASSWORD'] %> + <<: *sqlite_default + database: <%= ENV['QUEUE_DATABASE_PATH'] || 'db/queue.sqlite3' %> migrations_paths: db/queue_migrate cache: - <<: *default - database: <%= ENV['CACHE_DATABASE_NAME'] || 'dawarich_development_cache' %> - password: <%= ENV['CACHE_DATABASE_PASSWORD'] %> + <<: *sqlite_default + database: <%= ENV['CACHE_DATABASE_PATH'] || 'db/cache.sqlite3' %> migrations_paths: db/cache_migrate test: @@ -34,19 +37,16 @@ production: <<: *default database: <%= ENV['DATABASE_NAME'] || 'dawarich_production' %> queue: - <<: *default - database: <%= ENV['QUEUE_DATABASE_NAME'] || 'dawarich_production_queue' %> - password: <%= ENV['QUEUE_DATABASE_PASSWORD'] %> + <<: *sqlite_default + database: <%= ENV['QUEUE_DATABASE_PATH'] || 'db/queue.sqlite3' %> migrations_paths: db/queue_migrate cable: - <<: *default - database: <%= ENV['CABLE_DATABASE_NAME'] || 'dawarich_production_cable' %> - password: <%= ENV['CABLE_DATABASE_PASSWORD'] %> + <<: *sqlite_default + database: <%= ENV['CABLE_DATABASE_PATH'] || 'db/cable.sqlite3' %> migrations_paths: db/cable_migrate cache: - <<: *default - database: <%= ENV['CACHE_DATABASE_NAME'] || 'dawarich_production_cache' %> - password: <%= ENV['CACHE_DATABASE_PASSWORD'] %> + <<: *sqlite_default + database: <%= ENV['CACHE_DATABASE_PATH'] %> migrations_paths: db/cache_migrate staging: @@ -55,17 +55,14 @@ staging: database: <%= ENV['DATABASE_NAME'] || 'dawarich_staging' %> password: <%= ENV['DATABASE_PASSWORD'] %> cache: - <<: *default - database: <%= ENV['CACHE_DATABASE_NAME'] || 'dawarich_staging_cache' %> - password: <%= ENV['CACHE_DATABASE_PASSWORD'] %> + <<: *sqlite_default + database: <%= ENV['CACHE_DATABASE_PATH'] || 'db/cache.sqlite3' %> migrations_paths: db/cache_migrate queue: - <<: *default - database: <%= ENV['QUEUE_DATABASE_NAME'] || 'dawarich_staging_queue' %> - password: <%= ENV['QUEUE_DATABASE_PASSWORD'] %> + <<: *sqlite_default + database: <%= ENV['QUEUE_DATABASE_PATH'] || 'db/queue.sqlite3' %> migrations_paths: db/queue_migrate cable: - <<: *default - database: <%= ENV['CABLE_DATABASE_NAME'] || 'dawarich_staging_cable' %> - password: <%= ENV['CABLE_DATABASE_PASSWORD'] %> + <<: *sqlite_default + database: <%= ENV['CABLE_DATABASE_PATH'] || 'db/cable.sqlite3' %> migrations_paths: db/cable_migrate diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 9f15cb59..f590f6d7 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -45,6 +45,7 @@ services: - dawarich_public:/var/app/public - dawarich_watched:/var/app/tmp/imports/watched - dawarich_storage:/var/app/storage + - dawarich_db_data:/dawarich_db_data networks: - dawarich ports: @@ -62,12 +63,10 @@ services: DATABASE_USERNAME: postgres DATABASE_PASSWORD: password DATABASE_NAME: dawarich_development - QUEUE_DATABASE_NAME: dawarich_development_queue - QUEUE_DATABASE_PASSWORD: password - CACHE_DATABASE_NAME: dawarich_development_cache - CACHE_DATABASE_PASSWORD: password - CABLE_DATABASE_NAME: dawarich_development_cable - CABLE_DATABASE_PASSWORD: password + # SQLite database paths for secondary databases + QUEUE_DATABASE_PATH: /dawarich_db_data/dawarich_development_queue.sqlite3 + CACHE_DATABASE_PATH: /dawarich_db_data/dawarich_development_cache.sqlite3 + CABLE_DATABASE_PATH: /dawarich_db_data/dawarich_development_cable.sqlite3 MIN_MINUTES_SPENT_IN_CITY: 60 APPLICATION_HOSTS: localhost TIME_ZONE: Europe/London @@ -107,6 +106,7 @@ services: - dawarich_public:/var/app/public - dawarich_watched:/var/app/tmp/imports/watched - dawarich_storage:/var/app/storage + - dawarich_db_data:/dawarich_db_data networks: - dawarich stdin_open: true diff --git a/docker/web-entrypoint.sh b/docker/web-entrypoint.sh index 1d1b8d8b..8e882285 100644 --- a/docker/web-entrypoint.sh +++ b/docker/web-entrypoint.sh @@ -34,7 +34,7 @@ export DATABASE_NAME # Remove pre-existing puma/passenger server.pid rm -f $APP_PATH/tmp/pids/server.pid -# Function to check and create a database +# Function to check and create a PostgreSQL database create_database() { local db_name=$1 local db_password=$2 @@ -51,49 +51,49 @@ create_database() { echo "✅ PostgreSQL database $db_name is ready!" } -# Step 1: Create all databases -echo "Creating all required databases..." +# Set up SQLite database directory in the volume +SQLITE_DB_DIR="/dawarich_db_data" +mkdir -p $SQLITE_DB_DIR +echo "Created SQLite database directory at $SQLITE_DB_DIR" -# Create primary database +# Step 1: Database Setup +echo "Setting up all required databases..." + +# Create primary PostgreSQL database create_database "$DATABASE_NAME" "$DATABASE_PASSWORD" -# Create additional databases based on environment -if [ "$RAILS_ENV" = "development" ] || [ "$RAILS_ENV" = "production" ] || [ "$RAILS_ENV" = "staging" ]; then - # Setup Queue database - QUEUE_DATABASE_NAME=${QUEUE_DATABASE_NAME:-${DATABASE_NAME}_queue} - QUEUE_DATABASE_PASSWORD=${QUEUE_DATABASE_PASSWORD:-$DATABASE_PASSWORD} - export QUEUE_DATABASE_NAME - export QUEUE_DATABASE_PASSWORD - create_database "$QUEUE_DATABASE_NAME" "$QUEUE_DATABASE_PASSWORD" +# Setup SQLite databases based on environment - # Setup Cache database - CACHE_DATABASE_NAME=${CACHE_DATABASE_NAME:-${DATABASE_NAME}_cache} - CACHE_DATABASE_PASSWORD=${CACHE_DATABASE_PASSWORD:-$DATABASE_PASSWORD} - export CACHE_DATABASE_NAME - export CACHE_DATABASE_PASSWORD - create_database "$CACHE_DATABASE_NAME" "$CACHE_DATABASE_PASSWORD" -fi +# Setup Queue database with SQLite +QUEUE_DATABASE_PATH=${QUEUE_DATABASE_PATH:-"$SQLITE_DB_DIR/${DATABASE_NAME}_queue.sqlite3"} +export QUEUE_DATABASE_PATH +echo "✅ SQLite queue database configured at $QUEUE_DATABASE_PATH" -# Setup Cable database (only for production and staging) +# Setup Cache database with SQLite +CACHE_DATABASE_PATH=${CACHE_DATABASE_PATH:-"$SQLITE_DB_DIR/${DATABASE_NAME}_cache.sqlite3"} +export CACHE_DATABASE_PATH +echo "✅ SQLite cache database configured at $CACHE_DATABASE_PATH" + +# Setup Cable database with SQLite (only for production and staging) if [ "$RAILS_ENV" = "production" ] || [ "$RAILS_ENV" = "staging" ]; then - CABLE_DATABASE_NAME=${CABLE_DATABASE_NAME:-${DATABASE_NAME}_cable} - CABLE_DATABASE_PASSWORD=${CABLE_DATABASE_PASSWORD:-$DATABASE_PASSWORD} - export CABLE_DATABASE_NAME - export CABLE_DATABASE_PASSWORD - create_database "$CABLE_DATABASE_NAME" "$CABLE_DATABASE_PASSWORD" + CABLE_DATABASE_PATH=${CABLE_DATABASE_PATH:-"$SQLITE_DB_DIR/${DATABASE_NAME}_cable.sqlite3"} + export CABLE_DATABASE_PATH + echo "✅ SQLite cable database configured at $CABLE_DATABASE_PATH" fi # Step 2: Run migrations for all databases echo "Running migrations for all databases..." -# Run cache and queue migrations first (needed for app initialization) -if [ "$RAILS_ENV" = "development" ] || [ "$RAILS_ENV" = "production" ] || [ "$RAILS_ENV" = "staging" ]; then - echo "Running cache database migrations..." - bundle exec rails db:migrate:cache +# Run primary database migrations first (needed before SQLite migrations) +echo "Running primary database migrations..." +bundle exec rails db:migrate - echo "Running queue database migrations..." - bundle exec rails db:migrate:queue -fi +# Run SQLite database migrations +echo "Running cache database migrations..." +bundle exec rails db:migrate:cache + +echo "Running queue database migrations..." +bundle exec rails db:migrate:queue # Run cable migrations for production/staging if [ "$RAILS_ENV" = "production" ] || [ "$RAILS_ENV" = "staging" ]; then @@ -101,18 +101,12 @@ if [ "$RAILS_ENV" = "production" ] || [ "$RAILS_ENV" = "staging" ]; then bundle exec rails db:migrate:cable fi -# Run primary database migrations -echo "Running primary database migrations..." -bundle exec rails db:migrate - # Run data migrations echo "Running DATA migrations..." bundle exec rake data:migrate -# if [ "$RAILS_ENV" != "production" ]; then - echo "Running seeds..." - bundle exec rails db:seed -# fi +echo "Running seeds..." +bundle exec rails db:seed # run passed commands bundle exec ${@}