From 90538afb3b3634e95d2db1252abd0343afbd3f60 Mon Sep 17 00:00:00 2001 From: Kevin McConnell Date: Wed, 8 Jan 2025 13:49:01 +0000 Subject: [PATCH] Add Solid Queue We'll run jobs via the Puma plugin for now, to simplify the deployment. --- Gemfile | 2 + Gemfile.lock | 14 +++ bin/jobs | 6 ++ config/database.yml | 8 ++ config/environments/development.rb | 4 + config/environments/production.rb | 3 +- config/puma.rb | 3 + config/queue.yml | 18 ++++ config/recurring.yml | 10 ++ db/queue_schema.rb | 141 +++++++++++++++++++++++++++++ 10 files changed, 208 insertions(+), 1 deletion(-) create mode 100755 bin/jobs create mode 100644 config/queue.yml create mode 100644 config/recurring.yml create mode 100644 db/queue_schema.rb diff --git a/Gemfile b/Gemfile index 2ec7d8309..fbe6dc72f 100644 --- a/Gemfile +++ b/Gemfile @@ -44,3 +44,5 @@ group :test do gem "capybara" gem "selenium-webdriver" end + +gem "solid_queue", "~> 1.1" diff --git a/Gemfile.lock b/Gemfile.lock index b153fc0cf..f5da0e304 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -142,6 +142,8 @@ GEM reline (>= 0.3.8) drb (2.2.1) erubi (1.13.1) + et-orbi (1.2.11) + tzinfo ffi (1.17.1-aarch64-linux-gnu) ffi (1.17.1-aarch64-linux-musl) ffi (1.17.1-arm-linux-gnu) @@ -152,6 +154,9 @@ GEM ffi (1.17.1-x86_64-darwin) ffi (1.17.1-x86_64-linux-gnu) ffi (1.17.1-x86_64-linux-musl) + fugit (1.11.1) + et-orbi (~> 1, >= 1.2.11) + raabro (~> 1.4) globalid (1.2.1) activesupport (>= 6.1) hotwire-spark (0.1.12) @@ -238,6 +243,7 @@ GEM public_suffix (6.0.1) puma (6.5.0) nio4r (~> 2.0) + raabro (1.4.0) racc (1.8.1) rack (3.1.8) rack-session (2.0.0) @@ -322,6 +328,13 @@ GEM activejob (>= 7.2) activerecord (>= 7.2) railties (>= 7.2) + solid_queue (1.1.2) + activejob (>= 7.1) + activerecord (>= 7.1) + concurrent-ruby (>= 1.3.1) + fugit (~> 1.11.0) + railties (>= 7.1) + thor (~> 1.3.1) sqlite3 (2.5.0-aarch64-linux-gnu) sqlite3 (2.5.0-aarch64-linux-musl) sqlite3 (2.5.0-arm-linux-gnu) @@ -398,6 +411,7 @@ DEPENDENCIES sentry-ruby solid_cable (>= 3.0) solid_cache (~> 1.0) + solid_queue (~> 1.1) sqlite3 (>= 2.0) stimulus-rails thruster diff --git a/bin/jobs b/bin/jobs new file mode 100755 index 000000000..dcf59f309 --- /dev/null +++ b/bin/jobs @@ -0,0 +1,6 @@ +#!/usr/bin/env ruby + +require_relative "../config/environment" +require "solid_queue/cli" + +SolidQueue::Cli.start(ARGV) diff --git a/config/database.yml b/config/database.yml index 441886e48..a77427e19 100644 --- a/config/database.yml +++ b/config/database.yml @@ -21,6 +21,10 @@ development: <<: *default database: storage/development_cache.sqlite3 migrations_paths: db/cache_migrate + queue: + <<: *default + database: storage/development_queue.sqlite3 + migrations_paths: db/queue_migrate # Warning: The database defined as "test" will be erased and # re-generated from your development database when you run "rake". @@ -41,3 +45,7 @@ production: <<: *default database: storage/production_cache.sqlite3 migrations_paths: db/cache_migrate + queue: + <<: *default + database: storage/production_queue.sqlite3 + migrations_paths: db/queue_migrate diff --git a/config/environments/development.rb b/config/environments/development.rb index 838b0f7e3..7c75829a4 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -59,6 +59,10 @@ Rails.application.configure do # Highlight code that enqueued background job in logs. config.active_job.verbose_enqueue_logs = true + # Use Solid Queue in Development. + config.active_job.queue_adapter = :solid_queue + config.solid_queue.connects_to = { database: { writing: :queue } } + # Raises error for missing translations. # config.i18n.raise_on_missing_translations = true diff --git a/config/environments/production.rb b/config/environments/production.rb index 705d6fcd7..9e78f28c4 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -62,7 +62,8 @@ Rails.application.configure do config.cache_store = :solid_cache_store # Use a real queuing backend for Active Job (and separate queues per environment). - # config.active_job.queue_adapter = :resque + config.active_job.queue_adapter = :solid_queue + config.solid_queue.connects_to = { database: { writing: :queue } } # config.active_job.queue_name_prefix = "fizzy_production" config.action_mailer.perform_caching = false diff --git a/config/puma.rb b/config/puma.rb index 4b6627b44..7a484447a 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -19,3 +19,6 @@ pidfile ENV.fetch("PIDFILE", "tmp/pids/server.pid") # Allow puma to be restarted by `bin/rails restart` command. plugin :tmp_restart + +# Run Solid Queue with Puma +plugin :solid_queue diff --git a/config/queue.yml b/config/queue.yml new file mode 100644 index 000000000..9eace59c4 --- /dev/null +++ b/config/queue.yml @@ -0,0 +1,18 @@ +default: &default + dispatchers: + - polling_interval: 1 + batch_size: 500 + workers: + - queues: "*" + threads: 3 + processes: <%= ENV.fetch("JOB_CONCURRENCY", 1) %> + polling_interval: 0.1 + +development: + <<: *default + +test: + <<: *default + +production: + <<: *default diff --git a/config/recurring.yml b/config/recurring.yml new file mode 100644 index 000000000..d045b1914 --- /dev/null +++ b/config/recurring.yml @@ -0,0 +1,10 @@ +# production: +# periodic_cleanup: +# class: CleanSoftDeletedRecordsJob +# queue: background +# args: [ 1000, { batch_size: 500 } ] +# schedule: every hour +# periodic_command: +# command: "SoftDeletedRecord.due.delete_all" +# priority: 2 +# schedule: at 5am every day diff --git a/db/queue_schema.rb b/db/queue_schema.rb new file mode 100644 index 000000000..c4794c2b7 --- /dev/null +++ b/db/queue_schema.rb @@ -0,0 +1,141 @@ +# This file is auto-generated from the current state of the database. Instead +# of editing this file, please use the migrations feature of Active Record to +# incrementally modify your database, and then regenerate this schema definition. +# +# This file is the source Rails uses to define your schema when running `bin/rails +# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to +# be faster and is potentially less error prone than running all of your +# migrations from scratch. Old migrations may fail to apply correctly if those +# migrations use external dependencies or application code. +# +# It's strongly recommended that you check this file into your version control system. + +ActiveRecord::Schema[8.1].define(version: 1) do + create_table "solid_queue_blocked_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.string "concurrency_key", null: false + t.datetime "expires_at", null: false + t.datetime "created_at", null: false + t.index ["concurrency_key", "priority", "job_id"], name: "index_solid_queue_blocked_executions_for_release" + t.index ["expires_at", "concurrency_key"], name: "index_solid_queue_blocked_executions_for_maintenance" + t.index ["job_id"], name: "index_solid_queue_blocked_executions_on_job_id", unique: true + end + + create_table "solid_queue_claimed_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.bigint "process_id" + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_claimed_executions_on_job_id", unique: true + t.index ["process_id", "job_id"], name: "index_solid_queue_claimed_executions_on_process_id_and_job_id" + end + + create_table "solid_queue_failed_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.text "error" + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_failed_executions_on_job_id", unique: true + end + + create_table "solid_queue_jobs", force: :cascade do |t| + t.string "queue_name", null: false + t.string "class_name", null: false + t.text "arguments" + t.integer "priority", default: 0, null: false + t.string "active_job_id" + t.datetime "scheduled_at" + t.datetime "finished_at" + t.string "concurrency_key" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["active_job_id"], name: "index_solid_queue_jobs_on_active_job_id" + t.index ["class_name"], name: "index_solid_queue_jobs_on_class_name" + t.index ["finished_at"], name: "index_solid_queue_jobs_on_finished_at" + t.index ["queue_name", "finished_at"], name: "index_solid_queue_jobs_for_filtering" + t.index ["scheduled_at", "finished_at"], name: "index_solid_queue_jobs_for_alerting" + end + + create_table "solid_queue_pauses", force: :cascade do |t| + t.string "queue_name", null: false + t.datetime "created_at", null: false + t.index ["queue_name"], name: "index_solid_queue_pauses_on_queue_name", unique: true + end + + create_table "solid_queue_processes", force: :cascade do |t| + t.string "kind", null: false + t.datetime "last_heartbeat_at", null: false + t.bigint "supervisor_id" + t.integer "pid", null: false + t.string "hostname" + t.text "metadata" + t.datetime "created_at", null: false + t.string "name", null: false + t.index ["last_heartbeat_at"], name: "index_solid_queue_processes_on_last_heartbeat_at" + t.index ["name", "supervisor_id"], name: "index_solid_queue_processes_on_name_and_supervisor_id", unique: true + t.index ["supervisor_id"], name: "index_solid_queue_processes_on_supervisor_id" + end + + create_table "solid_queue_ready_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_ready_executions_on_job_id", unique: true + t.index ["priority", "job_id"], name: "index_solid_queue_poll_all" + t.index ["queue_name", "priority", "job_id"], name: "index_solid_queue_poll_by_queue" + end + + create_table "solid_queue_recurring_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "task_key", null: false + t.datetime "run_at", null: false + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_recurring_executions_on_job_id", unique: true + t.index ["task_key", "run_at"], name: "index_solid_queue_recurring_executions_on_task_key_and_run_at", unique: true + end + + create_table "solid_queue_recurring_tasks", force: :cascade do |t| + t.string "key", null: false + t.string "schedule", null: false + t.string "command", limit: 2048 + t.string "class_name" + t.text "arguments" + t.string "queue_name" + t.integer "priority", default: 0 + t.boolean "static", default: true, null: false + t.text "description" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["key"], name: "index_solid_queue_recurring_tasks_on_key", unique: true + t.index ["static"], name: "index_solid_queue_recurring_tasks_on_static" + end + + create_table "solid_queue_scheduled_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.datetime "scheduled_at", null: false + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_scheduled_executions_on_job_id", unique: true + t.index ["scheduled_at", "priority", "job_id"], name: "index_solid_queue_dispatch_all" + end + + create_table "solid_queue_semaphores", force: :cascade do |t| + t.string "key", null: false + t.integer "value", default: 1, null: false + t.datetime "expires_at", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["expires_at"], name: "index_solid_queue_semaphores_on_expires_at" + t.index ["key", "value"], name: "index_solid_queue_semaphores_on_key_and_value" + t.index ["key"], name: "index_solid_queue_semaphores_on_key", unique: true + end + + add_foreign_key "solid_queue_blocked_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_claimed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_failed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_ready_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_recurring_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_scheduled_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade +end