From 44429644083db38647ed9f5ccca3fc3f270c3020 Mon Sep 17 00:00:00 2001
From: Mike Dalessio <mike@37signals.com>
Date: Mon, 15 Sep 2025 10:42:47 -0400
Subject: [PATCH] Introduce yabeda for Solid Queue metrics

---
 Gemfile                            |  3 +++
 Gemfile.lock                       | 30 ++++++++++++++++++++++++++++++
 config/initializers/solid_queue.rb |  5 +++++
 config/initializers/yabeda.rb      |  7 +++++++
 config/recurring.yml               |  3 +++
 lib/yabeda/solid_queue.rb          | 27 +++++++++++++++++++++++++++
 6 files changed, 75 insertions(+)
 create mode 100644 config/initializers/solid_queue.rb
 create mode 100644 config/initializers/yabeda.rb
 create mode 100644 lib/yabeda/solid_queue.rb

diff --git a/Gemfile b/Gemfile
index 820e33d9d..f31c2c9f7 100644
--- a/Gemfile
+++ b/Gemfile
@@ -39,6 +39,9 @@ gem "mission_control-jobs"
 gem "sentry-ruby"
 gem "sentry-rails"
 gem "rails_structured_logging", bc: "rails-structured-logging"
+gem "yabeda"
+gem "prometheus-client-mmap", "~> 1.1"
+gem "yabeda-prometheus-mmap"
 
 # AI
 gem "ruby_llm", git: "https://github.com/crmne/ruby_llm.git"
diff --git a/Gemfile.lock b/Gemfile.lock
index e84496eef..7ca1a14ed 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -225,6 +225,8 @@ GEM
       activesupport (>= 7.0)
     addressable (2.8.7)
       public_suffix (>= 2.0.2, < 7.0)
+    anyway_config (2.7.2)
+      ruby-next-core (~> 1.0)
     ast (2.4.3)
     aws-eventstream (1.4.0)
     aws-partitions (1.1148.0)
@@ -284,6 +286,7 @@ GEM
       reline (>= 0.3.8)
     dotenv (3.1.8)
     drb (2.2.3)
+    dry-initializer (3.2.0)
     ed25519 (1.4.0)
     erb (5.0.2)
     erubi (1.13.1)
@@ -427,6 +430,22 @@ GEM
       prettyprint
     prettyprint (0.2.0)
     prism (1.4.0)
+    prometheus-client-mmap (1.2.6)
+      base64
+      bigdecimal
+      rb_sys (~> 0.9.86)
+    prometheus-client-mmap (1.2.6-arm64-darwin)
+      base64
+      bigdecimal
+      rb_sys (~> 0.9.86)
+    prometheus-client-mmap (1.2.6-x86_64-darwin)
+      base64
+      bigdecimal
+      rb_sys (~> 0.9.86)
+    prometheus-client-mmap (1.2.6-x86_64-linux-gnu)
+      base64
+      bigdecimal
+      rb_sys (~> 0.9.86)
     propshaft (1.2.1)
       actionpack (>= 7.0.0)
       activesupport (>= 7.0.0)
@@ -503,6 +522,7 @@ GEM
       rubocop (>= 1.72)
       rubocop-performance (>= 1.24)
       rubocop-rails (>= 2.30)
+    ruby-next-core (1.1.2)
     ruby-progressbar (1.13.0)
     ruby-vips (2.2.5)
       ffi (~> 1.12)
@@ -596,6 +616,13 @@ GEM
     websocket-extensions (0.1.5)
     xpath (3.2.0)
       nokogiri (~> 1.8)
+    yabeda (0.14.0)
+      anyway_config (>= 1.0, < 3)
+      concurrent-ruby
+      dry-initializer
+    yabeda-prometheus-mmap (0.4.0)
+      prometheus-client-mmap
+      yabeda (~> 0.10)
     zeitwerk (2.7.3)
 
 PLATFORMS
@@ -630,6 +657,7 @@ DEPENDENCIES
   net-http-persistent
   platform_agent
   portfolio (>= 4.6)!
+  prometheus-client-mmap (~> 1.1)
   propshaft
   puma (>= 5.0)
   queenbee!
@@ -658,6 +686,8 @@ DEPENDENCIES
   vcr
   web-push
   webmock
+  yabeda
+  yabeda-prometheus-mmap
 
 BUNDLED WITH
    2.7.0
diff --git a/config/initializers/solid_queue.rb b/config/initializers/solid_queue.rb
new file mode 100644
index 000000000..5fe44c026
--- /dev/null
+++ b/config/initializers/solid_queue.rb
@@ -0,0 +1,5 @@
+SolidQueue.on_start do
+  Process.warmup
+
+  Yabeda::Prometheus::Exporter.start_metrics_server!
+end
diff --git a/config/initializers/yabeda.rb b/config/initializers/yabeda.rb
new file mode 100644
index 000000000..daa3a17dc
--- /dev/null
+++ b/config/initializers/yabeda.rb
@@ -0,0 +1,7 @@
+require "prometheus/client/support/puma"
+
+Prometheus::Client.configuration.logger = Rails.logger
+Prometheus::Client.configuration.pid_provider = Prometheus::Client::Support::Puma.method(:worker_pid_provider)
+
+require "yabeda/solid_queue"
+Yabeda::SolidQueue.install!
diff --git a/config/recurring.yml b/config/recurring.yml
index d5577c4ff..45410859d 100644
--- a/config/recurring.yml
+++ b/config/recurring.yml
@@ -17,6 +17,9 @@ production: &production
   clear_solid_queue_finished_jobs:
     command: "SolidQueue::Job.clear_finished_in_batches(sleep_between_batches: 0.3)"
     schedule: every hour at minute 12
+  clear_solid_queue_recurring_executions:
+    command: "SolidQueue::RecurringExecution.clear_in_batches"
+    schedule: every hour at minute 52
   deliver_bundled_notifications:
     command: "Notification::Bundle.deliver_all_later"
     schedule: every 30 minutes
diff --git a/lib/yabeda/solid_queue.rb b/lib/yabeda/solid_queue.rb
new file mode 100644
index 000000000..082ac8936
--- /dev/null
+++ b/lib/yabeda/solid_queue.rb
@@ -0,0 +1,27 @@
+module Yabeda
+  module SolidQueue
+    def self.install!
+      Yabeda.configure do
+        group :solid_queue
+
+        gauge :jobs_failed_count, comment: "Number of failed jobs"
+        gauge :jobs_unreleased_count, comment: "Number of claimed jobs that don't belong to any process"
+        gauge :jobs_scheduled_and_delayed_count, comment: "Number of scheduled jobs that have over 5 minutes delay"
+        gauge :recurring_tasks_count, comment: "Number of recurring jobs scheduled"
+        gauge :recurring_tasks_delayed_count, comment: "Number of recurring jobs that haven't been enqueued within their schedule"
+
+        collect do
+          if ::SolidQueue.supervisor?
+            solid_queue.jobs_failed_count.set({}, ::SolidQueue::FailedExecution.count)
+            solid_queue.jobs_unreleased_count.set({}, ::SolidQueue::ClaimedExecution.where(process: nil).count)
+            solid_queue.jobs_scheduled_and_delayed_count.set({}, ::SolidQueue::ScheduledExecution.where(scheduled_at: ..5.minutes.ago).count)
+            solid_queue.recurring_tasks_count.set({}, ::SolidQueue::RecurringTask.count)
+            solid_queue.recurring_tasks_delayed_count.set({}, ::SolidQueue::RecurringTask.count do |task|
+              task.last_enqueued_time.present? && (task.previous_time - task.last_enqueued_time) > 5.minutes
+            end)
+          end
+        end
+      end
+    end
+  end
+end