Merge pull request #1109 from basecamp/flavorjones/yabeda-solid-queue

Introduce yabeda for Solid Queue metrics

Gemfile

@@ -39,6 +39,9 @@ gem "mission_control-jobs"
 gem "sentry-ruby"
 gem "sentry-rails"
 gem "rails_structured_logging", bc: "rails-structured-logging"
+gem "yabeda"
+gem "prometheus-client-mmap", "~> 1.1"
+gem "yabeda-prometheus-mmap"
 
 # AI
 gem "ruby_llm", git: "https://github.com/crmne/ruby_llm.git"

Gemfile.lock

@@ -225,6 +225,8 @@ GEM
       activesupport (>= 7.0)
     addressable (2.8.7)
       public_suffix (>= 2.0.2, < 7.0)
+    anyway_config (2.7.2)
+      ruby-next-core (~> 1.0)
     ast (2.4.3)
     aws-eventstream (1.4.0)
     aws-partitions (1.1148.0)
@@ -284,6 +286,7 @@ GEM
       reline (>= 0.3.8)
     dotenv (3.1.8)
     drb (2.2.3)
+    dry-initializer (3.2.0)
     ed25519 (1.4.0)
     erb (5.0.2)
     erubi (1.13.1)
@@ -427,6 +430,22 @@ GEM
       prettyprint
     prettyprint (0.2.0)
     prism (1.4.0)
+    prometheus-client-mmap (1.2.6)
+      base64
+      bigdecimal
+      rb_sys (~> 0.9.86)
+    prometheus-client-mmap (1.2.6-arm64-darwin)
+      base64
+      bigdecimal
+      rb_sys (~> 0.9.86)
+    prometheus-client-mmap (1.2.6-x86_64-darwin)
+      base64
+      bigdecimal
+      rb_sys (~> 0.9.86)
+    prometheus-client-mmap (1.2.6-x86_64-linux-gnu)
+      base64
+      bigdecimal
+      rb_sys (~> 0.9.86)
     propshaft (1.2.1)
       actionpack (>= 7.0.0)
       activesupport (>= 7.0.0)
@@ -503,6 +522,7 @@ GEM
       rubocop (>= 1.72)
       rubocop-performance (>= 1.24)
       rubocop-rails (>= 2.30)
+    ruby-next-core (1.1.2)
     ruby-progressbar (1.13.0)
     ruby-vips (2.2.5)
       ffi (~> 1.12)
@@ -596,6 +616,13 @@ GEM
     websocket-extensions (0.1.5)
     xpath (3.2.0)
       nokogiri (~> 1.8)
+    yabeda (0.14.0)
+      anyway_config (>= 1.0, < 3)
+      concurrent-ruby
+      dry-initializer
+    yabeda-prometheus-mmap (0.4.0)
+      prometheus-client-mmap
+      yabeda (~> 0.10)
     zeitwerk (2.7.3)
 
 PLATFORMS
@@ -630,6 +657,7 @@ DEPENDENCIES
   net-http-persistent
   platform_agent
   portfolio (>= 4.6)!
+  prometheus-client-mmap (~> 1.1)
   propshaft
   puma (>= 5.0)
   queenbee!
@@ -658,6 +686,8 @@ DEPENDENCIES
   vcr
   web-push
   webmock
+  yabeda
+  yabeda-prometheus-mmap
 
 BUNDLED WITH
    2.7.0

config/initializers/solid_queue.rb

@@ -0,0 +1,5 @@
+SolidQueue.on_start do
+  Process.warmup
+
+  Yabeda::Prometheus::Exporter.start_metrics_server!
+end

config/initializers/yabeda.rb

@@ -0,0 +1,7 @@
+require "prometheus/client/support/puma"
+
+Prometheus::Client.configuration.logger = Rails.logger
+Prometheus::Client.configuration.pid_provider = Prometheus::Client::Support::Puma.method(:worker_pid_provider)
+
+require "yabeda/solid_queue"
+Yabeda::SolidQueue.install!

config/recurring.yml

@@ -17,6 +17,9 @@ production: &production
   clear_solid_queue_finished_jobs:
     command: "SolidQueue::Job.clear_finished_in_batches(sleep_between_batches: 0.3)"
     schedule: every hour at minute 12
+  clear_solid_queue_recurring_executions:
+    command: "SolidQueue::RecurringExecution.clear_in_batches"
+    schedule: every hour at minute 52
   deliver_bundled_notifications:
     command: "Notification::Bundle.deliver_all_later"
     schedule: every 30 minutes

lib/yabeda/solid_queue.rb

@@ -0,0 +1,27 @@
+module Yabeda
+  module SolidQueue
+    def self.install!
+      Yabeda.configure do
+        group :solid_queue
+
+        gauge :jobs_failed_count, comment: "Number of failed jobs"
+        gauge :jobs_unreleased_count, comment: "Number of claimed jobs that don't belong to any process"
+        gauge :jobs_scheduled_and_delayed_count, comment: "Number of scheduled jobs that have over 5 minutes delay"
+        gauge :recurring_tasks_count, comment: "Number of recurring jobs scheduled"
+        gauge :recurring_tasks_delayed_count, comment: "Number of recurring jobs that haven't been enqueued within their schedule"
+
+        collect do
+          if ::SolidQueue.supervisor?
+            solid_queue.jobs_failed_count.set({}, ::SolidQueue::FailedExecution.count)
+            solid_queue.jobs_unreleased_count.set({}, ::SolidQueue::ClaimedExecution.where(process: nil).count)
+            solid_queue.jobs_scheduled_and_delayed_count.set({}, ::SolidQueue::ScheduledExecution.where(scheduled_at: ..5.minutes.ago).count)
+            solid_queue.recurring_tasks_count.set({}, ::SolidQueue::RecurringTask.count)
+            solid_queue.recurring_tasks_delayed_count.set({}, ::SolidQueue::RecurringTask.count do |task|
+              task.last_enqueued_time.present? && (task.previous_time - task.last_enqueued_time) > 5.minutes
+            end)
+          end
+        end
+      end
+    end
+  end
+end