Add GVL contention metrics via gvltools (#2538)

* Add GVL contention metrics via gvltools and Yabeda

Expose per-request and process-wide GVL wait time as Prometheus metrics
to diagnose suspected GVL contention from Solid Cable and Action Cable
threads in Puma workers.

Metrics: gvl_request_wait_seconds (histogram), gvl_waiting_threads
(gauge), gvl_global_timer_total_seconds (gauge).

* Add unit: :seconds to GVL histogram for consistency

Gemfile.saas

@@ -24,3 +24,4 @@ gem "yabeda-puma-plugin"
 gem "yabeda-rails"
 gem "webrick" # required for yabeda-prometheus metrics server
 gem "prometheus-client-mmap", "~> 1.3"
+gem "gvltools"

Gemfile.saas.lock

@@ -282,6 +282,7 @@ GEM
       addressable (>= 2.5.0)
     globalid (1.3.0)
       activesupport (>= 6.1)
+    gvltools (0.4.0)
     hashdiff (1.2.1)
     i18n (1.14.8)
       concurrent-ruby (~> 1.0)
@@ -657,6 +658,7 @@ DEPENDENCIES
   faker
   fizzy-saas!
   geared_pagination (~> 1.2)
+  gvltools
   image_processing (~> 1.14)
   importmap-rails
   jbuilder

saas/lib/fizzy/saas/engine.rb

@@ -1,6 +1,7 @@
 require_relative "transaction_pinning"
 require_relative "signup"
 require_relative "authorization"
+require_relative "gvl_instrumentation"
 require_relative "../../rails_ext/active_record_tasks_database_tasks.rb"
 
 module Fizzy
@@ -51,6 +52,10 @@ module Fizzy
         app.config.middleware.insert_after(ActiveRecord::Middleware::DatabaseSelector, TransactionPinning::Middleware)
       end
 
+      initializer "fizzy_saas.gvl_instrumentation" do |app|
+        app.config.middleware.insert_before(Rack::Runtime, GvlInstrumentation)
+      end
+
       initializer "fizzy_saas.solid_queue" do
         SolidQueue.on_start do
           Process.warmup
@@ -117,6 +122,9 @@ module Fizzy
           config.channel_class_name = "ActionCable::Channel::Base"
         end
 
+        require "yabeda/gvl"
+        Yabeda::GVL.install!
+
         require_relative "metrics"
       end
 

saas/lib/fizzy/saas/gvl_instrumentation.rb

@@ -0,0 +1,20 @@
+module Fizzy
+  module Saas
+    class GvlInstrumentation
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        GVLTools::LocalTimer.enable
+        before = GVLTools::LocalTimer.monotonic_time
+        result = @app.call(env)
+        gvl_wait_ns = GVLTools::LocalTimer.monotonic_time - before
+        Yabeda.gvl.request_wait_seconds.measure({}, gvl_wait_ns / 1_000_000_000.0)
+        result
+      ensure
+        GVLTools::LocalTimer.disable
+      end
+    end
+  end
+end

saas/lib/yabeda/gvl.rb

@@ -0,0 +1,30 @@
+module Yabeda
+  module GVL
+    WAIT_HISTOGRAM_BUCKETS = [ 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10 ]
+
+    def self.install!
+      GVLTools::GlobalTimer.enable
+      GVLTools::WaitingThreads.enable
+
+      Yabeda.configure do
+        group :gvl do
+          gauge :waiting_threads,
+            comment: "Number of threads currently waiting to acquire the GVL"
+
+          gauge :global_timer_total_seconds,
+            comment: "Total time all threads spent waiting on the GVL (seconds)"
+
+          histogram :request_wait_seconds,
+            unit: :seconds,
+            comment: "GVL wait time experienced during a single request (seconds)",
+            buckets: WAIT_HISTOGRAM_BUCKETS
+        end
+
+        collect do
+          gvl.waiting_threads.set({}, GVLTools::WaitingThreads.count)
+          gvl.global_timer_total_seconds.set({}, GVLTools::GlobalTimer.monotonic_time / 1_000_000_000.0)
+        end
+      end
+    end
+  end
+end