Merge pull request #1118 from basecamp/otel-collector

OTel collector for Prometheus metrics

Dockerfile

@@ -71,5 +71,5 @@ USER rails:rails
 ENTRYPOINT ["/rails/bin/docker-entrypoint"]
 
 # Start the server by default, this can be overwritten at runtime
-EXPOSE 80 443
+EXPOSE 80 443 9394
 CMD ["./bin/thrust", "./bin/rails", "server"]

Gemfile

@@ -42,6 +42,7 @@ gem "rails_structured_logging", bc: "rails-structured-logging"
 gem "yabeda"
 gem "yabeda-rails"
 gem "yabeda-puma-plugin"
+gem "webrick" # required for yabeda-prometheus metrics server
 gem "yabeda-prometheus-mmap"
 gem "prometheus-client-mmap", "~> 1.1"
 

Gemfile.lock

@@ -615,6 +615,7 @@ GEM
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
+    webrick (1.9.1)
     websocket (1.2.11)
     websocket-driver (0.8.0)
       base64
@@ -701,6 +702,7 @@ DEPENDENCIES
   vcr
   web-push
   webmock
+  webrick
   yabeda
   yabeda-prometheus-mmap
   yabeda-puma-plugin

config/deploy.yml

@@ -32,3 +32,17 @@ aliases:
   console: app exec -i --reuse "bin/rails console"
   ssh: app exec -i --reuse /bin/bash
   switch: accessory exec beamer --reuse "beamer switch --primary=<%= ENV["PRIMARY"] %>"
+
+accessories:
+  otel_collector:
+    image: otel/opentelemetry-collector-contrib:0.126.0
+    port: 9394
+    files:
+      - config/otel_collector.yml:/etc/otelcol-contrib/config.yaml
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    options:
+      user: 0
+    roles:
+      - web
+      - jobs

config/otel_collector.yml

@@ -0,0 +1,45 @@
+receivers:
+  prometheus:
+    config:
+      scrape_configs:
+        - job_name: "kamal-containers"
+          scrape_interval: 5s
+          docker_sd_configs:
+            - host: unix:///var/run/docker.sock
+              refresh_interval: 10s
+          relabel_configs:
+            # Limit scraping to labeled containers with port 9394 exposed
+            - source_labels: [__address__]
+              regex: .*:9394
+              action: keep
+            - source_labels: [__meta_docker_container_name]
+              regex: (/fizzy-(web|jobs)-.*|/kamal-proxy)
+              action: keep
+
+            # Pull service and role from Kamal Docker labels
+            - source_labels: [__meta_docker_container_label_service]
+              target_label: service
+            - source_labels: [__meta_docker_container_label_role]
+              target_label: role
+
+            - source_labels: [__meta_docker_container_name]
+              regex: ^/kamal-proxy$
+              target_label: service
+              replacement: kamal-proxy
+            - source_labels: [__meta_docker_container_name]
+              regex: ^/kamal-proxy$
+              target_label: role
+              replacement: proxy
+
+exporters:
+  prometheus:
+    endpoint: 0.0.0.0:9394
+
+service:
+  telemetry:
+    logs:
+      level: "info"
+  pipelines:
+    metrics:
+      receivers: [prometheus]
+      exporters: [prometheus]