Initial (wip) staging deployment

This adds an initial configuration for staging. Note that we are doing
this ahead of having the full infrastructure in all 3 DCs. So this
this will result in some cross-DC writes for now (e.g. we have a single cache in
IAD). We'll correct this as the infrastructure becomes available.

We'll also run jobs on every app server, until we split them out to
separate instances.
This commit is contained in:
Kevin McConnell
2025-11-13 11:43:40 +00:00
committed by Mike Dalessio
parent 4b4593dd01
commit 11a38a4526
7 changed files with 136 additions and 70 deletions
+7 -1
View File
@@ -1,6 +1,12 @@
SECRETS=$(kamal secrets fetch --adapter 1password --account basecamp --from Deploy/Fizzy Deployments/BASECAMP_REGISTRY_PASSWORD Deployments/DASH_BASIC_AUTH_SECRET Staging/RAILS_MASTER_KEY) SECRETS=$(kamal secrets fetch --adapter 1password --account basecamp --from Deploy/Fizzy Deployments/BASECAMP_REGISTRY_PASSWORD Deployments/DASH_BASIC_AUTH_SECRET Staging/RAILS_MASTER_KEY Staging/MYSQL_ALTER_PASSWORD Staging/MYSQL_ALTER_USER Staging/MYSQL_APP_PASSWORD Staging/MYSQL_APP_USER Staging/MYSQL_READONLY_PASSWORD Staging/MYSQL_READONLY_USER)
GITHUB_TOKEN=$(gh config get -h github.com oauth_token) GITHUB_TOKEN=$(gh config get -h github.com oauth_token)
BASECAMP_REGISTRY_PASSWORD=$(kamal secrets extract BASECAMP_REGISTRY_PASSWORD $SECRETS) BASECAMP_REGISTRY_PASSWORD=$(kamal secrets extract BASECAMP_REGISTRY_PASSWORD $SECRETS)
DASH_BASIC_AUTH_SECRET=$(kamal secrets extract DASH_BASIC_AUTH_SECRET $SECRETS) DASH_BASIC_AUTH_SECRET=$(kamal secrets extract DASH_BASIC_AUTH_SECRET $SECRETS)
RAILS_MASTER_KEY=$(kamal secrets extract RAILS_MASTER_KEY $SECRETS) RAILS_MASTER_KEY=$(kamal secrets extract RAILS_MASTER_KEY $SECRETS)
MYSQL_ALTER_PASSWORD=$(kamal secrets extract MYSQL_ALTER_PASSWORD $SECRETS)
MYSQL_ALTER_USER=$(kamal secrets extract MYSQL_ALTER_USER $SECRETS)
MYSQL_APP_PASSWORD=$(kamal secrets extract MYSQL_APP_PASSWORD $SECRETS)
MYSQL_APP_USER=$(kamal secrets extract MYSQL_APP_USER $SECRETS)
MYSQL_READONLY_PASSWORD=$(kamal secrets extract MYSQL_READONLY_PASSWORD $SECRETS)
MYSQL_READONLY_USER=$(kamal secrets extract MYSQL_READONLY_USER $SECRETS)
+1 -1
View File
@@ -2,7 +2,7 @@
# If running the rails server then create or migrate existing database # If running the rails server then create or migrate existing database
if [ "${1}" == "./bin/thrust" ] && [ "${2}" == "./bin/rails" ] && [ "${3}" == "server" ]; then if [ "${1}" == "./bin/thrust" ] && [ "${2}" == "./bin/rails" ] && [ "${3}" == "server" ]; then
./bin/rails db:prepare MIGRATE=1 ./bin/rails db:prepare
fi fi
exec "${@}" exec "${@}"
+48 -9
View File
@@ -1,7 +1,20 @@
<%
if ENV["MIGRATE"].present?
mysql_app_user_key = "MYSQL_ALTER_USER"
mysql_app_password_key = "MYSQL_ALTER_PASSWORD"
else
mysql_app_user_key = "MYSQL_APP_USER"
mysql_app_password_key = "MYSQL_APP_PASSWORD"
end
mysql_app_user = ENV[mysql_app_user_key]
mysql_app_password = ENV[mysql_app_password_key]
%>
default: &default default: &default
adapter: trilogy adapter: trilogy
host: <%= ENV.fetch "DB_HOST", "127.0.0.1" %> host: <%= ENV.fetch "FIZZY_DB_HOST", "127.0.0.1" %>
port: <%= ENV.fetch "DB_PORT", 33380 %> port: <%= ENV.fetch "FIZZY_DB_PORT", 3306 %>
pool: 50 pool: 50
timeout: 5000 timeout: 5000
@@ -9,22 +22,27 @@ development:
primary: primary:
<<: *default <<: *default
database: fizzy_development database: fizzy_development
port: <%= ENV.fetch "FIZZY_DB_PORT", 33380 %>
replica: replica:
<<: *default <<: *default
database: fizzy_development database: fizzy_development
replica: true replica: true
port: <%= ENV.fetch "FIZZY_DB_PORT", 33380 %>
cable: cable:
<<: *default <<: *default
database: development_cable database: development_cable
migrations_paths: db/cable_migrate migrations_paths: db/cable_migrate
port: <%= ENV.fetch "FIZZY_DB_PORT", 33380 %>
cache: cache:
<<: *default <<: *default
database: development_cache database: development_cache
migrations_paths: db/cache_migrate migrations_paths: db/cache_migrate
port: <%= ENV.fetch "FIZZY_DB_PORT", 33380 %>
queue: queue:
<<: *default <<: *default
database: development_queue database: development_queue
migrations_paths: db/queue_migrate migrations_paths: db/queue_migrate
port: <%= ENV.fetch "FIZZY_DB_PORT", 33380 %>
# Warning: The database defined as "test" will be erased and # Warning: The database defined as "test" will be erased and
# re-generated from your development database when you run "rake". # re-generated from your development database when you run "rake".
@@ -33,27 +51,48 @@ test:
primary: primary:
<<: *default <<: *default
database: fizzy_test database: fizzy_test
port: <%= ENV.fetch "FIZZY_DB_PORT", 33380 %>
replica: replica:
<<: *default <<: *default
database: fizzy_test database: fizzy_test
port: <%= ENV.fetch "FIZZY_DB_PORT", 33380 %>
replica: true replica: true
production: &production production: &production
primary: primary:
<<: *default <<: *default
database: fizzy_<%= Rails.env %> database: fizzy_production
host: <%= ENV["MYSQL_DATABASE_HOST"] %>
username: <%= mysql_app_user %>
password: <%= mysql_app_password %>
replica:
<<: *default
database: fizzy_production
host: <%= ENV["MYSQL_DATABASE_REPLICA_HOST"] %>
username: <%= ENV["MYSQL_READONLY_USER"] %>
password: <%= ENV["MYSQL_READONLY_PASSWORD"] %>
replica: true
cable: cable:
<<: *default <<: *default
database: <%= Rails.env %>_cable database: fizzy_solidcable_production
host: <%= ENV["MYSQL_SOLID_CABLE_HOST"] %>
username: <%= mysql_app_user %>
password: <%= mysql_app_password %>
migrations_paths: db/cable_migrate migrations_paths: db/cable_migrate
cache:
<<: *default
database: <%= Rails.env %>_cache
migrations_paths: db/cache_migrate
queue: queue:
<<: *default <<: *default
database: <%= Rails.env %>_queue database: fizzy_solidqueue_production
host: <%= ENV["MYSQL_SOLID_QUEUE_HOST"] %>
username: <%= mysql_app_user %>
password: <%= mysql_app_password %>
migrations_paths: db/queue_migrate migrations_paths: db/queue_migrate
cache:
<<: *default
database: fizzy_solidcache_production
host: <%= ENV["MYSQL_SOLID_CACHE_HOST"] %>
username: <%= mysql_app_user %>
password: <%= mysql_app_password %>
migrations_paths: db/cache_migrate
beta: *production beta: *production
staging: *production staging: *production
+28 -11
View File
@@ -1,21 +1,18 @@
servers: servers:
web: web:
hosts: hosts:
- fizzy-app-101.df-iad-int.37signals.com - fizzy-app-01.sc-chi-int.37signals.com: sc_chi
- fizzy-app-102.df-iad-int.37signals.com - fizzy-app-02.sc-chi-int.37signals.com: sc_chi
- fizzy-app-01.sc-chi-int.37signals.com - fizzy-app-101.df-iad-int.37signals.com: df_iad
- fizzy-app-02.sc-chi-int.37signals.com - fizzy-app-102.df-iad-int.37signals.com: df_iad
- fizzy-app-401.df-ams-int.37signals.com - fizzy-app-401.df-ams-int.37signals.com: df_ams
- fizzy-app-402.df-ams-int.37signals.com - fizzy-app-402.df-ams-int.37signals.com: df_ams
labels: labels:
otel_scrape_enabled: true otel_scrape_enabled: true
jobs: jobs:
hosts: hosts:
- fizzy-app-101.df-iad-int.37signals.com - fizzy-app-101.df-iad-int.37signals.com: df_iad
options: - fizzy-app-102.df-iad-int.37signals.com: df_iad
# NFS mount for backups
# See https://3.basecamp.com/2914079/buckets/37331921/todos/9175385847
mount: type=volume,src=backups,dst=/backups,volume-driver=local,volume-opt=type=nfs,volume-opt=device=:/fizzy-production-backups,"volume-opt=o=addr=purestorage.sc-chi-int.37signals.com,nfsvers=3,rw,noatime,nconnect=8,soft,timeo=30,retrans=2"
labels: labels:
otel_scrape_enabled: true otel_scrape_enabled: true
@@ -28,6 +25,26 @@ ssh:
env: env:
clear: clear:
RAILS_ENV: production RAILS_ENV: production
MYSQL_DATABASE_HOST: fizzy-mysql-primary
MYSQL_DATABASE_REPLICA_HOST: fizzy-mysql-replica
MYSQL_SOLID_CABLE_HOST: fizzy-mysql-primary
MYSQL_SOLID_QUEUE_HOST: fizzy-mysql-primary
secret:
- RAILS_MASTER_KEY
- MYSQL_ALTER_PASSWORD
- MYSQL_ALTER_USER
- MYSQL_APP_PASSWORD
- MYSQL_APP_USER
- MYSQL_READONLY_PASSWORD
- MYSQL_READONLY_USER
tags:
sc_chi:
MYSQL_SOLID_CACHE_HOST: fizzy-solidcache-db-101.df-iad-int.37signals.com
df_iad:
MYSQL_SOLID_CACHE_HOST: fizzy-solidcache-db-101.df-iad-int.37signals.com
df_ams:
MYSQL_SOLID_CACHE_HOST: fizzy-solidcache-db-101.df-iad-int.37signals.com
accessories: accessories:
load-balancer: load-balancer:
+28 -7
View File
@@ -1,17 +1,18 @@
servers: servers:
web: web:
hosts: hosts:
- fizzy-staging-app-01.sc-chi-int.37signals.com - fizzy-staging-app-101.df-iad-int.37signals.com: df_iad
- fizzy-staging-app-02.sc-chi-int.37signals.com - fizzy-staging-app-102.df-iad-int.37signals.com: df_iad
- fizzy-staging-app-101.df-iad-int.37signals.com - fizzy-staging-app-01.sc-chi-int.37signals.com: sc_chi
- fizzy-staging-app-102.df-iad-int.37signals.com - fizzy-staging-app-02.sc-chi-int.37signals.com: sc_chi
- fizzy-staging-app-401.df-ams-int.37signals.com - fizzy-staging-app-401.df-ams-int.37signals.com: df_ams
- fizzy-staging-app-402.df-ams-int.37signals.com - fizzy-staging-app-402.df-ams-int.37signals.com: df_ams
labels: labels:
otel_scrape_enabled: true otel_scrape_enabled: true
jobs: jobs:
hosts: hosts:
- fizzy-staging-app-01.sc-chi-int.37signals.com - fizzy-staging-app-101.df-iad-int.37signals.com: df_iad
- fizzy-staging-app-102.df-iad-int.37signals.com: df_iad
labels: labels:
otel_scrape_enabled: true otel_scrape_enabled: true
@@ -24,6 +25,26 @@ ssh:
env: env:
clear: clear:
RAILS_ENV: staging RAILS_ENV: staging
MYSQL_DATABASE_HOST: fizzy-staging-mysql-primary
MYSQL_DATABASE_REPLICA_HOST: fizzy-staging-mysql-replica
MYSQL_SOLID_CABLE_HOST: fizzy-staging-mysql-primary
MYSQL_SOLID_QUEUE_HOST: fizzy-staging-mysql-primary
secret:
- RAILS_MASTER_KEY
- MYSQL_ALTER_PASSWORD
- MYSQL_ALTER_USER
- MYSQL_APP_PASSWORD
- MYSQL_APP_USER
- MYSQL_READONLY_PASSWORD
- MYSQL_READONLY_USER
tags:
sc_chi:
MYSQL_SOLID_CACHE_HOST: fizzy-staging-solidcache-db-101.df-iad-int.37signals.com
df_iad:
MYSQL_SOLID_CACHE_HOST: fizzy-staging-solidcache-db-101.df-iad-int.37signals.com
df_ams:
MYSQL_SOLID_CACHE_HOST: fizzy-staging-solidcache-db-101.df-iad-int.37signals.com
accessories: accessories:
load-balancer: load-balancer:
+5
View File
@@ -3,3 +3,8 @@ ActiveSupport.on_load(:active_storage_blob) do
expires_in 5.minutes, public: true expires_in 5.minutes, public: true
end end
end end
# Use DB read/write splitting for Active Storage models
ActiveSupport.on_load(:active_storage_record) do
connects_to database: { writing: :primary, reading: :replica }
end
+19 -41
View File
@@ -4,66 +4,44 @@ set -e
# fizzy-staging-lb-01.sc-chi-int.37signals.com # fizzy-staging-lb-01.sc-chi-int.37signals.com
# #
# Service Host Path Target State TLS
# fizzy fizzy.37signals-staging.com / fizzy-staging-app-01.sc-chi-int.37signals.com,fizzy-staging-app-02.sc-chi-int.37signals.com running yes
# fizzy-admin fizzy.37signals-staging.com /admin fizzy-staging-app-01.sc-chi-int.37signals.com running yes
ssh app@fizzy-staging-lb-01.sc-chi-int.37signals.com \ ssh app@fizzy-staging-lb-01.sc-chi-int.37signals.com \
docker exec fizzy-load-balancer \ docker exec fizzy-load-balancer \
kamal-proxy deploy fizzy \ kamal-proxy deploy fizzy \
--force \
--tls \ --tls \
--host=fizzy.37signals-staging.com \ --host=fizzy.37signals-staging.com \
--target=fizzy-staging-app-01.sc-chi-int.37signals.com \ --writer-affinity-timeout=1s \
--read-target=fizzy-staging-app-02.sc-chi-int.37signals.com \ --tls-acme-cache-path=/certificates \
--tls-acme-cache-path=/certificates --target=fizzy-staging-app-101.df-iad-int.37signals.com \
--target=fizzy-staging-app-102.df-iad-int.37signals.com \
ssh app@fizzy-staging-lb-01.sc-chi-int.37signals.com \ --read-target=fizzy-staging-app-01.sc-chi-int.37signals.com \
docker exec fizzy-load-balancer \ --read-target=fizzy-staging-app-02.sc-chi-int.37signals.com
kamal-proxy deploy fizzy-admin \
--host=fizzy.37signals-staging.com \
--path-prefix /admin \
--strip-path-prefix=false \
--target=fizzy-staging-app-01.sc-chi-int.37signals.com
# fizzy-staging-lb-101.df-iad-int.37signals.com # fizzy-staging-lb-101.df-iad-int.37signals.com
# #
# Service Host Path Target State TLS
# fizzy fizzy.37signals-staging.com / fizzy-staging-app-01.sc-chi-int.37signals.com,fizzy-staging-app-102.df-iad-int.37signals.com running yes
# fizzy-admin fizzy.37signals-staging.com /admin fizzy-staging-app-01.sc-chi-int.37signals.com running yes
ssh app@fizzy-staging-lb-101.df-iad-int.37signals.com \ ssh app@fizzy-staging-lb-101.df-iad-int.37signals.com \
docker exec fizzy-load-balancer \ docker exec fizzy-load-balancer \
kamal-proxy deploy fizzy \ kamal-proxy deploy fizzy \
--force \
--tls \ --tls \
--host=fizzy.37signals-staging.com \ --host=fizzy.37signals-staging.com \
--target=fizzy-staging-app-01.sc-chi-int.37signals.com \ --writer-affinity-timeout=1s \
--read-target=fizzy-staging-app-102.df-iad-int.37signals.com \ --tls-acme-cache-path=/certificates \
--tls-acme-cache-path=/certificates --target=fizzy-staging-app-101.df-iad-int.37signals.com \
--target=fizzy-staging-app-102.df-iad-int.37signals.com
ssh app@fizzy-staging-lb-101.df-iad-int.37signals.com \
docker exec fizzy-load-balancer \
kamal-proxy deploy fizzy-admin \
--host=fizzy.37signals-staging.com \
--path-prefix /admin \
--strip-path-prefix=false \
--target=fizzy-staging-app-01.sc-chi-int.37signals.com
# fizzy-staging-lb-401.df-ams-int.37signals.com # fizzy-staging-lb-401.df-ams-int.37signals.com
# #
# Service Host Path Target State TLS
# fizzy fizzy.37signals-staging.com / fizzy-staging-app-01.sc-chi-int.37signals.com,fizzy-staging-app-402.df-ams-int.37signals.com running yes
# fizzy-admin fizzy.37signals-staging.com /admin fizzy-staging-app-01.sc-chi-int.37signals.com running yes
ssh app@fizzy-staging-lb-401.df-ams-int.37signals.com \ ssh app@fizzy-staging-lb-401.df-ams-int.37signals.com \
docker exec fizzy-load-balancer \ docker exec fizzy-load-balancer \
kamal-proxy deploy fizzy \ kamal-proxy deploy fizzy \
--force \
--tls \ --tls \
--host=fizzy.37signals-staging.com \ --host=fizzy.37signals-staging.com \
--target=fizzy-staging-app-01.sc-chi-int.37signals.com \ --writer-affinity-timeout=1s \
--read-target=fizzy-staging-app-402.df-ams-int.37signals.com \ --tls-acme-cache-path=/certificates \
--tls-acme-cache-path=/certificates --target=fizzy-staging-app-101.df-iad-int.37signals.com \
--target=fizzy-staging-app-102.df-iad-int.37signals.com \
--read-target=fizzy-staging-app-401.df-ams-int.37signals.com \
--read-target=fizzy-staging-app-402.df-ams-int.37signals.com
ssh app@fizzy-staging-lb-401.df-ams-int.37signals.com \
docker exec fizzy-load-balancer \
kamal-proxy deploy fizzy-admin \
--host=fizzy.37signals-staging.com \
--path-prefix /admin \
--strip-path-prefix=false \
--target=fizzy-staging-app-01.sc-chi-int.37signals.com