From 80057b0bf5f81ae1b8dfa075735caeea34fa5a84 Mon Sep 17 00:00:00 2001 From: Benjamin ter Kuile Date: Wed, 17 Apr 2013 16:25:38 +0200 Subject: [PATCH] Add monitoring basics --- Procfile | 2 + config/deploy.rb | 2 + config/recipes/monit.rb | 39 +++++++++++++++++++ config/recipes/postgresql.rb | 1 + config/recipes/templates/monit/monitrc.erb | 26 +++++++++++++ config/recipes/templates/monit/nginx.erb | 5 +++ config/recipes/templates/monit/postgresql.erb | 5 +++ config/recipes/templates/monit/unicorn.rb.erb | 8 ++++ config/recipes/templates/unicorn.erb | 15 +++++++ config/recipes/unicorn.rb | 1 + 10 files changed, 104 insertions(+) create mode 100644 Procfile create mode 100644 config/recipes/monit.rb create mode 100644 config/recipes/postgresql.rb create mode 100644 config/recipes/templates/monit/monitrc.erb create mode 100644 config/recipes/templates/monit/nginx.erb create mode 100644 config/recipes/templates/monit/postgresql.erb create mode 100644 config/recipes/templates/monit/unicorn.rb.erb create mode 100644 config/recipes/templates/unicorn.erb create mode 100644 config/recipes/unicorn.rb diff --git a/Procfile b/Procfile new file mode 100644 index 00000000..17ec0be5 --- /dev/null +++ b/Procfile @@ -0,0 +1,2 @@ +web: bundle exec thin start -p $PORT +faye: rackup faye.ru -s thin -E production diff --git a/config/deploy.rb b/config/deploy.rb index b7c83042..f8890b3b 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -1,6 +1,8 @@ #$:.unshift(File.expand_path('./lib', ENV['rvm_path'])) require "bundler/capistrano" #load 'deploy/assets' +# +load 'config/recipes/monit' # Load RVM's capistrano plugin. #require "rvm/capistrano" diff --git a/config/recipes/monit.rb b/config/recipes/monit.rb new file mode 100644 index 00000000..eb9819e4 --- /dev/null +++ b/config/recipes/monit.rb @@ -0,0 +1,39 @@ +namespace :monit do + desc "Install Monit" + task :install do + run "#{sudo} apt-get -y install monit" + end + after "deploy:install", "monit:install" + + desc "Setup all Monit configuration" + task :setup do + monit_config "monitrc", "/etc/monit/monitrc" + #nginx + #postgresql + #unicorn + faye + syntax + reload + end + after "deploy:setup", "monit:setup" + + task(:nginx, roles: :web) { monit_config "nginx" } + task(:postgresql, roles: :db) { monit_config "postgresql" } + task(:unicorn, roles: :app) { monit_config "unicorn" } + task(:faye, roles: :app) { monit_config "faye" } + + %w[start stop restart syntax reload].each do |command| + desc "Run Monit #{command} script" + task command do + run "#{sudo} service monit #{command}" + end + end +end + +def monit_config(name, destination = nil) + destination ||= "/etc/monit/conf.d/#{name}.conf" + template "monit/#{name}.erb", "/tmp/monit_#{name}" + run "#{sudo} mv /tmp/monit_#{name} #{destination}" + run "#{sudo} chown root #{destination}" + run "#{sudo} chmod 600 #{destination}" +end diff --git a/config/recipes/postgresql.rb b/config/recipes/postgresql.rb new file mode 100644 index 00000000..a52a27a4 --- /dev/null +++ b/config/recipes/postgresql.rb @@ -0,0 +1 @@ +set_default(:postgresql_pid) { "/var/run/postgresql/9.1-main.pid" } diff --git a/config/recipes/templates/monit/monitrc.erb b/config/recipes/templates/monit/monitrc.erb new file mode 100644 index 00000000..fb0efbaf --- /dev/null +++ b/config/recipes/templates/monit/monitrc.erb @@ -0,0 +1,26 @@ +set daemon 30 + +set logfile /var/log/monit.log +set idfile /var/lib/monit/id +set statefile /var/lib/monit/state + +set eventqueue + basedir /var/lib/monit/events + slots 100 + +# set mailserver smtp.gmail.com port 587 +# username "foo@example.com" password "secret" +# using tlsv1 +# with timeout 30 seconds + +set alert bterkuile+uflows-server-alert@gmail.com + +set httpd port 2812 + allow admin:"Monit22Secret" + +check system blog_server + if loadavg(5min) > 2 for 2 cycles then alert + if memory > 75% for 2 cycles then alert + if cpu(user) > 75% for 2 cycles then alert + +include /etc/monit/conf.d/* diff --git a/config/recipes/templates/monit/nginx.erb b/config/recipes/templates/monit/nginx.erb new file mode 100644 index 00000000..c1d6dd59 --- /dev/null +++ b/config/recipes/templates/monit/nginx.erb @@ -0,0 +1,5 @@ +check process nginx with pidfile /var/run/nginx.pid + start program = "/etc/init.d/nginx start" + stop program = "/etc/init.d/nginx stop" + if children > 250 then restart + if 5 restarts within 5 cycles then timeout diff --git a/config/recipes/templates/monit/postgresql.erb b/config/recipes/templates/monit/postgresql.erb new file mode 100644 index 00000000..22e56526 --- /dev/null +++ b/config/recipes/templates/monit/postgresql.erb @@ -0,0 +1,5 @@ +check process postgresql with pidfile <%= postgresql_pid %> + start program = "/etc/init.d/postgresql start" + stop program = "/etc/init.d/postgresql stop" + if failed host localhost port 5432 protocol pgsql then restart + if 5 restarts within 5 cycles then timeout diff --git a/config/recipes/templates/monit/unicorn.rb.erb b/config/recipes/templates/monit/unicorn.rb.erb new file mode 100644 index 00000000..5302a4db --- /dev/null +++ b/config/recipes/templates/monit/unicorn.rb.erb @@ -0,0 +1,8 @@ +after_fork do |server, worker| + # Start up the database connection again in the worker + if defined?(ActiveRecord::Base) + ActiveRecord::Base.establish_connection + end + child_pid = server.config[:pid].sub(".pid", ".#{worker.nr}.pid") + system("echo #{Process.pid} > #{child_pid}") +end diff --git a/config/recipes/templates/unicorn.erb b/config/recipes/templates/unicorn.erb new file mode 100644 index 00000000..c1ca0a90 --- /dev/null +++ b/config/recipes/templates/unicorn.erb @@ -0,0 +1,15 @@ +check process <%= application %>_unicorn with pidfile <%= unicorn_pid %> + start program = "/etc/init.d/unicorn_<%= application %> start" + stop program = "/etc/init.d/unicorn_<%= application %> stop" + +<% unicorn_workers.times do |n| %> + <% pid = unicorn_pid.sub(".pid", ".#{n}.pid") %> + check process <%= application %>_unicorn_worker_<%= n %> with pidfile <%= pid %> + start program = "/bin/true" + stop program = "/usr/bin/test -s <%= pid %> && /bin/kill -QUIT `cat <%= pid %>`" + if mem > 200.0 MB for 1 cycles then restart + if cpu > 50% for 3 cycles then restart + if 5 restarts within 5 cycles then timeout + alert foo@example.com only on { pid } + if changed pid 2 times within 60 cycles then alert +<% end %> diff --git a/config/recipes/unicorn.rb b/config/recipes/unicorn.rb new file mode 100644 index 00000000..06a416f2 --- /dev/null +++ b/config/recipes/unicorn.rb @@ -0,0 +1 @@ +set_default(:unicorn_workers, 2)