Add monitoring basics

Procfile

@@ -0,0 +1,2 @@
+web:     bundle exec thin start -p $PORT
+faye:    rackup faye.ru -s thin -E production

config/deploy.rb

@@ -1,6 +1,8 @@
 #$:.unshift(File.expand_path('./lib', ENV['rvm_path']))
 require "bundler/capistrano"
 #load 'deploy/assets'
+#
+load 'config/recipes/monit'
 
 # Load RVM's capistrano plugin.    
 #require "rvm/capistrano"

config/recipes/monit.rb

@@ -0,0 +1,39 @@
+namespace :monit do
+  desc "Install Monit"
+  task :install do
+    run "#{sudo} apt-get -y install monit"
+  end
+  after "deploy:install", "monit:install"
+
+  desc "Setup all Monit configuration"
+  task :setup do
+    monit_config "monitrc", "/etc/monit/monitrc"
+    #nginx
+    #postgresql
+    #unicorn
+    faye
+    syntax
+    reload
+  end
+  after "deploy:setup", "monit:setup"
+  
+  task(:nginx, roles: :web) { monit_config "nginx" }
+  task(:postgresql, roles: :db) { monit_config "postgresql" }
+  task(:unicorn, roles: :app) { monit_config "unicorn" }
+  task(:faye, roles: :app) { monit_config "faye" }
+
+  %w[start stop restart syntax reload].each do |command|
+    desc "Run Monit #{command} script"
+    task command do
+      run "#{sudo} service monit #{command}"
+    end
+  end
+end
+
+def monit_config(name, destination = nil)
+  destination ||= "/etc/monit/conf.d/#{name}.conf"
+  template "monit/#{name}.erb", "/tmp/monit_#{name}"
+  run "#{sudo} mv /tmp/monit_#{name} #{destination}"
+  run "#{sudo} chown root #{destination}"
+  run "#{sudo} chmod 600 #{destination}"
+end

config/recipes/postgresql.rb

@@ -0,0 +1 @@
+set_default(:postgresql_pid) { "/var/run/postgresql/9.1-main.pid" }

config/recipes/templates/monit/monitrc.erb

@@ -0,0 +1,26 @@
+set daemon 30
+
+set logfile /var/log/monit.log
+set idfile /var/lib/monit/id
+set statefile /var/lib/monit/state
+
+set eventqueue
+    basedir /var/lib/monit/events
+    slots 100
+
+# set mailserver smtp.gmail.com port 587
+#     username "foo@example.com" password "secret"
+#     using tlsv1
+#     with timeout 30 seconds
+
+set alert bterkuile+uflows-server-alert@gmail.com
+
+set httpd port 2812
+    allow admin:"Monit22Secret"
+
+check system blog_server
+  if loadavg(5min) > 2 for 2 cycles then alert
+  if memory > 75% for 2 cycles then alert
+  if cpu(user) > 75% for 2 cycles then alert
+
+include /etc/monit/conf.d/*

config/recipes/templates/monit/nginx.erb

@@ -0,0 +1,5 @@
+check process nginx with pidfile /var/run/nginx.pid
+  start program = "/etc/init.d/nginx start"
+  stop program = "/etc/init.d/nginx stop"
+  if children > 250 then restart
+  if 5 restarts within 5 cycles then timeout

config/recipes/templates/monit/postgresql.erb

@@ -0,0 +1,5 @@
+check process postgresql with pidfile <%= postgresql_pid %>
+  start program = "/etc/init.d/postgresql start"
+  stop program = "/etc/init.d/postgresql stop"
+  if failed host localhost port 5432 protocol pgsql then restart
+  if 5 restarts within 5 cycles then timeout

config/recipes/templates/monit/unicorn.rb.erb

@@ -0,0 +1,8 @@
+after_fork do |server, worker|
+  # Start up the database connection again in the worker
+  if defined?(ActiveRecord::Base)
+    ActiveRecord::Base.establish_connection
+  end
+  child_pid = server.config[:pid].sub(".pid", ".#{worker.nr}.pid")
+  system("echo #{Process.pid} > #{child_pid}")
+end

config/recipes/templates/unicorn.erb

@@ -0,0 +1,15 @@
+check process <%= application %>_unicorn with pidfile <%= unicorn_pid %>
+  start program = "/etc/init.d/unicorn_<%= application %> start"
+  stop program = "/etc/init.d/unicorn_<%= application %> stop"
+
+<% unicorn_workers.times do |n| %>
+  <% pid = unicorn_pid.sub(".pid", ".#{n}.pid") %>
+  check process <%= application %>_unicorn_worker_<%= n %> with pidfile <%= pid %>
+    start program = "/bin/true"
+    stop program = "/usr/bin/test -s <%= pid %> && /bin/kill -QUIT `cat <%= pid %>`"
+    if mem > 200.0 MB for 1 cycles then restart
+    if cpu > 50% for 3 cycles then restart
+    if 5 restarts within 5 cycles then timeout
+    alert foo@example.com only on { pid }
+    if changed pid 2 times within 60 cycles then alert
+<% end %>

config/recipes/unicorn.rb

@@ -0,0 +1 @@
+set_default(:unicorn_workers, 2)