Rewriting HAProxy config to include gunicorn and maintenance. Much better monitoring.

config/debug_haproxy.conf

@@ -17,11 +17,13 @@ defaults
   option log-health-checks
   option httplog
   option redispatch
+  option abortonclose
   timeout connect 5s
   timeout client 30s
   timeout server 30s
   timeout tunnel 1h
   retries 3
+  errorfile 502 /srv/newsblur/templates/502.http
   errorfile 503 /srv/newsblur/templates/502.http
   errorfile 504 /srv/newsblur/templates/502.http
  
@@ -32,20 +34,26 @@ frontend public
   # Redirect all HTTP traffic to HTTPS
   # redirect scheme https if !{ ssl_fc }
 
-  acl site_dead nbsrv(gunicorn) lt 1
+  acl gunicorn_dead nbsrv(gunicorn) lt 1
-  acl site_dead nbsrv(nginx) lt 1
+  acl nginx_dead nbsrv(nginx) lt 1
-  monitor fail  if site_dead
+  acl mx_mode nbsrv(maintenance) lt 1
+  monitor-uri /status
+  monitor fail  if gunicorn_dead
+  monitor fail  if nginx_dead
+  monitor fail  if mx_mode
   
   use_backend node if { path_beg /socket.io/ }
   use_backend nginx if { path_beg /media/ }
   use_backend nginx if { path_beg /static/ }
-  use_backend nginx if { path_beg /favicon/ }
+  use_backend nginx if { path_beg /favicon }
   use_backend nginx if { path_beg /crossdomain/ }
-  use_backend nginx if { path_beg /robots/ }
+  use_backend nginx if { path_beg /robots }
   use_backend nginx if { path_beg /munin/ }
   use_backend nginx if { path_beg /rss_feeds/icon }
   
-  default_backend gunicorn
+  use_backend nginx if mx_mode
+  
+  use_backend gunicorn unless gunicorn_dead || nginx_dead
  
 backend node
   balance roundrobin
@@ -59,6 +67,12 @@ backend gunicorn
   balance roundrobin
   server gunicorndebug 127.0.0.1:8000       check inter 500ms
 
+backend maintenance
+  option httpchk HEAD /maintenance HTTP/1.1\r\nHost:\ www
+  http-check expect status 404
+  http-check send-state
+  server nginxdebug 127.0.0.1:81       check inter 500ms
+  
 frontend stats
   bind :1936 ssl crt newsblur.pem
   default_backend stats
@@ -69,4 +83,4 @@ backend stats
     stats realm Haproxy\ Statistics
     stats uri /
     stats auth sclay:password
-    stats refresh 5s
+    stats refresh 15s

config/haproxy.conf

@@ -5,30 +5,55 @@ global
   daemon
   ca-base /srv/newsblur/config/certificates
   crt-base /srv/newsblur/config/certificates
+  log 127.0.0.1 local0 notice
+  log 127.0.0.1 local1 info
 
 defaults
   log global
   maxconn 4096
   mode http
-  # Add x-forwarded-for header.
   option forwardfor
   option http-server-close
+  option log-health-checks
+  option httplog
+  option redispatch
+  option abortonclose
   timeout connect 5s
   timeout client 30s
   timeout server 30s
-  # Long timeout for WebSocket connections.
   timeout tunnel 1h
-  errorfile 503 /srv/newsblur/templates/502.html
+  retries 3
-  errorfile 504 /srv/newsblur/templates/502.html
+  errorfile 502 /srv/newsblur/templates/502.http
+  errorfile 503 /srv/newsblur/templates/502.http
+  errorfile 504 /srv/newsblur/templates/502.http
  
 frontend public
   bind :80
   bind :443 ssl crt newsblur.pem
+  option http-server-close
   # Redirect all HTTP traffic to HTTPS
   # redirect scheme https if !{ ssl_fc }
 
+  acl gunicorn_dead nbsrv(gunicorn) lt 1
+  acl nginx_dead nbsrv(nginx) lt 1
+  acl mx_mode nbsrv(maintenance) lt 1
+  monitor-uri /status
+  monitor fail  if gunicorn_dead
+  monitor fail  if nginx_dead
+  monitor fail  if mx_mode
+  
   use_backend node if { path_beg /socket.io/ }
-  default_backend nginx
+  use_backend nginx if { path_beg /media/ }
+  use_backend nginx if { path_beg /static/ }
+  use_backend nginx if { path_beg /favicon }
+  use_backend nginx if { path_beg /crossdomain/ }
+  use_backend nginx if { path_beg /robots }
+  use_backend nginx if { path_beg /munin/ }
+  use_backend nginx if { path_beg /rss_feeds/icon }
+  
+  use_backend nginx if mx_mode
+  
+  use_backend gunicorn unless gunicorn_dead || nginx_dead
  
 backend node
   balance roundrobin
@@ -46,7 +71,7 @@ backend node
   server node21 198.211.106.215:8888    check inter 2000ms
   server node22 192.81.209.42:8888      check inter 2000ms
   server node23 198.211.102.245:8888    check inter 2000ms
- 
+
 backend nginx
   balance roundrobin
   server nginx10 198.211.109.197:81     check inter 2000ms
@@ -64,6 +89,28 @@ backend nginx
   server nginx22 192.81.209.42:80       check inter 2000ms
   server nginx23 198.211.102.245:80     check inter 2000ms
 
+backend gunicorn
+  balance roundrobin
+  server gunicorn10 198.211.109.197:81     check inter 2000ms
+  server gunicorn11 198.211.110.131:80     check inter 2000ms
+  server gunicorn12 198.211.110.230:80     check inter 2000ms
+  server gunicorn13 192.34.61.227:80       check inter 2000ms
+  server gunicorn14 198.211.109.155:80     check inter 2000ms
+  server gunicorn15 198.211.107.87:80      check inter 2000ms
+  server gunicorn16 198.211.105.155:80     check inter 2000ms
+  server gunicorn17 198.211.104.133:80     check inter 2000ms
+  server gunicorn18 198.211.103.214:80     check inter 2000ms
+  server gunicorn19 198.211.106.22:80      check inter 2000ms
+  server gunicorn20 198.211.110.189:80     check inter 2000ms
+  server gunicorn21 198.211.106.215:80     check inter 2000ms
+  server gunicorn22 192.81.209.42:80       check inter 2000ms
+  server gunicorn23 198.211.102.245:80     check inter 2000ms
+
+backend maintenance
+  option httpchk HEAD /maintenance HTTP/1.1\r\nHost:\ www
+  http-check expect status 404
+  http-check send-state
+  server nginx10 198.211.109.197:81     check inter 2000ms
   
 frontend stats
   bind :1936 ssl crt newsblur.pem
@@ -74,4 +121,5 @@ backend stats
     stats hide-version
     stats realm Haproxy\ Statistics
     stats uri /
-    stats auth sclay:password
+    stats auth sclay:password
+    stats refresh 15s

config/nginx.newsblur.conf

@@ -8,8 +8,8 @@ upstream icon_server {
 }
 
 server {
-    listen   80;
+    listen   81;
-    listen   443 default_server ssl;
+    # listen   443 default_server ssl;
 
     # ssl                     on;
     ssl_certificate         /srv/newsblur/config/certificates/newsblur.com.crt;
@@ -65,6 +65,12 @@ server {
         access_log off;
     }
 
+    location /maintenance {
+        alias      /srv/newsblur/templates/maintenance_on.png;
+        expires max;
+        access_log off;
+    }
+
     location ^~ /crossdomain.xml {
         expires max;
         alias /srv/newsblur/media/crossdomain.xml;

fabfile.py

@@ -52,6 +52,7 @@ env.roledefs ={
             '198.211.102.245',
             ],
     'dev': ['dev.newsblur.com'],
+    'debug': ['debug.newsblur.com'],
     'web': ['app01.newsblur.com', 
             'app02.newsblur.com', 
             'app04.newsblur.com',
@@ -154,6 +155,10 @@ def do():
     server()
     env.roles = ['do']
 
+def debug():
+    server()
+    env.roles = ['debug']
+
 def ec2():
     env.user = 'ubuntu'
     env.key_filename = ['/Users/sclay/.ec2/sclay.pem']