Blue green deploys are working!

ansible/deploy.yml

@@ -11,5 +11,3 @@
   when: '"postgres" in group_names or "mongo" in group_names or "redis" in group_names or "elasticsearch" in group_names'
 - import_playbook: playbooks/deploy_task.yml
   when: "'task' in group_names"
-- import_playbook: playbooks/deploy_staging.yml
-  when: "'staging' in group_names"

ansible/playbooks/deploy_app.yml

@@ -5,6 +5,8 @@
   # serial: "50%"
   vars_files:
     - ../env_vars/base.yml
+  vars:
+    haproxy_host: "{{ 'hstaging' if 'staging' in inventory_hostname else 'hwww' }}"
 
   tasks:
     # - name: Leave consul
@@ -153,9 +155,39 @@
       delay: 5
       until: result_8001.status == 200
 
-    - name: Wait 20 seconds
+    - name: Update HAProxy config to use backup server
+      delegate_to: "{{ haproxy_host }}"
+      block:
+        - name: Generate temporary HAProxy config with backup server
+          template:
+            src: /srv/newsblur/docker/haproxy/haproxy.consul.cfg.j2
+            dest: /srv/newsblur/docker/haproxy/haproxy.consul.cfg.new
+          when: disable_consul_services_ie_staging is not defined
+          vars:
+            maintenance_mode: true
+
+        - name: Generate temporary HAProxy config with backup server
+          template:
+            src: /srv/newsblur/docker/haproxy/haproxy.staging.cfg
+            dest: /srv/newsblur/docker/haproxy/haproxy.consul.cfg.new
+          when: disable_consul_services_ie_staging is defined
+          vars:
+            maintenance_mode: true
+
+        - name: Verify new HAProxy config
+          shell: docker exec haproxy haproxy -c -f /srv/newsblur/docker/haproxy/haproxy.consul.cfg.new
+          register: haproxy_verified
+          failed_when: haproxy_verified.rc != 0
+
+        - name: Replace active HAProxy config
+          command: mv /srv/newsblur/docker/haproxy/haproxy.consul.cfg.new /srv/newsblur/docker/haproxy/haproxy.consul.cfg
+
+        - name: Reload HAProxy configuration
+          shell: "docker kill -s HUP haproxy"
+
+    - name: Wait for HAProxy to start using new configuration
       pause:
-        seconds: 20
+        seconds: 2
 
     - name: Stop primary container
       docker_container:
@@ -163,10 +195,6 @@
         state: stopped
         container_default_behavior: no_defaults
 
-    - name: Wait 20 seconds more for fun
-      pause:
-        seconds: 20
-
     - name: Deploy new container
       docker_container:
         name: newsblur_web
@@ -201,9 +229,35 @@
       delay: 5
       until: result_8000.status == 200
 
-    - name: Wait 20 seconds one last time
-      pause:
-        seconds: 20
+    - name: Update HAProxy config to use new server
+      delegate_to: "{{ haproxy_host }}"
+      block:
+        - name: Generate temporary HAProxy config with new server
+          template:
+            src: /srv/newsblur/docker/haproxy/haproxy.consul.cfg.j2
+            dest: /srv/newsblur/docker/haproxy/haproxy.consul.cfg.new
+          when: disable_consul_services_ie_staging is not defined
+          vars:
+            maintenance_mode: false
+
+        - name: Generate temporary HAProxy config with new server
+          template:
+            src: /srv/newsblur/docker/haproxy/haproxy.staging.cfg
+            dest: /srv/newsblur/docker/haproxy/haproxy.consul.cfg.new
+          when: disable_consul_services_ie_staging is defined
+          vars:
+            maintenance_mode: false
+
+        - name: Verify new HAProxy config
+          shell: docker exec haproxy haproxy -c -f /srv/newsblur/docker/haproxy/haproxy.consul.cfg.new
+          register: haproxy_verified
+          failed_when: haproxy_verified.rc != 0
+
+        - name: Replace active HAProxy config
+          command: mv /srv/newsblur/docker/haproxy/haproxy.consul.cfg.new /srv/newsblur/docker/haproxy/haproxy.consul.cfg
+
+        - name: Reload HAProxy configuration
+          shell: "docker kill -s HUP haproxy"
 
     - name: Stop and remove backup container
       docker_container:

ansible/roles/haproxy/tasks/main.yml

@@ -96,3 +96,4 @@
   apt:
     name: socat
     state: present
+  become: yes

docker/haproxy/haproxy.consul.cfg.j2

@@ -25,11 +25,11 @@ defaults
     option httplog
     option redispatch
     option abortonclose
+    retries 2
     timeout connect 10s
     timeout client 10s
-    timeout server 30s
-    timeout tunnel 1h
-    retries 3
+    timeout server 10s
+    timeout queue 10s
     errorfile 502 /srv/newsblur/templates/502.http
     errorfile 503 /srv/newsblur/templates/502.http
     errorfile 504 /srv/newsblur/templates/502.http
@@ -104,26 +104,38 @@ backend nginx
 
 backend app_django
     option httpchk GET /_haproxychk
-    default-server check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
+    default-server check inter 1000ms on-error mark-down fall 2 rise 2 resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
     {% for host in groups.hdjango %}
-        server {{host}} {{host}}.node.nyc1.consul:8000
+        {% if maintenance_mode is defined and maintenance_mode %}
+            server {{host}} {{host}}.node.nyc1.consul:8001 backup
+        {% else %}
+            server {{host}} {{host}}.node.nyc1.consul:8000
+        {% endif %}
     {% endfor %}
 
 backend app_count
     balance roundrobin
     option httpchk GET /_haproxychk
-    default-server check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
+    default-server check inter 1000ms on-error mark-down fall 2 rise 2 resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
     {% for host in groups.hcount %}
-        server {{host}} {{host}}.node.nyc1.consul:8000
+        {% if maintenance_mode is defined and maintenance_mode %}
+            server {{host}} {{host}}.node.nyc1.consul:8001 backup
+        {% else %}
+            server {{host}} {{host}}.node.nyc1.consul:8000
+        {% endif %}
     {% endfor %}
     # server-template app-counts 1 _app-counts._tcp.service.nyc1.consul:8000 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
     
 backend app_refresh
     balance roundrobin
     option httpchk GET /_haproxychk
-    default-server check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
+    default-server check inter 2000ms on-error mark-down fall 2 rise 2 resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
     {% for host in groups.hrefresh %}
-        server {{host}} {{host}}.node.nyc1.consul:8000
+        {% if maintenance_mode is defined and maintenance_mode %}
+            server {{host}} {{host}}.node.nyc1.consul:8001 backup
+        {% else %}
+            server {{host}} {{host}}.node.nyc1.consul:8000
+        {% endif %}
     {% endfor %}
     # server-template app-refresh 1 _app-refresh._tcp.service.nyc1.consul:8000 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
 
@@ -132,7 +144,11 @@ backend app_push
     option httpchk GET /_haproxychk
     default-server check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
     {% for host in groups.hpush %}
-        server {{host}} {{host}}.node.nyc1.consul:8000
+        {% if maintenance_mode is defined and maintenance_mode %}
+            server {{host}} {{host}}.node.nyc1.consul:8001 backup
+        {% else %}
+            server {{host}} {{host}}.node.nyc1.consul:8000
+        {% endif %}
     {% endfor %}
     # server-template app-push 1 _app-push._tcp.service.nyc1.consul:8000 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
 
@@ -182,8 +198,11 @@ backend staging
     option httpchk GET /_haproxychk
     default-server check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
     {% for host in groups.staging %}
-        server {{host}} {{host}}.node.nyc1.consul:8000
-        server {{host}} {{host}}.node.nyc1.consul:8001 backup
+        {% if maintenance_mode is defined and maintenance_mode %}
+            server {{host}} {{host}}.node.nyc1.consul:8001 backup
+        {% else %}
+            server {{host}} {{host}}.node.nyc1.consul:8000
+        {% endif %}
     {% endfor %}
 
 backend blog

docker/haproxy/haproxy.staging.cfg

@@ -113,20 +113,29 @@ backend nginx
 backend app_django_counts
     balance roundrobin
     option httpchk GET /_haproxychk
+    {% if maintenance_mode is not defined or not maintenance_mode %}
     server app_django 127.0.0.1:8000 check inter 3000ms
+    {% else %}
     server app_django_backup 127.0.0.1:8001 check inter 3000ms backup
+    {% endif %}
 
 backend app_django_refresh
     balance roundrobin
     option httpchk GET /_haproxychk
+    {% if maintenance_mode is not defined or not maintenance_mode %}
     server app_django 127.0.0.1:8000 check inter 3000ms
+    {% else %}
     server app_django_backup 127.0.0.1:8001 check inter 3000ms backup
+    {% endif %}
 
 backend app_django
     balance roundrobin
     option httpchk GET /_haproxychk
+    {% if maintenance_mode is not defined or not maintenance_mode %}
     server app_django 127.0.0.1:8000 check inter 3000ms
+    {% else %}
     server app_django_backup 127.0.0.1:8001 check inter 3000ms backup
+    {% endif %}
 
 backend maintenance
     option httpchk HEAD /maintenance