Updating haproxy config, updating DB motd, and finishing up letsencrypt ansible role.

ansible/roles/base/files/motd.db.txt

@@ -13,51 +13,51 @@ white=`echo "\033[37;40m"`
 
 cat <<EOMOTD
 $bold  
-                                               $white               __
+                                               $yellow               __
-                                               $white         _   .'.-'.
+                                               $yellow         _   .'.-'.
-                                               $white       .'-'. ||  ||
+                                               $yellow       .'-'. ||  ||
-                                               $white   _   || || :\_.'/
+                                               $yellow   _   || || :\_.'/
-$yellow                               .-.       ___   $white .'-.\ :\_'/  '-''
+$magenta                               .-.       ___   $yellow .'-.\ :\_'/  '-''
-$yellow                              _L $cyano$yellow|_.-'''   '-.$white ;\_'/  '-'
+$magenta                              _L$cyan o$magenta|_.-'''   '-.$yellow ;\_'/  '-'
-$yellow                            _f $cyano$yellow\-'          /_. $white'-'
+$magenta                            _f$cyan o$magenta\-'          /_. $yellow'-'
-$yellow                        ,-'' '-'              /
+$magenta                        ,-'' '-'              /
-$yellow                       /                    .'
+$magenta                       /                    .'
-$yellow                      |                   .'             $white   .--.
+$magenta                      |                   .'             $yellow   .--.
-$yellow                       \              _.-'               $white   '-' |
+$magenta                       \              _.-'               $yellow   '-' |
-$yellow                        |        __.-'                   $white  .- ''
+$magenta                        |        __.-'                   $yellow  .- ''
-$yellow                       ,-.     ''|                       $white |
+$magenta                       $green,$magenta-.     ''|                       $yellow |
-$yellow                      /  |       |                       $white  '-.
+$magenta                    $green  /  $magenta|       |                       $yellow  '-.
-$yellow                      \_ |       |              _  .-.   $white   _ '
+$magenta                    $green  \_ $magenta|       |              _  .-.   $yellow   _ '
-$yellow                       /'|       |             ' . ' | $regular$yellow _.-$bold$red'7P'
+$magenta                    $green   /'$magenta|       |             ' . ' | $regular$yellow _.-$bold$red'7P'
-$yellow                      |  |       ;              \ \| |$regular$yellow-' _.-''
+$magenta                    $green  |  $magenta|       ;              \ \| |$regular$yellow-' _.-$bold$red''
-$yellow                      \_ |        ;           $regular$yellow_,-''$bold$yellow| |$regular$yellow.-'
+$magenta                    $green  \_ $magenta|        ;          $regular$yellow _,-''$bold$magenta| |$regular$yellow.-' $bold
-$yellow                       /'|        |          $regular$yellow()$bold$yellow_,.-j f.-.
+$magenta                    $green   /'$magenta|        |          $regular$yellow()_,-$bold$magenta j f$regular$magenta.-. $bold
-$bold $yellow                     |  |        |           .'') (_.-.j
+$magenta                    $green  |  $magenta|        |           .'') (_.-.j
-                      \_ ;        ;          (  <   (__.'
+$magenta                    $green  \_ $magenta;        ;          (  <   (__.'
-                       / '        ;           \  '    /
+$magenta                    $green   / $magenta'        ;           \  '    /
-                      |   |        \          /      ;
+$magenta                    $green  |  $magenta |        \          /      ;
-                       \_ |         ''--.___.'       |
+$magenta                    $green   \_$magenta |         ''--.___.'       |
-                      /' .'                          ;
+$magenta                    $green  /' $magenta.'                          ;
-                      \.'                  _        /
+$magenta                    $green  \.$magenta'                  _        /
-                      /                     '.    .'
+$magenta                      /                     '.    .'
                     .'                        \.-'
                    /                           '.
                   ;                              \ 
                   |                               '
                   '         /                      \ 
-                 ;         /       $regular$red NewsBlur $bold$yellow       \ 
+                 ;         /       $regular$red NewsBlur $bold$magenta       \ 
                  |        Y                          ;
-                 ;         \        $red DB Server$yellow        .
+                 ;         \        $red DB Server$magenta        .
                  '        t-'                         ;
                   \  \  \  \                          ;
-                  7'._f_.'-'                          |
+$magenta          $green        7$magenta'._f_.'-'                          |
-                  \_|                                 |
+$magenta          $green        \_$magenta|                                 |
-                  /'|                                 ;
+$magenta          $green        /'$magenta|                                 ;
-                  \_;                                 '
+$magenta          $green        \_$magenta;                                 '
-     .-.          f  '                               /
+$magenta     .-.  $green        f $magenta '                               /
-  .'  ,-'          \_|                              /
+$magenta  .'  ,-' $green         \_$magenta|                              /
- /   |     _  __ .-7 /                            .'
+$magenta /   |    $green _  __ .-7 $magenta/                            .'
- |    '-.r'_'Y__\L.-'                          _.'
+$magenta |    '-.r'_'Y__\L.-'                          _.'
  '                        ;              _..--' |
   '.                      ,              |      '.
     '-._                 /|              '.       |

ansible/roles/letsencrypt/tasks/certbot-dns.yml

@@ -78,10 +78,21 @@
   tags:
     - certbot-dns
 
+- name: Ensure permissions on certificates
+  become: yes
+  file: 
+    path: /etc/letsencrypt/
+    owner: nb
+    group: nb
+    recurse: yes
+
 - name: Symlink certificates to ssl_certificate and ssl_certificate_key
+  become: yes
   file:
     src: '{{ item.src }}'
     dest: '{{ item.dest }}'
+    owner: nb
+    group: nb
     state: link
   when: ssl_certificate is defined and ssl_certificate_key is defined and use_dns_plugin|bool
   with_items:

ansible/setup_redis.yml

@@ -2,7 +2,6 @@
 - name: Set up redis containers
   hosts: db_redis
   remote_user: nb
-  become: true
   vars:
     - update_apt_cache: yes
     - motd_role: db

docker/haproxy/Dockerfile

@@ -7,4 +7,4 @@ WORKDIR   /srv/newsblur
 RUN /bin/bash -c 'echo "ENABLED=1" | tee /etc/default/haproxy'
 RUN /bin/bash -c 'mkdir -p /srv/newsblur/config/certificates/'
 COPY 'config/haproxy_rsyslog.conf' '/etc/rsyslog.d/49-haproxy.conf'
-COPY './docker/haproxy/haproxy.conf' '/usr/local/etc/haproxy/haproxy.cfg'
+COPY './docker/haproxy/haproxy.dev.conf' '/usr/local/etc/haproxy/haproxy.cfg'

docker/haproxy/haproxy.cfg

@@ -9,6 +9,9 @@ global
     log 127.0.0.1 local0 notice
     # log 127.0.0.1 local1 info
 
+resolvers docker
+    nameserver dns1 127.0.0.11:53
+
 defaults
     log global
     maxconn 100000
@@ -32,7 +35,7 @@ defaults
 
 frontend public
     bind :80
-    bind :443 ssl crt /srv/newsblur/config/certificates/localhost.pem #ciphers ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES128-SHA:AES256-SHA256:AES256-SHA no-sslv3
+    bind :443 ssl crt /srv/newsblur/config/certificates/newsblur.com.pem #ciphers ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES128-SHA:AES256-SHA256:AES256-SHA no-sslv3
     rspadd Strict-Transport-Security:\ max-age=0;\ includeSubDomains
     option http-server-close
 
@@ -53,6 +56,7 @@ frontend public
     use_backend node_socket if { path_beg /v2/socket.io/ }
     use_backend node_favicon if { path_beg /rss_feeds/icon/ }
     use_backend node_text if { path_beg /rss_feeds/original_text_fetcher }
+    use_backend node_page if { path_beg /original_page/ }
     use_backend nginx if { path_beg /media/ }
     use_backend nginx if { path_beg /static/ }
     use_backend nginx if { path_beg /favicon }
@@ -62,60 +66,67 @@ frontend public
 
     use_backend nginx if mx_mode
 
-    use_backend gunicorn_counts if is_unread_count
+    use_backend app_counts if is_unread_count
-    use_backend gunicorn_refresh if is_refresh_feeds
+    use_backend app_refresh if is_refresh_feeds
     use_backend gunicorn unless gunicorn_dead || nginx_dead
 
 backend node_socket
     http-check expect rstatus 200|503
     balance roundrobin
-    server node_socket01 node_websocket:8888 check inter 3000ms
+    server node_socket01 node:8888 check inter 3000ms
 
 backend node_favicon
     http-check expect rstatus 200|503
     option httpchk GET /rss_feeds/icon/1
     balance roundrobin
-    server node_favicon01 favicons:3030 check inter 3000ms
+    server node_favicon01 node:3030 check inter 3000ms
 
 backend node_text
     http-check expect rstatus 200|503
     option httpchk GET /rss_feeds/original_text_fetcher?test=1
     balance roundrobin
-    server node_text01 original_text:4040 check inter 3000ms
+    server node_text01 node:4040 check inter 3000ms
+
+backend node_page
+    http-check expect rstatus 200|503
+    # check if there is a check for this
+    #option httpchk GET /rss_feeds/original_text_fetcher?test=1
+    balance roundrobin
+    server node_page01 node:3060 check inter 3000ms
 
 
 backend nginx
     balance roundrobin
     option httpchk GET /_nginxchk
     http-check expect rstatus 200|503
-    server nginx01 nginx:81 check inter 60000ms
+    server nginx01 app:81 check inter 3000ms
 
-backend gunicorn
+backend app
     balance roundrobin
     option httpchk GET /_haproxychk
-    server app01 nb.com:8000 check inter 3000ms
+    server app01 app:8000 check inter 3000ms
 
-backend gunicorn_counts
+backend app_counts
     balance roundrobin
     option httpchk GET /_haproxychk
-    server gunicorn_counts nb.com:8000 check inter 15000ms
+    server app_counts app_counts:8000 check inter 15000ms
 
 backend gunicorn_refresh
     balance roundrobin
     option httpchk GET /_haproxychk
-    server-template gunicorn_refresh 2 nb.com:8000 check inter 30000ms
+    server-template app_refresh 2 app_refresh:8000 check inter 30000ms
 
 
 backend push
     balance roundrobin
     option httpchk GET /_haproxychk
-    server push nb.com:8000 check inter 3000ms
+    server push app_push:8000 check inter 3000ms
 
 backend work
     balance roundrobin
     option httpchk GET http://monitor:5579/work_check/celeryd_work_queue
     http-check expect rstatus 200|503|301
-    server work celeryd_work_queue:82 check inter 3000ms
+    server work task_celeryd_work_queue:82 check inter 3000ms
 
 backend postgres
     option httpchk GET http://monitor:5579/db_check/postgres
@@ -125,18 +136,18 @@ backend mongo
     option httpchk GET http://monitor:5579/db_check/mongo
     server-template mongo 4 db_mongo:27017 check inter 2000ms
 
-backend redis
+backend db_redis
     option httpchk GET http://monitor:5579/db_check/redis
-    server redis-db40 redis:6579 check inter 2000ms
+    # redis, redis_pubsub, redis_sessions, redis_story droplets are necessary
-    server redis-pubsub-db43 monitor:5579 check inter 2000ms
+    # in docker swarm although not routed with haproxy
-
+backend db_redis_story
-backend redis_story
     option httpchk GET http://monitor:5579/db_check/redis_story
-    server redis-story-db42 redis:6579 check inter 2000ms
 
-backend redis_sessions
+backend db_redis_sessions
     option httpchk GET http://monitor:5579/db_check/redis_sessions
-    server redis-sess-db41 redis:6579 check inter 2000ms
+
+backend db_redis_pubsub
+    option httpchk GET http://monitor:5579/db_check/redis_pubsub
 
 backend elasticsearch
     option httpchk GET http://monitor:5579/db_check/elasticsearch
@@ -146,7 +157,7 @@ backend maintenance
     option httpchk HEAD /maintenance HTTP/1.1\r\nHost:\ www
     http-check expect status 404
     http-check send-state
-    server nginx nginx:443 check inter 3000ms
+    server nginx nginx:81 check inter 3000ms
 
 listen stats
 bind :1936 ssl crt localhost.pem

docker/haproxy/haproxy.dev.cfg

@@ -0,0 +1,158 @@
+global
+    maxconn 100000
+    daemon
+    ca-base /srv/newsblur/config/certificates
+    crt-base /srv/newsblur/config/certificates
+    tune.bufsize 32000
+    tune.maxrewrite 8196
+    tune.ssl.default-dh-param 2048
+    log 127.0.0.1 local0 notice
+    # log 127.0.0.1 local1 info
+
+defaults
+    log global
+    maxconn 100000
+    mode http
+    option forwardfor
+    option http-server-close
+    option httpclose
+    option log-health-checks
+    option log-separate-errors
+    option httplog
+    option redispatch
+    option abortonclose
+    timeout connect 10s
+    timeout client 10s
+    timeout server 30s
+    timeout tunnel 1h
+    retries 3
+    errorfile 502 /srv/newsblur/templates/502.http
+    errorfile 503 /srv/newsblur/templates/502.http
+    errorfile 504 /srv/newsblur/templates/502.http
+
+frontend public
+    bind :80
+    bind :443 ssl crt /srv/newsblur/config/certificates/localhost.pem #ciphers ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES128-SHA:AES256-SHA256:AES256-SHA no-sslv3
+    rspadd Strict-Transport-Security:\ max-age=0;\ includeSubDomains
+    option http-server-close
+
+    # Redirect all HTTP traffic to HTTPS
+    redirect scheme https code 301 if !{ ssl_fc }
+
+    acl gunicorn_dead nbsrv(gunicorn) lt 1
+    acl nginx_dead nbsrv(nginx) lt 1
+    acl mx_mode nbsrv(maintenance) lt 1
+    acl is_unread_count url_beg /reader/feed_unread_count
+    acl is_refresh_feeds url_beg /reader/refresh_feed
+
+    monitor-uri /status
+    monitor fail if gunicorn_dead
+    monitor fail if nginx_dead
+    monitor fail if mx_mode
+
+    use_backend node_socket if { path_beg /v2/socket.io/ }
+    use_backend node_favicon if { path_beg /rss_feeds/icon/ }
+    use_backend node_text if { path_beg /rss_feeds/original_text_fetcher }
+    use_backend nginx if { path_beg /media/ }
+    use_backend nginx if { path_beg /static/ }
+    use_backend nginx if { path_beg /favicon }
+    use_backend nginx if { path_beg /crossdomain/ }
+    use_backend nginx if { path_beg /robots }
+    #use_backend self if { path_beg /munin/ }
+
+    use_backend nginx if mx_mode
+
+    use_backend gunicorn_counts if is_unread_count
+    use_backend gunicorn_refresh if is_refresh_feeds
+    use_backend gunicorn unless gunicorn_dead || nginx_dead
+
+backend node_socket
+    http-check expect rstatus 200|503
+    balance roundrobin
+    server node_socket01 node_websocket:8888 check inter 3000ms
+
+backend node_favicon
+    http-check expect rstatus 200|503
+    option httpchk GET /rss_feeds/icon/1
+    balance roundrobin
+    server node_favicon01 favicons:3030 check inter 3000ms
+
+backend node_text
+    http-check expect rstatus 200|503
+    option httpchk GET /rss_feeds/original_text_fetcher?test=1
+    balance roundrobin
+    server node_text01 original_text:4040 check inter 3000ms
+
+
+backend nginx
+    balance roundrobin
+    option httpchk GET /_nginxchk
+    http-check expect rstatus 200|503
+    server nginx01 nginx:81 check inter 60000ms
+
+backend gunicorn
+    balance roundrobin
+    option httpchk GET /_haproxychk
+    server app01 nb.com:8000 check inter 3000ms
+
+backend gunicorn_counts
+    balance roundrobin
+    option httpchk GET /_haproxychk
+    server gunicorn_counts nb.com:8000 check inter 15000ms
+
+backend gunicorn_refresh
+    balance roundrobin
+    option httpchk GET /_haproxychk
+    server-template gunicorn_refresh 2 nb.com:8000 check inter 30000ms
+
+
+backend push
+    balance roundrobin
+    option httpchk GET /_haproxychk
+    server push nb.com:8000 check inter 3000ms
+
+backend work
+    balance roundrobin
+    option httpchk GET http://monitor:5579/work_check/celeryd_work_queue
+    http-check expect rstatus 200|503|301
+    server work celeryd_work_queue:82 check inter 3000ms
+
+backend postgres
+    option httpchk GET http://monitor:5579/db_check/postgres
+    server postgres-db02 postgres:5432 check inter 2000ms
+
+backend mongo
+    option httpchk GET http://monitor:5579/db_check/mongo
+    server-template mongo 4 db_mongo:27017 check inter 2000ms
+
+backend redis
+    option httpchk GET http://monitor:5579/db_check/redis
+    server redis-db40 redis:6579 check inter 2000ms
+    server redis-pubsub-db43 monitor:5579 check inter 2000ms
+
+backend redis_story
+    option httpchk GET http://monitor:5579/db_check/redis_story
+    server redis-story-db42 redis:6579 check inter 2000ms
+
+backend redis_sessions
+    option httpchk GET http://monitor:5579/db_check/redis_sessions
+    server redis-sess-db41 redis:6579 check inter 2000ms
+
+backend elasticsearch
+    option httpchk GET http://monitor:5579/db_check/elasticsearch
+    server es-search01 elasticsearch:9300 check inter 2000ms
+
+backend maintenance
+    option httpchk HEAD /maintenance HTTP/1.1\r\nHost:\ www
+    http-check expect status 404
+    http-check send-state
+    server nginx nginx:443 check inter 3000ms
+
+listen stats
+bind :1936 ssl crt localhost.pem
+stats enable
+stats hide-version
+stats realm Haproxy\ Statistics
+stats uri /
+stats auth gimmiestats:StatsGiver
+stats refresh 15s