Separating consul from haproxy health checks. Consul doesn't care what the status of the service is so long as it's reachable. Don't take out bad nodes, let the software do that.

ansible/roles/consul-client/templates/consul_client.json.j2

@@ -2,7 +2,8 @@
   "datacenter": "nyc1",
   "data_dir": "/opt/consul",
   "log_level": "INFO",
-  "enable_syslog": false,
+  "log_file": "/var/log/consul/consul.log",
+  "enable_syslog": true,
   "retry_join": [{{ consul_manager_ip.stdout|trim }}],
   "advertise_addr": "{% raw %}{{ GetAllInterfaces | include \"name\" \"^eth\" | include \"flags\" \"forwardable|up\" | attr \"address\" }}{% endraw %}",
   "bind_addr": "0.0.0.0",

ansible/roles/consul/tasks/main.yml

@@ -9,14 +9,12 @@
   become: yes
   apt_repository:
     repo: "deb [arch=amd64] https://apt.releases.hashicorp.com {{ ansible_distribution_release }} main"
-  when: ansible_distribution_version != '14.04'
 
 - name: Installing Consul
   become: yes
   apt:
     pkg: consul
     state: latest
-  when: ansible_distribution_version != '14.04'
 
 - name: Register Manager IP
   run_once: yes
@@ -34,6 +32,13 @@
     path: /etc/consul.d
     state: directory
 
+- name: Ensure /var/log/consul exists
+  become: yes
+  file:
+    path: /var/log/consul
+    state: directory
+    mode: 0777
+
 - name: Remove default /etc/consul.d/consul.hcl
   become: yes
   copy:

ansible/roles/grafana/tasks/main.yml

@@ -19,7 +19,7 @@
   docker_container:
     pull: true
     name: grafana
-    image: grafana/grafana:7.5.7
+    image: grafana/grafana:8.2.6
     restart_policy: unless-stopped
     hostname: "{{ inventory_hostname }}"
     user: root

ansible/roles/mongo/templates/consul_service.analytics.json

@@ -8,7 +8,7 @@
         "port": 27017,
         "checks": [{
             "id": "mongo-analytics-ping",
-            "http": "http://{{ ansible_ssh_host }}:5579/db_check/mongo_analytics",
+            "http": "http://{{ ansible_ssh_host }}:5579/db_check/mongo_analytics?consul=1,
             "interval": "15s"
         }]
     }

ansible/roles/postgres/templates/consul_service.json

@@ -7,7 +7,7 @@
         "port": 5432,
         "checks": [{
             "id": "postgres-ping",
-            "http": "{% if inventory_hostname.startswith('db-postgres') %}http://{{ ansible_ssh_host }}:5579/db_check/postgres{% else %}http://{{ ansible_ssh_host }}:5000/db_check/postgres{% endif %}",
+            "http": "http://{{ ansible_ssh_host }}:5579/db_check/postgres?consul=1",
             "interval": "15s",
             "failures_before_critical": 4
           }]

ansible/roles/redis/templates/consul_service.json

@@ -9,15 +9,15 @@
         "checks": [{
             "id": "{{inventory_hostname}}-ping",
             {% if inventory_hostname == 'db-redis-story' %}
-                "http": "http://{{ ansible_ssh_host }}:5579/db_check/redis_story",
+                "http": "http://{{ ansible_ssh_host }}:5579/db_check/redis_story?consul=1",
             {% elif inventory_hostname == 'db-redis-user' %}
-                "http": "http://{{ ansible_ssh_host }}:5579/db_check/redis_user",
+                "http": "http://{{ ansible_ssh_host }}:5579/db_check/redis_user?consul=1",
             {% elif inventory_hostname == 'db-redis-pubsub' %}
-                "http": "http://{{ ansible_ssh_host }}:5579/db_check/redis_pubsub",
+                "http": "http://{{ ansible_ssh_host }}:5579/db_check/redis_pubsub?consul=1",
             {% elif inventory_hostname == 'db-redis-sessions' %}
-                "http": "http://{{ ansible_ssh_host }}:5579/db_check/redis_sessions",            
+                "http": "http://{{ ansible_ssh_host }}:5579/db_check/redis_sessions?consul=1",            
             {% else %}
-                "http": "http://{{ ansible_ssh_host }}:5000/db_check/redis",            
+                "http": "http://{{ ansible_ssh_host }}:5000/db_check/redis?consul=1",
             {% endif %}
             "interval": "15s",
             "failures_before_critical": 4

flask_monitor/db_monitor.py

@@ -25,11 +25,14 @@ SECONDARY_STATE = 2
 
 @app.route("/db_check/postgres")
 def db_check_postgres():
+    if request.args.get('consul') == '1':
+        return str(1)
+
     connect_params = "dbname='%s' user='%s' password='%s' host='%s' port='%s'" % (
         settings.DATABASES['default']['NAME'],
         settings.DATABASES['default']['USER'],
         settings.DATABASES['default']['PASSWORD'],
-        'db-postgres.service.nyc1.consul',
+        f'{settings.SERVER_NAME}.node.nyc1.consul',
         settings.DATABASES['default']['PORT'],
     )
     try:
@@ -48,6 +51,9 @@ def db_check_postgres():
 
 @app.route("/db_check/mysql")
 def db_check_mysql():
+    if request.args.get('consul') == '1':
+        return str(1)
+
     connect_params = "dbname='%s' user='%s' password='%s' host='%s' port='%s'" % (
         settings.DATABASES['default']['NAME'],
         settings.DATABASES['default']['USER'],
@@ -76,12 +82,13 @@ def db_check_mysql():
 
 @app.route("/db_check/mongo")
 def db_check_mongo():
+    if request.args.get('consul') == '1':
+        return str(1)
+
     try:
         # The `mongo` hostname below is a reference to the newsblurnet docker network, where 172.18.0.0/16 is defined
-        client = pymongo.MongoClient(f"mongodb://{settings.MONGO_DB['username']}:{settings.MONGO_DB['password']}@{settings.SERVER_NAME}/?authSource=admin")
+        client = pymongo.MongoClient(f"mongodb://{settings.MONGO_DB['username']}:{settings.MONGO_DB['password']}@{settings.SERVER_NAME}.node.nyc1.consul/?authSource=admin")
         db = client.newsblur
-        if request.args.get('consul') == '1':
-            return str(1)
     except:
         abort(503)
 
@@ -122,6 +129,9 @@ def db_check_mongo():
 
 @app.route("/db_check/mongo_analytics")
 def db_check_mongo_analytics():
+    if request.args.get('consul') == '1':
+        return str(1)
+
     try:
         client = pymongo.MongoClient(f"mongodb://{settings.MONGO_ANALYTICS_DB['username']}:{settings.MONGO_ANALYTICS_DB['password']}@{settings.SERVER_NAME}/?authSource=admin")
         db = client.nbanalytics
@@ -144,6 +154,9 @@ def db_check_mongo_analytics():
 
 @app.route("/db_check/redis_user")
 def db_check_redis_user():
+    if request.args.get('consul') == '1':
+        return str(1)
+
     try:
         r = redis.Redis('db-redis-user.service.nyc1.consul', db=0)
     except:
@@ -160,7 +173,10 @@ def db_check_redis_user():
         abort(505)
 
 @app.route("/db_check/redis_story")
-def db_check_redis_story():
+def db_check_redis_story():    
+    if request.args.get('consul') == '1':
+        return str(1)
+    
     try:
         r = redis.Redis('db-redis-story.service.nyc1.consul', db=1)
     except:
@@ -178,6 +194,9 @@ def db_check_redis_story():
 
 @app.route("/db_check/redis_sessions")
 def db_check_redis_sessions():
+    if request.args.get('consul') == '1':
+        return str(1)
+
     try:
         r = redis.Redis('db-redis-sessions.service.nyc1.consul', db=5)
     except:
@@ -195,6 +214,9 @@ def db_check_redis_sessions():
 
 @app.route("/db_check/redis_pubsub")
 def db_check_redis_pubsub():
+    if request.args.get('consul') == '1':
+        return str(1)
+
     try:
         r = redis.Redis('db-redis-pubsub.service.nyc1.consul', db=1)
     except: