From ec33dfab484b9ea7fb4cbafb6962ced9802ca747 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Mon, 6 Dec 2021 15:12:12 -0500 Subject: [PATCH] Separating consul from haproxy health checks. Consul doesn't care what the status of the service is so long as it's reachable. Don't take out bad nodes, let the software do that. --- .../templates/consul_client.json.j2 | 3 +- ansible/roles/consul/tasks/main.yml | 9 ++++-- ansible/roles/grafana/tasks/main.yml | 2 +- .../templates/consul_service.analytics.json | 2 +- .../postgres/templates/consul_service.json | 2 +- .../roles/redis/templates/consul_service.json | 10 +++--- flask_monitor/db_monitor.py | 32 ++++++++++++++++--- 7 files changed, 44 insertions(+), 16 deletions(-) diff --git a/ansible/roles/consul-client/templates/consul_client.json.j2 b/ansible/roles/consul-client/templates/consul_client.json.j2 index 92d4d8d7f..b298796af 100644 --- a/ansible/roles/consul-client/templates/consul_client.json.j2 +++ b/ansible/roles/consul-client/templates/consul_client.json.j2 @@ -2,7 +2,8 @@ "datacenter": "nyc1", "data_dir": "/opt/consul", "log_level": "INFO", - "enable_syslog": false, + "log_file": "/var/log/consul/consul.log", + "enable_syslog": true, "retry_join": [{{ consul_manager_ip.stdout|trim }}], "advertise_addr": "{% raw %}{{ GetAllInterfaces | include \"name\" \"^eth\" | include \"flags\" \"forwardable|up\" | attr \"address\" }}{% endraw %}", "bind_addr": "0.0.0.0", diff --git a/ansible/roles/consul/tasks/main.yml b/ansible/roles/consul/tasks/main.yml index ef44173de..789de3600 100644 --- a/ansible/roles/consul/tasks/main.yml +++ b/ansible/roles/consul/tasks/main.yml @@ -9,14 +9,12 @@ become: yes apt_repository: repo: "deb [arch=amd64] https://apt.releases.hashicorp.com {{ ansible_distribution_release }} main" - when: ansible_distribution_version != '14.04' - name: Installing Consul become: yes apt: pkg: consul state: latest - when: ansible_distribution_version != '14.04' - name: Register Manager IP run_once: yes @@ -34,6 +32,13 @@ path: /etc/consul.d state: directory +- name: Ensure /var/log/consul exists + become: yes + file: + path: /var/log/consul + state: directory + mode: 0777 + - name: Remove default /etc/consul.d/consul.hcl become: yes copy: diff --git a/ansible/roles/grafana/tasks/main.yml b/ansible/roles/grafana/tasks/main.yml index d56f91afc..435243b1b 100644 --- a/ansible/roles/grafana/tasks/main.yml +++ b/ansible/roles/grafana/tasks/main.yml @@ -19,7 +19,7 @@ docker_container: pull: true name: grafana - image: grafana/grafana:7.5.7 + image: grafana/grafana:8.2.6 restart_policy: unless-stopped hostname: "{{ inventory_hostname }}" user: root diff --git a/ansible/roles/mongo/templates/consul_service.analytics.json b/ansible/roles/mongo/templates/consul_service.analytics.json index e3287710c..3fc55fe43 100644 --- a/ansible/roles/mongo/templates/consul_service.analytics.json +++ b/ansible/roles/mongo/templates/consul_service.analytics.json @@ -8,7 +8,7 @@ "port": 27017, "checks": [{ "id": "mongo-analytics-ping", - "http": "http://{{ ansible_ssh_host }}:5579/db_check/mongo_analytics", + "http": "http://{{ ansible_ssh_host }}:5579/db_check/mongo_analytics?consul=1, "interval": "15s" }] } diff --git a/ansible/roles/postgres/templates/consul_service.json b/ansible/roles/postgres/templates/consul_service.json index a924e65d9..caa43016f 100644 --- a/ansible/roles/postgres/templates/consul_service.json +++ b/ansible/roles/postgres/templates/consul_service.json @@ -7,7 +7,7 @@ "port": 5432, "checks": [{ "id": "postgres-ping", - "http": "{% if inventory_hostname.startswith('db-postgres') %}http://{{ ansible_ssh_host }}:5579/db_check/postgres{% else %}http://{{ ansible_ssh_host }}:5000/db_check/postgres{% endif %}", + "http": "http://{{ ansible_ssh_host }}:5579/db_check/postgres?consul=1", "interval": "15s", "failures_before_critical": 4 }] diff --git a/ansible/roles/redis/templates/consul_service.json b/ansible/roles/redis/templates/consul_service.json index 722d04655..21fffc268 100644 --- a/ansible/roles/redis/templates/consul_service.json +++ b/ansible/roles/redis/templates/consul_service.json @@ -9,15 +9,15 @@ "checks": [{ "id": "{{inventory_hostname}}-ping", {% if inventory_hostname == 'db-redis-story' %} - "http": "http://{{ ansible_ssh_host }}:5579/db_check/redis_story", + "http": "http://{{ ansible_ssh_host }}:5579/db_check/redis_story?consul=1", {% elif inventory_hostname == 'db-redis-user' %} - "http": "http://{{ ansible_ssh_host }}:5579/db_check/redis_user", + "http": "http://{{ ansible_ssh_host }}:5579/db_check/redis_user?consul=1", {% elif inventory_hostname == 'db-redis-pubsub' %} - "http": "http://{{ ansible_ssh_host }}:5579/db_check/redis_pubsub", + "http": "http://{{ ansible_ssh_host }}:5579/db_check/redis_pubsub?consul=1", {% elif inventory_hostname == 'db-redis-sessions' %} - "http": "http://{{ ansible_ssh_host }}:5579/db_check/redis_sessions", + "http": "http://{{ ansible_ssh_host }}:5579/db_check/redis_sessions?consul=1", {% else %} - "http": "http://{{ ansible_ssh_host }}:5000/db_check/redis", + "http": "http://{{ ansible_ssh_host }}:5000/db_check/redis?consul=1", {% endif %} "interval": "15s", "failures_before_critical": 4 diff --git a/flask_monitor/db_monitor.py b/flask_monitor/db_monitor.py index 5a1eaf05a..472d12bff 100644 --- a/flask_monitor/db_monitor.py +++ b/flask_monitor/db_monitor.py @@ -25,11 +25,14 @@ SECONDARY_STATE = 2 @app.route("/db_check/postgres") def db_check_postgres(): + if request.args.get('consul') == '1': + return str(1) + connect_params = "dbname='%s' user='%s' password='%s' host='%s' port='%s'" % ( settings.DATABASES['default']['NAME'], settings.DATABASES['default']['USER'], settings.DATABASES['default']['PASSWORD'], - 'db-postgres.service.nyc1.consul', + f'{settings.SERVER_NAME}.node.nyc1.consul', settings.DATABASES['default']['PORT'], ) try: @@ -48,6 +51,9 @@ def db_check_postgres(): @app.route("/db_check/mysql") def db_check_mysql(): + if request.args.get('consul') == '1': + return str(1) + connect_params = "dbname='%s' user='%s' password='%s' host='%s' port='%s'" % ( settings.DATABASES['default']['NAME'], settings.DATABASES['default']['USER'], @@ -76,12 +82,13 @@ def db_check_mysql(): @app.route("/db_check/mongo") def db_check_mongo(): + if request.args.get('consul') == '1': + return str(1) + try: # The `mongo` hostname below is a reference to the newsblurnet docker network, where 172.18.0.0/16 is defined - client = pymongo.MongoClient(f"mongodb://{settings.MONGO_DB['username']}:{settings.MONGO_DB['password']}@{settings.SERVER_NAME}/?authSource=admin") + client = pymongo.MongoClient(f"mongodb://{settings.MONGO_DB['username']}:{settings.MONGO_DB['password']}@{settings.SERVER_NAME}.node.nyc1.consul/?authSource=admin") db = client.newsblur - if request.args.get('consul') == '1': - return str(1) except: abort(503) @@ -122,6 +129,9 @@ def db_check_mongo(): @app.route("/db_check/mongo_analytics") def db_check_mongo_analytics(): + if request.args.get('consul') == '1': + return str(1) + try: client = pymongo.MongoClient(f"mongodb://{settings.MONGO_ANALYTICS_DB['username']}:{settings.MONGO_ANALYTICS_DB['password']}@{settings.SERVER_NAME}/?authSource=admin") db = client.nbanalytics @@ -144,6 +154,9 @@ def db_check_mongo_analytics(): @app.route("/db_check/redis_user") def db_check_redis_user(): + if request.args.get('consul') == '1': + return str(1) + try: r = redis.Redis('db-redis-user.service.nyc1.consul', db=0) except: @@ -160,7 +173,10 @@ def db_check_redis_user(): abort(505) @app.route("/db_check/redis_story") -def db_check_redis_story(): +def db_check_redis_story(): + if request.args.get('consul') == '1': + return str(1) + try: r = redis.Redis('db-redis-story.service.nyc1.consul', db=1) except: @@ -178,6 +194,9 @@ def db_check_redis_story(): @app.route("/db_check/redis_sessions") def db_check_redis_sessions(): + if request.args.get('consul') == '1': + return str(1) + try: r = redis.Redis('db-redis-sessions.service.nyc1.consul', db=5) except: @@ -195,6 +214,9 @@ def db_check_redis_sessions(): @app.route("/db_check/redis_pubsub") def db_check_redis_pubsub(): + if request.args.get('consul') == '1': + return str(1) + try: r = redis.Redis('db-redis-pubsub.service.nyc1.consul', db=1) except: