From dd13af5bf174ff17b8a53676ccf693a4d7e1859e Mon Sep 17 00:00:00 2001 From: Jonathan Math Date: Mon, 11 Oct 2021 11:11:42 -0500 Subject: [PATCH] add querystring logic to mongo db health check to accommodate consul vs haproxy checking --- .../roles/mongo/templates/consul_service.json | 2 +- config/debug_haproxy.conf | 2 +- config/haproxy.conf.template | 2 +- docker/haproxy/haproxy.consul.cfg.j2 | 2 +- flask_monitor/db_monitor.py | 63 ++++++++++--------- 5 files changed, 36 insertions(+), 35 deletions(-) diff --git a/ansible/roles/mongo/templates/consul_service.json b/ansible/roles/mongo/templates/consul_service.json index 15263611b..192d901a0 100644 --- a/ansible/roles/mongo/templates/consul_service.json +++ b/ansible/roles/mongo/templates/consul_service.json @@ -8,7 +8,7 @@ "port": 27017, "checks": [{ "id": "mongo-ping", - "http": "http://{{ ansible_ssh_host }}:5579/db_check/mongo", + "http": "http://{{ ansible_ssh_host }}:5579/db_check/mongo?consul=1", "interval": "15s", "failures_before_critical": 4 }] diff --git a/config/debug_haproxy.conf b/config/debug_haproxy.conf index ce5ca5067..ca210f023 100644 --- a/config/debug_haproxy.conf +++ b/config/debug_haproxy.conf @@ -124,7 +124,7 @@ backend postgres option httpchk GET /db_check/postgres server postgres-db01 db_pgsql:5000 check inter 2000ms backend mongo - option httpchk GET /db_check/mongo + option httpchk GET /db_check/mongo?haproxy=1 server mongo-db22 db_mongo:5000 check inter 2000ms backend redis option httpchk GET /db_check/redis diff --git a/config/haproxy.conf.template b/config/haproxy.conf.template index cb8245f73..71e6721ff 100644 --- a/config/haproxy.conf.template +++ b/config/haproxy.conf.template @@ -146,7 +146,7 @@ backend postgres server postgres-db02 db_pgsql:5000 check inter 2000ms backend mongo - option httpchk GET /db_check/mongo + option httpchk GET /db_check/mongo?haproxy=1 server mongo-db20d db20d:5000 check inter 2000ms server mongo-db22 db22:5000 check inter 2000ms server mongo-db23a db23a:5000 check inter 2000ms diff --git a/docker/haproxy/haproxy.consul.cfg.j2 b/docker/haproxy/haproxy.consul.cfg.j2 index 1ee58e647..303ae8992 100644 --- a/docker/haproxy/haproxy.consul.cfg.j2 +++ b/docker/haproxy/haproxy.consul.cfg.j2 @@ -167,7 +167,7 @@ backend postgres server db-postgres db-postgres.node.nyc1.consul:5579 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none backend mongo - option httpchk GET /db_check/mongo + option httpchk GET /db_check/mongo?haproxy=1 default-server check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none {% for host in groups.mongo %} server {{host}} {{host}}.node.nyc1.consul:5579 diff --git a/flask_monitor/db_monitor.py b/flask_monitor/db_monitor.py index b431b6a17..2d70054e1 100644 --- a/flask_monitor/db_monitor.py +++ b/flask_monitor/db_monitor.py @@ -1,4 +1,4 @@ -from flask import Flask, abort +from flask import Flask, abort, request import os import psycopg2 import pymysql @@ -80,42 +80,43 @@ def db_check_mongo(): # The `mongo` hostname below is a reference to the newsblurnet docker network, where 172.18.0.0/16 is defined client = pymongo.MongoClient(f"mongodb://{settings.MONGO_DB['username']}:{settings.MONGO_DB['password']}@{settings.SERVER_NAME}/?authSource=admin") db = client.newsblur - return str(1) + if request.args.get('consul') == '1': + return str(1) except: abort(503) - - try: - stories = db.stories.estimated_document_count() - except (pymongo.errors.NotMasterError, pymongo.errors.ServerSelectionTimeoutError): - abort(504) - except pymongo.errors.OperationFailure as e: - if 'Authentication failed' in str(e): - abort(505) - abort(506) + if request.args.get('haproxy') == '1': + try: + stories = db.stories.estimated_document_count() + except (pymongo.errors.NotMasterError, pymongo.errors.ServerSelectionTimeoutError): + abort(504) + except pymongo.errors.OperationFailure as e: + if 'Authentication failed' in str(e): + abort(505) + abort(506) + + if not stories: + abort(510) - if not stories: - abort(510) - - status = client.admin.command('replSetGetStatus') - members = status['members'] - primary_optime = None - oldest_secondary_optime = None - for member in members: - member_state = member['state'] - optime = member['optime'] - if member_state == PRIMARY_STATE: - primary_optime = optime['ts'].time - elif member_state == SECONDARY_STATE: - if not oldest_secondary_optime or optime['ts'].time < oldest_secondary_optime: - oldest_secondary_optime = optime['ts'].time + status = client.admin.command('replSetGetStatus') + members = status['members'] + primary_optime = None + oldest_secondary_optime = None + for member in members: + member_state = member['state'] + optime = member['optime'] + if member_state == PRIMARY_STATE: + primary_optime = optime['ts'].time + elif member_state == SECONDARY_STATE: + if not oldest_secondary_optime or optime['ts'].time < oldest_secondary_optime: + oldest_secondary_optime = optime['ts'].time - if not primary_optime or not oldest_secondary_optime: - abort(511) + if not primary_optime or not oldest_secondary_optime: + abort(511) - # if primary_optime - oldest_secondary_optime > 100: - # abort(512) + # if primary_optime - oldest_secondary_optime > 100: + # abort(512) - return str(stories) + return str(stories) @app.route("/db_check/mongo_analytics") def db_check_mongo_analytics():