diff --git a/ansible/roles/consul-client/templates/consul_client.json.j2 b/ansible/roles/consul-client/templates/consul_client.json.j2 index 92d4d8d7f..b298796af 100644 --- a/ansible/roles/consul-client/templates/consul_client.json.j2 +++ b/ansible/roles/consul-client/templates/consul_client.json.j2 @@ -2,7 +2,8 @@ "datacenter": "nyc1", "data_dir": "/opt/consul", "log_level": "INFO", - "enable_syslog": false, + "log_file": "/var/log/consul/consul.log", + "enable_syslog": true, "retry_join": [{{ consul_manager_ip.stdout|trim }}], "advertise_addr": "{% raw %}{{ GetAllInterfaces | include \"name\" \"^eth\" | include \"flags\" \"forwardable|up\" | attr \"address\" }}{% endraw %}", "bind_addr": "0.0.0.0", diff --git a/ansible/roles/consul/tasks/main.yml b/ansible/roles/consul/tasks/main.yml index ef44173de..789de3600 100644 --- a/ansible/roles/consul/tasks/main.yml +++ b/ansible/roles/consul/tasks/main.yml @@ -9,14 +9,12 @@ become: yes apt_repository: repo: "deb [arch=amd64] https://apt.releases.hashicorp.com {{ ansible_distribution_release }} main" - when: ansible_distribution_version != '14.04' - name: Installing Consul become: yes apt: pkg: consul state: latest - when: ansible_distribution_version != '14.04' - name: Register Manager IP run_once: yes @@ -34,6 +32,13 @@ path: /etc/consul.d state: directory +- name: Ensure /var/log/consul exists + become: yes + file: + path: /var/log/consul + state: directory + mode: 0777 + - name: Remove default /etc/consul.d/consul.hcl become: yes copy: diff --git a/ansible/roles/grafana/tasks/main.yml b/ansible/roles/grafana/tasks/main.yml index d56f91afc..435243b1b 100644 --- a/ansible/roles/grafana/tasks/main.yml +++ b/ansible/roles/grafana/tasks/main.yml @@ -19,7 +19,7 @@ docker_container: pull: true name: grafana - image: grafana/grafana:7.5.7 + image: grafana/grafana:8.2.6 restart_policy: unless-stopped hostname: "{{ inventory_hostname }}" user: root diff --git a/ansible/roles/mongo/templates/consul_service.analytics.json b/ansible/roles/mongo/templates/consul_service.analytics.json index e3287710c..3fc55fe43 100644 --- a/ansible/roles/mongo/templates/consul_service.analytics.json +++ b/ansible/roles/mongo/templates/consul_service.analytics.json @@ -8,7 +8,7 @@ "port": 27017, "checks": [{ "id": "mongo-analytics-ping", - "http": "http://{{ ansible_ssh_host }}:5579/db_check/mongo_analytics", + "http": "http://{{ ansible_ssh_host }}:5579/db_check/mongo_analytics?consul=1, "interval": "15s" }] } diff --git a/ansible/roles/postgres/templates/consul_service.json b/ansible/roles/postgres/templates/consul_service.json index a924e65d9..caa43016f 100644 --- a/ansible/roles/postgres/templates/consul_service.json +++ b/ansible/roles/postgres/templates/consul_service.json @@ -7,7 +7,7 @@ "port": 5432, "checks": [{ "id": "postgres-ping", - "http": "{% if inventory_hostname.startswith('db-postgres') %}http://{{ ansible_ssh_host }}:5579/db_check/postgres{% else %}http://{{ ansible_ssh_host }}:5000/db_check/postgres{% endif %}", + "http": "http://{{ ansible_ssh_host }}:5579/db_check/postgres?consul=1", "interval": "15s", "failures_before_critical": 4 }] diff --git a/ansible/roles/redis/templates/consul_service.json b/ansible/roles/redis/templates/consul_service.json index 722d04655..21fffc268 100644 --- a/ansible/roles/redis/templates/consul_service.json +++ b/ansible/roles/redis/templates/consul_service.json @@ -9,15 +9,15 @@ "checks": [{ "id": "{{inventory_hostname}}-ping", {% if inventory_hostname == 'db-redis-story' %} - "http": "http://{{ ansible_ssh_host }}:5579/db_check/redis_story", + "http": "http://{{ ansible_ssh_host }}:5579/db_check/redis_story?consul=1", {% elif inventory_hostname == 'db-redis-user' %} - "http": "http://{{ ansible_ssh_host }}:5579/db_check/redis_user", + "http": "http://{{ ansible_ssh_host }}:5579/db_check/redis_user?consul=1", {% elif inventory_hostname == 'db-redis-pubsub' %} - "http": "http://{{ ansible_ssh_host }}:5579/db_check/redis_pubsub", + "http": "http://{{ ansible_ssh_host }}:5579/db_check/redis_pubsub?consul=1", {% elif inventory_hostname == 'db-redis-sessions' %} - "http": "http://{{ ansible_ssh_host }}:5579/db_check/redis_sessions", + "http": "http://{{ ansible_ssh_host }}:5579/db_check/redis_sessions?consul=1", {% else %} - "http": "http://{{ ansible_ssh_host }}:5000/db_check/redis", + "http": "http://{{ ansible_ssh_host }}:5000/db_check/redis?consul=1", {% endif %} "interval": "15s", "failures_before_critical": 4 diff --git a/flask_monitor/db_monitor.py b/flask_monitor/db_monitor.py index 5a1eaf05a..472d12bff 100644 --- a/flask_monitor/db_monitor.py +++ b/flask_monitor/db_monitor.py @@ -25,11 +25,14 @@ SECONDARY_STATE = 2 @app.route("/db_check/postgres") def db_check_postgres(): + if request.args.get('consul') == '1': + return str(1) + connect_params = "dbname='%s' user='%s' password='%s' host='%s' port='%s'" % ( settings.DATABASES['default']['NAME'], settings.DATABASES['default']['USER'], settings.DATABASES['default']['PASSWORD'], - 'db-postgres.service.nyc1.consul', + f'{settings.SERVER_NAME}.node.nyc1.consul', settings.DATABASES['default']['PORT'], ) try: @@ -48,6 +51,9 @@ def db_check_postgres(): @app.route("/db_check/mysql") def db_check_mysql(): + if request.args.get('consul') == '1': + return str(1) + connect_params = "dbname='%s' user='%s' password='%s' host='%s' port='%s'" % ( settings.DATABASES['default']['NAME'], settings.DATABASES['default']['USER'], @@ -76,12 +82,13 @@ def db_check_mysql(): @app.route("/db_check/mongo") def db_check_mongo(): + if request.args.get('consul') == '1': + return str(1) + try: # The `mongo` hostname below is a reference to the newsblurnet docker network, where 172.18.0.0/16 is defined - client = pymongo.MongoClient(f"mongodb://{settings.MONGO_DB['username']}:{settings.MONGO_DB['password']}@{settings.SERVER_NAME}/?authSource=admin") + client = pymongo.MongoClient(f"mongodb://{settings.MONGO_DB['username']}:{settings.MONGO_DB['password']}@{settings.SERVER_NAME}.node.nyc1.consul/?authSource=admin") db = client.newsblur - if request.args.get('consul') == '1': - return str(1) except: abort(503) @@ -122,6 +129,9 @@ def db_check_mongo(): @app.route("/db_check/mongo_analytics") def db_check_mongo_analytics(): + if request.args.get('consul') == '1': + return str(1) + try: client = pymongo.MongoClient(f"mongodb://{settings.MONGO_ANALYTICS_DB['username']}:{settings.MONGO_ANALYTICS_DB['password']}@{settings.SERVER_NAME}/?authSource=admin") db = client.nbanalytics @@ -144,6 +154,9 @@ def db_check_mongo_analytics(): @app.route("/db_check/redis_user") def db_check_redis_user(): + if request.args.get('consul') == '1': + return str(1) + try: r = redis.Redis('db-redis-user.service.nyc1.consul', db=0) except: @@ -160,7 +173,10 @@ def db_check_redis_user(): abort(505) @app.route("/db_check/redis_story") -def db_check_redis_story(): +def db_check_redis_story(): + if request.args.get('consul') == '1': + return str(1) + try: r = redis.Redis('db-redis-story.service.nyc1.consul', db=1) except: @@ -178,6 +194,9 @@ def db_check_redis_story(): @app.route("/db_check/redis_sessions") def db_check_redis_sessions(): + if request.args.get('consul') == '1': + return str(1) + try: r = redis.Redis('db-redis-sessions.service.nyc1.consul', db=5) except: @@ -195,6 +214,9 @@ def db_check_redis_sessions(): @app.route("/db_check/redis_pubsub") def db_check_redis_pubsub(): + if request.args.get('consul') == '1': + return str(1) + try: r = redis.Redis('db-redis-pubsub.service.nyc1.consul', db=1) except: