Refactored state timeline to work as a flask metrics server on haproxy. Needs ansible configs to deploy on www server.

docker-compose.metrics.yml

@@ -15,10 +15,13 @@ services:
       - node-exporter
       - haproxy
       - flask_metrics_mongo
+      - flask_metrics_redis
+      - flask_metrics_haproxy
     external_links:
       - haproxy
       - flask_metrics_mongo
       - flask_metrics_redis
+      - flask_metrics_haproxy
 
   node-exporter:
     container_name: node-exporter
@@ -41,6 +44,7 @@ services:
       - ./docker/grafana/dashboards/:/etc/grafana/provisioning/dashboards/
     external_links:
       - prometheus
+  
   flask_metrics_mongo:
     container_name: flask_metrics_mongo
     image: newsblur/newsblur_monitor:latest
@@ -55,6 +59,7 @@ services:
       - nginx
     volumes:
       - ${PWD}:/srv/newsblur
+  
   flask_metrics_redis:
     container_name: flask_metrics_redis
     image: newsblur/newsblur_monitor:latest
@@ -69,6 +74,22 @@ services:
       - nginx
     volumes:
       - ${PWD}:/srv/newsblur
+
+  flask_metrics_haproxy:
+    container_name: flask_metrics_haproxy
+    image: newsblur/newsblur_monitor:latest
+    command: bash -c "python /srv/newsblur/flask_metrics/flask_metrics_haproxy.py"
+    environment: 
+      - DOCKERBUILD=True
+    ports:
+      - 5599:5569
+    depends_on: 
+      - db_mongo
+      - newsblur_web
+      - nginx
+    volumes:
+      - ${PWD}:/srv/newsblur
+  
   elasticsearch_exporter:
     container_name: elasticsearch_exporter
     image: prometheuscommunity/elasticsearch-exporter:latest
@@ -85,4 +106,4 @@ services:
     environment:
       DATA_SOURCE_NAME: 'postgresql://newsblur:newsblur@db_postgres:5432/postgres?sslmode=disable'
     ports:
-    - '9187:9187'
+    - '9187:9187'

docker/prometheus/prometheus.consul.yml.j2

@@ -239,7 +239,11 @@ scrape_configs:
         target_label: instance
 
     - job_name: 'haproxy state'
-      static_configs:
-        - targets: ['localhost:5569']
+      consul_sd_configs:
+      - server: 'consul.service.nyc1.consul:8500'
+        services: ['flask_metrics_haproxy']
+      relabel_configs:
+      - source_labels: ['__meta_consul_node']
+        target_label: instance
       metrics_path: /state/
       scrape_interval: 30s

docker/prometheus/prometheus.docker.yml

@@ -175,42 +175,42 @@ scrape_configs:
 
     - job_name: 'redis active connections'
       static_configs:
-        - targets: ['flask_metrics_redis:5569']
+        - targets: ['flask_metrics_redis:5589']
       metrics_path: /active-connections/
       scheme: http
       tls_config:
         insecure_skip_verify: true
     - job_name: 'redis commands'
       static_configs:
-        - targets: ['flask_metrics_redis:5569']
+        - targets: ['flask_metrics_redis:5589']
       metrics_path: /commands/
       scheme: http
       tls_config:
         insecure_skip_verify: true
     - job_name: 'redis connects'
       static_configs:
-        - targets: ['flask_metrics_redis:5569']
+        - targets: ['flask_metrics_redis:5589']
       metrics_path: /connects/
       scheme: http
       tls_config:
         insecure_skip_verify: true
     - job_name: 'redis size'
       static_configs:
-        - targets: ['flask_metrics_redis:5569']
+        - targets: ['flask_metrics_redis:5589']
       metrics_path: /size/
       scheme: http
       tls_config:
         insecure_skip_verify: true
     - job_name: 'redis memory'
       static_configs:
-        - targets: ['flask_metrics_redis:5569']
+        - targets: ['flask_metrics_redis:5589']
       metrics_path: /memory/
       scheme: http
       tls_config:
         insecure_skip_verify: true
     - job_name: 'redis used memory'
       static_configs:
-        - targets: ['flask_metrics_redis:5569']
+        - targets: ['flask_metrics_redis:5589']
       metrics_path: /used-memory/
       scheme: http
       tls_config:
@@ -230,18 +230,11 @@ scrape_configs:
       tls_config:
         insecure_skip_verify: true
 
-    - job_name: 'redis state'
+    - job_name: 'haproxy state'
       static_configs:
-        - targets: ['flask_metrics_redis:5569']
+        - targets: ['flask_metrics_haproxy:5599']
       metrics_path: /state/
       scheme: http
       tls_config:
         insecure_skip_verify: true
-    - job_name: 'mongo state'
-      static_configs:
-        - targets: ['flask_metrics_mongo:5569']
-      metrics_path: /state/
-      scheme: http
-      tls_config:
-        insecure_skip_verify: true
-      scrape_interval: 30s
+      scrape_interval: 30s

flask_metrics/flask_metrics_haproxy.py

@@ -0,0 +1,81 @@
+from flask import Flask, render_template, Response
+from newsblur_web import settings
+import sentry_sdk
+from sentry_sdk.integrations.flask import FlaskIntegration
+import requests
+from requests.auth import HTTPBasicAuth
+
+if settings.FLASK_SENTRY_DSN is not None:
+    sentry_sdk.init(
+        dsn=settings.FLASK_SENTRY_DSN,
+        integrations=[FlaskIntegration()],
+        traces_sample_rate=1.0,
+    )
+
+app = Flask(__name__)
+
+if settings.DOCKERBUILD:
+    pass
+
+
+STATUS_MAPPING = {
+    "UNK": 0,      # unknown
+    "INI": 1,      # initializing
+    "SOCKERR": 2,  # socket error
+    "L4OK": 3,     # check passed on layer 4, no upper layers testing enabled
+    "L4TOUT": 4,   # layer 1-4 timeout
+    "L4CON": 5,    # layer 1-4 connection problem, for example "Connection refused" (tcp rst) or "No route to host" (icmp)
+    "L6OK": 6,     # check passed on layer 6
+    "L6TOUT": 7,   # layer 6 (SSL) timeout
+    "L6RSP": 8,    # layer 6 invalid response - protocol error
+    "L7OK": 9,     # check passed on layer 7
+    "L7OKC": 10,   # check conditionally passed on layer 7, for example 404 with disable-on-404
+    "L7TOUT": 11,  # layer 7 (HTTP/SMTP) timeout
+    "L7RSP": 12,   # layer 7 invalid response - protocol error
+    "L7STS": 13,   # layer 7 response error, for example HTTP 5xx
+}
+
+def format_state_data(label, data):
+    formatted_data = {}
+    for k, v in data.items():
+        if v:
+            formatted_data[k] = f'{label}{{servername="{k}"}} {STATUS_MAPPING[v.strip()]}'
+    return formatted_data
+
+def fetch_states():
+    res = requests.get('https://newsblur.com:1936/;csv', auth=HTTPBasicAuth('gimmiestats', 'StatsGiver'))
+
+    lines = res.content.decode('utf-8').split('\n')
+    header_line = lines[0].split(",")
+    check_status_index = header_line.index('check_status')
+    servername_index = header_line.index('svname')
+
+    data = {}
+    backends = [line.split(",") for line in lines[1:]]
+    for backend_data in backends:
+        if len(backend_data) <= check_status_index: continue
+        if len(backend_data) <= servername_index: continue
+        if backend_data[servername_index] in ['FRONTEND', 'BACKEND']: continue
+        backend_status = backend_data[check_status_index].replace("*", "")
+        data[backend_data[servername_index]] = backend_status
+    
+    return data
+
+
+@app.route("/state/")
+def haproxy_state():
+    backends = fetch_states()
+    
+    formatted_data = format_state_data("haproxy_state", backends)
+    context = {
+        'chart_name': 'haproxy_state',
+        'chart_type': 'gauge',
+        'data': formatted_data
+    }
+    html_body = render_template('prometheus_data.html', **context)
+    return Response(html_body, content_type="text/plain")
+
+
+if __name__ == "__main__":
+    print(" ---> Starting NewsBlur Flask Metrics server for HAProxy...")
+    app.run(host="0.0.0.0", port=5569, debug=settings.DEBUG)

flask_metrics/flask_metrics_mongo.py

@@ -1,6 +1,5 @@
 from flask import Flask, render_template, Response
 import pymongo
-from flask_metrics.state_timeline import format_state_data, get_state
 from newsblur_web import settings
 import sentry_sdk
 from sentry_sdk.integrations.flask import FlaskIntegration
@@ -166,20 +165,6 @@ def page_queues():
     html_body = render_template('prometheus_data.html', **context)
     return Response(html_body, content_type="text/plain")
 
-@app.route("/state/")
-def mongo_state():
-    mongo_data = get_state("mongo")
-    if 'BACKEND' in mongo_data:
-        del mongo_data['BACKEND']
-    formatted_data = format_state_data("mongo_state", mongo_data)
-    context = {
-        'chart_name': 'mongo_state',
-        'chart_type': 'gauge',
-        'data': formatted_data
-    }
-    html_body = render_template('prometheus_data.html', **context)
-    return Response(html_body, content_type="text/plain")
-
 
 if __name__ == "__main__":
     print(" ---> Starting NewsBlur Flask Metrics server...")

flask_metrics/flask_metrics_redis.py

@@ -1,6 +1,5 @@
 from flask import Flask, render_template, Response
 from newsblur_web import settings
-from flask_metrics.state_timeline import format_state_data, get_state
 import sentry_sdk
 from sentry_sdk.integrations.flask import FlaskIntegration
 import redis
@@ -187,19 +186,6 @@ def memory_used():
     return Response(html_body, content_type="text/plain")
 
 
-@app.route("/state/")
-def redis_state():
-    redis_data = get_state("db_redis")
-    formatted_data = format_state_data("redis_state", redis_data)
-    context = {
-        'chart_name': 'redis_state',
-        'chart_type': 'gauge',
-        'data': formatted_data
-    }
-    html_body = render_template('prometheus_data.html', **context)
-    return Response(html_body, content_type="text/plain")
-
-
 if __name__ == "__main__":
     print(" ---> Starting NewsBlur Flask Metrics server...")
     app.run(host="0.0.0.0", port=5569)

flask_metrics/state_timeline.py

@@ -1,42 +0,0 @@
-from flask import render_template
-import requests
-from requests.auth import HTTPBasicAuth
-
-
-STATUS_MAPPING = {
-    "UNK": 0,      # unknown
-    "INI": 1,      # initializing
-    "SOCKERR": 2,  # socket error
-    "L4OK": 3,     # check passed on layer 4, no upper layers testing enabled
-    "L4TOUT": 4,   # layer 1-4 timeout
-    "L4CON": 5,    # layer 1-4 connection problem, for example "Connection refused" (tcp rst) or "No route to host" (icmp)
-    "L6OK": 6,     # check passed on layer 6
-    "L6TOUT": 7,   # layer 6 (SSL) timeout
-    "L6RSP": 8,    # layer 6 invalid response - protocol error
-    "L7OK": 9,     # check passed on layer 7
-    "L7OKC": 10,   # check conditionally passed on layer 7, for example 404 with disable-on-404
-    "L7TOUT": 11,  # layer 7 (HTTP/SMTP) timeout
-    "L7RSP": 12,   # layer 7 invalid response - protocol error
-    "L7STS": 13,   # layer 7 response error, for example HTTP 5xx
-}
-
-def format_state_data(label, data):
-    formatted_data = {}
-    for k, v in data.items():
-        if v:
-            formatted_data[k] = f'{label}{{servername="{k}"}} {STATUS_MAPPING[v.strip()]}'
-    return formatted_data
-
-def get_state(backend_name):
-    res = requests.get('https://newsblur.com:1936/;csv', auth=HTTPBasicAuth('gimmiestats', 'StatsGiver'))
-    lines = res.content.decode('utf-8').split('\n')
-    backends = [line.split(",") for line in lines if backend_name in line]
-
-    check_status_index = lines[0].split(",").index('check_status')
-    servername_index = lines[0].split(",").index('svname')
-
-    data = {}
-
-    for backend_data in backends:
-        data[backend_data[servername_index]] = backend_data[check_status_index].replace("*", "")
-    return data