Revert "Merge branch 'state_timeline'"

This reverts commit 83e4609af1, reversing
changes made to 089161b20f.

ansible/roles/flask_metrics/tasks/main.yml

@@ -51,8 +51,7 @@
       file: /srv/newsblur/flask_metrics/flask_metrics_mongo.py
     - service_name: redis
       file: /srv/newsblur/flask_metrics/flask_metrics_redis.py
-    - service_name: metrics
+
-      file: /srv/newsblur/flask_metrics/state_timeline.py
 - name: Restart flask_metrics
   become: yes
   shell:

docker-compose.metrics.yml

@@ -69,20 +69,6 @@ services:
       - nginx
     volumes:
       - ${PWD}:/srv/newsblur
-  flask_metrics_state:
-    container_name: flask_metrics_state
-    image: newsblur/newsblur_monitor:latest
-    command: bash -c "python /srv/newsblur/flask_metrics/state_timeline.py"
-    environment: 
-      - DOCKERBUILD=True
-    ports:
-      - 5599:5569
-    depends_on: 
-      - haproxy
-      - newsblur_web
-      - nginx
-    volumes:
-      - ${PWD}:/srv/newsblur
   elasticsearch_exporter:
     container_name: elasticsearch_exporter
     image: prometheuscommunity/elasticsearch-exporter:latest

docker/grafana/dashboards/newsblur_dashboard.json

@@ -144,8 +144,8 @@
         "overrides": []
       },
       "gridPos": {
-        "h": 27,
+        "h": 8,
-        "w": 24,
+        "w": 12,
         "x": 0,
         "y": 0
       },
@@ -166,26 +166,162 @@
       "targets": [
         {
           "exemplar": true,
-          "expr": "state",
+          "expr": "redis_state",
           "interval": "",
           "legendFormat": "{{ servername }}",
           "refId": "A"
         }
       ],
-      "title": "Service State",
+      "title": "Redis State",
       "type": "state-timeline"
     },
     {
-      "collapsed": true,
+      "datasource": "Prometheus",
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "fillOpacity": 70,
+            "lineWidth": 0
+          },
+          "mappings": [
+            {
+              "options": {
+                "0": {
+                  "color": "super-light-red",
+                  "index": 0,
+                  "text": "UNK"
+                },
+                "1": {
+                  "color": "semi-dark-yellow",
+                  "index": 1,
+                  "text": "INI"
+                },
+                "2": {
+                  "color": "semi-dark-red",
+                  "index": 2,
+                  "text": "SOCKERR"
+                },
+                "3": {
+                  "color": "dark-green",
+                  "index": 3,
+                  "text": "L4OK"
+                },
+                "4": {
+                  "color": "dark-red",
+                  "index": 4,
+                  "text": "L4TOUT"
+                },
+                "5": {
+                  "color": "dark-red",
+                  "index": 5,
+                  "text": "L4CON"
+                },
+                "6": {
+                  "color": "dark-green",
+                  "index": 6,
+                  "text": "L6OK"
+                },
+                "7": {
+                  "color": "dark-red",
+                  "index": 7,
+                  "text": "L6TOUT"
+                },
+                "8": {
+                  "color": "dark-red",
+                  "index": 8,
+                  "text": "L6RSP"
+                },
+                "9": {
+                  "color": "dark-green",
+                  "index": 9,
+                  "text": "L7OK"
+                },
+                "10": {
+                  "color": "semi-dark-red",
+                  "index": 10,
+                  "text": "L7OKC"
+                },
+                "11": {
+                  "color": "dark-red",
+                  "index": 11,
+                  "text": "L7TOUT"
+                },
+                "12": {
+                  "color": "dark-red",
+                  "index": 12,
+                  "text": "L7RSP"
+                },
+                "13": {
+                  "color": "dark-red",
+                  "index": 13,
+                  "text": "L7STS"
+                }
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 83,
+      "options": {
+        "alignValue": "left",
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "mergeValues": true,
+        "rowHeight": 0.9,
+        "showValue": "auto",
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "targets": [
+        {
+          "exemplar": true,
+          "expr": "mongo_state",
+          "interval": "",
+          "legendFormat": "{{ servername }}",
+          "refId": "A"
+        }
+      ],
+      "title": "Mongo State",
+      "type": "state-timeline"
+    },
+    {
+      "collapsed": false,
       "datasource": null,
       "gridPos": {
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 27
+        "y": 8
       },
       "id": 46,
-      "panels": [
+      "panels": [],
+      "title": "Top Line",
+      "type": "row"
+    },
     {
       "aliasColors": {},
       "bars": false,
@@ -204,7 +340,7 @@
         "h": 6,
         "w": 8,
         "x": 0,
-            "y": 28
+        "y": 9
       },
       "hiddenSeries": false,
       "id": 22,
@@ -317,7 +453,7 @@
         "h": 6,
         "w": 8,
         "x": 8,
-            "y": 28
+        "y": 9
       },
       "hiddenSeries": false,
       "id": 44,
@@ -443,7 +579,7 @@
         "h": 6,
         "w": 8,
         "x": 16,
-            "y": 28
+        "y": 9
       },
       "hiddenSeries": false,
       "id": 38,
@@ -540,10 +676,6 @@
         "align": false,
         "alignLevel": null
       }
-        }
-      ],
-      "title": "Top Line",
-      "type": "row"
     },
     {
       "collapsed": false,
@@ -552,7 +684,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 28
+        "y": 15
       },
       "id": 20,
       "panels": [],
@@ -579,7 +711,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 29
+        "y": 16
       },
       "hiddenSeries": false,
       "id": 2,
@@ -685,7 +817,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 29
+        "y": 16
       },
       "hiddenSeries": false,
       "id": 6,
@@ -781,7 +913,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 37
+        "y": 24
       },
       "id": 4,
       "panels": [],
@@ -818,7 +950,7 @@
         "h": 21,
         "w": 24,
         "x": 0,
-        "y": 38
+        "y": 25
       },
       "id": 10,
       "options": {
@@ -872,7 +1004,7 @@
         "h": 7,
         "w": 24,
         "x": 0,
-        "y": 59
+        "y": 46
       },
       "hiddenSeries": false,
       "id": 48,
@@ -963,7 +1095,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 66
+        "y": 53
       },
       "id": 42,
       "panels": [],
@@ -982,7 +1114,7 @@
         "h": 9,
         "w": 24,
         "x": 0,
-        "y": 67
+        "y": 54
       },
       "hiddenSeries": false,
       "id": 33,
@@ -1085,7 +1217,7 @@
         "h": 8,
         "w": 24,
         "x": 0,
-        "y": 76
+        "y": 63
       },
       "hiddenSeries": false,
       "id": 30,
@@ -1280,7 +1412,7 @@
         "h": 8,
         "w": 24,
         "x": 0,
-        "y": 84
+        "y": 71
       },
       "hiddenSeries": false,
       "id": 32,
@@ -1379,7 +1511,7 @@
         "h": 7,
         "w": 24,
         "x": 0,
-        "y": 92
+        "y": 79
       },
       "hiddenSeries": false,
       "id": 28,
@@ -1492,7 +1624,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 99
+        "y": 86
       },
       "hiddenSeries": false,
       "id": 12,
@@ -1586,7 +1718,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 99
+        "y": 86
       },
       "hiddenSeries": false,
       "id": 26,
@@ -1677,7 +1809,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 107
+        "y": 94
       },
       "hiddenSeries": false,
       "id": 14,
@@ -1763,7 +1895,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 115
+        "y": 102
       },
       "id": 50,
       "panels": [],
@@ -1782,7 +1914,7 @@
         "h": 8,
         "w": 24,
         "x": 0,
-        "y": 116
+        "y": 103
       },
       "hiddenSeries": false,
       "id": 56,
@@ -1877,7 +2009,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 124
+        "y": 111
       },
       "hiddenSeries": false,
       "id": 52,
@@ -1970,7 +2102,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 124
+        "y": 111
       },
       "hiddenSeries": false,
       "id": 54,
@@ -2063,7 +2195,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 132
+        "y": 119
       },
       "hiddenSeries": false,
       "id": 58,
@@ -2156,7 +2288,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 132
+        "y": 119
       },
       "hiddenSeries": false,
       "id": 60,
@@ -2244,7 +2376,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 140
+        "y": 127
       },
       "id": 62,
       "panels": [],
@@ -2263,7 +2395,7 @@
         "h": 8,
         "w": 24,
         "x": 0,
-        "y": 141
+        "y": 128
       },
       "hiddenSeries": false,
       "id": 75,
@@ -2358,7 +2490,7 @@
         "h": 8,
         "w": 24,
         "x": 0,
-        "y": 149
+        "y": 136
       },
       "hiddenSeries": false,
       "id": 66,
@@ -2519,7 +2651,7 @@
         "h": 8,
         "w": 24,
         "x": 0,
-        "y": 157
+        "y": 144
       },
       "hiddenSeries": false,
       "id": 71,
@@ -2665,7 +2797,7 @@
         "h": 8,
         "w": 24,
         "x": 0,
-        "y": 165
+        "y": 152
       },
       "hiddenSeries": false,
       "id": 72,
@@ -2797,7 +2929,7 @@
         "h": 7,
         "w": 24,
         "x": 0,
-        "y": 173
+        "y": 160
       },
       "hiddenSeries": false,
       "id": 73,
@@ -2897,7 +3029,7 @@
         "h": 8,
         "w": 24,
         "x": 0,
-        "y": 180
+        "y": 167
       },
       "hiddenSeries": false,
       "id": 78,
@@ -2990,7 +3122,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 188
+        "y": 175
       },
       "hiddenSeries": false,
       "id": 80,
@@ -3083,7 +3215,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 188
+        "y": 175
       },
       "hiddenSeries": false,
       "id": 64,
@@ -3171,7 +3303,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 196
+        "y": 183
       },
       "id": 68,
       "panels": [],
@@ -3190,7 +3322,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 197
+        "y": 184
       },
       "hiddenSeries": false,
       "id": 70,
@@ -3283,7 +3415,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 197
+        "y": 184
       },
       "hiddenSeries": false,
       "id": 76,
@@ -3371,7 +3503,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 205
+        "y": 192
       },
       "id": 16,
       "panels": [],
@@ -3390,7 +3522,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 206
+        "y": 193
       },
       "hiddenSeries": false,
       "id": 18,
@@ -3481,7 +3613,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 206
+        "y": 193
       },
       "hiddenSeries": false,
       "id": 24,
@@ -3572,7 +3704,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 214
+        "y": 201
       },
       "hiddenSeries": false,
       "id": 35,
@@ -3663,7 +3795,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 214
+        "y": 201
       },
       "hiddenSeries": false,
       "id": 8,
@@ -3749,7 +3881,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 222
+        "y": 209
       },
       "id": 40,
       "panels": [
@@ -3829,12 +3961,12 @@
     "list": []
   },
   "time": {
-    "from": "now-5m",
+    "from": "now-7d",
     "to": "now"
   },
   "timepicker": {},
   "timezone": "",
   "title": "NewsBlur",
   "uid": "T86VjXrG2",
-  "version": 22
+  "version": 17
 }

docker/prometheus/prometheus.consul.yml.j2

@@ -238,7 +238,19 @@ scrape_configs:
       - source_labels: ['__meta_consul_node']
         target_label: instance
 
-    - job_name: 'haproxy state'
+    - job_name: 'redis state'
-      static_configs:
+      consul_sd_configs:
-        - targets: ['localhost:5569']
+      - server: 'consul.service.nyc1.consul:8500'
+        services: ['flask_metrics_redis']
+      relabel_configs:
+      - source_labels: ['__meta_consul_node']
+        target_label: instance
+      metrics_path: /state/
+    - job_name: 'mongo state'
+      consul_sd_configs:
+      - server: 'consul.service.nyc1.consul:8500'
+        services: ['flask_metrics_mongo']
+      relabel_configs:
+      - source_labels: ['__meta_consul_node']
+        target_label: instance
       metrics_path: /state/

docker/prometheus/prometheus.docker.yml

@@ -230,9 +230,16 @@ scrape_configs:
       tls_config:
         insecure_skip_verify: true
 
-    - job_name: 'service state'
+    - job_name: 'redis state'
       static_configs:
-        - targets: ['flask_metrics_state:5569']
+        - targets: ['flask_metrics_redis:5569']
+      metrics_path: /state/
+      scheme: http
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'mongo state'
+      static_configs:
+        - targets: ['flask_metrics_mongo:5569']
       metrics_path: /state/
       scheme: http
       tls_config:

flask_metrics/flask_metrics_mongo.py

@@ -1,5 +1,6 @@
 from flask import Flask, render_template, Response
 import pymongo
+from flask_metrics.state_timeline import format_state_data, get_state
 from newsblur_web import settings
 import sentry_sdk
 from sentry_sdk.integrations.flask import FlaskIntegration
@@ -165,6 +166,20 @@ def page_queues():
     html_body = render_template('prometheus_data.html', **context)
     return Response(html_body, content_type="text/plain")
 
+@app.route("/state/")
+def mongo_state():
+    mongo_data = get_state("mongo")
+    if 'BACKEND' in mongo_data:
+        del mongo_data['BACKEND']
+    formatted_data = format_state_data("mongo_state", mongo_data)
+    context = {
+        'chart_name': 'mongo_state',
+        'chart_type': 'gauge',
+        'data': formatted_data
+    }
+    html_body = render_template('prometheus_data.html', **context)
+    return Response(html_body, content_type="text/plain")
+
 
 if __name__ == "__main__":
     print(" ---> Starting NewsBlur Flask Metrics server...")

flask_metrics/state_timeline.py

@@ -1,19 +1,7 @@
-from flask import Flask, render_template, Response
+from flask import render_template
 import requests
 from requests.auth import HTTPBasicAuth
 
-from newsblur_web import settings
-import sentry_sdk
-from sentry_sdk.integrations.flask import FlaskIntegration
-
-if settings.FLASK_SENTRY_DSN is not None:
-    sentry_sdk.init(
-        dsn=settings.FLASK_SENTRY_DSN,
-        integrations=[FlaskIntegration()],
-        traces_sample_rate=1.0,
-    )
-
-app = Flask(__name__)
 
 STATUS_MAPPING = {
     "UNK": 0,      # unknown
@@ -39,32 +27,16 @@ def format_state_data(label, data):
             formatted_data[k] = f'{label}{{servername="{k}"}} {STATUS_MAPPING[v.strip()]}'
     return formatted_data
 
-def get_state():
+def get_state(backend_name):
     res = requests.get('https://newsblur.com:1936/;csv', auth=HTTPBasicAuth('gimmiestats', 'StatsGiver'))
     lines = res.content.decode('utf-8').split('\n')
-    backends = [line.split(",") for line in lines][1:]
+    backends = [line.split(",") for line in lines if backend_name in line]
 
     check_status_index = lines[0].split(",").index('check_status')
     servername_index = lines[0].split(",").index('svname')
 
     data = {}
+
     for backend_data in backends:
-        if backend_data != [''] and backend_data[servername_index] != 'FRONTEND':
         data[backend_data[servername_index]] = backend_data[check_status_index].replace("*", "")
     return data
-
-@app.route("/state/")
-def state():
-    state_data = get_state()
-    formatted_data = format_state_data("state", state_data)
-    context = {
-        'chart_name': 'state',
-        'chart_type': 'gauge',
-        'data': formatted_data
-    }
-    html_body = render_template('prometheus_data.html', **context)
-    return Response(html_body, content_type="text/plain")
-
-if __name__ == "__main__":
-    print(" ---> Starting State Timeline Flask Metrics server...")
-    app.run(host="0.0.0.0", port=5569)