Merge branch 'state_timeline'

* state_timeline:
  fix flask_metrics ansible role
  add state timeline dashboard to grafana
  move state timeline to one single flask that gets astate for all services

ansible/roles/flask_metrics/tasks/main.yml

@@ -51,7 +51,8 @@
       file: /srv/newsblur/flask_metrics/flask_metrics_mongo.py
     - service_name: redis
       file: /srv/newsblur/flask_metrics/flask_metrics_redis.py
-
+    - service_name: metrics
+      file: /srv/newsblur/flask_metrics/state_timeline.py
 - name: Restart flask_metrics
   become: yes
   shell:

docker-compose.metrics.yml

@@ -69,6 +69,20 @@ services:
       - nginx
     volumes:
       - ${PWD}:/srv/newsblur
+  flask_metrics_state:
+    container_name: flask_metrics_state
+    image: newsblur/newsblur_monitor:latest
+    command: bash -c "python /srv/newsblur/flask_metrics/state_timeline.py"
+    environment: 
+      - DOCKERBUILD=True
+    ports:
+      - 5599:5569
+    depends_on: 
+      - haproxy
+      - newsblur_web
+      - nginx
+    volumes:
+      - ${PWD}:/srv/newsblur
   elasticsearch_exporter:
     container_name: elasticsearch_exporter
     image: prometheuscommunity/elasticsearch-exporter:latest

docker/prometheus/prometheus.consul.yml.j2

@@ -238,19 +238,7 @@ scrape_configs:
       - source_labels: ['__meta_consul_node']
         target_label: instance
 
-    - job_name: 'redis state'
-      consul_sd_configs:
-      - server: 'consul.service.nyc1.consul:8500'
-        services: ['flask_metrics_redis']
-      relabel_configs:
-      - source_labels: ['__meta_consul_node']
-        target_label: instance
-      metrics_path: /state/
-    - job_name: 'mongo state'
-      consul_sd_configs:
-      - server: 'consul.service.nyc1.consul:8500'
-        services: ['flask_metrics_mongo']
-      relabel_configs:
-      - source_labels: ['__meta_consul_node']
-        target_label: instance
+    - job_name: 'haproxy state'
+      static_configs:
+        - targets: ['localhost:5569']:
       metrics_path: /state/

docker/prometheus/prometheus.docker.yml

@@ -230,16 +230,9 @@ scrape_configs:
       tls_config:
         insecure_skip_verify: true
 
-    - job_name: 'redis state'
+    - job_name: 'service state'
       static_configs:
-        - targets: ['flask_metrics_redis:5569']
-      metrics_path: /state/
-      scheme: http
-      tls_config:
-        insecure_skip_verify: true
-    - job_name: 'mongo state'
-      static_configs:
-        - targets: ['flask_metrics_mongo:5569']
+        - targets: ['flask_metrics_state:5569']
       metrics_path: /state/
       scheme: http
       tls_config:

flask_metrics/flask_metrics_mongo.py

@@ -1,6 +1,5 @@
 from flask import Flask, render_template, Response
 import pymongo
-from flask_metrics.state_timeline import format_state_data, get_state
 from newsblur_web import settings
 import sentry_sdk
 from sentry_sdk.integrations.flask import FlaskIntegration
@@ -166,20 +165,6 @@ def page_queues():
     html_body = render_template('prometheus_data.html', **context)
     return Response(html_body, content_type="text/plain")
 
-@app.route("/state/")
-def mongo_state():
-    mongo_data = get_state("mongo")
-    if 'BACKEND' in mongo_data:
-        del mongo_data['BACKEND']
-    formatted_data = format_state_data("mongo_state", mongo_data)
-    context = {
-        'chart_name': 'mongo_state',
-        'chart_type': 'gauge',
-        'data': formatted_data
-    }
-    html_body = render_template('prometheus_data.html', **context)
-    return Response(html_body, content_type="text/plain")
-
 
 if __name__ == "__main__":
     print(" ---> Starting NewsBlur Flask Metrics server...")

flask_metrics/state_timeline.py

@@ -1,7 +1,19 @@
-from flask import render_template
+from flask import Flask, render_template, Response
 import requests
 from requests.auth import HTTPBasicAuth
 
+from newsblur_web import settings
+import sentry_sdk
+from sentry_sdk.integrations.flask import FlaskIntegration
+
+if settings.FLASK_SENTRY_DSN is not None:
+    sentry_sdk.init(
+        dsn=settings.FLASK_SENTRY_DSN,
+        integrations=[FlaskIntegration()],
+        traces_sample_rate=1.0,
+    )
+
+app = Flask(__name__)
 
 STATUS_MAPPING = {
     "UNK": 0,      # unknown
@@ -27,16 +39,32 @@ def format_state_data(label, data):
             formatted_data[k] = f'{label}{{servername="{k}"}} {STATUS_MAPPING[v.strip()]}'
     return formatted_data
 
-def get_state(backend_name):
+def get_state():
     res = requests.get('https://newsblur.com:1936/;csv', auth=HTTPBasicAuth('gimmiestats', 'StatsGiver'))
     lines = res.content.decode('utf-8').split('\n')
-    backends = [line.split(",") for line in lines if backend_name in line]
+    backends = [line.split(",") for line in lines][1:]
 
     check_status_index = lines[0].split(",").index('check_status')
     servername_index = lines[0].split(",").index('svname')
 
     data = {}
-
     for backend_data in backends:
-        data[backend_data[servername_index]] = backend_data[check_status_index].replace("*", "")
+        if backend_data != [''] and backend_data[servername_index] != 'FRONTEND':
+            data[backend_data[servername_index]] = backend_data[check_status_index].replace("*", "")
     return data
+
+@app.route("/state/")
+def state():
+    state_data = get_state()
+    formatted_data = format_state_data("state", state_data)
+    context = {
+        'chart_name': 'state',
+        'chart_type': 'gauge',
+        'data': formatted_data
+    }
+    html_body = render_template('prometheus_data.html', **context)
+    return Response(html_body, content_type="text/plain")
+
+if __name__ == "__main__":
+    print(" ---> Starting State Timeline Flask Metrics server...")
+    app.run(host="0.0.0.0", port=5569)