redo prometheus scrapers and build the jobs for them in prometheus.yml

apps/monitor/views/newsblur_app_servers.py

@@ -9,8 +9,14 @@ class AppServers(View):
         data = dict((("%s" % s['_id'].replace('-', ''), s['feeds']) for s in self.stats))
         if self.total:
             data['total'] = self.total[0]['feeds']
-
+        chart_name = "app_servers"
-        return render(request, 'monitor/prometheus_data.html', {"data": data})
+        chart_type = "histogram"
+        context = {
+            "data": data,
+            "chart_name": chart_name,
+            "chart_type": chart_type,
+        } 
+        return render(request, 'monitor/prometheus_data.html', context)
     
     @property
     def stats(self):

apps/monitor/views/newsblur_app_times.py

@@ -8,7 +8,15 @@ class AppTimes(View):
     def get(self, request):
         servers = dict((("%s" % s['_id'], s['page_load']) for s in self.stats))
         data = servers
-        return render(request, 'monitor/prometheus_data.html', {"data": data})
+        chart_name = "app_times"
+        chart_type = "histogram"
+
+        context = {
+            "data": data,
+            "chart_name": chart_name,
+            "chart_type": chart_type,
+        }
+        return render(request, 'monitor/prometheus_data.html', context)
     
     @property
     def stats(self):

apps/monitor/views/newsblur_classifiers.py

@@ -13,5 +13,13 @@ class Classifiers(View):
             'titles': MClassifierTitle.objects.count(),
         }
 
-        return render(request, 'monitor/prometheus_data.html', {"data": data})
+        chart_name = "classifiers"
+        chart_type = "histogram"
+
+        context = {
+            "data": data,
+            "chart_name": chart_name,
+            "chart_type": chart_type,
+        }
+        return render(request, 'monitor/prometheus_data.html', context)
 

apps/monitor/views/newsblur_dbtimes.py

@@ -16,5 +16,12 @@ class DbTimes(View):
             'task_mongo_avg': MStatistics.get('latest_task_mongo_avg'),
             'task_redis_avg': MStatistics.get('latest_task_redis_avg'),
         }
+        chart_name = "db_times"
+        chart_type = "histogram"
 
-        return render(request, 'monitor/prometheus_data.html', {"data": data})
+        context = {
+            "data": data,
+            "chart_name": chart_name,
+            "chart_type": chart_type,
+        }
+        return render(request, 'monitor/prometheus_data.html', context)

apps/monitor/views/newsblur_errors.py

@@ -10,5 +10,13 @@ class Errors(View):
         data = {
             'feed_success': statistics['feeds_fetched'],
         }
-        return render(request, 'monitor/prometheus_data.html', {"data": data})
+        chart_name = "errors"
+        chart_type = "histogram"
+
+        context = {
+            "data": data,
+            "chart_name": chart_name,
+            "chart_type": chart_type,
+        }
+        return render(request, 'monitor/prometheus_data.html', context)
 

apps/monitor/views/newsblur_feed_counts.py

@@ -45,6 +45,14 @@ class FeedCounts(View):
             'active_feeds': active_feeds,
             'push_feeds': push_feeds,
         }
-        return render(request, 'monitor/prometheus_data.html', {"data": data})
+        chart_name = "feed_counts"
+        chart_type = "histogram"
+
+        context = {
+            "data": data,
+            "chart_name": chart_name,
+            "chart_type": chart_type,
+        }
+        return render(request, 'monitor/prometheus_data.html', context)
 
 

apps/monitor/views/newsblur_feeds.py

@@ -26,6 +26,14 @@ class Feeds(View):
             'profiles': MSocialProfile.objects.count(),
             'social_subscriptions': MSocialSubscription.objects.count(),
         }
+        chart_name = "feeds"
+        chart_type = "histogram"
 
-        return render(request, 'monitor/prometheus_data.html', {"data": data})
+        context = {
+            "data": data,
+            "chart_name": chart_name,
+            "chart_type": chart_type,
+        }
+
+        return render(request, 'monitor/prometheus_data.html', context)
 

apps/monitor/views/newsblur_loadtimes.py

@@ -10,5 +10,13 @@ class LoadTimes(View):
             'feed_loadtimes_avg_hour': MStatistics.get('latest_avg_time_taken'),
             'feeds_loaded_hour': MStatistics.get('latest_sites_loaded'),
         }
-        return render(request, 'monitor/prometheus_data.html', {"data": data})
+        chart_name = "load_times"
+        chart_type = "histogram"
+
+        context = {
+            "data": data,
+            "chart_name": chart_name,
+            "chart_type": chart_type,
+        }
+        return render(request, 'monitor/prometheus_data.html', context)
 

apps/monitor/views/newsblur_stories.py

@@ -10,5 +10,13 @@ class Stories(View):
             'stories': MStory.objects.count(),
             'starred_stories': MStarredStory.objects.count(),
         }
-        return render(request, 'monitor/prometheus_data.html', {"data": data})
+        chart_name = "stories"
+        chart_type = "histogram"
+
+        context = {
+            "data": data,
+            "chart_name": chart_name,
+            "chart_type": chart_type,
+        }
+        return render(request, 'monitor/prometheus_data.html', context)
 

apps/monitor/views/newsblur_tasks_codes.py

@@ -7,8 +7,15 @@ class TasksCodes(View):
 
     def get(self, request):
         data = dict((("_%s" % s['_id'], s['feeds']) for s in self.stats))
-        
+        chart_name = "task_codes"
-        return render(request, 'monitor/prometheus_data.html', {"data": data})
+        chart_type = "histogram"
+
+        context = {
+            "data": data,
+            "chart_name": chart_name,
+            "chart_type": chart_type,
+        }
+        return render(request, 'monitor/prometheus_data.html', context)
     
     @property
     def stats(self):        

apps/monitor/views/newsblur_tasks_pipeline.py

@@ -8,7 +8,15 @@ class TasksPipeline(View):
 
     def get(self, request):
         data =self.stats
-        return render(request, 'monitor/prometheus_data.html', {"data": data})
+        chart_name = "task_pipeline"
+        chart_type = "histogram"
+
+        context = {
+            "data": data,
+            "chart_name": chart_name,
+            "chart_type": chart_type,
+        }
+        return render(request, 'monitor/prometheus_data.html', context)
     
     @property
     def stats(self):

apps/monitor/views/newsblur_tasks_servers.py

@@ -10,8 +10,15 @@ class TasksServers(View):
         data = dict((("%s" % s['_id'].replace('-', ''), s['feeds']) for s in self.stats))
         if self.total:
             data['total'] = self.total[0]['feeds']
+        chart_name = "task_servers"
+        chart_type = "histogram"
 
-        return render(request, 'monitor/prometheus_data.html', {"data": data})
+        context = {
+            "data": data,
+            "chart_name": chart_name,
+            "chart_type": chart_type,
+        }
+        return render(request, 'monitor/prometheus_data.html', context)
 
     
     @property

apps/monitor/views/newsblur_tasks_times.py

@@ -8,8 +8,15 @@ class TasksTimes(View):
 
     def get(self, request):
         data = dict((("%s" % s['_id'], s['total']) for s in self.stats))
+        chart_name = "task_times"
+        chart_type = "histogram"
 
-        return render(request, 'monitor/prometheus_data.html', {"data": data})
+        context = {
+            "data": data,
+            "chart_name": chart_name,
+            "chart_type": chart_type,
+        }
+        return render(request, 'monitor/prometheus_data.html', context)
 
     
     @property

apps/monitor/views/newsblur_updates.py

@@ -20,5 +20,13 @@ class Updates(View):
             'celery_work_queue': r.llen("work_queue"),
             'celery_search_queue': r.llen("search_indexer"),
         }
-        return render(request, 'monitor/prometheus_data.html', {"data": data})
+        chart_name = "updates"
+        chart_type = "histogram"
+
+        context = {
+            "data": data,
+            "chart_name": chart_name,
+            "chart_type": chart_type,
+        }
+        return render(request, 'monitor/prometheus_data.html', context)
 

apps/monitor/views/newsblur_users.py

@@ -19,5 +19,13 @@ class Users(View):
             'premium': Profile.objects.filter(is_premium=True).count(),
             'queued': RNewUserQueue.user_count(),
         }
-        return render(request, 'monitor/prometheus_data.html', {"data": data})
+        chart_name = "users"
+        chart_type = "histogram"
+
+        context = {
+            "data": data,
+            "chart_name": chart_name,
+            "chart_type": chart_type,
+        }
+        return render(request, 'monitor/prometheus_data.html', context)
 

docker-compose.yml

@@ -110,7 +110,7 @@ services:
       - 9200:9200
       - 9300:9300
     volumes:
-      - ./docker/volumes/elasticsearch:/usr/share/elasticsearch/data
+      - ./docker/volumes/elasticsearch:/elasticsearch/dat
       - ./config/elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
 
   db_mongo:
@@ -163,13 +163,18 @@ services:
       - ./docker/volumes/prometheus_data:/prometheus
     depends_on:
       - node-exporter
+      - haproxy
+    external_links:
+      - haproxy
 
   node-exporter:
+    container_name: node-exporter
     image: prom/node-exporter
     ports:
       - 9100:9100
 
   grafana:
+    container_name: grafana
     image: grafana/grafana
     environment:
       - GF_SECURITY_ADMIN_PASSWORD=pass

docker/prometheus/prometheus.yml

@@ -3,9 +3,115 @@ global:
     external_labels:
         monitor: 'my-monitor'
 scrape_configs:
-    - job_name: 'prometheus'
+    #- job_name: 'prometheus'
-      target_groups:
+    #  target_groups:
-          - targets: ['localhost:9090']
+    #    - targets: ['localhost:9090']
     - job_name: 'node-exporter'
       target_groups:
-          - targets: ['node-exporter:9100']
+        - targets: ['node-exporter:9100']
+
+    - job_name: 'app_servers'
+      target_groups:
+        - targets: ['haproxy']
+      metrics_path: /monitor/app-servers
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'app_times'
+      target_groups:
+        - targets: ['haproxy']
+      metrics_path: /monitor/app-times
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'classifiers'
+      target_groups:
+        - targets: ['haproxy']
+      metrics_path: /monitor/classifiers
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'db_times'
+      target_groups:
+        - targets: ['haproxy']
+      metrics_path: /monitor/db-times
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'errors'
+      target_groups:
+        - targets: ['haproxy']
+      metrics_path: /monitor/errors
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'feed_counts'
+      target_groups:
+        - targets: ['haproxy']
+      metrics_path: /monitor/feed-counts
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'feeds'
+      target_groups:
+        - targets: ['haproxy']
+      metrics_path: /monitor/feeds
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'load_times'
+      target_groups:
+        - targets: ['haproxy']
+      metrics_path: /monitor/load-times
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'stories'
+      target_groups:
+        - targets: ['haproxy']
+      metrics_path: /monitor/stories
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'task_codes'
+      target_groups:
+        - targets: ['haproxy']
+      metrics_path: /monitor/task-codes
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'task_pipeline'
+      target_groups:
+        - targets: ['haproxy']
+      metrics_path: /monitor/task-pipeline
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'task_servers'
+      target_groups:
+        - targets: ['haproxy']
+      metrics_path: /monitor/task-servers
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'task_times'
+      target_groups:
+        - targets: ['haproxy']
+      metrics_path: /monitor/task-times
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'updates'
+      target_groups:
+        - targets: ['haproxy']
+      metrics_path: /monitor/updates
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'users'
+      target_groups:
+        - targets: ['haproxy']
+      metrics_path: /monitor/users
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true

docker/prometheus/prometheus.yml.j2

@@ -5,7 +5,114 @@ global:
 scrape_configs:
     - job_name: 'prometheus'
       target_groups:
-          - targets: ['localhost:9090']
+        - targets: ['localhost:9090']
+
     - job_name: 'node-exporter'
       target_groups:
-          - targets: ['node-exporter:9100']
+        - targets: ['node-exporter:9100']
+
+    - job_name: 'app_servers'
+      target_groups:
+        - targets: ['{{ ansible_ssh_host }}']
+      metrics_path: /monitor/app-servers
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'app_times'
+      target_groups:
+        - targets: ['{{ ansible_ssh_host }}']
+      metrics_path: /monitor/app-times
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'classifiers'
+      target_groups:
+        - targets: ['{{ ansible_ssh_host }}']
+      metrics_path: /monitor/classifiers
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'db_times'
+      target_groups:
+        - targets: ['{{ ansible_ssh_host }}']
+      metrics_path: /monitor/db-times
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'errors'
+      target_groups:
+        - targets: ['{{ ansible_ssh_host }}']
+      metrics_path: /monitor/errors
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'feed_counts'
+      target_groups:
+        - targets: ['{{ ansible_ssh_host }}']
+      metrics_path: /monitor/feed-counts
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'feeds'
+      target_groups:
+        - targets: ['{{ ansible_ssh_host }}']
+      metrics_path: /monitor/feeds
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'load_times'
+      target_groups:
+        - targets: ['{{ ansible_ssh_host }}']
+      metrics_path: /monitor/load-times
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'stories'
+      target_groups:
+        - targets: ['{{ ansible_ssh_host }}']
+      metrics_path: /monitor/stories
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'task_codes'
+      target_groups:
+        - targets: ['{{ ansible_ssh_host }}']
+      metrics_path: /monitor/task-codes
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'task_pipeline'
+      target_groups:
+        - targets: ['{{ ansible_ssh_host }}']
+      metrics_path: /monitor/task-pipeline
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'task_servers'
+      target_groups:
+        - targets: ['{{ ansible_ssh_host }}']
+      metrics_path: /monitor/task-servers
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'task_times'
+      target_groups:
+        - targets: ['{{ ansible_ssh_host }}']
+      metrics_path: /monitor/task-times
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'updates'
+      target_groups:
+        - targets: ['{{ ansible_ssh_host }}']
+      metrics_path: /monitor/updates
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+    - job_name: 'users'
+      target_groups:
+        - targets: ['{{ ansible_ssh_host }}']
+      metrics_path: /monitor/users
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true

templates/monitor/prometheus_data.html

@@ -1,3 +1,4 @@
 {% for k, v in data.items %}
-{{ k }} {{ v }}<br>
+# TYPE {{chart_name}}_{{k}} {{chart_type}}
+{{chart_name}}_{{k}} {{ v }}
 {% endfor %}