From 9cbad888c77d2fa74313d38a803037b253dd9b24 Mon Sep 17 00:00:00 2001 From: Jonathan Math Date: Fri, 30 Apr 2021 13:48:48 -0500 Subject: [PATCH] redo prometheus scrapers and build the jobs for them in prometheus.yml --- apps/monitor/views/newsblur_app_servers.py | 10 +- apps/monitor/views/newsblur_app_times.py | 10 +- apps/monitor/views/newsblur_classifiers.py | 10 +- apps/monitor/views/newsblur_dbtimes.py | 9 +- apps/monitor/views/newsblur_errors.py | 10 +- apps/monitor/views/newsblur_feed_counts.py | 10 +- apps/monitor/views/newsblur_feeds.py | 10 +- apps/monitor/views/newsblur_loadtimes.py | 10 +- apps/monitor/views/newsblur_stories.py | 10 +- apps/monitor/views/newsblur_tasks_codes.py | 11 +- apps/monitor/views/newsblur_tasks_pipeline.py | 10 +- apps/monitor/views/newsblur_tasks_servers.py | 9 +- apps/monitor/views/newsblur_tasks_times.py | 9 +- apps/monitor/views/newsblur_updates.py | 10 +- apps/monitor/views/newsblur_users.py | 10 +- docker-compose.yml | 7 +- docker/prometheus/prometheus.yml | 114 +++++++++++++++++- docker/prometheus/prometheus.yml.j2 | 111 ++++++++++++++++- templates/monitor/prometheus_data.html | 3 +- 19 files changed, 358 insertions(+), 25 deletions(-) diff --git a/apps/monitor/views/newsblur_app_servers.py b/apps/monitor/views/newsblur_app_servers.py index d60b0dc5d..29948357d 100755 --- a/apps/monitor/views/newsblur_app_servers.py +++ b/apps/monitor/views/newsblur_app_servers.py @@ -9,8 +9,14 @@ class AppServers(View): data = dict((("%s" % s['_id'].replace('-', ''), s['feeds']) for s in self.stats)) if self.total: data['total'] = self.total[0]['feeds'] - - return render(request, 'monitor/prometheus_data.html', {"data": data}) + chart_name = "app_servers" + chart_type = "histogram" + context = { + "data": data, + "chart_name": chart_name, + "chart_type": chart_type, + } + return render(request, 'monitor/prometheus_data.html', context) @property def stats(self): diff --git a/apps/monitor/views/newsblur_app_times.py b/apps/monitor/views/newsblur_app_times.py index fff819fdb..b0d54d91d 100755 --- a/apps/monitor/views/newsblur_app_times.py +++ b/apps/monitor/views/newsblur_app_times.py @@ -8,7 +8,15 @@ class AppTimes(View): def get(self, request): servers = dict((("%s" % s['_id'], s['page_load']) for s in self.stats)) data = servers - return render(request, 'monitor/prometheus_data.html', {"data": data}) + chart_name = "app_times" + chart_type = "histogram" + + context = { + "data": data, + "chart_name": chart_name, + "chart_type": chart_type, + } + return render(request, 'monitor/prometheus_data.html', context) @property def stats(self): diff --git a/apps/monitor/views/newsblur_classifiers.py b/apps/monitor/views/newsblur_classifiers.py index 21b12bff1..102cca770 100755 --- a/apps/monitor/views/newsblur_classifiers.py +++ b/apps/monitor/views/newsblur_classifiers.py @@ -13,5 +13,13 @@ class Classifiers(View): 'titles': MClassifierTitle.objects.count(), } - return render(request, 'monitor/prometheus_data.html', {"data": data}) + chart_name = "classifiers" + chart_type = "histogram" + + context = { + "data": data, + "chart_name": chart_name, + "chart_type": chart_type, + } + return render(request, 'monitor/prometheus_data.html', context) diff --git a/apps/monitor/views/newsblur_dbtimes.py b/apps/monitor/views/newsblur_dbtimes.py index 459003391..08fe269d3 100755 --- a/apps/monitor/views/newsblur_dbtimes.py +++ b/apps/monitor/views/newsblur_dbtimes.py @@ -16,5 +16,12 @@ class DbTimes(View): 'task_mongo_avg': MStatistics.get('latest_task_mongo_avg'), 'task_redis_avg': MStatistics.get('latest_task_redis_avg'), } + chart_name = "db_times" + chart_type = "histogram" - return render(request, 'monitor/prometheus_data.html', {"data": data}) + context = { + "data": data, + "chart_name": chart_name, + "chart_type": chart_type, + } + return render(request, 'monitor/prometheus_data.html', context) diff --git a/apps/monitor/views/newsblur_errors.py b/apps/monitor/views/newsblur_errors.py index a34e51059..d81630ea0 100755 --- a/apps/monitor/views/newsblur_errors.py +++ b/apps/monitor/views/newsblur_errors.py @@ -10,5 +10,13 @@ class Errors(View): data = { 'feed_success': statistics['feeds_fetched'], } - return render(request, 'monitor/prometheus_data.html', {"data": data}) + chart_name = "errors" + chart_type = "histogram" + + context = { + "data": data, + "chart_name": chart_name, + "chart_type": chart_type, + } + return render(request, 'monitor/prometheus_data.html', context) diff --git a/apps/monitor/views/newsblur_feed_counts.py b/apps/monitor/views/newsblur_feed_counts.py index 165522acd..741db2883 100755 --- a/apps/monitor/views/newsblur_feed_counts.py +++ b/apps/monitor/views/newsblur_feed_counts.py @@ -45,6 +45,14 @@ class FeedCounts(View): 'active_feeds': active_feeds, 'push_feeds': push_feeds, } - return render(request, 'monitor/prometheus_data.html', {"data": data}) + chart_name = "feed_counts" + chart_type = "histogram" + + context = { + "data": data, + "chart_name": chart_name, + "chart_type": chart_type, + } + return render(request, 'monitor/prometheus_data.html', context) diff --git a/apps/monitor/views/newsblur_feeds.py b/apps/monitor/views/newsblur_feeds.py index 67d999e87..4330461b0 100755 --- a/apps/monitor/views/newsblur_feeds.py +++ b/apps/monitor/views/newsblur_feeds.py @@ -26,6 +26,14 @@ class Feeds(View): 'profiles': MSocialProfile.objects.count(), 'social_subscriptions': MSocialSubscription.objects.count(), } + chart_name = "feeds" + chart_type = "histogram" - return render(request, 'monitor/prometheus_data.html', {"data": data}) + context = { + "data": data, + "chart_name": chart_name, + "chart_type": chart_type, + } + + return render(request, 'monitor/prometheus_data.html', context) diff --git a/apps/monitor/views/newsblur_loadtimes.py b/apps/monitor/views/newsblur_loadtimes.py index d79fa3dac..4c2db04e6 100755 --- a/apps/monitor/views/newsblur_loadtimes.py +++ b/apps/monitor/views/newsblur_loadtimes.py @@ -10,5 +10,13 @@ class LoadTimes(View): 'feed_loadtimes_avg_hour': MStatistics.get('latest_avg_time_taken'), 'feeds_loaded_hour': MStatistics.get('latest_sites_loaded'), } - return render(request, 'monitor/prometheus_data.html', {"data": data}) + chart_name = "load_times" + chart_type = "histogram" + + context = { + "data": data, + "chart_name": chart_name, + "chart_type": chart_type, + } + return render(request, 'monitor/prometheus_data.html', context) diff --git a/apps/monitor/views/newsblur_stories.py b/apps/monitor/views/newsblur_stories.py index 9ae2a2485..daf294073 100755 --- a/apps/monitor/views/newsblur_stories.py +++ b/apps/monitor/views/newsblur_stories.py @@ -10,5 +10,13 @@ class Stories(View): 'stories': MStory.objects.count(), 'starred_stories': MStarredStory.objects.count(), } - return render(request, 'monitor/prometheus_data.html', {"data": data}) + chart_name = "stories" + chart_type = "histogram" + + context = { + "data": data, + "chart_name": chart_name, + "chart_type": chart_type, + } + return render(request, 'monitor/prometheus_data.html', context) diff --git a/apps/monitor/views/newsblur_tasks_codes.py b/apps/monitor/views/newsblur_tasks_codes.py index 2679ae7af..af96f707a 100755 --- a/apps/monitor/views/newsblur_tasks_codes.py +++ b/apps/monitor/views/newsblur_tasks_codes.py @@ -7,8 +7,15 @@ class TasksCodes(View): def get(self, request): data = dict((("_%s" % s['_id'], s['feeds']) for s in self.stats)) - - return render(request, 'monitor/prometheus_data.html', {"data": data}) + chart_name = "task_codes" + chart_type = "histogram" + + context = { + "data": data, + "chart_name": chart_name, + "chart_type": chart_type, + } + return render(request, 'monitor/prometheus_data.html', context) @property def stats(self): diff --git a/apps/monitor/views/newsblur_tasks_pipeline.py b/apps/monitor/views/newsblur_tasks_pipeline.py index 45b163ae0..ab9a2c83f 100755 --- a/apps/monitor/views/newsblur_tasks_pipeline.py +++ b/apps/monitor/views/newsblur_tasks_pipeline.py @@ -8,7 +8,15 @@ class TasksPipeline(View): def get(self, request): data =self.stats - return render(request, 'monitor/prometheus_data.html', {"data": data}) + chart_name = "task_pipeline" + chart_type = "histogram" + + context = { + "data": data, + "chart_name": chart_name, + "chart_type": chart_type, + } + return render(request, 'monitor/prometheus_data.html', context) @property def stats(self): diff --git a/apps/monitor/views/newsblur_tasks_servers.py b/apps/monitor/views/newsblur_tasks_servers.py index fecf57b35..503b8d609 100755 --- a/apps/monitor/views/newsblur_tasks_servers.py +++ b/apps/monitor/views/newsblur_tasks_servers.py @@ -10,8 +10,15 @@ class TasksServers(View): data = dict((("%s" % s['_id'].replace('-', ''), s['feeds']) for s in self.stats)) if self.total: data['total'] = self.total[0]['feeds'] + chart_name = "task_servers" + chart_type = "histogram" - return render(request, 'monitor/prometheus_data.html', {"data": data}) + context = { + "data": data, + "chart_name": chart_name, + "chart_type": chart_type, + } + return render(request, 'monitor/prometheus_data.html', context) @property diff --git a/apps/monitor/views/newsblur_tasks_times.py b/apps/monitor/views/newsblur_tasks_times.py index b0ad56669..23722f579 100755 --- a/apps/monitor/views/newsblur_tasks_times.py +++ b/apps/monitor/views/newsblur_tasks_times.py @@ -8,8 +8,15 @@ class TasksTimes(View): def get(self, request): data = dict((("%s" % s['_id'], s['total']) for s in self.stats)) + chart_name = "task_times" + chart_type = "histogram" - return render(request, 'monitor/prometheus_data.html', {"data": data}) + context = { + "data": data, + "chart_name": chart_name, + "chart_type": chart_type, + } + return render(request, 'monitor/prometheus_data.html', context) @property diff --git a/apps/monitor/views/newsblur_updates.py b/apps/monitor/views/newsblur_updates.py index e67a762b0..8fc6fbb25 100755 --- a/apps/monitor/views/newsblur_updates.py +++ b/apps/monitor/views/newsblur_updates.py @@ -20,5 +20,13 @@ class Updates(View): 'celery_work_queue': r.llen("work_queue"), 'celery_search_queue': r.llen("search_indexer"), } - return render(request, 'monitor/prometheus_data.html', {"data": data}) + chart_name = "updates" + chart_type = "histogram" + + context = { + "data": data, + "chart_name": chart_name, + "chart_type": chart_type, + } + return render(request, 'monitor/prometheus_data.html', context) diff --git a/apps/monitor/views/newsblur_users.py b/apps/monitor/views/newsblur_users.py index 2e4f06d19..a9345f865 100755 --- a/apps/monitor/views/newsblur_users.py +++ b/apps/monitor/views/newsblur_users.py @@ -19,5 +19,13 @@ class Users(View): 'premium': Profile.objects.filter(is_premium=True).count(), 'queued': RNewUserQueue.user_count(), } - return render(request, 'monitor/prometheus_data.html', {"data": data}) + chart_name = "users" + chart_type = "histogram" + + context = { + "data": data, + "chart_name": chart_name, + "chart_type": chart_type, + } + return render(request, 'monitor/prometheus_data.html', context) diff --git a/docker-compose.yml b/docker-compose.yml index d4e100304..80e794890 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -110,7 +110,7 @@ services: - 9200:9200 - 9300:9300 volumes: - - ./docker/volumes/elasticsearch:/usr/share/elasticsearch/data + - ./docker/volumes/elasticsearch:/elasticsearch/dat - ./config/elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml db_mongo: @@ -163,13 +163,18 @@ services: - ./docker/volumes/prometheus_data:/prometheus depends_on: - node-exporter + - haproxy + external_links: + - haproxy node-exporter: + container_name: node-exporter image: prom/node-exporter ports: - 9100:9100 grafana: + container_name: grafana image: grafana/grafana environment: - GF_SECURITY_ADMIN_PASSWORD=pass diff --git a/docker/prometheus/prometheus.yml b/docker/prometheus/prometheus.yml index bf420f0c0..db548881f 100644 --- a/docker/prometheus/prometheus.yml +++ b/docker/prometheus/prometheus.yml @@ -3,9 +3,115 @@ global: external_labels: monitor: 'my-monitor' scrape_configs: - - job_name: 'prometheus' - target_groups: - - targets: ['localhost:9090'] + #- job_name: 'prometheus' + # target_groups: + # - targets: ['localhost:9090'] - job_name: 'node-exporter' target_groups: - - targets: ['node-exporter:9100'] \ No newline at end of file + - targets: ['node-exporter:9100'] + + - job_name: 'app_servers' + target_groups: + - targets: ['haproxy'] + metrics_path: /monitor/app-servers + scheme: https + tls_config: + insecure_skip_verify: true + - job_name: 'app_times' + target_groups: + - targets: ['haproxy'] + metrics_path: /monitor/app-times + scheme: https + tls_config: + insecure_skip_verify: true + - job_name: 'classifiers' + target_groups: + - targets: ['haproxy'] + metrics_path: /monitor/classifiers + scheme: https + tls_config: + insecure_skip_verify: true + - job_name: 'db_times' + target_groups: + - targets: ['haproxy'] + metrics_path: /monitor/db-times + scheme: https + tls_config: + insecure_skip_verify: true + - job_name: 'errors' + target_groups: + - targets: ['haproxy'] + metrics_path: /monitor/errors + scheme: https + tls_config: + insecure_skip_verify: true + - job_name: 'feed_counts' + target_groups: + - targets: ['haproxy'] + metrics_path: /monitor/feed-counts + scheme: https + tls_config: + insecure_skip_verify: true + - job_name: 'feeds' + target_groups: + - targets: ['haproxy'] + metrics_path: /monitor/feeds + scheme: https + tls_config: + insecure_skip_verify: true + - job_name: 'load_times' + target_groups: + - targets: ['haproxy'] + metrics_path: /monitor/load-times + scheme: https + tls_config: + insecure_skip_verify: true + - job_name: 'stories' + target_groups: + - targets: ['haproxy'] + metrics_path: /monitor/stories + scheme: https + tls_config: + insecure_skip_verify: true + - job_name: 'task_codes' + target_groups: + - targets: ['haproxy'] + metrics_path: /monitor/task-codes + scheme: https + tls_config: + insecure_skip_verify: true + - job_name: 'task_pipeline' + target_groups: + - targets: ['haproxy'] + metrics_path: /monitor/task-pipeline + scheme: https + tls_config: + insecure_skip_verify: true + - job_name: 'task_servers' + target_groups: + - targets: ['haproxy'] + metrics_path: /monitor/task-servers + scheme: https + tls_config: + insecure_skip_verify: true + - job_name: 'task_times' + target_groups: + - targets: ['haproxy'] + metrics_path: /monitor/task-times + scheme: https + tls_config: + insecure_skip_verify: true + - job_name: 'updates' + target_groups: + - targets: ['haproxy'] + metrics_path: /monitor/updates + scheme: https + tls_config: + insecure_skip_verify: true + - job_name: 'users' + target_groups: + - targets: ['haproxy'] + metrics_path: /monitor/users + scheme: https + tls_config: + insecure_skip_verify: true \ No newline at end of file diff --git a/docker/prometheus/prometheus.yml.j2 b/docker/prometheus/prometheus.yml.j2 index bf420f0c0..872b4181f 100644 --- a/docker/prometheus/prometheus.yml.j2 +++ b/docker/prometheus/prometheus.yml.j2 @@ -5,7 +5,114 @@ global: scrape_configs: - job_name: 'prometheus' target_groups: - - targets: ['localhost:9090'] + - targets: ['localhost:9090'] + - job_name: 'node-exporter' target_groups: - - targets: ['node-exporter:9100'] \ No newline at end of file + - targets: ['node-exporter:9100'] + + - job_name: 'app_servers' + target_groups: + - targets: ['{{ ansible_ssh_host }}'] + metrics_path: /monitor/app-servers + scheme: https + tls_config: + insecure_skip_verify: true + - job_name: 'app_times' + target_groups: + - targets: ['{{ ansible_ssh_host }}'] + metrics_path: /monitor/app-times + scheme: https + tls_config: + insecure_skip_verify: true + - job_name: 'classifiers' + target_groups: + - targets: ['{{ ansible_ssh_host }}'] + metrics_path: /monitor/classifiers + scheme: https + tls_config: + insecure_skip_verify: true + - job_name: 'db_times' + target_groups: + - targets: ['{{ ansible_ssh_host }}'] + metrics_path: /monitor/db-times + scheme: https + tls_config: + insecure_skip_verify: true + - job_name: 'errors' + target_groups: + - targets: ['{{ ansible_ssh_host }}'] + metrics_path: /monitor/errors + scheme: https + tls_config: + insecure_skip_verify: true + - job_name: 'feed_counts' + target_groups: + - targets: ['{{ ansible_ssh_host }}'] + metrics_path: /monitor/feed-counts + scheme: https + tls_config: + insecure_skip_verify: true + - job_name: 'feeds' + target_groups: + - targets: ['{{ ansible_ssh_host }}'] + metrics_path: /monitor/feeds + scheme: https + tls_config: + insecure_skip_verify: true + - job_name: 'load_times' + target_groups: + - targets: ['{{ ansible_ssh_host }}'] + metrics_path: /monitor/load-times + scheme: https + tls_config: + insecure_skip_verify: true + - job_name: 'stories' + target_groups: + - targets: ['{{ ansible_ssh_host }}'] + metrics_path: /monitor/stories + scheme: https + tls_config: + insecure_skip_verify: true + - job_name: 'task_codes' + target_groups: + - targets: ['{{ ansible_ssh_host }}'] + metrics_path: /monitor/task-codes + scheme: https + tls_config: + insecure_skip_verify: true + - job_name: 'task_pipeline' + target_groups: + - targets: ['{{ ansible_ssh_host }}'] + metrics_path: /monitor/task-pipeline + scheme: https + tls_config: + insecure_skip_verify: true + - job_name: 'task_servers' + target_groups: + - targets: ['{{ ansible_ssh_host }}'] + metrics_path: /monitor/task-servers + scheme: https + tls_config: + insecure_skip_verify: true + - job_name: 'task_times' + target_groups: + - targets: ['{{ ansible_ssh_host }}'] + metrics_path: /monitor/task-times + scheme: https + tls_config: + insecure_skip_verify: true + - job_name: 'updates' + target_groups: + - targets: ['{{ ansible_ssh_host }}'] + metrics_path: /monitor/updates + scheme: https + tls_config: + insecure_skip_verify: true + - job_name: 'users' + target_groups: + - targets: ['{{ ansible_ssh_host }}'] + metrics_path: /monitor/users + scheme: https + tls_config: + insecure_skip_verify: true \ No newline at end of file diff --git a/templates/monitor/prometheus_data.html b/templates/monitor/prometheus_data.html index c89e2f385..676f41f04 100644 --- a/templates/monitor/prometheus_data.html +++ b/templates/monitor/prometheus_data.html @@ -1,3 +1,4 @@ {% for k, v in data.items %} -{{ k }} {{ v }}
+# TYPE {{chart_name}}_{{k}} {{chart_type}} +{{chart_name}}_{{k}} {{ v }} {% endfor %} \ No newline at end of file