From c08ac7d999c235eaa93076e63b8e60a5f30d0816 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Wed, 11 May 2022 15:23:47 -0400 Subject: [PATCH] Adding feed_sizes to prometheus. Also adding archive and pro subscribers. --- apps/monitor/urls.py | 3 +- apps/monitor/views/__init__.py | 1 + apps/monitor/views/newsblur_feed_sizes.py | 42 ++++++++++++++++++++++ apps/monitor/views/newsblur_users.py | 4 +++ docker/prometheus/prometheus.consul.yml.j2 | 5 +++ docker/prometheus/prometheus.docker.yml | 7 ++++ 6 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 apps/monitor/views/newsblur_feed_sizes.py diff --git a/apps/monitor/urls.py b/apps/monitor/urls.py index 4eaf440e0..2ae4c0fdc 100644 --- a/apps/monitor/urls.py +++ b/apps/monitor/urls.py @@ -2,7 +2,7 @@ from django.conf.urls import url from apps.monitor.views import ( AppServers, AppTimes, Classifiers, DbTimes, Errors, FeedCounts, Feeds, LoadTimes, Stories, TasksCodes, TasksPipeline, TasksServers, TasksTimes, - Updates, Users + Updates, Users, FeedSizes ) urlpatterns = [ url(r'^app-servers?$', AppServers.as_view(), name="app_servers"), @@ -11,6 +11,7 @@ urlpatterns = [ url(r'^db-times?$', DbTimes.as_view(), name="db_times"), url(r'^errors?$', Errors.as_view(), name="errors"), url(r'^feed-counts?$', FeedCounts.as_view(), name="feed_counts"), + url(r'^feed-sizes?$', FeedSizes.as_view(), name="feed_sizes"), url(r'^feeds?$', Feeds.as_view(), name="feeds"), url(r'^load-times?$', LoadTimes.as_view(), name="load_times"), url(r'^stories?$', Stories.as_view(), name="stories"), diff --git a/apps/monitor/views/__init__.py b/apps/monitor/views/__init__.py index 419637896..e7e079a69 100644 --- a/apps/monitor/views/__init__.py +++ b/apps/monitor/views/__init__.py @@ -4,6 +4,7 @@ from apps.monitor.views.newsblur_classifiers import Classifiers from apps.monitor.views.newsblur_dbtimes import DbTimes from apps.monitor.views.newsblur_errors import Errors from apps.monitor.views.newsblur_feed_counts import FeedCounts +from apps.monitor.views.newsblur_feed_sizes import FeedSizes from apps.monitor.views.newsblur_feeds import Feeds from apps.monitor.views.newsblur_loadtimes import LoadTimes from apps.monitor.views.newsblur_stories import Stories diff --git a/apps/monitor/views/newsblur_feed_sizes.py b/apps/monitor/views/newsblur_feed_sizes.py new file mode 100644 index 000000000..8fb5a2555 --- /dev/null +++ b/apps/monitor/views/newsblur_feed_sizes.py @@ -0,0 +1,42 @@ +from django.conf import settings +from django.shortcuts import render +from django.views import View +from django.db.models import Sum +import redis +from apps.rss_feeds.models import Feed, DuplicateFeed +from apps.push.models import PushSubscription +from apps.statistics.models import MStatistics + +class FeedSizes(View): + + def get(self, request): + + fs_size_bytes = MStatistics.get('munin:fs_size_bytes') + if not fs_size_bytes: + fs_size_bytes = Feed.objects.aggregate(Sum('fs_size_bytes')) + MStatistics.set('munin:fs_size_bytes', fs_size_bytes, 60*60*12) + + archive_users_size_bytes = MStatistics.get('munin:archive_users_size_bytes') + if not archive_users_size_bytes: + archive_users_size_bytes = Feed.objects.filter(archive_subscribers__gte=1).aggregate(Sum('fs_size_bytes')) + MStatistics.set('munin:archive_users_size_bytes', archive_users_size_bytes, 60*60*12) + + data = { + 'fs_size_bytes': fs_size_bytes, + 'archive_users_size_bytes': archive_users_size_bytes, + } + chart_name = "feed_sizes" + chart_type = "counter" + + formatted_data = {} + for k, v in data.items(): + formatted_data[k] = f'{chart_name}{{category="{k}"}} {v}' + + context = { + "data": formatted_data, + "chart_name": chart_name, + "chart_type": chart_type, + } + return render(request, 'monitor/prometheus_data.html', context, content_type="text/plain") + + diff --git a/apps/monitor/views/newsblur_users.py b/apps/monitor/views/newsblur_users.py index d1a7d861a..62421f5b3 100755 --- a/apps/monitor/views/newsblur_users.py +++ b/apps/monitor/views/newsblur_users.py @@ -9,14 +9,18 @@ from apps.profile.models import Profile, RNewUserQueue class Users(View): def get(self, request): + last_year = datetime.datetime.utcnow() - datetime.timedelta(days=365) last_month = datetime.datetime.utcnow() - datetime.timedelta(days=30) last_day = datetime.datetime.utcnow() - datetime.timedelta(minutes=60*24) data = { 'all': User.objects.count(), + 'yearly': Profile.objects.filter(last_seen_on__gte=last_year).count(), 'monthly': Profile.objects.filter(last_seen_on__gte=last_month).count(), 'daily': Profile.objects.filter(last_seen_on__gte=last_day).count(), 'premium': Profile.objects.filter(is_premium=True).count(), + 'archive': Profile.objects.filter(is_archive=True).count(), + 'pro': Profile.objects.filter(is_pro=True).count(), 'queued': RNewUserQueue.user_count(), } chart_name = "users" diff --git a/docker/prometheus/prometheus.consul.yml.j2 b/docker/prometheus/prometheus.consul.yml.j2 index 751bd87a7..537c45353 100644 --- a/docker/prometheus/prometheus.consul.yml.j2 +++ b/docker/prometheus/prometheus.consul.yml.j2 @@ -65,6 +65,11 @@ scrape_configs: - targets: ['{{ monitor_server }}'] metrics_path: /monitor/errors scheme: https + - job_name: 'feed_sizes' + static_configs: + - targets: ['{{ monitor_server }}'] + metrics_path: /monitor/feed-sizes + scheme: https - job_name: 'feed_counts' static_configs: - targets: ['{{ monitor_server }}'] diff --git a/docker/prometheus/prometheus.docker.yml b/docker/prometheus/prometheus.docker.yml index 31e93572a..9a8199bf4 100644 --- a/docker/prometheus/prometheus.docker.yml +++ b/docker/prometheus/prometheus.docker.yml @@ -52,6 +52,13 @@ scrape_configs: scheme: https tls_config: insecure_skip_verify: true + - job_name: 'feed_sizes' + static_configs: + - targets: ['haproxy'] + metrics_path: /monitor/feed-szies + scheme: https + tls_config: + insecure_skip_verify: true - job_name: 'feed_counts' static_configs: - targets: ['haproxy']