Adding feed_sizes to prometheus. Also adding archive and pro subscribers.

apps/monitor/urls.py

@@ -2,7 +2,7 @@ from django.conf.urls import url
 from apps.monitor.views import ( AppServers, AppTimes,
 Classifiers, DbTimes, Errors, FeedCounts, Feeds, LoadTimes,
  Stories, TasksCodes, TasksPipeline, TasksServers, TasksTimes,
- Updates, Users
+ Updates, Users, FeedSizes
 )
 urlpatterns = [
     url(r'^app-servers?$', AppServers.as_view(), name="app_servers"),
@@ -11,6 +11,7 @@ urlpatterns = [
     url(r'^db-times?$', DbTimes.as_view(), name="db_times"),
     url(r'^errors?$', Errors.as_view(), name="errors"),
     url(r'^feed-counts?$', FeedCounts.as_view(), name="feed_counts"),
+    url(r'^feed-sizes?$', FeedSizes.as_view(), name="feed_sizes"),
     url(r'^feeds?$', Feeds.as_view(), name="feeds"),
     url(r'^load-times?$', LoadTimes.as_view(), name="load_times"),
     url(r'^stories?$', Stories.as_view(), name="stories"),

apps/monitor/views/__init__.py

@@ -4,6 +4,7 @@ from apps.monitor.views.newsblur_classifiers import Classifiers
 from apps.monitor.views.newsblur_dbtimes import DbTimes
 from apps.monitor.views.newsblur_errors import Errors
 from apps.monitor.views.newsblur_feed_counts import FeedCounts
+from apps.monitor.views.newsblur_feed_sizes import FeedSizes
 from apps.monitor.views.newsblur_feeds import Feeds
 from apps.monitor.views.newsblur_loadtimes import LoadTimes
 from apps.monitor.views.newsblur_stories import Stories

apps/monitor/views/newsblur_feed_sizes.py

@@ -0,0 +1,42 @@
+from django.conf import settings
+from django.shortcuts import render
+from django.views import View
+from django.db.models import Sum
+import redis
+from apps.rss_feeds.models import Feed, DuplicateFeed
+from apps.push.models import PushSubscription
+from apps.statistics.models import MStatistics
+
+class FeedSizes(View):
+
+    def get(self, request):
+        
+        fs_size_bytes = MStatistics.get('munin:fs_size_bytes')
+        if not fs_size_bytes:
+            fs_size_bytes = Feed.objects.aggregate(Sum('fs_size_bytes'))
+            MStatistics.set('munin:fs_size_bytes', fs_size_bytes, 60*60*12)
+
+        archive_users_size_bytes = MStatistics.get('munin:archive_users_size_bytes')
+        if not archive_users_size_bytes:
+            archive_users_size_bytes = Feed.objects.filter(archive_subscribers__gte=1).aggregate(Sum('fs_size_bytes'))
+            MStatistics.set('munin:archive_users_size_bytes', archive_users_size_bytes, 60*60*12)
+
+        data = {
+            'fs_size_bytes': fs_size_bytes,
+            'archive_users_size_bytes': archive_users_size_bytes,
+        }
+        chart_name = "feed_sizes"
+        chart_type = "counter"
+
+        formatted_data = {}
+        for k, v in data.items():
+            formatted_data[k] = f'{chart_name}{{category="{k}"}} {v}'
+
+        context = {
+            "data": formatted_data,
+            "chart_name": chart_name,
+            "chart_type": chart_type,
+        }
+        return render(request, 'monitor/prometheus_data.html', context, content_type="text/plain")
+
+

apps/monitor/views/newsblur_users.py

@@ -9,14 +9,18 @@ from apps.profile.models import Profile, RNewUserQueue
 class Users(View):
 
     def get(self, request):
+        last_year = datetime.datetime.utcnow() - datetime.timedelta(days=365)
         last_month = datetime.datetime.utcnow() - datetime.timedelta(days=30)
         last_day = datetime.datetime.utcnow() - datetime.timedelta(minutes=60*24)
 
         data = {
             'all': User.objects.count(),
+            'yearly': Profile.objects.filter(last_seen_on__gte=last_year).count(),
             'monthly': Profile.objects.filter(last_seen_on__gte=last_month).count(),
             'daily': Profile.objects.filter(last_seen_on__gte=last_day).count(),
             'premium': Profile.objects.filter(is_premium=True).count(),
+            'archive': Profile.objects.filter(is_archive=True).count(),
+            'pro': Profile.objects.filter(is_pro=True).count(),
             'queued': RNewUserQueue.user_count(),
         }
         chart_name = "users"

docker/prometheus/prometheus.consul.yml.j2

@@ -65,6 +65,11 @@ scrape_configs:
         - targets: ['{{ monitor_server }}']
       metrics_path: /monitor/errors
       scheme: https
+    - job_name: 'feed_sizes'
+      static_configs:
+        - targets: ['{{ monitor_server }}']
+      metrics_path: /monitor/feed-sizes
+      scheme: https
     - job_name: 'feed_counts'
       static_configs:
         - targets: ['{{ monitor_server }}']

docker/prometheus/prometheus.docker.yml

@@ -52,6 +52,13 @@ scrape_configs:
       scheme: https
       tls_config:
         insecure_skip_verify: true
+    - job_name: 'feed_sizes'
+      static_configs:
+        - targets: ['haproxy']
+      metrics_path: /monitor/feed-szies
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
     - job_name: 'feed_counts'
       static_configs:
         - targets: ['haproxy']