Fixing feed fetch status screen to better show how subs are being fetched.

apps/rss_feeds/models.py

@@ -645,7 +645,7 @@ class Feed(models.Model):
 
     def setup_feed_for_premium_subscribers(self):
         self.count_subscribers()
-        self.set_next_scheduled_update()
+        self.set_next_scheduled_update(verbose=settings.DEBUG)
         
     def check_feed_link_for_feed_address(self):
         @timelimit(10)
@@ -713,7 +713,7 @@ class Feed(models.Model):
         if status_code not in (200, 304):
             self.errors_since_good += 1
             self.count_errors_in_history('feed', status_code, fetch_history=fetch_history)
-            self.set_next_scheduled_update()
+            self.set_next_scheduled_update(verbose=settings.DEBUG)
         elif self.has_feed_exception or self.errors_since_good:
             self.errors_since_good = 0
             self.has_feed_exception = False
@@ -1249,7 +1249,7 @@ class Feed(models.Model):
             feed = Feed.get_by_id(feed.pk)
         if feed:
             feed.last_update = datetime.datetime.utcnow()
-            feed.set_next_scheduled_update()
+            feed.set_next_scheduled_update(verbose=settings.DEBUG)
             r.zadd('fetched_feeds_last_hour', { feed.pk: int(datetime.datetime.now().strftime('%s')) })
         
         if not feed or original_feed_id != feed.pk:
@@ -2248,7 +2248,10 @@ class Feed(models.Model):
         
         # Pro subscribers get absolute minimum
         if self.pro_subscribers >= 1:
-            total = min(total, 5)
+            if self.stories_last_month == 0:
+                total = min(total, 60)
+            else:
+                total = min(total, settings.PRO_MINUTES_BETWEEN_FETCHES)
 
         if verbose:
             logging.debug("   ---> [%-30s] Fetched every %s min - Subs: %s/%s/%s/%s/%s Stories/day: %s" % (

apps/rss_feeds/views.py

@@ -1,5 +1,6 @@
 import datetime
 import base64
+import redis
 from urllib.parse import urlparse
 from utils import log as logging
 from django.shortcuts import get_object_or_404, render
@@ -501,16 +502,35 @@ def exception_change_feed_link(request):
 
 @login_required
 def status(request):
-    if not request.user.is_staff:
+    if not request.user.is_staff and not settings.DEBUG:
         logging.user(request, "~SKNON-STAFF VIEWING RSS FEEDS STATUS!")
         assert False
         return HttpResponseForbidden()
     minutes  = int(request.GET.get('minutes', 1))
     now      = datetime.datetime.now()
-    hour_ago = now - datetime.timedelta(minutes=minutes)
-    feeds    = Feed.objects.filter(last_update__gte=hour_ago).order_by('-last_update')
+    username = request.GET.get('user', '') or request.GET.get('username', '')
+    if username:
+        user = User.objects.get(username=username)
+    else:
+        user = request.user
+    usersubs = UserSubscription.objects.filter(user=user)
+    feed_ids = usersubs.values('feed_id')
+    if minutes > 0:
+        hour_ago = now + datetime.timedelta(minutes=minutes)
+        feeds    = Feed.objects.filter(pk__in=feed_ids, next_scheduled_update__lte=hour_ago).order_by('next_scheduled_update')
+    else:
+        hour_ago = now + datetime.timedelta(minutes=minutes)
+        feeds    = Feed.objects.filter(pk__in=feed_ids, last_update__gte=hour_ago).order_by('-last_update')
+    
+    r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL)
+    queues = {
+        'tasked_feeds': r.zcard('tasked_feeds'),
+        'queued_feeds': r.scard('queued_feeds'),
+        'scheduled_updates': r.zcard('scheduled_updates'),
+    }
     return render(request, 'rss_feeds/status.xhtml', {
-        'feeds': feeds
+        'feeds': feeds,
+        'queues': queues
     })
 
 @json.json_view

newsblur_web/docker_local_settings.py

@@ -33,6 +33,7 @@ DEBUG_ASSETS = True
 # down verbosity.
 DEBUG_QUERIES = DEBUG
 DEBUG_QUERIES_SUMMARY_ONLY = True
+# DEBUG_QUERIES_SUMMARY_ONLY = False
 
 MEDIA_URL = '/media/'
 IMAGES_URL = '/imageproxy'
@@ -41,6 +42,8 @@ AUTO_PREMIUM_NEW_USERS = True
 AUTO_ENABLE_NEW_USERS = True
 ENFORCE_SIGNUP_CAPTCHA = False
 
+PRO_MINUTES_BETWEEN_FETCHES = 15
+
 CACHES = {
     'default': {
         'BACKEND': 'redis_cache.RedisCache',

newsblur_web/settings.py

@@ -259,6 +259,11 @@ DAYS_OF_STORY_HASHES    = 30
 # is no longer considered an active subscriber
 SUBSCRIBER_EXPIRE       = 7
 
+# PRO_MINUTES_BETWEEN_FETCHES sets the number of minutes to fetch feeds for 
+# Premium Pro accounts. Defaults to every 5 minutes, but that's for NewsBlur
+# servers. On your local, you should probably set this to 10-15 minutes
+PRO_MINUTES_BETWEEN_FETCHES = 5
+
 ROOT_URLCONF            = 'newsblur_web.urls'
 INTERNAL_IPS            = ('127.0.0.1',)
 LOGGING_LOG_SQL         = True

templates/rss_feeds/status.xhtml

@@ -1,6 +1,6 @@
 {% extends 'base.html' %}
 
-{% load utils_tags %}
+{% load utils_tags tz %}
 
 {% block bodyclass %}NB-body-status{% endblock %}
 
@@ -8,16 +8,24 @@
 
 <div class="NB-module">
 
+  <div class="queues">
+    Tasked: {{ queues.tasked_feeds }}, Queued: {{ queues.queued_feeds }}, Scheduled: {{ queues.scheduled_updates }}
+  </div>
+  
 <table class="NB-status">
 
   <tr>
     <th>ID</th>
     <th>Title</th>
-    <th>Last Update<br>Next Update</th>
+    <th>Min since<br>last update</th>
+    <th style="white-space: nowrap">Last Update<br>Next Update</th>
+    <th>Min to<br>next update</th>
     <th>Decay</th>
     <th>Subs</th>
     <th>Active</th>
     <th>Premium</th>
+    <th>Archive</th>
+    <th>Pro</th>
     <th>Act. Prem</th>
     <th>Per Month</th>
     <th>Last Month</th>
@@ -26,15 +34,19 @@
     <tr>
       <td>{{ feed.pk }}</td>
       <td><img class="NB-favicon" src="/rss_feeds/icon/{{ feed.pk }}" /> {{ feed.feed_title|truncatewords:4 }}</td>
-      <td class="NB-status-update">
-        {{ feed.last_update|date:"M j H:i:s" }}
+      <td>{{ feed.last_update|smooth_timedelta }}</td>
+      <td class="NB-status-update" style="white-space: nowrap">
+        {% localdatetime feed.last_update "%b %d, %Y %H:%M:%S" %}
         <br>
-        {{ feed.next_scheduled_update|date:"M j H:i:s" }}
+        {% localdatetime feed.next_scheduled_update "%b %d, %Y %H:%M:%S" %}
       </td>
+      <td>{{ feed.next_scheduled_update|smooth_timedelta }}</td>
       <td>{{ feed.min_to_decay }}</td>
       <td>{{ feed.num_subscribers }}</td>
       <td style="color: {% if feed.active_subscribers == 0 %}lightgrey{% else %}darkblue{% endif %}">{{ feed.active_subscribers }}</td>
       <td style="color: {% if feed.premium_subscribers == 0 %}lightgrey{% else %}darkblue{% endif %}">{{ feed.premium_subscribers }}</td>
+      <td style="color: {% if feed.premium_subscribers == 0 %}lightgrey{% else %}darkblue{% endif %}">{{ feed.archive_subscribers }}</td>
+      <td style="color: {% if feed.premium_subscribers == 0 %}lightgrey{% else %}darkblue{% endif %}">{{ feed.pro_subscribers }}</td>
       <td style="color: {% if feed.active_premium_subscribers == 0 %}lightgrey{% else %}darkblue{% endif %}">{{ feed.active_premium_subscribers }}</td>
       <td style="color: {% if feed.average_stories_per_month == 0 %}lightgrey{% else %}{% endif %}">{{ feed.average_stories_per_month }}</td>
       <td style="color: {% if feed.stories_last_month == 0 %}lightgrey{% else %}{% endif %}">{{ feed.stories_last_month }}</td>
@@ -45,4 +57,4 @@
 
 </div>
 
-{% endblock content %}
+{% endblock content %}

utils/templatetags/utils_tags.py

@@ -1,4 +1,5 @@
 import struct
+import datetime
 from django.contrib.sites.models import Site
 from django.conf import settings
 from django import template
@@ -212,6 +213,40 @@ def commify(n):
         out += '.' + cents
     return out
 
+@register.filter
+def smooth_timedelta(timedeltaobj):
+    """Convert a datetime.timedelta object into Days, Hours, Minutes, Seconds."""
+    if isinstance(timedeltaobj, datetime.datetime):
+        timedeltaobj = timedeltaobj - datetime.datetime.now()
+    secs = timedeltaobj.total_seconds()
+    overdue = secs < 0
+    secs = abs(secs)
+    timetot = ""
+    if not overdue:
+        timetot += "in "
+        
+    if secs > 86400: # 60sec * 60min * 24hrs
+        days = secs // 86400
+        timetot += "{} days".format(int(days))
+        secs = secs - days*86400
+
+    if secs > 3600:
+        hrs = secs // 3600
+        timetot += " {} hours".format(int(hrs))
+        secs = secs - hrs*3600
+
+    if secs > 60:
+        mins = secs // 60
+        timetot += " {} min".format(int(mins))
+        secs = secs - mins*60
+
+    if secs > 0:
+        timetot += " {} sec".format(int(secs))
+
+    if overdue:
+        timetot += " ago"
+    
+    return timetot
 
 @register.tag
 def include_javascripts(parser, token):