diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index b5b1d74a0..aa300b1c1 100755 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -645,7 +645,7 @@ class Feed(models.Model): def setup_feed_for_premium_subscribers(self): self.count_subscribers() - self.set_next_scheduled_update() + self.set_next_scheduled_update(verbose=settings.DEBUG) def check_feed_link_for_feed_address(self): @timelimit(10) @@ -713,7 +713,7 @@ class Feed(models.Model): if status_code not in (200, 304): self.errors_since_good += 1 self.count_errors_in_history('feed', status_code, fetch_history=fetch_history) - self.set_next_scheduled_update() + self.set_next_scheduled_update(verbose=settings.DEBUG) elif self.has_feed_exception or self.errors_since_good: self.errors_since_good = 0 self.has_feed_exception = False @@ -1249,7 +1249,7 @@ class Feed(models.Model): feed = Feed.get_by_id(feed.pk) if feed: feed.last_update = datetime.datetime.utcnow() - feed.set_next_scheduled_update() + feed.set_next_scheduled_update(verbose=settings.DEBUG) r.zadd('fetched_feeds_last_hour', { feed.pk: int(datetime.datetime.now().strftime('%s')) }) if not feed or original_feed_id != feed.pk: @@ -2248,7 +2248,10 @@ class Feed(models.Model): # Pro subscribers get absolute minimum if self.pro_subscribers >= 1: - total = min(total, 5) + if self.stories_last_month == 0: + total = min(total, 60) + else: + total = min(total, settings.PRO_MINUTES_BETWEEN_FETCHES) if verbose: logging.debug(" ---> [%-30s] Fetched every %s min - Subs: %s/%s/%s/%s/%s Stories/day: %s" % ( diff --git a/apps/rss_feeds/views.py b/apps/rss_feeds/views.py index 1c57ffd3d..df13dcdc8 100644 --- a/apps/rss_feeds/views.py +++ b/apps/rss_feeds/views.py @@ -1,5 +1,6 @@ import datetime import base64 +import redis from urllib.parse import urlparse from utils import log as logging from django.shortcuts import get_object_or_404, render @@ -501,16 +502,35 @@ def exception_change_feed_link(request): @login_required def status(request): - if not request.user.is_staff: + if not request.user.is_staff and not settings.DEBUG: logging.user(request, "~SKNON-STAFF VIEWING RSS FEEDS STATUS!") assert False return HttpResponseForbidden() minutes = int(request.GET.get('minutes', 1)) now = datetime.datetime.now() - hour_ago = now - datetime.timedelta(minutes=minutes) - feeds = Feed.objects.filter(last_update__gte=hour_ago).order_by('-last_update') + username = request.GET.get('user', '') or request.GET.get('username', '') + if username: + user = User.objects.get(username=username) + else: + user = request.user + usersubs = UserSubscription.objects.filter(user=user) + feed_ids = usersubs.values('feed_id') + if minutes > 0: + hour_ago = now + datetime.timedelta(minutes=minutes) + feeds = Feed.objects.filter(pk__in=feed_ids, next_scheduled_update__lte=hour_ago).order_by('next_scheduled_update') + else: + hour_ago = now + datetime.timedelta(minutes=minutes) + feeds = Feed.objects.filter(pk__in=feed_ids, last_update__gte=hour_ago).order_by('-last_update') + + r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL) + queues = { + 'tasked_feeds': r.zcard('tasked_feeds'), + 'queued_feeds': r.scard('queued_feeds'), + 'scheduled_updates': r.zcard('scheduled_updates'), + } return render(request, 'rss_feeds/status.xhtml', { - 'feeds': feeds + 'feeds': feeds, + 'queues': queues }) @json.json_view diff --git a/newsblur_web/docker_local_settings.py b/newsblur_web/docker_local_settings.py index 39f1f8b3e..01b496282 100644 --- a/newsblur_web/docker_local_settings.py +++ b/newsblur_web/docker_local_settings.py @@ -33,6 +33,7 @@ DEBUG_ASSETS = True # down verbosity. DEBUG_QUERIES = DEBUG DEBUG_QUERIES_SUMMARY_ONLY = True +# DEBUG_QUERIES_SUMMARY_ONLY = False MEDIA_URL = '/media/' IMAGES_URL = '/imageproxy' @@ -41,6 +42,8 @@ AUTO_PREMIUM_NEW_USERS = True AUTO_ENABLE_NEW_USERS = True ENFORCE_SIGNUP_CAPTCHA = False +PRO_MINUTES_BETWEEN_FETCHES = 15 + CACHES = { 'default': { 'BACKEND': 'redis_cache.RedisCache', diff --git a/newsblur_web/settings.py b/newsblur_web/settings.py index 60807a961..687d5040b 100644 --- a/newsblur_web/settings.py +++ b/newsblur_web/settings.py @@ -259,6 +259,11 @@ DAYS_OF_STORY_HASHES = 30 # is no longer considered an active subscriber SUBSCRIBER_EXPIRE = 7 +# PRO_MINUTES_BETWEEN_FETCHES sets the number of minutes to fetch feeds for +# Premium Pro accounts. Defaults to every 5 minutes, but that's for NewsBlur +# servers. On your local, you should probably set this to 10-15 minutes +PRO_MINUTES_BETWEEN_FETCHES = 5 + ROOT_URLCONF = 'newsblur_web.urls' INTERNAL_IPS = ('127.0.0.1',) LOGGING_LOG_SQL = True diff --git a/templates/rss_feeds/status.xhtml b/templates/rss_feeds/status.xhtml index 95221ed09..f88a361d1 100644 --- a/templates/rss_feeds/status.xhtml +++ b/templates/rss_feeds/status.xhtml @@ -1,6 +1,6 @@ {% extends 'base.html' %} -{% load utils_tags %} +{% load utils_tags tz %} {% block bodyclass %}NB-body-status{% endblock %} @@ -8,16 +8,24 @@
ID | Title | -Last Update Next Update |
+ Min since last update |
+ Last Update Next Update |
+ Min to next update |
Decay | Subs | Active | Premium | +Archive | +Pro | Act. Prem | Per Month | Last Month | @@ -26,15 +34,19 @@
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
{{ feed.pk }} | - {{ feed.last_update|date:"M j H:i:s" }} + | {{ feed.last_update|smooth_timedelta }} | +
+ {% localdatetime feed.last_update "%b %d, %Y %H:%M:%S" %}
- {{ feed.next_scheduled_update|date:"M j H:i:s" }} + {% localdatetime feed.next_scheduled_update "%b %d, %Y %H:%M:%S" %} |
+ {{ feed.next_scheduled_update|smooth_timedelta }} | {{ feed.min_to_decay }} | {{ feed.num_subscribers }} | {{ feed.active_subscribers }} | {{ feed.premium_subscribers }} | +{{ feed.archive_subscribers }} | +{{ feed.pro_subscribers }} | {{ feed.active_premium_subscribers }} | {{ feed.average_stories_per_month }} | {{ feed.stories_last_month }} | @@ -45,4 +57,4 @@ -{% endblock content %} \ No newline at end of file +{% endblock content %} diff --git a/utils/templatetags/utils_tags.py b/utils/templatetags/utils_tags.py index 7bd0a55d2..daaeca063 100644 --- a/utils/templatetags/utils_tags.py +++ b/utils/templatetags/utils_tags.py @@ -1,4 +1,5 @@ import struct +import datetime from django.contrib.sites.models import Site from django.conf import settings from django import template @@ -212,6 +213,40 @@ def commify(n): out += '.' + cents return out +@register.filter +def smooth_timedelta(timedeltaobj): + """Convert a datetime.timedelta object into Days, Hours, Minutes, Seconds.""" + if isinstance(timedeltaobj, datetime.datetime): + timedeltaobj = timedeltaobj - datetime.datetime.now() + secs = timedeltaobj.total_seconds() + overdue = secs < 0 + secs = abs(secs) + timetot = "" + if not overdue: + timetot += "in " + + if secs > 86400: # 60sec * 60min * 24hrs + days = secs // 86400 + timetot += "{} days".format(int(days)) + secs = secs - days*86400 + + if secs > 3600: + hrs = secs // 3600 + timetot += " {} hours".format(int(hrs)) + secs = secs - hrs*3600 + + if secs > 60: + mins = secs // 60 + timetot += " {} min".format(int(mins)) + secs = secs - mins*60 + + if secs > 0: + timetot += " {} sec".format(int(secs)) + + if overdue: + timetot += " ago" + + return timetot @register.tag def include_javascripts(parser, token):