Queue date for feeds for better metrics.

apps/rss_feeds/management/commands/task_feeds.py

@@ -28,6 +28,7 @@ class Command(BaseCommand):
         i = 0
         feed_queue = []
         for f in feeds:
+            f.queued_date = datetime.datetime.now()
             f.set_next_scheduled_update()
             i += 1
             feed_queue.append(f.pk)

apps/rss_feeds/migrations/0023_queued_date.py

@@ -0,0 +1,125 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Adding field 'Feed.queued_date'
+        db.add_column('feeds', 'queued_date', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime.now, db_index=True), keep_default=False)
+
+
+    def backwards(self, orm):
+        
+        # Deleting field 'Feed.queued_date'
+        db.delete_column('feeds', 'queued_date')
+
+
+    models = {
+        'rss_feeds.duplicatefeed': {
+            'Meta': {'object_name': 'DuplicateFeed'},
+            'duplicate_address': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+            'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'duplicate_addresses'", 'to': "orm['rss_feeds.Feed']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        },
+        'rss_feeds.feed': {
+            'Meta': {'ordering': "['feed_title']", 'object_name': 'Feed', 'db_table': "'feeds'"},
+            'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'average_stories_per_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'creation': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'days_to_trim': ('django.db.models.fields.IntegerField', [], {'default': '90'}),
+            'etag': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'exception_code': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'feed_address': ('django.db.models.fields.URLField', [], {'unique': 'True', 'max_length': '255'}),
+            'feed_link': ('django.db.models.fields.URLField', [], {'default': "''", 'max_length': '1000', 'null': 'True', 'blank': 'True'}),
+            'feed_tagline': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1024', 'null': 'True', 'blank': 'True'}),
+            'feed_title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'fetched_once': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'has_feed_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'has_page_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_load_time': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'last_modified': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'last_update': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
+            'min_to_decay': ('django.db.models.fields.IntegerField', [], {'default': '15'}),
+            'next_scheduled_update': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
+            'num_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'popular_authors': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}),
+            'popular_tags': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
+            'queued_date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
+            'stories_last_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'story_count_history': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'})
+        },
+        'rss_feeds.feedfetchhistory': {
+            'Meta': {'object_name': 'FeedFetchHistory'},
+            'exception': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'feed_fetch_history'", 'to': "orm['rss_feeds.Feed']"}),
+            'fetch_date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'message': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'status_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'})
+        },
+        'rss_feeds.feedpage': {
+            'Meta': {'object_name': 'FeedPage'},
+            'feed': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'feed_page'", 'unique': 'True', 'to': "orm['rss_feeds.Feed']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'page_data': ('utils.compressed_textfield.StoryField', [], {'null': 'True', 'blank': 'True'})
+        },
+        'rss_feeds.feedupdatehistory': {
+            'Meta': {'object_name': 'FeedUpdateHistory'},
+            'average_per_feed': ('django.db.models.fields.DecimalField', [], {'max_digits': '4', 'decimal_places': '1'}),
+            'fetch_date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'number_of_feeds': ('django.db.models.fields.IntegerField', [], {}),
+            'seconds_taken': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'rss_feeds.feedxml': {
+            'Meta': {'object_name': 'FeedXML'},
+            'feed': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'feed_xml'", 'unique': 'True', 'to': "orm['rss_feeds.Feed']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'rss_xml': ('utils.compressed_textfield.StoryField', [], {'null': 'True', 'blank': 'True'})
+        },
+        'rss_feeds.pagefetchhistory': {
+            'Meta': {'object_name': 'PageFetchHistory'},
+            'exception': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'page_fetch_history'", 'to': "orm['rss_feeds.Feed']"}),
+            'fetch_date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'message': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'status_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'})
+        },
+        'rss_feeds.story': {
+            'Meta': {'ordering': "['-story_date']", 'unique_together': "(('story_feed', 'story_guid_hash'),)", 'object_name': 'Story', 'db_table': "'stories'"},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'story_author': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.StoryAuthor']"}),
+            'story_author_name': ('django.db.models.fields.CharField', [], {'max_length': '500', 'null': 'True', 'blank': 'True'}),
+            'story_content': ('utils.compressed_textfield.StoryField', [], {'null': 'True', 'blank': 'True'}),
+            'story_content_type': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'story_date': ('django.db.models.fields.DateTimeField', [], {}),
+            'story_feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'stories'", 'to': "orm['rss_feeds.Feed']"}),
+            'story_guid': ('django.db.models.fields.CharField', [], {'max_length': '1000'}),
+            'story_guid_hash': ('django.db.models.fields.CharField', [], {'max_length': '40'}),
+            'story_original_content': ('utils.compressed_textfield.StoryField', [], {'null': 'True', 'blank': 'True'}),
+            'story_past_trim_date': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'story_permalink': ('django.db.models.fields.CharField', [], {'max_length': '1000'}),
+            'story_tags': ('django.db.models.fields.CharField', [], {'max_length': '2000', 'null': 'True', 'blank': 'True'}),
+            'story_title': ('django.db.models.fields.CharField', [], {'max_length': '255'})
+        },
+        'rss_feeds.storyauthor': {
+            'Meta': {'object_name': 'StoryAuthor'},
+            'author_name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        },
+        'rss_feeds.tag': {
+            'Meta': {'object_name': 'Tag'},
+            'feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '255'})
+        }
+    }
+
+    complete_apps = ['rss_feeds']

apps/rss_feeds/models.py

@@ -47,6 +47,7 @@ class Feed(models.Model):
     average_stories_per_month = models.IntegerField(default=0)
     story_count_history = models.TextField(blank=True, null=True)
     next_scheduled_update = models.DateTimeField(default=datetime.datetime.now, db_index=True)
+    queued_date = models.DateTimeField(default=datetime.datetime.now, db_index=True)
     last_load_time = models.IntegerField(default=0)
     popular_tags = models.CharField(max_length=1024, blank=True, null=True)
     popular_authors = models.CharField(max_length=2048, blank=True, null=True)

utils/munin/newsblur_feeds.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python 
-import datetime
 from utils.munin.base import MuninGraph
 from apps.rss_feeds.models import Feed
 from apps.reader.models import UserSubscription
@@ -12,12 +11,14 @@ graph_config = {
     'feeds.label': 'feeds',
     'subscriptions.label': 'subscriptions',
     'exception_feeds.label': 'exception_feeds',
+    'inactive_feeds.label': 'inactive_feeds',
 }
 
 metrics = {
     'feeds': Feed.objects.count(),
     'subscriptions': UserSubscription.objects.count(),
     'exception_feeds': Feed.objects.filter(Q(has_feed_exception=True) | Q(has_page_exception=True)).count(),
+    'inactive_feeds': Feed.objects.filter(active=False).count(),
 }
 
 if __name__ == '__main__':

utils/munin/newsblur_updates.py

@@ -14,7 +14,7 @@ graph_config = {
 hour_ago = datetime.datetime.now() - datetime.timedelta(hours=1)
 
 metrics = {
-    'update_queue': Feed.objects.filter(next_scheduled_update__lte=datetime.datetime.now(), active=True).count(),
+    'update_queue': Feed.objects.filter(queued_date__gte=hour_ago).count(),
     'feeds_fetched': Feed.objects.filter(last_update__gte=hour_ago).count()
 }