Merge branch 'master' into social

* master:
  Showing feed pushes in statistics.
  Instrumentation to count feed pushes.
  Migration to fix starred stories with incorrectly double-encoded url strings.
  Fixing bug around missing feed links being checked for length.

Conflicts:
	apps/rss_feeds/views.py

apps/push/views.py

@@ -8,6 +8,7 @@ from django.shortcuts import get_object_or_404
 
 from apps.push.models import PushSubscription
 from apps.push.signals import verified
+from apps.rss_feeds.models import MFeedPushHistory
 
 def push_callback(request, push_id):
     if request.method == 'GET':
@@ -42,6 +43,7 @@ def push_callback(request, push_id):
         # Don't give fat ping, just fetch.
         # subscription.feed.queue_pushed_feed_xml(request.raw_post_data)
         subscription.feed.queue_pushed_feed_xml("Fetch me")
-
+        MFeedPushHistory.objects.create(feed_id=subscription.feed_id)
+        
         return HttpResponse('')
     return Http404

apps/rss_feeds/migrations/0055_starred_story_permalinks.py

@@ -0,0 +1,98 @@
+# encoding: utf-8
+from south.v2 import DataMigration
+
+from urllib import unquote
+from apps.rss_feeds.models import MStarredStory
+
+
+class Migration(DataMigration):
+
+    def forwards(self, orm):
+        ss_count = MStarredStory.objects.count()
+        print "%s starred stories" % ss_count
+        group_size = ss_count/1000
+        for group in range(1000):
+            offset = group_size*group
+            print "Group offset: %s/%s-%s" % (group, offset, group_size*(group+1))
+            stories = MStarredStory.objects.order_by('pk')[offset:group_size*(group+1)]
+            for i, story in enumerate(stories):
+                original_permalink = story.story_permalink
+                try:
+                    story.story_permalink = unquote(story.story_permalink.encode('utf-8', 'ignore'))
+                    if original_permalink != story.story_permalink:
+                        story.save()
+                        print " ---> Fixing %s" % (story.pk)
+                        print "      Story #%s from: %s" % (i+offset, original_permalink)
+                        print "      Story #%s to  : %s" % (i+offset, story.story_permalink)
+                except UnicodeDecodeError:
+                    print " ***> Did not like: %s" % story.story_permalink
+                    pass
+
+    def backwards(self, orm):
+        "Write your backwards methods here."
+        pass
+
+    models = {
+        'rss_feeds.duplicatefeed': {
+            'Meta': {'object_name': 'DuplicateFeed'},
+            'duplicate_address': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'duplicate_feed_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}),
+            'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'duplicate_addresses'", 'to': "orm['rss_feeds.Feed']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        },
+        'rss_feeds.feed': {
+            'Meta': {'ordering': "['feed_title']", 'object_name': 'Feed', 'db_table': "'feeds'"},
+            'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
+            'active_premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}),
+            'active_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}),
+            'average_stories_per_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'branch_from_feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']", 'null': 'True', 'blank': 'True'}),
+            'creation': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'days_to_trim': ('django.db.models.fields.IntegerField', [], {'default': '90'}),
+            'etag': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'exception_code': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'favicon_color': ('django.db.models.fields.CharField', [], {'max_length': '6', 'null': 'True', 'blank': 'True'}),
+            'favicon_not_found': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'feed_address': ('django.db.models.fields.URLField', [], {'max_length': '255'}),
+            'feed_address_locked': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
+            'feed_link': ('django.db.models.fields.URLField', [], {'default': "''", 'max_length': '1000', 'null': 'True', 'blank': 'True'}),
+            'feed_link_locked': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'feed_title': ('django.db.models.fields.CharField', [], {'default': "'[Untitled]'", 'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'fetched_once': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'has_feed_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'has_page': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'has_page_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'hash_address_and_link': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_push': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
+            'known_good': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'last_load_time': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'last_modified': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'last_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+            'min_to_decay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'next_scheduled_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+            'num_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+            'premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+            'queued_date': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+            'stories_last_month': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
+        'rss_feeds.feeddata': {
+            'Meta': {'object_name': 'FeedData'},
+            'feed': ('utils.fields.AutoOneToOneField', [], {'related_name': "'data'", 'unique': 'True', 'to': "orm['rss_feeds.Feed']"}),
+            'feed_classifier_counts': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'feed_tagline': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'popular_authors': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}),
+            'popular_tags': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
+            'story_count_history': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'})
+        },
+        'rss_feeds.feedloadtime': {
+            'Meta': {'object_name': 'FeedLoadtime'},
+            'date_accessed': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'loadtime': ('django.db.models.fields.FloatField', [], {})
+        }
+    }
+
+    complete_apps = ['rss_feeds']

apps/rss_feeds/models.py

@@ -141,11 +141,11 @@ class Feed(models.Model):
         if len(self.feed_title) > max_feed_title:
             self.feed_title = self.feed_title[:max_feed_title]
         max_feed_address = Feed._meta.get_field('feed_address').max_length
-        if len(self.feed_address) > max_feed_address:
-            self.feed_address = self.feed_address[:max_feed_address]
+        if len(feed_address) > max_feed_address:
+            self.feed_address = feed_address[:max_feed_address]
         max_feed_link = Feed._meta.get_field('feed_link').max_length
-        if len(self.feed_link) > max_feed_link:
-            self.feed_link = self.feed_link[:max_feed_link]
+        if len(feed_link) > max_feed_link:
+            self.feed_link = feed_link[:max_feed_link]
         
         try:
             super(Feed, self).save(*args, **kwargs)
@@ -1355,7 +1355,7 @@ class MFeedFetchHistory(mongo.Document):
         for fetch in fetches:
             history                = {}
             history['message']     = fetch.message
-            history['fetch_date']  = fetch.fetch_date
+            history['fetch_date']  = fetch.fetch_date.strftime("%Y-%m-%d %H:%M:%S")
             history['status_code'] = fetch.status_code
             history['exception']   = fetch.exception
             fetch_history.append(history)
@@ -1388,12 +1388,35 @@ class MPageFetchHistory(mongo.Document):
         for fetch in fetches:
             history                = {}
             history['message']     = fetch.message
-            history['fetch_date']  = fetch.fetch_date
+            history['fetch_date']  = fetch.fetch_date.strftime("%Y-%m-%d %H:%M:%S")
             history['status_code'] = fetch.status_code
             history['exception']   = fetch.exception
             fetch_history.append(history)
         return fetch_history
-
+        
+        
+class MFeedPushHistory(mongo.Document):
+    feed_id = mongo.IntField()
+    push_date = mongo.DateTimeField(default=datetime.datetime.now)
+    
+    meta = {
+        'collection': 'feed_push_history',
+        'allow_inheritance': False,
+        'ordering': ['-push_date'],
+        'indexes': ['feed_id', '-push_date'],
+    }
+    
+    @classmethod
+    def feed_history(cls, feed_id):
+        pushes = cls.objects(feed_id=feed_id).order_by('-push_date')[:5]
+        push_history = []
+        for push in pushes:
+            history = {}
+            history['push_date']  = push.push_date.strftime("%Y-%m-%d %H:%M:%S")
+            push_history.append(history)
+        return push_history
+        
+        
 class FeedLoadtime(models.Model):
     feed = models.ForeignKey(Feed)
     date_accessed = models.DateTimeField(auto_now=True)

apps/rss_feeds/views.py

@@ -9,7 +9,8 @@ from django.contrib.auth.decorators import login_required
 from django.template import RequestContext
 # from django.db import IntegrityError
 from apps.rss_feeds.models import Feed, merge_feeds
-from apps.rss_feeds.models import MFeedFetchHistory, MPageFetchHistory, MFeedIcon
+from apps.rss_feeds.models import MFeedFetchHistory, MPageFetchHistory, MFeedPushHistory
+from apps.rss_feeds.models import MFeedIcon
 from apps.analyzer.models import get_classifiers_for_user
 from apps.reader.models import UserSubscription
 from utils.user_functions import ajax_login_required
@@ -150,6 +151,7 @@ def load_feed_statistics(request, feed_id):
     # Fetch histories
     stats['feed_fetch_history'] = MFeedFetchHistory.feed_history(feed_id)
     stats['page_fetch_history'] = MPageFetchHistory.feed_history(feed_id)
+    stats['feed_push_history'] = MFeedPushHistory.feed_history(feed_id)
     
     logging.user(request, "~FBStatistics: ~SB%s ~FG(%s/%s/%s subs)" % (feed, feed.num_subscribers, feed.active_subscribers, feed.premium_subscribers,))
 

media/css/reader.css

@@ -4984,8 +4984,11 @@ background: transparent;
 
 .NB-modal-statistics .NB-statistics-stat .NB-statistics-fetches-half {
   float: left;
-  width: 50%;
   text-align: center;
+  margin-right: 18px;
+}
+.NB-modal-statistics .NB-statistics-stat .NB-statistics-fetches-half:last-child {
+    margin-right: 0;
 }
 
 .NB-modal-statistics .NB-statistics-stat .NB-statistics-history-stat {
@@ -5027,12 +5030,17 @@ background: transparent;
 }
 .NB-modal-statistics .NB-statistics-history-fetch .NB-statistics-history-fetch-message {
   padding-right: 4px;
-  margin-left: 120px;
+  margin-left: 110px;
   font-weight: bold;
 }
 .NB-modal-statistics .NB-statistics-history-fetch .NB-statistics-history-fetch-exception {
   display: none;
 }
+.NB-modal-statistics .NB-statistics-history-empty {
+    color: #C0C0C0;
+    font-size: 10px;
+    padding: 4px 12px;
+}
 .NB-modal-statistics .NB-statistics-classifiers {
     border: 1px solid #e6e6e6;
     clear: both;

media/js/newsblur/reader/reader_statistics.js

@@ -129,12 +129,16 @@ _.extend(NEWSBLUR.ReaderStatistics.prototype, {
             ])),
             (!this.options.social_feed && $.make('div', { className: 'NB-statistics-stat NB-statistics-fetches'}, [
                 $.make('div', { className: 'NB-statistics-fetches-half'}, [
-                    $.make('div', { className: 'NB-statistics-label' }, 'Feed'),
-                    $.make('div', this.make_history(data, 'feed'))
+                    $.make('div', { className: 'NB-statistics-label' }, 'Feed Fetch'),
+                    $.make('div', this.make_history(data, 'feed_fetch'))
                 ]),
                 $.make('div', { className: 'NB-statistics-fetches-half'}, [
-                    $.make('div', { className: 'NB-statistics-label' }, 'Page'),
-                    $.make('div', this.make_history(data, 'page'))
+                    $.make('div', { className: 'NB-statistics-label' }, 'Page Fetch'),
+                    $.make('div', this.make_history(data, 'page_fetch'))
+                ]),
+                $.make('div', { className: 'NB-statistics-fetches-half'}, [
+                    $.make('div', { className: 'NB-statistics-label' }, 'Feed Push'),
+                    $.make('div', this.make_history(data, 'feed_push'))
                 ])
             ]))
         ]);
@@ -212,20 +216,24 @@ _.extend(NEWSBLUR.ReaderStatistics.prototype, {
     },
     
     make_history: function(data, fetch_type) {
-        var fetches = data[fetch_type+'_fetch_history'];
-        if (!fetches) return;
+        var fetches = data[fetch_type+'_history'];
+        var $history;
         
-        var $history = _.map(fetches, function(fetch) {
-            var feed_ok = _.contains([200, 304], fetch.status_code);
-            var status_class = feed_ok ? ' NB-ok ' : ' NB-errorcode ';
-            return $.make('div', { className: 'NB-statistics-history-fetch' + status_class, title: feed_ok ? '' : fetch.exception }, [
-                $.make('div', { className: 'NB-statistics-history-fetch-date' }, fetch.fetch_date),
-                $.make('div', { className: 'NB-statistics-history-fetch-message' }, [
-                    fetch.message,
-                    $.make('div', { className: 'NB-statistics-history-fetch-code' }, ' ('+fetch.status_code+')')
-                ])
-            ]);
-        });
+        if (!fetches || !fetches.length) {
+            $history = $.make('div', { className: 'NB-statistics-history-empty' }, "Nothing recorded.");
+        } else {
+            $history = _.map(fetches, function(fetch) {
+                var feed_ok = _.contains([200, 304], fetch.status_code) || !fetch.status_code;
+                var status_class = feed_ok ? ' NB-ok ' : ' NB-errorcode ';
+                return $.make('div', { className: 'NB-statistics-history-fetch' + status_class, title: feed_ok ? '' : fetch.exception }, [
+                    $.make('div', { className: 'NB-statistics-history-fetch-date' }, fetch.fetch_date || fetch.push_date),
+                    $.make('div', { className: 'NB-statistics-history-fetch-message' }, [
+                        fetch.message,
+                        (fetch.status_code && $.make('div', { className: 'NB-statistics-history-fetch-code' }, ' ('+fetch.status_code+')'))
+                    ])
+                ]);
+            });
+        }
 
         return $history;
     },