Attempting to fix dashboard stories by no longer relying on full river reloads when a new story comes in but to fetch that story if it is newer (or older) than last visible dashboard story.

apps/newsletters/models.py

@@ -1,9 +1,6 @@
 import datetime
 import re
 import redis
-from cgi import escape
-from django.db import models
-from django.contrib.auth.models import User
 from django.contrib.sites.models import Site
 from django.core.mail import EmailMultiAlternatives
 from django.core.urlresolvers import reverse
@@ -87,7 +84,7 @@ class EmailNewsletter:
         usersub.needs_unread_recalc = True
         usersub.save()
         
-        self._publish_to_subscribers(feed)
+        self._publish_to_subscribers(feed, story.story_hash)
         
         MFetchHistory.add(feed_id=feed.pk, fetch_type='push')
         logging.user(user, "~FCNewsletter feed story: ~SB%s~SN / ~SB%s" % (story.story_title, feed))
@@ -109,12 +106,12 @@ class EmailNewsletter:
         
         params = dict(receiver_user_id=user.pk, email_type='first_newsletter')
         try:
-            sent_email = MSentEmail.objects.get(**params)
+            MSentEmail.objects.get(**params)
             if not force:
                 # Return if email already sent
                 return
         except MSentEmail.DoesNotExist:
-            sent_email = MSentEmail.objects.create(**params)
+            MSentEmail.objects.create(**params)
                 
         text    = render_to_string('mail/email_first_newsletter.txt', {})
         html    = render_to_string('mail/email_first_newsletter.xhtml', {})
@@ -177,10 +174,10 @@ class EmailNewsletter:
         content = content.replace('!important', '')
         return content
         
-    def _publish_to_subscribers(self, feed):
+    def _publish_to_subscribers(self, feed, story_hash):
         try:
             r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
-            listeners_count = r.publish(str(feed.pk), 'story:new')
+            listeners_count = r.publish(str(feed.pk), 'story:new:%s' % story_hash)
             if listeners_count:
                 logging.debug("   ---> [%-30s] ~FMPublished to %s subscribers" % (feed.title[:30], listeners_count))
         except redis.ConnectionError:

apps/reader/views.py

@@ -1227,6 +1227,7 @@ def load_river_stories__redis(request):
     query             = request.REQUEST.get('query', '').strip()
     include_hidden    = is_true(request.REQUEST.get('include_hidden', False))
     include_feeds     = is_true(request.REQUEST.get('include_feeds', False))
+    initial_dashboard = is_true(request.GET.get('initial_dashboard', False))
     now               = localtime_for_timezone(datetime.datetime.now(), user.profile.timezone)
     usersubs          = []
     code              = 1
@@ -1234,7 +1235,7 @@ def load_river_stories__redis(request):
     offset            = (page-1) * limit
     story_date_order  = "%sstory_date" % ('' if order == 'oldest' else '-')
     
-    if limit == 4 and not request.user.is_staff:
+    if limit == 4 and not initial_dashboard:
         logging.user(request, "~FRIgnoring ~FCdashboard river stories")
         return dict(code=-1, message="Had to turn off dashboard river for now.", stories=[])
         

apps/rss_feeds/models.py

@@ -1169,6 +1169,7 @@ class Feed(models.Model):
                 try:
                     s.save()
                     ret_values['new'] += 1
+                    s.publish_to_subscribers()
                 except (IntegrityError, OperationError), e:
                     ret_values['error'] += 1
                     if settings.DEBUG:
@@ -2232,6 +2233,13 @@ class MStory(mongo.Document):
         
         super(MStory, self).delete(*args, **kwargs)
     
+    def publish_to_subscribers(self):
+        try:
+            r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
+            r.publish("%s:story" % (self.story_feed_id), '%s,%s' % (self.story_hash, self.story_date.strftime('%s')))
+        except redis.ConnectionError:
+            logging.debug("   ***> [%-30s] ~BMRedis is unavailable for real-time." % (Feed.get_by_id(self.story_feed_id).title[:30],))
+    
     @classmethod
     def purge_feed_stories(cls, feed, cutoff, verbose=True):
         stories = cls.objects(story_feed_id=feed.pk)

apps/social/models.py

@@ -1863,7 +1863,7 @@ class MSharedStory(mongo.DynamicDocument):
         try:
             r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
             feed_id = "social:%s" % self.user_id
-            listeners_count = r.publish(feed_id, 'story:new')
+            listeners_count = r.publish(feed_id, 'story:new:%s' % self.story_hash)
             if listeners_count:
                 logging.debug("   ---> ~FMPublished to %s subscribers" % (listeners_count))
         except redis.ConnectionError:

media/js/newsblur/common/assetmodel.js

@@ -569,6 +569,7 @@ NEWSBLUR.AssetModel = Backbone.Router.extend({
             }
             
             if (data.stories && first_load) {
+                // console.log(['first load river', data.stories.length, ' stories']);
                 this.feed_tags = data.feed_tags || {};
                 this.feed_authors = data.feed_authors || {};
                 this.active_feed = this.get_feed(feed_id);
@@ -583,6 +584,7 @@ NEWSBLUR.AssetModel = Backbone.Router.extend({
                 this.starred_stories = data.starred_stories;
                 this.stories.reset(data.stories, {added: data.stories.length});
             } else if (data.stories) {
+                // console.log(['adding to river', data.stories.length, ' stories']);
                 this.stories.add(data.stories, {silent: true});
                 this.stories.trigger('add', {added: data.stories.length});
             }
@@ -707,6 +709,12 @@ NEWSBLUR.AssetModel = Backbone.Router.extend({
     fetch_dashboard_stories: function(feed_id, feeds, page, callback, error_callback) {
         var self = this;
         
+        this.dashboard_stories.comparator = function(a, b) {
+            var a_time = parseInt(a.get('story_timestamp'), 10);
+            var b_time = parseInt(b.get('story_timestamp'), 10);
+            return a_time < b_time ? 1 : (a_time == b_time) ? 0 : -1;
+        };
+        
         var pre_callback = function(data) {
             if (data.user_profiles) {
                 self.add_user_profiles(data.user_profiles);
@@ -731,13 +739,32 @@ NEWSBLUR.AssetModel = Backbone.Router.extend({
             order: this.view_setting(feed_id, 'order'),
             read_filter: this.view_setting(feed_id, 'read_filter'),
             include_hidden: false,
-            dashboard: true
+            dashboard: true,
+            initial_dashboard: true
         }, pre_callback, error_callback, {
             'ajax_group': 'dashboard',
             'request_type': 'GET'
         });
     },
     
+    add_dashboard_story: function(story_hash) {
+        var self = this;
+        
+        var pre_callback = function(data) {
+            self.dashboard_stories.add(data.stories, {silent: true});
+            self.dashboard_stories.limit(NEWSBLUR.Globals.is_premium ? 5 : 3);
+            self.dashboard_stories.trigger('reset', {added: 1});
+        };
+        
+        this.make_request('/reader/river_stories', {
+            h: story_hash,
+            dashboard: true
+        }, pre_callback, null, {
+            'ajax_group': 'dashboard',
+            'request_type': 'GET'
+        });
+    },
+    
     fetch_river_blurblogs_stories: function(feed_id, page, options, callback, error_callback, first_load) {
         var self = this;
         

media/js/newsblur/models/stories.js

@@ -258,7 +258,7 @@ NEWSBLUR.Models.Story = Backbone.Model.extend({
 NEWSBLUR.Collections.Stories = Backbone.Collection.extend({
     
     model: NEWSBLUR.Models.Story,
-   
+    
     read_stories: [],
     
     previous_stories_stack: [],
@@ -459,6 +459,10 @@ NEWSBLUR.Collections.Stories = Backbone.Collection.extend({
         });
     },
     
+    limit: function(count) {
+        this.models = this.models.slice(0, count);
+    },
+    
     // ===========
     // = Getters =
     // ===========

media/js/newsblur/reader/reader.js

@@ -1541,7 +1541,7 @@
             var $original_tabs = $('.task_view_page, .task_view_story');
             var $page_tab = $('.task_view_page');
             view = view || NEWSBLUR.assets.view_setting(feed_id);
-            console.log(['set_correct_story_view_for_feed', feed_id, view]);
+            // console.log(['set_correct_story_view_for_feed', feed_id, view]);
             
             $original_tabs.removeClass('NB-disabled-page')
                           .removeClass('NB-disabled')
@@ -1632,7 +1632,7 @@
                 $page_tab.addClass('NB-disabled');
             }
             
-            console.log(['setting reader.story_view', view, " was:", this.story_view]);
+            // console.log(['setting reader.story_view', view, " was:", this.story_view]);
             this.story_view = view;
         },
         
@@ -1963,9 +1963,12 @@
                 NEWSBLUR.assets.stories.reset(NEWSBLUR.assets.dashboard_stories.map(function(story) {
                     return story.toJSON();
                 }));
+                this.model.fetch_river_stories(this.active_feed, feeds, 1, 
+                    _.bind(this.post_open_river_stories, this), NEWSBLUR.app.taskbar_info.show_stories_error, false);
+            } else {
+                this.model.fetch_river_stories(this.active_feed, feeds, 1, 
+                    _.bind(this.post_open_river_stories, this), NEWSBLUR.app.taskbar_info.show_stories_error, true);
             }
-            this.model.fetch_river_stories(this.active_feed, feeds, 1, 
-                _.bind(this.post_open_river_stories, this), NEWSBLUR.app.taskbar_info.show_stories_error, true);
         },
         
         post_open_river_stories: function(data, first_load) {
@@ -2323,7 +2326,7 @@
 
             this.model.mark_feed_as_read([feed_id], cutoff_timestamp, direction, 
                                          feed_id == this.active_feed,  _.bind(function() {
-                this.feeds_unread_count(feed_id);
+                this.feed_unread_count(feed_id);
             }, this));
 
             if (!direction && NEWSBLUR.assets.preference('markread_nextfeed') == 'nextfeed' && 
@@ -2865,7 +2868,7 @@
             var feed_id = this.active_story_view();
             var feed = this.model.get_feed(feed_id);
             view = view || this.story_view;
-            NEWSBLUR.log(['switch_taskbar_view', view, options.skip_save_type, feed]);
+            // NEWSBLUR.log(['switch_taskbar_view', view, options.skip_save_type, feed]);
             
             if (view == 'page' && feed && feed.get('has_exception') && 
                 feed.get('exception_type') == 'page') {
@@ -4756,12 +4759,19 @@
                 this.socket.on('feed:update', _.bind(function(feed_id, message) {
                     NEWSBLUR.log(['Real-time feed update', feed_id, message]);
                     this.feed_unread_count(feed_id, {
-                        realtime: true,
+                        realtime: true
-                        new_story: true
                     });
                     
                 }, this));
-                
+
+                this.socket.removeAllListeners('feed:story:new');
+                this.socket.on('feed:story:new', _.bind(function(feed_id, message) {
+                    var story_hash = message.split(',')[0];
+                    var timestamp = message.split(',')[1];
+                    // NEWSBLUR.log(['Real-time new story', feed_id, story_hash, timestamp]);
+                    NEWSBLUR.app.dashboard_river.new_story(story_hash, timestamp);
+                }, this));
+
                 this.socket.removeAllListeners(NEWSBLUR.Globals.username);
                 this.socket.removeAllListeners("user:update");
                 this.socket.on('user:update', _.bind(function(username, message) {
@@ -4974,18 +4984,9 @@
             
             _.delay(_.bind(function() {
                 this.model.feed_unread_count(feed_id, options.callback);
-                if (options.new_story) {
-                    NEWSBLUR.app.dashboard_river.load_stories();
-                }
             }, this), Math.random() * delay);
         },
         
-        feeds_unread_count: function(feed_ids, options) {
-            options = options || {};
-            
-            this.model.feed_unread_count(feed_ids, options.callback);
-        },
-        
         update_interactions_count: function() {
             this.model.interactions_count(function(data) {
                 NEWSBLUR.app.sidebar_header.update_interactions_count(data.interactions_count);

media/js/newsblur/views/dashboard_river.js

@@ -47,6 +47,7 @@ NEWSBLUR.Views.DashboardRiver = Backbone.View.extend({
     // ==========
     
     load_stories: function() {
+        // console.log(['dashboard river load_stories', this.page]);
         // var feeds = NEWSBLUR.assets.folders.feed_ids_in_folder();
         var feeds = this.feeds();
         if (!feeds.length) return;
@@ -54,7 +55,7 @@ NEWSBLUR.Views.DashboardRiver = Backbone.View.extend({
         
         this.page = 1;
         this.story_titles.show_loading();
-        NEWSBLUR.assets.fetch_dashboard_stories("river:", feeds, this.page, 
+        NEWSBLUR.assets.fetch_dashboard_stories(this.active_feed, feeds, this.page, 
             _.bind(this.post_load_stories, this), NEWSBLUR.app.taskbar_info.show_stories_error);
     },
     
@@ -74,7 +75,7 @@ NEWSBLUR.Views.DashboardRiver = Backbone.View.extend({
         var counts = NEWSBLUR.assets.folders.unread_counts();
         var unread_view = NEWSBLUR.assets.preference('unread_view');
         if (unread_view >= 1) {
-            console.log(['counts', counts['ps'], visible, this.page]);
+            // console.log(['counts', counts['ps'], visible, this.page]);
             if (counts['ps'] <= visible) {
                 this.show_end_line();
                 return;
@@ -91,7 +92,7 @@ NEWSBLUR.Views.DashboardRiver = Backbone.View.extend({
         var feeds = this.feeds();
         this.page += 1;
         this.story_titles.show_loading();
-        NEWSBLUR.assets.fetch_dashboard_stories("river:", feeds, this.page, 
+        NEWSBLUR.assets.fetch_dashboard_stories(this.active_feed, feeds, this.page, 
             _.bind(this.post_load_stories, this), NEWSBLUR.app.taskbar_info.show_stories_error);        
     },
     
@@ -118,6 +119,31 @@ NEWSBLUR.Views.DashboardRiver = Backbone.View.extend({
     show_end_line: function() {
         this.story_titles.show_no_more_stories();
         this.$(".NB-end-line").addClass("NB-visible");
+    },
+    
+    new_story: function(story_hash, timestamp) {
+        var oldest_story = NEWSBLUR.assets.dashboard_stories.last();
+
+        if (oldest_story) {
+            var last_timestamp = parseInt(oldest_story.get('story_timestamp'), 10);
+            timestamp = parseInt(timestamp, 10);
+            
+            if (NEWSBLUR.assets.view_setting(this.active_feed, 'order') == 'newest') {
+                if (timestamp < last_timestamp) {
+                    // console.log(['New story older than last/oldest dashboard story', timestamp, '<', last_timestamp]);
+                    return;
+                }
+            } else {
+                if (timestamp > last_timestamp) {
+                    // console.log(['New story older than last/newest dashboard story', timestamp, '<', last_timestamp]);
+                    return;
+                }
+            }
+        }
+        
+        console.log(['Fetching dashboard story', story_hash]);
+        NEWSBLUR.assets.add_dashboard_story(story_hash);
+        
     }
     
 });

media/js/newsblur/views/text_tab_view.js

@@ -60,6 +60,8 @@ NEWSBLUR.Views.TextTabView = Backbone.View.extend({
     },
     
     render: function(data) {
+        if (!this.story) return;
+        
         if (data && (data.story_id != this.story.get('id') || 
                      data.feed_id != this.story.get('story_feed_id'))) {
             return;

media/js/vendor/backbone-1.0.0.js

@@ -1464,7 +1464,6 @@
     // route callback be fired (not usually desirable), or `replace: true`, if
     // you wish to modify the current URL without adding an entry to the history.
     navigate: function(fragment, options) {
-        console.log(['backbone navigate', fragment, options]);
       if (!History.started) return false;
       if (!options || options === true) options = {trigger: options};
       fragment = this.getFragment(fragment || '');

node/unread_counts.coffee

@@ -59,12 +59,16 @@ io.on 'connection', (socket) ->
                      " (#{io.engine.clientsCount} connected) " +
                      " #{if SECURE then "(SSL)" else "(non-SSL)"}"
             socket.subscribe.subscribe @feeds
+            feeds_story = @feeds.map (f) -> "#{f}:story"
+            socket.subscribe.subscribe feeds_story
             socket.subscribe.subscribe @username
 
         socket.subscribe.on 'message', (channel, message) =>
             log.info @username, "Update on #{channel}: #{message}"
             if channel == @username
                 socket.emit 'user:update', channel, message
+            else if channel.indexOf(':story') >= 0
+                socket.emit 'feed:story:new', channel, message
             else
                 socket.emit 'feed:update', channel, message
 

node/unread_counts.js

@@ -63,8 +63,13 @@
       })(this));
       socket.subscribe.on("connect", (function(_this) {
         return function() {
+          var feeds_story;
           log.info(_this.username, ("Connected (" + _this.feeds.length + " feeds, " + ip + "),") + (" (" + io.engine.clientsCount + " connected) ") + (" " + (SECURE ? "(SSL)" : "(non-SSL)")));
           socket.subscribe.subscribe(_this.feeds);
+          feeds_story = _this.feeds.map(function(f) {
+            return "" + f + ":story";
+          });
+          socket.subscribe.subscribe(feeds_story);
           return socket.subscribe.subscribe(_this.username);
         };
       })(this));
@@ -73,6 +78,8 @@
           log.info(_this.username, "Update on " + channel + ": " + message);
           if (channel === _this.username) {
             return socket.emit('user:update', channel, message);
+          } else if (channel.indexOf(':story') >= 0) {
+            return socket.emit('feed:story:new', channel, message);
           } else {
             return socket.emit('feed:update', channel, message);
           }

templates/reader/dashboard.xhtml

@@ -72,7 +72,7 @@
             {% render_getting_started %}
         {% endif %}
           
-        {% if user.is_staff or user.username == "chrome" %}
+        {% if user.is_staff %}
         <div class="NB-module NB-module-river">
             <h5 class="NB-module-header">
                 <div class="NB-module-river-settings NB-javascript"></div>

utils/feed_fetcher.py

@@ -954,7 +954,7 @@ class Dispatcher:
                 logging.debug("   ---> [%-30s] ~FRIntegrityError on feed: %s" % (feed.title[:30], feed.feed_address,))
             
             if ret_entries and ret_entries['new']:
-                self.publish_to_subscribers(feed)
+                self.publish_to_subscribers(feed, ret_entries['new'])
                 
             done_msg = (u'%2s ---> [%-30s] ~FYProcessed in ~FM~SB%.4ss~FY~SN (~FB%s~FY) [%s]' % (
                 identity, feed.title[:30], delta,
@@ -973,10 +973,10 @@ class Dispatcher:
         
         # time_taken = datetime.datetime.utcnow() - self.time_start
     
-    def publish_to_subscribers(self, feed):
+    def publish_to_subscribers(self, feed, new_count):
         try:
             r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
-            listeners_count = r.publish(str(feed.pk), 'story:new')
+            listeners_count = r.publish(str(feed.pk), 'story:new_count:%s' % new_count)
             if listeners_count:
                 logging.debug("   ---> [%-30s] ~FMPublished to %s subscribers" % (feed.title[:30], listeners_count))
         except redis.ConnectionError: