From c5d5ea500158ecddca438f1f86c372ac03808a7a Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Tue, 13 Dec 2016 16:29:42 -0800 Subject: [PATCH] Attempting to fix dashboard stories by no longer relying on full river reloads when a new story comes in but to fetch that story if it is newer (or older) than last visible dashboard story. --- apps/newsletters/models.py | 13 +++----- apps/reader/views.py | 3 +- apps/rss_feeds/models.py | 8 +++++ apps/social/models.py | 2 +- media/js/newsblur/common/assetmodel.js | 29 ++++++++++++++++- media/js/newsblur/models/stories.js | 6 +++- media/js/newsblur/reader/reader.js | 37 +++++++++++----------- media/js/newsblur/views/dashboard_river.js | 32 +++++++++++++++++-- media/js/newsblur/views/text_tab_view.js | 2 ++ media/js/vendor/backbone-1.0.0.js | 1 - node/unread_counts.coffee | 4 +++ node/unread_counts.js | 7 ++++ templates/reader/dashboard.xhtml | 2 +- utils/feed_fetcher.py | 6 ++-- 14 files changed, 114 insertions(+), 38 deletions(-) diff --git a/apps/newsletters/models.py b/apps/newsletters/models.py index e676f4d0a..33b5e401e 100644 --- a/apps/newsletters/models.py +++ b/apps/newsletters/models.py @@ -1,9 +1,6 @@ import datetime import re import redis -from cgi import escape -from django.db import models -from django.contrib.auth.models import User from django.contrib.sites.models import Site from django.core.mail import EmailMultiAlternatives from django.core.urlresolvers import reverse @@ -87,7 +84,7 @@ class EmailNewsletter: usersub.needs_unread_recalc = True usersub.save() - self._publish_to_subscribers(feed) + self._publish_to_subscribers(feed, story.story_hash) MFetchHistory.add(feed_id=feed.pk, fetch_type='push') logging.user(user, "~FCNewsletter feed story: ~SB%s~SN / ~SB%s" % (story.story_title, feed)) @@ -109,12 +106,12 @@ class EmailNewsletter: params = dict(receiver_user_id=user.pk, email_type='first_newsletter') try: - sent_email = MSentEmail.objects.get(**params) + MSentEmail.objects.get(**params) if not force: # Return if email already sent return except MSentEmail.DoesNotExist: - sent_email = MSentEmail.objects.create(**params) + MSentEmail.objects.create(**params) text = render_to_string('mail/email_first_newsletter.txt', {}) html = render_to_string('mail/email_first_newsletter.xhtml', {}) @@ -177,10 +174,10 @@ class EmailNewsletter: content = content.replace('!important', '') return content - def _publish_to_subscribers(self, feed): + def _publish_to_subscribers(self, feed, story_hash): try: r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL) - listeners_count = r.publish(str(feed.pk), 'story:new') + listeners_count = r.publish(str(feed.pk), 'story:new:%s' % story_hash) if listeners_count: logging.debug(" ---> [%-30s] ~FMPublished to %s subscribers" % (feed.title[:30], listeners_count)) except redis.ConnectionError: diff --git a/apps/reader/views.py b/apps/reader/views.py index 06c64d33c..ec2c01f0a 100644 --- a/apps/reader/views.py +++ b/apps/reader/views.py @@ -1227,6 +1227,7 @@ def load_river_stories__redis(request): query = request.REQUEST.get('query', '').strip() include_hidden = is_true(request.REQUEST.get('include_hidden', False)) include_feeds = is_true(request.REQUEST.get('include_feeds', False)) + initial_dashboard = is_true(request.GET.get('initial_dashboard', False)) now = localtime_for_timezone(datetime.datetime.now(), user.profile.timezone) usersubs = [] code = 1 @@ -1234,7 +1235,7 @@ def load_river_stories__redis(request): offset = (page-1) * limit story_date_order = "%sstory_date" % ('' if order == 'oldest' else '-') - if limit == 4 and not request.user.is_staff: + if limit == 4 and not initial_dashboard: logging.user(request, "~FRIgnoring ~FCdashboard river stories") return dict(code=-1, message="Had to turn off dashboard river for now.", stories=[]) diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index 7140f145a..89dc6ac94 100644 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -1169,6 +1169,7 @@ class Feed(models.Model): try: s.save() ret_values['new'] += 1 + s.publish_to_subscribers() except (IntegrityError, OperationError), e: ret_values['error'] += 1 if settings.DEBUG: @@ -2232,6 +2233,13 @@ class MStory(mongo.Document): super(MStory, self).delete(*args, **kwargs) + def publish_to_subscribers(self): + try: + r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL) + r.publish("%s:story" % (self.story_feed_id), '%s,%s' % (self.story_hash, self.story_date.strftime('%s'))) + except redis.ConnectionError: + logging.debug(" ***> [%-30s] ~BMRedis is unavailable for real-time." % (Feed.get_by_id(self.story_feed_id).title[:30],)) + @classmethod def purge_feed_stories(cls, feed, cutoff, verbose=True): stories = cls.objects(story_feed_id=feed.pk) diff --git a/apps/social/models.py b/apps/social/models.py index 43e1ba9bb..d836398d5 100644 --- a/apps/social/models.py +++ b/apps/social/models.py @@ -1863,7 +1863,7 @@ class MSharedStory(mongo.DynamicDocument): try: r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL) feed_id = "social:%s" % self.user_id - listeners_count = r.publish(feed_id, 'story:new') + listeners_count = r.publish(feed_id, 'story:new:%s' % self.story_hash) if listeners_count: logging.debug(" ---> ~FMPublished to %s subscribers" % (listeners_count)) except redis.ConnectionError: diff --git a/media/js/newsblur/common/assetmodel.js b/media/js/newsblur/common/assetmodel.js index f03d82581..910285852 100644 --- a/media/js/newsblur/common/assetmodel.js +++ b/media/js/newsblur/common/assetmodel.js @@ -569,6 +569,7 @@ NEWSBLUR.AssetModel = Backbone.Router.extend({ } if (data.stories && first_load) { + // console.log(['first load river', data.stories.length, ' stories']); this.feed_tags = data.feed_tags || {}; this.feed_authors = data.feed_authors || {}; this.active_feed = this.get_feed(feed_id); @@ -583,6 +584,7 @@ NEWSBLUR.AssetModel = Backbone.Router.extend({ this.starred_stories = data.starred_stories; this.stories.reset(data.stories, {added: data.stories.length}); } else if (data.stories) { + // console.log(['adding to river', data.stories.length, ' stories']); this.stories.add(data.stories, {silent: true}); this.stories.trigger('add', {added: data.stories.length}); } @@ -707,6 +709,12 @@ NEWSBLUR.AssetModel = Backbone.Router.extend({ fetch_dashboard_stories: function(feed_id, feeds, page, callback, error_callback) { var self = this; + this.dashboard_stories.comparator = function(a, b) { + var a_time = parseInt(a.get('story_timestamp'), 10); + var b_time = parseInt(b.get('story_timestamp'), 10); + return a_time < b_time ? 1 : (a_time == b_time) ? 0 : -1; + }; + var pre_callback = function(data) { if (data.user_profiles) { self.add_user_profiles(data.user_profiles); @@ -731,13 +739,32 @@ NEWSBLUR.AssetModel = Backbone.Router.extend({ order: this.view_setting(feed_id, 'order'), read_filter: this.view_setting(feed_id, 'read_filter'), include_hidden: false, - dashboard: true + dashboard: true, + initial_dashboard: true }, pre_callback, error_callback, { 'ajax_group': 'dashboard', 'request_type': 'GET' }); }, + add_dashboard_story: function(story_hash) { + var self = this; + + var pre_callback = function(data) { + self.dashboard_stories.add(data.stories, {silent: true}); + self.dashboard_stories.limit(NEWSBLUR.Globals.is_premium ? 5 : 3); + self.dashboard_stories.trigger('reset', {added: 1}); + }; + + this.make_request('/reader/river_stories', { + h: story_hash, + dashboard: true + }, pre_callback, null, { + 'ajax_group': 'dashboard', + 'request_type': 'GET' + }); + }, + fetch_river_blurblogs_stories: function(feed_id, page, options, callback, error_callback, first_load) { var self = this; diff --git a/media/js/newsblur/models/stories.js b/media/js/newsblur/models/stories.js index bafcefbee..94e5b678a 100644 --- a/media/js/newsblur/models/stories.js +++ b/media/js/newsblur/models/stories.js @@ -258,7 +258,7 @@ NEWSBLUR.Models.Story = Backbone.Model.extend({ NEWSBLUR.Collections.Stories = Backbone.Collection.extend({ model: NEWSBLUR.Models.Story, - + read_stories: [], previous_stories_stack: [], @@ -459,6 +459,10 @@ NEWSBLUR.Collections.Stories = Backbone.Collection.extend({ }); }, + limit: function(count) { + this.models = this.models.slice(0, count); + }, + // =========== // = Getters = // =========== diff --git a/media/js/newsblur/reader/reader.js b/media/js/newsblur/reader/reader.js index 8dc85f8db..a749106ac 100644 --- a/media/js/newsblur/reader/reader.js +++ b/media/js/newsblur/reader/reader.js @@ -1541,7 +1541,7 @@ var $original_tabs = $('.task_view_page, .task_view_story'); var $page_tab = $('.task_view_page'); view = view || NEWSBLUR.assets.view_setting(feed_id); - console.log(['set_correct_story_view_for_feed', feed_id, view]); + // console.log(['set_correct_story_view_for_feed', feed_id, view]); $original_tabs.removeClass('NB-disabled-page') .removeClass('NB-disabled') @@ -1632,7 +1632,7 @@ $page_tab.addClass('NB-disabled'); } - console.log(['setting reader.story_view', view, " was:", this.story_view]); + // console.log(['setting reader.story_view', view, " was:", this.story_view]); this.story_view = view; }, @@ -1963,9 +1963,12 @@ NEWSBLUR.assets.stories.reset(NEWSBLUR.assets.dashboard_stories.map(function(story) { return story.toJSON(); })); + this.model.fetch_river_stories(this.active_feed, feeds, 1, + _.bind(this.post_open_river_stories, this), NEWSBLUR.app.taskbar_info.show_stories_error, false); + } else { + this.model.fetch_river_stories(this.active_feed, feeds, 1, + _.bind(this.post_open_river_stories, this), NEWSBLUR.app.taskbar_info.show_stories_error, true); } - this.model.fetch_river_stories(this.active_feed, feeds, 1, - _.bind(this.post_open_river_stories, this), NEWSBLUR.app.taskbar_info.show_stories_error, true); }, post_open_river_stories: function(data, first_load) { @@ -2323,7 +2326,7 @@ this.model.mark_feed_as_read([feed_id], cutoff_timestamp, direction, feed_id == this.active_feed, _.bind(function() { - this.feeds_unread_count(feed_id); + this.feed_unread_count(feed_id); }, this)); if (!direction && NEWSBLUR.assets.preference('markread_nextfeed') == 'nextfeed' && @@ -2865,7 +2868,7 @@ var feed_id = this.active_story_view(); var feed = this.model.get_feed(feed_id); view = view || this.story_view; - NEWSBLUR.log(['switch_taskbar_view', view, options.skip_save_type, feed]); + // NEWSBLUR.log(['switch_taskbar_view', view, options.skip_save_type, feed]); if (view == 'page' && feed && feed.get('has_exception') && feed.get('exception_type') == 'page') { @@ -4756,12 +4759,19 @@ this.socket.on('feed:update', _.bind(function(feed_id, message) { NEWSBLUR.log(['Real-time feed update', feed_id, message]); this.feed_unread_count(feed_id, { - realtime: true, - new_story: true + realtime: true }); }, this)); - + + this.socket.removeAllListeners('feed:story:new'); + this.socket.on('feed:story:new', _.bind(function(feed_id, message) { + var story_hash = message.split(',')[0]; + var timestamp = message.split(',')[1]; + // NEWSBLUR.log(['Real-time new story', feed_id, story_hash, timestamp]); + NEWSBLUR.app.dashboard_river.new_story(story_hash, timestamp); + }, this)); + this.socket.removeAllListeners(NEWSBLUR.Globals.username); this.socket.removeAllListeners("user:update"); this.socket.on('user:update', _.bind(function(username, message) { @@ -4974,18 +4984,9 @@ _.delay(_.bind(function() { this.model.feed_unread_count(feed_id, options.callback); - if (options.new_story) { - NEWSBLUR.app.dashboard_river.load_stories(); - } }, this), Math.random() * delay); }, - feeds_unread_count: function(feed_ids, options) { - options = options || {}; - - this.model.feed_unread_count(feed_ids, options.callback); - }, - update_interactions_count: function() { this.model.interactions_count(function(data) { NEWSBLUR.app.sidebar_header.update_interactions_count(data.interactions_count); diff --git a/media/js/newsblur/views/dashboard_river.js b/media/js/newsblur/views/dashboard_river.js index afbd195b2..cfcadad5b 100644 --- a/media/js/newsblur/views/dashboard_river.js +++ b/media/js/newsblur/views/dashboard_river.js @@ -47,6 +47,7 @@ NEWSBLUR.Views.DashboardRiver = Backbone.View.extend({ // ========== load_stories: function() { + // console.log(['dashboard river load_stories', this.page]); // var feeds = NEWSBLUR.assets.folders.feed_ids_in_folder(); var feeds = this.feeds(); if (!feeds.length) return; @@ -54,7 +55,7 @@ NEWSBLUR.Views.DashboardRiver = Backbone.View.extend({ this.page = 1; this.story_titles.show_loading(); - NEWSBLUR.assets.fetch_dashboard_stories("river:", feeds, this.page, + NEWSBLUR.assets.fetch_dashboard_stories(this.active_feed, feeds, this.page, _.bind(this.post_load_stories, this), NEWSBLUR.app.taskbar_info.show_stories_error); }, @@ -74,7 +75,7 @@ NEWSBLUR.Views.DashboardRiver = Backbone.View.extend({ var counts = NEWSBLUR.assets.folders.unread_counts(); var unread_view = NEWSBLUR.assets.preference('unread_view'); if (unread_view >= 1) { - console.log(['counts', counts['ps'], visible, this.page]); + // console.log(['counts', counts['ps'], visible, this.page]); if (counts['ps'] <= visible) { this.show_end_line(); return; @@ -91,7 +92,7 @@ NEWSBLUR.Views.DashboardRiver = Backbone.View.extend({ var feeds = this.feeds(); this.page += 1; this.story_titles.show_loading(); - NEWSBLUR.assets.fetch_dashboard_stories("river:", feeds, this.page, + NEWSBLUR.assets.fetch_dashboard_stories(this.active_feed, feeds, this.page, _.bind(this.post_load_stories, this), NEWSBLUR.app.taskbar_info.show_stories_error); }, @@ -118,6 +119,31 @@ NEWSBLUR.Views.DashboardRiver = Backbone.View.extend({ show_end_line: function() { this.story_titles.show_no_more_stories(); this.$(".NB-end-line").addClass("NB-visible"); + }, + + new_story: function(story_hash, timestamp) { + var oldest_story = NEWSBLUR.assets.dashboard_stories.last(); + + if (oldest_story) { + var last_timestamp = parseInt(oldest_story.get('story_timestamp'), 10); + timestamp = parseInt(timestamp, 10); + + if (NEWSBLUR.assets.view_setting(this.active_feed, 'order') == 'newest') { + if (timestamp < last_timestamp) { + // console.log(['New story older than last/oldest dashboard story', timestamp, '<', last_timestamp]); + return; + } + } else { + if (timestamp > last_timestamp) { + // console.log(['New story older than last/newest dashboard story', timestamp, '<', last_timestamp]); + return; + } + } + } + + console.log(['Fetching dashboard story', story_hash]); + NEWSBLUR.assets.add_dashboard_story(story_hash); + } }); \ No newline at end of file diff --git a/media/js/newsblur/views/text_tab_view.js b/media/js/newsblur/views/text_tab_view.js index 7007fb052..b1f2e84e0 100644 --- a/media/js/newsblur/views/text_tab_view.js +++ b/media/js/newsblur/views/text_tab_view.js @@ -60,6 +60,8 @@ NEWSBLUR.Views.TextTabView = Backbone.View.extend({ }, render: function(data) { + if (!this.story) return; + if (data && (data.story_id != this.story.get('id') || data.feed_id != this.story.get('story_feed_id'))) { return; diff --git a/media/js/vendor/backbone-1.0.0.js b/media/js/vendor/backbone-1.0.0.js index f7e809c07..8f5e0cf5c 100644 --- a/media/js/vendor/backbone-1.0.0.js +++ b/media/js/vendor/backbone-1.0.0.js @@ -1464,7 +1464,6 @@ // route callback be fired (not usually desirable), or `replace: true`, if // you wish to modify the current URL without adding an entry to the history. navigate: function(fragment, options) { - console.log(['backbone navigate', fragment, options]); if (!History.started) return false; if (!options || options === true) options = {trigger: options}; fragment = this.getFragment(fragment || ''); diff --git a/node/unread_counts.coffee b/node/unread_counts.coffee index f6e4e7e3c..1b72e2e16 100644 --- a/node/unread_counts.coffee +++ b/node/unread_counts.coffee @@ -59,12 +59,16 @@ io.on 'connection', (socket) -> " (#{io.engine.clientsCount} connected) " + " #{if SECURE then "(SSL)" else "(non-SSL)"}" socket.subscribe.subscribe @feeds + feeds_story = @feeds.map (f) -> "#{f}:story" + socket.subscribe.subscribe feeds_story socket.subscribe.subscribe @username socket.subscribe.on 'message', (channel, message) => log.info @username, "Update on #{channel}: #{message}" if channel == @username socket.emit 'user:update', channel, message + else if channel.indexOf(':story') >= 0 + socket.emit 'feed:story:new', channel, message else socket.emit 'feed:update', channel, message diff --git a/node/unread_counts.js b/node/unread_counts.js index 113cf6680..67d4e5e66 100644 --- a/node/unread_counts.js +++ b/node/unread_counts.js @@ -63,8 +63,13 @@ })(this)); socket.subscribe.on("connect", (function(_this) { return function() { + var feeds_story; log.info(_this.username, ("Connected (" + _this.feeds.length + " feeds, " + ip + "),") + (" (" + io.engine.clientsCount + " connected) ") + (" " + (SECURE ? "(SSL)" : "(non-SSL)"))); socket.subscribe.subscribe(_this.feeds); + feeds_story = _this.feeds.map(function(f) { + return "" + f + ":story"; + }); + socket.subscribe.subscribe(feeds_story); return socket.subscribe.subscribe(_this.username); }; })(this)); @@ -73,6 +78,8 @@ log.info(_this.username, "Update on " + channel + ": " + message); if (channel === _this.username) { return socket.emit('user:update', channel, message); + } else if (channel.indexOf(':story') >= 0) { + return socket.emit('feed:story:new', channel, message); } else { return socket.emit('feed:update', channel, message); } diff --git a/templates/reader/dashboard.xhtml b/templates/reader/dashboard.xhtml index 171549768..254f1c0c2 100644 --- a/templates/reader/dashboard.xhtml +++ b/templates/reader/dashboard.xhtml @@ -72,7 +72,7 @@ {% render_getting_started %} {% endif %} - {% if user.is_staff or user.username == "chrome" %} + {% if user.is_staff %}
diff --git a/utils/feed_fetcher.py b/utils/feed_fetcher.py index fcf2482f9..6f71766b9 100644 --- a/utils/feed_fetcher.py +++ b/utils/feed_fetcher.py @@ -954,7 +954,7 @@ class Dispatcher: logging.debug(" ---> [%-30s] ~FRIntegrityError on feed: %s" % (feed.title[:30], feed.feed_address,)) if ret_entries and ret_entries['new']: - self.publish_to_subscribers(feed) + self.publish_to_subscribers(feed, ret_entries['new']) done_msg = (u'%2s ---> [%-30s] ~FYProcessed in ~FM~SB%.4ss~FY~SN (~FB%s~FY) [%s]' % ( identity, feed.title[:30], delta, @@ -973,10 +973,10 @@ class Dispatcher: # time_taken = datetime.datetime.utcnow() - self.time_start - def publish_to_subscribers(self, feed): + def publish_to_subscribers(self, feed, new_count): try: r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL) - listeners_count = r.publish(str(feed.pk), 'story:new') + listeners_count = r.publish(str(feed.pk), 'story:new_count:%s' % new_count) if listeners_count: logging.debug(" ---> [%-30s] ~FMPublished to %s subscribers" % (feed.title[:30], listeners_count)) except redis.ConnectionError: