public-mirrors/NewsBlur-viq
1
0
Fork 0
mirror of https://github.com/viq/NewsBlur.git synced 2025-09-18 21:43:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Attempting to fix dashboard stories by no longer relying on full river reloads when a new story comes in but to fetch that story if it is newer (or older) than last visible dashboard story.

Browse source
This commit is contained in:
Samuel Clay 2016-12-13 16:29:42 -08:00
parent 5cfd564308
commit c5d5ea5001
14 changed files with 114 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
apps/newsletters/models.py
View file

@ -1,9 +1,6 @@
import datetime
import re
import redis
from cgi import escape
from django.db import models
from django.contrib.auth.models import User
from django.contrib.sites.models import Site
from django.core.mail import EmailMultiAlternatives
from django.core.urlresolvers import reverse
@ -87,7 +84,7 @@ class EmailNewsletter:
usersub.needs_unread_recalc = True
usersub.save()
self._publish_to_subscribers(feed)
self._publish_to_subscribers(feed, story.story_hash)
MFetchHistory.add(feed_id=feed.pk, fetch_type='push')
logging.user(user, "~FCNewsletter feed story: ~SB%s~SN / ~SB%s" % (story.story_title, feed))
@ -109,12 +106,12 @@ class EmailNewsletter:
params = dict(receiver_user_id=user.pk, email_type='first_newsletter')
try:
sent_email = MSentEmail.objects.get(**params)
MSentEmail.objects.get(**params)
if not force:
# Return if email already sent
return
except MSentEmail.DoesNotExist:
sent_email = MSentEmail.objects.create(**params)
MSentEmail.objects.create(**params)
text = render_to_string('mail/email_first_newsletter.txt', {})
html = render_to_string('mail/email_first_newsletter.xhtml', {})
@ -177,10 +174,10 @@ class EmailNewsletter:
content = content.replace('!important', '')
return content
def _publish_to_subscribers(self, feed):
def _publish_to_subscribers(self, feed, story_hash):
try:
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
listeners_count = r.publish(str(feed.pk), 'story:new')
listeners_count = r.publish(str(feed.pk), 'story:new:%s' % story_hash)
if listeners_count:
logging.debug(" ---> [%-30s] ~FMPublished to %s subscribers" % (feed.title[:30], listeners_count))
except redis.ConnectionError:

3
apps/reader/views.py
View file

@ -1227,6 +1227,7 @@ def load_river_stories__redis(request):
query = request.REQUEST.get('query', '').strip()
include_hidden = is_true(request.REQUEST.get('include_hidden', False))
include_feeds = is_true(request.REQUEST.get('include_feeds', False))
initial_dashboard = is_true(request.GET.get('initial_dashboard', False))
now = localtime_for_timezone(datetime.datetime.now(), user.profile.timezone)
usersubs = []
code = 1
@ -1234,7 +1235,7 @@ def load_river_stories__redis(request):
offset = (page-1) * limit
story_date_order = "%sstory_date" % ('' if order == 'oldest' else '-')
if limit == 4 and not request.user.is_staff:
if limit == 4 and not initial_dashboard:
logging.user(request, "~FRIgnoring ~FCdashboard river stories")
return dict(code=-1, message="Had to turn off dashboard river for now.", stories=[])

8
apps/rss_feeds/models.py
View file

@ -1169,6 +1169,7 @@ class Feed(models.Model):
try:
s.save()
ret_values['new'] += 1
s.publish_to_subscribers()
except (IntegrityError, OperationError), e:
ret_values['error'] += 1
if settings.DEBUG:
@ -2232,6 +2233,13 @@ class MStory(mongo.Document):
super(MStory, self).delete(*args, **kwargs)
def publish_to_subscribers(self):
try:
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish("%s:story" % (self.story_feed_id), '%s,%s' % (self.story_hash, self.story_date.strftime('%s')))
except redis.ConnectionError:
logging.debug(" ***> [%-30s] ~BMRedis is unavailable for real-time." % (Feed.get_by_id(self.story_feed_id).title[:30],))
@classmethod
def purge_feed_stories(cls, feed, cutoff, verbose=True):
stories = cls.objects(story_feed_id=feed.pk)

2
apps/social/models.py
View file

@ -1863,7 +1863,7 @@ class MSharedStory(mongo.DynamicDocument):
try:
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
feed_id = "social:%s" % self.user_id
listeners_count = r.publish(feed_id, 'story:new')
listeners_count = r.publish(feed_id, 'story:new:%s' % self.story_hash)
if listeners_count:
logging.debug(" ---> ~FMPublished to %s subscribers" % (listeners_count))
except redis.ConnectionError:

29
media/js/newsblur/common/assetmodel.js
View file

@ -569,6 +569,7 @@ NEWSBLUR.AssetModel = Backbone.Router.extend({
}
if (data.stories && first_load) {
// console.log(['first load river', data.stories.length, ' stories']);
this.feed_tags = data.feed_tags || {};
this.feed_authors = data.feed_authors || {};
this.active_feed = this.get_feed(feed_id);
@ -583,6 +584,7 @@ NEWSBLUR.AssetModel = Backbone.Router.extend({
this.starred_stories = data.starred_stories;
this.stories.reset(data.stories, {added: data.stories.length});
} else if (data.stories) {
// console.log(['adding to river', data.stories.length, ' stories']);
this.stories.add(data.stories, {silent: true});
this.stories.trigger('add', {added: data.stories.length});
}
@ -707,6 +709,12 @@ NEWSBLUR.AssetModel = Backbone.Router.extend({
fetch_dashboard_stories: function(feed_id, feeds, page, callback, error_callback) {
var self = this;
this.dashboard_stories.comparator = function(a, b) {
var a_time = parseInt(a.get('story_timestamp'), 10);
var b_time = parseInt(b.get('story_timestamp'), 10);
return a_time < b_time ? 1 : (a_time == b_time) ? 0 : -1;
};
var pre_callback = function(data) {
if (data.user_profiles) {
self.add_user_profiles(data.user_profiles);
@ -731,13 +739,32 @@ NEWSBLUR.AssetModel = Backbone.Router.extend({
order: this.view_setting(feed_id, 'order'),
read_filter: this.view_setting(feed_id, 'read_filter'),
include_hidden: false,
dashboard: true
dashboard: true,
initial_dashboard: true
}, pre_callback, error_callback, {
'ajax_group': 'dashboard',
'request_type': 'GET'
});
},
add_dashboard_story: function(story_hash) {
var self = this;
var pre_callback = function(data) {
self.dashboard_stories.add(data.stories, {silent: true});
self.dashboard_stories.limit(NEWSBLUR.Globals.is_premium ? 5 : 3);
self.dashboard_stories.trigger('reset', {added: 1});
};
this.make_request('/reader/river_stories', {
h: story_hash,
dashboard: true
}, pre_callback, null, {
'ajax_group': 'dashboard',
'request_type': 'GET'
});
},
fetch_river_blurblogs_stories: function(feed_id, page, options, callback, error_callback, first_load) {
var self = this;

6
media/js/newsblur/models/stories.js
View file

@ -258,7 +258,7 @@ NEWSBLUR.Models.Story = Backbone.Model.extend({
NEWSBLUR.Collections.Stories = Backbone.Collection.extend({
model: NEWSBLUR.Models.Story,
read_stories: [],
previous_stories_stack: [],
@ -459,6 +459,10 @@ NEWSBLUR.Collections.Stories = Backbone.Collection.extend({
});
},
limit: function(count) {
this.models = this.models.slice(0, count);
},
// ===========
// = Getters =
// ===========

37
media/js/newsblur/reader/reader.js
View file

@ -1541,7 +1541,7 @@
var $original_tabs = $('.task_view_page, .task_view_story');
var $page_tab = $('.task_view_page');
view = view || NEWSBLUR.assets.view_setting(feed_id);
console.log(['set_correct_story_view_for_feed', feed_id, view]);
// console.log(['set_correct_story_view_for_feed', feed_id, view]);
$original_tabs.removeClass('NB-disabled-page')
.removeClass('NB-disabled')
@ -1632,7 +1632,7 @@
$page_tab.addClass('NB-disabled');
}
console.log(['setting reader.story_view', view, " was:", this.story_view]);
// console.log(['setting reader.story_view', view, " was:", this.story_view]);
this.story_view = view;
},
@ -1963,9 +1963,12 @@
NEWSBLUR.assets.stories.reset(NEWSBLUR.assets.dashboard_stories.map(function(story) {
return story.toJSON();
}));
this.model.fetch_river_stories(this.active_feed, feeds, 1,
_.bind(this.post_open_river_stories, this), NEWSBLUR.app.taskbar_info.show_stories_error, false);
} else {
this.model.fetch_river_stories(this.active_feed, feeds, 1,
_.bind(this.post_open_river_stories, this), NEWSBLUR.app.taskbar_info.show_stories_error, true);
}
this.model.fetch_river_stories(this.active_feed, feeds, 1,
_.bind(this.post_open_river_stories, this), NEWSBLUR.app.taskbar_info.show_stories_error, true);
},
post_open_river_stories: function(data, first_load) {
@ -2323,7 +2326,7 @@
this.model.mark_feed_as_read([feed_id], cutoff_timestamp, direction,
feed_id == this.active_feed, _.bind(function() {
this.feeds_unread_count(feed_id);
this.feed_unread_count(feed_id);
}, this));
if (!direction && NEWSBLUR.assets.preference('markread_nextfeed') == 'nextfeed' &&
@ -2865,7 +2868,7 @@
var feed_id = this.active_story_view();
var feed = this.model.get_feed(feed_id);
view = view || this.story_view;
NEWSBLUR.log(['switch_taskbar_view', view, options.skip_save_type, feed]);
// NEWSBLUR.log(['switch_taskbar_view', view, options.skip_save_type, feed]);
if (view == 'page' && feed && feed.get('has_exception') &&
feed.get('exception_type') == 'page') {
@ -4756,12 +4759,19 @@
this.socket.on('feed:update', _.bind(function(feed_id, message) {
NEWSBLUR.log(['Real-time feed update', feed_id, message]);
this.feed_unread_count(feed_id, {
realtime: true,
new_story: true
realtime: true
});
}, this));
this.socket.removeAllListeners('feed:story:new');
this.socket.on('feed:story:new', _.bind(function(feed_id, message) {
var story_hash = message.split(',')[0];
var timestamp = message.split(',')[1];
// NEWSBLUR.log(['Real-time new story', feed_id, story_hash, timestamp]);
NEWSBLUR.app.dashboard_river.new_story(story_hash, timestamp);
}, this));
this.socket.removeAllListeners(NEWSBLUR.Globals.username);
this.socket.removeAllListeners("user:update");
this.socket.on('user:update', _.bind(function(username, message) {
@ -4974,18 +4984,9 @@
_.delay(_.bind(function() {
this.model.feed_unread_count(feed_id, options.callback);
if (options.new_story) {
NEWSBLUR.app.dashboard_river.load_stories();
}
}, this), Math.random() * delay);
},
feeds_unread_count: function(feed_ids, options) {
options = options || {};
this.model.feed_unread_count(feed_ids, options.callback);
},
update_interactions_count: function() {
this.model.interactions_count(function(data) {
NEWSBLUR.app.sidebar_header.update_interactions_count(data.interactions_count);

32
media/js/newsblur/views/dashboard_river.js
View file

@ -47,6 +47,7 @@ NEWSBLUR.Views.DashboardRiver = Backbone.View.extend({
// ==========
load_stories: function() {
// console.log(['dashboard river load_stories', this.page]);
// var feeds = NEWSBLUR.assets.folders.feed_ids_in_folder();
var feeds = this.feeds();
if (!feeds.length) return;
@ -54,7 +55,7 @@ NEWSBLUR.Views.DashboardRiver = Backbone.View.extend({
this.page = 1;
this.story_titles.show_loading();
NEWSBLUR.assets.fetch_dashboard_stories("river:", feeds, this.page,
NEWSBLUR.assets.fetch_dashboard_stories(this.active_feed, feeds, this.page,
_.bind(this.post_load_stories, this), NEWSBLUR.app.taskbar_info.show_stories_error);
},
@ -74,7 +75,7 @@ NEWSBLUR.Views.DashboardRiver = Backbone.View.extend({
var counts = NEWSBLUR.assets.folders.unread_counts();
var unread_view = NEWSBLUR.assets.preference('unread_view');
if (unread_view >= 1) {
console.log(['counts', counts['ps'], visible, this.page]);
// console.log(['counts', counts['ps'], visible, this.page]);
if (counts['ps'] <= visible) {
this.show_end_line();
return;
@ -91,7 +92,7 @@ NEWSBLUR.Views.DashboardRiver = Backbone.View.extend({
var feeds = this.feeds();
this.page += 1;
this.story_titles.show_loading();
NEWSBLUR.assets.fetch_dashboard_stories("river:", feeds, this.page,
NEWSBLUR.assets.fetch_dashboard_stories(this.active_feed, feeds, this.page,
_.bind(this.post_load_stories, this), NEWSBLUR.app.taskbar_info.show_stories_error);
},
@ -118,6 +119,31 @@ NEWSBLUR.Views.DashboardRiver = Backbone.View.extend({
show_end_line: function() {
this.story_titles.show_no_more_stories();
this.$(".NB-end-line").addClass("NB-visible");
},
new_story: function(story_hash, timestamp) {
var oldest_story = NEWSBLUR.assets.dashboard_stories.last();
if (oldest_story) {
var last_timestamp = parseInt(oldest_story.get('story_timestamp'), 10);
timestamp = parseInt(timestamp, 10);
if (NEWSBLUR.assets.view_setting(this.active_feed, 'order') == 'newest') {
if (timestamp < last_timestamp) {
// console.log(['New story older than last/oldest dashboard story', timestamp, '<', last_timestamp]);
return;
}
} else {
if (timestamp > last_timestamp) {
// console.log(['New story older than last/newest dashboard story', timestamp, '<', last_timestamp]);
return;
}
}
}
console.log(['Fetching dashboard story', story_hash]);
NEWSBLUR.assets.add_dashboard_story(story_hash);
}
});

2
media/js/newsblur/views/text_tab_view.js
View file

@ -60,6 +60,8 @@ NEWSBLUR.Views.TextTabView = Backbone.View.extend({
},
render: function(data) {
if (!this.story) return;
if (data && (data.story_id != this.story.get('id') ||
data.feed_id != this.story.get('story_feed_id'))) {
return;

1
media/js/vendor/backbone-1.0.0.js vendored
View file

@ -1464,7 +1464,6 @@
// route callback be fired (not usually desirable), or `replace: true`, if
// you wish to modify the current URL without adding an entry to the history.
navigate: function(fragment, options) {
console.log(['backbone navigate', fragment, options]);
if (!History.started) return false;
if (!options || options === true) options = {trigger: options};
fragment = this.getFragment(fragment || '');

4
node/unread_counts.coffee
View file

@ -59,12 +59,16 @@ io.on 'connection', (socket) ->
" (#{io.engine.clientsCount} connected) " +
" #{if SECURE then "(SSL)" else "(non-SSL)"}"
socket.subscribe.subscribe @feeds
feeds_story = @feeds.map (f) -> "#{f}:story"
socket.subscribe.subscribe feeds_story
socket.subscribe.subscribe @username
socket.subscribe.on 'message', (channel, message) =>
log.info @username, "Update on #{channel}: #{message}"
if channel == @username
socket.emit 'user:update', channel, message
else if channel.indexOf(':story') >= 0
socket.emit 'feed:story:new', channel, message
else
socket.emit 'feed:update', channel, message

7
node/unread_counts.js
View file

@ -63,8 +63,13 @@
})(this));
socket.subscribe.on("connect", (function(_this) {
return function() {
var feeds_story;
log.info(_this.username, ("Connected (" + _this.feeds.length + " feeds, " + ip + "),") + (" (" + io.engine.clientsCount + " connected) ") + (" " + (SECURE ? "(SSL)" : "(non-SSL)")));
socket.subscribe.subscribe(_this.feeds);
feeds_story = _this.feeds.map(function(f) {
return "" + f + ":story";
});
socket.subscribe.subscribe(feeds_story);
return socket.subscribe.subscribe(_this.username);
};
})(this));
@ -73,6 +78,8 @@
log.info(_this.username, "Update on " + channel + ": " + message);
if (channel === _this.username) {
return socket.emit('user:update', channel, message);
} else if (channel.indexOf(':story') >= 0) {
return socket.emit('feed:story:new', channel, message);
} else {
return socket.emit('feed:update', channel, message);
}

2
templates/reader/dashboard.xhtml
View file

@ -72,7 +72,7 @@
{% render_getting_started %}
{% endif %}
{% if user.is_staff or user.username == "chrome" %}
{% if user.is_staff %}
<div class="NB-module NB-module-river">
<h5 class="NB-module-header">
<div class="NB-module-river-settings NB-javascript"></div>

6
utils/feed_fetcher.py
View file

@ -954,7 +954,7 @@ class Dispatcher:
logging.debug(" ---> [%-30s] ~FRIntegrityError on feed: %s" % (feed.title[:30], feed.feed_address,))
if ret_entries and ret_entries['new']:
self.publish_to_subscribers(feed)
self.publish_to_subscribers(feed, ret_entries['new'])
done_msg = (u'%2s ---> [%-30s] ~FYProcessed in ~FM~SB%.4ss~FY~SN (~FB%s~FY) [%s]' % (
identity, feed.title[:30], delta,
@ -973,10 +973,10 @@ class Dispatcher:
# time_taken = datetime.datetime.utcnow() - self.time_start
def publish_to_subscribers(self, feed):
def publish_to_subscribers(self, feed, new_count):
try:
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
listeners_count = r.publish(str(feed.pk), 'story:new')
listeners_count = r.publish(str(feed.pk), 'story:new_count:%s' % new_count)
if listeners_count:
logging.debug(" ---> [%-30s] ~FMPublished to %s subscribers" % (feed.title[:30], listeners_count))
except redis.ConnectionError: