public-mirrors/NewsBlur-viq
1
0
Fork 0
mirror of https://github.com/viq/NewsBlur.git synced 2025-09-18 21:43:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' of github.com:samuelclay/NewsBlur

Browse source 
* 'master' of github.com:samuelclay/NewsBlur:
  Update media/js/newsblur/reader.js
  Upgrading feedparser from 5.0.1 to 5.1.
  Catching errors in page links with ill-formed urls.
  Not showing the world favicon until favicons are downloaded.
  Fixing feed exception dialog to actually reload feeds on the page.
  Geometrically decaying feed fetches on known good feeds that have gone bad.
  Fixing issue around empty original pages.
  New preference: action to perform when opening a feed -- jump to either newest or oldest story automatically.
  Style tweaks for Windows.
This commit is contained in:
Samuel Clay 2012-02-24 08:01:56 -08:00
commit 7152cd335a
16 changed files with 1292 additions and 1114 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
apps/reader/views.py
View file

@ -274,7 +274,7 @@ def load_feeds_flat(request):
data = dict(flat_folders=flat_folders, feeds=feeds, user=user.username, iphone_version=iphone_version)
return data
@ratelimit(minutes=1, requests=10)
@ratelimit(minutes=1, requests=20)
@json.json_view
def refresh_feeds(request):
start = datetime.datetime.utcnow()

26
apps/rss_feeds/models.py
View file

@ -304,9 +304,7 @@ class Feed(models.Model):
# for history in old_fetch_histories:
# history.delete()
if status_code not in (200, 304):
fetch_history = map(lambda h: h.status_code,
MFeedFetchHistory.objects(feed_id=self.pk)[:50])
self.count_errors_in_history(fetch_history, status_code, 'feed')
self.count_errors_in_history('feed', status_code)
elif self.has_feed_exception:
self.has_feed_exception = False
self.active = True
@ -323,15 +321,16 @@ class Feed(models.Model):
# history.delete()
if status_code not in (200, 304):
fetch_history = map(lambda h: h.status_code,
MPageFetchHistory.objects(feed_id=self.pk)[:50])
self.count_errors_in_history(fetch_history, status_code, 'page')
self.count_errors_in_history('page', status_code)
elif self.has_page_exception:
self.has_page_exception = False
self.active = True
self.save()
def count_errors_in_history(self, fetch_history, status_code, exception_type):
def count_errors_in_history(self, exception_type='feed', status_code=None):
history_class = MFeedFetchHistory if exception_type == 'feed' else MPageFetchHistory
fetch_history = map(lambda h: h.status_code,
history_class.objects(feed_id=self.pk)[:50])
non_errors = [h for h in fetch_history if int(h) in (200, 304)]
errors = [h for h in fetch_history if int(h) not in (200, 304)]
@ -341,12 +340,14 @@ class Feed(models.Model):
self.active = False
elif exception_type == 'page':
self.has_page_exception = True
self.exception_code = status_code
self.exception_code = status_code or int(errors[0])
self.save()
elif self.exception_code > 0:
self.active = True
self.exception_code = 0
self.save()
return errors, non_errors
def count_subscribers(self, verbose=False):
SUBSCRIBER_EXPIRE = datetime.datetime.now() - datetime.timedelta(days=settings.SUBSCRIBER_EXPIRE)
@ -1006,9 +1007,12 @@ class Feed(models.Model):
return total, random_factor*2
def set_next_scheduled_update(self):
def set_next_scheduled_update(self, multiplier=1):
total, random_factor = self.get_next_scheduled_update(force=True, verbose=False)
if multiplier > 1:
total = total * multiplier
next_scheduled_update = datetime.datetime.utcnow() + datetime.timedelta(
minutes = total + random_factor)
@ -1022,6 +1026,10 @@ class Feed(models.Model):
self.save()
def schedule_feed_fetch_geometrically(self):
errors, non_errors = self.count_errors_in_history('feed')
self.set_next_scheduled_update(multiplier=len(errors))
# def calculate_collocations_story_content(self,
# collocation_measures=TrigramAssocMeasures,
# collocation_finder=TrigramCollocationFinder):

15
apps/rss_feeds/page_importer.py
View file

@ -63,9 +63,18 @@ class PageImporter(object):
self.save_no_page()
return
else:
data = open(feed_link, 'r').read()
html = self.rewrite_page(data)
self.save_page(html)
try:
data = open(feed_link, 'r').read()
except IOError:
self.feed.feed_link = 'http://' + feed_link
self.fetch_page(urllib_fallback=True)
return
if data:
html = self.rewrite_page(data)
self.save_page(html)
else:
self.save_no_page()
return
except (ValueError, urllib2.URLError, httplib.BadStatusLine, httplib.InvalidURL), e:
self.feed.save_page_history(401, "Bad URL", e)
fp = feedparser.parse(self.feed.feed_address)

15
config/com.redis.plist Normal file
View file

@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key>
<string>com.redis</string>
<key>RunAtLoad</key>
<true/>
<key>ProgramArguments</key>
<array>
<string>/usr/local/bin/redis-server</string>
<string>/Users/sclay/projects/newsblur/config/redis.conf</string>
</array>
</dict>
</plist>

6
config/mongodb.dev.conf
View file

@ -5,12 +5,10 @@
# Note: if you run mongodb as a non-root user (recommended) you may
# need to create and set permissions for this directory manually,
# e.g., if the parent directory isn't mutable by the mongodb user.
dbpath=/Users/conesus/newsblur/data/db/unsharded
dbpath=/Users/sclay/projects/data/db/unsharded
#where to log
logpath=/Users/conesus/newsblur/data/unsharded.log
logappend=false
logpath=/Users/sclay/projects/data/unsharded.log
#port = 27017

6
fabfile.py vendored
View file

@ -303,11 +303,11 @@ def setup_libxml_code():
run('./configure && make && sudo make install')
def setup_psycopg():
sudo('easy_install psycopg2')
sudo('easy_install -U psycopg2')
def setup_python():
sudo('easy_install pip')
sudo('easy_install fabric django celery django-celery django-compress South django-extensions pymongo BeautifulSoup pyyaml nltk==0.9.9 lxml oauth2 pytz boto seacucumber django_ses mongoengine redis requests')
sudo('easy_install -U pip')
sudo('easy_install -U fabric django readline pyflakes iconv celery django-celery django-compress South django-extensions pymongo BeautifulSoup pyyaml nltk==0.9.9 lxml oauth2 pytz boto seacucumber django_ses mongoengine redis requests')
put('config/pystartup.py', '.pystartup')
with cd(os.path.join(env.NEWSBLUR_PATH, 'vendor/cjson')):

0
logs/__init__.py Normal file
View file

2
media/css/modals.css
View file

@ -89,7 +89,7 @@
border-bottom: 1px solid #A1A9CF;
color:#505050;
font-size: 14px;
font-family: "Gill Sans", inherit;
font-family: "Gill Sans", "Lucida Sans", "Lucida Grande", Verdana, Arial, Helvetica, sans-serif, inherit;
font-weight: normal;
text-transform: uppercase;
position: relative;

4
media/css/reader.css
View file

@ -3060,7 +3060,7 @@ a.NB-splash-link:hover {
text-transform: none;
position: absolute;
right: 8px;
bottom: 4px;
top: 4px;
}
.NB-add .NB-add-folders select {
@ -3078,7 +3078,7 @@ a.NB-splash-link:hover {
}
.NB-add input[type=text] {
width: 308px;
width: 300px;
margin-bottom: 16px;
}
.NB-add .NB-add-add-folder input[type=text] {

4
media/js/newsblur/assetmodel.js
View file

@ -856,7 +856,7 @@ NEWSBLUR.AssetModel.Reader.prototype = {
var pre_callback = function(data) {
// NEWSBLUR.log(['save_exception_change_feed_link pre_callback', feed_id, feed_link, data]);
self.post_refresh_feeds(data, callback);
NEWSBLUR.reader.force_feed_refresh(feed_id, null, data.new_feed_id);
NEWSBLUR.reader.force_feed_refresh(feed_id, data.new_feed_id);
};
if (NEWSBLUR.Globals.is_authenticated) {
@ -875,7 +875,7 @@ NEWSBLUR.AssetModel.Reader.prototype = {
var pre_callback = function(data) {
// NEWSBLUR.log(['save_exception_change_feed_address pre_callback', feed_id, feed_address, data]);
self.post_refresh_feeds(data, callback);
NEWSBLUR.reader.force_feed_refresh(feed_id, null, data.new_feed_id);
NEWSBLUR.reader.force_feed_refresh(feed_id, data.new_feed_id);
};
if (NEWSBLUR.Globals.is_authenticated) {

59
media/js/newsblur/reader.js
View file

@ -41,6 +41,7 @@
};
this.locks = {};
this.counts = {
'page': 1,
'feature_page': 0,
'unfetched_feeds': 0,
'fetched_feeds': 0,
@ -754,7 +755,8 @@
this.open_story(story, $next_story);
this.scroll_story_titles_to_show_selected_story_title($next_story);
}
} else if (this.counts['find_next_unread_on_page_of_feed_stories_load'] < this.constants.FILL_OUT_PAGES) {
} else if (this.counts['find_next_unread_on_page_of_feed_stories_load'] < this.constants.FILL_OUT_PAGES &&
!this.flags['no_more_stories']) {
// Nothing up, nothing down, but still unread. Load 1 page then find it.
this.counts['find_next_unread_on_page_of_feed_stories_load'] += 1;
this.load_page_of_feed_stories();
@ -808,7 +810,8 @@
this.open_story(story, $next_story);
this.scroll_story_titles_to_show_selected_story_title($next_story);
}
} else if (this.counts['find_last_unread_on_page_of_feed_stories_load'] < this.constants.FILL_OUT_PAGES) {
} else if (this.counts['find_last_unread_on_page_of_feed_stories_load'] < this.constants.FILL_OUT_PAGES &&
!this.flags['no_more_stories']) {
// Nothing up, nothing down, but still unread. Load 1 page then find it.
this.counts['find_last_unread_on_page_of_feed_stories_load'] += 1;
this.load_page_of_feed_stories();
@ -1009,6 +1012,18 @@
}
},
find_story_with_action_preference_on_open_feed: function() {
var open_feed_action = this.model.preference('open_feed_action');
console.log(["action_preference_on_open_feed", open_feed_action, this.counts['page']]);
if (this.counts['page'] != 1) return;
if (open_feed_action == 'newest') {
this.show_next_unread_story();
} else if (open_feed_action == 'oldest') {
this.show_last_unread_story();
}
},
// =============
// = Feed Pane =
// =============
@ -1111,7 +1126,7 @@
return -1;
} else if (!feedA && feedB) {
return 1;
} else if (!feedA && !feedB && a && b && !_.isNumber(a) && !_.isNumber(b) && !(a.e instanceof jQuery) && (!b.e instanceof jQuery)) {
} else if (!feedA && !feedB && a && b && !_.isNumber(a) && !_.isNumber(b) && !(a.e instanceof jQuery) && !(b.e instanceof jQuery)) {
// console.log(['a b 1', a, b, feedA, feedB]);
var folderA = _.keys(a)[0];
var folderB = _.keys(b)[0];
@ -1126,7 +1141,7 @@
return -1;
} else if (!feedA && feedB) {
return 1;
} else if (!feedA && !feedB && a && b && !_.isNumber(a) && !_.isNumber(b) && !(a.e instanceof jQuery) && (!b.e instanceof jQuery)) {
} else if (!feedA && !feedB && a && b && !_.isNumber(a) && !_.isNumber(b) && !(a.e instanceof jQuery) && !(b.e instanceof jQuery)) {
// console.log(['a b 2', a, b]);
var folderA = _.keys(a)[0];
var folderB = _.keys(b)[0];
@ -1225,7 +1240,7 @@
<div class="feed_counts">\
<%= feed_counts_floater %>\
</div>\
<img class="feed_favicon" src="<%= $.favicon(feed.favicon, true) %>">\
<img class="feed_favicon" src="<%= $.favicon(feed.favicon, !favicons_downloaded) %>">\
<span class="feed_title">\
<%= feed.feed_title %>\
<% if (type == "story") { %>\
@ -1257,7 +1272,8 @@
unread_class : unread_class,
exception_class : exception_class,
toplevel : depth == 0,
list_type : type == 'feed' ? 'li' : 'div'
list_type : type == 'feed' ? 'li' : 'div',
favicons_downloaded : this.flags['favicons_downloaded']
});
return $feed;
@ -1676,7 +1692,8 @@
'feed_list_showing_manage_menu': false,
'unread_threshold_temporarily': null,
'river_view': false,
'non_premium_river_view': false
'non_premium_river_view': false,
'no_more_stories': false
});
$.extend(this.cache, {
@ -1700,6 +1717,7 @@
});
$.extend(this.counts, {
'page': 1,
'page_fill_outs': 0,
'find_next_unread_on_page_of_feed_stories_load': 0,
'find_last_unread_on_page_of_feed_stories_load': 0
@ -1707,7 +1725,6 @@
this.active_feed = null;
this.active_story = null;
this.$s.$story_titles.data('page', 1);
this.$s.$story_titles.data('feed_id', null);
this.$s.$feed_stories.scrollTop(0);
this.$s.$feed_stories.empty();
@ -1741,7 +1758,6 @@
this.next_feed = feed_id;
this.show_stories_progress_bar();
$story_titles.data('page', 1);
$story_titles.data('feed_id', feed_id);
this.iframe_scroll = null;
this.set_correct_story_view_for_feed(feed_id);
@ -1792,6 +1808,7 @@
this.story_titles_clear_loading_endbar();
this.create_story_titles(stories);
this.make_story_feed_entries(stories, first_load);
this.find_story_with_action_preference_on_open_feed();
this.show_feed_hidden_story_title_indicator(true);
this.show_story_titles_above_intelligence_level({'animate': false});
this.fill_out_story_titles();
@ -1959,7 +1976,6 @@
this.hide_splash_page();
this.active_feed = 'starred';
$story_titles.data('page', 1);
$story_titles.data('feed_id', null);
this.iframe_scroll = null;
this.mark_feed_as_selected(null, null);
@ -2016,7 +2032,6 @@
$folder.addClass('NB-selected');
}
$story_titles.data('page', 1);
$story_titles.data('feed_id', null);
this.iframe_scroll = null;
this.flags['opening_feed'] = true;
@ -2062,6 +2077,7 @@
this.story_titles_clear_loading_endbar();
this.create_story_titles(data.stories, {'river_stories': true});
this.make_story_feed_entries(data.stories, first_load, {'river_stories': true});
this.find_story_with_action_preference_on_open_feed();
this.show_story_titles_above_intelligence_level({'animate': false});
// $('.NB-feedbar-last-updated-date').text(data.last_update + ' ago');
this.flags['story_titles_loaded'] = true;
@ -3183,23 +3199,23 @@
options = $.extend({'show_loading': true}, options);
var $story_titles = this.$s.$story_titles;
var feed_id = $story_titles.data('feed_id');
var page = $story_titles.data('page');
if (!this.flags['opening_feed']) {
this.flags.pre_load_page_scroll_position = $('#story_titles').scrollTop();
if (options.show_loading) this.show_feedbar_loading();
this.flags.post_load_page_scroll_position = $('#story_titles').scrollTop();
// console.log(["setting scroll positions", this.flags.pre_load_page_scroll_position, this.flags.post_load_page_scroll_position]);
$story_titles.data('page', page+1);
this.flags['opening_feed'] = true;
this.counts['page'] += 1;
if (this.active_feed == 'starred') {
this.model.fetch_starred_stories(page+1, _.bind(this.post_open_starred_stories, this),
this.model.fetch_starred_stories(this.counts['page'], _.bind(this.post_open_starred_stories, this),
this.show_stories_error, false);
} else if (this.flags['river_view']) {
this.model.fetch_river_stories(this.active_feed, this.cache['river_feeds_with_unreads'],
page+1, _.bind(this.post_open_river_stories, this),
this.counts['page'], _.bind(this.post_open_river_stories, this),
this.show_stories_error, false);
} else {
this.model.load_feed(feed_id, page+1, false,
this.model.load_feed(feed_id, this.counts['page'], false,
$.rescope(this.post_open_feed, this), this.show_stories_error);
}
}
@ -3216,7 +3232,8 @@
($last.length == 0 ||
($('#story_titles').scrollTop() == 0 &&
$last.position().top + $last.height() - 13 < container_height))) {
if (this.counts['page_fill_outs'] < this.constants.FILL_OUT_PAGES) {
if (this.counts['page_fill_outs'] < this.constants.FILL_OUT_PAGES &&
!this.flags['no_more_stories']) {
this.counts['page_fill_outs'] += 1;
// this.check_story_titles_last_story();
// _.delay(_.bind(this.load_page_of_feed_stories, this, {'show_loading': false}), 250);
@ -3903,6 +3920,8 @@
if (!($('.NB-story-titles-end-stories-line', $story_titles).length)) {
$story_titles.append($end_stories_line);
}
this.flags['no_more_stories'] = true;
},
append_river_premium_only_notification: function() {
@ -5198,7 +5217,7 @@
var $feed = this.find_feed_in_feed_list(feed_id);
$feed.addClass('NB-feed-unfetched').removeClass('NB-feed-exception');
this.model.save_exception_retry(feed_id, _.bind(this.force_feed_refresh, this, feed_id, $feed), this.show_stories_error);
this.model.save_exception_retry(feed_id, _.bind(this.force_feed_refresh, this, feed_id), this.show_stories_error);
},
setup_socket_realtime_unread_counts: function(force) {
@ -5257,10 +5276,10 @@
}, refresh_interval);
},
force_feed_refresh: function(feed_id, $feed, new_feed_id) {
force_feed_refresh: function(feed_id, new_feed_id) {
var self = this;
feed_id = feed_id || this.active_feed;
$feed = $feed || this.find_feed_in_feed_list(feed_id);
var $feed = this.find_feed_in_feed_list(feed_id);
new_feed_id = new_feed_id || feed_id;
this.force_feeds_refresh(function(feeds) {

13
media/js/newsblur/reader_feed_exception.js
View file

@ -223,15 +223,14 @@ _.extend(NEWSBLUR.ReaderFeedException.prototype, {
},
save_retry_feed: function() {
var self = this;
var $loading = $('.NB-modal-loading', this.$modal);
$loading.addClass('NB-active');
var feed_id = this.feed_id;
$('.NB-modal-submit-retry', this.$modal).addClass('NB-disabled').attr('value', 'Fetching...');
this.model.save_exception_retry(this.feed_id, function() {
// NEWSBLUR.reader.flags['has_unfetched_feeds'] = true;
// NEWSBLUR.reader.force_instafetch_stories(self.feed_id);
this.model.save_exception_retry(feed_id, function() {
NEWSBLUR.reader.force_feed_refresh(feed_id);
$.modal.close();
});
},
@ -261,8 +260,7 @@ _.extend(NEWSBLUR.ReaderFeedException.prototype, {
if (feed_address.length) {
this.model.save_exception_change_feed_address(feed_id, feed_address, function(code) {
// NEWSBLUR.reader.flags['has_unfetched_feeds'] = true;
// NEWSBLUR.reader.load_feeds();
NEWSBLUR.reader.force_feed_refresh(feed_id);
$.modal.close();
});
}
@ -279,8 +277,7 @@ _.extend(NEWSBLUR.ReaderFeedException.prototype, {
if (feed_link.length) {
this.model.save_exception_change_feed_link(feed_id, feed_link, function(code) {
// NEWSBLUR.reader.flags['has_unfetched_feeds'] = true;
// NEWSBLUR.reader.load_feeds();
NEWSBLUR.reader.force_feed_refresh(feed_id);
$.modal.close();
});
}

31
media/js/newsblur/reader_preferences.js
View file

@ -268,6 +268,31 @@ _.extend(NEWSBLUR.ReaderPreferences.prototype, {
'Window title'
])
]),
$.make('div', { className: 'NB-preference NB-preference-openfeedaction' }, [
$.make('div', { className: 'NB-preference-options' }, [
$.make('div', [
$.make('input', { id: 'NB-preference-openfeedaction-1', type: 'radio', name: 'open_feed_action', value: 'newest' }),
$.make('label', { 'for': 'NB-preference-openfeedaction-1' }, [
'Open the newest unread story'
])
]),
$.make('div', [
$.make('input', { id: 'NB-preference-openfeedaction-2', type: 'radio', name: 'open_feed_action', value: 'oldest' }),
$.make('label', { 'for': 'NB-preference-openfeedaction-2' }, [
'Open the oldest unread story'
])
]),
$.make('div', [
$.make('input', { id: 'NB-preference-openfeedaction-0', type: 'radio', name: 'open_feed_action', value: 0 }),
$.make('label', { 'for': 'NB-preference-openfeedaction-0' }, [
'Show all stories'
])
])
]),
$.make('div', { className: 'NB-preference-label'}, [
'When opening a site'
])
]),
$.make('div', { className: 'NB-preference NB-preference-hidestorychanges' }, [
$.make('div', { className: 'NB-preference-options' }, [
$.make('div', [
@ -515,6 +540,12 @@ _.extend(NEWSBLUR.ReaderPreferences.prototype, {
return false;
}
});
$('input[name=open_feed_action]', this.$modal).each(function() {
if ($(this).val() == NEWSBLUR.Preferences.open_feed_action) {
$(this).attr('checked', true);
return false;
}
});
$('input[name=hide_story_changes]', this.$modal).each(function() {
if ($(this).val() == NEWSBLUR.Preferences.hide_story_changes) {
$(this).attr('checked', true);

1
templates/base.html
View file

@ -36,6 +36,7 @@
'hide_read_feeds' : 0,
'show_tooltips' : 1,
'feed_order' : 'ALPHABETICAL',
'open_feed_action' : 0,
'hide_story_changes' : 1,
'feed_view_single_story' : 0,
'animations' : true,

2
utils/feed_fetcher.py
View file

@ -144,7 +144,7 @@ class ProcessFeed:
self.feed.save_feed_history(self.fpf.status, "HTTP Error")
else:
self.feed.has_feed_exception = True
self.feed.schedule_feed_fetch_immediately()
self.feed.schedule_feed_fetch_geometrically()
self.feed.save()
return FEED_ERRHTTP, ret_values

2220
utils/feedparser.py
View file

File diff suppressed because it is too large Load diff