public-mirrors/NewsBlur
1
0
Fork 0
mirror of https://github.com/samuelclay/NewsBlur.git synced 2025-09-18 21:50:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into social

Browse source 
* master:
  Adding asset-less deployment.
  Upgrading feedparser from 5.0.1 to 5.1.
  Catching errors in page links with ill-formed urls.
  Not showing the world favicon until favicons are downloaded.
  Fixing feed exception dialog to actually reload feeds on the page.
  Geometrically decaying feed fetches on known good feeds that have gone bad.
  Fixing issue around empty original pages.

Conflicts:
	media/js/newsblur/reader/reader.js
This commit is contained in:
Samuel Clay 2012-02-23 15:40:38 -08:00
commit 307e36622b
12 changed files with 1229 additions and 1099 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
apps/reader/views.py
View file

@ -292,7 +292,7 @@ def load_feeds_flat(request):
data = dict(flat_folders=flat_folders, feeds=feeds, user=user.username, iphone_version=iphone_version)
return data
@ratelimit(minutes=1, requests=10)
@ratelimit(minutes=1, requests=20)
@json.json_view
def refresh_feeds(request):
start = datetime.datetime.utcnow()

26
apps/rss_feeds/models.py
View file

@ -307,9 +307,7 @@ class Feed(models.Model):
# for history in old_fetch_histories:
# history.delete()
if status_code not in (200, 304):
fetch_history = map(lambda h: h.status_code,
MFeedFetchHistory.objects(feed_id=self.pk)[:50])
self.count_errors_in_history(fetch_history, status_code, 'feed')
self.count_errors_in_history('feed', status_code)
elif self.has_feed_exception:
self.has_feed_exception = False
self.active = True
@ -326,15 +324,16 @@ class Feed(models.Model):
# history.delete()
if status_code not in (200, 304):
fetch_history = map(lambda h: h.status_code,
MPageFetchHistory.objects(feed_id=self.pk)[:50])
self.count_errors_in_history(fetch_history, status_code, 'page')
self.count_errors_in_history('page', status_code)
elif self.has_page_exception:
self.has_page_exception = False
self.active = True
self.save()
def count_errors_in_history(self, fetch_history, status_code, exception_type):
def count_errors_in_history(self, exception_type='feed', status_code=None):
history_class = MFeedFetchHistory if exception_type == 'feed' else MPageFetchHistory
fetch_history = map(lambda h: h.status_code,
history_class.objects(feed_id=self.pk)[:50])
non_errors = [h for h in fetch_history if int(h) in (200, 304)]
errors = [h for h in fetch_history if int(h) not in (200, 304)]
@ -344,12 +343,14 @@ class Feed(models.Model):
self.active = False
elif exception_type == 'page':
self.has_page_exception = True
self.exception_code = status_code
self.exception_code = status_code or int(errors[0])
self.save()
elif self.exception_code > 0:
self.active = True
self.exception_code = 0
self.save()
return errors, non_errors
def count_subscribers(self, verbose=False):
SUBSCRIBER_EXPIRE = datetime.datetime.now() - datetime.timedelta(days=settings.SUBSCRIBER_EXPIRE)
@ -1016,9 +1017,12 @@ class Feed(models.Model):
return total, random_factor*2
def set_next_scheduled_update(self):
def set_next_scheduled_update(self, multiplier=1):
total, random_factor = self.get_next_scheduled_update(force=True, verbose=False)
if multiplier > 1:
total = total * multiplier
next_scheduled_update = datetime.datetime.utcnow() + datetime.timedelta(
minutes = total + random_factor)
@ -1032,6 +1036,10 @@ class Feed(models.Model):
self.save()
def schedule_feed_fetch_geometrically(self):
errors, non_errors = self.count_errors_in_history('feed')
self.set_next_scheduled_update(multiplier=len(errors))
# def calculate_collocations_story_content(self,
# collocation_measures=TrigramAssocMeasures,
# collocation_finder=TrigramCollocationFinder):

15
apps/rss_feeds/page_importer.py
View file

@ -63,9 +63,18 @@ class PageImporter(object):
self.save_no_page()
return
else:
data = open(feed_link, 'r').read()
html = self.rewrite_page(data)
self.save_page(html)
try:
data = open(feed_link, 'r').read()
except IOError:
self.feed.feed_link = 'http://' + feed_link
self.fetch_page(urllib_fallback=True)
return
if data:
html = self.rewrite_page(data)
self.save_page(html)
else:
self.save_no_page()
return
except (ValueError, urllib2.URLError, httplib.BadStatusLine, httplib.InvalidURL), e:
self.feed.save_page_history(401, "Bad URL", e)
fp = feedparser.parse(self.feed.feed_address)

15
config/com.redis.plist Normal file
View file

@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key>
<string>com.redis</string>
<key>RunAtLoad</key>
<true/>
<key>ProgramArguments</key>
<array>
<string>/usr/local/bin/redis-server</string>
<string>/Users/sclay/projects/newsblur/config/redis.conf</string>
</array>
</dict>
</plist>

6
config/mongodb.dev.conf
View file

@ -5,12 +5,10 @@
# Note: if you run mongodb as a non-root user (recommended) you may
# need to create and set permissions for this directory manually,
# e.g., if the parent directory isn't mutable by the mongodb user.
dbpath=/Users/conesus/newsblur/data/db/unsharded
dbpath=/Users/sclay/projects/data/db/unsharded
#where to log
logpath=/Users/conesus/newsblur/data/unsharded.log
logappend=false
logpath=/Users/sclay/projects/data/unsharded.log
#port = 27017

14
fabfile.py vendored
View file

@ -77,20 +77,22 @@ def post_deploy():
@parallel
def deploy():
deploy_code()
deploy_code(copy_assets=True)
post_deploy()
def deploy_full():
deploy_code(full=True)
post_deploy()
def deploy_code(full=False):
@parallel
def deploy_code(copy_assets=False, full=False):
with cd(env.NEWSBLUR_PATH):
run('git pull')
run('mkdir -p static')
if full:
run('rm -fr static/*')
transfer_assets()
if copy_assets:
transfer_assets()
if full:
with settings(warn_only=True):
run('sudo supervisorctl restart gunicorn')
@ -303,11 +305,11 @@ def setup_libxml_code():
run('./configure && make && sudo make install')
def setup_psycopg():
sudo('easy_install psycopg2')
sudo('easy_install -U psycopg2')
def setup_python():
sudo('easy_install pip')
sudo('easy_install fabric django celery django-celery django-compress South django-extensions pymongo BeautifulSoup pyyaml nltk==0.9.9 lxml oauth2 pytz boto seacucumber django_ses mongoengine redis requests')
sudo('easy_install -U pip')
sudo('easy_install -U fabric django readline pyflakes iconv celery django-celery django-compress South django-extensions pymongo BeautifulSoup pyyaml nltk==0.9.9 lxml oauth2 pytz boto seacucumber django_ses mongoengine redis requests')
put('config/pystartup.py', '.pystartup')
with cd(os.path.join(env.NEWSBLUR_PATH, 'vendor/cjson')):

0
logs/__init__.py Normal file
View file

4
media/js/newsblur/common/assetmodel.js
View file

@ -959,7 +959,7 @@ NEWSBLUR.AssetModel.Reader.prototype = {
var pre_callback = function(data) {
// NEWSBLUR.log(['save_exception_change_feed_link pre_callback', feed_id, feed_link, data]);
self.post_refresh_feeds(data, callback);
NEWSBLUR.reader.force_feed_refresh(feed_id, null, data.new_feed_id);
NEWSBLUR.reader.force_feed_refresh(feed_id, data.new_feed_id);
};
if (NEWSBLUR.Globals.is_authenticated) {
@ -978,7 +978,7 @@ NEWSBLUR.AssetModel.Reader.prototype = {
var pre_callback = function(data) {
// NEWSBLUR.log(['save_exception_change_feed_address pre_callback', feed_id, feed_address, data]);
self.post_refresh_feeds(data, callback);
NEWSBLUR.reader.force_feed_refresh(feed_id, null, data.new_feed_id);
NEWSBLUR.reader.force_feed_refresh(feed_id, data.new_feed_id);
};
if (NEWSBLUR.Globals.is_authenticated) {

11
media/js/newsblur/reader/reader.js
View file

@ -1287,7 +1287,7 @@
<div class="feed_counts">\
<%= feed_counts_floater %>\
</div>\
<img class="feed_favicon" src="<%= $.favicon(feed, true) %>">\
<img class="feed_favicon" src="<%= $.favicon(feed, !favicons_downloaded) %>">\
<span class="feed_title">\
<%= feed.feed_title %>\
<% if (type == "story") { %>\
@ -1319,7 +1319,8 @@
unread_class : unread_class,
exception_class : exception_class,
toplevel : depth == 0,
list_type : type == 'feed' ? 'li' : 'div'
list_type : type == 'feed' ? 'li' : 'div',
favicons_downloaded : this.flags['favicons_downloaded']
});
return $feed;
@ -5770,7 +5771,7 @@
var $feed = this.find_feed_in_feed_list(feed_id);
$feed.addClass('NB-feed-unfetched').removeClass('NB-feed-exception');
this.model.save_exception_retry(feed_id, _.bind(this.force_feed_refresh, this, feed_id, $feed), this.show_stories_error);
this.model.save_exception_retry(feed_id, _.bind(this.force_feed_refresh, this, feed_id), this.show_stories_error);
},
setup_socket_realtime_unread_counts: function(force) {
@ -5828,10 +5829,10 @@
}, refresh_interval);
},
force_feed_refresh: function(feed_id, $feed, new_feed_id) {
force_feed_refresh: function(feed_id, new_feed_id) {
var self = this;
feed_id = feed_id || this.active_feed;
$feed = $feed || this.find_feed_in_feed_list(feed_id);
var $feed = this.find_feed_in_feed_list(feed_id);
new_feed_id = new_feed_id || feed_id;
this.force_feeds_refresh(function(feeds) {

13
media/js/newsblur/reader/reader_feed_exception.js
View file

@ -225,15 +225,14 @@ _.extend(NEWSBLUR.ReaderFeedException.prototype, {
},
save_retry_feed: function() {
var self = this;
var $loading = $('.NB-modal-loading', this.$modal);
$loading.addClass('NB-active');
var feed_id = this.feed_id;
$('.NB-modal-submit-retry', this.$modal).addClass('NB-disabled').attr('value', 'Fetching...');
this.model.save_exception_retry(this.feed_id, function() {
// NEWSBLUR.reader.flags['has_unfetched_feeds'] = true;
// NEWSBLUR.reader.force_instafetch_stories(self.feed_id);
this.model.save_exception_retry(feed_id, function() {
NEWSBLUR.reader.force_feed_refresh(feed_id);
$.modal.close();
});
},
@ -263,8 +262,7 @@ _.extend(NEWSBLUR.ReaderFeedException.prototype, {
if (feed_address.length) {
this.model.save_exception_change_feed_address(feed_id, feed_address, function(code) {
// NEWSBLUR.reader.flags['has_unfetched_feeds'] = true;
// NEWSBLUR.reader.load_feeds();
NEWSBLUR.reader.force_feed_refresh(feed_id);
$.modal.close();
});
}
@ -281,8 +279,7 @@ _.extend(NEWSBLUR.ReaderFeedException.prototype, {
if (feed_link.length) {
this.model.save_exception_change_feed_link(feed_id, feed_link, function(code) {
// NEWSBLUR.reader.flags['has_unfetched_feeds'] = true;
// NEWSBLUR.reader.load_feeds();
NEWSBLUR.reader.force_feed_refresh(feed_id);
$.modal.close();
});
}

2
utils/feed_fetcher.py
View file

@ -144,7 +144,7 @@ class ProcessFeed:
self.feed.save_feed_history(self.fpf.status, "HTTP Error")
else:
self.feed.has_feed_exception = True
self.feed.schedule_feed_fetch_immediately()
self.feed.schedule_feed_fetch_geometrically()
self.feed.save()
return FEED_ERRHTTP, ret_values

2220
utils/feedparser.py
View file

File diff suppressed because it is too large Load diff