public-mirrors/NewsBlur-viq
1
0
Fork 0
mirror of https://github.com/viq/NewsBlur.git synced 2025-08-05 16:49:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into social

Browse source 
* master:
  Fixing starred stories from not loading past the second page due to a missing flag reset.
  Fixing user-found bug around tooltips continuing to show if the original dom element (and the reference to the tooltip) are destroyed too early, leaving a tooltip floating in the air.
  Geometrically delaying the fetching of bad feeds, while they're still somewhat fresh.
  Fixing error codes on statistics. Also fixing simple error on invalid feed.
  Update media/js/newsblur/reader.js

Conflicts:
	media/js/newsblur/reader/reader.js
This commit is contained in:
Samuel Clay 2012-02-24 13:01:48 -08:00
commit 408cb7e801
8 changed files with 49 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
apps/rss_feeds/models.py
View file

@ -4,7 +4,6 @@ import random
import re import re
import math import math
import mongoengine as mongo import mongoengine as mongo
import redis
import zlib import zlib
import urllib import urllib
import hashlib import hashlib
@ -13,7 +12,6 @@ from operator import itemgetter
# from nltk.collocations import TrigramCollocationFinder, BigramCollocationFinder, TrigramAssocMeasures, BigramAssocMeasures # from nltk.collocations import TrigramCollocationFinder, BigramCollocationFinder, TrigramAssocMeasures, BigramAssocMeasures
from django.db import models from django.db import models
from django.db import IntegrityError from django.db import IntegrityError
from django.core.cache import cache
from django.conf import settings from django.conf import settings
from django.db.models.query import QuerySet from django.db.models.query import QuerySet
from django.contrib.auth.models import User from django.contrib.auth.models import User
@ -290,6 +288,10 @@ class Feed(models.Model):
self.save_feed_history(505, 'Timeout', '') self.save_feed_history(505, 'Timeout', '')
feed_address = None feed_address = None
if feed_address:
self.feed.has_feed_exception = True
self.feed.schedule_feed_fetch_immediately()
return not not feed_address return not not feed_address
def save_feed_history(self, status_code, message, exception=None): def save_feed_history(self, status_code, message, exception=None):
@ -307,7 +309,8 @@ class Feed(models.Model):
# for history in old_fetch_histories: # for history in old_fetch_histories:
# history.delete() # history.delete()
if status_code not in (200, 304): if status_code not in (200, 304):
self.count_errors_in_history('feed', status_code) errors, non_errors = self.count_errors_in_history('feed', status_code)
self.set_next_scheduled_update(error_count=len(errors), non_error_count=len(non_errors))
elif self.has_feed_exception: elif self.has_feed_exception:
self.has_feed_exception = False self.has_feed_exception = False
self.active = True self.active = True
@ -337,7 +340,7 @@ class Feed(models.Model):
non_errors = [h for h in fetch_history if int(h) in (200, 304)] non_errors = [h for h in fetch_history if int(h) in (200, 304)]
errors = [h for h in fetch_history if int(h) not in (200, 304)] errors = [h for h in fetch_history if int(h) not in (200, 304)]
if len(non_errors) == 0 and len(errors) >= 1: if len(non_errors) == 0 and len(errors) > 1:
if exception_type == 'feed': if exception_type == 'feed':
self.has_feed_exception = True self.has_feed_exception = True
self.active = False self.active = False
@ -348,6 +351,10 @@ class Feed(models.Model):
elif self.exception_code > 0: elif self.exception_code > 0:
self.active = True self.active = True
self.exception_code = 0 self.exception_code = 0
if exception_type == 'feed':
self.has_feed_exception = False
elif exception_type == 'page':
self.has_page_exception = False
self.save() self.save()
return errors, non_errors return errors, non_errors
@ -1017,11 +1024,12 @@ class Feed(models.Model):
return total, random_factor*2 return total, random_factor*2
def set_next_scheduled_update(self, multiplier=1): def set_next_scheduled_update(self, error_count=0, non_error_count=0):
total, random_factor = self.get_next_scheduled_update(force=True, verbose=False) total, random_factor = self.get_next_scheduled_update(force=True, verbose=False)
if multiplier > 1: if error_count:
total = total * multiplier logging.debug(' ---> [%-30s] ~FBScheduling feed fetch geometrically: ~SB%s errors, %s non-errors' % (unicode(self)[:30], error_count, non_error_count))
total = total * error_count
next_scheduled_update = datetime.datetime.utcnow() + datetime.timedelta( next_scheduled_update = datetime.datetime.utcnow() + datetime.timedelta(
minutes = total + random_factor) minutes = total + random_factor)
@ -1032,14 +1040,11 @@ class Feed(models.Model):
self.save() self.save()
def schedule_feed_fetch_immediately(self): def schedule_feed_fetch_immediately(self):
logging.debug(' ---> [%-30s] Scheduling feed fetch immediately...' % (unicode(self)[:30]))
self.next_scheduled_update = datetime.datetime.utcnow() self.next_scheduled_update = datetime.datetime.utcnow()
self.save() self.save()
def schedule_feed_fetch_geometrically(self):
errors, non_errors = self.count_errors_in_history('feed')
self.set_next_scheduled_update(multiplier=len(errors))
# def calculate_collocations_story_content(self, # def calculate_collocations_story_content(self,
# collocation_measures=TrigramAssocMeasures, # collocation_measures=TrigramAssocMeasures,
# collocation_finder=TrigramCollocationFinder): # collocation_finder=TrigramCollocationFinder):

6
apps/statistics/models.py
View file

@ -52,7 +52,7 @@ class MStatistics(mongo.Document):
def collect_statistics_feeds_fetched(cls, last_day=None): def collect_statistics_feeds_fetched(cls, last_day=None):
if not last_day: if not last_day:
last_day = datetime.datetime.now() - datetime.timedelta(hours=24) last_day = datetime.datetime.now() - datetime.timedelta(hours=24)
last_biweek = datetime.datetime.now() - datetime.timedelta(days=14) last_month = datetime.datetime.now() - datetime.timedelta(days=30)
feeds_fetched = MFeedFetchHistory.objects.filter(fetch_date__lt=last_day).count() feeds_fetched = MFeedFetchHistory.objects.filter(fetch_date__lt=last_day).count()
cls.objects(key='feeds_fetched').update_one(upsert=True, key='feeds_fetched', value=feeds_fetched) cls.objects(key='feeds_fetched').update_one(upsert=True, key='feeds_fetched', value=feeds_fetched)
@ -64,8 +64,8 @@ class MStatistics(mongo.Document):
def delete_old_history(): def delete_old_history():
MFeedFetchHistory.objects(fetch_date__lt=last_day, status_code__in=[200, 304]).delete() MFeedFetchHistory.objects(fetch_date__lt=last_day, status_code__in=[200, 304]).delete()
MPageFetchHistory.objects(fetch_date__lt=last_day, status_code__in=[200, 304]).delete() MPageFetchHistory.objects(fetch_date__lt=last_day, status_code__in=[200, 304]).delete()
MFeedFetchHistory.objects(fetch_date__lt=last_biweek).delete() MFeedFetchHistory.objects(fetch_date__lt=last_month).delete()
MPageFetchHistory.objects(fetch_date__lt=last_biweek).delete() MPageFetchHistory.objects(fetch_date__lt=last_month).delete()
try: try:
delete_old_history() delete_old_history()
except TimeoutError: except TimeoutError:

1
fabfile.py vendored
View file

@ -65,6 +65,7 @@ def task():
# = Deploy = # = Deploy =
# ========== # ==========
@parallel
def pull(): def pull():
with cd(env.NEWSBLUR_PATH): with cd(env.NEWSBLUR_PATH):
run('git pull') run('git pull')

2
media/css/reader.css
View file

@ -4915,7 +4915,7 @@ background: transparent;
.NB-modal-statistics .NB-statistics-history-fetch.NB-ok { .NB-modal-statistics .NB-statistics-history-fetch.NB-ok {
color: #135500; color: #135500;
} }
.NB-modal-statistics .NB-statistics-history-fetch.NB-error { .NB-modal-statistics .NB-statistics-history-fetch.NB-errorcode {
color: #6A1000; color: #6A1000;
} }
.NB-modal-statistics .NB-statistics-history-fetch .NB-statistics-history-fetch-code { .NB-modal-statistics .NB-statistics-history-fetch .NB-statistics-history-fetch-code {

29
media/js/newsblur/reader/reader.js
View file

@ -1030,7 +1030,7 @@
find_story_with_action_preference_on_open_feed: function() { find_story_with_action_preference_on_open_feed: function() {
var open_feed_action = this.model.preference('open_feed_action'); var open_feed_action = this.model.preference('open_feed_action');
console.log(["action_preference_on_open_feed", open_feed_action, this.counts['page']]);
if (this.counts['page'] != 1) return; if (this.counts['page'] != 1) return;
if (open_feed_action == 'newest') { if (open_feed_action == 'newest') {
@ -1173,7 +1173,7 @@
return -1; return -1;
} else if (!feedA && feedB) { } else if (!feedA && feedB) {
return 1; return 1;
} else if (!feedA && !feedB && a && b && !_.isNumber(a) && !_.isNumber(b) && !(a.e instanceof jQuery) && (!b.e instanceof jQuery)) { } else if (!feedA && !feedB && a && b && !_.isNumber(a) && !_.isNumber(b) && !(a.e instanceof jQuery) && !(b.e instanceof jQuery)) {
// console.log(['a b 1', a, b, feedA, feedB]); // console.log(['a b 1', a, b, feedA, feedB]);
var folderA = _.keys(a)[0]; var folderA = _.keys(a)[0];
var folderB = _.keys(b)[0]; var folderB = _.keys(b)[0];
@ -1188,7 +1188,7 @@
return -1; return -1;
} else if (!feedA && feedB) { } else if (!feedA && feedB) {
return 1; return 1;
} else if (!feedA && !feedB && a && b && !_.isNumber(a) && !_.isNumber(b) && !(a.e instanceof jQuery) && (!b.e instanceof jQuery)) { } else if (!feedA && !feedB && a && b && !_.isNumber(a) && !_.isNumber(b) && !(a.e instanceof jQuery) && !(b.e instanceof jQuery)) {
// console.log(['a b 2', a, b]); // console.log(['a b 2', a, b]);
var folderA = _.keys(a)[0]; var folderA = _.keys(a)[0];
var folderB = _.keys(b)[0]; var folderB = _.keys(b)[0];
@ -2064,6 +2064,7 @@
post_open_starred_stories: function(data, first_load) { post_open_starred_stories: function(data, first_load) {
if (this.active_feed == 'starred') { if (this.active_feed == 'starred') {
// NEWSBLUR.log(['post_open_starred_stories', data.stories.length, first_load]); // NEWSBLUR.log(['post_open_starred_stories', data.stories.length, first_load]);
this.flags['opening_feed'] = false;
this.flags['feed_view_positions_calculated'] = false; this.flags['feed_view_positions_calculated'] = false;
this.counts['feed_view_positions_timer'] = 0; this.counts['feed_view_positions_timer'] = 0;
this.story_titles_clear_loading_endbar(); this.story_titles_clear_loading_endbar();
@ -2876,8 +2877,9 @@
trigger: 'manual', trigger: 'manual',
offsetOpposite: -1 offsetOpposite: -1
}); });
$star.tipsy('enable'); var tipsy = $star.data('tipsy');
$star.tipsy('show'); tipsy.enable();
tipsy.show();
$star.animate({ $star.animate({
'opacity': 1 'opacity': 1
@ -2885,8 +2887,10 @@
'duration': 850, 'duration': 850,
'queue': false, 'queue': false,
'complete': function() { 'complete': function() {
$(this).tipsy('hide'); if (tipsy.enabled) {
$(this).tipsy('disable'); tipsy.hide();
tipsy.disable();
}
} }
}); });
@ -2920,11 +2924,14 @@
trigger: 'manual', trigger: 'manual',
offsetOpposite: -1 offsetOpposite: -1
}); });
$star.tipsy('enable'); var tipsy = $star.data('tipsy');
$star.tipsy('show'); tipsy.enable();
tipsy.show();
_.delay(function() { _.delay(function() {
$star.tipsy('hide'); if (tipsy.enabled) {
$star.tipsy('disable'); tipsy.hide();
tipsy.disable();
}
}, 850); }, 850);
$story_title.removeClass('NB-story-starred'); $story_title.removeClass('NB-story-starred');

2
media/js/newsblur/reader/reader_statistics.js
View file

@ -217,7 +217,7 @@ _.extend(NEWSBLUR.ReaderStatistics.prototype, {
var $history = _.map(fetches, function(fetch) { var $history = _.map(fetches, function(fetch) {
var feed_ok = _.contains([200, 304], fetch.status_code); var feed_ok = _.contains([200, 304], fetch.status_code);
var status_class = feed_ok ? ' NB-ok ' : ' NB-error '; var status_class = feed_ok ? ' NB-ok ' : ' NB-errorcode ';
return $.make('div', { className: 'NB-statistics-history-fetch' + status_class, title: feed_ok ? '' : fetch.exception }, [ return $.make('div', { className: 'NB-statistics-history-fetch' + status_class, title: feed_ok ? '' : fetch.exception }, [
$.make('div', { className: 'NB-statistics-history-fetch-date' }, fetch.fetch_date), $.make('div', { className: 'NB-statistics-history-fetch-date' }, fetch.fetch_date),
$.make('div', { className: 'NB-statistics-history-fetch-message' }, [ $.make('div', { className: 'NB-statistics-history-fetch-message' }, [

17
utils/feed_fetcher.py
View file

@ -47,11 +47,10 @@ class FetchFeed:
datetime.datetime.now() - self.feed.last_update) datetime.datetime.now() - self.feed.last_update)
logging.debug(log_msg) logging.debug(log_msg)
self.feed.set_next_scheduled_update()
etag=self.feed.etag etag=self.feed.etag
modified = self.feed.last_modified.utctimetuple()[:7] if self.feed.last_modified else None modified = self.feed.last_modified.utctimetuple()[:7] if self.feed.last_modified else None
if self.options.get('force') or not self.feed.fetched_once: if self.options.get('force') or not self.feed.fetched_once or not self.feed.known_good:
modified = None modified = None
etag = None etag = None
@ -126,10 +125,9 @@ class ProcessFeed:
if self.fpf.status in (302, 301): if self.fpf.status in (302, 301):
if not self.fpf.href.endswith('feedburner.com/atom.xml'): if not self.fpf.href.endswith('feedburner.com/atom.xml'):
self.feed.feed_address = self.fpf.href self.feed.feed_address = self.fpf.href
if not self.feed.fetched_once: if not self.feed.known_good:
self.feed.has_feed_exception = True
self.feed.fetched_once = True self.feed.fetched_once = True
logging.debug(" ---> [%-30s] Feed is 302'ing, but it's not new. Refetching..." % (unicode(self.feed)[:30])) logging.debug(" ---> [%-30s] Feed is %s'ing. Refetching..." % (unicode(self.feed)[:30], self.fpf.status))
self.feed.schedule_feed_fetch_immediately() self.feed.schedule_feed_fetch_immediately()
if not self.fpf.entries: if not self.fpf.entries:
self.feed.save() self.feed.save()
@ -142,9 +140,6 @@ class ProcessFeed:
fixed_feed = self.feed.check_feed_link_for_feed_address() fixed_feed = self.feed.check_feed_link_for_feed_address()
if not fixed_feed: if not fixed_feed:
self.feed.save_feed_history(self.fpf.status, "HTTP Error") self.feed.save_feed_history(self.fpf.status, "HTTP Error")
else:
self.feed.has_feed_exception = True
self.feed.schedule_feed_fetch_geometrically()
self.feed.save() self.feed.save()
return FEED_ERRHTTP, ret_values return FEED_ERRHTTP, ret_values
@ -156,9 +151,6 @@ class ProcessFeed:
fixed_feed = self.feed.check_feed_link_for_feed_address() fixed_feed = self.feed.check_feed_link_for_feed_address()
if not fixed_feed: if not fixed_feed:
self.feed.save_feed_history(502, 'Non-xml feed', self.fpf.bozo_exception) self.feed.save_feed_history(502, 'Non-xml feed', self.fpf.bozo_exception)
else:
self.feed.has_feed_exception = True
self.feed.schedule_feed_fetch_immediately()
self.feed.save() self.feed.save()
return FEED_ERRPARSE, ret_values return FEED_ERRPARSE, ret_values
elif self.fpf.bozo and isinstance(self.fpf.bozo_exception, xml.sax._exceptions.SAXException): elif self.fpf.bozo and isinstance(self.fpf.bozo_exception, xml.sax._exceptions.SAXException):
@ -169,9 +161,6 @@ class ProcessFeed:
fixed_feed = self.feed.check_feed_link_for_feed_address() fixed_feed = self.feed.check_feed_link_for_feed_address()
if not fixed_feed: if not fixed_feed:
self.feed.save_feed_history(503, 'SAX Exception', self.fpf.bozo_exception) self.feed.save_feed_history(503, 'SAX Exception', self.fpf.bozo_exception)
else:
self.feed.has_feed_exception = True
self.feed.schedule_feed_fetch_immediately()
self.feed.save() self.feed.save()
return FEED_ERRPARSE, ret_values return FEED_ERRPARSE, ret_values

5
utils/feedparser.py
View file

@ -3924,7 +3924,10 @@ def parse(url_file_stream_or_string, etag=None, modified=None, agent=None, refer
break break
# if no luck and we have auto-detection library, try that # if no luck and we have auto-detection library, try that
if (not known_encoding) and chardet: if (not known_encoding) and chardet:
proposed_encoding = unicode(chardet.detect(data)['encoding'], 'ascii', 'ignore') # import pdb; pdb.set_trace()
proposed_encoding = chardet.detect(data)['encoding']
if proposed_encoding:
proposed_encoding = unicode(proposed_encoding, 'ascii', 'ignore')
if proposed_encoding and (proposed_encoding not in tried_encodings): if proposed_encoding and (proposed_encoding not in tried_encodings):
tried_encodings.append(proposed_encoding) tried_encodings.append(proposed_encoding)
try: try: