From bf3bd8c49c406f662f6b7896edb9e890f154e1cf Mon Sep 17 00:00:00 2001 From: Laptop765 Date: Thu, 23 Feb 2012 21:40:03 -0800 Subject: [PATCH 1/5] Update media/js/newsblur/reader.js --- media/js/newsblur/reader.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/media/js/newsblur/reader.js b/media/js/newsblur/reader.js index ba92e30f9..ffb7d5bdc 100644 --- a/media/js/newsblur/reader.js +++ b/media/js/newsblur/reader.js @@ -1126,7 +1126,7 @@ return -1; } else if (!feedA && feedB) { return 1; - } else if (!feedA && !feedB && a && b && !_.isNumber(a) && !_.isNumber(b) && !(a.e instanceof jQuery) && (!b.e instanceof jQuery)) { + } else if (!feedA && !feedB && a && b && !_.isNumber(a) && !_.isNumber(b) && !(a.e instanceof jQuery) && !(b.e instanceof jQuery)) { // console.log(['a b 1', a, b, feedA, feedB]); var folderA = _.keys(a)[0]; var folderB = _.keys(b)[0]; @@ -1141,7 +1141,7 @@ return -1; } else if (!feedA && feedB) { return 1; - } else if (!feedA && !feedB && a && b && !_.isNumber(a) && !_.isNumber(b) && !(a.e instanceof jQuery) && (!b.e instanceof jQuery)) { + } else if (!feedA && !feedB && a && b && !_.isNumber(a) && !_.isNumber(b) && !(a.e instanceof jQuery) && !(b.e instanceof jQuery)) { // console.log(['a b 2', a, b]); var folderA = _.keys(a)[0]; var folderB = _.keys(b)[0]; From 48b69c5e57b025ee83a1c2dad30b3f1e06371e70 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Fri, 24 Feb 2012 10:30:39 -0800 Subject: [PATCH 2/5] Fixing error codes on statistics. Also fixing simple error on invalid feed. --- apps/statistics/models.py | 6 +++--- media/css/reader.css | 2 +- media/js/newsblur/reader_statistics.js | 2 +- utils/feedparser.py | 5 ++++- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/apps/statistics/models.py b/apps/statistics/models.py index 339c2aaa4..4b5e4b02e 100644 --- a/apps/statistics/models.py +++ b/apps/statistics/models.py @@ -51,7 +51,7 @@ class MStatistics(mongo.Document): def collect_statistics_feeds_fetched(cls, last_day=None): if not last_day: last_day = datetime.datetime.now() - datetime.timedelta(hours=24) - last_biweek = datetime.datetime.now() - datetime.timedelta(days=14) + last_month = datetime.datetime.now() - datetime.timedelta(days=30) feeds_fetched = MFeedFetchHistory.objects.filter(fetch_date__lt=last_day).count() cls.objects(key='feeds_fetched').update_one(upsert=True, key='feeds_fetched', value=feeds_fetched) @@ -63,8 +63,8 @@ class MStatistics(mongo.Document): def delete_old_history(): MFeedFetchHistory.objects(fetch_date__lt=last_day, status_code__in=[200, 304]).delete() MPageFetchHistory.objects(fetch_date__lt=last_day, status_code__in=[200, 304]).delete() - MFeedFetchHistory.objects(fetch_date__lt=last_biweek).delete() - MPageFetchHistory.objects(fetch_date__lt=last_biweek).delete() + MFeedFetchHistory.objects(fetch_date__lt=last_month).delete() + MPageFetchHistory.objects(fetch_date__lt=last_month).delete() try: delete_old_history() except TimeoutError: diff --git a/media/css/reader.css b/media/css/reader.css index 5d362e692..d027aef88 100644 --- a/media/css/reader.css +++ b/media/css/reader.css @@ -4601,7 +4601,7 @@ background: transparent; .NB-modal-statistics .NB-statistics-history-fetch.NB-ok { color: #135500; } -.NB-modal-statistics .NB-statistics-history-fetch.NB-error { +.NB-modal-statistics .NB-statistics-history-fetch.NB-errorcode { color: #6A1000; } .NB-modal-statistics .NB-statistics-history-fetch .NB-statistics-history-fetch-code { diff --git a/media/js/newsblur/reader_statistics.js b/media/js/newsblur/reader_statistics.js index 7f9f53b09..4d40eceda 100644 --- a/media/js/newsblur/reader_statistics.js +++ b/media/js/newsblur/reader_statistics.js @@ -214,7 +214,7 @@ _.extend(NEWSBLUR.ReaderStatistics.prototype, { var $history = _.map(fetches, function(fetch) { var feed_ok = _.contains([200, 304], fetch.status_code); - var status_class = feed_ok ? ' NB-ok ' : ' NB-error '; + var status_class = feed_ok ? ' NB-ok ' : ' NB-errorcode '; return $.make('div', { className: 'NB-statistics-history-fetch' + status_class, title: feed_ok ? '' : fetch.exception }, [ $.make('div', { className: 'NB-statistics-history-fetch-date' }, fetch.fetch_date), $.make('div', { className: 'NB-statistics-history-fetch-message' }, [ diff --git a/utils/feedparser.py b/utils/feedparser.py index d88dfb412..5e6b9cecd 100755 --- a/utils/feedparser.py +++ b/utils/feedparser.py @@ -3924,7 +3924,10 @@ def parse(url_file_stream_or_string, etag=None, modified=None, agent=None, refer break # if no luck and we have auto-detection library, try that if (not known_encoding) and chardet: - proposed_encoding = unicode(chardet.detect(data)['encoding'], 'ascii', 'ignore') + # import pdb; pdb.set_trace() + proposed_encoding = chardet.detect(data)['encoding'] + if proposed_encoding: + proposed_encoding = unicode(proposed_encoding, 'ascii', 'ignore') if proposed_encoding and (proposed_encoding not in tried_encodings): tried_encodings.append(proposed_encoding) try: From 1ffefaa419b60cbfaea4475582671aef565f5516 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Fri, 24 Feb 2012 11:47:38 -0800 Subject: [PATCH 3/5] Geometrically delaying the fetching of bad feeds, while they're still somewhat fresh. --- apps/rss_feeds/models.py | 29 +++++++++++++++++------------ utils/feed_fetcher.py | 17 +++-------------- 2 files changed, 20 insertions(+), 26 deletions(-) diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index bb0fdf06e..180aaedde 100644 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -4,7 +4,6 @@ import random import re import math import mongoengine as mongo -import redis import zlib import urllib import hashlib @@ -13,7 +12,6 @@ from operator import itemgetter # from nltk.collocations import TrigramCollocationFinder, BigramCollocationFinder, TrigramAssocMeasures, BigramAssocMeasures from django.db import models from django.db import IntegrityError -from django.core.cache import cache from django.conf import settings from django.db.models.query import QuerySet from mongoengine.queryset import OperationError @@ -287,6 +285,10 @@ class Feed(models.Model): self.save_feed_history(505, 'Timeout', '') feed_address = None + if feed_address: + self.feed.has_feed_exception = True + self.feed.schedule_feed_fetch_immediately() + return not not feed_address def save_feed_history(self, status_code, message, exception=None): @@ -304,7 +306,8 @@ class Feed(models.Model): # for history in old_fetch_histories: # history.delete() if status_code not in (200, 304): - self.count_errors_in_history('feed', status_code) + errors, non_errors = self.count_errors_in_history('feed', status_code) + self.set_next_scheduled_update(error_count=len(errors), non_error_count=len(non_errors)) elif self.has_feed_exception: self.has_feed_exception = False self.active = True @@ -333,8 +336,8 @@ class Feed(models.Model): history_class.objects(feed_id=self.pk)[:50]) non_errors = [h for h in fetch_history if int(h) in (200, 304)] errors = [h for h in fetch_history if int(h) not in (200, 304)] - - if len(non_errors) == 0 and len(errors) >= 1: + + if len(non_errors) == 0 and len(errors) > 1: if exception_type == 'feed': self.has_feed_exception = True self.active = False @@ -345,6 +348,10 @@ class Feed(models.Model): elif self.exception_code > 0: self.active = True self.exception_code = 0 + if exception_type == 'feed': + self.has_feed_exception = False + elif exception_type == 'page': + self.has_page_exception = False self.save() return errors, non_errors @@ -1007,11 +1014,12 @@ class Feed(models.Model): return total, random_factor*2 - def set_next_scheduled_update(self, multiplier=1): + def set_next_scheduled_update(self, error_count=0, non_error_count=0): total, random_factor = self.get_next_scheduled_update(force=True, verbose=False) - if multiplier > 1: - total = total * multiplier + if error_count: + logging.debug(' ---> [%-30s] ~FBScheduling feed fetch geometrically: ~SB%s errors, %s non-errors' % (unicode(self)[:30], error_count, non_error_count)) + total = total * error_count next_scheduled_update = datetime.datetime.utcnow() + datetime.timedelta( minutes = total + random_factor) @@ -1022,14 +1030,11 @@ class Feed(models.Model): self.save() def schedule_feed_fetch_immediately(self): + logging.debug(' ---> [%-30s] Scheduling feed fetch immediately...' % (unicode(self)[:30])) self.next_scheduled_update = datetime.datetime.utcnow() self.save() - def schedule_feed_fetch_geometrically(self): - errors, non_errors = self.count_errors_in_history('feed') - self.set_next_scheduled_update(multiplier=len(errors)) - # def calculate_collocations_story_content(self, # collocation_measures=TrigramAssocMeasures, # collocation_finder=TrigramCollocationFinder): diff --git a/utils/feed_fetcher.py b/utils/feed_fetcher.py index f629b292f..9d3ad34bb 100644 --- a/utils/feed_fetcher.py +++ b/utils/feed_fetcher.py @@ -47,11 +47,10 @@ class FetchFeed: datetime.datetime.now() - self.feed.last_update) logging.debug(log_msg) - self.feed.set_next_scheduled_update() etag=self.feed.etag modified = self.feed.last_modified.utctimetuple()[:7] if self.feed.last_modified else None - if self.options.get('force') or not self.feed.fetched_once: + if self.options.get('force') or not self.feed.fetched_once or not self.feed.known_good: modified = None etag = None @@ -126,10 +125,9 @@ class ProcessFeed: if self.fpf.status in (302, 301): if not self.fpf.href.endswith('feedburner.com/atom.xml'): self.feed.feed_address = self.fpf.href - if not self.feed.fetched_once: - self.feed.has_feed_exception = True + if not self.feed.known_good: self.feed.fetched_once = True - logging.debug(" ---> [%-30s] Feed is 302'ing, but it's not new. Refetching..." % (unicode(self.feed)[:30])) + logging.debug(" ---> [%-30s] Feed is %s'ing. Refetching..." % (unicode(self.feed)[:30], self.fpf.status)) self.feed.schedule_feed_fetch_immediately() if not self.fpf.entries: self.feed.save() @@ -142,9 +140,6 @@ class ProcessFeed: fixed_feed = self.feed.check_feed_link_for_feed_address() if not fixed_feed: self.feed.save_feed_history(self.fpf.status, "HTTP Error") - else: - self.feed.has_feed_exception = True - self.feed.schedule_feed_fetch_geometrically() self.feed.save() return FEED_ERRHTTP, ret_values @@ -156,9 +151,6 @@ class ProcessFeed: fixed_feed = self.feed.check_feed_link_for_feed_address() if not fixed_feed: self.feed.save_feed_history(502, 'Non-xml feed', self.fpf.bozo_exception) - else: - self.feed.has_feed_exception = True - self.feed.schedule_feed_fetch_immediately() self.feed.save() return FEED_ERRPARSE, ret_values elif self.fpf.bozo and isinstance(self.fpf.bozo_exception, xml.sax._exceptions.SAXException): @@ -169,9 +161,6 @@ class ProcessFeed: fixed_feed = self.feed.check_feed_link_for_feed_address() if not fixed_feed: self.feed.save_feed_history(503, 'SAX Exception', self.fpf.bozo_exception) - else: - self.feed.has_feed_exception = True - self.feed.schedule_feed_fetch_immediately() self.feed.save() return FEED_ERRPARSE, ret_values From 3dd04f521f5f99b89f6d2d571a98ef3c8da72265 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Fri, 24 Feb 2012 12:08:11 -0800 Subject: [PATCH 4/5] Fixing user-found bug around tooltips continuing to show if the original dom element (and the reference to the tooltip) are destroyed too early, leaving a tooltip floating in the air. --- fabfile.py | 1 + media/js/newsblur/reader.js | 22 ++++++++++++++-------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/fabfile.py b/fabfile.py index e20033860..182e7d0e4 100644 --- a/fabfile.py +++ b/fabfile.py @@ -65,6 +65,7 @@ def task(): # = Deploy = # ========== +@parallel def pull(): with cd(env.NEWSBLUR_PATH): run('git pull') diff --git a/media/js/newsblur/reader.js b/media/js/newsblur/reader.js index ffb7d5bdc..26f9e2ab3 100644 --- a/media/js/newsblur/reader.js +++ b/media/js/newsblur/reader.js @@ -2684,16 +2684,19 @@ trigger: 'manual', offsetOpposite: -1 }); - $star.tipsy('enable'); - $star.tipsy('show'); + var tipsy = $star.data('tipsy'); + tipsy.enable(); + tipsy.show(); $star.animate({ 'opacity': 1 }, { 'duration': 850, 'queue': false, 'complete': function() { - $(this).tipsy('hide'); - $(this).tipsy('disable'); + if (tipsy.enabled) { + tipsy.hide(); + tipsy.disable(); + } } }); this.model.mark_story_as_starred(story_id, story.story_feed_id, function() {}); @@ -2712,11 +2715,14 @@ trigger: 'manual', offsetOpposite: -1 }); - $star.tipsy('enable'); - $star.tipsy('show'); + var tipsy = $star.data('tipsy'); + tipsy.enable(); + tipsy.show(); _.delay(function() { - $star.tipsy('hide'); - $star.tipsy('disable'); + if (tipsy.enabled) { + tipsy.hide(); + tipsy.disable(); + } }, 850); $story.removeClass('NB-story-starred'); this.model.mark_story_as_unstarred(story_id, function() {}); From 3f42018f910b036fcfbb21d561eb2bfa2de173cf Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Fri, 24 Feb 2012 12:47:39 -0800 Subject: [PATCH 5/5] Fixing starred stories from not loading past the second page due to a missing flag reset. --- media/js/newsblur/reader.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/media/js/newsblur/reader.js b/media/js/newsblur/reader.js index 26f9e2ab3..49d8ed749 100644 --- a/media/js/newsblur/reader.js +++ b/media/js/newsblur/reader.js @@ -1014,7 +1014,7 @@ find_story_with_action_preference_on_open_feed: function() { var open_feed_action = this.model.preference('open_feed_action'); - console.log(["action_preference_on_open_feed", open_feed_action, this.counts['page']]); + if (this.counts['page'] != 1) return; if (open_feed_action == 'newest') { @@ -2001,6 +2001,7 @@ post_open_starred_stories: function(data, first_load) { if (this.active_feed == 'starred') { // NEWSBLUR.log(['post_open_starred_stories', data.stories.length, first_load]); + this.flags['opening_feed'] = false; this.flags['feed_view_positions_calculated'] = false; this.story_titles_clear_loading_endbar(); this.create_story_titles(data.stories, {'river_stories': true});