From 288d6c8e49da7c9d7e3b713f3638c1ae586d59eb Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Thu, 5 Jan 2012 20:59:48 -0800 Subject: [PATCH 1/3] The @kennethreitz commit: logging all discrepencies between requests and urllib2. This should take a few hours. --- apps/rss_feeds/page_importer.py | 6 ++++-- utils/feed_functions.py | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/apps/rss_feeds/page_importer.py b/apps/rss_feeds/page_importer.py index 240a24593..e720be99d 100644 --- a/apps/rss_feeds/page_importer.py +++ b/apps/rss_feeds/page_importer.py @@ -41,7 +41,7 @@ class PageImporter(object): } @timelimit(15) - def fetch_page(self, urllib_fallback=False): + def fetch_page(self, urllib_fallback=False, requests_exception=None): feed_link = self.feed.feed_link if not feed_link: self.save_no_page() @@ -66,6 +66,8 @@ class PageImporter(object): data = open(feed_link, 'r').read() html = self.rewrite_page(data) self.save_page(html) + if urllib_fallback: + mail_feed_error_to_admin(self.feed, requests_exception, locals(), subject="REQUESTS DIFFERENCE") except (ValueError, urllib2.URLError, httplib.BadStatusLine, httplib.InvalidURL), e: self.feed.save_page_history(401, "Bad URL", e) fp = feedparser.parse(self.feed.feed_address) @@ -79,7 +81,7 @@ class PageImporter(object): LookupError, requests.packages.urllib3.exceptions.HTTPError), e: logging.debug(' ***> [%-30s] Page fetch failed using requests: %s' % (self.feed, e)) - return self.fetch_page(urllib_fallback=True) + return self.fetch_page(urllib_fallback=True, requests_exception=e) except Exception, e: logging.debug('[%d] ! -------------------------' % (self.feed.id,)) tb = traceback.format_exc() diff --git a/utils/feed_functions.py b/utils/feed_functions.py index 234255a91..8b8c04089 100644 --- a/utils/feed_functions.py +++ b/utils/feed_functions.py @@ -176,10 +176,12 @@ def add_object_to_folder(obj, folder, folders, parent='', added=False): folders[k][f_k] = add_object_to_folder(obj, folder, f_v, f_k, added) return folders -def mail_feed_error_to_admin(feed, e, local_vars=None): +def mail_feed_error_to_admin(feed, e, local_vars=None, subject=None): # Mail the admins with the error + if not subject: + subject = "Feed update error" exc_info = sys.exc_info() - subject = 'Feed update error: %s' % repr(e) + subject = '%s: %s' % (subject, repr(e)) message = 'Traceback:\n%s\n\Feed:\n%s\nLocals:\n%s' % ( '\n'.join(traceback.format_exception(*exc_info)), pprint.pformat(feed.__dict__), From 3334383a04d75daf0b4971091d8c8fab379c37fb Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Thu, 5 Jan 2012 21:57:18 -0800 Subject: [PATCH 2/3] Turning off the page fetcher requests vs. urllib2 discrepency checker. I bet I've caught them all by now. --- apps/rss_feeds/page_importer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/apps/rss_feeds/page_importer.py b/apps/rss_feeds/page_importer.py index e720be99d..9d12d3a4a 100644 --- a/apps/rss_feeds/page_importer.py +++ b/apps/rss_feeds/page_importer.py @@ -66,8 +66,6 @@ class PageImporter(object): data = open(feed_link, 'r').read() html = self.rewrite_page(data) self.save_page(html) - if urllib_fallback: - mail_feed_error_to_admin(self.feed, requests_exception, locals(), subject="REQUESTS DIFFERENCE") except (ValueError, urllib2.URLError, httplib.BadStatusLine, httplib.InvalidURL), e: self.feed.save_page_history(401, "Bad URL", e) fp = feedparser.parse(self.feed.feed_address) From aa96e288d2737e4baff7c610fb0ff9887eca849b Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Sun, 8 Jan 2012 14:15:22 -0800 Subject: [PATCH 3/3] Adding error handling for insta-fetching stories. --- apps/rss_feeds/models.py | 2 +- media/js/newsblur/assetmodel.js | 14 +++++++------- media/js/newsblur/reader.js | 26 ++++++++++++++++---------- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index 0a3a1ddaf..b3926fe57 100644 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -95,7 +95,7 @@ class Feed(models.Model): if include_favicon: try: - feed_icon = MFeedIcon.objects.get(feed_id=self.pk) + feed_icon = MFeedIcon.objects.filter(feed_id=self.pk).first() feed['favicon'] = feed_icon.data except MFeedIcon.DoesNotExist: pass diff --git a/media/js/newsblur/assetmodel.js b/media/js/newsblur/assetmodel.js index 9e7c1ffe8..53d634097 100644 --- a/media/js/newsblur/assetmodel.js +++ b/media/js/newsblur/assetmodel.js @@ -73,7 +73,7 @@ NEWSBLUR.AssetModel.Reader.prototype = { if (clear_queue) { this.ajax[options['ajax_group']].clear(true); } - + this.ajax[options['ajax_group']].add(_.extend({ url: url, data: data, @@ -93,12 +93,12 @@ NEWSBLUR.AssetModel.Reader.prototype = { } }, error: function(e, textStatus, errorThrown) { - NEWSBLUR.log(['AJAX Error', e, textStatus, errorThrown]); + NEWSBLUR.log(['AJAX Error', e, textStatus, errorThrown, !!error_callback, error_callback]); if (errorThrown == 'abort') { return; } - if ($.isFunction(error_callback)) { + if (error_callback) { error_callback(); } else if ($.isFunction(callback)) { var message = "Please create an account. Not much to do without an account."; @@ -435,7 +435,7 @@ NEWSBLUR.AssetModel.Reader.prototype = { } }, - refresh_feeds: function(callback, has_unfetched_feeds, feed_id) { + refresh_feeds: function(callback, has_unfetched_feeds, feed_id, error_callback) { var self = this; var pre_callback = function(data) { @@ -461,7 +461,7 @@ NEWSBLUR.AssetModel.Reader.prototype = { } if (NEWSBLUR.Globals.is_authenticated || feed_id) { - this.make_request('/reader/refresh_feeds', data, pre_callback); + this.make_request('/reader/refresh_feeds', data, pre_callback, error_callback); } }, @@ -833,7 +833,7 @@ NEWSBLUR.AssetModel.Reader.prototype = { } }, - save_exception_retry: function(feed_id, callback) { + save_exception_retry: function(feed_id, callback, error_callback) { var self = this; var pre_callback = function(data) { @@ -844,7 +844,7 @@ NEWSBLUR.AssetModel.Reader.prototype = { this.make_request('/rss_feeds/exception_retry', { 'feed_id': feed_id, 'reset_fetch': !!(this.feeds[feed_id].has_feed_exception || this.feeds[feed_id].has_page_exception) - }, pre_callback); + }, pre_callback, error_callback); }, save_exception_change_feed_link: function(feed_id, feed_link, callback) { diff --git a/media/js/newsblur/reader.js b/media/js/newsblur/reader.js index 1b4377359..2cbe57d15 100644 --- a/media/js/newsblur/reader.js +++ b/media/js/newsblur/reader.js @@ -79,6 +79,12 @@ this.$s.$feed_stories.bind('mousemove', $.rescope(this.handle_mousemove_feed_view, this)); this.handle_keystrokes(); + // ============ + // = Bindings = + // ============ + + _.bindAll(this, 'show_stories_error'); + // ================== // = Initialization = // ================== @@ -1740,7 +1746,7 @@ _.delay(_.bind(function() { if (!delay || feed_id == self.next_feed) { this.model.load_feed(feed_id, 1, true, $.rescope(this.post_open_feed, this), - _.bind(this.show_stories_error, this)); + this.show_stories_error); } }, this), delay || 0); @@ -1963,7 +1969,7 @@ this.setup_mousemove_on_views(); this.model.fetch_starred_stories(1, _.bind(this.post_open_starred_stories, this), - _.bind(this.show_stories_error, this), true); + this.show_stories_error, true); }, post_open_starred_stories: function(data, first_load) { @@ -2026,7 +2032,7 @@ this.cache['river_feeds_with_unreads'] = feeds; this.show_stories_progress_bar(feeds.length); this.model.fetch_river_stories(this.active_feed, feeds, 1, - _.bind(this.post_open_river_stories, this), _.bind(this.show_stories_error, this), true); + _.bind(this.post_open_river_stories, this), this.show_stories_error, true); }, post_open_river_stories: function(data, first_load) { @@ -3156,14 +3162,14 @@ $story_titles.data('page', page+1); if (this.active_feed == 'starred') { this.model.fetch_starred_stories(page+1, _.bind(this.post_open_starred_stories, this), - _.bind(this.show_stories_error, this), false); + this.show_stories_error, false); } else if (this.flags['river_view']) { this.model.fetch_river_stories(this.active_feed, this.cache['river_feeds_with_unreads'], page+1, _.bind(this.post_open_river_stories, this), - _.bind(this.show_stories_error, this), false); + this.show_stories_error, false); } else { this.model.load_feed(feed_id, page+1, false, - $.rescope(this.post_open_feed, this), _.bind(this.show_stories_error, this)); + $.rescope(this.post_open_feed, this), this.show_stories_error); } } }, @@ -5118,7 +5124,7 @@ var $feed = this.find_feed_in_feed_list(feed_id); $feed.addClass('NB-feed-unfetched').removeClass('NB-feed-exception'); - this.model.save_exception_retry(feed_id, _.bind(this.force_feed_refresh, this, feed_id, $feed)); + this.model.save_exception_retry(feed_id, _.bind(this.force_feed_refresh, this, feed_id, $feed), this.show_stories_error); }, setup_socket_realtime_unread_counts: function(force) { @@ -5194,10 +5200,10 @@ if (self.active_feed == feed_id || self.active_feed == new_feed_id) { self.open_feed(new_feed_id, true, $new_feed); } - }, true, new_feed_id); + }, true, new_feed_id, this.show_stories_error); }, - force_feeds_refresh: function(callback, replace_active_feed, feed_id) { + force_feeds_refresh: function(callback, replace_active_feed, feed_id, error_callback) { if (callback) { this.cache.refresh_callback = callback; } else { @@ -5208,7 +5214,7 @@ this.model.refresh_feeds(_.bind(function(updated_feeds) { this.post_feed_refresh(updated_feeds, replace_active_feed, feed_id); - }, this), this.flags['has_unfetched_feeds'], feed_id); + }, this), this.flags['has_unfetched_feeds'], feed_id, error_callback); }, post_feed_refresh: function(updated_feeds, replace_active_feed, single_feed_id) {