From 2954ef7318734beaeca89411c4a8946ea3efc83c Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Fri, 17 Feb 2017 11:45:43 -0800 Subject: [PATCH] Adding new debug to feed fetcher. --- apps/rss_feeds/models.py | 1 + utils/feed_fetcher.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index a63a58d02..98dd2b6f4 100644 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -1046,6 +1046,7 @@ class Feed(models.Model): 'timeout': 10, 'single_threaded': kwargs.get('single_threaded', True), 'force': kwargs.get('force'), + 'force_fp': kwargs.get('force_fp'), 'compute_scores': kwargs.get('compute_scores', True), 'mongodb_replication_lag': kwargs.get('mongodb_replication_lag', None), 'fake': kwargs.get('fake'), diff --git a/utils/feed_fetcher.py b/utils/feed_fetcher.py index 6f71766b9..89b64967b 100644 --- a/utils/feed_fetcher.py +++ b/utils/feed_fetcher.py @@ -145,10 +145,14 @@ class FetchFeed: response_headers['Content-Location'] = raw_feed.url self.fpf = feedparser.parse(smart_unicode(raw_feed.content), response_headers=response_headers) + if self.options.get('debug', False): + logging.debug(" ---> [%-30s] ~FBFeed fetch status %s: %s length / %s" % (self.feed.title[:30], raw_feed.status_code, len(smart_unicode(raw_feed.content)), raw_feed.headers)) + else: + logging.debug(" ---> [%-30s] ~FRFeed fetch was empty, trying feedparser: %s" % (self.feed.title[:30], raw_feed.headers)) except Exception, e: logging.debug(" ---> [%-30s] ~FRFeed failed to fetch with request, trying feedparser: %s" % (self.feed.title[:30], unicode(e)[:100])) - if not self.fpf: + if not self.fpf or self.options.get('force_fp', False): try: self.fpf = feedparser.parse(address, agent=USER_AGENT,