public-mirrors/NewsBlur
1
0
Fork 0
mirror of https://github.com/samuelclay/NewsBlur.git synced 2025-08-31 21:41:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Adding verbose flag to feed updates to diagnose errors.

Browse source
This commit is contained in:
Samuel Clay 2011-11-24 15:29:26 -05:00
parent c4a9068472
commit 7ea2e01496
2 changed files with 14 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
apps/rss_feeds/models.py
View file

@ -562,7 +562,7 @@ class Feed(models.Model):
self.data.feed_classifier_counts = json.encode(scores) self.data.feed_classifier_counts = json.encode(scores)
self.data.save() self.data.save()
def update(self, force=False, single_threaded=True, compute_scores=True, slave_db=None): def update(self, verbose=False, force=False, single_threaded=True, compute_scores=True, slave_db=None):
from utils import feed_fetcher from utils import feed_fetcher
try: try:
self.feed_address = self.feed_address % {'NEWSBLUR_DIR': settings.NEWSBLUR_DIR} self.feed_address = self.feed_address % {'NEWSBLUR_DIR': settings.NEWSBLUR_DIR}
@ -573,7 +573,7 @@ class Feed(models.Model):
self.set_next_scheduled_update() self.set_next_scheduled_update()
options = { options = {
'verbose': 1 if not force else 2, 'verbose': verbose,
'timeout': 10, 'timeout': 10,
'single_threaded': single_threaded, 'single_threaded': single_threaded,
'force': force, 'force': force,
@ -594,7 +594,7 @@ class Feed(models.Model):
return feed return feed
def add_update_stories(self, stories, existing_stories): def add_update_stories(self, stories, existing_stories, verbose=False):
ret_values = { ret_values = {
ENTRY_NEW:0, ENTRY_NEW:0,
ENTRY_UPDATED:0, ENTRY_UPDATED:0,
@ -629,9 +629,10 @@ class Feed(models.Model):
s.save() s.save()
ret_values[ENTRY_NEW] += 1 ret_values[ENTRY_NEW] += 1
cache.set('updated_feed:%s' % self.id, 1) cache.set('updated_feed:%s' % self.id, 1)
except (IntegrityError, OperationError): except (IntegrityError, OperationError), e:
ret_values[ENTRY_ERR] += 1 ret_values[ENTRY_ERR] += 1
# logging.info('Saving new story, IntegrityError: %s - %s: %s' % (self.feed_title, story.get('title'), e)) if verbose:
logging.info('Saving new story, IntegrityError: %s - %s: %s' % (self.feed_title, story.get('title'), e))
elif existing_story and story_has_changed: elif existing_story and story_has_changed:
# update story # update story
# logging.debug('- Updated story in feed (%s - %s): %s / %s' % (self.feed_title, story.get('title'), len(existing_story.story_content), len(story_content))) # logging.debug('- Updated story in feed (%s - %s): %s / %s' % (self.feed_title, story.get('title'), len(existing_story.story_content), len(story_content)))
@ -644,8 +645,10 @@ class Feed(models.Model):
existing_story = MStory.objects.get(story_feed_id=existing_story.story_feed_id, story_guid=existing_story.story_guid) existing_story = MStory.objects.get(story_feed_id=existing_story.story_feed_id, story_guid=existing_story.story_guid)
else: else:
raise MStory.DoesNotExist raise MStory.DoesNotExist
except (MStory.DoesNotExist, OperationError): except (MStory.DoesNotExist, OperationError), e:
ret_values[ENTRY_ERR] += 1 ret_values[ENTRY_ERR] += 1
if verbose:
logging.info('Saving existing story, OperationError: %s - %s: %s' % (self.feed_title, story.get('title'), e))
continue continue
if existing_story.story_original_content_z: if existing_story.story_original_content_z:
original_content = zlib.decompress(existing_story.story_original_content_z) original_content = zlib.decompress(existing_story.story_original_content_z)
@ -679,10 +682,12 @@ class Feed(models.Model):
cache.set('updated_feed:%s' % self.id, 1) cache.set('updated_feed:%s' % self.id, 1)
except (IntegrityError, OperationError): except (IntegrityError, OperationError):
ret_values[ENTRY_ERR] += 1 ret_values[ENTRY_ERR] += 1
logging.info('Saving updated story, IntegrityError: %s - %s' % (self.feed_title, story.get('title'))) if verbose:
logging.info('Saving updated story, IntegrityError: %s - %s' % (self.feed_title, story.get('title')))
except ValidationError, e: except ValidationError, e:
ret_values[ENTRY_ERR] += 1 ret_values[ENTRY_ERR] += 1
logging.info('Saving updated story, ValidationError: %s - %s: %s' % (self.feed_title, story.get('title'), e)) if verbose:
logging.info('Saving updated story, ValidationError: %s - %s: %s' % (self.feed_title, story.get('title'), e))
else: else:
ret_values[ENTRY_SAME] += 1 ret_values[ENTRY_SAME] += 1
# logging.debug("Unchanged story: %s " % story.get('title')) # logging.debug("Unchanged story: %s " % story.get('title'))

2
utils/feed_fetcher.py
View file

@ -241,7 +241,7 @@ class ProcessFeed:
# | (Q(story_guid__in=story_guids)), # | (Q(story_guid__in=story_guids)),
# story_feed=self.feed # story_feed=self.feed
# ).order_by('-story_date') # ).order_by('-story_date')
ret_values = self.feed.add_update_stories(self.fpf.entries, existing_stories) ret_values = self.feed.add_update_stories(self.fpf.entries, existing_stories, verbose=self.options['verbose'])
logging.debug(u' ---> [%-30s] ~FYParsed Feed: new~FG=~FG~SB%s~SN~FY up~FG=~FY~SB%s~SN same~FG=~FY%s err~FG=~FR~SB%s' % ( logging.debug(u' ---> [%-30s] ~FYParsed Feed: new~FG=~FG~SB%s~SN~FY up~FG=~FY~SB%s~SN same~FG=~FY%s err~FG=~FR~SB%s' % (
unicode(self.feed)[:30], unicode(self.feed)[:30],