public-mirrors/NewsBlur-viq
1
0
Fork 0
mirror of https://github.com/viq/NewsBlur.git synced 2025-08-31 22:20:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fixing rss feed tests.

Browse source
This commit is contained in:
Samuel Clay 2012-03-06 16:11:27 -08:00
parent 74ba777099
commit d5926fe577
8 changed files with 6746 additions and 107 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
apps/rss_feeds/fixtures/slashdot1.json
View file

@ -7,6 +7,7 @@
"days_to_trim": 90,
"feed_link": "%(NEWSBLUR_DIR)s/apps/rss_feeds/fixtures/slashdot1.html",
"feed_link_locked": true,
"fetched_once": true,
"num_subscribers": 1,
"active_subscribers": 1,
"creation": "2009-01-12",

3325
apps/rss_feeds/fixtures/slashdot1.xml
View file

File diff suppressed because one or more lines are too long

1
apps/rss_feeds/fixtures/slashdot2.json
View file

@ -7,6 +7,7 @@
"days_to_trim": 90,
"feed_link": "%(NEWSBLUR_DIR)s/apps/rss_feeds/fixtures/slashdot1.html",
"feed_link_locked": true,
"fetched_once": true,
"num_subscribers": 0,
"creation": "2009-01-12",
"feed_title": "Slashdot",

3325
apps/rss_feeds/fixtures/slashdot2.xml
View file

File diff suppressed because one or more lines are too long

179
apps/rss_feeds/models.py
View file

@ -608,7 +608,7 @@ class Feed(models.Model):
from utils import feed_fetcher
if not options:
options = {}
if settings.DEBUG:
if getattr(settings, 'TEST_DEBUG', False):
self.feed_address = self.feed_address % {'NEWSBLUR_DIR': settings.NEWSBLUR_DIR}
self.feed_link = self.feed_link % {'NEWSBLUR_DIR': settings.NEWSBLUR_DIR}
@ -645,97 +645,99 @@ class Feed(models.Model):
ENTRY_SAME:0,
ENTRY_ERR:0
}
for story in stories:
story = pre_process_story(story)
if story.get('title'):
story_content = story.get('story_content')
story_tags = self.get_tags(story)
story_link = self.get_permalink(story)
existing_story, story_has_changed = self._exists_story(story, story_content, existing_stories)
if existing_story is None:
s = MStory(story_feed_id = self.pk,
story_date = story.get('published'),
story_title = story.get('title'),
story_content = story_content,
story_author_name = story.get('author'),
story_permalink = story_link,
story_guid = story.get('guid'),
story_tags = story_tags
)
try:
s.save()
ret_values[ENTRY_NEW] += 1
except (IntegrityError, OperationError), e:
ret_values[ENTRY_ERR] += 1
if verbose:
logging.info('Saving new story, IntegrityError: %s - %s: %s' % (self.feed_title, story.get('title'), e))
elif existing_story and story_has_changed:
# update story
# logging.debug('- Updated story in feed (%s - %s): %s / %s' % (self.feed_title, story.get('title'), len(existing_story.story_content), len(story_content)))
original_content = None
try:
if existing_story and existing_story.id:
try:
existing_story = MStory.objects.get(story_feed_id=existing_story.story_feed_id,
id=existing_story.id)
except ValidationError:
existing_story = MStory.objects.get(story_feed_id=existing_story.story_feed_id,
story_guid=existing_story.id)
elif existing_story and existing_story.story_guid:
existing_story = MStory.objects.get(story_feed_id=existing_story.story_feed_id,
story_guid=existing_story.story_guid)
else:
raise MStory.DoesNotExist
except (MStory.DoesNotExist, OperationError), e:
ret_values[ENTRY_ERR] += 1
if verbose:
logging.info('Saving existing story, OperationError: %s - %s: %s' % (self.feed_title, story.get('title'), e))
continue
if existing_story.story_original_content_z:
original_content = zlib.decompress(existing_story.story_original_content_z)
elif existing_story.story_content_z:
original_content = zlib.decompress(existing_story.story_content_z)
# print 'Type: %s %s' % (type(original_content), type(story_content))
if story_content and len(story_content) > 10:
diff = HTMLDiff(unicode(original_content), story_content)
story_content_diff = diff.getDiff()
if not story.get('title'):
continue
story_content = story.get('story_content')
story_tags = self.get_tags(story)
story_link = self.get_permalink(story)
existing_story, story_has_changed = self._exists_story(story, story_content, existing_stories)
if existing_story is None:
s = MStory(story_feed_id = self.pk,
story_date = story.get('published'),
story_title = story.get('title'),
story_content = story_content,
story_author_name = story.get('author'),
story_permalink = story_link,
story_guid = story.get('guid'),
story_tags = story_tags
)
try:
s.save()
ret_values[ENTRY_NEW] += 1
except (IntegrityError, OperationError), e:
ret_values[ENTRY_ERR] += 1
if verbose:
logging.info('Saving new story, IntegrityError: %s - %s: %s' % (self.feed_title, story.get('title'), e))
elif existing_story and story_has_changed:
# update story
# logging.debug('- Updated story in feed (%s - %s): %s / %s' % (self.feed_title, story.get('title'), len(existing_story.story_content), len(story_content)))
original_content = None
try:
if existing_story and existing_story.id:
try:
existing_story = MStory.objects.get(story_feed_id=existing_story.story_feed_id,
id=existing_story.id)
except ValidationError:
existing_story = MStory.objects.get(story_feed_id=existing_story.story_feed_id,
story_guid=existing_story.id)
elif existing_story and existing_story.story_guid:
existing_story = MStory.objects.get(story_feed_id=existing_story.story_feed_id,
story_guid=existing_story.story_guid)
else:
story_content_diff = original_content
# logging.debug("\t\tDiff: %s %s %s" % diff.getStats())
# logging.debug("\t\tDiff content: %s" % diff.getDiff())
# if existing_story.story_title != story.get('title'):
# logging.debug('\tExisting title / New: : \n\t\t- %s\n\t\t- %s' % (existing_story.story_title, story.get('title')))
if existing_story.story_guid != story.get('guid'):
self.update_read_stories_with_new_guid(existing_story.story_guid, story.get('guid'))
existing_story.story_feed = self.pk
existing_story.story_date = story.get('published')
existing_story.story_title = story.get('title')
existing_story.story_content = story_content_diff
existing_story.story_original_content = original_content
existing_story.story_author_name = story.get('author')
existing_story.story_permalink = story_link
existing_story.story_guid = story.get('guid')
existing_story.story_tags = story_tags
try:
existing_story.save()
ret_values[ENTRY_UPDATED] += 1
except (IntegrityError, OperationError):
ret_values[ENTRY_ERR] += 1
if verbose:
logging.info('Saving updated story, IntegrityError: %s - %s' % (self.feed_title, story.get('title')))
except ValidationError, e:
ret_values[ENTRY_ERR] += 1
if verbose:
logging.info('Saving updated story, ValidationError: %s - %s: %s' % (self.feed_title, story.get('title'), e))
raise MStory.DoesNotExist
except (MStory.DoesNotExist, OperationError), e:
ret_values[ENTRY_ERR] += 1
if verbose:
logging.info('Saving existing story, OperationError: %s - %s: %s' % (self.feed_title, story.get('title'), e))
continue
if existing_story.story_original_content_z:
original_content = zlib.decompress(existing_story.story_original_content_z)
elif existing_story.story_content_z:
original_content = zlib.decompress(existing_story.story_content_z)
# print 'Type: %s %s' % (type(original_content), type(story_content))
if story_content and len(story_content) > 10:
diff = HTMLDiff(unicode(original_content), story_content)
story_content_diff = diff.getDiff()
else:
ret_values[ENTRY_SAME] += 1
# logging.debug("Unchanged story: %s " % story.get('title'))
story_content_diff = original_content
# logging.debug("\t\tDiff: %s %s %s" % diff.getStats())
# logging.debug("\t\tDiff content: %s" % diff.getDiff())
# if existing_story.story_title != story.get('title'):
# logging.debug('\tExisting title / New: : \n\t\t- %s\n\t\t- %s' % (existing_story.story_title, story.get('title')))
if existing_story.story_guid != story.get('guid'):
self.update_read_stories_with_new_guid(existing_story.story_guid, story.get('guid'))
existing_story.story_feed = self.pk
existing_story.story_date = story.get('published')
existing_story.story_title = story.get('title')
existing_story.story_content = story_content_diff
existing_story.story_original_content = original_content
existing_story.story_author_name = story.get('author')
existing_story.story_permalink = story_link
existing_story.story_guid = story.get('guid')
existing_story.story_tags = story_tags
try:
existing_story.save()
ret_values[ENTRY_UPDATED] += 1
except (IntegrityError, OperationError):
ret_values[ENTRY_ERR] += 1
if verbose:
logging.info('Saving updated story, IntegrityError: %s - %s' % (self.feed_title, story.get('title')))
except ValidationError, e:
ret_values[ENTRY_ERR] += 1
if verbose:
logging.info('Saving updated story, ValidationError: %s - %s: %s' % (self.feed_title, story.get('title'), e))
else:
ret_values[ENTRY_SAME] += 1
# logging.debug("Unchanged story: %s " % story.get('title'))
return ret_values
def update_read_stories_with_new_guid(self, old_story_guid, new_story_guid):
@ -913,8 +915,9 @@ class Feed(models.Model):
story_link = self.get_permalink(story)
start_date = story_pub_date - datetime.timedelta(hours=8)
end_date = story_pub_date + datetime.timedelta(hours=8)
for existing_story in existing_stories:
content_ratio = 0
existing_story_pub_date = existing_story.story_date
# print 'Story pub date: %s %s' % (story_published_now, story_pub_date)

18
apps/rss_feeds/tests.py
View file

@ -37,7 +37,7 @@ class FeedTest(TestCase):
url = reverse('load-single-feed', kwargs=dict(feed_id=1))
response = self.client.get(url)
feed = json.decode(response.content)
self.assertEquals(len(feed['stories']), 12)
self.assertEquals(len(feed['stories']), 6)
def test_load_feeds__gothamist(self):
self.client.login(username='conesus', password='test')
@ -55,7 +55,7 @@ class FeedTest(TestCase):
url = reverse('load-single-feed', kwargs=dict(feed_id=4))
response = self.client.get(url)
content = json.decode(response.content)
self.assertEquals(len(content['stories']), 12)
self.assertEquals(len(content['stories']), 6)
management.call_command('loaddata', 'gothamist_aug_2009_2.json', verbosity=0)
management.call_command('refresh_feed', force=1, feed=4, single_threaded=True, daemonize=False)
@ -68,13 +68,13 @@ class FeedTest(TestCase):
# print [c['story_title'] for c in json.decode(response.content)]
content = json.decode(response.content)
# Test: 1 changed char in title
self.assertEquals(len(content['stories']), 12)
self.assertEquals(len(content['stories']), 6)
def test_load_feeds__slashdot(self):
self.client.login(username='conesus', password='test')
old_story_guid = "{'original-id': u'http://yro.slashdot.org/story/09/09/05/0112254/Court-Allows-Microsoft-To-Sell-Word-During-Appeal?from=rss', 'gr:original-id': u'http://yro.slashdot.org/story/09/09/05/0112254/Court-Allows-Microsoft-To-Sell-Word-During-Appeal?from=rss'}"
new_story_guid = "{'original-id': u'http://yro.slashdot.org/story/09/09/05/0112254/Court-Allows-Microsoft-To-Sell-Word-During-Appeal?from=rss!!', 'gr:original-id': u'http://yro.slashdot.org/story/09/09/05/0112254/Court-Allows-Microsoft-To-Sell-Word-During-Appeal?from=rss!!'}"
old_story_guid = "tag:google.com,2005:reader/item/4528442633bc7b2b"
new_story_guid = "tag:google.com,2005:reader/item/4528442633bc7b2b!!'}"
management.call_command('loaddata', 'slashdot1.json', verbosity=0)
@ -90,7 +90,7 @@ class FeedTest(TestCase):
response = self.client.get(reverse('load-feeds'))
content = json.decode(response.content)
self.assertEquals(content['feeds']['5']['nt'], 38)
self.client.post(reverse('mark-story-as-read'), {'story_id': old_story_guid, 'feed_id': 5})
response = self.client.get(reverse('refresh-feeds'))
@ -110,12 +110,12 @@ class FeedTest(TestCase):
feed = json.decode(response.content)
# Test: 1 changed char in title
self.assertEquals(len(feed['stories']), 12)
self.assertEquals(len(feed['stories']), 6)
response = self.client.get(reverse('refresh-feeds'))
content = json.decode(response.content)
self.assertEquals(content['feeds']['5']['nt'], 37)
def test_load_feeds__brokelyn__invalid_xml(self):
self.client.login(username='conesus', password='test')
@ -129,7 +129,7 @@ class FeedTest(TestCase):
feed = json.decode(response.content)
# Test: 1 changed char in title
self.assertEquals(len(feed['stories']), 10)
self.assertEquals(len(feed['stories']), 6)
def test_all_feeds(self):
pass

2
utils/feed_fetcher.py
View file

@ -208,7 +208,7 @@ class ProcessFeed:
# if story.get('published') > end_date:
# end_date = story.get('published')
story_guids.append(story.get('guid') or story.get('link'))
existing_stories = list(MStory.objects(
# story_guid__in=story_guids,
story_date__gte=start_date,

2
utils/test-settings.py
View file

@ -15,7 +15,7 @@ MONGO_DB = {
}
TEST_DATABASE_NAME = ":memory:"
DAYS_OF_UNREAD = 9999
TEST_DEBUG = True
# from django.db import connection