diff --git a/apps/reader/views.py b/apps/reader/views.py index 5fefd8590..7b550c15b 100644 --- a/apps/reader/views.py +++ b/apps/reader/views.py @@ -335,7 +335,7 @@ def load_single_feed(request, feed_id): page = int(request.REQUEST.get('page', 1)) dupe_feed_id = None userstories_db = None - + if page: offset = limit * (page-1) if not feed_id: raise Http404 @@ -350,7 +350,7 @@ def load_single_feed(request, feed_id): else: raise Http404 - stories = feed.get_stories(offset, limit) + stories = feed.get_stories(offset, limit, slave=True) # Get intelligence classifier for user classifier_feeds = list(MClassifierFeed.objects(user_id=user.pk, feed_id=feed_id)) diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index f3a62137f..c56b33148 100644 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -29,6 +29,7 @@ from utils.feed_functions import timelimit, TimeoutError from utils.feed_functions import relative_timesince from utils.feed_functions import seconds_timesince from utils.story_functions import pre_process_story +from utils.story_functions import bunch from utils.diff import HTMLDiff ENTRY_NEW, ENTRY_UPDATED, ENTRY_SAME, ENTRY_ERR = range(4) @@ -742,11 +743,19 @@ class Feed(models.Model): # print "Found %s user stories. Deleting..." % userstories.count() userstories.delete() - def get_stories(self, offset=0, limit=25, force=False): + def get_stories(self, offset=0, limit=25, force=False, slave=False): stories = cache.get('feed_stories:%s-%s-%s' % (self.id, offset, limit), []) if not stories or force: - stories_db = MStory.objects(story_feed_id=self.pk)[offset:offset+limit] + if slave: + import pymongo + db = pymongo.Connection(['localhost:27017'], slave_okay=True, replicaset='nbset').newsblur + stories_db_orig = db.stories.find({"story_feed_id": self.pk})[offset:offset+limit] + stories_db = [] + for story in stories_db_orig: + stories_db.append(bunch(story)) + else: + stories_db = MStory.objects(story_feed_id=self.pk)[offset:offset+limit] stories = Feed.format_stories(stories_db, self.pk) cache.set('feed_stories:%s-%s-%s' % (self.id, offset, limit), stories) diff --git a/apps/statistics/models.py b/apps/statistics/models.py index f307de4cb..e3be02933 100644 --- a/apps/statistics/models.py +++ b/apps/statistics/models.py @@ -55,12 +55,15 @@ class MStatistics(mongo.Document): feeds_fetched = MFeedFetchHistory.objects.count() cls.objects(key='feeds_fetched').update_one(upsert=True, key='feeds_fetched', value=feeds_fetched) - from utils.feed_functions import timelimit + from utils.feed_functions import timelimit, TimeoutError @timelimit(60) def delete_old_history(): MFeedFetchHistory.objects(fetch_date__lt=last_day).delete() MPageFetchHistory.objects(fetch_date__lt=last_day).delete() - delete_old_history() + try: + delete_old_history() + except TimeoutError: + print "Timed out on deleting old history. Shit." return feeds_fetched diff --git a/settings.py b/settings.py index 350be4dd4..9d4b6f104 100644 --- a/settings.py +++ b/settings.py @@ -452,7 +452,6 @@ DEBUG_TOOLBAR_CONFIG = { MONGO_DB_DEFAULTS = { 'name': 'newsblur', 'host': 'mongodb://db01,db02/?slaveOk=true', - # 'tz_aware': True, } MONGO_DB = dict(MONGO_DB_DEFAULTS, **MONGO_DB) MONGODB = connect(MONGO_DB.pop('name'), **MONGO_DB)