Testing out a slave read for getting stories.

apps/reader/views.py

@@ -335,7 +335,7 @@ def load_single_feed(request, feed_id):
     page         = int(request.REQUEST.get('page', 1))
     dupe_feed_id = None
     userstories_db = None
-    
+        
     if page: offset = limit * (page-1)
     if not feed_id: raise Http404
         
@@ -350,7 +350,7 @@ def load_single_feed(request, feed_id):
         else:
             raise Http404
         
-    stories = feed.get_stories(offset, limit) 
+    stories = feed.get_stories(offset, limit, slave=True) 
         
     # Get intelligence classifier for user
     classifier_feeds   = list(MClassifierFeed.objects(user_id=user.pk, feed_id=feed_id))

apps/rss_feeds/models.py

@@ -29,6 +29,7 @@ from utils.feed_functions import timelimit, TimeoutError
 from utils.feed_functions import relative_timesince
 from utils.feed_functions import seconds_timesince
 from utils.story_functions import pre_process_story
+from utils.story_functions import bunch
 from utils.diff import HTMLDiff
 
 ENTRY_NEW, ENTRY_UPDATED, ENTRY_SAME, ENTRY_ERR = range(4)
@@ -742,11 +743,19 @@ class Feed(models.Model):
                 # print "Found %s user stories. Deleting..." % userstories.count()
                 userstories.delete()
         
-    def get_stories(self, offset=0, limit=25, force=False):
+    def get_stories(self, offset=0, limit=25, force=False, slave=False):
         stories = cache.get('feed_stories:%s-%s-%s' % (self.id, offset, limit), [])
         
         if not stories or force:
-            stories_db = MStory.objects(story_feed_id=self.pk)[offset:offset+limit]
+            if slave:
+                import pymongo
+                db = pymongo.Connection(['localhost:27017'], slave_okay=True, replicaset='nbset').newsblur
+                stories_db_orig = db.stories.find({"story_feed_id": self.pk})[offset:offset+limit]
+                stories_db = []
+                for story in stories_db_orig:
+                    stories_db.append(bunch(story))
+            else:
+                stories_db = MStory.objects(story_feed_id=self.pk)[offset:offset+limit]
             stories = Feed.format_stories(stories_db, self.pk)
             cache.set('feed_stories:%s-%s-%s' % (self.id, offset, limit), stories)
         

apps/statistics/models.py

@@ -55,12 +55,15 @@ class MStatistics(mongo.Document):
         feeds_fetched = MFeedFetchHistory.objects.count()
         cls.objects(key='feeds_fetched').update_one(upsert=True, key='feeds_fetched', value=feeds_fetched)
         
-        from utils.feed_functions import timelimit
+        from utils.feed_functions import timelimit, TimeoutError
         @timelimit(60)
         def delete_old_history():
             MFeedFetchHistory.objects(fetch_date__lt=last_day).delete()
             MPageFetchHistory.objects(fetch_date__lt=last_day).delete()
-        delete_old_history()
+        try:
+            delete_old_history()
+        except TimeoutError:
+            print "Timed out on deleting old history. Shit."
         
         return feeds_fetched
         

settings.py

@@ -452,7 +452,6 @@ DEBUG_TOOLBAR_CONFIG = {
 MONGO_DB_DEFAULTS = {
     'name': 'newsblur',
     'host': 'mongodb://db01,db02/?slaveOk=true',
-    # 'tz_aware': True,
 }
 MONGO_DB = dict(MONGO_DB_DEFAULTS, **MONGO_DB)
 MONGODB = connect(MONGO_DB.pop('name'), **MONGO_DB)