Merge branch 'master' into chooser

apps/rss_feeds/models.py

@@ -18,6 +18,7 @@ from mongoengine.queryset import OperationError
 from utils import json
 from utils import feedfinder
 from utils.feed_functions import levenshtein_distance
+from utils.feed_functions import timelimit
 from utils.story_functions import format_story_link_date__short
 from utils.story_functions import format_story_link_date__long
 from utils.story_functions import pre_process_story
@@ -77,6 +78,7 @@ class Feed(models.Model):
         self.save_popular_authors(lock=lock)
         self.save_popular_tags(lock=lock)
     
+    @timelimit(20)
     def check_feed_address_for_feed_link(self):
         feed_address = None
 
@@ -111,7 +113,7 @@ class Feed(models.Model):
                           message=message,
                           exception=exception,
                           fetch_date=datetime.datetime.now()).save()
-        old_fetch_histories = MFeedFetchHistory.objects(feed_id=self.pk).order_by('-fetch_date')[10:]
+        old_fetch_histories = MFeedFetchHistory.objects(feed_id=self.pk).order_by('-fetch_date')[5:]
         for history in old_fetch_histories:
             history.delete()
             
@@ -130,7 +132,7 @@ class Feed(models.Model):
                           message=message,
                           exception=exception,
                           fetch_date=datetime.datetime.now()).save()
-        old_fetch_histories = MPageFetchHistory.objects(feed_id=self.pk).order_by('-fetch_date')[10:]
+        old_fetch_histories = MPageFetchHistory.objects(feed_id=self.pk).order_by('-fetch_date')[5:]
         for history in old_fetch_histories:
             history.delete()
             
@@ -798,7 +800,7 @@ class MFeedFetchHistory(mongo.Document):
     meta = {
         'collection': 'feed_fetch_history',
         'allow_inheritance': False,
-        'indexes': ['feed_id', ('feed_id', 'status_code'), ('feed_id', 'fetch_date')],
+        'indexes': ['feed_id', ('fetch_date', 'status_code'), ('feed_id', 'status_code'), ('feed_id', 'fetch_date')],
     }
     
     def save(self, *args, **kwargs):
@@ -833,7 +835,7 @@ class MPageFetchHistory(mongo.Document):
     meta = {
         'collection': 'page_fetch_history',
         'allow_inheritance': False,
-        'indexes': ['feed_id', ('feed_id', 'status_code'), ('feed_id', 'fetch_date')],
+        'indexes': ['feed_id', ('fetch_date', 'status_code'), ('feed_id', 'status_code'), ('feed_id', 'fetch_date')],
     }
     
     def save(self, *args, **kwargs):

utils/feed_fetcher.py

@@ -9,6 +9,7 @@ from utils import feedparser
 from django.db import IntegrityError
 from utils.story_functions import pre_process_story
 from utils import log as logging
+from utils.feed_functions import timelimit
 import sys
 import time
 import datetime
@@ -35,36 +36,6 @@ def mtime(ttime):
     """
     return datetime.datetime.fromtimestamp(time.mktime(ttime))
     
-import threading
-class TimeoutError(Exception): pass
-def timelimit(timeout):
-    """borrowed from web.py"""
-    def _1(function):
-        def _2(*args, **kw):
-            class Dispatch(threading.Thread):
-                def __init__(self):
-                    threading.Thread.__init__(self)
-                    self.result = None
-                    self.error = None
-                    
-                    self.setDaemon(True)
-                    self.start()
-
-                def run(self):
-                    try:
-                        self.result = function(*args, **kw)
-                    except:
-                        self.error = sys.exc_info()
-
-            c = Dispatch()
-            c.join(timeout)
-            if c.isAlive():
-                raise TimeoutError, 'took too long'
-            if c.error:
-                raise c.error[0], c.error[1]
-            return c.result
-        return _2
-    return _1
     
 class FetchFeed:
     def __init__(self, feed, options):

utils/feed_functions.py

@@ -1,7 +1,39 @@
 import datetime
+import threading
+import sys
 from django.utils.translation import ungettext
 from utils import feedfinder
 
+class TimeoutError(Exception): pass
+def timelimit(timeout):
+    """borrowed from web.py"""
+    def _1(function):
+        def _2(*args, **kw):
+            class Dispatch(threading.Thread):
+                def __init__(self):
+                    threading.Thread.__init__(self)
+                    self.result = None
+                    self.error = None
+                    
+                    self.setDaemon(True)
+                    self.start()
+
+                def run(self):
+                    try:
+                        self.result = function(*args, **kw)
+                    except:
+                        self.error = sys.exc_info()
+
+            c = Dispatch()
+            c.join(timeout)
+            if c.isAlive():
+                raise TimeoutError, 'took too long'
+            if c.error:
+                raise c.error[0], c.error[1]
+            return c.result
+        return _2
+    return _1
+    
 def encode(tstr):
     """ Encodes a unicode string in utf-8
     """