Merge branch 'master' of github.com:samuelclay/NewsBlur

.gitignore

@@ -16,3 +16,4 @@ build/
 **/*.perspectivev*
 *.pbxuser
 data/
+logs

apps/rss_feeds/models.py

@@ -794,6 +794,7 @@ class MFeedFetchHistory(mongo.Document):
     meta = {
         'collection': 'feed_fetch_history',
         'allow_inheritance': False,
+        'indexes': ['feed_id', ('feed_id', 'status_code'), ('feed_id', 'fetch_date')],
     }
     
     def save(self, *args, **kwargs):
@@ -828,6 +829,7 @@ class MPageFetchHistory(mongo.Document):
     meta = {
         'collection': 'page_fetch_history',
         'allow_inheritance': False,
+        'indexes': ['feed_id', ('feed_id', 'status_code'), ('feed_id', 'fetch_date')],
     }
         
 class DuplicateFeed(models.Model):

media/js/newsblur/reader.js

@@ -2330,7 +2330,7 @@
         
         setup_feed_refresh: function() {
             var self = this;
-            var FEED_REFRESH_INTERVAL = (1000 * 60) / 12; // 1/2 minutes
+            var FEED_REFRESH_INTERVAL = (1000 * 60) / 2; // 1/2 minutes
             
             clearInterval(this.flags.feed_refresh);
             

media/maintenance.html.unused

@@ -0,0 +1,5 @@
+<h1>NewsBlur is upgrading...</h1>
+
+<p>As of Sunday, August 22nd 8:30pm, NewsBlur is going through wonderful backend changes that will make it much, much faster. (PostgreSQL -> MongoDB, for you nerd types.)</p>
+
+<p>This change will take a few hours. Rest easy knowing I won't sleep until it's done.</p>

utils/bootstrap_mongo.py

@@ -112,7 +112,7 @@ def bootstrap_feedpages():
     # db.feed_pages.drop()
     print "Dropped! Mongo DB feed_pages: %s" % MFeedPage.objects().count()
 
-    print "FeedPages: %s" % MFeedPage.objects().count()
+    print "FeedPages: %s" % FeedPage.objects.count()
     pprint(db.feed_pages.index_information())
 
     feeds = Feed.objects.all().order_by('-average_stories_per_month')

utils/feed_fetch.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+ps aux | grep refresh_feeds | egrep -v grep | awk '{print $2}' | xargs kill > /dev/null 2>&1
+python /home/conesus/newsblur/manage.py refresh_feeds -s &
+python /home/conesus/newsblur/manage.py refresh_feeds -s &
+python /home/conesus/newsblur/manage.py refresh_feeds -s &
+python /home/conesus/newsblur/manage.py refresh_feeds -s &
+python /home/conesus/newsblur/manage.py refresh_feeds -s &
+python /home/conesus/newsblur/manage.py refresh_feeds -s &

utils/feed_fetch_silent.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+ps aux | grep refresh_feeds | egrep -v grep | awk '{print $2}' | xargs kill > /dev/null 2>&1
+python /home/conesus/newsblur/manage.py refresh_feeds -s > /dev/null 2>&1 &
+python /home/conesus/newsblur/manage.py refresh_feeds -s > /dev/null 2>&1 &
+python /home/conesus/newsblur/manage.py refresh_feeds -s > /dev/null 2>&1 &
+python /home/conesus/newsblur/manage.py refresh_feeds -s > /dev/null 2>&1 &
+python /home/conesus/newsblur/manage.py refresh_feeds -s > /dev/null 2>&1 &
+python /home/conesus/newsblur/manage.py refresh_feeds -s > /dev/null 2>&1 &
+python /home/conesus/newsblur/manage.py refresh_feeds -s > /dev/null 2>&1 &
+

utils/feed_fetcher.py

@@ -33,12 +33,13 @@ def mtime(ttime):
     """
     return datetime.datetime.fromtimestamp(time.mktime(ttime))
     
+    
 class FetchFeed:
     def __init__(self, feed, options):
         self.feed = feed
         self.options = options
         self.fpf = None
-
+    
     def fetch(self):
         """ Downloads and parses a feed.
         """
@@ -57,7 +58,9 @@ class FetchFeed:
             logging.debug(log_msg)
             feed.save_feed_history(303, "Already fetched")
             return FEED_SAME, None
-        
+        else:
+            feed.set_next_scheduled_update()
+            
         etag=self.feed.etag
         modified = self.feed.last_modified.utctimetuple()[:7] if self.feed.last_modified else None
         
@@ -263,9 +266,7 @@ class Dispatcher:
                 ENTRY_ERR: 0
             }
             start_time = datetime.datetime.now()
-        
-            feed.set_next_scheduled_update()
-            
+                    
             ### Uncomment to test feed fetcher
             # from random import randint
             # if randint(0,10) < 10:

utils/munin/newsblur_errors.py

@@ -7,7 +7,7 @@ import datetime
 
 graph_config = {
     'graph_category' : 'NewsBlur',
-    'graph_title' : 'NewsBlur Errors',
+    'graph_title' : 'NewsBlur Fetching History',
     'graph_vlabel' : 'errors',
     'feed_errors.label': 'Feed Errors',
     'feed_success.label': 'Feed Success',

utils/munin/newsblur_feeds.py

@@ -7,7 +7,7 @@ from django.db.models import Q
 
 graph_config = {
     'graph_category' : 'NewsBlur',
-    'graph_title' : 'NewsBlur Feeds',
+    'graph_title' : 'NewsBlur Feeds & Subscriptions',
     'graph_vlabel' : 'Feeds & Subscribers',
     'feeds.label': 'feeds',
     'subscriptions.label': 'subscriptions',