Merge branch 'master' into realtime

* master:
  Adding 'E' as a keyboard shortcut for the Everything view.
  Doubling the amount Space pages in a story.
  Auto-linkify urls in stories.
  Reading stories from slave db during feed update.

Conflicts:
	settings.py

apps/rss_feeds/management/commands/refresh_feeds.py

@@ -64,6 +64,12 @@ class Command(BaseCommand):
         
         options['compute_scores'] = True
         
+            
+        import pymongo
+        db = pymongo.Connection(settings.MONGODB_SLAVE['host'], slave_okay=True, replicaset='nbset').newsblur
+        
+        options['slave_db'] = db
+        
         disp = feed_fetcher.Dispatcher(options, num_workers)        
         
         feeds_queue = []

apps/rss_feeds/models.py

@@ -9,7 +9,6 @@ import zlib
 import urllib
 from collections import defaultdict
 from operator import itemgetter
-from BeautifulSoup import BeautifulStoneSoup
 # from nltk.collocations import TrigramCollocationFinder, BigramCollocationFinder, TrigramAssocMeasures, BigramAssocMeasures
 from django.db import models
 from django.db import IntegrityError
@@ -551,7 +550,7 @@ class Feed(models.Model):
             self.data.feed_classifier_counts = json.encode(scores)
             self.data.save()
         
-    def update(self, force=False, single_threaded=True, compute_scores=True):
+    def update(self, force=False, single_threaded=True, compute_scores=True, slave_db=None):
         from utils import feed_fetcher
         try:
             self.feed_address = self.feed_address % {'NEWSBLUR_DIR': settings.NEWSBLUR_DIR}
@@ -567,6 +566,7 @@ class Feed(models.Model):
             'single_threaded': single_threaded,
             'force': force,
             'compute_scores': compute_scores,
+            'slave_db': slave_db,
         }
         disp = feed_fetcher.Dispatcher(options, 1)        
         disp.add_jobs([[self.pk]])
@@ -625,6 +625,7 @@ class Feed(models.Model):
                     # logging.debug('- Updated story in feed (%s - %s): %s / %s' % (self.feed_title, story.get('title'), len(existing_story.story_content), len(story_content)))
                     story_guid = story.get('guid') or story.get('id') or story.get('link')
                     original_content = None
+                    existing_story = MStory.objects.get(story_feed_id=existing_story.story_feed_id, story_guid=existing_story.story_guid)
                     if existing_story.story_original_content_z:
                         original_content = zlib.decompress(existing_story.story_original_content_z)
                     elif existing_story.story_content_z:

apps/rss_feeds/tasks.py

@@ -1,5 +1,6 @@
 from celery.task import Task
 from utils import log as logging
+from django.conf import settings
 
 class UpdateFeeds(Task):
     name = 'update-feeds'
@@ -11,10 +12,13 @@ class UpdateFeeds(Task):
         if not isinstance(feed_pks, list):
             feed_pks = [feed_pks]
             
+        import pymongo
+        db = pymongo.Connection(settings.MONGODB_SLAVE['host'], slave_okay=True, replicaset='nbset').newsblur
+
         for feed_pk in feed_pks:
             try:
                 feed = Feed.objects.get(pk=feed_pk)
-                feed.update()
+                feed.update(slave_db=db)
             except Feed.DoesNotExist:
                 logging.info(" ---> Feed doesn't exist: [%s]" % feed_pk)
             # logging.debug(' Updating: [%s] %s' % (feed_pks, feed))

media/js/jquery.newsblur.js

@@ -5,6 +5,8 @@ if (typeof NEWSBLUR.Globals == 'undefined') NEWSBLUR.Globals = {};
 /* = Core NewsBlur Javascript = */
 /* ============================= */
 
+var URL_REGEX = /((https?\:\/\/)|(www\.))(\S+)(\w{2,4})(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?/gi;
+
 NEWSBLUR.log = function(msg) {
     try {
         if (typeof o == "object")
@@ -31,6 +33,36 @@ NEWSBLUR.log = function(msg) {
 
     $.fn.extend({
         
+        autolink: function() {
+            return this.each(function(){
+                var desc = $(this);
+                desc.textNodes().each(function(){
+                    var text = $(this);
+                    if(text && text.parent() && text.parent()[0] && text.parent()[0].nodeName != 'A') {
+                        text.replaceWith(this.data.replace(URL_REGEX, function($0, $1) {
+                            return '<a href="' + $0 +'">' + $0 + '</a>';
+                        }));
+                    }
+                });
+            });
+        },
+
+        textNodes: function() {
+            var ret = [];
+
+            (function(el){
+                if (!el) return;
+                if ((el.nodeType == 3)) {
+                    ret.push(el);
+                } else {
+                    for (var i=0; i < el.childNodes.length; ++i) {
+                        arguments.callee(el.childNodes[i]);
+                    }
+                }
+            })(this[0]);
+            return $(ret);
+        },
+    
         isScrollVisible: function($elem) {
             var docViewTop = 0; // $(this).scrollTop();
             var docViewBottom = docViewTop + $(this).height();

media/js/newsblur/reader.js

@@ -3401,7 +3401,7 @@
                                 $.make('span', { className: 'NB-feed-story-starred-date' }, story.starred_date))
                         ])
                     ]),
-                    $.make('div', { className: 'NB-feed-story-content' }, story.story_content)                
+                    $.make('div', { className: 'NB-feed-story-content' }, this.make_story_content(story.story_content))
                 ]).data('story', story.id).data('story_id', story.id).data('feed_id', story.story_feed_id);
                 
                 if (story_has_modifications && this.model.preference('show_tooltips')) {
@@ -3457,6 +3457,11 @@
             if (first_load) this.show_stories_preference_in_feed_view(true);
         },
         
+        make_story_content: function(story_content) {
+            var $story_content = $('<div>').html(story_content).autolink();
+            return $story_content;
+        },
+        
         make_story_feed_title: function(story) {
             var title = story.story_title;
             var feed_titles = this.model.classifiers[story.story_feed_id] && 
@@ -6486,11 +6491,11 @@
             });
             $document.bind('keydown', 'space', function(e) {
                 e.preventDefault();
-                self.page_in_story(0.2, 1);
+                self.page_in_story(0.4, 1);
             });
             $document.bind('keydown', 'shift+space', function(e) {
                 e.preventDefault();
-                self.page_in_story(0.4, -1);
+                self.page_in_story(0.6, -1);
             });
             $document.bind('keydown', 'u', function(e) {
                 e.preventDefault();
@@ -6556,6 +6561,10 @@
                     self.mark_feed_as_read();
                 }
             });
+            $document.bind('keydown', 'shift+e', function(e) {
+                e.preventDefault();
+                self.open_river_stories();
+            });
         }
         
     };

settings.py

@@ -422,6 +422,7 @@ class MasterSlaveRouter(object):
     def allow_syncdb(self, db, model):
         "Explicitly put all models on all databases."
         return True
+<<<<<<< HEAD
         
 # =========
 # = Redis =
@@ -429,6 +430,15 @@ class MasterSlaveRouter(object):
 
 REDIS = {
     'host': 'db02',
+=======
+
+# ===========
+# = MongoDB =
+# ===========
+
+MONGODB_SLAVE = {
+    'host': 'db01'
+>>>>>>> master
 }
 
 # ==================
@@ -470,4 +480,4 @@ MONGODB = connect(MONGO_DB.pop('name'), **MONGO_DB)
 # = Redis =
 # =========
 
-REDIS_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=0)
+REDIS_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=0)

utils/feed_fetcher.py

@@ -11,6 +11,7 @@ from utils import feedparser
 from utils.story_functions import pre_process_story
 from utils import log as logging
 from utils.feed_functions import timelimit, TimeoutError, mail_feed_error_to_admin, utf8encode
+from utils.story_functions import bunch
 import time
 import datetime
 import traceback
@@ -215,11 +216,24 @@ class ProcessFeed:
             # if story.get('published') > end_date:
             #     end_date = story.get('published')
             story_guids.append(story.get('guid') or story.get('link'))
-        existing_stories = list(MStory.objects(
-            # story_guid__in=story_guids,
-            story_date__gte=start_date,
-            story_feed_id=self.feed.pk
-        ).limit(len(story_guids)))
+        
+        if self.options['slave_db']:
+            slave_db = self.options['slave_db']
+            stories_db_orig = slave_db.stories.find({
+                "story_feed_id": self.feed.pk,
+                "story_date": {
+                    "$gte": start_date,
+                },
+            }).limit(len(story_guids))
+            existing_stories = []
+            for story in stories_db_orig:
+                existing_stories.append(bunch(story))
+        else:
+            existing_stories = list(MStory.objects(
+                # story_guid__in=story_guids,
+                story_date__gte=start_date,
+                story_feed_id=self.feed.pk
+            ).limit(len(story_guids)))
         
         # MStory.objects(
         #     (Q(story_date__gte=start_date) & Q(story_date__lte=end_date))
@@ -228,10 +242,9 @@ class ProcessFeed:
         # ).order_by('-story_date')
         ret_values = self.feed.add_update_stories(self.fpf.entries, existing_stories)
         
-        logging.debug(u'   ---> [%-30s] Parsed Feed: %s' % (
+        logging.debug(u'   ---> [%-30s] ~FYParsed Feed: new~FG=~FG~SB%s~SN~FY up~FG=~FY~SB%s~SN same~FG=~FY%s err~FG=~FR~SB%s' % (
                       unicode(self.feed)[:30], 
-                      u' '.join(u'%s=%d' % (self.entry_trans[key],
-                              ret_values[key]) for key in self.entry_keys),))
+                      ret_values[ENTRY_NEW], ret_values[ENTRY_UPDATED], ret_values[ENTRY_SAME], ret_values[ENTRY_ERR]))
         self.feed.update_all_statistics()
         self.feed.trim_feed()
         self.feed.save_feed_history(200, "OK")
@@ -414,8 +427,21 @@ class Dispatcher:
                       unicode(feed)[:30], user_subs.count(),
                       feed.num_subscribers, feed.active_subscribers, feed.premium_subscribers))
         
-        stories_db = MStory.objects(story_feed_id=feed.pk,
-                                    story_date__gte=UNREAD_CUTOFF)
+        if self.options['slave_db']:
+            slave_db = self.options['slave_db']
+
+            stories_db_orig = slave_db.stories.find({
+                "story_feed_id": feed.pk,
+                "story_date": {
+                    "$gte": UNREAD_CUTOFF,
+                },
+            })
+            stories_db = []
+            for story in stories_db_orig:
+                stories_db.append(bunch(story))
+        else:
+            stories_db = MStory.objects(story_feed_id=feed.pk,
+                                        story_date__gte=UNREAD_CUTOFF)
         for sub in user_subs:
             cache.delete('usersub:%s' % sub.user_id)
             sub.needs_unread_recalc = True