Merge branch 'master' into story_hash_2

# By Samuel Clay (10) and others
* master:
  Further pushing out feed updates.
  Using @dcramer's excellent getsentry.com for all exceptions. Turning off exception emails.
  Further pushing down feed fetches to handle load until db is scaled out.
  Scrolling code blocks.
  New task servers.
  Turning down feed fetches.
  If requests.raw won't work, sut fudge it with a StringIO for now.
  Further turning down feed fetches until mongo is ready to replicate.
  Fixing Facebook image calculation.
  Fixing facebook posting.
  Fixed API docs for /reader/river_stories
  make fabfile.py use requirements.txt
  Make sure to send the proper content-type with social feeds
  Restore default whitespace to pre/code blocks
  document keyboard shortcut to add site/folder

apps/rss_feeds/models.py

@@ -1233,7 +1233,7 @@ class Feed(models.Model):
         # .5 hours for 2 subscribers.
         # .25 hours for 3 subscribers.
         # 1 min for 10 subscribers.
-        subscriber_bonus = 6 * 60 / max(.167, max(0, self.active_subscribers)**3)
+        subscriber_bonus = 12 * 60 / max(.167, max(0, self.active_subscribers)**3)
         if self.premium_subscribers > 0:
             subscriber_bonus /= min(self.active_subscribers+self.premium_subscribers, 5)
         
@@ -1245,13 +1245,13 @@ class Feed(models.Model):
                 slow_punishment = 2 * self.last_load_time
             elif self.last_load_time >= 200:
                 slow_punishment = 6 * self.last_load_time
-        total = max(5, int(updates_per_day_delay + subscriber_bonus + slow_punishment))
+        total = max(10, int(updates_per_day_delay + subscriber_bonus + slow_punishment))
         
-        if self.active_premium_subscribers > 3:
+        if self.active_premium_subscribers > 5:
             total = min(total, 60) # 1 hour minimum for premiums
 
         if ((self.stories_last_month == 0 or self.average_stories_per_month == 0)):
-            total = total * random.randint(1, 12)
+            total = total * random.randint(1, 24)
         
         if self.is_push:
             total = total * 20

apps/rss_feeds/page_importer.py

@@ -11,7 +11,8 @@ from django.conf import settings
 from django.utils.text import compress_string
 from utils import log as logging
 from apps.rss_feeds.models import MFeedPage
-from utils.feed_functions import timelimit, mail_feed_error_to_admin
+from utils.feed_functions import timelimit
+# from utils.feed_functions import mail_feed_error_to_admin
 
 BROKEN_PAGES = [
     'tag:', 
@@ -120,7 +121,10 @@ class PageImporter(object):
             logging.debug(tb)
             logging.debug('[%d] ! -------------------------' % (self.feed.id,))
             self.feed.save_page_history(500, "Error", tb)
-            mail_feed_error_to_admin(self.feed, e, local_vars=locals())
+            # mail_feed_error_to_admin(self.feed, e, local_vars=locals())
+            if (not settings.DEBUG and hasattr(settings, 'RAVEN_CLIENT') and
+                settings.RAVEN_CLIENT):
+                settings.RAVEN_CLIENT.captureException()
             if not urllib_fallback:
                 self.fetch_page(urllib_fallback=True)
         else:

apps/rss_feeds/tasks.py

@@ -20,7 +20,7 @@ class TaskFeeds(Task):
             next_scheduled_update__lte=now,
             active=True,
             active_premium_subscribers__gte=1
-        ).order_by('?')[:600]
+        ).order_by('?')[:400]
         popular_count = popular_feeds.count()
         
         # Regular feeds
@@ -28,7 +28,7 @@ class TaskFeeds(Task):
             next_scheduled_update__lte=now,
             active=True,
             active_subscribers__gte=1
-        ).order_by('?')[:400]
+        ).order_by('?')[:200]
         active_count = feeds.count()
         
         # Mistakenly inactive feeds

apps/social/models.py

@@ -33,6 +33,7 @@ from utils.feed_functions import relative_timesince
 from utils.story_functions import truncate_chars, strip_tags, linkify, image_size
 from utils.scrubber import SelectiveScriptScrubber
 from utils import s3_utils
+from StringIO import StringIO
 
 RECOMMENDATIONS_LIMIT = 5
 IGNORE_IMAGE_SOURCES = [
@@ -1207,7 +1208,7 @@ class MSharedStory(mongo.Document):
                     ('user_id', 'story_db_id'),
                     'shared_date', 'story_guid', 'story_feed_id'],
         'index_drop_dups': True,
-        'ordering': ['shared_date'],
+        'ordering': ['-shared_date'],
         'allow_inheritance': False,
     }
 
@@ -1879,11 +1880,13 @@ class MSharedStory(mongo.Document):
         soup = BeautifulSoup(zlib.decompress(self.story_content_z))
         image_sources = [img.get('src') for img in soup.findAll('img')]
         image_sizes = []
+        
         for image_source in image_sources[:10]:
             if any(ignore in image_source for ignore in IGNORE_IMAGE_SOURCES):
                 continue
-            r = requests.get(image_source, prefetch=False, headers=headers)
+            req = requests.get(image_source, headers=headers, stream=True)
-            _, width, height = image_size(r.raw)
+            datastream = StringIO(req.content[:30])
+            _, width, height = image_size(datastream)
             if width <= 16 or height <= 16:
                 continue
             image_sizes.append({'src': image_source, 'size': (width, height)})

apps/social/views.py

@@ -1206,7 +1206,7 @@ def shared_stories_rss_feed(request, user_id, username):
         user.username,
         request.META['HTTP_USER_AGENT'][:24]
     ))
-    return HttpResponse(rss.writeString('utf-8'))
+    return HttpResponse(rss.writeString('utf-8'), content_type='application/rss+xml')
 
 @required_params('user_id')
 @json.json_view

config/requirements.txt

@@ -1,10 +1,17 @@
-django
+fabric
-mongoengine
+django==1.3.1
+readline
+chardet
+pyflakes
+iconv
+celery
 django-celery
+django-celery-with-redis
 django-compress
 South
 django-extensions
-psycopg2
+pymongo==2.2.0
+stripe
 BeautifulSoup
 pyyaml
 nltk
@@ -12,4 +19,15 @@ lxml
 oauth2
 pytz
 boto
-gunicorn
+seacucumber
+django_ses
+django-mailgun
+mongoengine
+redis
+requests
+django-subdomains
+psutil
+python-gflags
+cssutils
+raven
+pyes

fabfile.py

@@ -96,6 +96,9 @@ env.roledefs ={
                 
                 'ec2-54-234-211-75.compute-1.amazonaws.com',
                 'ec2-50-16-97-13.compute-1.amazonaws.com',
+                'ec2-54-242-131-232.compute-1.amazonaws.com',
+                'ec2-75-101-195-131.compute-1.amazonaws.com',
+                'ec2-54-242-105-17.compute-1.amazonaws.com',
                 ],
     'vps': ['task01.newsblur.com', 
             'task03.newsblur.com', 
@@ -492,9 +495,10 @@ def setup_psycopg():
 
 def setup_python():
     # sudo('easy_install -U pip')
-    sudo('easy_install -U fabric django==1.3.1 readline chardet pyflakes iconv celery django-celery django-celery-with-redis django-compress South django-extensions pymongo==2.2.0 stripe BeautifulSoup pyyaml nltk lxml oauth2 pytz boto seacucumber django_ses django-mailgun mongoengine redis requests django-subdomains psutil python-gflags cssutils raven pyes')
+    sudo('easy_install -U $(<%s)' %
-    
+         os.path.join(env.NEWSBLUR_PATH, 'config/requirements.txt'))
     put('config/pystartup.py', '.pystartup')
+
     # with cd(os.path.join(env.NEWSBLUR_PATH, 'vendor/cjson')):
     #     sudo('python setup.py install')
 

media/css/reader.css

@@ -1800,6 +1800,10 @@ background: transparent;
 #story_pane .NB-feed-story:first-child .NB-feed-story-header {
     padding-top: 0;
 }
+#story_pane .NB-feed-stories pre {
+    overflow-x: auto;
+    max-width: 100%;
+}
 
 #story_pane .NB-feed-story-header-info {
     font-weight: bold;
@@ -1901,10 +1905,6 @@ background: transparent;
 .NB-feed-story .NB-feed-story-content div {
     max-width: 100%;
 }
-.NB-feed-story .NB-feed-story-content pre,
-.NB-feed-story .NB-feed-story-content code {
-    white-space: normal;
-}
 .NB-feed-story .NB-feed-story-content img {
   max-width: 100% !important;
   width: auto;

media/js/newsblur/reader/reader_keyboard.js

@@ -270,6 +270,14 @@ NEWSBLUR.ReaderKeyboard.prototype = {
                     '?'
                 ])
               ])
+            ]),
+            $.make('div', { className: 'NB-keyboard-group' }, [
+              $.make('div', { className: 'NB-keyboard-shortcut NB-last' }, [
+                $.make('div', { className: 'NB-keyboard-shortcut-explanation' }, 'Add Site/Folder'),
+                $.make('div', { className: 'NB-keyboard-shortcut-key' }, [
+                    'a'
+                ])
+              ])
             ])
         ]);
     },

settings.py

@@ -149,7 +149,7 @@ LOGGING = {
     },
     'loggers': {
         'django.request': {
-            'handlers': ['mail_admins'],
+            'handlers': ['console', 'log_file'],
             'level': 'ERROR',
             'propagate': True,
         },

templates/static/api.yml

@@ -229,6 +229,11 @@
           optional: true
           default: newest
           example: oldest
+        - key: read_filter
+          desc: "Show all stories or only unread stories"
+          optional: true
+          default: unread
+          example: all
             
     - url: /reader/mark_story_as_read
       method: POST

utils/feed_fetcher.py

@@ -18,7 +18,8 @@ from apps.statistics.models import MAnalyticsFetcher
 from utils import feedparser
 from utils.story_functions import pre_process_story
 from utils import log as logging
-from utils.feed_functions import timelimit, TimeoutError, mail_feed_error_to_admin, utf8encode
+from utils.feed_functions import timelimit, TimeoutError, utf8encode
+# from utils.feed_functions import mail_feed_error_to_admin
 
 
 # Refresh feed code adapted from Feedjack.
@@ -386,7 +387,7 @@ class Dispatcher:
                 feed.save_feed_history(500, "Error", tb)
                 feed_code = 500
                 fetched_feed = None
-                mail_feed_error_to_admin(feed, e, local_vars=locals())
+                # mail_feed_error_to_admin(feed, e, local_vars=locals())
                 if (not settings.DEBUG and hasattr(settings, 'RAVEN_CLIENT') and
                     settings.RAVEN_CLIENT):
                     settings.RAVEN_CLIENT.captureException()
@@ -431,7 +432,9 @@ class Dispatcher:
                     feed.save_page_history(550, "Page Error", tb)
                     fetched_feed = None
                     page_data = None
-                    mail_feed_error_to_admin(feed, e, local_vars=locals())
+                    # mail_feed_error_to_admin(feed, e, local_vars=locals())
+                    if (not settings.DEBUG and hasattr(settings, 'RAVEN_CLIENT') and
+                        settings.RAVEN_CLIENT):
                         settings.RAVEN_CLIENT.captureException()
 
                 feed = self.refresh_feed(feed.pk)
@@ -449,7 +452,9 @@ class Dispatcher:
                     logging.error(tb)
                     logging.debug('[%d] ! -------------------------' % (feed_id,))
                     # feed.save_feed_history(560, "Icon Error", tb)
-                    mail_feed_error_to_admin(feed, e, local_vars=locals())
+                    # mail_feed_error_to_admin(feed, e, local_vars=locals())
+                    if (not settings.DEBUG and hasattr(settings, 'RAVEN_CLIENT') and
+                        settings.RAVEN_CLIENT):
                         settings.RAVEN_CLIENT.captureException()
             else:
                 logging.debug(u'   ---> [%-30s] ~FBSkipping page fetch: (%s on %s stories) %s' % (feed.title[:30], self.feed_trans[ret_feed], feed.stories_last_month, '' if feed.has_page else ' [HAS NO PAGE]'))