From 359edd5830cf2a3bd372a5f7ac171d57b0e95cc8 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Mon, 11 Feb 2013 17:02:30 -0500 Subject: [PATCH 01/15] document keyboard shortcut to add site/folder --- media/js/newsblur/reader/reader_keyboard.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/media/js/newsblur/reader/reader_keyboard.js b/media/js/newsblur/reader/reader_keyboard.js index 2772d880d..22d1660f0 100644 --- a/media/js/newsblur/reader/reader_keyboard.js +++ b/media/js/newsblur/reader/reader_keyboard.js @@ -270,6 +270,14 @@ NEWSBLUR.ReaderKeyboard.prototype = { '?' ]) ]) + ]), + $.make('div', { className: 'NB-keyboard-group' }, [ + $.make('div', { className: 'NB-keyboard-shortcut NB-last' }, [ + $.make('div', { className: 'NB-keyboard-shortcut-explanation' }, 'Add Site/Folder'), + $.make('div', { className: 'NB-keyboard-shortcut-key' }, [ + 'a' + ]) + ]) ]) ]); }, From 5481d11623063628449db6955caaf1391e474bce Mon Sep 17 00:00:00 2001 From: Rowan Crawford Date: Tue, 19 Mar 2013 00:02:00 +1300 Subject: [PATCH 02/15] Restore default whitespace to pre/code blocks --- media/css/reader.css | 4 ---- 1 file changed, 4 deletions(-) diff --git a/media/css/reader.css b/media/css/reader.css index 720a8b137..65adc36da 100644 --- a/media/css/reader.css +++ b/media/css/reader.css @@ -1901,10 +1901,6 @@ background: transparent; .NB-feed-story .NB-feed-story-content div { max-width: 100%; } -.NB-feed-story .NB-feed-story-content pre, -.NB-feed-story .NB-feed-story-content code { - white-space: normal; -} .NB-feed-story .NB-feed-story-content img { max-width: 100% !important; width: auto; From bffe3b11df78f1b5ccaaedbb53598564660bc36e Mon Sep 17 00:00:00 2001 From: Glendon Solsberry Date: Tue, 19 Mar 2013 08:12:48 -0400 Subject: [PATCH 03/15] Make sure to send the proper content-type with social feeds --- apps/social/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/social/views.py b/apps/social/views.py index 178487aa4..2c1eaa28e 100644 --- a/apps/social/views.py +++ b/apps/social/views.py @@ -1208,7 +1208,7 @@ def shared_stories_rss_feed(request, user_id, username): user.username, request.META['HTTP_USER_AGENT'][:24] )) - return HttpResponse(rss.writeString('utf-8')) + return HttpResponse(rss.writeString('utf-8'), content_type='application/rss+xml') @required_params('user_id') @json.json_view @@ -1322,4 +1322,4 @@ def comment(request, comment_id): except MSharedStory.DoesNotExist: raise Http404 - return shared_story.comments_with_author() \ No newline at end of file + return shared_story.comments_with_author() From 88812a77f90ae3d231251ab4db2602335f968348 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Tue, 19 Mar 2013 18:06:01 +0000 Subject: [PATCH 04/15] make fabfile.py use requirements.txt The '$( /usr/lib/python2.7/sitecustomize.py\'') From f81b239287cdce6b7d0c4df442aeb0691606ea19 Mon Sep 17 00:00:00 2001 From: Liwen Guo Date: Tue, 19 Mar 2013 20:18:33 +0100 Subject: [PATCH 05/15] Fixed API docs for /reader/river_stories read_filter on /reader/river_stories defaults to unread, however the parameter wasn't even documented before on this endpoint. --- templates/static/api.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/templates/static/api.yml b/templates/static/api.yml index 410b33cc2..45c7e9549 100644 --- a/templates/static/api.yml +++ b/templates/static/api.yml @@ -229,6 +229,11 @@ optional: true default: newest example: oldest + - key: read_filter + desc: "Show all stories or only unread stories" + optional: true + default: unread + example: all - url: /reader/mark_story_as_read method: POST From d5b72663803846a367683095201f45e2fe9aedf4 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Tue, 19 Mar 2013 17:00:22 -0700 Subject: [PATCH 06/15] Fixing facebook posting. --- apps/social/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/social/models.py b/apps/social/models.py index f9597f6b8..f68964ded 100644 --- a/apps/social/models.py +++ b/apps/social/models.py @@ -1882,7 +1882,7 @@ class MSharedStory(mongo.Document): for image_source in image_sources[:10]: if any(ignore in image_source for ignore in IGNORE_IMAGE_SOURCES): continue - r = requests.get(image_source, prefetch=False, headers=headers) + r = requests.get(image_source, headers=headers) _, width, height = image_size(r.raw) if width <= 16 or height <= 16: continue From 1854f719d33b468fc4a9fd27de8bf69290240f6c Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Tue, 19 Mar 2013 17:27:04 -0700 Subject: [PATCH 07/15] Fixing Facebook image calculation. --- apps/social/models.py | 7 ++++--- utils/story_functions.py | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/apps/social/models.py b/apps/social/models.py index f68964ded..8bb412e79 100644 --- a/apps/social/models.py +++ b/apps/social/models.py @@ -1207,7 +1207,7 @@ class MSharedStory(mongo.Document): ('user_id', 'story_db_id'), 'shared_date', 'story_guid', 'story_feed_id'], 'index_drop_dups': True, - 'ordering': ['shared_date'], + 'ordering': ['-shared_date'], 'allow_inheritance': False, } @@ -1879,11 +1879,12 @@ class MSharedStory(mongo.Document): soup = BeautifulSoup(zlib.decompress(self.story_content_z)) image_sources = [img.get('src') for img in soup.findAll('img')] image_sizes = [] + for image_source in image_sources[:10]: if any(ignore in image_source for ignore in IGNORE_IMAGE_SOURCES): continue - r = requests.get(image_source, headers=headers) - _, width, height = image_size(r.raw) + req = requests.get(image_source, headers=headers, stream=True) + _, width, height = image_size(req.content) if width <= 16 or height <= 16: continue image_sizes.append({'src': image_source, 'size': (width, height)}) diff --git a/utils/story_functions.py b/utils/story_functions.py index daceac605..6b766bef9 100644 --- a/utils/story_functions.py +++ b/utils/story_functions.py @@ -13,7 +13,7 @@ from django.utils.html import strip_tags as strip_tags_django from django.conf import settings from utils.tornado_escape import linkify as linkify_tornado from utils.tornado_escape import xhtml_unescape as xhtml_unescape_tornado -from vendor import reseekfile +# from vendor import reseekfile COMMENTS_RE = re.compile('\') @@ -239,8 +239,9 @@ def truncate_chars(value, max_length): return truncd_val + "..." def image_size(datastream): - datastream = reseekfile.ReseekFile(datastream) - data = str(datastream.read(30)) + # datastream = reseekfile.ReseekFile(datastream) + # data = str(datastream.read(30)) + data = datastream[:30] size = len(data) height = -1 width = -1 From 0db41bc8538290bef840f37b266ce7ba68961a42 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Tue, 19 Mar 2013 17:38:36 -0700 Subject: [PATCH 08/15] Further turning down feed fetches until mongo is ready to replicate. --- apps/rss_feeds/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/rss_feeds/tasks.py b/apps/rss_feeds/tasks.py index bc57ca4b0..51c6e42ee 100644 --- a/apps/rss_feeds/tasks.py +++ b/apps/rss_feeds/tasks.py @@ -20,7 +20,7 @@ class TaskFeeds(Task): next_scheduled_update__lte=now, active=True, active_premium_subscribers__gte=1 - ).order_by('?')[:600] + ).order_by('?')[:500] popular_count = popular_feeds.count() # Regular feeds @@ -28,7 +28,7 @@ class TaskFeeds(Task): next_scheduled_update__lte=now, active=True, active_subscribers__gte=1 - ).order_by('?')[:400] + ).order_by('?')[:300] active_count = feeds.count() # Mistakenly inactive feeds From 61fac023045ecacdcadc2f68f095b3667c73183c Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Tue, 19 Mar 2013 17:49:56 -0700 Subject: [PATCH 09/15] If requests.raw won't work, sut fudge it with a StringIO for now. --- apps/social/models.py | 4 +++- utils/story_functions.py | 7 +++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/apps/social/models.py b/apps/social/models.py index 8bb412e79..2d9471d55 100644 --- a/apps/social/models.py +++ b/apps/social/models.py @@ -33,6 +33,7 @@ from utils.feed_functions import relative_timesince from utils.story_functions import truncate_chars, strip_tags, linkify, image_size from utils.scrubber import SelectiveScriptScrubber from utils import s3_utils +from StringIO import StringIO RECOMMENDATIONS_LIMIT = 5 IGNORE_IMAGE_SOURCES = [ @@ -1884,7 +1885,8 @@ class MSharedStory(mongo.Document): if any(ignore in image_source for ignore in IGNORE_IMAGE_SOURCES): continue req = requests.get(image_source, headers=headers, stream=True) - _, width, height = image_size(req.content) + datastream = StringIO(req.content[:30]) + _, width, height = image_size(datastream) if width <= 16 or height <= 16: continue image_sizes.append({'src': image_source, 'size': (width, height)}) diff --git a/utils/story_functions.py b/utils/story_functions.py index 6b766bef9..daceac605 100644 --- a/utils/story_functions.py +++ b/utils/story_functions.py @@ -13,7 +13,7 @@ from django.utils.html import strip_tags as strip_tags_django from django.conf import settings from utils.tornado_escape import linkify as linkify_tornado from utils.tornado_escape import xhtml_unescape as xhtml_unescape_tornado -# from vendor import reseekfile +from vendor import reseekfile COMMENTS_RE = re.compile('\') @@ -239,9 +239,8 @@ def truncate_chars(value, max_length): return truncd_val + "..." def image_size(datastream): - # datastream = reseekfile.ReseekFile(datastream) - # data = str(datastream.read(30)) - data = datastream[:30] + datastream = reseekfile.ReseekFile(datastream) + data = str(datastream.read(30)) size = len(data) height = -1 width = -1 From 7ad6c594a373d624c468e3484e42ab6db1dd8e06 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Tue, 19 Mar 2013 17:57:24 -0700 Subject: [PATCH 10/15] Turning down feed fetches. --- apps/rss_feeds/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/rss_feeds/tasks.py b/apps/rss_feeds/tasks.py index 51c6e42ee..a5508573d 100644 --- a/apps/rss_feeds/tasks.py +++ b/apps/rss_feeds/tasks.py @@ -28,7 +28,7 @@ class TaskFeeds(Task): next_scheduled_update__lte=now, active=True, active_subscribers__gte=1 - ).order_by('?')[:300] + ).order_by('?')[:250] active_count = feeds.count() # Mistakenly inactive feeds From 991318face827016afa097df91aca251a142c66b Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Tue, 19 Mar 2013 18:27:00 -0700 Subject: [PATCH 11/15] New task servers. --- fabfile.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fabfile.py b/fabfile.py index a01d5ebba..5cad3eb8d 100644 --- a/fabfile.py +++ b/fabfile.py @@ -96,6 +96,9 @@ env.roledefs ={ 'ec2-54-234-211-75.compute-1.amazonaws.com', 'ec2-50-16-97-13.compute-1.amazonaws.com', + 'ec2-54-242-131-232.compute-1.amazonaws.com', + 'ec2-75-101-195-131.compute-1.amazonaws.com', + 'ec2-54-242-105-17.compute-1.amazonaws.com', ], 'vps': ['task01.newsblur.com', 'task03.newsblur.com', From ff1f79eddfdf071c1bf61a206931d714c5cc99e9 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Tue, 19 Mar 2013 18:29:10 -0700 Subject: [PATCH 12/15] Scrolling code blocks. --- media/css/reader.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/media/css/reader.css b/media/css/reader.css index eb96f48b8..97964652d 100644 --- a/media/css/reader.css +++ b/media/css/reader.css @@ -1800,6 +1800,10 @@ background: transparent; #story_pane .NB-feed-story:first-child .NB-feed-story-header { padding-top: 0; } +#story_pane .NB-feed-stories pre { + overflow-x: auto; + max-width: 100%; +} #story_pane .NB-feed-story-header-info { font-weight: bold; From ca43ea8555830fc207981765b4237d357485aa0b Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Wed, 20 Mar 2013 08:13:07 -0700 Subject: [PATCH 13/15] Further pushing down feed fetches to handle load until db is scaled out. --- apps/rss_feeds/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/rss_feeds/tasks.py b/apps/rss_feeds/tasks.py index a5508573d..f43243ec6 100644 --- a/apps/rss_feeds/tasks.py +++ b/apps/rss_feeds/tasks.py @@ -20,7 +20,7 @@ class TaskFeeds(Task): next_scheduled_update__lte=now, active=True, active_premium_subscribers__gte=1 - ).order_by('?')[:500] + ).order_by('?')[:400] popular_count = popular_feeds.count() # Regular feeds @@ -28,7 +28,7 @@ class TaskFeeds(Task): next_scheduled_update__lte=now, active=True, active_subscribers__gte=1 - ).order_by('?')[:250] + ).order_by('?')[:200] active_count = feeds.count() # Mistakenly inactive feeds From a75554374b34447ebdcf7111b1faade44325ef1d Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Wed, 20 Mar 2013 08:24:11 -0700 Subject: [PATCH 14/15] Using @dcramer's excellent getsentry.com for all exceptions. Turning off exception emails. --- apps/rss_feeds/page_importer.py | 8 ++++++-- settings.py | 2 +- utils/feed_fetcher.py | 17 +++++++++++------ 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/apps/rss_feeds/page_importer.py b/apps/rss_feeds/page_importer.py index ba4cc68a0..2f6209775 100644 --- a/apps/rss_feeds/page_importer.py +++ b/apps/rss_feeds/page_importer.py @@ -11,7 +11,8 @@ from django.conf import settings from django.utils.text import compress_string from utils import log as logging from apps.rss_feeds.models import MFeedPage -from utils.feed_functions import timelimit, mail_feed_error_to_admin +from utils.feed_functions import timelimit +# from utils.feed_functions import mail_feed_error_to_admin BROKEN_PAGES = [ 'tag:', @@ -120,7 +121,10 @@ class PageImporter(object): logging.debug(tb) logging.debug('[%d] ! -------------------------' % (self.feed.id,)) self.feed.save_page_history(500, "Error", tb) - mail_feed_error_to_admin(self.feed, e, local_vars=locals()) + # mail_feed_error_to_admin(self.feed, e, local_vars=locals()) + if (not settings.DEBUG and hasattr(settings, 'RAVEN_CLIENT') and + settings.RAVEN_CLIENT): + settings.RAVEN_CLIENT.captureException() if not urllib_fallback: self.fetch_page(urllib_fallback=True) else: diff --git a/settings.py b/settings.py index 745e79f9c..0a1909fb5 100644 --- a/settings.py +++ b/settings.py @@ -149,7 +149,7 @@ LOGGING = { }, 'loggers': { 'django.request': { - 'handlers': ['mail_admins'], + 'handlers': ['console', 'log_file'], 'level': 'ERROR', 'propagate': True, }, diff --git a/utils/feed_fetcher.py b/utils/feed_fetcher.py index e6546450e..e33a90bfb 100644 --- a/utils/feed_fetcher.py +++ b/utils/feed_fetcher.py @@ -18,7 +18,8 @@ from apps.statistics.models import MAnalyticsFetcher from utils import feedparser from utils.story_functions import pre_process_story from utils import log as logging -from utils.feed_functions import timelimit, TimeoutError, mail_feed_error_to_admin, utf8encode +from utils.feed_functions import timelimit, TimeoutError, utf8encode +# from utils.feed_functions import mail_feed_error_to_admin # Refresh feed code adapted from Feedjack. @@ -386,7 +387,7 @@ class Dispatcher: feed.save_feed_history(500, "Error", tb) feed_code = 500 fetched_feed = None - mail_feed_error_to_admin(feed, e, local_vars=locals()) + # mail_feed_error_to_admin(feed, e, local_vars=locals()) if (not settings.DEBUG and hasattr(settings, 'RAVEN_CLIENT') and settings.RAVEN_CLIENT): settings.RAVEN_CLIENT.captureException() @@ -431,8 +432,10 @@ class Dispatcher: feed.save_page_history(550, "Page Error", tb) fetched_feed = None page_data = None - mail_feed_error_to_admin(feed, e, local_vars=locals()) - settings.RAVEN_CLIENT.captureException() + # mail_feed_error_to_admin(feed, e, local_vars=locals()) + if (not settings.DEBUG and hasattr(settings, 'RAVEN_CLIENT') and + settings.RAVEN_CLIENT): + settings.RAVEN_CLIENT.captureException() feed = self.refresh_feed(feed.pk) logging.debug(u' ---> [%-30s] ~FYFetching icon: %s' % (feed.title[:30], feed.feed_link)) @@ -449,8 +452,10 @@ class Dispatcher: logging.error(tb) logging.debug('[%d] ! -------------------------' % (feed_id,)) # feed.save_feed_history(560, "Icon Error", tb) - mail_feed_error_to_admin(feed, e, local_vars=locals()) - settings.RAVEN_CLIENT.captureException() + # mail_feed_error_to_admin(feed, e, local_vars=locals()) + if (not settings.DEBUG and hasattr(settings, 'RAVEN_CLIENT') and + settings.RAVEN_CLIENT): + settings.RAVEN_CLIENT.captureException() else: logging.debug(u' ---> [%-30s] ~FBSkipping page fetch: (%s on %s stories) %s' % (feed.title[:30], self.feed_trans[ret_feed], feed.stories_last_month, '' if feed.has_page else ' [HAS NO PAGE]')) From d167788f296e7fae06cf0ffca97f4601918b8893 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Wed, 20 Mar 2013 10:04:25 -0700 Subject: [PATCH 15/15] Further pushing out feed updates. --- apps/rss_feeds/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index bdc43d344..679988c76 100644 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -1230,7 +1230,7 @@ class Feed(models.Model): # .5 hours for 2 subscribers. # .25 hours for 3 subscribers. # 1 min for 10 subscribers. - subscriber_bonus = 6 * 60 / max(.167, max(0, self.active_subscribers)**3) + subscriber_bonus = 12 * 60 / max(.167, max(0, self.active_subscribers)**3) if self.premium_subscribers > 0: subscriber_bonus /= min(self.active_subscribers+self.premium_subscribers, 5) @@ -1242,13 +1242,13 @@ class Feed(models.Model): slow_punishment = 2 * self.last_load_time elif self.last_load_time >= 200: slow_punishment = 6 * self.last_load_time - total = max(5, int(updates_per_day_delay + subscriber_bonus + slow_punishment)) + total = max(10, int(updates_per_day_delay + subscriber_bonus + slow_punishment)) - if self.active_premium_subscribers > 3: + if self.active_premium_subscribers > 5: total = min(total, 60) # 1 hour minimum for premiums if ((self.stories_last_month == 0 or self.average_stories_per_month == 0)): - total = total * random.randint(1, 12) + total = total * random.randint(1, 24) if self.is_push: total = total * 20