diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index 57ddf5462..590e97e3e 100644 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -1233,7 +1233,7 @@ class Feed(models.Model): # .5 hours for 2 subscribers. # .25 hours for 3 subscribers. # 1 min for 10 subscribers. - subscriber_bonus = 6 * 60 / max(.167, max(0, self.active_subscribers)**3) + subscriber_bonus = 12 * 60 / max(.167, max(0, self.active_subscribers)**3) if self.premium_subscribers > 0: subscriber_bonus /= min(self.active_subscribers+self.premium_subscribers, 5) @@ -1245,13 +1245,13 @@ class Feed(models.Model): slow_punishment = 2 * self.last_load_time elif self.last_load_time >= 200: slow_punishment = 6 * self.last_load_time - total = max(5, int(updates_per_day_delay + subscriber_bonus + slow_punishment)) + total = max(10, int(updates_per_day_delay + subscriber_bonus + slow_punishment)) - if self.active_premium_subscribers > 3: + if self.active_premium_subscribers > 5: total = min(total, 60) # 1 hour minimum for premiums if ((self.stories_last_month == 0 or self.average_stories_per_month == 0)): - total = total * random.randint(1, 12) + total = total * random.randint(1, 24) if self.is_push: total = total * 20 diff --git a/apps/rss_feeds/page_importer.py b/apps/rss_feeds/page_importer.py index ba4cc68a0..2f6209775 100644 --- a/apps/rss_feeds/page_importer.py +++ b/apps/rss_feeds/page_importer.py @@ -11,7 +11,8 @@ from django.conf import settings from django.utils.text import compress_string from utils import log as logging from apps.rss_feeds.models import MFeedPage -from utils.feed_functions import timelimit, mail_feed_error_to_admin +from utils.feed_functions import timelimit +# from utils.feed_functions import mail_feed_error_to_admin BROKEN_PAGES = [ 'tag:', @@ -120,7 +121,10 @@ class PageImporter(object): logging.debug(tb) logging.debug('[%d] ! -------------------------' % (self.feed.id,)) self.feed.save_page_history(500, "Error", tb) - mail_feed_error_to_admin(self.feed, e, local_vars=locals()) + # mail_feed_error_to_admin(self.feed, e, local_vars=locals()) + if (not settings.DEBUG and hasattr(settings, 'RAVEN_CLIENT') and + settings.RAVEN_CLIENT): + settings.RAVEN_CLIENT.captureException() if not urllib_fallback: self.fetch_page(urllib_fallback=True) else: diff --git a/apps/rss_feeds/tasks.py b/apps/rss_feeds/tasks.py index bc57ca4b0..f43243ec6 100644 --- a/apps/rss_feeds/tasks.py +++ b/apps/rss_feeds/tasks.py @@ -20,7 +20,7 @@ class TaskFeeds(Task): next_scheduled_update__lte=now, active=True, active_premium_subscribers__gte=1 - ).order_by('?')[:600] + ).order_by('?')[:400] popular_count = popular_feeds.count() # Regular feeds @@ -28,7 +28,7 @@ class TaskFeeds(Task): next_scheduled_update__lte=now, active=True, active_subscribers__gte=1 - ).order_by('?')[:400] + ).order_by('?')[:200] active_count = feeds.count() # Mistakenly inactive feeds diff --git a/apps/social/models.py b/apps/social/models.py index f9597f6b8..2d9471d55 100644 --- a/apps/social/models.py +++ b/apps/social/models.py @@ -33,6 +33,7 @@ from utils.feed_functions import relative_timesince from utils.story_functions import truncate_chars, strip_tags, linkify, image_size from utils.scrubber import SelectiveScriptScrubber from utils import s3_utils +from StringIO import StringIO RECOMMENDATIONS_LIMIT = 5 IGNORE_IMAGE_SOURCES = [ @@ -1207,7 +1208,7 @@ class MSharedStory(mongo.Document): ('user_id', 'story_db_id'), 'shared_date', 'story_guid', 'story_feed_id'], 'index_drop_dups': True, - 'ordering': ['shared_date'], + 'ordering': ['-shared_date'], 'allow_inheritance': False, } @@ -1879,11 +1880,13 @@ class MSharedStory(mongo.Document): soup = BeautifulSoup(zlib.decompress(self.story_content_z)) image_sources = [img.get('src') for img in soup.findAll('img')] image_sizes = [] + for image_source in image_sources[:10]: if any(ignore in image_source for ignore in IGNORE_IMAGE_SOURCES): continue - r = requests.get(image_source, prefetch=False, headers=headers) - _, width, height = image_size(r.raw) + req = requests.get(image_source, headers=headers, stream=True) + datastream = StringIO(req.content[:30]) + _, width, height = image_size(datastream) if width <= 16 or height <= 16: continue image_sizes.append({'src': image_source, 'size': (width, height)}) diff --git a/apps/social/views.py b/apps/social/views.py index 52133d9ed..76b8d7581 100644 --- a/apps/social/views.py +++ b/apps/social/views.py @@ -1206,7 +1206,7 @@ def shared_stories_rss_feed(request, user_id, username): user.username, request.META['HTTP_USER_AGENT'][:24] )) - return HttpResponse(rss.writeString('utf-8')) + return HttpResponse(rss.writeString('utf-8'), content_type='application/rss+xml') @required_params('user_id') @json.json_view @@ -1320,4 +1320,4 @@ def comment(request, comment_id): except MSharedStory.DoesNotExist: raise Http404 - return shared_story.comments_with_author() \ No newline at end of file + return shared_story.comments_with_author() diff --git a/config/requirements.txt b/config/requirements.txt index a2aff7f66..26e4eb32b 100644 --- a/config/requirements.txt +++ b/config/requirements.txt @@ -1,10 +1,17 @@ -django -mongoengine +fabric +django==1.3.1 +readline +chardet +pyflakes +iconv +celery django-celery +django-celery-with-redis django-compress South django-extensions -psycopg2 +pymongo==2.2.0 +stripe BeautifulSoup pyyaml nltk @@ -12,4 +19,15 @@ lxml oauth2 pytz boto -gunicorn +seacucumber +django_ses +django-mailgun +mongoengine +redis +requests +django-subdomains +psutil +python-gflags +cssutils +raven +pyes diff --git a/fabfile.py b/fabfile.py index 48f60ccc6..5cad3eb8d 100644 --- a/fabfile.py +++ b/fabfile.py @@ -96,6 +96,9 @@ env.roledefs ={ 'ec2-54-234-211-75.compute-1.amazonaws.com', 'ec2-50-16-97-13.compute-1.amazonaws.com', + 'ec2-54-242-131-232.compute-1.amazonaws.com', + 'ec2-75-101-195-131.compute-1.amazonaws.com', + 'ec2-54-242-105-17.compute-1.amazonaws.com', ], 'vps': ['task01.newsblur.com', 'task03.newsblur.com', @@ -489,15 +492,16 @@ def setup_libxml_code(): def setup_psycopg(): sudo('easy_install -U psycopg2') - + def setup_python(): # sudo('easy_install -U pip') - sudo('easy_install -U fabric django==1.3.1 readline chardet pyflakes iconv celery django-celery django-celery-with-redis django-compress South django-extensions pymongo==2.2.0 stripe BeautifulSoup pyyaml nltk lxml oauth2 pytz boto seacucumber django_ses django-mailgun mongoengine redis requests django-subdomains psutil python-gflags cssutils raven pyes') - + sudo('easy_install -U $(<%s)' % + os.path.join(env.NEWSBLUR_PATH, 'config/requirements.txt')) put('config/pystartup.py', '.pystartup') + # with cd(os.path.join(env.NEWSBLUR_PATH, 'vendor/cjson')): # sudo('python setup.py install') - + with settings(warn_only=True): sudo('su -c \'echo "import sys; sys.setdefaultencoding(\\\\"utf-8\\\\")" > /usr/lib/python2.7/sitecustomize.py\'') diff --git a/media/css/reader.css b/media/css/reader.css index 9bca661cf..97964652d 100644 --- a/media/css/reader.css +++ b/media/css/reader.css @@ -1800,6 +1800,10 @@ background: transparent; #story_pane .NB-feed-story:first-child .NB-feed-story-header { padding-top: 0; } +#story_pane .NB-feed-stories pre { + overflow-x: auto; + max-width: 100%; +} #story_pane .NB-feed-story-header-info { font-weight: bold; @@ -1901,10 +1905,6 @@ background: transparent; .NB-feed-story .NB-feed-story-content div { max-width: 100%; } -.NB-feed-story .NB-feed-story-content pre, -.NB-feed-story .NB-feed-story-content code { - white-space: normal; -} .NB-feed-story .NB-feed-story-content img { max-width: 100% !important; width: auto; diff --git a/media/js/newsblur/reader/reader_keyboard.js b/media/js/newsblur/reader/reader_keyboard.js index 2772d880d..22d1660f0 100644 --- a/media/js/newsblur/reader/reader_keyboard.js +++ b/media/js/newsblur/reader/reader_keyboard.js @@ -270,6 +270,14 @@ NEWSBLUR.ReaderKeyboard.prototype = { '?' ]) ]) + ]), + $.make('div', { className: 'NB-keyboard-group' }, [ + $.make('div', { className: 'NB-keyboard-shortcut NB-last' }, [ + $.make('div', { className: 'NB-keyboard-shortcut-explanation' }, 'Add Site/Folder'), + $.make('div', { className: 'NB-keyboard-shortcut-key' }, [ + 'a' + ]) + ]) ]) ]); }, diff --git a/settings.py b/settings.py index 745e79f9c..0a1909fb5 100644 --- a/settings.py +++ b/settings.py @@ -149,7 +149,7 @@ LOGGING = { }, 'loggers': { 'django.request': { - 'handlers': ['mail_admins'], + 'handlers': ['console', 'log_file'], 'level': 'ERROR', 'propagate': True, }, diff --git a/templates/static/api.yml b/templates/static/api.yml index 410b33cc2..45c7e9549 100644 --- a/templates/static/api.yml +++ b/templates/static/api.yml @@ -229,6 +229,11 @@ optional: true default: newest example: oldest + - key: read_filter + desc: "Show all stories or only unread stories" + optional: true + default: unread + example: all - url: /reader/mark_story_as_read method: POST diff --git a/utils/feed_fetcher.py b/utils/feed_fetcher.py index e6546450e..e33a90bfb 100644 --- a/utils/feed_fetcher.py +++ b/utils/feed_fetcher.py @@ -18,7 +18,8 @@ from apps.statistics.models import MAnalyticsFetcher from utils import feedparser from utils.story_functions import pre_process_story from utils import log as logging -from utils.feed_functions import timelimit, TimeoutError, mail_feed_error_to_admin, utf8encode +from utils.feed_functions import timelimit, TimeoutError, utf8encode +# from utils.feed_functions import mail_feed_error_to_admin # Refresh feed code adapted from Feedjack. @@ -386,7 +387,7 @@ class Dispatcher: feed.save_feed_history(500, "Error", tb) feed_code = 500 fetched_feed = None - mail_feed_error_to_admin(feed, e, local_vars=locals()) + # mail_feed_error_to_admin(feed, e, local_vars=locals()) if (not settings.DEBUG and hasattr(settings, 'RAVEN_CLIENT') and settings.RAVEN_CLIENT): settings.RAVEN_CLIENT.captureException() @@ -431,8 +432,10 @@ class Dispatcher: feed.save_page_history(550, "Page Error", tb) fetched_feed = None page_data = None - mail_feed_error_to_admin(feed, e, local_vars=locals()) - settings.RAVEN_CLIENT.captureException() + # mail_feed_error_to_admin(feed, e, local_vars=locals()) + if (not settings.DEBUG and hasattr(settings, 'RAVEN_CLIENT') and + settings.RAVEN_CLIENT): + settings.RAVEN_CLIENT.captureException() feed = self.refresh_feed(feed.pk) logging.debug(u' ---> [%-30s] ~FYFetching icon: %s' % (feed.title[:30], feed.feed_link)) @@ -449,8 +452,10 @@ class Dispatcher: logging.error(tb) logging.debug('[%d] ! -------------------------' % (feed_id,)) # feed.save_feed_history(560, "Icon Error", tb) - mail_feed_error_to_admin(feed, e, local_vars=locals()) - settings.RAVEN_CLIENT.captureException() + # mail_feed_error_to_admin(feed, e, local_vars=locals()) + if (not settings.DEBUG and hasattr(settings, 'RAVEN_CLIENT') and + settings.RAVEN_CLIENT): + settings.RAVEN_CLIENT.captureException() else: logging.debug(u' ---> [%-30s] ~FBSkipping page fetch: (%s on %s stories) %s' % (feed.title[:30], self.feed_trans[ret_feed], feed.stories_last_month, '' if feed.has_page else ' [HAS NO PAGE]'))