public-mirrors/NewsBlur
1
0
Fork 0
mirror of https://github.com/samuelclay/NewsBlur.git synced 2025-08-05 16:58:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into story_hash_2

Browse source 
# By Samuel Clay (10) and others
* master:
  Further pushing out feed updates.
  Using @dcramer's excellent getsentry.com for all exceptions. Turning off exception emails.
  Further pushing down feed fetches to handle load until db is scaled out.
  Scrolling code blocks.
  New task servers.
  Turning down feed fetches.
  If requests.raw won't work, sut fudge it with a StringIO for now.
  Further turning down feed fetches until mongo is ready to replicate.
  Fixing Facebook image calculation.
  Fixing facebook posting.
  Fixed API docs for /reader/river_stories
  make fabfile.py use requirements.txt
  Make sure to send the proper content-type with social feeds
  Restore default whitespace to pre/code blocks
  document keyboard shortcut to add site/folder
This commit is contained in:
Samuel Clay 2013-03-20 10:22:30 -07:00
commit 5eebcfc546
12 changed files with 79 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
apps/rss_feeds/models.py
View file

@ -1233,7 +1233,7 @@ class Feed(models.Model):
# .5 hours for 2 subscribers.
# .25 hours for 3 subscribers.
# 1 min for 10 subscribers.
subscriber_bonus = 6 * 60 / max(.167, max(0, self.active_subscribers)**3)
subscriber_bonus = 12 * 60 / max(.167, max(0, self.active_subscribers)**3)
if self.premium_subscribers > 0:
subscriber_bonus /= min(self.active_subscribers+self.premium_subscribers, 5)
@ -1245,13 +1245,13 @@ class Feed(models.Model):
slow_punishment = 2 * self.last_load_time
elif self.last_load_time >= 200:
slow_punishment = 6 * self.last_load_time
total = max(5, int(updates_per_day_delay + subscriber_bonus + slow_punishment))
total = max(10, int(updates_per_day_delay + subscriber_bonus + slow_punishment))
if self.active_premium_subscribers > 3:
if self.active_premium_subscribers > 5:
total = min(total, 60) # 1 hour minimum for premiums
if ((self.stories_last_month == 0 or self.average_stories_per_month == 0)):
total = total * random.randint(1, 12)
total = total * random.randint(1, 24)
if self.is_push:
total = total * 20

8
apps/rss_feeds/page_importer.py
View file

@ -11,7 +11,8 @@ from django.conf import settings
from django.utils.text import compress_string
from utils import log as logging
from apps.rss_feeds.models import MFeedPage
from utils.feed_functions import timelimit, mail_feed_error_to_admin
from utils.feed_functions import timelimit
# from utils.feed_functions import mail_feed_error_to_admin
BROKEN_PAGES = [
'tag:',
@ -120,7 +121,10 @@ class PageImporter(object):
logging.debug(tb)
logging.debug('[%d] ! -------------------------' % (self.feed.id,))
self.feed.save_page_history(500, "Error", tb)
mail_feed_error_to_admin(self.feed, e, local_vars=locals())
# mail_feed_error_to_admin(self.feed, e, local_vars=locals())
if (not settings.DEBUG and hasattr(settings, 'RAVEN_CLIENT') and
settings.RAVEN_CLIENT):
settings.RAVEN_CLIENT.captureException()
if not urllib_fallback:
self.fetch_page(urllib_fallback=True)
else:

4
apps/rss_feeds/tasks.py
View file

@ -20,7 +20,7 @@ class TaskFeeds(Task):
next_scheduled_update__lte=now,
active=True,
active_premium_subscribers__gte=1
).order_by('?')[:600]
).order_by('?')[:400]
popular_count = popular_feeds.count()
# Regular feeds
@ -28,7 +28,7 @@ class TaskFeeds(Task):
next_scheduled_update__lte=now,
active=True,
active_subscribers__gte=1
).order_by('?')[:400]
).order_by('?')[:200]
active_count = feeds.count()
# Mistakenly inactive feeds

9
apps/social/models.py
View file

@ -33,6 +33,7 @@ from utils.feed_functions import relative_timesince
from utils.story_functions import truncate_chars, strip_tags, linkify, image_size
from utils.scrubber import SelectiveScriptScrubber
from utils import s3_utils
from StringIO import StringIO
RECOMMENDATIONS_LIMIT = 5
IGNORE_IMAGE_SOURCES = [
@ -1207,7 +1208,7 @@ class MSharedStory(mongo.Document):
('user_id', 'story_db_id'),
'shared_date', 'story_guid', 'story_feed_id'],
'index_drop_dups': True,
'ordering': ['shared_date'],
'ordering': ['-shared_date'],
'allow_inheritance': False,
}
@ -1879,11 +1880,13 @@ class MSharedStory(mongo.Document):
soup = BeautifulSoup(zlib.decompress(self.story_content_z))
image_sources = [img.get('src') for img in soup.findAll('img')]
image_sizes = []
for image_source in image_sources[:10]:
if any(ignore in image_source for ignore in IGNORE_IMAGE_SOURCES):
continue
r = requests.get(image_source, prefetch=False, headers=headers)
_, width, height = image_size(r.raw)
req = requests.get(image_source, headers=headers, stream=True)
datastream = StringIO(req.content[:30])
_, width, height = image_size(datastream)
if width <= 16 or height <= 16:
continue
image_sizes.append({'src': image_source, 'size': (width, height)})

4
apps/social/views.py
View file

@ -1206,7 +1206,7 @@ def shared_stories_rss_feed(request, user_id, username):
user.username,
request.META['HTTP_USER_AGENT'][:24]
))
return HttpResponse(rss.writeString('utf-8'))
return HttpResponse(rss.writeString('utf-8'), content_type='application/rss+xml')
@required_params('user_id')
@json.json_view
@ -1320,4 +1320,4 @@ def comment(request, comment_id):
except MSharedStory.DoesNotExist:
raise Http404
return shared_story.comments_with_author()
return shared_story.comments_with_author()

26
config/requirements.txt
View file

@ -1,10 +1,17 @@
django
mongoengine
fabric
django==1.3.1
readline
chardet
pyflakes
iconv
celery
django-celery
django-celery-with-redis
django-compress
South
django-extensions
psycopg2
pymongo==2.2.0
stripe
BeautifulSoup
pyyaml
nltk
@ -12,4 +19,15 @@ lxml
oauth2
pytz
boto
gunicorn
seacucumber
django_ses
django-mailgun
mongoengine
redis
requests
django-subdomains
psutil
python-gflags
cssutils
raven
pyes

12
fabfile.py vendored
View file

@ -96,6 +96,9 @@ env.roledefs ={
'ec2-54-234-211-75.compute-1.amazonaws.com',
'ec2-50-16-97-13.compute-1.amazonaws.com',
'ec2-54-242-131-232.compute-1.amazonaws.com',
'ec2-75-101-195-131.compute-1.amazonaws.com',
'ec2-54-242-105-17.compute-1.amazonaws.com',
],
'vps': ['task01.newsblur.com',
'task03.newsblur.com',
@ -489,15 +492,16 @@ def setup_libxml_code():
def setup_psycopg():
sudo('easy_install -U psycopg2')
def setup_python():
# sudo('easy_install -U pip')
sudo('easy_install -U fabric django==1.3.1 readline chardet pyflakes iconv celery django-celery django-celery-with-redis django-compress South django-extensions pymongo==2.2.0 stripe BeautifulSoup pyyaml nltk lxml oauth2 pytz boto seacucumber django_ses django-mailgun mongoengine redis requests django-subdomains psutil python-gflags cssutils raven pyes')
sudo('easy_install -U $(<%s)' %
os.path.join(env.NEWSBLUR_PATH, 'config/requirements.txt'))
put('config/pystartup.py', '.pystartup')
# with cd(os.path.join(env.NEWSBLUR_PATH, 'vendor/cjson')):
# sudo('python setup.py install')
with settings(warn_only=True):
sudo('su -c \'echo "import sys; sys.setdefaultencoding(\\\\"utf-8\\\\")" > /usr/lib/python2.7/sitecustomize.py\'')

8
media/css/reader.css
View file

@ -1800,6 +1800,10 @@ background: transparent;
#story_pane .NB-feed-story:first-child .NB-feed-story-header {
padding-top: 0;
}
#story_pane .NB-feed-stories pre {
overflow-x: auto;
max-width: 100%;
}
#story_pane .NB-feed-story-header-info {
font-weight: bold;
@ -1901,10 +1905,6 @@ background: transparent;
.NB-feed-story .NB-feed-story-content div {
max-width: 100%;
}
.NB-feed-story .NB-feed-story-content pre,
.NB-feed-story .NB-feed-story-content code {
white-space: normal;
}
.NB-feed-story .NB-feed-story-content img {
max-width: 100% !important;
width: auto;

8
media/js/newsblur/reader/reader_keyboard.js
View file

@ -270,6 +270,14 @@ NEWSBLUR.ReaderKeyboard.prototype = {
'?'
])
])
]),
$.make('div', { className: 'NB-keyboard-group' }, [
$.make('div', { className: 'NB-keyboard-shortcut NB-last' }, [
$.make('div', { className: 'NB-keyboard-shortcut-explanation' }, 'Add Site/Folder'),
$.make('div', { className: 'NB-keyboard-shortcut-key' }, [
'a'
])
])
])
]);
},

2
settings.py
View file

@ -149,7 +149,7 @@ LOGGING = {
},
'loggers': {
'django.request': {
'handlers': ['mail_admins'],
'handlers': ['console', 'log_file'],
'level': 'ERROR',
'propagate': True,
},

5
templates/static/api.yml
View file

@ -229,6 +229,11 @@
optional: true
default: newest
example: oldest
- key: read_filter
desc: "Show all stories or only unread stories"
optional: true
default: unread
example: all
- url: /reader/mark_story_as_read
method: POST

17
utils/feed_fetcher.py
View file

@ -18,7 +18,8 @@ from apps.statistics.models import MAnalyticsFetcher
from utils import feedparser
from utils.story_functions import pre_process_story
from utils import log as logging
from utils.feed_functions import timelimit, TimeoutError, mail_feed_error_to_admin, utf8encode
from utils.feed_functions import timelimit, TimeoutError, utf8encode
# from utils.feed_functions import mail_feed_error_to_admin
# Refresh feed code adapted from Feedjack.
@ -386,7 +387,7 @@ class Dispatcher:
feed.save_feed_history(500, "Error", tb)
feed_code = 500
fetched_feed = None
mail_feed_error_to_admin(feed, e, local_vars=locals())
# mail_feed_error_to_admin(feed, e, local_vars=locals())
if (not settings.DEBUG and hasattr(settings, 'RAVEN_CLIENT') and
settings.RAVEN_CLIENT):
settings.RAVEN_CLIENT.captureException()
@ -431,8 +432,10 @@ class Dispatcher:
feed.save_page_history(550, "Page Error", tb)
fetched_feed = None
page_data = None
mail_feed_error_to_admin(feed, e, local_vars=locals())
settings.RAVEN_CLIENT.captureException()
# mail_feed_error_to_admin(feed, e, local_vars=locals())
if (not settings.DEBUG and hasattr(settings, 'RAVEN_CLIENT') and
settings.RAVEN_CLIENT):
settings.RAVEN_CLIENT.captureException()
feed = self.refresh_feed(feed.pk)
logging.debug(u' ---> [%-30s] ~FYFetching icon: %s' % (feed.title[:30], feed.feed_link))
@ -449,8 +452,10 @@ class Dispatcher:
logging.error(tb)
logging.debug('[%d] ! -------------------------' % (feed_id,))
# feed.save_feed_history(560, "Icon Error", tb)
mail_feed_error_to_admin(feed, e, local_vars=locals())
settings.RAVEN_CLIENT.captureException()
# mail_feed_error_to_admin(feed, e, local_vars=locals())
if (not settings.DEBUG and hasattr(settings, 'RAVEN_CLIENT') and
settings.RAVEN_CLIENT):
settings.RAVEN_CLIENT.captureException()
else:
logging.debug(u' ---> [%-30s] ~FBSkipping page fetch: (%s on %s stories) %s' % (feed.title[:30], self.feed_trans[ret_feed], feed.stories_last_month, '' if feed.has_page else ' [HAS NO PAGE]'))