public-mirrors/NewsBlur
1
0
Fork 0
mirror of https://github.com/samuelclay/NewsBlur.git synced 2025-09-18 21:50:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into ios_story

Browse source 
* master:
  Facebook being a PITA.
  Facebook wants ISO8601 datetime format.
  Facebook doesn't like going to other URLs.
  Facebook doesn't like going to other URLs.
  Dumb typo in new facebook og crap.
  Using blurblog permalink for new facebook shares.
  Fixing Facebook share to use fancy actions.
  Adding scroll to comments button to share bar. Thanks @afita.
  Turning off microformats for more errors.
  Fixing errors in timeouts to show the correct error. Also fixing microformats parsing issue and allow IPv6 URLs in enclosures to be ignored, fixing a bunch of feeds.
  Cleaning redis stories for 1% of all feed fetches.
  Refreshing feed on fetch.
  Fiddling with logging on dupe feeds.
  New share by facebook.
This commit is contained in:
Samuel Clay 2012-11-26 15:03:09 -08:00
commit ab265c0899
12 changed files with 173 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
apps/rss_feeds/models.py
View file

@ -182,13 +182,14 @@ class Feed(models.Model):
return self
except IntegrityError:
duplicate_feed = Feed.objects.filter(feed_address=self.feed_address, feed_link=self.feed_link)
logging.debug("%s: %s" % (self.feed_address, duplicate_feed))
logging.debug(' ***> [%-30s] Feed deleted.' % (unicode(self)[:30]))
if duplicate_feed:
if self.pk != duplicate_feed[0].pk:
merge_feeds(self.pk, duplicate_feed[0].pk)
merge_feeds(self.pk, duplicate_feed[0].pk, force=True)
return duplicate_feed[0]
# Feed has been deleted. Just ignore it.
logging.debug("%s: %s" % (self.feed_address, duplicate_feed))
logging.debug(' ***> [%-30s] Feed deleted (%s).' % (unicode(self)[:30], self.pk))
return
def sync_redis(self):
@ -1596,9 +1597,13 @@ def merge_feeds(original_feed_id, duplicate_feed_id, force=False):
return
logging.info(" ---> Feed: [%s - %s] %s - %s" % (original_feed_id, duplicate_feed_id,
original_feed, original_feed.feed_link))
logging.info(" --> %s / %s" % (original_feed.feed_address, original_feed.feed_link))
logging.info(" --> %s / %s" % (duplicate_feed.feed_address, duplicate_feed.feed_link))
original_feed, original_feed.feed_link))
logging.info(" ++> %s: %s / %s" % (original_feed.pk,
original_feed.feed_address,
original_feed.feed_link))
logging.info(" --> %s: %s / %s" % (duplicate_feed.pk,
duplicate_feed.feed_address,
duplicate_feed.feed_link))
user_subs = UserSubscription.objects.filter(feed=duplicate_feed)
for user_sub in user_subs:
@ -1629,9 +1634,13 @@ def merge_feeds(original_feed_id, duplicate_feed_id, force=False):
dupe_feed.feed = original_feed
dupe_feed.duplicate_feed_id = duplicate_feed.pk
dupe_feed.save()
logging.debug(' ---> Dupe subscribers: %s, Original subscribers: %s' %
(duplicate_feed.num_subscribers, original_feed.num_subscribers))
duplicate_feed.delete()
original_feed.count_subscribers()
logging.debug(' ---> Now original subscribers: %s' %
(original_feed.num_subscribers))
MSharedStory.switch_feed(original_feed_id, duplicate_feed_id)

28
apps/social/migrations/0006_guid_hash.py Normal file
View file

@ -0,0 +1,28 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import DataMigration
from django.db import models
from apps.social.models import MSharedStory
class Migration(DataMigration):
def forwards(self, orm):
shared_stories = MSharedStory.objects.filter(story_guid_hash__exists=False)
count = shared_stories.count()
print "%s shared stories..." % count
for s, story in enumerate(shared_stories):
if s % 100 == 0:
print "%s/%s" % (s+1, count)
story.story_guid
def backwards(self, orm):
"Write your backwards methods here."
models = {
}
complete_apps = ['social']
symmetrical = True

45
apps/social/models.py
View file

@ -1144,6 +1144,7 @@ class MSharedStory(mongo.Document):
story_author_name = mongo.StringField()
story_permalink = mongo.StringField()
story_guid = mongo.StringField(unique_with=('user_id',))
story_guid_hash = mongo.StringField()
story_tags = mongo.ListField(mongo.StringField(max_length=250))
posted_to_services = mongo.ListField(mongo.StringField(max_length=20))
mute_email_users = mongo.ListField(mongo.IntField())
@ -1173,7 +1174,13 @@ class MSharedStory(mongo.Document):
@property
def guid_hash(self):
return hashlib.sha1(self.story_guid).hexdigest()
if self.story_guid_hash:
return self.story_guid_hash
self.story_guid_hash = hashlib.sha1(self.story_guid).hexdigest()
self.save()
return self.story_guid_hash
def to_json(self):
return {
@ -1194,7 +1201,8 @@ class MSharedStory(mongo.Document):
if self.story_original_content:
self.story_original_content_z = zlib.compress(self.story_original_content)
self.story_original_content = None
self.story_guid_hash = hashlib.sha1(self.story_guid).hexdigest()
self.story_title = strip_tags(self.story_title)
self.comments = linkify(strip_tags(self.comments))
@ -1626,13 +1634,14 @@ class MSharedStory(mongo.Document):
self.guid_hash[:6]
)
def generate_post_to_service_message(self):
def generate_post_to_service_message(self, include_url=True):
message = self.comments
if not message or len(message) < 1:
message = self.story_title
message = truncate_chars(message, 116)
message += " " + self.blurblog_permalink()
if include_url:
message = truncate_chars(message, 116)
message += " " + self.blurblog_permalink()
return message
@ -1641,16 +1650,16 @@ class MSharedStory(mongo.Document):
return
posted = False
message = self.generate_post_to_service_message()
social_service = MSocialServices.objects.get(user_id=self.user_id)
user = User.objects.get(pk=self.user_id)
message = self.generate_post_to_service_message()
logging.user(user, "~BM~FGPosting to %s: ~SB%s" % (service, message))
if service == 'twitter':
posted = social_service.post_to_twitter(message)
posted = social_service.post_to_twitter(self)
elif service == 'facebook':
posted = social_service.post_to_facebook(message)
posted = social_service.post_to_facebook(self)
if posted:
self.posted_to_services.append(service)
@ -2113,7 +2122,9 @@ class MSocialServices(mongo.Document):
profile.save()
return profile
def post_to_twitter(self, message):
def post_to_twitter(self, shared_story):
message = shared_story.generate_post_to_service_message()
try:
api = self.twitter_api()
api.update_status(status=message)
@ -2123,10 +2134,22 @@ class MSocialServices(mongo.Document):
return True
def post_to_facebook(self, message):
def post_to_facebook(self, shared_story):
message = shared_story.generate_post_to_service_message(include_url=False)
shared_story.calculate_image_sizes()
content = zlib.decompress(shared_story.story_content_z)[:1024]
try:
api = self.facebook_api()
api.put_wall_post(message=message)
# api.put_wall_post(message=message)
api.put_object('me', '%s:share' % settings.FACEBOOK_NAMESPACE,
link=shared_story.blurblog_permalink(),
type="link",
name=shared_story.story_title,
description=content,
article=shared_story.blurblog_permalink(),
message=message,
)
except facebook.GraphAPIError, e:
print e
return

26
apps/social/views.py
View file

@ -399,7 +399,27 @@ def load_social_page(request, user_id, username=None, **kwargs):
story['user_comments'] = shared_story.comments
stories = MSharedStory.attach_users_to_stories(stories, profiles)
active_story = None
path = request.META['PATH_INFO']
if '/story/' in path and format != 'html':
story_id = path.replace('/story/', '')
active_story_db = MSharedStory.objects.filter(user_id=social_user.pk,
story_guid_hash__startswith=story_id).limit(1)
if active_story_db:
active_story_db = active_story_db[0]
active_story = Feed.format_story(active_story_db)
if active_story_db.image_count:
active_story['image_url'] = active_story_db.image_sizes[0]['src']
active_story['tags'] = ', '.join(active_story_db.story_tags)
active_story['blurblog_permalink'] = active_story_db.blurblog_permalink()
active_story['iso8601'] = active_story_db.story_date.isoformat()
if active_story['story_feed_id']:
feed = Feed.get_by_id(active_story['story_feed_id'])
if feed:
active_story['feed'] = feed.canonical()
print active_story
params = {
'social_user' : social_user,
'stories' : stories,
@ -412,7 +432,9 @@ def load_social_page(request, user_id, username=None, **kwargs):
'feeds' : feeds,
'user_profile' : hasattr(user, 'profile') and user.profile,
'has_next_page' : has_next_page,
'holzer_truism' : random.choice(jennyholzer.TRUISMS) #if not has_next_page else None
'holzer_truism' : random.choice(jennyholzer.TRUISMS), #if not has_next_page else None
'facebook_app_id': settings.FACEBOOK_APP_ID,
'active_story' : active_story,
}
diff1 = checkpoint1-start

15
fabfile.py vendored
View file

@ -192,11 +192,20 @@ def staging_full():
@parallel
def celery():
celery_slow()
def celery_slow():
with cd(env.NEWSBLUR_PATH):
run('git pull')
celery_stop()
celery_start()
@parallel
def celery_fast():
with cd(env.NEWSBLUR_PATH):
run('git pull')
celery_reload()
@parallel
def celery_stop():
with cd(env.NEWSBLUR_PATH):
@ -210,6 +219,12 @@ def celery_start():
run('sudo supervisorctl start celery')
run('tail logs/newsblur.log')
@parallel
def celery_reload():
with cd(env.NEWSBLUR_PATH):
run('sudo supervisorctl reload celery')
run('tail logs/newsblur.log')
def kill_celery():
with cd(env.NEWSBLUR_PATH):
run('ps aux | grep celeryd | egrep -v grep | awk \'{print $2}\' | sudo xargs kill -9')

1
media/css/reader.css
View file

@ -2439,6 +2439,7 @@ background: transparent;
#story_pane .NB-story-comments-label {
float: left;
margin-right: 12px;
cursor: pointer;
}
#story_pane .NB-story-comments-label b {
font-size: 12px;

3
media/js/newsblur/views/module_follow_requests.js
View file

@ -10,7 +10,8 @@ NEWSBLUR.Views.FollowRequestsModule = Backbone.View.extend({
},
start_polling: function() {
if (NEWSBLUR.assets.user_profile.get('protected')) {
if (NEWSBLUR.assets.user_profile.get('protected') &&
NEWSBLUR.Globals.is_authenticated) {
this.poll = setInterval(_.bind(function() {
this.fetch_follow_requests();
}, this), this.POLL_INTERVAL);

10
media/js/newsblur/views/story_detail_view.js
View file

@ -18,7 +18,8 @@ NEWSBLUR.Views.StoryDetailView = Backbone.View.extend({
"click .NB-feed-story-tag" : "save_classifier",
"click .NB-feed-story-author" : "save_classifier",
"click .NB-feed-story-train" : "open_story_trainer",
"click .NB-feed-story-save" : "star_story"
"click .NB-feed-story-save" : "star_story",
"click .NB-story-comments-label" : "scroll_to_comments"
},
initialize: function() {
@ -475,6 +476,13 @@ NEWSBLUR.Views.StoryDetailView = Backbone.View.extend({
this.model.mark_read({skip_delay: true});
window.open(this.model.get('story_permalink'), '_blank');
window.focus();
},
scroll_to_comments: function() {
NEWSBLUR.app.story_list.scroll_to_selected_story(this.model, {
scroll_to_comments: true,
scroll_offset: -50
});
}

3
settings.py
View file

@ -221,7 +221,6 @@ INSTALLED_APPS = (
'django_extensions',
'djcelery',
'django_ses',
'raven.contrib.django',
'apps.rss_feeds',
'apps.reader',
'apps.analyzer',
@ -246,6 +245,7 @@ INSTALLED_APPS = (
if not DEVELOPMENT:
INSTALLED_APPS += (
'gunicorn',
'raven.contrib.django',
)
# ==========
@ -415,6 +415,7 @@ REDIS = {
FACEBOOK_APP_ID = '111111111111111'
FACEBOOK_SECRET = '99999999999999999999999999999999'
FACEBOOK_NAMESPACE = 'newsblur'
TWITTER_CONSUMER_KEY = 'ooooooooooooooooooooo'
TWITTER_CONSUMER_SECRET = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'

15
templates/social/social_page.xhtml
View file

@ -10,6 +10,21 @@
<link rel="icon" href="{{ social_profile.photo_url }}">
<meta name="viewport" content="initial-scale=1, maximum-scale=1.5">
{% if active_story %}
<meta property="fb:app_id" content="{{ facebook_app_id }}">
<meta property="og:type" content="website">
<meta property="og:url" content="{{ active_story.blurblog_permalink }}">
{% if active_story.feed %}
<meta property="og:site_name" content="{{ active_story.feed.feed_title }}">
{% endif %}
<meta property="og:image" content="{{ active_story.image_url }}">
<meta property="og:title" content="{{ active_story.story_title }}">
<meta property="og:description" content="{{ active_story.story_content|addslashes|safe }}">
<meta property="article:published_time" content="{{ active_story.iso8601 }}">
<meta property="article:author" content="{{ active_story.story_authors }}">
<meta property="article:tag" content="{{ active_story.tags }}">
{% endif %}
{% include_stylesheets "blurblog" %}
{% if social_profile.custom_css %}

31
utils/feed_fetcher.py
View file

@ -58,7 +58,7 @@ class FetchFeed:
if self.options.get('force') or not self.feed.fetched_once or not self.feed.known_good:
modified = None
etag = None
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/536.2.3 (KHTML, like Gecko) Version/5.2 (NewsBlur Feed Fetcher - %s subscriber%s - %s)' % (
self.feed.num_subscribers,
's' if self.feed.num_subscribers != 1 else '',
@ -67,19 +67,31 @@ class FetchFeed:
if self.options.get('feed_xml'):
logging.debug(u' ---> [%-30s] ~FM~BKFeed has been fat pinged. Ignoring fat: %s' % (
self.feed.title[:30], len(self.options.get('feed_xml'))))
if self.options.get('fpf'):
self.fpf = self.options.get('fpf')
logging.debug(u' ---> [%-30s] ~FM~BKFeed fetched in real-time with fat ping.' % (
self.feed.title[:30]))
else:
return FEED_OK, self.fpf
try:
self.fpf = feedparser.parse(self.feed.feed_address,
agent=USER_AGENT,
etag=etag,
modified=modified)
except (TypeError, ValueError), e:
logging.debug(u' ***> [%-30s] ~FR%s, turning off microformats.' %
(self.feed.title[:30], e))
feedparser.PARSE_MICROFORMATS = False
self.fpf = feedparser.parse(self.feed.feed_address,
agent=USER_AGENT,
etag=etag,
modified=modified)
feedparser.PARSE_MICROFORMATS = True
logging.debug(u' ---> [%-30s] ~FYFeed fetch in ~FM%.4ss' % (
self.feed.title[:30], time.time() - start))
return FEED_OK, self.fpf
def get_identity(self):
@ -105,6 +117,9 @@ class ProcessFeed:
def refresh_feed(self):
self.feed = Feed.get_by_id(self.feed_id)
if self.feed_id != self.feed.pk:
logging.debug(" ***> Feed has changed: from %s to %s" % (self.feed_id, self.feed.pk))
self.feed_id = self.feed.pk
def process(self):
""" Downloads and parses a feed.
@ -207,6 +222,7 @@ class ProcessFeed:
guids.append(entry.link)
elif entry.get('title'):
guids.append(entry.title)
self.feed = self.feed.save()
# Compare new stories to existing stories, adding and updating
@ -226,7 +242,7 @@ class ProcessFeed:
existing_stories = list(MStory.objects(
# story_guid__in=story_guids,
story_date__gte=start_date,
story_feed_id=self.feed_id
story_feed_id=self.feed.pk
).limit(max(int(len(story_guids)*1.5), 10)))
ret_values = self.feed.add_update_stories(stories, existing_stories,
@ -361,6 +377,8 @@ class Dispatcher:
feed.fetched_once = True
feed = feed.save()
# MUserStory.delete_old_stories(feed_id=feed.pk)
if random.random() <= 0.01:
feed.sync_redis()
try:
self.count_unreads_for_subscribers(feed)
except TimeoutError:
@ -395,7 +413,8 @@ class Dispatcher:
feed_code = 500
fetched_feed = None
mail_feed_error_to_admin(feed, e, local_vars=locals())
settings.RAVEN_CLIENT.captureException(e)
if not settings.DEBUG:
settings.RAVEN_CLIENT.captureException(e)
if not feed_code:
if ret_feed == FEED_OK:

6
utils/feed_functions.py
View file

@ -31,14 +31,12 @@ def timelimit(timeout):
c.join(timeout)
if c.isAlive():
raise TimeoutError, 'took too long'
if not settings.DEBUG and not settings.TEST_DEBUG and c.error:
if c.error:
tb = ''.join(traceback.format_exception(c.error[0], c.error[1], c.error[2]))
logging.debug(tb)
mail_admins('Error in timeout: %s' % c.error[0], tb)
raise c.error[0], c.error[1]
raise c.error[0], c.error[1], c.error[2]
return c.result
# else:
# return function(*args, **kw)
return _2
return _1