public-mirrors/NewsBlur
1
0
Fork 0
mirror of https://github.com/samuelclay/NewsBlur.git synced 2025-08-31 21:41:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into jammit

Browse source 
* master:
  Fixing bogus type error, caused by duplicate feeds.
  Correcting styling on site settings loader icon.
  Removing dupe migration from feed_settings.
  Revert "Switching to requests from urllib2/httplib. Bring on the page errors."
  Fixing #41: Adding REDIS to local_settings.py.template.
  Revert "Revert "Switching to requests from urllib2/httplib. Bring on the page errors.""
  Revert "Switching to requests from urllib2/httplib. Bring on the page errors."
  Switching to requests from urllib2/httplib. Bring on the page errors.
  Adding a bunch fo keyboard shortcuts.
  Adding elapsed time field to river and feed view.
  Fixing autolinking of text links to no longer indiscriminately replace text, causing double encoding of html entities.
  Fixing the space bar key to not queue, so it's actually useful now!
  Refining media enclosures by adding all types and linking to their source.
  Refining media enclosure detection.

Conflicts:
	config/nginx.newsblur.conf
This commit is contained in:
Samuel Clay 2011-11-27 03:17:23 -05:00
commit d247ab6206
15 changed files with 117 additions and 161 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
apps/reader/views.py
View file

@ -432,7 +432,8 @@ def load_single_feed(request, feed_id):
feed_authors=feed_authors,
classifiers=classifiers,
last_update=last_update,
feed_id=feed.pk)
feed_id=feed.pk,
elapsed_time=round(float(timediff), 2))
if dupe_feed_id: data['dupe_feed_id'] = dupe_feed_id
if not usersub:
@ -486,13 +487,12 @@ def load_starred_stories(request):
def load_river_stories(request):
limit = 18
offset = 0
start = datetime.datetime.utcnow()
start = time.time()
user = get_user(request)
feed_ids = [int(feed_id) for feed_id in request.REQUEST.getlist('feeds') if feed_id]
original_feed_ids = list(feed_ids)
page = int(request.REQUEST.get('page', 1))
read_stories_count = int(request.REQUEST.get('read_stories_count', 0))
new_flag = request.REQUEST.get('new_flag', False)
bottom_delta = datetime.timedelta(days=settings.DAYS_OF_UNREAD)
if not feed_ids:
@ -615,19 +615,15 @@ def load_river_stories(request):
'tags': apply_classifier_tags(classifier_tags[story['story_feed_id']], story),
'title': apply_classifier_titles(classifier_titles[story['story_feed_id']], story),
}
diff = datetime.datetime.utcnow() - start
timediff = float("%s.%.2s" % (diff.seconds, (diff.microseconds / 1000)))
diff = time.time() - start
timediff = round(float(diff), 2)
logging.user(request, "~FCLoading river stories: page %s - ~SB%s/%s "
"stories ~SN(%s/%s/%s feeds) ~FB(%s seconds)" %
(page, len(stories), len(mstories), len(found_feed_ids),
len(feed_ids), len(original_feed_ids), timediff))
if new_flag:
return dict(stories=stories, classifiers=classifiers)
else:
logging.user(request, "~BR~FCNo new flag on river")
return dict(stories=stories)
return dict(stories=stories, classifiers=classifiers, elapsed_time=timediff)
@ajax_login_required

105
apps/rss_feeds/migrations/0050_unique_hash.py
View file

@ -1,105 +0,0 @@
# encoding: utf-8
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Removing unique constraint on 'Feed', fields ['feed_address']
db.delete_unique('feeds', ['feed_address'])
# Adding field 'Feed.branch_from_feed'
db.add_column('feeds', 'branch_from_feed', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['rss_feeds.Feed'], null=True, blank=True), keep_default=False)
# Changing field 'Feed.hash_address_and_link'
db.alter_column('feeds', 'hash_address_and_link', self.gf('django.db.models.fields.CharField')(default=123, unique=True, max_length=64))
# Adding index on 'Feed', fields ['hash_address_and_link']
db.create_index('feeds', ['hash_address_and_link'])
# Adding unique constraint on 'Feed', fields ['hash_address_and_link']
db.create_unique('feeds', ['hash_address_and_link'])
def backwards(self, orm):
# Removing unique constraint on 'Feed', fields ['hash_address_and_link']
db.delete_unique('feeds', ['hash_address_and_link'])
# Removing index on 'Feed', fields ['hash_address_and_link']
db.delete_index('feeds', ['hash_address_and_link'])
# Deleting field 'Feed.branch_from_feed'
db.delete_column('feeds', 'branch_from_feed_id')
# Adding unique constraint on 'Feed', fields ['feed_address']
db.create_unique('feeds', ['feed_address'])
# Changing field 'Feed.hash_address_and_link'
db.alter_column('feeds', 'hash_address_and_link', self.gf('django.db.models.fields.CharField')(max_length=64, null=True))
models = {
'rss_feeds.duplicatefeed': {
'Meta': {'object_name': 'DuplicateFeed'},
'duplicate_address': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'duplicate_feed_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}),
'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'duplicate_addresses'", 'to': "orm['rss_feeds.Feed']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'rss_feeds.feed': {
'Meta': {'ordering': "['feed_title']", 'object_name': 'Feed', 'db_table': "'feeds'"},
'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
'active_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}),
'average_stories_per_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'branch_from_feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']", 'null': 'True', 'blank': 'True'}),
'creation': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'days_to_trim': ('django.db.models.fields.IntegerField', [], {'default': '90'}),
'etag': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
'exception_code': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'favicon_color': ('django.db.models.fields.CharField', [], {'max_length': '6', 'null': 'True', 'blank': 'True'}),
'favicon_not_found': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'feed_address': ('django.db.models.fields.URLField', [], {'max_length': '255'}),
'feed_address_locked': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
'feed_link': ('django.db.models.fields.URLField', [], {'default': "''", 'max_length': '1000', 'null': 'True', 'blank': 'True'}),
'feed_link_locked': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'feed_title': ('django.db.models.fields.CharField', [], {'default': "'[Untitled]'", 'max_length': '255', 'null': 'True', 'blank': 'True'}),
'fetched_once': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'has_feed_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
'has_page': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'has_page_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
'hash_address_and_link': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'last_load_time': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'last_modified': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
'last_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
'min_to_decay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'next_scheduled_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
'num_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
'premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
'queued_date': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
'stories_last_month': ('django.db.models.fields.IntegerField', [], {'default': '0'})
},
'rss_feeds.feeddata': {
'Meta': {'object_name': 'FeedData'},
'feed': ('utils.fields.AutoOneToOneField', [], {'related_name': "'data'", 'unique': 'True', 'to': "orm['rss_feeds.Feed']"}),
'feed_classifier_counts': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'feed_tagline': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'popular_authors': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}),
'popular_tags': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
'story_count_history': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'})
},
'rss_feeds.feedloadtime': {
'Meta': {'object_name': 'FeedLoadtime'},
'date_accessed': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'loadtime': ('django.db.models.fields.FloatField', [], {})
}
}
complete_apps = ['rss_feeds']

7
apps/rss_feeds/models.py
View file

@ -619,13 +619,8 @@ class Feed(models.Model):
story = pre_process_story(story)
if story.get('title'):
story_contents = story.get('content')
story_content = story.get('story_content')
story_tags = self.get_tags(story)
if story_contents is not None:
story_content = story_contents[0]['value']
else:
story_content = story.get('summary')
existing_story, story_has_changed = self._exists_story(story, story_content, existing_stories)
if existing_story is None:

12
apps/rss_feeds/views.py
View file

@ -159,9 +159,9 @@ def exception_change_feed_address(request):
feed.fetched_once = False
feed.feed_address = feed_address
feed.next_scheduled_update = datetime.datetime.utcnow()
duplicate_feed_id = feed.save()
if duplicate_feed_id:
new_feed = Feed.objects.get(pk=duplicate_feed_id)
duplicate_feed = feed.save()
if duplicate_feed:
new_feed = Feed.objects.get(pk=duplicate_feed.pk)
feed = new_feed
new_feed.next_scheduled_update = datetime.datetime.utcnow()
new_feed.has_feed_exception = False
@ -213,9 +213,9 @@ def exception_change_feed_link(request):
feed.feed_link = feed_link
feed.feed_address = feed_address
feed.next_scheduled_update = datetime.datetime.utcnow()
duplicate_feed_id = feed.save()
if duplicate_feed_id:
new_feed = Feed.objects.get(pk=duplicate_feed_id)
duplicate_feed = feed.save()
if duplicate_feed:
new_feed = Feed.objects.get(pk=duplicate_feed.pk)
feed = new_feed
new_feed.next_scheduled_update = datetime.datetime.utcnow()
new_feed.has_page_exception = False

12
config/nginx.newsblur.conf
View file

@ -20,7 +20,12 @@ server {
if ($host = 'newsblur.com') {
rewrite ^/(.*)$ http://www.newsblur.com/$1 permanent;
}
error_page 503 @maintenance;
location @maintenance {
rewrite ^(.*)$ /home/sclay/newsblur/media/maintenance.html break;
}
location /media/ {
expires max;
keepalive_timeout 1;
@ -61,11 +66,6 @@ server {
return 503;
}
error_page 503 @maintenance;
location @maintenance {
rewrite ^(.*)$ /home/sclay/newsblur/media/maintenance.html break;
}
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
proxy_redirect off;

2
fabfile.py vendored
View file

@ -289,7 +289,7 @@ def setup_psycopg():
def setup_python():
sudo('easy_install pip')
sudo('easy_install fabric django celery django-celery django-compress South django-extensions pymongo BeautifulSoup pyyaml nltk==0.9.9 lxml oauth2 pytz boto seacucumber django_ses mongoengine redis')
sudo('easy_install fabric django celery django-celery django-compress South django-extensions pymongo BeautifulSoup pyyaml nltk==0.9.9 lxml oauth2 pytz boto seacucumber django_ses mongoengine redis requests')
put('config/pystartup.py', '.pystartup')
with cd(os.path.join(env.NEWSBLUR_PATH, 'vendor/cjson')):

8
local_settings.py.template
View file

@ -42,6 +42,14 @@ OAUTH_SECRET = 'SECRET_KEY_FROM_GOOGLE'
# Celery RabbitMQ Broker
BROKER_HOST = "127.0.0.1"
REDIS = {
'host': '127.0.0.1',
}
# ===========
# = Logging =
# ===========
# Logging (setup for development)
LOG_TO_STREAM = True

3
media/css/reader.css
View file

@ -4936,6 +4936,9 @@ background: transparent;
.NB-modal-exception .NB-exception-submit-wrapper {
margin: 0 0 2px 100px;
}
.NB-modal-exception .NB-modal-loading {
margin: 6px 8px 0;
}
/* ===================== */
/* = Feedchooser Modal = */
/* ===================== */

13
media/js/jquery.newsblur.js
View file

@ -35,13 +35,16 @@ NEWSBLUR.log = function(msg) {
autolink: function() {
return this.each(function(){
var desc = $(this);
desc.textNodes().each(function(){
var $desc = $(this);
$desc.textNodes().each(function(){
var text = $(this);
if(text && text.parent() && text.parent()[0] && text.parent()[0].nodeName != 'A') {
text.replaceWith(this.data.replace(URL_REGEX, function($0, $1) {
return '<a href="' + $0 +'">' + $0 + '</a>';
}));
if (this.data.indexOf('http') != -1) {
text.replaceWith(this.data.replace(URL_REGEX, function($0, $1) {
console.log(["Replacing text link", $0]);
return '<a href="' + $0 +'">' + $0 + '</a>';
}));
}
}
});
});

4
media/js/newsblur/assetmodel.js
View file

@ -405,9 +405,7 @@ NEWSBLUR.AssetModel.Reader.prototype = {
this.make_request('/reader/river_stories', {
feeds: feeds,
page: page,
read_stories_count: this.read_stories_river_count,
// TODO: Remove new flag
new_flag: true
read_stories_count: this.read_stories_river_count
}, pre_callback, error_callback, {
'ajax_group': (page ? 'feed_page' : 'feed'),
'request_type': 'GET'

6
media/js/newsblur/reader.js
View file

@ -974,9 +974,9 @@
}
// NEWSBLUR.log(['page_in_story', this.$s.$story_pane, direction, page_height, scroll_height]);
if (this.story_view == 'page') {
this.$s.$feed_iframe.scrollTo({top:dir+'='+scroll_height, left:'+=0'}, 260);
this.$s.$feed_iframe.scrollTo({top:dir+'='+scroll_height, left:'+=0'}, 230, {queue: false});
} else if (this.story_view == 'feed') {
this.$s.$feed_stories.scrollTo({top:dir+'='+scroll_height, left:'+=0'}, 370);
this.$s.$feed_stories.scrollTo({top:dir+'='+scroll_height, left:'+=0'}, 340, {queue: false});
}
},
@ -6650,7 +6650,7 @@
});
$document.bind('keydown', 'shift+space', function(e) {
e.preventDefault();
self.page_in_story(0.6, -1);
self.page_in_story(0.65, -1);
});
$document.bind('keydown', 'u', function(e) {
e.preventDefault();

34
media/js/newsblur/reader_keyboard.js
View file

@ -88,7 +88,7 @@ NEWSBLUR.ReaderKeyboard.prototype = {
])
]),
$.make('div', { className: 'NB-keyboard-shortcut NB-last' }, [
$.make('div', { className: 'NB-keyboard-shortcut-explanation' }, 'Open Site'),
$.make('div', { className: 'NB-keyboard-shortcut-explanation' }, 'Open in Story view'),
$.make('div', { className: 'NB-keyboard-shortcut-key' }, [
'enter'
]),
@ -166,13 +166,43 @@ NEWSBLUR.ReaderKeyboard.prototype = {
])
]),
$.make('div', { className: 'NB-keyboard-shortcut NB-last' }, [
$.make('div', { className: 'NB-keyboard-shortcut-explanation' }, 'Mark site/folder as read'),
$.make('div', { className: 'NB-keyboard-shortcut-explanation' }, 'Mark all as read'),
$.make('div', { className: 'NB-keyboard-shortcut-key' }, [
'shift',
$.make('span', '+'),
'a'
])
])
]),
$.make('div', { className: 'NB-keyboard-group' }, [
$.make('div', { className: 'NB-keyboard-shortcut' }, [
$.make('div', { className: 'NB-keyboard-shortcut-explanation' }, 'Return to dashboard'),
$.make('div', { className: 'NB-keyboard-shortcut-key' }, [
'd'
])
]),
$.make('div', { className: 'NB-keyboard-shortcut NB-last' }, [
$.make('div', { className: 'NB-keyboard-shortcut-explanation' }, 'Open Everything'),
$.make('div', { className: 'NB-keyboard-shortcut-key' }, [
'shift',
$.make('span', '+'),
'e'
])
])
]),
$.make('div', { className: 'NB-keyboard-group' }, [
$.make('div', { className: 'NB-keyboard-shortcut' }, [
$.make('div', { className: 'NB-keyboard-shortcut-explanation' }, 'View keyboard shortcuts'),
$.make('div', { className: 'NB-keyboard-shortcut-key' }, [
'?'
])
]),
$.make('div', { className: 'NB-keyboard-shortcut NB-last' }, [
$.make('div', { className: 'NB-keyboard-shortcut-explanation' }, 'Find oldest unread story'),
$.make('div', { className: 'NB-keyboard-shortcut-key' }, [
'm'
])
])
])
]);
},

15
media/maintenance.html.unused
View file

@ -1,3 +1,16 @@
<html>
<head>
<title>NewsBlur is upgrading...</title>
<META HTTP-EQUIV="CACHE-CONTROL" CONTENT="NO-CACHE">
<META HTTP-EQUIV="EXPIRES" CONTENT="0">
<META HTTP-EQUIV="PRAGMA" CONTENT="NO-CACHE">
</head>
<body>
<h1>NewsBlur is upgrading...</h1>
<p>Upgrading the database will take about an hour. We should be back by 2am ET later tonight.</p>
<p>.</p>
</body>
</html>

2
utils/feed_fetcher.py
View file

@ -247,7 +247,7 @@ class ProcessFeed:
# ).order_by('-story_date')
ret_values = self.feed.add_update_stories(self.fpf.entries, existing_stories, verbose=self.options['verbose'])
logging.debug(u' ---> [%-30s] ~FYParsed Feed: new~FG=~FG~SB%s~SN~FY up~FG=~FY~SB%s~SN same~FG=~FY%s err~FG=~FR~SB%s' % (
logging.debug(u' ---> [%-30s] ~FYParsed Feed: new=~FG~SB%s~SN~FY up=~FY~SB%s~SN same=~FY%s err=~FR~SB%s' % (
unicode(self.feed)[:30],
ret_values[ENTRY_NEW], ret_values[ENTRY_UPDATED], ret_values[ENTRY_SAME], ret_values[ENTRY_ERR]))
self.feed.update_all_statistics()

37
utils/story_functions.py
View file

@ -67,18 +67,33 @@ def pre_process_story(entry):
entry['link'] = urlquote(entry_link)
if isinstance(entry.get('guid'), dict):
entry['guid'] = unicode(entry['guid'])
entry_content = ""
# Normalize story content/summary
if entry.get('content'):
entry_content = entry['content'][0]['value']
if entry.get('media_content') and 'audio controls' not in entry_content:
media_url = entry['media_content'][0].get('url') and entry['media_content'][0]['url']
media_type = entry['media_content'][0].get('type') and entry['media_content'][0]['type']
if media_url and media_type:
entry['content'][0]['value'] += """<br><br>
<audio controls="controls">
<source src="%(media_url)s" type="%(media_type)s" />
<a href="%(media_url)s">%(media_url)s</a>
</audio>""" % {'media_url': media_url, 'media_type': media_type}
entry['story_content'] = entry['content'][0].get('value', '')
else:
entry['story_content'] = entry.get('summary', '')
# Add each media enclosure as a Download link
for media_content in entry.get('media_content', []):
media_url = media_content.get('url', '')
media_type = media_content.get('type', '')
if media_url and media_type and media_url not in entry['story_content']:
if 'audio' in media_type and media_url:
entry['story_content'] += """<br><br>
<audio controls="controls">
<source src="%(media_url)s" type="%(media_type)s" />
</audio>""" % {
'media_url': media_url,
'media_type': media_type
}
elif 'image' in media_type and media_url:
entry['story_content'] += """<br><br><img src="%s" />""" % media_url
entry['story_content'] += """<br><br>
Download %(media_type)s: <a href="%(media_url)s">%(media_url)s</a>""" % {
'media_url': media_url,
'media_type': media_type.split('/')[0]
}
entry['guid'] = entry.get('guid') or entry.get('id') or entry.get('link') or str(entry.get('published'))