diff --git a/apps/reader/views.py b/apps/reader/views.py index c8ffee26b..529d049fd 100644 --- a/apps/reader/views.py +++ b/apps/reader/views.py @@ -432,7 +432,8 @@ def load_single_feed(request, feed_id): feed_authors=feed_authors, classifiers=classifiers, last_update=last_update, - feed_id=feed.pk) + feed_id=feed.pk, + elapsed_time=round(float(timediff), 2)) if dupe_feed_id: data['dupe_feed_id'] = dupe_feed_id if not usersub: @@ -486,13 +487,12 @@ def load_starred_stories(request): def load_river_stories(request): limit = 18 offset = 0 - start = datetime.datetime.utcnow() + start = time.time() user = get_user(request) feed_ids = [int(feed_id) for feed_id in request.REQUEST.getlist('feeds') if feed_id] original_feed_ids = list(feed_ids) page = int(request.REQUEST.get('page', 1)) read_stories_count = int(request.REQUEST.get('read_stories_count', 0)) - new_flag = request.REQUEST.get('new_flag', False) bottom_delta = datetime.timedelta(days=settings.DAYS_OF_UNREAD) if not feed_ids: @@ -615,19 +615,15 @@ def load_river_stories(request): 'tags': apply_classifier_tags(classifier_tags[story['story_feed_id']], story), 'title': apply_classifier_titles(classifier_titles[story['story_feed_id']], story), } - - diff = datetime.datetime.utcnow() - start - timediff = float("%s.%.2s" % (diff.seconds, (diff.microseconds / 1000))) + + diff = time.time() - start + timediff = round(float(diff), 2) logging.user(request, "~FCLoading river stories: page %s - ~SB%s/%s " "stories ~SN(%s/%s/%s feeds) ~FB(%s seconds)" % (page, len(stories), len(mstories), len(found_feed_ids), len(feed_ids), len(original_feed_ids), timediff)) - if new_flag: - return dict(stories=stories, classifiers=classifiers) - else: - logging.user(request, "~BR~FCNo new flag on river") - return dict(stories=stories) + return dict(stories=stories, classifiers=classifiers, elapsed_time=timediff) @ajax_login_required diff --git a/apps/rss_feeds/migrations/0050_unique_hash.py b/apps/rss_feeds/migrations/0050_unique_hash.py deleted file mode 100644 index 451c0260d..000000000 --- a/apps/rss_feeds/migrations/0050_unique_hash.py +++ /dev/null @@ -1,105 +0,0 @@ -# encoding: utf-8 -import datetime -from south.db import db -from south.v2 import SchemaMigration -from django.db import models - -class Migration(SchemaMigration): - - def forwards(self, orm): - - # Removing unique constraint on 'Feed', fields ['feed_address'] - db.delete_unique('feeds', ['feed_address']) - - # Adding field 'Feed.branch_from_feed' - db.add_column('feeds', 'branch_from_feed', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['rss_feeds.Feed'], null=True, blank=True), keep_default=False) - - # Changing field 'Feed.hash_address_and_link' - db.alter_column('feeds', 'hash_address_and_link', self.gf('django.db.models.fields.CharField')(default=123, unique=True, max_length=64)) - - # Adding index on 'Feed', fields ['hash_address_and_link'] - db.create_index('feeds', ['hash_address_and_link']) - - # Adding unique constraint on 'Feed', fields ['hash_address_and_link'] - db.create_unique('feeds', ['hash_address_and_link']) - - - def backwards(self, orm): - - # Removing unique constraint on 'Feed', fields ['hash_address_and_link'] - db.delete_unique('feeds', ['hash_address_and_link']) - - # Removing index on 'Feed', fields ['hash_address_and_link'] - db.delete_index('feeds', ['hash_address_and_link']) - - # Deleting field 'Feed.branch_from_feed' - db.delete_column('feeds', 'branch_from_feed_id') - - # Adding unique constraint on 'Feed', fields ['feed_address'] - db.create_unique('feeds', ['feed_address']) - - # Changing field 'Feed.hash_address_and_link' - db.alter_column('feeds', 'hash_address_and_link', self.gf('django.db.models.fields.CharField')(max_length=64, null=True)) - - - models = { - 'rss_feeds.duplicatefeed': { - 'Meta': {'object_name': 'DuplicateFeed'}, - 'duplicate_address': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'duplicate_feed_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}), - 'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'duplicate_addresses'", 'to': "orm['rss_feeds.Feed']"}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}) - }, - 'rss_feeds.feed': { - 'Meta': {'ordering': "['feed_title']", 'object_name': 'Feed', 'db_table': "'feeds'"}, - 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}), - 'active_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}), - 'average_stories_per_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}), - 'branch_from_feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']", 'null': 'True', 'blank': 'True'}), - 'creation': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}), - 'days_to_trim': ('django.db.models.fields.IntegerField', [], {'default': '90'}), - 'etag': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), - 'exception_code': ('django.db.models.fields.IntegerField', [], {'default': '0'}), - 'favicon_color': ('django.db.models.fields.CharField', [], {'max_length': '6', 'null': 'True', 'blank': 'True'}), - 'favicon_not_found': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), - 'feed_address': ('django.db.models.fields.URLField', [], {'max_length': '255'}), - 'feed_address_locked': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}), - 'feed_link': ('django.db.models.fields.URLField', [], {'default': "''", 'max_length': '1000', 'null': 'True', 'blank': 'True'}), - 'feed_link_locked': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), - 'feed_title': ('django.db.models.fields.CharField', [], {'default': "'[Untitled]'", 'max_length': '255', 'null': 'True', 'blank': 'True'}), - 'fetched_once': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), - 'has_feed_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), - 'has_page': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), - 'has_page_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), - 'hash_address_and_link': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64', 'db_index': 'True'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'last_load_time': ('django.db.models.fields.IntegerField', [], {'default': '0'}), - 'last_modified': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), - 'last_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), - 'min_to_decay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), - 'next_scheduled_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), - 'num_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), - 'premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), - 'queued_date': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), - 'stories_last_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}) - }, - 'rss_feeds.feeddata': { - 'Meta': {'object_name': 'FeedData'}, - 'feed': ('utils.fields.AutoOneToOneField', [], {'related_name': "'data'", 'unique': 'True', 'to': "orm['rss_feeds.Feed']"}), - 'feed_classifier_counts': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), - 'feed_tagline': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'popular_authors': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}), - 'popular_tags': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}), - 'story_count_history': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) - }, - 'rss_feeds.feedloadtime': { - 'Meta': {'object_name': 'FeedLoadtime'}, - 'date_accessed': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), - 'feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']"}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'loadtime': ('django.db.models.fields.FloatField', [], {}) - } - } - - complete_apps = ['rss_feeds'] diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index 7559c9756..5c434e7ba 100644 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -619,13 +619,8 @@ class Feed(models.Model): story = pre_process_story(story) if story.get('title'): - story_contents = story.get('content') + story_content = story.get('story_content') story_tags = self.get_tags(story) - - if story_contents is not None: - story_content = story_contents[0]['value'] - else: - story_content = story.get('summary') existing_story, story_has_changed = self._exists_story(story, story_content, existing_stories) if existing_story is None: diff --git a/apps/rss_feeds/views.py b/apps/rss_feeds/views.py index 1bee9c80d..b99b2223b 100644 --- a/apps/rss_feeds/views.py +++ b/apps/rss_feeds/views.py @@ -159,9 +159,9 @@ def exception_change_feed_address(request): feed.fetched_once = False feed.feed_address = feed_address feed.next_scheduled_update = datetime.datetime.utcnow() - duplicate_feed_id = feed.save() - if duplicate_feed_id: - new_feed = Feed.objects.get(pk=duplicate_feed_id) + duplicate_feed = feed.save() + if duplicate_feed: + new_feed = Feed.objects.get(pk=duplicate_feed.pk) feed = new_feed new_feed.next_scheduled_update = datetime.datetime.utcnow() new_feed.has_feed_exception = False @@ -213,9 +213,9 @@ def exception_change_feed_link(request): feed.feed_link = feed_link feed.feed_address = feed_address feed.next_scheduled_update = datetime.datetime.utcnow() - duplicate_feed_id = feed.save() - if duplicate_feed_id: - new_feed = Feed.objects.get(pk=duplicate_feed_id) + duplicate_feed = feed.save() + if duplicate_feed: + new_feed = Feed.objects.get(pk=duplicate_feed.pk) feed = new_feed new_feed.next_scheduled_update = datetime.datetime.utcnow() new_feed.has_page_exception = False diff --git a/config/nginx.newsblur.conf b/config/nginx.newsblur.conf index a064d40fd..492bb1073 100644 --- a/config/nginx.newsblur.conf +++ b/config/nginx.newsblur.conf @@ -20,7 +20,12 @@ server { if ($host = 'newsblur.com') { rewrite ^/(.*)$ http://www.newsblur.com/$1 permanent; } - + + error_page 503 @maintenance; + location @maintenance { + rewrite ^(.*)$ /home/sclay/newsblur/media/maintenance.html break; + } + location /media/ { expires max; keepalive_timeout 1; @@ -61,11 +66,6 @@ server { return 503; } - error_page 503 @maintenance; - location @maintenance { - rewrite ^(.*)$ /home/sclay/newsblur/media/maintenance.html break; - } - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Host $http_host; proxy_redirect off; diff --git a/fabfile.py b/fabfile.py index 1829425d5..c9dbfeafc 100644 --- a/fabfile.py +++ b/fabfile.py @@ -289,7 +289,7 @@ def setup_psycopg(): def setup_python(): sudo('easy_install pip') - sudo('easy_install fabric django celery django-celery django-compress South django-extensions pymongo BeautifulSoup pyyaml nltk==0.9.9 lxml oauth2 pytz boto seacucumber django_ses mongoengine redis') + sudo('easy_install fabric django celery django-celery django-compress South django-extensions pymongo BeautifulSoup pyyaml nltk==0.9.9 lxml oauth2 pytz boto seacucumber django_ses mongoengine redis requests') put('config/pystartup.py', '.pystartup') with cd(os.path.join(env.NEWSBLUR_PATH, 'vendor/cjson')): diff --git a/local_settings.py.template b/local_settings.py.template index 2970894bd..4e25452c9 100644 --- a/local_settings.py.template +++ b/local_settings.py.template @@ -42,6 +42,14 @@ OAUTH_SECRET = 'SECRET_KEY_FROM_GOOGLE' # Celery RabbitMQ Broker BROKER_HOST = "127.0.0.1" +REDIS = { + 'host': '127.0.0.1', +} + +# =========== +# = Logging = +# =========== + # Logging (setup for development) LOG_TO_STREAM = True diff --git a/media/css/reader.css b/media/css/reader.css index f4c5b4c47..99284240b 100644 --- a/media/css/reader.css +++ b/media/css/reader.css @@ -4936,6 +4936,9 @@ background: transparent; .NB-modal-exception .NB-exception-submit-wrapper { margin: 0 0 2px 100px; } +.NB-modal-exception .NB-modal-loading { + margin: 6px 8px 0; +} /* ===================== */ /* = Feedchooser Modal = */ /* ===================== */ diff --git a/media/js/jquery.newsblur.js b/media/js/jquery.newsblur.js index a281e3978..fcc15d11b 100644 --- a/media/js/jquery.newsblur.js +++ b/media/js/jquery.newsblur.js @@ -35,13 +35,16 @@ NEWSBLUR.log = function(msg) { autolink: function() { return this.each(function(){ - var desc = $(this); - desc.textNodes().each(function(){ + var $desc = $(this); + $desc.textNodes().each(function(){ var text = $(this); if(text && text.parent() && text.parent()[0] && text.parent()[0].nodeName != 'A') { - text.replaceWith(this.data.replace(URL_REGEX, function($0, $1) { - return '' + $0 + ''; - })); + if (this.data.indexOf('http') != -1) { + text.replaceWith(this.data.replace(URL_REGEX, function($0, $1) { + console.log(["Replacing text link", $0]); + return '' + $0 + ''; + })); + } } }); }); diff --git a/media/js/newsblur/assetmodel.js b/media/js/newsblur/assetmodel.js index 6f2535b88..f1143a26e 100644 --- a/media/js/newsblur/assetmodel.js +++ b/media/js/newsblur/assetmodel.js @@ -405,9 +405,7 @@ NEWSBLUR.AssetModel.Reader.prototype = { this.make_request('/reader/river_stories', { feeds: feeds, page: page, - read_stories_count: this.read_stories_river_count, - // TODO: Remove new flag - new_flag: true + read_stories_count: this.read_stories_river_count }, pre_callback, error_callback, { 'ajax_group': (page ? 'feed_page' : 'feed'), 'request_type': 'GET' diff --git a/media/js/newsblur/reader.js b/media/js/newsblur/reader.js index 1e6f997db..9c1e90b87 100644 --- a/media/js/newsblur/reader.js +++ b/media/js/newsblur/reader.js @@ -974,9 +974,9 @@ } // NEWSBLUR.log(['page_in_story', this.$s.$story_pane, direction, page_height, scroll_height]); if (this.story_view == 'page') { - this.$s.$feed_iframe.scrollTo({top:dir+'='+scroll_height, left:'+=0'}, 260); + this.$s.$feed_iframe.scrollTo({top:dir+'='+scroll_height, left:'+=0'}, 230, {queue: false}); } else if (this.story_view == 'feed') { - this.$s.$feed_stories.scrollTo({top:dir+'='+scroll_height, left:'+=0'}, 370); + this.$s.$feed_stories.scrollTo({top:dir+'='+scroll_height, left:'+=0'}, 340, {queue: false}); } }, @@ -6650,7 +6650,7 @@ }); $document.bind('keydown', 'shift+space', function(e) { e.preventDefault(); - self.page_in_story(0.6, -1); + self.page_in_story(0.65, -1); }); $document.bind('keydown', 'u', function(e) { e.preventDefault(); diff --git a/media/js/newsblur/reader_keyboard.js b/media/js/newsblur/reader_keyboard.js index da7525215..89d039c59 100644 --- a/media/js/newsblur/reader_keyboard.js +++ b/media/js/newsblur/reader_keyboard.js @@ -88,7 +88,7 @@ NEWSBLUR.ReaderKeyboard.prototype = { ]) ]), $.make('div', { className: 'NB-keyboard-shortcut NB-last' }, [ - $.make('div', { className: 'NB-keyboard-shortcut-explanation' }, 'Open Site'), + $.make('div', { className: 'NB-keyboard-shortcut-explanation' }, 'Open in Story view'), $.make('div', { className: 'NB-keyboard-shortcut-key' }, [ 'enter' ]), @@ -166,13 +166,43 @@ NEWSBLUR.ReaderKeyboard.prototype = { ]) ]), $.make('div', { className: 'NB-keyboard-shortcut NB-last' }, [ - $.make('div', { className: 'NB-keyboard-shortcut-explanation' }, 'Mark site/folder as read'), + $.make('div', { className: 'NB-keyboard-shortcut-explanation' }, 'Mark all as read'), $.make('div', { className: 'NB-keyboard-shortcut-key' }, [ 'shift', $.make('span', '+'), 'a' ]) ]) + ]), + $.make('div', { className: 'NB-keyboard-group' }, [ + $.make('div', { className: 'NB-keyboard-shortcut' }, [ + $.make('div', { className: 'NB-keyboard-shortcut-explanation' }, 'Return to dashboard'), + $.make('div', { className: 'NB-keyboard-shortcut-key' }, [ + 'd' + ]) + ]), + $.make('div', { className: 'NB-keyboard-shortcut NB-last' }, [ + $.make('div', { className: 'NB-keyboard-shortcut-explanation' }, 'Open Everything'), + $.make('div', { className: 'NB-keyboard-shortcut-key' }, [ + 'shift', + $.make('span', '+'), + 'e' + ]) + ]) + ]), + $.make('div', { className: 'NB-keyboard-group' }, [ + $.make('div', { className: 'NB-keyboard-shortcut' }, [ + $.make('div', { className: 'NB-keyboard-shortcut-explanation' }, 'View keyboard shortcuts'), + $.make('div', { className: 'NB-keyboard-shortcut-key' }, [ + '?' + ]) + ]), + $.make('div', { className: 'NB-keyboard-shortcut NB-last' }, [ + $.make('div', { className: 'NB-keyboard-shortcut-explanation' }, 'Find oldest unread story'), + $.make('div', { className: 'NB-keyboard-shortcut-key' }, [ + 'm' + ]) + ]) ]) ]); }, diff --git a/media/maintenance.html.unused b/media/maintenance.html.unused index 7b5c3eaa2..0bccad374 100644 --- a/media/maintenance.html.unused +++ b/media/maintenance.html.unused @@ -1,3 +1,16 @@ + + +NewsBlur is upgrading... + + + + + +

NewsBlur is upgrading...

-

Upgrading the database will take about an hour. We should be back by 2am ET later tonight.

+

.

+ + + + diff --git a/utils/feed_fetcher.py b/utils/feed_fetcher.py index b3ce5b79f..223f05e40 100644 --- a/utils/feed_fetcher.py +++ b/utils/feed_fetcher.py @@ -247,7 +247,7 @@ class ProcessFeed: # ).order_by('-story_date') ret_values = self.feed.add_update_stories(self.fpf.entries, existing_stories, verbose=self.options['verbose']) - logging.debug(u' ---> [%-30s] ~FYParsed Feed: new~FG=~FG~SB%s~SN~FY up~FG=~FY~SB%s~SN same~FG=~FY%s err~FG=~FR~SB%s' % ( + logging.debug(u' ---> [%-30s] ~FYParsed Feed: new=~FG~SB%s~SN~FY up=~FY~SB%s~SN same=~FY%s err=~FR~SB%s' % ( unicode(self.feed)[:30], ret_values[ENTRY_NEW], ret_values[ENTRY_UPDATED], ret_values[ENTRY_SAME], ret_values[ENTRY_ERR])) self.feed.update_all_statistics() diff --git a/utils/story_functions.py b/utils/story_functions.py index d78b21115..e2769b47f 100644 --- a/utils/story_functions.py +++ b/utils/story_functions.py @@ -67,18 +67,33 @@ def pre_process_story(entry): entry['link'] = urlquote(entry_link) if isinstance(entry.get('guid'), dict): entry['guid'] = unicode(entry['guid']) - entry_content = "" + + # Normalize story content/summary if entry.get('content'): - entry_content = entry['content'][0]['value'] - if entry.get('media_content') and 'audio controls' not in entry_content: - media_url = entry['media_content'][0].get('url') and entry['media_content'][0]['url'] - media_type = entry['media_content'][0].get('type') and entry['media_content'][0]['type'] - if media_url and media_type: - entry['content'][0]['value'] += """

- """ % {'media_url': media_url, 'media_type': media_type} + entry['story_content'] = entry['content'][0].get('value', '') + else: + entry['story_content'] = entry.get('summary', '') + + # Add each media enclosure as a Download link + for media_content in entry.get('media_content', []): + media_url = media_content.get('url', '') + media_type = media_content.get('type', '') + if media_url and media_type and media_url not in entry['story_content']: + if 'audio' in media_type and media_url: + entry['story_content'] += """

+ """ % { + 'media_url': media_url, + 'media_type': media_type + } + elif 'image' in media_type and media_url: + entry['story_content'] += """

""" % media_url + entry['story_content'] += """

+ Download %(media_type)s: %(media_url)s""" % { + 'media_url': media_url, + 'media_type': media_type.split('/')[0] + } entry['guid'] = entry.get('guid') or entry.get('id') or entry.get('link') or str(entry.get('published'))