diff --git a/apps/push/views.py b/apps/push/views.py index f9112e2e2..28864a03f 100644 --- a/apps/push/views.py +++ b/apps/push/views.py @@ -8,6 +8,7 @@ from django.shortcuts import get_object_or_404 from apps.push.models import PushSubscription from apps.push.signals import verified +from apps.rss_feeds.models import MFeedPushHistory def push_callback(request, push_id): if request.method == 'GET': @@ -42,6 +43,7 @@ def push_callback(request, push_id): # Don't give fat ping, just fetch. # subscription.feed.queue_pushed_feed_xml(request.raw_post_data) subscription.feed.queue_pushed_feed_xml("Fetch me") - + MFeedPushHistory.objects.create(feed_id=subscription.feed_id) + return HttpResponse('') return Http404 diff --git a/apps/rss_feeds/migrations/0055_starred_story_permalinks.py b/apps/rss_feeds/migrations/0055_starred_story_permalinks.py new file mode 100644 index 000000000..0ac9a376a --- /dev/null +++ b/apps/rss_feeds/migrations/0055_starred_story_permalinks.py @@ -0,0 +1,98 @@ +# encoding: utf-8 +from south.v2 import DataMigration + +from urllib import unquote +from apps.rss_feeds.models import MStarredStory + + +class Migration(DataMigration): + + def forwards(self, orm): + ss_count = MStarredStory.objects.count() + print "%s starred stories" % ss_count + group_size = ss_count/1000 + for group in range(1000): + offset = group_size*group + print "Group offset: %s/%s-%s" % (group, offset, group_size*(group+1)) + stories = MStarredStory.objects.order_by('pk')[offset:group_size*(group+1)] + for i, story in enumerate(stories): + original_permalink = story.story_permalink + try: + story.story_permalink = unquote(story.story_permalink.encode('utf-8', 'ignore')) + if original_permalink != story.story_permalink: + story.save() + print " ---> Fixing %s" % (story.pk) + print " Story #%s from: %s" % (i+offset, original_permalink) + print " Story #%s to : %s" % (i+offset, story.story_permalink) + except UnicodeDecodeError: + print " ***> Did not like: %s" % story.story_permalink + pass + + def backwards(self, orm): + "Write your backwards methods here." + pass + + models = { + 'rss_feeds.duplicatefeed': { + 'Meta': {'object_name': 'DuplicateFeed'}, + 'duplicate_address': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'duplicate_feed_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}), + 'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'duplicate_addresses'", 'to': "orm['rss_feeds.Feed']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}) + }, + 'rss_feeds.feed': { + 'Meta': {'ordering': "['feed_title']", 'object_name': 'Feed', 'db_table': "'feeds'"}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}), + 'active_premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}), + 'active_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}), + 'average_stories_per_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'branch_from_feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']", 'null': 'True', 'blank': 'True'}), + 'creation': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'days_to_trim': ('django.db.models.fields.IntegerField', [], {'default': '90'}), + 'etag': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'exception_code': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'favicon_color': ('django.db.models.fields.CharField', [], {'max_length': '6', 'null': 'True', 'blank': 'True'}), + 'favicon_not_found': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'feed_address': ('django.db.models.fields.URLField', [], {'max_length': '255'}), + 'feed_address_locked': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}), + 'feed_link': ('django.db.models.fields.URLField', [], {'default': "''", 'max_length': '1000', 'null': 'True', 'blank': 'True'}), + 'feed_link_locked': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'feed_title': ('django.db.models.fields.CharField', [], {'default': "'[Untitled]'", 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'fetched_once': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'has_feed_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'has_page': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_page_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'hash_address_and_link': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_push': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}), + 'known_good': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'last_load_time': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'last_modified': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'last_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'min_to_decay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'next_scheduled_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'num_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'queued_date': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'stories_last_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}) + }, + 'rss_feeds.feeddata': { + 'Meta': {'object_name': 'FeedData'}, + 'feed': ('utils.fields.AutoOneToOneField', [], {'related_name': "'data'", 'unique': 'True', 'to': "orm['rss_feeds.Feed']"}), + 'feed_classifier_counts': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'feed_tagline': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'popular_authors': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}), + 'popular_tags': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}), + 'story_count_history': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) + }, + 'rss_feeds.feedloadtime': { + 'Meta': {'object_name': 'FeedLoadtime'}, + 'date_accessed': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'loadtime': ('django.db.models.fields.FloatField', [], {}) + } + } + + complete_apps = ['rss_feeds'] diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index d45c0d724..b7a52871b 100644 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -141,11 +141,11 @@ class Feed(models.Model): if len(self.feed_title) > max_feed_title: self.feed_title = self.feed_title[:max_feed_title] max_feed_address = Feed._meta.get_field('feed_address').max_length - if len(self.feed_address) > max_feed_address: - self.feed_address = self.feed_address[:max_feed_address] + if len(feed_address) > max_feed_address: + self.feed_address = feed_address[:max_feed_address] max_feed_link = Feed._meta.get_field('feed_link').max_length - if len(self.feed_link) > max_feed_link: - self.feed_link = self.feed_link[:max_feed_link] + if len(feed_link) > max_feed_link: + self.feed_link = feed_link[:max_feed_link] try: super(Feed, self).save(*args, **kwargs) @@ -1355,7 +1355,7 @@ class MFeedFetchHistory(mongo.Document): for fetch in fetches: history = {} history['message'] = fetch.message - history['fetch_date'] = fetch.fetch_date + history['fetch_date'] = fetch.fetch_date.strftime("%Y-%m-%d %H:%M:%S") history['status_code'] = fetch.status_code history['exception'] = fetch.exception fetch_history.append(history) @@ -1388,12 +1388,35 @@ class MPageFetchHistory(mongo.Document): for fetch in fetches: history = {} history['message'] = fetch.message - history['fetch_date'] = fetch.fetch_date + history['fetch_date'] = fetch.fetch_date.strftime("%Y-%m-%d %H:%M:%S") history['status_code'] = fetch.status_code history['exception'] = fetch.exception fetch_history.append(history) return fetch_history - + + +class MFeedPushHistory(mongo.Document): + feed_id = mongo.IntField() + push_date = mongo.DateTimeField(default=datetime.datetime.now) + + meta = { + 'collection': 'feed_push_history', + 'allow_inheritance': False, + 'ordering': ['-push_date'], + 'indexes': ['feed_id', '-push_date'], + } + + @classmethod + def feed_history(cls, feed_id): + pushes = cls.objects(feed_id=feed_id).order_by('-push_date')[:5] + push_history = [] + for push in pushes: + history = {} + history['push_date'] = push.push_date.strftime("%Y-%m-%d %H:%M:%S") + push_history.append(history) + return push_history + + class FeedLoadtime(models.Model): feed = models.ForeignKey(Feed) date_accessed = models.DateTimeField(auto_now=True) diff --git a/apps/rss_feeds/views.py b/apps/rss_feeds/views.py index deb6f0675..59b873ace 100644 --- a/apps/rss_feeds/views.py +++ b/apps/rss_feeds/views.py @@ -9,7 +9,8 @@ from django.contrib.auth.decorators import login_required from django.template import RequestContext # from django.db import IntegrityError from apps.rss_feeds.models import Feed, merge_feeds -from apps.rss_feeds.models import MFeedFetchHistory, MPageFetchHistory, MFeedIcon +from apps.rss_feeds.models import MFeedFetchHistory, MPageFetchHistory, MFeedPushHistory +from apps.rss_feeds.models import MFeedIcon from apps.analyzer.models import get_classifiers_for_user from apps.reader.models import UserSubscription from utils.user_functions import ajax_login_required @@ -150,6 +151,7 @@ def load_feed_statistics(request, feed_id): # Fetch histories stats['feed_fetch_history'] = MFeedFetchHistory.feed_history(feed_id) stats['page_fetch_history'] = MPageFetchHistory.feed_history(feed_id) + stats['feed_push_history'] = MFeedPushHistory.feed_history(feed_id) logging.user(request, "~FBStatistics: ~SB%s ~FG(%s/%s/%s subs)" % (feed, feed.num_subscribers, feed.active_subscribers, feed.premium_subscribers,)) diff --git a/media/css/reader.css b/media/css/reader.css index 9a3590366..8c9e96be9 100644 --- a/media/css/reader.css +++ b/media/css/reader.css @@ -4984,8 +4984,11 @@ background: transparent; .NB-modal-statistics .NB-statistics-stat .NB-statistics-fetches-half { float: left; - width: 50%; text-align: center; + margin-right: 18px; +} +.NB-modal-statistics .NB-statistics-stat .NB-statistics-fetches-half:last-child { + margin-right: 0; } .NB-modal-statistics .NB-statistics-stat .NB-statistics-history-stat { @@ -5027,12 +5030,17 @@ background: transparent; } .NB-modal-statistics .NB-statistics-history-fetch .NB-statistics-history-fetch-message { padding-right: 4px; - margin-left: 120px; + margin-left: 110px; font-weight: bold; } .NB-modal-statistics .NB-statistics-history-fetch .NB-statistics-history-fetch-exception { display: none; } +.NB-modal-statistics .NB-statistics-history-empty { + color: #C0C0C0; + font-size: 10px; + padding: 4px 12px; +} .NB-modal-statistics .NB-statistics-classifiers { border: 1px solid #e6e6e6; clear: both; diff --git a/media/js/newsblur/reader/reader_statistics.js b/media/js/newsblur/reader/reader_statistics.js index 8809419c3..2fdfb7ec5 100644 --- a/media/js/newsblur/reader/reader_statistics.js +++ b/media/js/newsblur/reader/reader_statistics.js @@ -129,12 +129,16 @@ _.extend(NEWSBLUR.ReaderStatistics.prototype, { ])), (!this.options.social_feed && $.make('div', { className: 'NB-statistics-stat NB-statistics-fetches'}, [ $.make('div', { className: 'NB-statistics-fetches-half'}, [ - $.make('div', { className: 'NB-statistics-label' }, 'Feed'), - $.make('div', this.make_history(data, 'feed')) + $.make('div', { className: 'NB-statistics-label' }, 'Feed Fetch'), + $.make('div', this.make_history(data, 'feed_fetch')) ]), $.make('div', { className: 'NB-statistics-fetches-half'}, [ - $.make('div', { className: 'NB-statistics-label' }, 'Page'), - $.make('div', this.make_history(data, 'page')) + $.make('div', { className: 'NB-statistics-label' }, 'Page Fetch'), + $.make('div', this.make_history(data, 'page_fetch')) + ]), + $.make('div', { className: 'NB-statistics-fetches-half'}, [ + $.make('div', { className: 'NB-statistics-label' }, 'Feed Push'), + $.make('div', this.make_history(data, 'feed_push')) ]) ])) ]); @@ -212,20 +216,24 @@ _.extend(NEWSBLUR.ReaderStatistics.prototype, { }, make_history: function(data, fetch_type) { - var fetches = data[fetch_type+'_fetch_history']; - if (!fetches) return; + var fetches = data[fetch_type+'_history']; + var $history; - var $history = _.map(fetches, function(fetch) { - var feed_ok = _.contains([200, 304], fetch.status_code); - var status_class = feed_ok ? ' NB-ok ' : ' NB-errorcode '; - return $.make('div', { className: 'NB-statistics-history-fetch' + status_class, title: feed_ok ? '' : fetch.exception }, [ - $.make('div', { className: 'NB-statistics-history-fetch-date' }, fetch.fetch_date), - $.make('div', { className: 'NB-statistics-history-fetch-message' }, [ - fetch.message, - $.make('div', { className: 'NB-statistics-history-fetch-code' }, ' ('+fetch.status_code+')') - ]) - ]); - }); + if (!fetches || !fetches.length) { + $history = $.make('div', { className: 'NB-statistics-history-empty' }, "Nothing recorded."); + } else { + $history = _.map(fetches, function(fetch) { + var feed_ok = _.contains([200, 304], fetch.status_code) || !fetch.status_code; + var status_class = feed_ok ? ' NB-ok ' : ' NB-errorcode '; + return $.make('div', { className: 'NB-statistics-history-fetch' + status_class, title: feed_ok ? '' : fetch.exception }, [ + $.make('div', { className: 'NB-statistics-history-fetch-date' }, fetch.fetch_date || fetch.push_date), + $.make('div', { className: 'NB-statistics-history-fetch-message' }, [ + fetch.message, + (fetch.status_code && $.make('div', { className: 'NB-statistics-history-fetch-code' }, ' ('+fetch.status_code+')')) + ]) + ]); + }); + } return $history; },