public-mirrors/NewsBlur
1
0
Fork 0
mirror of https://github.com/samuelclay/NewsBlur.git synced 2025-08-31 21:41:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into social

Browse source 
* master:
  Showing feed pushes in statistics.
  Instrumentation to count feed pushes.
  Migration to fix starred stories with incorrectly double-encoded url strings.
  Fixing bug around missing feed links being checked for length.

Conflicts:
	apps/rss_feeds/views.py
This commit is contained in:
Samuel Clay 2012-04-09 15:19:25 -07:00
commit ef6c92bf92
6 changed files with 169 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
apps/push/views.py
View file

@ -8,6 +8,7 @@ from django.shortcuts import get_object_or_404
from apps.push.models import PushSubscription from apps.push.models import PushSubscription
from apps.push.signals import verified from apps.push.signals import verified
from apps.rss_feeds.models import MFeedPushHistory
def push_callback(request, push_id): def push_callback(request, push_id):
if request.method == 'GET': if request.method == 'GET':
@ -42,6 +43,7 @@ def push_callback(request, push_id):
# Don't give fat ping, just fetch. # Don't give fat ping, just fetch.
# subscription.feed.queue_pushed_feed_xml(request.raw_post_data) # subscription.feed.queue_pushed_feed_xml(request.raw_post_data)
subscription.feed.queue_pushed_feed_xml("Fetch me") subscription.feed.queue_pushed_feed_xml("Fetch me")
MFeedPushHistory.objects.create(feed_id=subscription.feed_id)
return HttpResponse('') return HttpResponse('')
return Http404 return Http404

98
apps/rss_feeds/migrations/0055_starred_story_permalinks.py Normal file
View file

@ -0,0 +1,98 @@
# encoding: utf-8
from south.v2 import DataMigration
from urllib import unquote
from apps.rss_feeds.models import MStarredStory
class Migration(DataMigration):
def forwards(self, orm):
ss_count = MStarredStory.objects.count()
print "%s starred stories" % ss_count
group_size = ss_count/1000
for group in range(1000):
offset = group_size*group
print "Group offset: %s/%s-%s" % (group, offset, group_size*(group+1))
stories = MStarredStory.objects.order_by('pk')[offset:group_size*(group+1)]
for i, story in enumerate(stories):
original_permalink = story.story_permalink
try:
story.story_permalink = unquote(story.story_permalink.encode('utf-8', 'ignore'))
if original_permalink != story.story_permalink:
story.save()
print " ---> Fixing %s" % (story.pk)
print " Story #%s from: %s" % (i+offset, original_permalink)
print " Story #%s to : %s" % (i+offset, story.story_permalink)
except UnicodeDecodeError:
print " ***> Did not like: %s" % story.story_permalink
pass
def backwards(self, orm):
"Write your backwards methods here."
pass
models = {
'rss_feeds.duplicatefeed': {
'Meta': {'object_name': 'DuplicateFeed'},
'duplicate_address': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'duplicate_feed_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}),
'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'duplicate_addresses'", 'to': "orm['rss_feeds.Feed']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'rss_feeds.feed': {
'Meta': {'ordering': "['feed_title']", 'object_name': 'Feed', 'db_table': "'feeds'"},
'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
'active_premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}),
'active_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}),
'average_stories_per_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'branch_from_feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']", 'null': 'True', 'blank': 'True'}),
'creation': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'days_to_trim': ('django.db.models.fields.IntegerField', [], {'default': '90'}),
'etag': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
'exception_code': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'favicon_color': ('django.db.models.fields.CharField', [], {'max_length': '6', 'null': 'True', 'blank': 'True'}),
'favicon_not_found': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'feed_address': ('django.db.models.fields.URLField', [], {'max_length': '255'}),
'feed_address_locked': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
'feed_link': ('django.db.models.fields.URLField', [], {'default': "''", 'max_length': '1000', 'null': 'True', 'blank': 'True'}),
'feed_link_locked': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'feed_title': ('django.db.models.fields.CharField', [], {'default': "'[Untitled]'", 'max_length': '255', 'null': 'True', 'blank': 'True'}),
'fetched_once': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'has_feed_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
'has_page': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'has_page_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
'hash_address_and_link': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'is_push': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
'known_good': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
'last_load_time': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'last_modified': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
'last_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
'min_to_decay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'next_scheduled_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
'num_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
'premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
'queued_date': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
'stories_last_month': ('django.db.models.fields.IntegerField', [], {'default': '0'})
},
'rss_feeds.feeddata': {
'Meta': {'object_name': 'FeedData'},
'feed': ('utils.fields.AutoOneToOneField', [], {'related_name': "'data'", 'unique': 'True', 'to': "orm['rss_feeds.Feed']"}),
'feed_classifier_counts': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'feed_tagline': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'popular_authors': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}),
'popular_tags': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
'story_count_history': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'})
},
'rss_feeds.feedloadtime': {
'Meta': {'object_name': 'FeedLoadtime'},
'date_accessed': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'loadtime': ('django.db.models.fields.FloatField', [], {})
}
}
complete_apps = ['rss_feeds']

35
apps/rss_feeds/models.py
View file

@ -141,11 +141,11 @@ class Feed(models.Model):
if len(self.feed_title) > max_feed_title: if len(self.feed_title) > max_feed_title:
self.feed_title = self.feed_title[:max_feed_title] self.feed_title = self.feed_title[:max_feed_title]
max_feed_address = Feed._meta.get_field('feed_address').max_length max_feed_address = Feed._meta.get_field('feed_address').max_length
if len(self.feed_address) > max_feed_address: if len(feed_address) > max_feed_address:
self.feed_address = self.feed_address[:max_feed_address] self.feed_address = feed_address[:max_feed_address]
max_feed_link = Feed._meta.get_field('feed_link').max_length max_feed_link = Feed._meta.get_field('feed_link').max_length
if len(self.feed_link) > max_feed_link: if len(feed_link) > max_feed_link:
self.feed_link = self.feed_link[:max_feed_link] self.feed_link = feed_link[:max_feed_link]
try: try:
super(Feed, self).save(*args, **kwargs) super(Feed, self).save(*args, **kwargs)
@ -1355,7 +1355,7 @@ class MFeedFetchHistory(mongo.Document):
for fetch in fetches: for fetch in fetches:
history = {} history = {}
history['message'] = fetch.message history['message'] = fetch.message
history['fetch_date'] = fetch.fetch_date history['fetch_date'] = fetch.fetch_date.strftime("%Y-%m-%d %H:%M:%S")
history['status_code'] = fetch.status_code history['status_code'] = fetch.status_code
history['exception'] = fetch.exception history['exception'] = fetch.exception
fetch_history.append(history) fetch_history.append(history)
@ -1388,12 +1388,35 @@ class MPageFetchHistory(mongo.Document):
for fetch in fetches: for fetch in fetches:
history = {} history = {}
history['message'] = fetch.message history['message'] = fetch.message
history['fetch_date'] = fetch.fetch_date history['fetch_date'] = fetch.fetch_date.strftime("%Y-%m-%d %H:%M:%S")
history['status_code'] = fetch.status_code history['status_code'] = fetch.status_code
history['exception'] = fetch.exception history['exception'] = fetch.exception
fetch_history.append(history) fetch_history.append(history)
return fetch_history return fetch_history
class MFeedPushHistory(mongo.Document):
feed_id = mongo.IntField()
push_date = mongo.DateTimeField(default=datetime.datetime.now)
meta = {
'collection': 'feed_push_history',
'allow_inheritance': False,
'ordering': ['-push_date'],
'indexes': ['feed_id', '-push_date'],
}
@classmethod
def feed_history(cls, feed_id):
pushes = cls.objects(feed_id=feed_id).order_by('-push_date')[:5]
push_history = []
for push in pushes:
history = {}
history['push_date'] = push.push_date.strftime("%Y-%m-%d %H:%M:%S")
push_history.append(history)
return push_history
class FeedLoadtime(models.Model): class FeedLoadtime(models.Model):
feed = models.ForeignKey(Feed) feed = models.ForeignKey(Feed)
date_accessed = models.DateTimeField(auto_now=True) date_accessed = models.DateTimeField(auto_now=True)

4
apps/rss_feeds/views.py
View file

@ -9,7 +9,8 @@ from django.contrib.auth.decorators import login_required
from django.template import RequestContext from django.template import RequestContext
# from django.db import IntegrityError # from django.db import IntegrityError
from apps.rss_feeds.models import Feed, merge_feeds from apps.rss_feeds.models import Feed, merge_feeds
from apps.rss_feeds.models import MFeedFetchHistory, MPageFetchHistory, MFeedIcon from apps.rss_feeds.models import MFeedFetchHistory, MPageFetchHistory, MFeedPushHistory
from apps.rss_feeds.models import MFeedIcon
from apps.analyzer.models import get_classifiers_for_user from apps.analyzer.models import get_classifiers_for_user
from apps.reader.models import UserSubscription from apps.reader.models import UserSubscription
from utils.user_functions import ajax_login_required from utils.user_functions import ajax_login_required
@ -150,6 +151,7 @@ def load_feed_statistics(request, feed_id):
# Fetch histories # Fetch histories
stats['feed_fetch_history'] = MFeedFetchHistory.feed_history(feed_id) stats['feed_fetch_history'] = MFeedFetchHistory.feed_history(feed_id)
stats['page_fetch_history'] = MPageFetchHistory.feed_history(feed_id) stats['page_fetch_history'] = MPageFetchHistory.feed_history(feed_id)
stats['feed_push_history'] = MFeedPushHistory.feed_history(feed_id)
logging.user(request, "~FBStatistics: ~SB%s ~FG(%s/%s/%s subs)" % (feed, feed.num_subscribers, feed.active_subscribers, feed.premium_subscribers,)) logging.user(request, "~FBStatistics: ~SB%s ~FG(%s/%s/%s subs)" % (feed, feed.num_subscribers, feed.active_subscribers, feed.premium_subscribers,))

12
media/css/reader.css
View file

@ -4984,8 +4984,11 @@ background: transparent;
.NB-modal-statistics .NB-statistics-stat .NB-statistics-fetches-half { .NB-modal-statistics .NB-statistics-stat .NB-statistics-fetches-half {
float: left; float: left;
width: 50%;
text-align: center; text-align: center;
margin-right: 18px;
}
.NB-modal-statistics .NB-statistics-stat .NB-statistics-fetches-half:last-child {
margin-right: 0;
} }
.NB-modal-statistics .NB-statistics-stat .NB-statistics-history-stat { .NB-modal-statistics .NB-statistics-stat .NB-statistics-history-stat {
@ -5027,12 +5030,17 @@ background: transparent;
} }
.NB-modal-statistics .NB-statistics-history-fetch .NB-statistics-history-fetch-message { .NB-modal-statistics .NB-statistics-history-fetch .NB-statistics-history-fetch-message {
padding-right: 4px; padding-right: 4px;
margin-left: 120px; margin-left: 110px;
font-weight: bold; font-weight: bold;
} }
.NB-modal-statistics .NB-statistics-history-fetch .NB-statistics-history-fetch-exception { .NB-modal-statistics .NB-statistics-history-fetch .NB-statistics-history-fetch-exception {
display: none; display: none;
} }
.NB-modal-statistics .NB-statistics-history-empty {
color: #C0C0C0;
font-size: 10px;
padding: 4px 12px;
}
.NB-modal-statistics .NB-statistics-classifiers { .NB-modal-statistics .NB-statistics-classifiers {
border: 1px solid #e6e6e6; border: 1px solid #e6e6e6;
clear: both; clear: both;

28
media/js/newsblur/reader/reader_statistics.js
View file

@ -129,12 +129,16 @@ _.extend(NEWSBLUR.ReaderStatistics.prototype, {
])), ])),
(!this.options.social_feed && $.make('div', { className: 'NB-statistics-stat NB-statistics-fetches'}, [ (!this.options.social_feed && $.make('div', { className: 'NB-statistics-stat NB-statistics-fetches'}, [
$.make('div', { className: 'NB-statistics-fetches-half'}, [ $.make('div', { className: 'NB-statistics-fetches-half'}, [
$.make('div', { className: 'NB-statistics-label' }, 'Feed'), $.make('div', { className: 'NB-statistics-label' }, 'Feed Fetch'),
$.make('div', this.make_history(data, 'feed')) $.make('div', this.make_history(data, 'feed_fetch'))
]), ]),
$.make('div', { className: 'NB-statistics-fetches-half'}, [ $.make('div', { className: 'NB-statistics-fetches-half'}, [
$.make('div', { className: 'NB-statistics-label' }, 'Page'), $.make('div', { className: 'NB-statistics-label' }, 'Page Fetch'),
$.make('div', this.make_history(data, 'page')) $.make('div', this.make_history(data, 'page_fetch'))
]),
$.make('div', { className: 'NB-statistics-fetches-half'}, [
$.make('div', { className: 'NB-statistics-label' }, 'Feed Push'),
$.make('div', this.make_history(data, 'feed_push'))
]) ])
])) ]))
]); ]);
@ -212,20 +216,24 @@ _.extend(NEWSBLUR.ReaderStatistics.prototype, {
}, },
make_history: function(data, fetch_type) { make_history: function(data, fetch_type) {
var fetches = data[fetch_type+'_fetch_history']; var fetches = data[fetch_type+'_history'];
if (!fetches) return; var $history;
var $history = _.map(fetches, function(fetch) { if (!fetches || !fetches.length) {
var feed_ok = _.contains([200, 304], fetch.status_code); $history = $.make('div', { className: 'NB-statistics-history-empty' }, "Nothing recorded.");
} else {
$history = _.map(fetches, function(fetch) {
var feed_ok = _.contains([200, 304], fetch.status_code) || !fetch.status_code;
var status_class = feed_ok ? ' NB-ok ' : ' NB-errorcode '; var status_class = feed_ok ? ' NB-ok ' : ' NB-errorcode ';
return $.make('div', { className: 'NB-statistics-history-fetch' + status_class, title: feed_ok ? '' : fetch.exception }, [ return $.make('div', { className: 'NB-statistics-history-fetch' + status_class, title: feed_ok ? '' : fetch.exception }, [
$.make('div', { className: 'NB-statistics-history-fetch-date' }, fetch.fetch_date), $.make('div', { className: 'NB-statistics-history-fetch-date' }, fetch.fetch_date || fetch.push_date),
$.make('div', { className: 'NB-statistics-history-fetch-message' }, [ $.make('div', { className: 'NB-statistics-history-fetch-message' }, [
fetch.message, fetch.message,
$.make('div', { className: 'NB-statistics-history-fetch-code' }, ' ('+fetch.status_code+')') (fetch.status_code && $.make('div', { className: 'NB-statistics-history-fetch-code' }, ' ('+fetch.status_code+')'))
]) ])
]); ]);
}); });
}
return $history; return $history;
}, },