Adding classification types for publishers vs. stories.

This commit is contained in:
Samuel Clay 2010-01-12 01:19:37 +00:00
parent 0c33bc84cb
commit 35df77cdf6
10 changed files with 302 additions and 63 deletions

View file

@ -31,7 +31,7 @@ class ClassifierTitle(models.Model):
score = models.SmallIntegerField()
title = models.CharField(max_length=255)
feed = models.ForeignKey(Feed)
original_story = models.ForeignKey(Story)
original_story = models.ForeignKey(Story, null=True)
creation_date = models.DateTimeField(auto_now=True)
def __unicode__(self):
@ -42,7 +42,7 @@ class ClassifierAuthor(models.Model):
score = models.SmallIntegerField()
author = models.ForeignKey(StoryAuthor)
feed = models.ForeignKey(Feed)
original_story = models.ForeignKey(Story)
original_story = models.ForeignKey(Story, null=True)
creation_date = models.DateTimeField(auto_now=True)
def __unicode__(self):
@ -52,7 +52,7 @@ class ClassifierFeed(models.Model):
user = models.ForeignKey(User)
score = models.SmallIntegerField()
feed = models.ForeignKey(Feed)
original_story = models.ForeignKey(Story)
original_story = models.ForeignKey(Story, null=True)
creation_date = models.DateTimeField(auto_now=True)
def __unicode__(self):
@ -63,7 +63,7 @@ class ClassifierTag(models.Model):
score = models.SmallIntegerField()
tag = models.ForeignKey(Tag)
feed = models.ForeignKey(Feed)
original_story = models.ForeignKey(Story)
original_story = models.ForeignKey(Story, null=True)
creation_date = models.DateTimeField(auto_now=True)
def __unicode__(self):

View file

@ -3,5 +3,6 @@ from apps.analyzer import views
urlpatterns = patterns('',
(r'^$', views.index),
(r'^save/?', views.save_classifier),
(r'^save/story/?', views.save_classifier_story),
(r'^save/publisher/?', views.save_classifier_publisher),
)

View file

@ -23,7 +23,7 @@ def index(requst):
pass
@require_POST
def save_classifier(request):
def save_classifier_story(request):
post = request.POST
facets = post.getlist('facet')
code = 0
@ -64,5 +64,41 @@ def save_classifier(request):
feed=feed,
original_story=story)
response = dict(code=code, message=message, payload=payload)
return HttpResponse(response)
@require_POST
def save_classifier_publisher(request):
post = request.POST
facets = post.getlist('facet')
code = 0
message = 'OK'
payload = {}
feed = Feed.objects.get(pk=post['feed_id'])
score = int(post['score'])
if 'author' in post:
authors = post.getlist('authors')
for author_name in authors:
author = StoryAuthor.objects.get(author_name=author_name, feed=feed)
ClassifierAuthor.objects.create(user=request.user,
score=score,
author=author,
feed=feed)
if 'publisher' in facets:
ClassifierFeed.objects.create(user=request.user,
score=score,
feed=feed)
if 'tag' in post:
tags = post.getlist('tag')
for tag_name in tags:
tag = Tag.objects.get(name=tag_name, feed=feed)
ClassifierTag.objects.create(user=request.user,
score=score,
tag=tag,
feed=feed)
response = dict(code=code, message=message, payload=payload)
return HttpResponse(response)

View file

@ -3,7 +3,7 @@ from django.contrib.auth.decorators import login_required
from django.template import RequestContext
from django.db import IntegrityError
try:
from apps.rss_feeds.models import Feed, Story, Tag
from apps.rss_feeds.models import Feed, Story, Tag, StoryAuthor
except:
pass
from django.core.cache import cache
@ -119,9 +119,15 @@ def load_single_feed(request):
all_tags = Tag.objects.filter(feed=feed)\
.annotate(stories_count=Count('story'))\
.order_by('-stories_count')[:20]
tags = [(tag.name, tag.stories_count) for tag in all_tags if tag.stories_count > 1]
feed_tags = [(tag.name, tag.stories_count) for tag in all_tags if tag.stories_count > 1]
context = dict(stories=stories, tags=tags, intelligence={})
all_authors = StoryAuthor.objects.filter(feed=feed)\
.annotate(stories_count=Count('story'))\
.order_by('-stories_count')[:20]
feed_authors = [(author.author_name, author.stories_count) for author in all_authors\
if author.stories_count > 1]
context = dict(stories=stories, feed_tags=feed_tags, feed_authors=feed_authors, intelligence={})
data = json.encode(context)
return HttpResponse(data, mimetype='application/json')

View file

@ -2,6 +2,7 @@ from django.core.management.base import BaseCommand
from django.core.handlers.wsgi import WSGIHandler
from apps.rss_feeds.models import Feed, Story
from django.core.cache import cache
from django.db.models import Q
from apps.reader.models import UserSubscription, UserSubscriptionFolders, UserStory
from optparse import OptionParser, make_option
from utils.management_functions import daemonize
@ -12,14 +13,18 @@ import errno
class Command(BaseCommand):
option_list = BaseCommand.option_list + (
make_option("-f", "--feed", dest="feed", default=None),
make_option("-t", "--title", dest="title", default=None),
make_option("-d", "--daemon", dest="daemonize", action="store_true"),
)
def handle(self, *args, **options):
if options['daemonize']:
daemonize()
feed = Feed.objects.get(id=options['feed'])
if options['title']:
feed = Feed.objects.get(feed_title__contains=options['title'])
else:
feed = Feed.objects.get(pk=options['feed'])
self._refresh_feeds([feed])
def _refresh_feeds(self, feeds):

View file

@ -189,6 +189,19 @@ a img {
font-size: 9px;
}
#story_titles .story .story_title .NB-storytitles-tags {
padding-left: 12px;
font-weight: normal;
}
#story_titles .story .story_title .NB-storytitles-tag {
font-size: 9px;
padding: 0px 4px;
margin: 0 2px;
color: #808080;
background-color: #E9E9E9;
}
#story_titles .story .story_id {
display: none;
}
@ -752,13 +765,31 @@ form.opml_import_form input {
margin: 6px 0 0 0;
}
.NB-classifier .NB-classifier-tag {
background-color: #E0E0FF;
.NB-classifier .NB-publisher .NB-classifier-authors {
margin-left: -24px;
}
.NB-classifier .NB-publisher .NB-classifier-authors .NB-classifier-author {
float: left;
margin: 4px 16px 0 0;
font-weight: bold;
}
.NB-classifier .NB-publisher .NB-classifier-authors input {
float: none;
margin: 4px 6px 0 0;
}
.NB-classifier .NB-classifier-tag-container {
white-space: nowrap;
float: left;
display: block;
margin: 2px 6px 6px 0;
}
.NB-classifier .NB-classifier-tag {
background-color: #E0E0FF;
padding: 2px 6px 2px 0px;
margin: 2px 6px 2px 0;
}
.NB-classifier .NB-classifier-tag input[type=checkbox] {
@ -769,6 +800,12 @@ form.opml_import_form input {
.NB-classifier .NB-classifier-tag label {
}
.NB-classifier .NB-classifier-tag-count {
font-size: 10px;
font-weight: bold;
margin: 0 6px 0 2px;
}
.NB-classifier .NB-classifier-tags {
margin-left: -24px;
}

View file

@ -159,6 +159,8 @@ NEWSBLUR.AssetModel.Reader.prototype = {
var pre_callback = function(data) {
if (feed_id != self.feed_id) {
self.stories = data.stories;
self.feed_tags = data.feed_tags;
self.feed_authors = data.feed_authors;
self.feed_id = feed_id;
} else {
$.merge(self.stories, data.stories);
@ -198,6 +200,14 @@ NEWSBLUR.AssetModel.Reader.prototype = {
return null;
},
get_feed_tags: function() {
return this.feed_tags;
},
get_feed_authors: function() {
return this.feed_authors;
},
get_story: function(story_id, callback) {
var self = this;
for (s in this.stories) {
@ -214,8 +224,12 @@ NEWSBLUR.AssetModel.Reader.prototype = {
this.make_request('/opml_import/process', data, callback);
},
save_classifier: function(story_id, data, callback) {
this.make_request('/classifier/save', data, callback);
save_classifier_story: function(story_id, data, callback) {
this.make_request('/classifier/save/story/', data, callback);
},
save_classifier_publisher: function(data, callback) {
this.make_request('/classifier/save/publisher', data, callback);
}
};

View file

@ -39,10 +39,18 @@
var read = story.read_status
? ' read'
: '';
var $story_tags = $.make('span', { className: 'NB-storytitles-tags'});
for (var t in story.story_tags) {
var tag = story.story_tags[t];
var $tag = $.make('span', { className: 'NB-storytitles-tag'}, tag).corners('4px');
$story_tags.append($tag);
break;
}
var $story_title = $.make('div', { className: 'story' + read }, [
$.make('a', { href: story.story_permalink, className: 'story_title' }, [
story.story_title,
$.make('span', { className: 'NB-storytitles-author'}, story.story_authors)
$.make('span', { className: 'NB-storytitles-title' }, story.story_title),
$.make('span', { className: 'NB-storytitles-author' }, story.story_authors),
$story_tags
]),
$.make('span', { className: 'story_date' }, story.short_parsed_date),
$.make('span', { className: 'story_id' }, ''+story.id),
@ -266,7 +274,7 @@
var folders = self.model.folders;
$('#story_taskbar').css({'display': 'block'});
NEWSBLUR.log(['Subscriptions', {'folders':folders}]);
// NEWSBLUR.log(['Subscriptions', {'folders':folders}]);
for (fo in folders) {
var feeds = folders[fo].feeds;
var $folder = $.make('div', { className: 'folder' }, [
@ -540,6 +548,12 @@
$feed_link.parent('.feed').next('.feed').children('a').addClass('after_selected');
},
open_feed_intelligence_modal: function() {
var feed_id = this.active_feed;
NEWSBLUR.classifier = new NEWSBLUR.ReaderClassifierFeed(feed_id, 1);
},
// ===================
// = Taskbar - Story =
// ===================
@ -675,7 +689,7 @@
})
.not('script')
.each(function() {
NEWSBLUR.log(['Accepted 1 $elem', $(this), $(this).is(':visible')]);
// NEWSBLUR.log(['Accepted 1 $elem', $(this), $(this).is(':visible')]);
});
if (!$stories.length) {
@ -692,7 +706,7 @@
if ($(this).is(':visible')) {
$stories.push(this);
}
NEWSBLUR.log(['Accepted 2 $elem', $(this)]);
// NEWSBLUR.log(['Accepted 2 $elem', $(this)]);
});
}
}
@ -700,7 +714,7 @@
if (!$stories.length) {
// Try slicing words off the title, from the beginning.
title_words = title.match(/[^ ]+/g);
NEWSBLUR.log(['Words', title_words.length, title_words, title_words.slice(1).join(' '), title_words.slice(0, -1).join(' '), title_words.slice(1, -1).join(' ')]);
// NEWSBLUR.log(['Words', title_words.length, title_words, title_words.slice(1).join(' '), title_words.slice(0, -1).join(' '), title_words.slice(1, -1).join(' ')]);
if (title_words.length > 2) {
for (var i=0; i < 3; i++) {
if (i==0) shortened_title = title_words.slice(1).join(' ');
@ -717,7 +731,7 @@
if ($(this).is(':visible')) {
$stories.push(this);
}
NEWSBLUR.log(['Accepted 3 $elem', $(this)]);
// NEWSBLUR.log(['Accepted 3 $elem', $(this)]);
});
// NEWSBLUR.log(['Cutting words off title', $stories.length, $stories]);
if ($stories.length) break;
@ -730,7 +744,7 @@
content_words = story_content.replace(/<([^<>\s]*)(\s[^<>]*)?>/, '')
.replace(/\(.*?\)/, '')
.match(/[^ ]+/g);
NEWSBLUR.log(['content_words', content_words]);
// NEWSBLUR.log(['content_words', content_words]);
if (content_words.length > 2) {
var shortened_content = content_words.slice(0, 8).join(' ');
$iframe_contents.find(':contains('+shortened_content+')')
@ -742,7 +756,7 @@
if ($(this).is(':visible')) {
$stories.push(this);
}
NEWSBLUR.log(['Accepted 4 $elem', $(this)]);
// NEWSBLUR.log(['Accepted 4 $elem', $(this)]);
});
}
}
@ -821,13 +835,13 @@
mark_story_as_like: function(story_id, $button) {
var feed_id = this.active_feed;
NEWSBLUR.classifier = new NEWSBLUR.ReaderClassifier(story_id, feed_id, 1);
NEWSBLUR.classifier = new NEWSBLUR.ReaderClassifierStory(story_id, feed_id, 1);
},
mark_story_as_dislike: function(story_id, $button) {
var feed_id = this.active_feed;
NEWSBLUR.classifier = new NEWSBLUR.ReaderClassifier(story_id, feed_id, -1);
NEWSBLUR.classifier = new NEWSBLUR.ReaderClassifierStory(story_id, feed_id, -1);
},
@ -860,7 +874,7 @@
handle_opml_upload: function() {
var self = this;
NEWSBLUR.log(['Uploading']);
// NEWSBLUR.log(['Uploading']);
$.ajaxFileUpload({
url: '/opml/opml_upload',
secureuri: false,
@ -870,9 +884,9 @@
{
if (typeof data.code != 'undefined') {
if (data.code <= 0) {
NEWSBLUR.log(['Success - Error', data.code]);
// NEWSBLUR.log(['Success - Error', data.code]);
} else {
NEWSBLUR.log(['Success', data]);
// NEWSBLUR.log(['Success', data]);
self.load_feeds();
}
}
@ -890,10 +904,10 @@
var self = this;
var $form = $('form.opml_import_form');
NEWSBLUR.log(['OPML Form:', $form]);
// NEWSBLUR.log(['OPML Form:', $form]);
var callback = function(e) {
NEWSBLUR.log(['OPML Callback', e]);
// NEWSBLUR.log(['OPML Callback', e]);
};
$form.submit(function() {
@ -921,6 +935,15 @@
self.mark_feed_as_read(feed_id, $t);
});
// ============
// = Feed Bar =
// ============
$.targetIs(e, { tagSelector: '.NB-button-intelligence' }, function($t, $p){
e.preventDefault();
self.open_feed_intelligence_modal();
});
// ===========
// = Stories =
// ===========
@ -993,13 +1016,13 @@
$.targetIs(e, { tagSelector: '#story_titles .story' }, function($t, $p){
e.preventDefault();
NEWSBLUR.log(['Story dblclick', $t]);
// NEWSBLUR.log(['Story dblclick', $t]);
var story_id = $('.story_id', $t).text();
self.open_story_link(story_id, $t);
});
$.targetIs(e, { tagSelector: '#feed_list .feed' }, function($t, $p){
e.preventDefault();
NEWSBLUR.log(['Feed dblclick', $('.feed_id', $t), $t]);
// NEWSBLUR.log(['Feed dblclick', $('.feed_id', $t), $t]);
var feed_id = $t.data('feed_id');
self.open_feed_link(feed_id, $t);
});

View file

@ -1,4 +1,16 @@
NEWSBLUR.ReaderClassifier = function(story_id, feed_id, score, options) {
NEWSBLUR.ReaderClassifierFeed = function(feed_id, score, options) {
var defaults = {};
this.feed_id = feed_id;
this.score = score;
this.options = $.extend({}, defaults, options);
this.model = NEWSBLUR.AssetModel.reader();
this.google_favicon_url = 'http://www.google.com/s2/favicons?domain_url=';
this.runner_feed();
};
NEWSBLUR.ReaderClassifierStory = function(story_id, feed_id, score, options) {
var defaults = {};
this.story_id = story_id;
@ -7,14 +19,24 @@ NEWSBLUR.ReaderClassifier = function(story_id, feed_id, score, options) {
this.options = $.extend({}, defaults, options);
this.model = NEWSBLUR.AssetModel.reader();
this.google_favicon_url = 'http://www.google.com/s2/favicons?domain_url=';
this.runner();
this.runner_story();
};
NEWSBLUR.ReaderClassifier.prototype = {
NEWSBLUR.ReaderClassifierStory.prototype = {
runner: function() {
runner_feed: function() {
this.find_story_and_feed();
this.make_modal();
this.make_modal_feed();
this.handle_text_highlight();
this.handle_select_checkboxes();
this.handle_cancel();
this.handle_select_title();
this.open_modal();
},
runner_story: function() {
this.find_story_and_feed();
this.make_modal_story();
this.handle_text_highlight();
this.handle_select_checkboxes();
this.handle_cancel();
@ -23,17 +45,113 @@ NEWSBLUR.ReaderClassifier.prototype = {
},
find_story_and_feed: function() {
this.story = this.model.get_story(this.story_id);
if (this.story_id) {
this.story = this.model.get_story(this.story_id);
}
this.feed = this.model.get_feed(this.feed_id);
this.feed_tags = this.model.get_feed_tags();
this.feed_authors = this.model.get_feed_authors();
},
make_modal: function() {
make_modal_feed: function() {
var self = this;
var feed = this.feed;
var $feed_authors = [];
var $feed_tags = [];
if (this.feed_authors) {
for (var fa in this.feed_authors) {
var feed_author = this.feed_authors[fa];
var $author = $.make('span', { className: 'NB-classifier-author' }, [
$.make('input', { type: 'checkbox', name: 'author', value: feed_author[0], id: 'classifier_author_'+fa }),
$.make('label', { 'for': 'classifier_author_'+fa }, feed_author[0])
]);
$feed_authors.push($author);
}
}
if (this.feed_tags) {
for (var t in this.feed_tags) {
var tag = this.feed_tags[t];
var $tag = $.make('span', { className: 'NB-classifier-tag-container' }, [
$.make('span', { className: 'NB-classifier-tag' }, [
$.make('input', { type: 'checkbox', name: 'tag', value: tag[0], id: 'classifier_tag_'+t }),
$.make('label', { 'for': 'classifier_tag_'+t }, [
$.make('b', tag[0])
])
]),
$.make('span', { className: 'NB-classifier-tag-count' }, [
'&times;&nbsp;',
tag[1]
])
]);
$feed_tags.push($tag);
}
}
this.$classifier = $.make('div', { className: 'NB-classifier' }, [
$.make('h2', { className: 'NB-classifier-modal-title' }),
$.make('form', { method: 'post', className: 'NB-publisher' }, [
($feed_authors.length && $.make('div', { className: 'NB-classifier-field' }, [
$.make('h5', 'Authors'),
$.make('div', { className: 'NB-classifier-authors' }, $feed_authors)
])),
($feed_tags.length && $.make('div', { className: 'NB-classifier-field' }, [
$.make('h5', 'Categories &amp; Tags'),
$.make('div', { className: 'NB-classifier-tags' }, $feed_tags)
])),
$.make('div', { className: 'NB-classifier-field' }, [
$.make('h5', 'Everything by This Publisher'),
$.make('input', { type: 'checkbox', name: 'facet', value: 'publisher', id: 'classifier_publisher' }),
$.make('label', { 'for': 'classifier_publisher' }, [
$.make('img', { className: 'feed_favicon', src: this.google_favicon_url + feed.feed_link }),
$.make('span', { className: 'feed_title' }, feed.feed_title)
])
]),
$.make('div', { className: 'NB-classifier-submit' }, [
$.make('input', { name: 'score', value: this.score, type: 'hidden' }),
$.make('input', { name: 'feed_id', value: this.feed_id, type: 'hidden' }),
$.make('input', { name: 'story_id', value: this.story_id, type: 'hidden' }),
$.make('input', { type: 'submit', disabled: 'true', className: 'NB-disabled', value: 'Check what you like above...' }),
' or ',
$.make('a', { href: '#', className: 'NB-classifier-cancel' }, 'cancel')
])
]).bind('submit', function(e) {
e.preventDefault();
self.save_publisher();
return false;
})
]);
var $modal_title = $('.NB-classifier-modal-title', this.$classifier);
if (this.score == 1) {
$modal_title.html('What do you <b class="NB-classifier-like">like</b> about this publisher?');
} else if (this.score == -1) {
$modal_title.html('What do you <b class="NB-classifier-dislike">dislike</b> about publisher?');
}
},
make_modal_story: function() {
var self = this;
var story = this.story;
var feed = this.feed;
var $story_tags = [];
// HTML entities decoding.
story.story_title = $('<div/>').html(story.story_title).text();
for (var t in story.story_tags) {
var tag = story.story_tags[t];
var $tag = $.make('span', { className: 'NB-classifier-tag-container' }, [
$.make('span', { className: 'NB-classifier-tag' }, [
$.make('input', { type: 'checkbox', name: 'tag', value: tag, id: 'classifier_tag_'+t }),
$.make('label', { 'for': 'classifier_tag_'+t }, [
$.make('b', tag)
])
])
]);
$story_tags.push($tag);
}
this.$classifier = $.make('div', { className: 'NB-classifier' }, [
$.make('h2', { className: 'NB-classifier-modal-title' }),
@ -57,9 +175,9 @@ NEWSBLUR.ReaderClassifier.prototype = {
$.make('b', story.story_authors)
])
])),
(story.story_tags.length && $.make('div', { className: 'NB-classifier-field' }, [
($story_tags.length && $.make('div', { className: 'NB-classifier-field' }, [
$.make('h5', 'Story Categories &amp; Tags'),
$.make('div', { className: 'NB-classifier-tags' })
$.make('div', { className: 'NB-classifier-tags' }, $story_tags)
])),
$.make('div', { className: 'NB-classifier-field' }, [
$.make('h5', 'Everything by This Publisher'),
@ -79,22 +197,11 @@ NEWSBLUR.ReaderClassifier.prototype = {
])
]).bind('submit', function(e) {
e.preventDefault();
self.save();
self.save_story();
return false;
})
]);
for (var t in story.story_tags) {
var tag = story.story_tags[t];
var $tag = $.make('span', { className: 'NB-classifier-tag' }, [
$.make('input', { type: 'checkbox', name: 'tag', value: tag, id: 'classifier_tag_'+t }),
$.make('label', { 'for': 'classifier_tag_'+t }, [
$.make('b', tag)
])
]);
$('.NB-classifier-tags', this.$classifier).append($tag);
}
var $modal_title = $('.NB-classifier-modal-title', this.$classifier);
if (this.score == 1) {
$modal_title.html('What do you <b class="NB-classifier-like">like</b> about this story?');
@ -108,7 +215,6 @@ NEWSBLUR.ReaderClassifier.prototype = {
var $holder = $.make('div', { className: 'NB-classifier-holder' }).append(this.$classifier).appendTo('body').css({'visibility': 'hidden', 'display': 'block', 'width': 600});
var height = $('.NB-classifier', $holder).outerHeight(true);
NEWSBLUR.log(['Classifier height', height]);
$holder.css({'visibility': 'visible', 'display': 'none'});
this.$classifier.modal({
@ -126,7 +232,6 @@ NEWSBLUR.ReaderClassifier.prototype = {
$('.NB-classifier-tag', self.$classifier).corners('4px');
},
'onClose': function(dialog) {
NEWSBLUR.log(['Dialog Close', dialog]);
dialog.data.hide().empty().remove();
dialog.container.hide().empty().remove();
dialog.overlay.fadeOut(200, function() {
@ -202,15 +307,29 @@ NEWSBLUR.ReaderClassifier.prototype = {
return data;
},
save: function() {
save_publisher: function() {
var $save = $('.NB-classifier input[type=submit]');
var story_id = this.story_id;
var data = this.serialize_classifier();
$save.text('Saving...').addClass('NB-disabled').attr('disabled', true);
this.model.save_classifier(story_id, data, function() {
this.model.save_classifier_publisher(data, function() {
$.modal.close();
});
},
save_story: function() {
var $save = $('.NB-classifier input[type=submit]');
var story_id = this.story_id;
var data = this.serialize_classifier();
$save.text('Saving...').addClass('NB-disabled').attr('disabled', true);
this.model.save_classifier_story(story_id, data, function() {
$.modal.close();
});
}
};
};
NEWSBLUR.ReaderClassifierFeed.prototype = NEWSBLUR.ReaderClassifierStory.prototype;

View file

@ -185,9 +185,7 @@ class ProcessFeed:
story_feed=self.feed,
story_date__gte=start_date,
story_date__lte=end_date,
).order_by('-story_date').values()
if len(existing_stories) > 100:
existing_stories = existing_stories[:10] + existing_stories[-100:]
).order_by('-story_date')[:100].values()
ret_values = self.feed.add_update_stories(self.fpf.entries, existing_stories)
return FEED_OK, ret_values