public-mirrors/NewsBlur
1
0
Fork 0
mirror of https://github.com/samuelclay/NewsBlur.git synced 2025-09-18 21:50:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Refactoring how feeds are found using urls in the add modal. Also fixing a display issue around showing the river icon in a collapsed feed and showing the manage menu. Lastly, changing the feedfinder's user agent because some assclown website was disallowing on the string 'feedfinder'

Browse source
This commit is contained in:
Samuel Clay 2011-02-08 22:07:59 -05:00
parent 5b982d61b5
commit 48afe91758
7 changed files with 79 additions and 60 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
apps/reader/models.py
View file

@ -10,8 +10,6 @@ from apps.reader.managers import UserSubscriptionManager
from apps.rss_feeds.models import Feed, MStory, DuplicateFeed
from apps.analyzer.models import MClassifierFeed, MClassifierAuthor, MClassifierTag, MClassifierTitle
from apps.analyzer.models import apply_classifier_titles, apply_classifier_feeds, apply_classifier_authors, apply_classifier_tags
from utils import urlnorm
from utils.feed_functions import fetch_address_from_page
from utils.feed_functions import add_object_to_folder
from utils.feed_functions import relative_timesince
@ -91,31 +89,14 @@ class UserSubscription(models.Model):
logging.info(" ---> [%s] ~FRAdding URL: ~SB%s (in %s)" % (user, feed_address, folder))
if feed_address:
feed_address = urlnorm.normalize(feed_address)
# See if it exists as a duplicate first
duplicate_feed = DuplicateFeed.objects.filter(duplicate_address=feed_address).order_by('pk')
if duplicate_feed:
feed = [duplicate_feed[0].feed]
else:
feed = Feed.objects.filter(feed_address=feed_address).order_by('pk')
if feed:
feed = feed[0]
else:
try:
feed = fetch_address_from_page(feed_address)
except:
code = -2
message = "This feed has been added, but something went wrong"\
" when downloading it. Maybe the server's busy."
feed = Feed.get_feed_from_url(feed_address)
if not feed:
code = -1
if bookmarklet:
message = "This site does not have an RSS feed. Nothing is linked to from this page."
else:
message = "This site does not point to an RSS feed or a website with an RSS feed."
message = "This address does not point to an RSS feed or a website with an RSS feed."
else:
us, subscription_created = cls.objects.get_or_create(
feed=feed,

37
apps/rss_feeds/models.py
View file

@ -18,12 +18,13 @@ from apps.rss_feeds.tasks import UpdateFeeds
from celery.task import Task
from utils import json_functions as json
from utils import feedfinder
from utils import urlnorm
from utils import log as logging
from utils.fields import AutoOneToOneField
from utils.feed_functions import levenshtein_distance
from utils.feed_functions import timelimit, TimeoutError
from utils.story_functions import pre_process_story
from utils.diff import HTMLDiff
from utils import log as logging
ENTRY_NEW, ENTRY_UPDATED, ENTRY_SAME, ENTRY_ERR = range(4)
@ -78,6 +79,40 @@ class Feed(models.Model):
# Feed has been deleted. Just ignore it.
pass
@classmethod
def get_feed_from_url(cls, url):
feed = None
def by_url(address):
duplicate_feed = DuplicateFeed.objects.filter(duplicate_address=address).order_by('pk')
if duplicate_feed:
feed = [duplicate_feed[0].feed]
else:
feed = cls.objects.filter(feed_address=address).order_by('pk')
return feed
url = urlnorm.normalize(url)
feed = by_url(url)
if not feed:
if feedfinder.isFeed(url):
feed = cls.objects.create(feed_address=url)
feed.update()
feed = Feed.objects.get(pk=feed.pk)
else:
feed_finder_url = feedfinder.feed(url)
print "URL: %s %s" % (url, feed_finder_url)
if feed_finder_url:
feed = by_url(feed_finder_url)
if not feed:
feed = cls.objects.create(feed_address=feed_finder_url)
feed.update()
feed = Feed.objects.get(pk=feed.pk)
else:
feed = feed[0]
return feed
@classmethod
def task_feeds(cls, feeds, queue_size=12):
print " ---> Tasking %s feeds..." % feeds.count()

15
fabfile.py vendored
View file

@ -93,6 +93,21 @@ def backup_postgresql():
with cd('~/newsblur/utils/backups'):
run('./postgresql_backup.sh')
# =============
# = Bootstrap =
# =============
def setup():
env.user = 'root'
setup_user()
setup_installs()
def setup_user():
run('useradd conesus ')
def setup_installs():
run('apt-get -y install sysstat git')
# ======
# = S3 =
# ======

6
media/css/reader.css
View file

@ -588,12 +588,14 @@ body.NB-theme-serif #story_pane .NB-feed-story-content {
}
.NB-is-premium .NB-feedlist .folder .folder_title:hover .NB-feedlist-river-icon {
.NB-is-premium .NB-feedlist .folder .folder_title:hover .NB-feedlist-river-icon,
.NB-is-premium .NB-feedlist .folder.NB-showing-menu > .folder_title .NB-feedlist-river-icon {
display: block;
opacity: .6;
}
.NB-is-premium .NB-feedlist .folder .folder_title:hover .feed_counts_floater {
.NB-is-premium .NB-feedlist .folder .folder_title:hover .feed_counts_floater,
.NB-is-premium .NB-feedlist .folder.NB-showing-menu > .folder_title .feed_counts_floater {
margin-right: 24px;
}
.NB-feedlist .folder .folder_title.NB-feedlist-folder-title-recently-collapsed:hover .feed_counts_floater {

22
media/js/newsblur/reader.js
View file

@ -3467,14 +3467,16 @@
// If this menu is already open, then hide it instead.
if (($item && $item[0] == $manage_menu_container.data('item')) &&
parseInt($manage_menu_container.css('opacity'), 10) == 1) {
this.hide_manage_menu(type);
this.hide_manage_menu(type, $item);
return;
} else {
this.hide_manage_menu(type);
this.hide_manage_menu(type, $item);
}
if ($item.hasClass('NB-empty')) return;
$item.addClass('NB-showing-menu');
// Create menu, size and position it, then attach to the right place.
var feed_id, inverse, story_id;
if (type == 'folder') {
@ -3607,7 +3609,9 @@
this.flags['feed_list_showing_manage_menu'] = false;
$(document).unbind('click.menu');
$manage_menu_container.uncorner();
$item.removeClass('NB-showing-menu');
if (animate) {
$manage_menu_container.stop().animate({
'opacity': 0
@ -4065,8 +4069,12 @@
var self = this;
var refresh_interval = this.FEED_REFRESH_INTERVAL;
if (!NEWSBLUR.Globals.is_premium) {
refresh_interval *= 5;
}
if (new_feeds) {
refresh_interval = (1000 * 60) * 1/6;
refresh_interval = (1000 * 60) * 1/4;
}
clearInterval(this.flags.feed_refresh);
@ -4531,9 +4539,11 @@
if (!self.flags['sorting_feed']) {
stopPropagation = true;
if ($t.parent().hasClass('feed')) {
self.show_manage_menu('feed', $t.parents('.feed').eq(0));
self.show_manage_menu('feed', $t.closest('.feed'));
} else {
self.show_manage_menu('folder', $t.parents('.folder').eq(0));
var $folder = $t.closest('.folder');
self.show_manage_menu('folder', $folder);
$folder.addClass('NB-hover');
}
}
});

27
utils/feed_functions.py
View file

@ -5,7 +5,6 @@ import traceback
import pprint
from django.core.mail import mail_admins
from django.utils.translation import ungettext
from utils import feedfinder
from utils import log as logging
class TimeoutError(Exception): pass
@ -79,32 +78,6 @@ def levenshtein_distance(first, second):
distance_matrix[i][j] = min(insertion, deletion, substitution)
return distance_matrix[first_length-1][second_length-1]
def fetch_address_from_page(url, existing_feed=None):
from apps.rss_feeds.models import Feed, DuplicateFeed
feed_finder_url = feedfinder.feed(url)
if feed_finder_url:
if existing_feed:
if Feed.objects.filter(feed_address=feed_finder_url):
return None
existing_feed.feed_address = feed_finder_url
existing_feed.save()
feed = existing_feed
else:
duplicate_feed = DuplicateFeed.objects.filter(duplicate_address=feed_finder_url)
if duplicate_feed:
feed = [duplicate_feed[0].feed]
else:
feed = Feed.objects.filter(feed_address=feed_finder_url)
if not feed:
feed = Feed(feed_address=feed_finder_url)
feed.save()
feed.update()
feed = Feed.objects.get(pk=feed.pk)
else:
feed = feed[0]
return feed
def _do_timesince(d, chunks, now=None):
"""
Started as a copy of django.util.timesince.timesince, but modified to

9
utils/feedfinder.py
View file

@ -44,7 +44,7 @@ __license__ = "Python"
__credits__ = """Abe Fettig for a patch to sort Syndic8 feeds by popularity
Also Jason Diamond, Brian Lalor for bug reporting and patches"""
_debug = 0
_debug = 1
import sgmllib, urllib, urlparse, re, sys, robotparser
from StringIO import StringIO
@ -73,7 +73,7 @@ class URLGatekeeper:
def __init__(self):
self.rpcache = {} # a dictionary of RobotFileParser objects, by domain
self.urlopener = urllib.FancyURLopener()
self.urlopener.version = "feedfinder/" + __version__ + " " + self.urlopener.version + " +http://www.aaronsw.com/2002/feedfinder/"
self.urlopener.version = "NewsBlur Feed Finder"
_debuglog(self.urlopener.version)
self.urlopener.addheaders = [('User-agent', self.urlopener.version)]
robotparser.URLopener.version = self.urlopener.version
@ -211,7 +211,10 @@ def isFeed(uri):
protocol = urlparse.urlparse(uri)
if protocol[0] not in ('http', 'https'): return 0
data = _gatekeeper.get(uri)
return couldBeFeedData(data)
count = couldBeFeedData(data)
print count
print data
return count
def sortFeeds(feed1Info, feed2Info):
return cmp(feed2Info['headlines_rank'], feed1Info['headlines_rank'])