mirror of
https://github.com/samuelclay/NewsBlur.git
synced 2025-04-13 09:42:01 +00:00
5a3884c3bd
* django1.7: (102 commits) Beginning fix of rss_feeds unit tests. Fixing unit test for profile app and signup. Remove highlights count when it reaches zero. Android v10.1b1. Stubbing in profile tests. Adding nginx.local.conf Adding original text and original story to API docs. #1282 Adding feed to root folder #1319 In app and external browser options Adding a smarter wakeup for real-time to handle cases where a laptop is re-opened but real-time is not immediately reestablished. #1348 (scroll indicators theme) #1344 (search loses focus) #1335 Auto theme option for OS level dark mode Fixing signup flow. #1347 Show pager with stories after using the intel trainer and refreshing #1272 Load HTML in comments New icon for Infrequent Site Stories. Allowing selection in private notes. Autoresizing private notes field. For #1035: Adding private notes to saved stories. ...
233 lines
No EOL
9.1 KiB
Python
233 lines
No EOL
9.1 KiB
Python
import datetime
|
|
import mongoengine as mongo
|
|
import pickle
|
|
import base64
|
|
from StringIO import StringIO
|
|
from oauth2client.client import Error as OAuthError
|
|
from xml.etree.ElementTree import Element, SubElement, Comment, tostring
|
|
from lxml import etree
|
|
from django.db import models
|
|
from django.contrib.auth.models import User
|
|
from mongoengine.queryset import OperationError
|
|
import vendor.opml as opml
|
|
from apps.rss_feeds.models import Feed, DuplicateFeed
|
|
from apps.reader.models import UserSubscription, UserSubscriptionFolders
|
|
from utils import json_functions as json, urlnorm
|
|
from utils import log as logging
|
|
from utils.feed_functions import timelimit
|
|
from utils.feed_functions import add_object_to_folder
|
|
|
|
|
|
class OAuthToken(models.Model):
|
|
user = models.OneToOneField(User, null=True, blank=True)
|
|
session_id = models.CharField(max_length=50, null=True, blank=True)
|
|
uuid = models.CharField(max_length=50, null=True, blank=True)
|
|
remote_ip = models.CharField(max_length=50, null=True, blank=True)
|
|
request_token = models.CharField(max_length=50)
|
|
request_token_secret = models.CharField(max_length=50)
|
|
access_token = models.CharField(max_length=50)
|
|
access_token_secret = models.CharField(max_length=50)
|
|
credential = models.TextField(null=True, blank=True)
|
|
created_date = models.DateTimeField(default=datetime.datetime.now)
|
|
|
|
|
|
class Importer:
|
|
|
|
def clear_feeds(self):
|
|
UserSubscription.objects.filter(user=self.user).delete()
|
|
|
|
def clear_folders(self):
|
|
UserSubscriptionFolders.objects.filter(user=self.user).delete()
|
|
|
|
def get_folders(self):
|
|
self.usf, _ = UserSubscriptionFolders.objects.get_or_create(user=self.user,
|
|
defaults={'folders': '[]'})
|
|
return json.decode(self.usf.folders)
|
|
|
|
|
|
class OPMLExporter(Importer):
|
|
|
|
def __init__(self, user):
|
|
self.user = user
|
|
self.fetch_feeds()
|
|
|
|
def process(self, verbose=False):
|
|
now = str(datetime.datetime.now())
|
|
|
|
root = Element('opml')
|
|
root.set('version', '1.1')
|
|
root.append(Comment('Generated by NewsBlur - www.newsblur.com'))
|
|
|
|
head = SubElement(root, 'head')
|
|
title = SubElement(head, 'title')
|
|
title.text = 'NewsBlur Feeds'
|
|
dc = SubElement(head, 'dateCreated')
|
|
dc.text = now
|
|
dm = SubElement(head, 'dateModified')
|
|
dm.text = now
|
|
folders = self.get_folders()
|
|
body = SubElement(root, 'body')
|
|
self.process_outline(body, folders, verbose=verbose)
|
|
return tostring(root)
|
|
|
|
def process_outline(self, body, folders, verbose=False):
|
|
for obj in folders:
|
|
if isinstance(obj, int) and obj in self.feeds:
|
|
feed = self.feeds[obj]
|
|
if verbose:
|
|
print " ---> Adding feed: %s - %s" % (feed['id'],
|
|
feed['feed_title'][:30])
|
|
feed_attrs = self.make_feed_row(feed)
|
|
body.append(Element('outline', feed_attrs))
|
|
elif isinstance(obj, dict):
|
|
for folder_title, folder_objs in obj.items():
|
|
if verbose:
|
|
print " ---> Adding folder: %s" % folder_title
|
|
folder_element = Element('outline', {'text': folder_title, 'title': folder_title})
|
|
body.append(self.process_outline(folder_element, folder_objs, verbose=verbose))
|
|
return body
|
|
|
|
def make_feed_row(self, feed):
|
|
feed_attrs = {
|
|
'text': feed['feed_title'],
|
|
'title': feed['feed_title'],
|
|
'type': 'rss',
|
|
'version': 'RSS',
|
|
'htmlUrl': feed['feed_link'] or "",
|
|
'xmlUrl': feed['feed_address'] or "",
|
|
}
|
|
return feed_attrs
|
|
|
|
def fetch_feeds(self):
|
|
subs = UserSubscription.objects.filter(user=self.user)
|
|
self.feeds = []
|
|
for sub in subs:
|
|
try:
|
|
self.feeds.append((sub.feed_id, sub.canonical()))
|
|
except Feed.DoesNotExist:
|
|
continue
|
|
self.feeds = dict(self.feeds)
|
|
|
|
|
|
class OPMLImporter(Importer):
|
|
|
|
def __init__(self, opml_xml, user):
|
|
self.user = user
|
|
self.opml_xml = opml_xml
|
|
|
|
@timelimit(10)
|
|
def try_processing(self):
|
|
folders = self.process()
|
|
return folders
|
|
|
|
def process(self):
|
|
# self.clear_feeds()
|
|
|
|
outline = opml.from_string(self.opml_xml)
|
|
folders = self.get_folders()
|
|
try:
|
|
folders = self.process_outline(outline, folders)
|
|
except AttributeError:
|
|
folders = None
|
|
else:
|
|
# self.clear_folders()
|
|
self.usf.folders = json.encode(folders)
|
|
self.usf.save()
|
|
|
|
return folders
|
|
|
|
def process_outline(self, outline, folders, in_folder=''):
|
|
for item in outline:
|
|
if (not hasattr(item, 'xmlUrl') and
|
|
(hasattr(item, 'text') or hasattr(item, 'title'))):
|
|
folder = item
|
|
title = getattr(item, 'text', None) or getattr(item, 'title', None)
|
|
# if hasattr(folder, 'text'):
|
|
# logging.info(' ---> [%s] ~FRNew Folder: %s' % (self.user, folder.text))
|
|
obj = {title: []}
|
|
folders = add_object_to_folder(obj, in_folder, folders)
|
|
folders = self.process_outline(folder, folders, title)
|
|
elif hasattr(item, 'xmlUrl'):
|
|
feed = item
|
|
if not hasattr(feed, 'htmlUrl'):
|
|
setattr(feed, 'htmlUrl', None)
|
|
# If feed title matches what's in the DB, don't override it on subscription.
|
|
feed_title = getattr(feed, 'title', None) or getattr(feed, 'text', None)
|
|
if not feed_title:
|
|
setattr(feed, 'title', feed.htmlUrl or feed.xmlUrl)
|
|
user_feed_title = None
|
|
else:
|
|
setattr(feed, 'title', feed_title)
|
|
user_feed_title = feed.title
|
|
|
|
feed_address = urlnorm.normalize(feed.xmlUrl)
|
|
feed_link = urlnorm.normalize(feed.htmlUrl)
|
|
if len(feed_address) > Feed._meta.get_field('feed_address').max_length:
|
|
continue
|
|
if feed_link and len(feed_link) > Feed._meta.get_field('feed_link').max_length:
|
|
continue
|
|
# logging.info(' ---> \t~FR%s - %s - %s' % (feed.title, feed_link, feed_address,))
|
|
feed_data = dict(feed_address=feed_address, feed_link=feed_link, feed_title=feed.title)
|
|
# feeds.append(feed_data)
|
|
|
|
# See if it exists as a duplicate first
|
|
duplicate_feed = DuplicateFeed.objects.filter(duplicate_address=feed_address)
|
|
if duplicate_feed:
|
|
feed_db = duplicate_feed[0].feed
|
|
else:
|
|
feed_data['active_subscribers'] = 1
|
|
feed_data['num_subscribers'] = 1
|
|
feed_db, _ = Feed.find_or_create(feed_address=feed_address,
|
|
feed_link=feed_link,
|
|
defaults=dict(**feed_data))
|
|
|
|
if user_feed_title == feed_db.feed_title:
|
|
user_feed_title = None
|
|
|
|
try:
|
|
us = UserSubscription.objects.get(
|
|
feed=feed_db,
|
|
user=self.user)
|
|
except UserSubscription.DoesNotExist:
|
|
us = UserSubscription(
|
|
feed=feed_db,
|
|
user=self.user,
|
|
needs_unread_recalc=True,
|
|
mark_read_date=datetime.datetime.utcnow() - datetime.timedelta(days=1),
|
|
active=self.user.profile.is_premium,
|
|
user_title=user_feed_title)
|
|
us.save()
|
|
|
|
if self.user.profile.is_premium and not us.active:
|
|
us.active = True
|
|
us.save()
|
|
if not us.needs_unread_recalc:
|
|
us.needs_unread_recalc = True
|
|
us.save()
|
|
|
|
folders = add_object_to_folder(feed_db.pk, in_folder, folders)
|
|
|
|
return folders
|
|
|
|
def count_feeds_in_opml(self):
|
|
opml_count = len(opml.from_string(self.opml_xml))
|
|
sub_count = UserSubscription.objects.filter(user=self.user).count()
|
|
return max(sub_count, opml_count)
|
|
|
|
|
|
class UploadedOPML(mongo.Document):
|
|
user_id = mongo.IntField()
|
|
opml_file = mongo.StringField()
|
|
upload_date = mongo.DateTimeField(default=datetime.datetime.now)
|
|
|
|
def __unicode__(self):
|
|
user = User.objects.get(pk=self.user_id)
|
|
return "%s: %s characters" % (user.username, len(self.opml_file))
|
|
|
|
meta = {
|
|
'collection': 'uploaded_opml',
|
|
'allow_inheritance': False,
|
|
'order': '-upload_date',
|
|
'indexes': ['user_id', '-upload_date'],
|
|
}
|
|
|