import base64 import datetime import pickle from xml.etree.ElementTree import Comment, Element, SubElement, tostring import mongoengine as mongo from django.contrib.auth.models import User from django.db import models from lxml import etree from mongoengine.queryset import OperationError from oauth2client.client import Error as OAuthError import vendor.opml as opml from apps.reader.models import UserSubscription, UserSubscriptionFolders from apps.rss_feeds.models import DuplicateFeed, Feed from utils import json_functions as json from utils import log as logging from utils import urlnorm from utils.feed_functions import add_object_to_folder, timelimit class OAuthToken(models.Model): user = models.OneToOneField(User, null=True, blank=True, on_delete=models.CASCADE) session_id = models.CharField(max_length=50, null=True, blank=True) uuid = models.CharField(max_length=50, null=True, blank=True) remote_ip = models.CharField(max_length=50, null=True, blank=True) request_token = models.CharField(max_length=50) request_token_secret = models.CharField(max_length=50) access_token = models.CharField(max_length=50) access_token_secret = models.CharField(max_length=50) credential = models.TextField(null=True, blank=True) created_date = models.DateTimeField(default=datetime.datetime.now) class Importer: def clear_feeds(self): UserSubscription.objects.filter(user=self.user).delete() def clear_folders(self): UserSubscriptionFolders.objects.filter(user=self.user).delete() def get_folders(self): self.usf, _ = UserSubscriptionFolders.objects.get_or_create( user=self.user, defaults={"folders": "[]"} ) return json.decode(self.usf.folders) class OPMLExporter(Importer): def __init__(self, user): self.user = user self.fetch_feeds() def process(self, verbose=False): now = str(datetime.datetime.now()) root = Element("opml") root.set("version", "1.1") root.append(Comment("Generated by NewsBlur - newsblur.com")) head = SubElement(root, "head") title = SubElement(head, "title") title.text = "NewsBlur Feeds" dc = SubElement(head, "dateCreated") dc.text = now dm = SubElement(head, "dateModified") dm.text = now folders = self.get_folders() body = SubElement(root, "body") self.process_outline(body, folders, verbose=verbose) return tostring(root, encoding="utf8", method="xml") def process_outline(self, body, folders, verbose=False): for obj in folders: if isinstance(obj, int) and obj in self.feeds: feed = self.feeds[obj] if verbose: print(" ---> Adding feed: %s - %s" % (feed["id"], feed["feed_title"][:30])) feed_attrs = self.make_feed_row(feed) body.append(Element("outline", feed_attrs)) elif isinstance(obj, dict): for folder_title, folder_objs in list(obj.items()): if verbose: print(" ---> Adding folder: %s" % folder_title) folder_element = Element("outline", {"text": folder_title, "title": folder_title}) body.append(self.process_outline(folder_element, folder_objs, verbose=verbose)) return body def make_feed_row(self, feed): feed_attrs = { "text": feed["feed_title"], "title": feed["feed_title"], "type": "rss", "version": "RSS", "htmlUrl": feed["feed_link"] or "", "xmlUrl": feed["feed_address"] or "", } return feed_attrs def fetch_feeds(self): subs = UserSubscription.objects.filter(user=self.user) self.feeds = [] for sub in subs: try: self.feeds.append((sub.feed_id, sub.canonical())) except Feed.DoesNotExist: continue self.feeds = dict(self.feeds) @property def feed_count(self): return len(self.feeds) class OPMLImporter(Importer): def __init__(self, opml_xml, user): self.user = user self.opml_xml = opml_xml @timelimit(10) def try_processing(self): folders = self.process() return folders def process(self): # self.clear_feeds() outline = opml.from_string(self.opml_xml) folders = self.get_folders() try: folders = self.process_outline(outline, folders) except AttributeError: folders = None else: # self.clear_folders() self.usf.folders = json.encode(folders) self.usf.save() return folders def process_outline(self, outline, folders, in_folder=""): for item in outline: if not hasattr(item, "xmlUrl") and (hasattr(item, "text") or hasattr(item, "title")): folder = item title = getattr(item, "text", None) or getattr(item, "title", None) # if hasattr(folder, 'text'): # logging.info(' ---> [%s] ~FRNew Folder: %s' % (self.user, folder.text)) obj = {title: []} folders = add_object_to_folder(obj, in_folder, folders) folders = self.process_outline(folder, folders, title) elif hasattr(item, "xmlUrl"): feed = item if not hasattr(feed, "htmlUrl"): setattr(feed, "htmlUrl", None) # If feed title matches what's in the DB, don't override it on subscription. feed_title = getattr(feed, "title", None) or getattr(feed, "text", None) if not feed_title: setattr(feed, "title", feed.htmlUrl or feed.xmlUrl) user_feed_title = None else: setattr(feed, "title", feed_title) user_feed_title = feed.title feed_address = urlnorm.normalize(feed.xmlUrl) feed_link = urlnorm.normalize(feed.htmlUrl) if len(feed_address) > Feed._meta.get_field("feed_address").max_length: continue if feed_link and len(feed_link) > Feed._meta.get_field("feed_link").max_length: continue # logging.info(' ---> \t~FR%s - %s - %s' % (feed.title, feed_link, feed_address,)) feed_data = dict(feed_address=feed_address, feed_link=feed_link, feed_title=feed.title) # feeds.append(feed_data) # See if it exists as a duplicate first duplicate_feed = DuplicateFeed.objects.filter(duplicate_address=feed_address) if duplicate_feed: feed_db = duplicate_feed[0].feed else: feed_data["active_subscribers"] = 1 feed_data["num_subscribers"] = 1 feed_db, _ = Feed.find_or_create( feed_address=feed_address, feed_link=feed_link, defaults=dict(**feed_data) ) if user_feed_title == feed_db.feed_title: user_feed_title = None try: us = UserSubscription.objects.get(feed=feed_db, user=self.user) except UserSubscription.DoesNotExist: us = None if not us: us = UserSubscription( feed=feed_db, user=self.user, needs_unread_recalc=True, mark_read_date=datetime.datetime.utcnow() - datetime.timedelta(days=1), active=self.user.profile.is_premium, user_title=user_feed_title, ) us.save() if self.user.profile.is_premium and not us.active: us.active = True us.save() if not us.needs_unread_recalc: us.needs_unread_recalc = True us.save() folders = add_object_to_folder(feed_db.pk, in_folder, folders) return folders def count_feeds_in_opml(self): opml_count = len(opml.from_string(self.opml_xml)) sub_count = UserSubscription.objects.filter(user=self.user).count() return max(sub_count, opml_count) class UploadedOPML(mongo.Document): user_id = mongo.IntField() opml_file = mongo.StringField() upload_date = mongo.DateTimeField(default=datetime.datetime.now) def __str__(self): user = User.objects.get(pk=self.user_id) return "%s: %s characters" % (user.username, len(self.opml_file)) meta = { "collection": "uploaded_opml", "allow_inheritance": False, "order": "-upload_date", "indexes": ["user_id", "-upload_date"], }