diff --git a/apps/feed_import/management/commands/fetch_starred_stories.py b/apps/feed_import/management/commands/fetch_starred_stories.py index 118aa7ab5..7d64ba184 100644 --- a/apps/feed_import/management/commands/fetch_starred_stories.py +++ b/apps/feed_import/management/commands/fetch_starred_stories.py @@ -1,7 +1,6 @@ from django.core.management.base import BaseCommand from django.conf import settings from django.contrib.auth.models import User -from apps.feed_import.models import GoogleReaderImporter from optparse import make_option from utils.management_functions import daemonize diff --git a/apps/feed_import/models.py b/apps/feed_import/models.py index 38da7e093..eb7a8c70f 100644 --- a/apps/feed_import/models.py +++ b/apps/feed_import/models.py @@ -1,6 +1,5 @@ import datetime import mongoengine as mongo -import httplib2 import pickle import base64 from StringIO import StringIO @@ -235,197 +234,4 @@ class UploadedOPML(mongo.Document): 'order': '-upload_date', 'indexes': ['user_id', '-upload_date'], } - - -class GoogleReaderImporter(Importer): - - def __init__(self, user, xml=None): - self.user = user - self.scope = "http://www.google.com/reader/api" - self.xml = xml - self.auto_active = False - - @timelimit(10) - def try_import_feeds(self, auto_active=False): - code = 0 - try: - self.import_feeds(auto_active=auto_active) - self.import_starred_items(count=10) - except AssertionError: - code = -1 - else: - code = 1 - return code - - def import_feeds(self, auto_active=False): - self.auto_active = auto_active - sub_url = "%s/0/subscription/list" % self.scope - if not self.xml: - feeds_xml = self.send_request(sub_url) - else: - feeds_xml = self.xml - if feeds_xml: - self.process_feeds(feeds_xml) - - def send_request(self, url): - if not self.user.is_authenticated(): - return - - user_tokens = OAuthToken.objects.filter(user=self.user) - - if user_tokens.count(): - user_token = user_tokens[0] - if user_token.credential: - credential = pickle.loads(base64.b64decode(user_token.credential)) - http = httplib2.Http() - http = credential.authorize(http) - content = http.request(url) - return content and content[1] - - def process_feeds(self, feeds_xml): - # self.clear_feeds() - # self.clear_folders() - folders = self.get_folders() - self.feeds = self.parse(feeds_xml) - - for item in self.feeds: - folders = self.process_item(item, folders) - - logging.user(self.user, "~BB~FW~SBGoogle Reader import: ~BT~FW%s" % (folders)) - - self.usf.folders = json.encode(folders) - self.usf.save() - - def parse(self, feeds_xml): - parser = etree.XMLParser(recover=True) - tree = etree.parse(StringIO(feeds_xml), parser) - feeds = tree.xpath('/object/list/object') - return feeds - - def process_item(self, item, folders): - feed_title = item.xpath('./string[@name="title"]') and \ - item.xpath('./string[@name="title"]')[0].text - feed_address = item.xpath('./string[@name="id"]') and \ - item.xpath('./string[@name="id"]')[0].text.replace('feed/', '') - feed_link = item.xpath('./string[@name="htmlUrl"]') and \ - item.xpath('./string[@name="htmlUrl"]')[0].text - category = item.xpath('./list[@name="categories"]/object/string[@name="label"]') and \ - item.xpath('./list[@name="categories"]/object/string[@name="label"]')[0].text - - if not feed_address: - feed_address = feed_link - - try: - feed_link = urlnorm.normalize(feed_link) - feed_address = urlnorm.normalize(feed_address) - - if len(feed_address) > Feed._meta.get_field('feed_address').max_length: - return folders - - # See if it exists as a duplicate first - duplicate_feed = DuplicateFeed.objects.filter(duplicate_address=feed_address) - if duplicate_feed: - feed_db = duplicate_feed[0].feed - else: - feed_data = dict(feed_title=feed_title) - feed_data['active_subscribers'] = 1 - feed_data['num_subscribers'] = 1 - feed_db, _ = Feed.find_or_create(feed_address=feed_address, feed_link=feed_link, - defaults=dict(**feed_data)) - - us, _ = UserSubscription.objects.get_or_create( - feed=feed_db, - user=self.user, - defaults={ - 'needs_unread_recalc': True, - 'mark_read_date': datetime.datetime.utcnow() - datetime.timedelta(days=1), - 'active': self.user.profile.is_premium or self.auto_active, - } - ) - if not us.needs_unread_recalc: - us.needs_unread_recalc = True - us.save() - if not category: category = "" - - if category: - obj = {category: []} - folders = add_object_to_folder(obj, '', folders) - folders = add_object_to_folder(feed_db.pk, category, folders) - # if feed_db.pk not in folders[category]: - # folders[category].append(feed_db.pk) - except Exception, e: - logging.info(' *** -> Exception: %s: %s' % (e, item)) - - return folders - - def test(self): - sub_url = "%s/0/token" % (self.scope) - try: - resp = self.send_request(sub_url) - except OAuthError: - return False - return resp - - @timelimit(10) - def try_import_starred_stories(self): - self.import_starred_items(count=1000) - - starred_count = MStarredStory.objects.filter(user_id=self.user.pk).count() - return starred_count - - def import_starred_items(self, count=10): - sub_url = "%s/0/stream/contents/user/-/state/com.google/starred?n=%s" % (self.scope, count) - stories_str = self.send_request(sub_url) - try: - stories = json.decode(stories_str) - except: - logging.user(self.user, "~BB~FW~SBGoogle Reader starred stories: ~BT~FWNo stories") - stories = None - if stories: - logging.user(self.user, "~BB~FW~SBGoogle Reader starred stories: ~BT~FW%s stories" % (len(stories['items']))) - self.process_starred_items(stories['items']) - - starred_count = MStarredStory.objects.filter(user_id=self.user.pk).count() - return starred_count - - def process_starred_items(self, stories): - counts = { - 'created': 0, - 'existed': 0, - 'failed': 0, - } - logging.user(self.user, "~FCBeginning starring...") - - for story in stories: - try: - original_feed = Feed.get_feed_from_url(story['origin']['streamId'], create=False, fetch=False) - if not original_feed: - original_feed = Feed.get_feed_from_url(story['origin']['htmlUrl'], create=False, fetch=False) - content = story.get('content') or story.get('summary') - story_db = { - "user_id": self.user.pk, - "starred_date": datetime.datetime.fromtimestamp(story['updated']), - "story_date": datetime.datetime.fromtimestamp(story['published']), - "story_title": story.get('title', story.get('origin', {}).get('title', '[Untitled]')), - "story_permalink": story['alternate'][0]['href'], - "story_guid": story['id'], - "story_content": content.get('content'), - "story_author_name": story.get('author'), - "story_feed_id": original_feed and original_feed.pk, - "story_tags": [tag for tag in story.get('categories', []) if 'user/' not in tag] - } - # logging.user(self.user, "~FCStarring: ~SB%s~SN in ~SB%s" % (story_db['story_title'][:50], original_feed and original_feed)) - MStarredStory.objects.create(**story_db) - counts['created'] += 1 - except OperationError: - # logging.user(self.user, "~FCAlready starred: ~SB%s" % (story_db['story_title'][:50])) - counts['existed'] += 1 - except Exception: - # logging.user(self.user, "~FC~BRFailed to star: ~SB%s / %s" % (story, e)) - counts['failed'] += 1 - - logging.user(self.user, "~FCStarred: ~SB%s~SN/~SB%s%s~SN/~SB%s%s~SN" % ( - counts['created'], - '~FM' if counts['existed'] else '~SN', counts['existed'], - '~FR' if counts['failed'] else '~SN', counts['failed'])) - return counts + \ No newline at end of file diff --git a/apps/feed_import/tasks.py b/apps/feed_import/tasks.py index ee6001343..6b831ebf4 100644 --- a/apps/feed_import/tasks.py +++ b/apps/feed_import/tasks.py @@ -1,6 +1,6 @@ from celery.task import Task from django.contrib.auth.models import User -from apps.feed_import.models import UploadedOPML, OPMLImporter, GoogleReaderImporter +from apps.feed_import.models import UploadedOPML, OPMLImporter from apps.reader.models import UserSubscription from apps.rss_feeds.models import MStarredStory from utils import log as logging @@ -20,33 +20,3 @@ class ProcessOPML(Task): user.profile.send_upload_opml_finished_email(feed_count) logging.user(user, "~FR~SBOPML upload (task): ~SK%s~SN~SB~FR feeds" % (feed_count)) - -class ProcessReaderImport(Task): - - def run(self, user_id, auto_active=False): - user = User.objects.get(pk=user_id) - logging.user(user, "~FR~SBGoogle Reader import (task) starting...") - - importer = GoogleReaderImporter(user=user) - importer.import_feeds(auto_active=auto_active) - importer.import_starred_items(count=10) - - feed_count = UserSubscription.objects.filter(user=user).count() - user.profile.send_import_reader_finished_email(feed_count) - logging.user(user, "~FR~SBGoogle Reader import (task): ~SK%s~SN~SB~FR feeds" % (feed_count)) - - -class ProcessReaderStarredImport(Task): - - def run(self, user_id): - user = User.objects.get(pk=user_id) - logging.user(user, "~FR~SBGoogle Reader starred stories import (task) starting...") - - importer = GoogleReaderImporter(user=user) - importer.import_starred_items(count=1000) - - feed_count = UserSubscription.objects.filter(user=user).count() - starred_count = MStarredStory.objects.filter(user_id=user.pk).count() - user.profile.send_import_reader_starred_finished_email(feed_count, starred_count) - logging.user(user, "~FR~SBGoogle Reader starred stories import (task): ~SK%s~SN~SB~FR feeds, ~SK%s~SN~SB~FR starred stories" % (feed_count, starred_count)) - diff --git a/apps/feed_import/tests.py b/apps/feed_import/tests.py index af5ac283c..0c4ac24a6 100644 --- a/apps/feed_import/tests.py +++ b/apps/feed_import/tests.py @@ -4,7 +4,6 @@ from django.test import TestCase from django.contrib.auth.models import User from django.core.urlresolvers import reverse from apps.reader.models import UserSubscription, UserSubscriptionFolders -from apps.feed_import.models import GoogleReaderImporter from utils import json_functions as json class ImportTest(TestCase): diff --git a/apps/feed_import/views.py b/apps/feed_import/views.py index 0a4608799..8200189a6 100644 --- a/apps/feed_import/views.py +++ b/apps/feed_import/views.py @@ -1,7 +1,6 @@ import datetime import pickle import base64 -import httplib2 from utils import log as logging from oauth2client.client import OAuth2WebServerFlow, FlowExchangeError from bson.errors import InvalidStringData @@ -17,9 +16,9 @@ from django.contrib.auth import login as login_user from django.shortcuts import render_to_response from apps.reader.forms import SignupForm from apps.reader.models import UserSubscription -from apps.feed_import.models import OAuthToken, GoogleReaderImporter +from apps.feed_import.models import OAuthToken from apps.feed_import.models import OPMLImporter, OPMLExporter, UploadedOPML -from apps.feed_import.tasks import ProcessOPML, ProcessReaderImport, ProcessReaderStarredImport +from apps.feed_import.tasks import ProcessOPML from utils import json_functions as json from utils.user_functions import ajax_login_required, get_user from utils.feed_functions import TimeoutError diff --git a/apps/profile/models.py b/apps/profile/models.py index 7415fa29d..b5e728cd2 100644 --- a/apps/profile/models.py +++ b/apps/profile/models.py @@ -20,7 +20,7 @@ from django.core.urlresolvers import reverse from django.template.loader import render_to_string from apps.rss_feeds.models import Feed, MStory, MStarredStory from apps.rss_feeds.tasks import SchedulePremiumSetup -from apps.feed_import.models import GoogleReaderImporter, OPMLExporter +from apps.feed_import.models import OPMLExporter from apps.reader.models import UserSubscription from apps.reader.models import RUserStory from utils import log as logging @@ -651,10 +651,6 @@ class Profile(models.Model): pipeline.zrem(premium_key, user.pk) pipeline.execute() - def import_reader_starred_items(self, count=20): - importer = GoogleReaderImporter(self.user) - importer.import_starred_items(count=count) - def send_new_user_email(self): if not self.user.email or not self.send_emails: return diff --git a/config/requirements.txt b/config/requirements.txt index 510271535..805ece47d 100755 --- a/config/requirements.txt +++ b/config/requirements.txt @@ -23,7 +23,7 @@ dnspython==1.15.0 Fabric==1.14.0 gunicorn==19.7 hiredis==0.2.0 -httplib2==0.9.2 +httplib2==0.18.0 image==1.5.27 isodate==0.5.4 lxml==3.6.4 @@ -31,7 +31,7 @@ mock==2.0.0 mongoengine==0.10.7 PyMySQL==0.9.3 ndg-httpsclient==0.4.2 -nltk==3.2.1 +nltk==3.4.5 numpy==1.11.2 oauth2==1.9.0.post1 pillow==3.4.2 diff --git a/fabfile.py b/fabfile.py index 12ca8b679..c62e709a6 100644 --- a/fabfile.py +++ b/fabfile.py @@ -468,7 +468,7 @@ def setup_virtualenv(): # sudo('rmvirtualenv newsblur') # sudo('rm -fr venv') with settings(warn_only=True): - run('mkvirtualenv --no-site-packages newsblur') + run('mkvirtualenv newsblur') run('echo "import sys; sys.setdefaultencoding(\'utf-8\')" | sudo tee venv/newsblur/lib/python2.7/sitecustomize.py') run('echo "/srv/newsblur" | sudo tee venv/newsblur/lib/python2.7/site-packages/newsblur.pth') @@ -761,8 +761,8 @@ def setup_staging(): run('touch logs/newsblur.log') def setup_node_app(): - sudo('curl -sL https://deb.nodesource.com/setup_10.x | sudo -E bash -') - sudo('apt-get install -y nodejs') + sudo('curl -sL https://deb.nodesource.com/setup_14.x | sudo bash -') + sudo('apt-get install -y nodejs npm') # run('curl -L https://npmjs.org/install.sh | sudo sh') # sudo('apt-get install npm') sudo('sudo npm install -g npm')