Removing additional traces of Google Reader.

apps/feed_import/management/commands/fetch_starred_stories.py

@@ -1,7 +1,6 @@
 from django.core.management.base import BaseCommand
 from django.conf import settings
 from django.contrib.auth.models import User
-from apps.feed_import.models import GoogleReaderImporter
 from optparse import make_option
 from utils.management_functions import daemonize
 

apps/feed_import/models.py

@@ -1,6 +1,5 @@
 import datetime
 import mongoengine as mongo
-import httplib2
 import pickle
 import base64
 from StringIO import StringIO
@@ -235,197 +234,4 @@ class UploadedOPML(mongo.Document):
         'order': '-upload_date',
         'indexes': ['user_id', '-upload_date'],
     }
-    
+    
-
-class GoogleReaderImporter(Importer):
-    
-    def __init__(self, user, xml=None):
-        self.user = user
-        self.scope = "http://www.google.com/reader/api"
-        self.xml = xml
-        self.auto_active = False
-    
-    @timelimit(10)
-    def try_import_feeds(self, auto_active=False):
-        code = 0
-        try:
-            self.import_feeds(auto_active=auto_active)
-            self.import_starred_items(count=10)
-        except AssertionError:
-            code = -1
-        else:
-            code = 1
-        return code
-
-    def import_feeds(self, auto_active=False):
-        self.auto_active = auto_active
-        sub_url = "%s/0/subscription/list" % self.scope
-        if not self.xml:
-            feeds_xml = self.send_request(sub_url)
-        else:
-            feeds_xml = self.xml
-        if feeds_xml:
-            self.process_feeds(feeds_xml)
-        
-    def send_request(self, url):
-        if not self.user.is_authenticated():
-            return
-            
-        user_tokens = OAuthToken.objects.filter(user=self.user)
-
-        if user_tokens.count():
-            user_token = user_tokens[0]
-            if user_token.credential:
-                credential = pickle.loads(base64.b64decode(user_token.credential))
-                http = httplib2.Http()
-                http = credential.authorize(http)
-                content = http.request(url)
-                return content and content[1]
-        
-    def process_feeds(self, feeds_xml):
-        # self.clear_feeds()
-        # self.clear_folders()
-        folders = self.get_folders()
-        self.feeds = self.parse(feeds_xml)
-
-        for item in self.feeds:
-            folders = self.process_item(item, folders)
-
-        logging.user(self.user, "~BB~FW~SBGoogle Reader import: ~BT~FW%s" % (folders))
-        
-        self.usf.folders = json.encode(folders)
-        self.usf.save()
-
-    def parse(self, feeds_xml):
-        parser = etree.XMLParser(recover=True)
-        tree = etree.parse(StringIO(feeds_xml), parser)
-        feeds = tree.xpath('/object/list/object')
-        return feeds
-    
-    def process_item(self, item, folders):
-        feed_title = item.xpath('./string[@name="title"]') and \
-                        item.xpath('./string[@name="title"]')[0].text
-        feed_address = item.xpath('./string[@name="id"]') and \
-                        item.xpath('./string[@name="id"]')[0].text.replace('feed/', '')
-        feed_link = item.xpath('./string[@name="htmlUrl"]') and \
-                        item.xpath('./string[@name="htmlUrl"]')[0].text
-        category = item.xpath('./list[@name="categories"]/object/string[@name="label"]') and \
-                        item.xpath('./list[@name="categories"]/object/string[@name="label"]')[0].text
-        
-        if not feed_address:
-            feed_address = feed_link
-        
-        try:
-            feed_link = urlnorm.normalize(feed_link)
-            feed_address = urlnorm.normalize(feed_address)
-
-            if len(feed_address) > Feed._meta.get_field('feed_address').max_length:
-                return folders
-
-            # See if it exists as a duplicate first
-            duplicate_feed = DuplicateFeed.objects.filter(duplicate_address=feed_address)
-            if duplicate_feed:
-                feed_db = duplicate_feed[0].feed
-            else:
-                feed_data = dict(feed_title=feed_title)
-                feed_data['active_subscribers'] = 1
-                feed_data['num_subscribers'] = 1
-                feed_db, _ = Feed.find_or_create(feed_address=feed_address, feed_link=feed_link,
-                                                 defaults=dict(**feed_data))
-
-            us, _ = UserSubscription.objects.get_or_create(
-                feed=feed_db, 
-                user=self.user,
-                defaults={
-                    'needs_unread_recalc': True,
-                    'mark_read_date': datetime.datetime.utcnow() - datetime.timedelta(days=1),
-                    'active': self.user.profile.is_premium or self.auto_active,
-                }
-            )
-            if not us.needs_unread_recalc:
-                us.needs_unread_recalc = True
-                us.save()
-            if not category: category = ""
-            
-            if category:
-                obj = {category: []}
-                folders = add_object_to_folder(obj, '', folders)
-            folders = add_object_to_folder(feed_db.pk, category, folders)
-            # if feed_db.pk not in folders[category]:
-            #     folders[category].append(feed_db.pk)
-        except Exception, e:
-            logging.info(' *** -> Exception: %s: %s' % (e, item))
-
-        return folders
-        
-    def test(self):
-        sub_url = "%s/0/token" % (self.scope)
-        try:
-            resp = self.send_request(sub_url)
-        except OAuthError:
-            return False
-        return resp
-    
-    @timelimit(10)
-    def try_import_starred_stories(self):
-        self.import_starred_items(count=1000)
-        
-        starred_count = MStarredStory.objects.filter(user_id=self.user.pk).count()
-        return starred_count
-
-    def import_starred_items(self, count=10):
-        sub_url = "%s/0/stream/contents/user/-/state/com.google/starred?n=%s" % (self.scope, count)
-        stories_str = self.send_request(sub_url)
-        try:
-            stories = json.decode(stories_str)
-        except:
-            logging.user(self.user, "~BB~FW~SBGoogle Reader starred stories: ~BT~FWNo stories")
-            stories = None
-        if stories:
-            logging.user(self.user, "~BB~FW~SBGoogle Reader starred stories: ~BT~FW%s stories" % (len(stories['items'])))
-            self.process_starred_items(stories['items'])
-            
-        starred_count = MStarredStory.objects.filter(user_id=self.user.pk).count()
-        return starred_count
-        
-    def process_starred_items(self, stories):
-        counts = {
-            'created': 0,
-            'existed': 0,
-            'failed': 0,
-        }
-        logging.user(self.user, "~FCBeginning starring...")
-        
-        for story in stories:
-            try:
-                original_feed = Feed.get_feed_from_url(story['origin']['streamId'], create=False, fetch=False)
-                if not original_feed:
-                    original_feed = Feed.get_feed_from_url(story['origin']['htmlUrl'], create=False, fetch=False)
-                content = story.get('content') or story.get('summary')
-                story_db = {
-                    "user_id": self.user.pk,
-                    "starred_date": datetime.datetime.fromtimestamp(story['updated']),
-                    "story_date": datetime.datetime.fromtimestamp(story['published']),
-                    "story_title": story.get('title', story.get('origin', {}).get('title', '[Untitled]')),
-                    "story_permalink": story['alternate'][0]['href'],
-                    "story_guid": story['id'],
-                    "story_content": content.get('content'),
-                    "story_author_name": story.get('author'),
-                    "story_feed_id": original_feed and original_feed.pk,
-                    "story_tags": [tag for tag in story.get('categories', []) if 'user/' not in tag]
-                }
-                # logging.user(self.user, "~FCStarring: ~SB%s~SN in ~SB%s" % (story_db['story_title'][:50], original_feed and original_feed))
-                MStarredStory.objects.create(**story_db)
-                counts['created'] += 1
-            except OperationError:
-                # logging.user(self.user, "~FCAlready starred: ~SB%s" % (story_db['story_title'][:50]))
-                counts['existed'] += 1
-            except Exception:
-                # logging.user(self.user, "~FC~BRFailed to star: ~SB%s / %s" % (story, e))
-                counts['failed'] += 1
-
-        logging.user(self.user, "~FCStarred: ~SB%s~SN/~SB%s%s~SN/~SB%s%s~SN" % (
-                     counts['created'],
-                     '~FM' if counts['existed'] else '~SN', counts['existed'],
-                     '~FR' if counts['failed'] else '~SN', counts['failed']))
-        return counts

apps/feed_import/tasks.py

@@ -1,6 +1,6 @@
 from celery.task import Task
 from django.contrib.auth.models import User
-from apps.feed_import.models import UploadedOPML, OPMLImporter, GoogleReaderImporter
+from apps.feed_import.models import UploadedOPML, OPMLImporter
 from apps.reader.models import UserSubscription
 from apps.rss_feeds.models import MStarredStory
 from utils import log as logging
@@ -20,33 +20,3 @@ class ProcessOPML(Task):
         user.profile.send_upload_opml_finished_email(feed_count)
         logging.user(user, "~FR~SBOPML upload (task): ~SK%s~SN~SB~FR feeds" % (feed_count))
 
-
-class ProcessReaderImport(Task):
-    
-    def run(self, user_id, auto_active=False):
-        user = User.objects.get(pk=user_id)
-        logging.user(user, "~FR~SBGoogle Reader import (task) starting...")
-
-        importer = GoogleReaderImporter(user=user)
-        importer.import_feeds(auto_active=auto_active)
-        importer.import_starred_items(count=10)
-        
-        feed_count = UserSubscription.objects.filter(user=user).count()
-        user.profile.send_import_reader_finished_email(feed_count)
-        logging.user(user, "~FR~SBGoogle Reader import (task): ~SK%s~SN~SB~FR feeds" % (feed_count))
-
-
-class ProcessReaderStarredImport(Task):
-    
-    def run(self, user_id):
-        user = User.objects.get(pk=user_id)
-        logging.user(user, "~FR~SBGoogle Reader starred stories import (task) starting...")
-
-        importer = GoogleReaderImporter(user=user)
-        importer.import_starred_items(count=1000)
-        
-        feed_count = UserSubscription.objects.filter(user=user).count()
-        starred_count = MStarredStory.objects.filter(user_id=user.pk).count()
-        user.profile.send_import_reader_starred_finished_email(feed_count, starred_count)
-        logging.user(user, "~FR~SBGoogle Reader starred stories import (task): ~SK%s~SN~SB~FR feeds, ~SK%s~SN~SB~FR starred stories" % (feed_count, starred_count))
-

apps/feed_import/tests.py

@@ -4,7 +4,6 @@ from django.test import TestCase
 from django.contrib.auth.models import User
 from django.core.urlresolvers import reverse
 from apps.reader.models import UserSubscription, UserSubscriptionFolders
-from apps.feed_import.models import GoogleReaderImporter
 from utils import json_functions as json
 
 class ImportTest(TestCase):

apps/feed_import/views.py

@@ -1,7 +1,6 @@
 import datetime
 import pickle
 import base64
-import httplib2
 from utils import log as logging
 from oauth2client.client import OAuth2WebServerFlow, FlowExchangeError
 from bson.errors import InvalidStringData
@@ -17,9 +16,9 @@ from django.contrib.auth import login as login_user
 from django.shortcuts import render_to_response
 from apps.reader.forms import SignupForm
 from apps.reader.models import UserSubscription
-from apps.feed_import.models import OAuthToken, GoogleReaderImporter
+from apps.feed_import.models import OAuthToken
 from apps.feed_import.models import OPMLImporter, OPMLExporter, UploadedOPML
-from apps.feed_import.tasks import ProcessOPML, ProcessReaderImport, ProcessReaderStarredImport
+from apps.feed_import.tasks import ProcessOPML
 from utils import json_functions as json
 from utils.user_functions import ajax_login_required, get_user
 from utils.feed_functions import TimeoutError

apps/profile/models.py

@@ -20,7 +20,7 @@ from django.core.urlresolvers import reverse
 from django.template.loader import render_to_string
 from apps.rss_feeds.models import Feed, MStory, MStarredStory
 from apps.rss_feeds.tasks import SchedulePremiumSetup
-from apps.feed_import.models import GoogleReaderImporter, OPMLExporter
+from apps.feed_import.models import OPMLExporter
 from apps.reader.models import UserSubscription
 from apps.reader.models import RUserStory
 from utils import log as logging
@@ -651,10 +651,6 @@ class Profile(models.Model):
                         pipeline.zrem(premium_key, user.pk)
                 pipeline.execute()
     
-    def import_reader_starred_items(self, count=20):
-        importer = GoogleReaderImporter(self.user)
-        importer.import_starred_items(count=count)
-                     
     def send_new_user_email(self):
         if not self.user.email or not self.send_emails:
             return

config/requirements.txt

@@ -23,7 +23,7 @@ dnspython==1.15.0
 Fabric==1.14.0
 gunicorn==19.7
 hiredis==0.2.0
-httplib2==0.9.2
+httplib2==0.18.0
 image==1.5.27
 isodate==0.5.4
 lxml==3.6.4
@@ -31,7 +31,7 @@ mock==2.0.0
 mongoengine==0.10.7
 PyMySQL==0.9.3
 ndg-httpsclient==0.4.2
-nltk==3.2.1
+nltk==3.4.5
 numpy==1.11.2
 oauth2==1.9.0.post1
 pillow==3.4.2

fabfile.py

@@ -468,7 +468,7 @@ def setup_virtualenv():
                 # sudo('rmvirtualenv newsblur')
                 # sudo('rm -fr venv')
                 with settings(warn_only=True):
-                    run('mkvirtualenv --no-site-packages newsblur')
+                    run('mkvirtualenv newsblur')
                 run('echo "import sys; sys.setdefaultencoding(\'utf-8\')" | sudo tee venv/newsblur/lib/python2.7/sitecustomize.py')
                 run('echo "/srv/newsblur" | sudo tee venv/newsblur/lib/python2.7/site-packages/newsblur.pth')
     
@@ -761,8 +761,8 @@ def setup_staging():
         run('touch logs/newsblur.log')
 
 def setup_node_app():
-    sudo('curl -sL https://deb.nodesource.com/setup_10.x | sudo -E bash -')
+    sudo('curl -sL https://deb.nodesource.com/setup_14.x | sudo bash -')
-    sudo('apt-get install -y nodejs')
+    sudo('apt-get install -y nodejs npm')
     # run('curl -L https://npmjs.org/install.sh | sudo sh')
     # sudo('apt-get install npm')
     sudo('sudo npm install -g npm')