mirror of
https://github.com/viq/NewsBlur.git
synced 2025-04-13 09:38:09 +00:00
Removing additional traces of Google Reader.
This commit is contained in:
Samuel Clay
parent
49a48cea01
commit
d60ec99f3e
8 changed files with 10 additions and 241 deletions
|
|
@ -1,7 +1,6 @@
|
|||
from django.core.management.base import BaseCommand
|
||||
from django.conf import settings
|
||||
from django.contrib.auth.models import User
|
||||
from apps.feed_import.models import GoogleReaderImporter
|
||||
from optparse import make_option
|
||||
from utils.management_functions import daemonize
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import datetime
|
||||
import mongoengine as mongo
|
||||
import httplib2
|
||||
import pickle
|
||||
import base64
|
||||
from StringIO import StringIO
|
||||
|
|
@ -235,197 +234,4 @@ class UploadedOPML(mongo.Document):
|
|||
'order': '-upload_date',
|
||||
'indexes': ['user_id', '-upload_date'],
|
||||
}
|
||||
|
||||
|
||||
class GoogleReaderImporter(Importer):
|
||||
|
||||
def __init__(self, user, xml=None):
|
||||
self.user = user
|
||||
self.scope = "http://www.google.com/reader/api"
|
||||
self.xml = xml
|
||||
self.auto_active = False
|
||||
|
||||
@timelimit(10)
|
||||
def try_import_feeds(self, auto_active=False):
|
||||
code = 0
|
||||
try:
|
||||
self.import_feeds(auto_active=auto_active)
|
||||
self.import_starred_items(count=10)
|
||||
except AssertionError:
|
||||
code = -1
|
||||
else:
|
||||
code = 1
|
||||
return code
|
||||
|
||||
def import_feeds(self, auto_active=False):
|
||||
self.auto_active = auto_active
|
||||
sub_url = "%s/0/subscription/list" % self.scope
|
||||
if not self.xml:
|
||||
feeds_xml = self.send_request(sub_url)
|
||||
else:
|
||||
feeds_xml = self.xml
|
||||
if feeds_xml:
|
||||
self.process_feeds(feeds_xml)
|
||||
|
||||
def send_request(self, url):
|
||||
if not self.user.is_authenticated():
|
||||
return
|
||||
|
||||
user_tokens = OAuthToken.objects.filter(user=self.user)
|
||||
|
||||
if user_tokens.count():
|
||||
user_token = user_tokens[0]
|
||||
if user_token.credential:
|
||||
credential = pickle.loads(base64.b64decode(user_token.credential))
|
||||
http = httplib2.Http()
|
||||
http = credential.authorize(http)
|
||||
content = http.request(url)
|
||||
return content and content[1]
|
||||
|
||||
def process_feeds(self, feeds_xml):
|
||||
# self.clear_feeds()
|
||||
# self.clear_folders()
|
||||
folders = self.get_folders()
|
||||
self.feeds = self.parse(feeds_xml)
|
||||
|
||||
for item in self.feeds:
|
||||
folders = self.process_item(item, folders)
|
||||
|
||||
logging.user(self.user, "~BB~FW~SBGoogle Reader import: ~BT~FW%s" % (folders))
|
||||
|
||||
self.usf.folders = json.encode(folders)
|
||||
self.usf.save()
|
||||
|
||||
def parse(self, feeds_xml):
|
||||
parser = etree.XMLParser(recover=True)
|
||||
tree = etree.parse(StringIO(feeds_xml), parser)
|
||||
feeds = tree.xpath('/object/list/object')
|
||||
return feeds
|
||||
|
||||
def process_item(self, item, folders):
|
||||
feed_title = item.xpath('./string[@name="title"]') and \
|
||||
item.xpath('./string[@name="title"]')[0].text
|
||||
feed_address = item.xpath('./string[@name="id"]') and \
|
||||
item.xpath('./string[@name="id"]')[0].text.replace('feed/', '')
|
||||
feed_link = item.xpath('./string[@name="htmlUrl"]') and \
|
||||
item.xpath('./string[@name="htmlUrl"]')[0].text
|
||||
category = item.xpath('./list[@name="categories"]/object/string[@name="label"]') and \
|
||||
item.xpath('./list[@name="categories"]/object/string[@name="label"]')[0].text
|
||||
|
||||
if not feed_address:
|
||||
feed_address = feed_link
|
||||
|
||||
try:
|
||||
feed_link = urlnorm.normalize(feed_link)
|
||||
feed_address = urlnorm.normalize(feed_address)
|
||||
|
||||
if len(feed_address) > Feed._meta.get_field('feed_address').max_length:
|
||||
return folders
|
||||
|
||||
# See if it exists as a duplicate first
|
||||
duplicate_feed = DuplicateFeed.objects.filter(duplicate_address=feed_address)
|
||||
if duplicate_feed:
|
||||
feed_db = duplicate_feed[0].feed
|
||||
else:
|
||||
feed_data = dict(feed_title=feed_title)
|
||||
feed_data['active_subscribers'] = 1
|
||||
feed_data['num_subscribers'] = 1
|
||||
feed_db, _ = Feed.find_or_create(feed_address=feed_address, feed_link=feed_link,
|
||||
defaults=dict(**feed_data))
|
||||
|
||||
us, _ = UserSubscription.objects.get_or_create(
|
||||
feed=feed_db,
|
||||
user=self.user,
|
||||
defaults={
|
||||
'needs_unread_recalc': True,
|
||||
'mark_read_date': datetime.datetime.utcnow() - datetime.timedelta(days=1),
|
||||
'active': self.user.profile.is_premium or self.auto_active,
|
||||
}
|
||||
)
|
||||
if not us.needs_unread_recalc:
|
||||
us.needs_unread_recalc = True
|
||||
us.save()
|
||||
if not category: category = ""
|
||||
|
||||
if category:
|
||||
obj = {category: []}
|
||||
folders = add_object_to_folder(obj, '', folders)
|
||||
folders = add_object_to_folder(feed_db.pk, category, folders)
|
||||
# if feed_db.pk not in folders[category]:
|
||||
# folders[category].append(feed_db.pk)
|
||||
except Exception, e:
|
||||
logging.info(' *** -> Exception: %s: %s' % (e, item))
|
||||
|
||||
return folders
|
||||
|
||||
def test(self):
|
||||
sub_url = "%s/0/token" % (self.scope)
|
||||
try:
|
||||
resp = self.send_request(sub_url)
|
||||
except OAuthError:
|
||||
return False
|
||||
return resp
|
||||
|
||||
@timelimit(10)
|
||||
def try_import_starred_stories(self):
|
||||
self.import_starred_items(count=1000)
|
||||
|
||||
starred_count = MStarredStory.objects.filter(user_id=self.user.pk).count()
|
||||
return starred_count
|
||||
|
||||
def import_starred_items(self, count=10):
|
||||
sub_url = "%s/0/stream/contents/user/-/state/com.google/starred?n=%s" % (self.scope, count)
|
||||
stories_str = self.send_request(sub_url)
|
||||
try:
|
||||
stories = json.decode(stories_str)
|
||||
except:
|
||||
logging.user(self.user, "~BB~FW~SBGoogle Reader starred stories: ~BT~FWNo stories")
|
||||
stories = None
|
||||
if stories:
|
||||
logging.user(self.user, "~BB~FW~SBGoogle Reader starred stories: ~BT~FW%s stories" % (len(stories['items'])))
|
||||
self.process_starred_items(stories['items'])
|
||||
|
||||
starred_count = MStarredStory.objects.filter(user_id=self.user.pk).count()
|
||||
return starred_count
|
||||
|
||||
def process_starred_items(self, stories):
|
||||
counts = {
|
||||
'created': 0,
|
||||
'existed': 0,
|
||||
'failed': 0,
|
||||
}
|
||||
logging.user(self.user, "~FCBeginning starring...")
|
||||
|
||||
for story in stories:
|
||||
try:
|
||||
original_feed = Feed.get_feed_from_url(story['origin']['streamId'], create=False, fetch=False)
|
||||
if not original_feed:
|
||||
original_feed = Feed.get_feed_from_url(story['origin']['htmlUrl'], create=False, fetch=False)
|
||||
content = story.get('content') or story.get('summary')
|
||||
story_db = {
|
||||
"user_id": self.user.pk,
|
||||
"starred_date": datetime.datetime.fromtimestamp(story['updated']),
|
||||
"story_date": datetime.datetime.fromtimestamp(story['published']),
|
||||
"story_title": story.get('title', story.get('origin', {}).get('title', '[Untitled]')),
|
||||
"story_permalink": story['alternate'][0]['href'],
|
||||
"story_guid": story['id'],
|
||||
"story_content": content.get('content'),
|
||||
"story_author_name": story.get('author'),
|
||||
"story_feed_id": original_feed and original_feed.pk,
|
||||
"story_tags": [tag for tag in story.get('categories', []) if 'user/' not in tag]
|
||||
}
|
||||
# logging.user(self.user, "~FCStarring: ~SB%s~SN in ~SB%s" % (story_db['story_title'][:50], original_feed and original_feed))
|
||||
MStarredStory.objects.create(**story_db)
|
||||
counts['created'] += 1
|
||||
except OperationError:
|
||||
# logging.user(self.user, "~FCAlready starred: ~SB%s" % (story_db['story_title'][:50]))
|
||||
counts['existed'] += 1
|
||||
except Exception:
|
||||
# logging.user(self.user, "~FC~BRFailed to star: ~SB%s / %s" % (story, e))
|
||||
counts['failed'] += 1
|
||||
|
||||
logging.user(self.user, "~FCStarred: ~SB%s~SN/~SB%s%s~SN/~SB%s%s~SN" % (
|
||||
counts['created'],
|
||||
'~FM' if counts['existed'] else '~SN', counts['existed'],
|
||||
'~FR' if counts['failed'] else '~SN', counts['failed']))
|
||||
return counts
|
||||
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
from celery.task import Task
|
||||
from django.contrib.auth.models import User
|
||||
from apps.feed_import.models import UploadedOPML, OPMLImporter, GoogleReaderImporter
|
||||
from apps.feed_import.models import UploadedOPML, OPMLImporter
|
||||
from apps.reader.models import UserSubscription
|
||||
from apps.rss_feeds.models import MStarredStory
|
||||
from utils import log as logging
|
||||
|
|
@ -20,33 +20,3 @@ class ProcessOPML(Task):
|
|||
user.profile.send_upload_opml_finished_email(feed_count)
|
||||
logging.user(user, "~FR~SBOPML upload (task): ~SK%s~SN~SB~FR feeds" % (feed_count))
|
||||
|
||||
|
||||
class ProcessReaderImport(Task):
|
||||
|
||||
def run(self, user_id, auto_active=False):
|
||||
user = User.objects.get(pk=user_id)
|
||||
logging.user(user, "~FR~SBGoogle Reader import (task) starting...")
|
||||
|
||||
importer = GoogleReaderImporter(user=user)
|
||||
importer.import_feeds(auto_active=auto_active)
|
||||
importer.import_starred_items(count=10)
|
||||
|
||||
feed_count = UserSubscription.objects.filter(user=user).count()
|
||||
user.profile.send_import_reader_finished_email(feed_count)
|
||||
logging.user(user, "~FR~SBGoogle Reader import (task): ~SK%s~SN~SB~FR feeds" % (feed_count))
|
||||
|
||||
|
||||
class ProcessReaderStarredImport(Task):
|
||||
|
||||
def run(self, user_id):
|
||||
user = User.objects.get(pk=user_id)
|
||||
logging.user(user, "~FR~SBGoogle Reader starred stories import (task) starting...")
|
||||
|
||||
importer = GoogleReaderImporter(user=user)
|
||||
importer.import_starred_items(count=1000)
|
||||
|
||||
feed_count = UserSubscription.objects.filter(user=user).count()
|
||||
starred_count = MStarredStory.objects.filter(user_id=user.pk).count()
|
||||
user.profile.send_import_reader_starred_finished_email(feed_count, starred_count)
|
||||
logging.user(user, "~FR~SBGoogle Reader starred stories import (task): ~SK%s~SN~SB~FR feeds, ~SK%s~SN~SB~FR starred stories" % (feed_count, starred_count))
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ from django.test import TestCase
|
|||
from django.contrib.auth.models import User
|
||||
from django.core.urlresolvers import reverse
|
||||
from apps.reader.models import UserSubscription, UserSubscriptionFolders
|
||||
from apps.feed_import.models import GoogleReaderImporter
|
||||
from utils import json_functions as json
|
||||
|
||||
class ImportTest(TestCase):
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
import datetime
|
||||
import pickle
|
||||
import base64
|
||||
import httplib2
|
||||
from utils import log as logging
|
||||
from oauth2client.client import OAuth2WebServerFlow, FlowExchangeError
|
||||
from bson.errors import InvalidStringData
|
||||
|
|
@ -17,9 +16,9 @@ from django.contrib.auth import login as login_user
|
|||
from django.shortcuts import render_to_response
|
||||
from apps.reader.forms import SignupForm
|
||||
from apps.reader.models import UserSubscription
|
||||
from apps.feed_import.models import OAuthToken, GoogleReaderImporter
|
||||
from apps.feed_import.models import OAuthToken
|
||||
from apps.feed_import.models import OPMLImporter, OPMLExporter, UploadedOPML
|
||||
from apps.feed_import.tasks import ProcessOPML, ProcessReaderImport, ProcessReaderStarredImport
|
||||
from apps.feed_import.tasks import ProcessOPML
|
||||
from utils import json_functions as json
|
||||
from utils.user_functions import ajax_login_required, get_user
|
||||
from utils.feed_functions import TimeoutError
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ from django.core.urlresolvers import reverse
|
|||
from django.template.loader import render_to_string
|
||||
from apps.rss_feeds.models import Feed, MStory, MStarredStory
|
||||
from apps.rss_feeds.tasks import SchedulePremiumSetup
|
||||
from apps.feed_import.models import GoogleReaderImporter, OPMLExporter
|
||||
from apps.feed_import.models import OPMLExporter
|
||||
from apps.reader.models import UserSubscription
|
||||
from apps.reader.models import RUserStory
|
||||
from utils import log as logging
|
||||
|
|
@ -651,10 +651,6 @@ class Profile(models.Model):
|
|||
pipeline.zrem(premium_key, user.pk)
|
||||
pipeline.execute()
|
||||
|
||||
def import_reader_starred_items(self, count=20):
|
||||
importer = GoogleReaderImporter(self.user)
|
||||
importer.import_starred_items(count=count)
|
||||
|
||||
def send_new_user_email(self):
|
||||
if not self.user.email or not self.send_emails:
|
||||
return
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ dnspython==1.15.0
|
|||
Fabric==1.14.0
|
||||
gunicorn==19.7
|
||||
hiredis==0.2.0
|
||||
httplib2==0.9.2
|
||||
httplib2==0.18.0
|
||||
image==1.5.27
|
||||
isodate==0.5.4
|
||||
lxml==3.6.4
|
||||
|
|
@ -31,7 +31,7 @@ mock==2.0.0
|
|||
mongoengine==0.10.7
|
||||
PyMySQL==0.9.3
|
||||
ndg-httpsclient==0.4.2
|
||||
nltk==3.2.1
|
||||
nltk==3.4.5
|
||||
numpy==1.11.2
|
||||
oauth2==1.9.0.post1
|
||||
pillow==3.4.2
|
||||
|
|
|
|||
6
fabfile.py
vendored
6
fabfile.py
vendored
|
|
@ -468,7 +468,7 @@ def setup_virtualenv():
|
|||
# sudo('rmvirtualenv newsblur')
|
||||
# sudo('rm -fr venv')
|
||||
with settings(warn_only=True):
|
||||
run('mkvirtualenv --no-site-packages newsblur')
|
||||
run('mkvirtualenv newsblur')
|
||||
run('echo "import sys; sys.setdefaultencoding(\'utf-8\')" | sudo tee venv/newsblur/lib/python2.7/sitecustomize.py')
|
||||
run('echo "/srv/newsblur" | sudo tee venv/newsblur/lib/python2.7/site-packages/newsblur.pth')
|
||||
|
||||
|
|
@ -761,8 +761,8 @@ def setup_staging():
|
|||
run('touch logs/newsblur.log')
|
||||
|
||||
def setup_node_app():
|
||||
sudo('curl -sL https://deb.nodesource.com/setup_10.x | sudo -E bash -')
|
||||
sudo('apt-get install -y nodejs')
|
||||
sudo('curl -sL https://deb.nodesource.com/setup_14.x | sudo bash -')
|
||||
sudo('apt-get install -y nodejs npm')
|
||||
# run('curl -L https://npmjs.org/install.sh | sudo sh')
|
||||
# sudo('apt-get install npm')
|
||||
sudo('sudo npm install -g npm')
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue