mirror of
https://github.com/samuelclay/NewsBlur.git
synced 2025-04-13 09:42:01 +00:00
203 lines
7.5 KiB
Python
203 lines
7.5 KiB
Python
import requests
|
|
import re
|
|
import urlparse
|
|
import traceback
|
|
import feedparser
|
|
import time
|
|
import urllib2
|
|
import httplib
|
|
import gzip
|
|
import StringIO
|
|
from boto.s3.key import Key
|
|
from django.conf import settings
|
|
from utils import log as logging
|
|
from apps.rss_feeds.models import MFeedPage
|
|
from utils.feed_functions import timelimit, mail_feed_error_to_admin
|
|
|
|
BROKEN_PAGES = [
|
|
'tag:',
|
|
'info:',
|
|
'uuid:',
|
|
'urn:',
|
|
'[]',
|
|
]
|
|
|
|
# Also change in reader_utils.js.
|
|
BROKEN_PAGE_URLS = [
|
|
'nytimes.com',
|
|
'stackoverflow.com',
|
|
'stackexchange.com',
|
|
'twitter.com',
|
|
'rankexploits',
|
|
]
|
|
|
|
class PageImporter(object):
|
|
|
|
def __init__(self, feed):
|
|
self.feed = feed
|
|
|
|
@property
|
|
def headers(self):
|
|
s = requests.session()
|
|
s.config['keep_alive'] = False
|
|
return {
|
|
'User-Agent': 'NewsBlur Page Fetcher (%s subscriber%s) - %s '
|
|
'(Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) '
|
|
'AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 '
|
|
'Safari/534.48.3)' % (
|
|
self.feed.num_subscribers,
|
|
's' if self.feed.num_subscribers != 1 else '',
|
|
settings.NEWSBLUR_URL
|
|
),
|
|
'Connection': 'close',
|
|
}
|
|
|
|
@timelimit(15)
|
|
def fetch_page(self, urllib_fallback=False, requests_exception=None):
|
|
html = None
|
|
feed_link = self.feed.feed_link
|
|
if not feed_link:
|
|
self.save_no_page()
|
|
return
|
|
|
|
if feed_link.startswith('www'):
|
|
self.feed.feed_link = 'http://' + feed_link
|
|
try:
|
|
if any(feed_link.startswith(s) for s in BROKEN_PAGES):
|
|
self.save_no_page()
|
|
return
|
|
elif any(s in feed_link.lower() for s in BROKEN_PAGE_URLS):
|
|
self.save_no_page()
|
|
return
|
|
elif feed_link.startswith('http'):
|
|
if urllib_fallback:
|
|
request = urllib2.Request(feed_link, headers=self.headers)
|
|
response = urllib2.urlopen(request)
|
|
time.sleep(0.01) # Grrr, GIL.
|
|
data = response.read()
|
|
else:
|
|
try:
|
|
response = requests.get(feed_link, headers=self.headers)
|
|
except requests.exceptions.TooManyRedirects:
|
|
response = requests.get(feed_link)
|
|
try:
|
|
data = response.text
|
|
except (LookupError, TypeError):
|
|
data = response.content
|
|
else:
|
|
try:
|
|
data = open(feed_link, 'r').read()
|
|
except IOError:
|
|
self.feed.feed_link = 'http://' + feed_link
|
|
self.fetch_page(urllib_fallback=True)
|
|
return
|
|
if data:
|
|
html = self.rewrite_page(data)
|
|
self.save_page(html)
|
|
else:
|
|
self.save_no_page()
|
|
return
|
|
except (ValueError, urllib2.URLError, httplib.BadStatusLine, httplib.InvalidURL,
|
|
requests.exceptions.ConnectionError), e:
|
|
self.feed.save_page_history(401, "Bad URL", e)
|
|
fp = feedparser.parse(self.feed.feed_address)
|
|
feed_link = fp.feed.get('link', "")
|
|
self.feed.save()
|
|
except (urllib2.HTTPError), e:
|
|
self.feed.save_page_history(e.code, e.msg, e.fp.read())
|
|
except (httplib.IncompleteRead), e:
|
|
self.feed.save_page_history(500, "IncompleteRead", e)
|
|
except (requests.exceptions.RequestException,
|
|
requests.packages.urllib3.exceptions.HTTPError), e:
|
|
logging.debug(' ***> [%-30s] Page fetch failed using requests: %s' % (self.feed, e))
|
|
# mail_feed_error_to_admin(self.feed, e, local_vars=locals())
|
|
return self.fetch_page(urllib_fallback=True, requests_exception=e)
|
|
except Exception, e:
|
|
logging.debug('[%d] ! -------------------------' % (self.feed.id,))
|
|
tb = traceback.format_exc()
|
|
logging.debug(tb)
|
|
logging.debug('[%d] ! -------------------------' % (self.feed.id,))
|
|
self.feed.save_page_history(500, "Error", tb)
|
|
mail_feed_error_to_admin(self.feed, e, local_vars=locals())
|
|
if not urllib_fallback:
|
|
self.fetch_page(urllib_fallback=True)
|
|
else:
|
|
self.feed.save_page_history(200, "OK")
|
|
|
|
return html
|
|
|
|
def save_no_page(self):
|
|
logging.debug(' --->> [%-30s] ~FYNo original page: %s' % (self.feed, self.feed.feed_link))
|
|
self.feed.has_page = False
|
|
self.feed.save()
|
|
self.feed.save_page_history(404, "Feed has no original page.")
|
|
|
|
def rewrite_page(self, response):
|
|
BASE_RE = re.compile(r'<head(.*?\>)', re.I)
|
|
base_code = u'<base href="%s" />' % (self.feed.feed_link,)
|
|
try:
|
|
html = BASE_RE.sub(r'<head\1 '+base_code, response)
|
|
except:
|
|
response = response.decode('latin1').encode('utf-8')
|
|
html = BASE_RE.sub(r'<head\1 '+base_code, response)
|
|
|
|
if '<base href' not in html:
|
|
html = "%s %s" % (base_code, html)
|
|
|
|
# html = self.fix_urls(html)
|
|
|
|
return html.strip()
|
|
|
|
def fix_urls(self, document):
|
|
# BEWARE: This will rewrite URLs inside of <script> tags. You know, like
|
|
# Google Analytics. Ugh.
|
|
|
|
FIND_RE = re.compile(r'\b(href|src)\s*=\s*("[^"]*"|\'[^\']*\'|[^"\'<>=\s]+)')
|
|
ret = []
|
|
last_end = 0
|
|
|
|
for match in FIND_RE.finditer(document):
|
|
url = match.group(2)
|
|
if url[0] in "\"'":
|
|
url = url.strip(url[0])
|
|
parsed = urlparse.urlparse(url)
|
|
if parsed.scheme == parsed.netloc == '': #relative to domain
|
|
url = urlparse.urljoin(self.feed.feed_link, url)
|
|
ret.append(document[last_end:match.start(2)])
|
|
ret.append('"%s"' % (url,))
|
|
last_end = match.end(2)
|
|
ret.append(document[last_end:])
|
|
|
|
return ''.join(ret)
|
|
|
|
def save_page(self, html):
|
|
if html and len(html) > 100:
|
|
if settings.BACKED_BY_AWS.get('pages_on_s3'):
|
|
k = Key(settings.S3_PAGES_BUCKET)
|
|
k.key = self.feed.s3_pages_key
|
|
k.set_metadata('Content-Encoding', 'gzip')
|
|
k.set_metadata('Content-Type', 'text/html')
|
|
k.set_metadata('Access-Control-Allow-Origin', '*')
|
|
out = StringIO.StringIO()
|
|
f = gzip.GzipFile(fileobj=out, mode='w')
|
|
f.write(html)
|
|
f.close()
|
|
compressed_html = out.getvalue()
|
|
k.set_contents_from_string(compressed_html)
|
|
k.set_acl('public-read')
|
|
|
|
if False and not self.feed.s3_page:
|
|
try:
|
|
feed_page = MFeedPage.objects.get(feed_id=self.feed.pk)
|
|
feed_page.delete()
|
|
logging.debug(' --->> [%-30s] ~FYTransfering page data to S3...' % (self.feed))
|
|
except MFeedPage.DoesNotExist:
|
|
pass
|
|
|
|
self.feed.s3_page = True
|
|
self.feed.save()
|
|
else:
|
|
feed_page, _ = MFeedPage.objects.get_or_create(feed_id=self.feed.pk)
|
|
feed_page.page_data = html
|
|
feed_page.save()
|
|
return feed_page
|