Merge pull request #835 from sv0/text_importer

Text importer

apps/rss_feeds/models.py

@@ -24,6 +24,7 @@ from django.core.urlresolvers import reverse
 from django.contrib.auth.models import User
 from django.contrib.sites.models import Site
 from django.template.defaultfilters import slugify
+from django.utils.encoding import smart_str
 from mongoengine.queryset import OperationError, Q, NotUniqueError
 from mongoengine.base import ValidationError
 from vendor.timezones.utilities import localtime_for_timezone
@@ -84,7 +85,6 @@ class Feed(models.Model):
     s3_icon = models.NullBooleanField(default=False, blank=True, null=True)
     search_indexed = models.NullBooleanField(default=None, null=True, blank=True)
 
-
     class Meta:
         db_table="feeds"
         ordering=["feed_title"]
@@ -1900,13 +1900,13 @@ class MStory(mongo.Document):
         self.story_hash = self.feed_guid_hash
         
         if self.story_content:
-            self.story_content_z = zlib.compress(self.story_content)
+            self.story_content_z = zlib.compress(smart_str(self.story_content))
             self.story_content = None
         if self.story_original_content:
-            self.story_original_content_z = zlib.compress(self.story_original_content)
+            self.story_original_content_z = zlib.compress(smart_str(self.story_original_content))
             self.story_original_content = None
         if self.story_latest_content:
-            self.story_latest_content_z = zlib.compress(self.story_latest_content)
+            self.story_latest_content_z = zlib.compress(smart_str(self.story_latest_content))
             self.story_latest_content = None
         if self.story_title and len(self.story_title) > story_title_max:
             self.story_title = self.story_title[:story_title_max]

apps/rss_feeds/tests.py

@@ -7,6 +7,7 @@ from django.conf import settings
 from apps.rss_feeds.models import Feed, MStory
 from mongoengine.connection import connect, disconnect
 
+
 class FeedTest(TestCase):
     fixtures = ['rss_feeds.json']
 
@@ -121,7 +122,6 @@ class FeedTest(TestCase):
         content = json.decode(response.content)
         self.assertEquals(content['feeds']['5']['nt'], 37)
 
-        
     def test_load_feeds__motherjones(self):
         self.client.login(username='conesus', password='test')
 

apps/rss_feeds/text_importer.py

@@ -1,7 +1,6 @@
 import requests
 import zlib
 from requests.packages.urllib3.exceptions import LocationParseError
-from django.conf import settings
 from socket import error as SocketError
 from mongoengine.queryset import NotUniqueError
 from vendor.readability import readability
@@ -9,11 +8,13 @@ from utils import log as logging
 from utils.feed_functions import timelimit, TimeoutError
 from OpenSSL.SSL import Error as OpenSSLError
 from pyasn1.error import PyAsn1Error
+from django.utils.encoding import smart_str
 
 BROKEN_URLS = [
     "gamespot.com",
 ]
 
+
 class TextImporter:
 
     def __init__(self, story=None, feed=None, story_url=None, request=None, debug=False):
@@ -25,15 +26,16 @@ class TextImporter:
 
     @property
     def headers(self):
+        num_subscribers = getattr(self.feed, 'num_subscribers', 0)
         return {
             'User-Agent': 'NewsBlur Content Fetcher - %s subscriber%s - %s '
                           '(Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) '
                           'AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 '
                           'Safari/534.48.3)' % (
-                self.feed.num_subscribers,
-                's' if self.feed.num_subscribers != 1 else '',
-                self.feed.permalink,
-            ),
+                              num_subscribers,
+                              's' if num_subscribers != 1 else '',
+                              getattr(self.feed, 'permalink', '')
+                          ),
         }
 
     def fetch(self, skip_save=False, return_document=False):
@@ -53,17 +55,7 @@ class TextImporter:
         if not resp:
             return
 
-        try:
-            text = resp.text
-        except (LookupError, TypeError):
-            text = resp.content
-        
-        charset_declared = 'charset' in resp.headers.get('content-type', "")
-        if resp.encoding and resp.encoding != 'utf-8' and not charset_declared:
-            try:
-                text = text.encode(resp.encoding)
-            except (LookupError, UnicodeEncodeError):
-                pass
+        text = resp.text
         original_text_doc = readability.Document(text, url=resp.url,
                                                  debug=self.debug,
                                                  positive_keywords=["postContent", "postField"])
@@ -80,13 +72,13 @@ class TextImporter:
 
         if content:
             if self.story and not skip_save:
-                self.story.original_text_z = zlib.compress(content)
+                self.story.original_text_z = zlib.compress(smart_str(content))
                 try:
                     self.story.save()
                 except NotUniqueError:
                     pass
             logging.user(self.request, ("~SN~FYFetched ~FGoriginal text~FY: now ~SB%s bytes~SN vs. was ~SB%s bytes" % (
-                len(unicode(content)),
+                len(content),
                 self.story and self.story.story_content_z and len(zlib.decompress(self.story.story_content_z))
             )), warn_color=False)
         else: