Merge branch 'dashboard3' of https://github.com/samuelclay/NewsBlur into dashboard3

apps/notifications/models.py

@@ -1,5 +1,6 @@
 import datetime
 import enum
+import html
 import redis
 import mongoengine as mongo
 from boto.ses.connection import BotoServerError
@@ -17,7 +18,6 @@ from utils.view_functions import is_true
 from utils.story_functions import truncate_chars
 from utils import log as logging
 from utils import mongoengine_fields
-from html.parser import HTMLParser
 from vendor.apns import APNs, Payload
 from bs4 import BeautifulSoup, Tag
 import urllib.parse
@@ -154,7 +154,7 @@ class MUserFeedNotification(mongo.Document):
                     user_feed_notification.last_notification_date = story['story_date']
                     user_feed_notification.save()
                 
-                story['story_content'] = HTMLParser().unescape(story['story_content'])
+                story['story_content'] = html.unescape(story['story_content'])
                 
                 sent = user_feed_notification.push_story_notification(story, classifiers, usersub)
                 if sent: 
@@ -190,9 +190,9 @@ class MUserFeedNotification(mongo.Document):
         title = feed_title
         if notification_title_only:
             subtitle = None
-            body = HTMLParser().unescape(story['story_title'])
+            body = html.unescape(story['story_title'])
         else:
-            subtitle = HTMLParser().unescape(story['story_title'])
+            subtitle = html.unescape(story['story_title'])
             soup = BeautifulSoup(story['story_content'].strip(), features="lxml")
             body = replace_with_newlines(soup)
         body = truncate_chars(body.strip(), 600)

apps/rss_feeds/models.py

@@ -11,7 +11,7 @@ import hashlib
 import redis
 import base64
 import pymongo
-import html.parser
+import html
 import urllib.parse
 from collections import defaultdict
 from operator import itemgetter
@@ -2457,8 +2457,7 @@ class MStory(mongo.Document):
     
     @property
     def decoded_story_title(self):
-        h = html.parser.HTMLParser()
-        return h.unescape(self.story_title)
+        return html.unescape(self.story_title)
 
     def save(self, *args, **kwargs):
         story_title_max = MStory._fields['story_title'].max_length

apps/social/models.py

@@ -1494,8 +1494,7 @@ class MSharedStory(mongo.DynamicDocument):
     
     @property
     def decoded_story_title(self):
-        h = html_parser.HTMLParser()
-        return h.unescape(self.story_title)
+        return html.unescape(self.story_title)
         
     def canonical(self):
         return {

utils/story_functions.py

@@ -4,9 +4,9 @@ import struct
 import dateutil
 import hashlib
 import base64
+import html
 import sys
 from random import randint
-from html.parser import HTMLParser
 from lxml.html.diff import tokenize, fixup_ins_del_tags, htmldiff_tokens
 from lxml.etree import ParserError, XMLSyntaxError, SerialisationError
 import lxml.html, lxml.etree
@@ -208,25 +208,11 @@ def attach_media_scripts(content):
     if 'imgur-embed-pub' in content and '<script' not in content:
         content += '<script async src="https://s.imgur.com/min/embed.js" charset="utf-8"></script>'
     return content
-        
-    
-class MLStripper(HTMLParser):
-    def __init__(self):
-        self.reset()
-        self.fed = []
-    def handle_data(self, d):
-        self.fed.append(d)
-    def get_data(self):
-        return ' '.join(self.fed)
 
 def strip_tags(html):
     if not html:
         return ''
     return strip_tags_django(html)
-    
-    s = MLStripper()
-    s.feed(html)
-    return s.get_data()
 
 def strip_comments(html_string):
     return COMMENTS_RE.sub('', html_string)
@@ -288,7 +274,7 @@ def truncate_chars(value, max_length):
     except UnicodeDecodeError:
         pass
     if len(value) <= max_length:
-        return value
+        return value.decode('utf-8', 'ignore')
  
     truncd_val = value[:max_length]
     if value[max_length] != " ":