Fixing a substantial number of encoding errors. Also fixing OPML uploads.

apps/feed_import/models.py

@@ -188,6 +188,9 @@ class OPMLImporter(Importer):
                         feed=feed_db, 
                         user=self.user)
                 except UserSubscription.DoesNotExist:
+                    us = None
+                
+                if not us:
                     us = UserSubscription(
                         feed=feed_db, 
                         user=self.user,

apps/reader/managers.py

@@ -8,7 +8,7 @@ class UserSubscriptionManager(models.Manager):
     def get(self, *args, **kwargs):
         try:
             return super(UserSubscriptionManager, self).get(*args, **kwargs)
-        except self.model.DoesNotExist:
+        except self.model.DoesNotExist as exception:
             if isinstance(kwargs.get('feed'), int):
                 feed_id = kwargs.get('feed')
             elif 'feed' in kwargs:
@@ -32,5 +32,4 @@ class UserSubscriptionManager(models.Manager):
                 logging.debug(" ---> [%s] ~BRFound dupe UserSubscription: ~SB%s (%s)" % (user and user.username, feed, feed_id))
                 return super(UserSubscriptionManager, self).get(*args, **kwargs)
             else:
-                exc_info = sys.exc_info()
-                raise exc_info[1]
+                raise exception

apps/rss_feeds/models.py

@@ -365,7 +365,7 @@ class Feed(models.Model):
             return feed, False
         except cls.DoesNotExist:
             feed = cls(**defaults)
-            feed.save()
+            feed = feed.save()
             return feed, True
         
     @classmethod

apps/rss_feeds/page_importer.py

@@ -199,15 +199,18 @@ class PageImporter(object):
                     requests.adapters.ReadTimeout) as e:
                 logging.debug('   ***> [%-30s] Original story fetch failed using requests: %s' % (self.feed.log_title[:30], e))
                 return
-        try:
-            data = response.text
-        except (LookupError, TypeError):
-            data = response.content
+        # try:
+        data = response.text
+        # except (LookupError, TypeError):
+        #     data = response.content
+        # import pdb; pdb.set_trace()
 
-        if response.encoding and response.encoding != 'utf-8':
+        if response.encoding and response.encoding.lower() != 'utf-8':
+            logging.debug(f" -> ~FBEncoding is {response.encoding}, re-encoding...")
             try:
-                data = data.encode(response.encoding)
+                data = data.encode('utf-8').decode('utf-8')
             except (LookupError, UnicodeEncodeError):
+                logging.debug(f" -> ~FRRe-encoding failed!")
                 pass
 
         if data:
@@ -221,7 +224,7 @@ class PageImporter(object):
         return html
     
     def save_story(self, html):
-        self.story.original_page_z = zlib.compress(html)
+        self.story.original_page_z = zlib.compress(html.encode('utf-8'))
         try:
             self.story.save()
         except NotUniqueError:

config/mongodb.prod.conf

@@ -88,5 +88,9 @@ noauth = true
 #master = true
 #source = slave.example.com
 
+# Mongo 3.6 requires binding to non-localhost, but because we use ufw for a firewall,
+# allow all IPs to connect
+net.bindIpAll = true
+
 # in replica set configuration, specify the name of the replica set
 replSet = nbset

utils/exception_middleware.py

@@ -9,7 +9,7 @@ class ConsoleExceptionMiddleware:
         print("######################## Exception #############################")
         print(('\n'.join(traceback.format_exception(*(exc_info or sys.exc_info())))))
         print("----------------------------------------------------------------")
-        pprint(inspect.trace()[-1][0].f_locals)
+        # pprint(inspect.trace()[-1][0].f_locals)
         print("################################################################")
         
         #pprint(request)
@@ -22,4 +22,4 @@ class ConsoleExceptionMiddleware:
 
         response = self.get_response(request)
 
-        return response
+        return response

utils/feed_functions.py

@@ -23,6 +23,7 @@ def timelimit(timeout):
                     threading.Thread.__init__(self)
                     self.result = None
                     self.error = None
+                    self.exc_info = None
                     
                     self.setDaemon(True)
                     self.start()
@@ -30,17 +31,19 @@ def timelimit(timeout):
                 def run(self):
                     try:
                         self.result = function(*args, **kw)
-                    except:
-                        self.error = sys.exc_info()
+                    except BaseException as e:
+                        self.error = e
+                        self.exc_info = sys.exc_info()
             c = Dispatch()
+            dispatch = c
             c.join(timeout)
             if c.is_alive():
                 raise TimeoutError('took too long')
             if c.error:
-                tb = ''.join(traceback.format_exception(c.error[0], c.error[1], c.error[2]))
+                tb = ''.join(traceback.format_exception(c.exc_info[0], c.exc_info[1], c.exc_info[2]))
                 logging.debug(tb)
-                mail_admins('Error in timeout: %s' % c.error[0], tb)
-                raise c.error[0](c.error[1]).with_traceback(c.error[2])
+                mail_admins('Error in timeout: %s' % c.exc_info[0], tb)
+                raise c.error
             return c.result
         return _2
     return _1