From 21f40c253f5a78ac59d00c5b38e1a30b50cdcb95 Mon Sep 17 00:00:00 2001
From: Samuel Clay <samuel@ofbrooklyn.com>
Date: Mon, 30 Nov 2020 15:48:59 -0500
Subject: [PATCH] Updating User Agent for all fetchers.

---
 apps/rss_feeds/icon_importer.py |  8 +++-----
 apps/rss_feeds/models.py        | 10 +++++-----
 apps/rss_feeds/page_importer.py |  6 ++----
 apps/rss_feeds/text_importer.py |  8 +++-----
 apps/social/models.py           |  5 +----
 5 files changed, 14 insertions(+), 23 deletions(-)

diff --git a/apps/rss_feeds/icon_importer.py b/apps/rss_feeds/icon_importer.py
index 969b17c41..2ed5f6066 100644
--- a/apps/rss_feeds/icon_importer.py
+++ b/apps/rss_feeds/icon_importer.py
@@ -275,14 +275,12 @@ class IconImporter(object):
         @timelimit(30)
         def _1(url):
             headers = {
-                'User-Agent': 'NewsBlur Favicon Fetcher - %s subscriber%s - %s '
-                              '(Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) '
-                              'AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 '
-                              'Safari/534.48.3)' %
+                'User-Agent': 'NewsBlur Favicon Fetcher - %s subscriber%s - %s %s' %
                               (
                                   self.feed.num_subscribers,
                                   's' if self.feed.num_subscribers != 1 else '',
-                                  self.feed.permalink
+                                  self.feed.permalink,
+                                  self.feed.fake_user_agent,
                               ),
                 'Connection': 'close',
                 'Accept': 'image/png,image/x-icon,image/*;q=0.9,*/*;q=0.8'
diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py
index cc6764a3a..1c9e32c2f 100644
--- a/apps/rss_feeds/models.py
+++ b/apps/rss_feeds/models.py
@@ -1117,20 +1117,20 @@ class Feed(models.Model):
             # A known workaround is using facebook's user agent.
             return 'facebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)'
 
-        ua = ('NewsBlur Feed Fetcher - %s subscriber%s - %s '
-              '(Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) '
-              'AppleWebKit/537.36 (KHTML, like Gecko) '
-              'Chrome/56.0.2924.87 Safari/537.36)' % (
+        ua = ('NewsBlur Feed Fetcher - %s subscriber%s - %s %s' % (
                    self.num_subscribers,
                    's' if self.num_subscribers != 1 else '',
                    self.permalink,
+                   self.fake_user_agent,
               ))
 
         return ua
     
     @property
     def fake_user_agent(self):
-        ua = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0"
+        ua = ('("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) '
+              'AppleWebKit/605.1.15 (KHTML, like Gecko) '
+              'Version/14.0.1 Safari/605.1.15")')
         
         return ua
     
diff --git a/apps/rss_feeds/page_importer.py b/apps/rss_feeds/page_importer.py
index f25383f50..911b85bae 100644
--- a/apps/rss_feeds/page_importer.py
+++ b/apps/rss_feeds/page_importer.py
@@ -51,13 +51,11 @@ class PageImporter(object):
     @property
     def headers(self):
         return {
-            'User-Agent': 'NewsBlur Page Fetcher - %s subscriber%s - %s '
-                          '(Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) '
-                          'AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 '
-                          'Safari/534.48.3)' % (
+            'User-Agent': 'NewsBlur Page Fetcher - %s subscriber%s - %s %s' % (
                 self.feed.num_subscribers,
                 's' if self.feed.num_subscribers != 1 else '',
                 self.feed.permalink,
+                self.feed.fake_user_agent,
             ),
         }
     
diff --git a/apps/rss_feeds/text_importer.py b/apps/rss_feeds/text_importer.py
index 7fd1b8ea9..95d47c0df 100644
--- a/apps/rss_feeds/text_importer.py
+++ b/apps/rss_feeds/text_importer.py
@@ -37,13 +37,11 @@ class TextImporter:
     def headers(self):
         num_subscribers = getattr(self.feed, 'num_subscribers', 0)
         return {
-            'User-Agent': 'NewsBlur Content Fetcher - %s subscriber%s - %s '
-                          '(Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) '
-                          'AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 '
-                          'Safari/534.48.3)' % (
+            'User-Agent': 'NewsBlur Content Fetcher - %s subscriber%s - %s %s' % (
                               num_subscribers,
                               's' if num_subscribers != 1 else '',
-                              getattr(self.feed, 'permalink', '')
+                              getattr(self.feed, 'permalink', ''),
+                              self.feed.fake_user_agent,
                           ),
         }
 
diff --git a/apps/social/models.py b/apps/social/models.py
index 476c1f8b1..90f1acf31 100644
--- a/apps/social/models.py
+++ b/apps/social/models.py
@@ -2312,10 +2312,7 @@ class MSharedStory(mongo.DynamicDocument):
             return self.image_sizes
             
         headers = {
-            'User-Agent': 'NewsBlur Image Fetcher - %s '
-                          '(Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) '
-                          'AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 '
-                          'Safari/534.48.3)' % (
+            'User-Agent': 'NewsBlur Image Fetcher - %s' % (
                 settings.NEWSBLUR_URL
             ),
         }