From 7c8f85c24a73b61b855c32a6f50c46a424b610d5 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Tue, 9 Feb 2016 17:14:59 -0800 Subject: [PATCH] Newsletter icons now work. Time to try it out on prod. --- apps/newsletters/models.py | 14 +++++--- apps/newsletters/views.py | 62 ++++++++++++++++----------------- apps/rss_feeds/icon_importer.py | 10 ++++-- apps/rss_feeds/models.py | 12 +++---- 4 files changed, 55 insertions(+), 43 deletions(-) diff --git a/apps/newsletters/models.py b/apps/newsletters/models.py index 1ad8930cf..4ac105ab6 100644 --- a/apps/newsletters/models.py +++ b/apps/newsletters/models.py @@ -15,7 +15,7 @@ class EmailNewsletter: if not user: return - sender_name, sender_domain = params['sender'].split('@') + sender_name, sender_domain = self.split_sender(params['from']) feed_address = self.feed_address(user, params['sender']) usf = UserSubscriptionFolders.objects.get(user=user) @@ -25,7 +25,7 @@ class EmailNewsletter: feed = Feed.objects.get(feed_address=feed_address) except Feed.DoesNotExist: feed = Feed.objects.create(feed_address=feed_address, - feed_link=sender_domain, + feed_link='http://' + sender_domain, feed_title=sender_name, fetched_once=True, known_good=True) @@ -46,7 +46,7 @@ class EmailNewsletter: "story_date": datetime.datetime.fromtimestamp(int(params['timestamp'])), "story_title": params['subject'], "story_content": params['body-html'], - "story_author_name": sender_name, + "story_author_name": params['from'], "story_permalink": reverse('newsletter-story', kwargs={'story_hash': story_hash}), "story_guid": params['signature'], @@ -79,4 +79,10 @@ class EmailNewsletter: def feed_address(self, user, sender): return 'newsletter:%s:%s' % (user.pk, sender) - \ No newline at end of file + + def split_sender(self, sender): + tokens = re.search('(.*?) <(.*?)@(.*?)>', sender) + if not tokens: + return params['sender'].split('@') + + return tokens.group(0), tokens.group(2) \ No newline at end of file diff --git a/apps/newsletters/views.py b/apps/newsletters/views.py index bc10f9121..a988f985f 100644 --- a/apps/newsletters/views.py +++ b/apps/newsletters/views.py @@ -4,40 +4,40 @@ from apps.newsletters.models import EmailNewsletter from apps.rss_feeds.models import Feed, MStory def newsletter_receive(request): - params = { - 'stripped-signature':'Thanks,\nBob', - 'From':'Bob ', - 'attachment-count':'2', - 'To':'Alice ', - 'subject':'Re: Sample POST request', - 'from':'Bob ', - 'User-Agent':'Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130308 Thunderbird/17.0.4', - 'stripped-html':'\n
\n
Hi Alice,
\n

\n
This is Bob. 
\n

\n I also attached a file.

\n
Thanks,
\n
Bob
\n

\n
', - 'In-Reply-To':'<517AC78B.5060404@newsletters.newsblur.com>', - 'Date':'Fri, 26 Apr 2013 11:50:29 -0700', - 'Message-Id':'<517ACC75.5010709@newsletters.newsblur.com>', - 'body-plain':'Hi Alice,\n\nThis is Bob.\n\nI also attached a file.\n\nThanks,\nBob\n\nOn 04/26/2013 11:29 AM, Alice wrote:\n> Hi Bob,\n>\n> This is Alice. How are you doing?\n>\n> Thanks,\n> Alice\n\n', - 'Mime-Version':'1.0', - 'Received':'from [10.20.76.69] (Unknown [50.56.129.169]) by mxa.mailgun.org with ESMTP id 517acc75.4b341f0-worker2; Fri, 26 Apr 2013 18:50:29 -0000 (UTC)', - 'content-id-map':'{"": "attachment-1"}', - 'Sender':'bob@newsletters.newsblur.com', - 'timestamp':'1455054490', - 'message-headers':'[["Received", "by luna.mailgun.net with SMTP mgrt 8788212249833; Fri, 26 Apr 2013 18:50:30 +0000"], ["Received", "from [10.20.76.69] (Unknown [50.56.129.169]) by mxa.mailgun.org with ESMTP id 517acc75.4b341f0-worker2; Fri, 26 Apr 2013 18:50:29 -0000 (UTC)"], ["Message-Id", "<517ACC75.5010709@newsletters.newsblur.com>"], ["Date", "Fri, 26 Apr 2013 11:50:29 -0700"], ["From", "Bob "], ["User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130308 Thunderbird/17.0.4"], ["Mime-Version", "1.0"], ["To", "Alice "], ["Subject", "Re: Sample POST request"], ["References", "<517AC78B.5060404@newsletters.newsblur.com>"], ["In-Reply-To", "<517AC78B.5060404@newsletters.newsblur.com>"], ["X-Mailgun-Variables", "{\\"my_var_1\\": \\"Mailgun Variable #1\\", \\"my-var-2\\": \\"awesome\\"}"], ["Content-Type", "multipart/mixed; boundary=\\"------------020601070403020003080006\\""], ["Sender", "bob@newsletters.newsblur.com"]]', - 'stripped-text':'Hi Alice,\n\nThis is Bob.\n\nI also attached a file.', - 'recipient':'alice+555551235342@newsletters.newsblur.com', - 'sender':'bob@newsletters.newsblur.com', - 'X-Mailgun-Variables':'{"my_var_1": "Mailgun Variable #1", "my-var-2": "awesome"}', - 'token':'cb2ef40ca2fee03a099f7da78ca07384228f00f023026c77a4', - 'body-html':'\n \n \n \n \n
\n
Hi Alice,
\n

\n
\n
This is Bob. 
\n

\n I also attached a file.
\n
\n
\n
Thanks,
\n
Bob
\n
\n On 04/26/2013 11:29 AM, Alice wrote:
\n
\n
Hi\n Bob,\n
\n
\n This is Alice. How are you doing?\n
\n
\n Thanks,\n
\n Alice\n
\n
\n
\n \n\n', - 'References':'<517AC78B.5060404@newsletters.newsblur.com>', - 'signature':'0369fa4dcc7de7fac51f5bb408bd5c9daa8730e80d394e8a128658d74e669049', - 'Content-Type':'multipart/mixed; boundary="------------020601070403020003080006"', - 'Subject':'Re: Sample POST request' - } + # params = { + # 'stripped-signature':'Thanks,\nBob', + # 'From':'Test mailer ', + # 'attachment-count':'2', + # 'To':'Alice ', + # 'subject':'Test Newsletter #2', + # 'from':'Test mailer ', + # 'User-Agent':'Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130308 Thunderbird/17.0.4', + # 'stripped-html':'\n
\n
Hi Alice,
\n

\n
This is Bob. 
\n

\n I also attached a file.

\n
Thanks,
\n
Bob
\n

\n
', + # 'In-Reply-To':'<517AC78B.5060404@newsletters.newsblur.com>', + # 'Date':'Fri, 26 Apr 2013 11:50:29 -0700', + # 'Message-Id':'<517ACC75.5010709@newsletters.newsblur.com>', + # 'body-plain':'Hi Alice,\n\nThis is Bob.\n\nI also attached a file.\n\nThanks,\nBob\n\nOn 04/26/2013 11:29 AM, Alice wrote:\n> Hi Bob,\n>\n> This is Alice. How are you doing?\n>\n> Thanks,\n> Alice\n\n', + # 'Mime-Version':'1.0', + # 'Received':'from [10.20.76.69] (Unknown [50.56.129.169]) by mxa.mailgun.org with ESMTP id 517acc75.4b341f0-worker2; Fri, 26 Apr 2013 18:50:29 -0000 (UTC)', + # 'content-id-map':'{"": "attachment-1"}', + # 'Sender':'bob@newsletters.newsblur.com', + # 'timestamp':'1455054990', + # 'message-headers':'[["Received", "by luna.mailgun.net with SMTP mgrt 8788212249833; Fri, 26 Apr 2013 18:50:30 +0000"], ["Received", "from [10.20.76.69] (Unknown [50.56.129.169]) by mxa.mailgun.org with ESMTP id 517acc75.4b341f0-worker2; Fri, 26 Apr 2013 18:50:29 -0000 (UTC)"], ["Message-Id", "<517ACC75.5010709@newsletters.newsblur.com>"], ["Date", "Fri, 26 Apr 2013 11:50:29 -0700"], ["From", "Test mailer "], ["User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130308 Thunderbird/17.0.4"], ["Mime-Version", "1.0"], ["To", "Alice "], ["Subject", "Re: Sample POST request"], ["References", "<517AC78B.5060404@newsletters.newsblur.com>"], ["In-Reply-To", "<517AC78B.5060404@newsletters.newsblur.com>"], ["X-Mailgun-Variables", "{\\"my_var_1\\": \\"Mailgun Variable #1\\", \\"my-var-2\\": \\"awesome\\"}"], ["Content-Type", "multipart/mixed; boundary=\\"------------020601070403020003080006\\""], ["Sender", "bob@newsletters.newsblur.com"]]', + # 'stripped-text':'Hi Alice,\n\nThis is Bob.\n\nI also attached a file.', + # 'recipient':'alice+555551235342@newsletters.newsblur.com', + # 'sender':'samuel@ofbrooklyn.com', + # 'X-Mailgun-Variables':'{"my_var_1": "Mailgun Variable #1", "my-var-2": "awesome"}', + # 'token':'cb2ef40ca2fee03a099f7da78ca07384228f00f023026c77a4', + # 'body-html':'\n \n \n \n \n
\n
Hi Alice,
\n

\n
\n
This is Bob. 
\n

\n I also attached a file.
\n
\n
\n
Thanks,
\n
Bob
\n
\n On 04/26/2013 11:29 AM, Alice wrote:
\n
\n
Hi\n Bob,\n
\n
\n This is Alice. How are you doing?\n
\n
\n Thanks,\n
\n Alice\n
\n
\n
\n \n\n', + # 'References':'<517AC78B.5060404@newsletters.newsblur.com>', + # 'signature':'1369fa4dcc7de7fac51f5bb408bd5c9daa8730e80d394e8a128658d74e669049', + # 'Content-Type':'multipart/mixed; boundary="------------020601070403020003080006"', + # 'Subject':'Test Newsletter #1' + # } response = HttpResponse('OK') email_newsletter = EmailNewsletter() - story = email_newsletter.receive_newsletter(params) + story = email_newsletter.receive_newsletter(request.REQUEST) if not story: raise Http404 diff --git a/apps/rss_feeds/icon_importer.py b/apps/rss_feeds/icon_importer.py index 86b1b641f..db0110814 100644 --- a/apps/rss_feeds/icon_importer.py +++ b/apps/rss_feeds/icon_importer.py @@ -204,7 +204,7 @@ class IconImporter(object): url = self._url_from_html(content) if not url: try: - content = requests.get(self.feed.feed_link).content + content = requests.get(self.cleaned_feed_link).content url = self._url_from_html(content) except (AttributeError, SocketError, requests.ConnectionError, requests.models.MissingSchema, requests.sessions.InvalidSchema, @@ -217,7 +217,13 @@ class IconImporter(object): if url: image, image_file = self.get_image_from_url(url) return image, image_file, url - + + @property + def cleaned_feed_link(self): + if self.feed.feed_link.startswith('http'): + return self.feed.feed_link + return 'http://' + self.feed.feed_link + def fetch_image_from_path(self, path='favicon.ico', force=False): image = None url = None diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index 6f911b403..67ff5aa4a 100644 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -1003,9 +1003,6 @@ class Feed(models.Model): r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL) original_feed_id = int(self.pk) - if self.is_newsletter: - return self.update_newsletter_icon() - if getattr(settings, 'TEST_DEBUG', False): original_feed_address = self.feed_address original_feed_link = self.feed_link @@ -1029,9 +1026,12 @@ class Feed(models.Model): 'fpf': kwargs.get('fpf'), 'feed_xml': kwargs.get('feed_xml'), } - disp = feed_fetcher.Dispatcher(options, 1) - disp.add_jobs([[self.pk]]) - feed = disp.run_jobs() + if self.is_newsletter: + feed = self.update_newsletter_icon() + else: + disp = feed_fetcher.Dispatcher(options, 1) + disp.add_jobs([[self.pk]]) + feed = disp.run_jobs() if feed: feed = Feed.get_by_id(feed.pk)