NewsBlur-viq/apps/newsletters/models.py

import datetime
import re

import redis
from django.conf import settings
from django.contrib.sites.models import Site
from django.core.mail import EmailMultiAlternatives
from django.template.loader import render_to_string
from django.urls import reverse
from django.utils.html import linebreaks

from apps.notifications.models import MUserFeedNotification
from apps.notifications.tasks import QueueNotifications
from apps.profile.models import MSentEmail, Profile
from apps.reader.models import UserSubscription, UserSubscriptionFolders
from apps.rss_feeds.models import Feed, MFetchHistory, MStory
from utils import log as logging
from utils.scrubber import Scrubber
from utils.story_functions import linkify


class EmailNewsletter:
    def receive_newsletter(self, params):
        user = self._user_from_email(params["recipient"])
        if not user:
            return

        sender_name, sender_username, sender_domain = self._split_sender(params["from"])
        feed_address = self._feed_address(user, "%s@%s" % (sender_username, sender_domain))

        try:
            usf = UserSubscriptionFolders.objects.get(user=user)
        except UserSubscriptionFolders.DoesNotExist:
            logging.user(user, "~FRUser does not have a USF, ignoring newsletter.")
            return
        usf.add_folder("", "Newsletters")

        # First look for the email address
        try:
            feed = Feed.objects.get(feed_address=feed_address)
        except Feed.MultipleObjectsReturned:
            feeds = Feed.objects.filter(feed_address=feed_address)[:1]
            if feeds.count():
                feed = feeds[0]
        except Feed.DoesNotExist:
            feed = None

        # If not found, check among titles user has subscribed to
        if not feed:
            newsletter_subs = UserSubscription.objects.filter(
                user=user, feed__feed_address__contains="newsletter:"
            ).only("feed")
            newsletter_feed_ids = [us.feed.pk for us in newsletter_subs]
            feeds = Feed.objects.filter(feed_title__iexact=sender_name, pk__in=newsletter_feed_ids)
            if feeds.count():
                feed = feeds[0]

        # Create a new feed if it doesn't exist by sender name or email
        if not feed:
            feed = Feed.objects.create(
                feed_address=feed_address,
                feed_link="http://" + sender_domain,
                feed_title=sender_name,
                fetched_once=True,
                known_good=True,
            )
            feed.update()
            logging.user(user, "~FCCreating newsletter feed: ~SB%s" % (feed))
            r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
            r.publish(user.username, "reload:%s" % feed.pk)
            self._check_if_first_newsletter(user)

        feed.last_update = datetime.datetime.now()
        feed.last_story_date = datetime.datetime.now()
        feed.save()

        if feed.feed_title != sender_name:
            feed.feed_title = sender_name
            feed.save()

        try:
            usersub = UserSubscription.objects.get(user=user, feed=feed)
        except UserSubscription.DoesNotExist:
            _, _, usersub = UserSubscription.add_subscription(
                user=user, feed_address=feed_address, folder="Newsletters"
            )
            r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
            r.publish(user.username, "reload:feeds")

        story_hash = MStory.ensure_story_hash(params["signature"], feed.pk)
        story_content = self._get_content(params)
        plain_story_content = self._get_content(params, force_plain=True)
        if len(plain_story_content) > len(story_content):
            story_content = plain_story_content
        story_content = self._clean_content(story_content)
        story_params = {
            "story_feed_id": feed.pk,
            "story_date": datetime.datetime.fromtimestamp(int(params["timestamp"])),
            "story_title": params["subject"],
            "story_content": story_content,
            "story_author_name": params["from"],
            "story_permalink": "https://%s%s"
            % (
                Site.objects.get_current().domain,
                reverse("newsletter-story", kwargs={"story_hash": story_hash}),
            ),
            "story_guid": params["signature"],
        }

        try:
            story = MStory.objects.get(story_hash=story_hash)
        except MStory.DoesNotExist:
            story = MStory(**story_params)
            story.save()

        usersub.needs_unread_recalc = True
        usersub.save()

        self._publish_to_subscribers(feed, story.story_hash)

        MFetchHistory.add(feed_id=feed.pk, fetch_type="push")
        logging.user(user, "~FCNewsletter feed story: ~SB%s~SN / ~SB%s" % (story.story_title, feed))

        return story

    def _check_if_first_newsletter(self, user, force=False):
        if not user.email:
            return

        subs = UserSubscription.objects.filter(user=user)
        found_newsletter = False
        for sub in subs:
            if sub.feed.is_newsletter:
                found_newsletter = True
                break
        if not found_newsletter and not force:
            return

        params = dict(receiver_user_id=user.pk, email_type="first_newsletter")
        try:
            MSentEmail.objects.get(**params)
            if not force:
                # Return if email already sent
                return
        except MSentEmail.DoesNotExist:
            MSentEmail.objects.create(**params)

        text = render_to_string("mail/email_first_newsletter.txt", {})
        html = render_to_string("mail/email_first_newsletter.xhtml", {})
        subject = "Your email newsletters are now being sent to NewsBlur"
        msg = EmailMultiAlternatives(
            subject,
            text,
            from_email="NewsBlur <%s>" % settings.HELLO_EMAIL,
            to=["%s <%s>" % (user, user.email)],
        )
        msg.attach_alternative(html, "text/html")
        msg.send()

        logging.user(user, "~BB~FM~SBSending first newsletter email to: %s" % user.email)

    def _user_from_email(self, email):
        tokens = re.search("(\w+)[\+\-\.](\w+)@newsletters.newsblur.com", email)
        if not tokens:
            return

        username, secret_token = tokens.groups()
        try:
            profiles = Profile.objects.filter(secret_token=secret_token)
            if not profiles:
                return
            profile = profiles[0]
        except Profile.DoesNotExist:
            return

        return profile.user

    def _feed_address(self, user, sender_email):
        return "newsletter:%s:%s" % (user.pk, sender_email)

    def _split_sender(self, sender):
        tokens = re.search("(.*?) <(.*?)@(.*?)>", sender)

        if not tokens:
            name, domain = sender.split("@")
            return name, sender, domain

        sender_name, sender_username, sender_domain = tokens.group(1), tokens.group(2), tokens.group(3)
        sender_name = sender_name.replace('"', "")

        return sender_name, sender_username, sender_domain

    def _get_content(self, params, force_plain=False):
        if "body-enriched" in params and not force_plain:
            return params["body-enriched"]
        if "body-html" in params and not force_plain:
            return params["body-html"]
        if "stripped-html" in params and not force_plain:
            return params["stripped-html"]
        if "body-plain" in params:
            return linkify(linebreaks(params["body-plain"]))

        if force_plain:
            return self._get_content(params, force_plain=False)

    def _clean_content(self, content):
        original = content
        scrubber = Scrubber()
        content = scrubber.scrub(content)
        if len(content) < len(original) * 0.01:
            content = original
        content = content.replace("!important", "")
        return content

    def _publish_to_subscribers(self, feed, story_hash):
        try:
            r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
            listeners_count = r.publish("%s:story" % feed.pk, "story:new:%s" % story_hash)
            if listeners_count:
                logging.debug(
                    "   ---> [%-30s] ~FMPublished to %s subscribers" % (feed.log_title[:30], listeners_count)
                )
        except redis.ConnectionError:
            logging.debug("   ***> [%-30s] ~BMRedis is unavailable for real-time." % (feed.log_title[:30],))

        if MUserFeedNotification.feed_has_users(feed.pk) > 0:
            QueueNotifications.delay(feed.pk, 1)