NewsBlur-viq/apps/notifications/models.py

import datetime
import enum
import html
import re
import urllib.parse

import mongoengine as mongo
import redis
from apns2.client import APNsClient
from apns2.errors import BadDeviceToken, DeviceTokenNotForTopic, Unregistered
from apns2.payload import Payload
from bs4 import BeautifulSoup
from django.conf import settings
from django.contrib.auth.models import User
from django.contrib.sites.models import Site
from django.core.mail import EmailMultiAlternatives
from django.template.loader import render_to_string

from apps.analyzer.models import (
    MClassifierAuthor,
    MClassifierFeed,
    MClassifierTag,
    MClassifierTitle,
    compute_story_score,
)
from apps.reader.models import UserSubscription

# from django.utils.html import strip_tags
from apps.rss_feeds.models import Feed, MStory
from utils import log as logging
from utils import mongoengine_fields
from utils.story_functions import truncate_chars
from utils.view_functions import is_true


class NotificationFrequency(enum.Enum):
    immediately = 1
    hour_1 = 2
    hour_6 = 3
    hour_12 = 4
    hour_24 = 5


class MUserNotificationTokens(mongo.Document):
    """A user's push notification tokens"""

    user_id = mongo.IntField()
    ios_tokens = mongo.ListField(mongo.StringField(max_length=1024))
    use_sandbox = mongo.BooleanField(default=False)

    meta = {
        "collection": "notification_tokens",
        "indexes": [
            {
                "fields": ["user_id"],
                "unique": True,
            }
        ],
        "allow_inheritance": False,
    }

    @classmethod
    def get_tokens_for_user(cls, user_id):
        try:
            tokens = cls.objects.get(user_id=user_id)
        except cls.DoesNotExist:
            tokens = cls.objects.create(user_id=user_id)

        return tokens


class MUserFeedNotification(mongo.Document):
    """A user's notifications of a single feed."""

    user_id = mongo.IntField()
    feed_id = mongo.IntField()
    frequency = mongoengine_fields.IntEnumField(NotificationFrequency)
    is_focus = mongo.BooleanField()
    last_notification_date = mongo.DateTimeField(default=datetime.datetime.now)
    is_email = mongo.BooleanField()
    is_web = mongo.BooleanField()
    is_ios = mongo.BooleanField()
    is_android = mongo.BooleanField()
    ios_tokens = mongo.ListField(mongo.StringField(max_length=1024))

    meta = {
        "collection": "notifications",
        "indexes": [
            "feed_id",
            {
                "fields": ["user_id", "feed_id"],
                "unique": True,
            },
        ],
        "allow_inheritance": False,
    }

    def __str__(self):
        notification_types = []
        if self.is_email:
            notification_types.append("email")
        if self.is_web:
            notification_types.append("web")
        if self.is_ios:
            notification_types.append("ios")
        if self.is_android:
            notification_types.append("android")

        return "%s/%s: %s -> %s" % (
            User.objects.get(pk=self.user_id).username,
            Feed.get_by_id(self.feed_id),
            ",".join(notification_types),
            self.last_notification_date,
        )

    @classmethod
    def feed_has_users(cls, feed_id):
        return cls.users_for_feed(feed_id).count()

    @classmethod
    def users_for_feed(cls, feed_id):
        notifications = cls.objects.filter(feed_id=feed_id)

        return notifications

    @classmethod
    def feeds_for_user(cls, user_id):
        notifications = cls.objects.filter(user_id=user_id)
        notifications_by_feed = {}

        for feed in notifications:
            notifications_by_feed[feed.feed_id] = {
                "notification_types": [],
                "notification_filter": "focus" if feed.is_focus else "unread",
            }
            if feed.is_email:
                notifications_by_feed[feed.feed_id]["notification_types"].append("email")
            if feed.is_web:
                notifications_by_feed[feed.feed_id]["notification_types"].append("web")
            if feed.is_ios:
                notifications_by_feed[feed.feed_id]["notification_types"].append("ios")
            if feed.is_android:
                notifications_by_feed[feed.feed_id]["notification_types"].append("android")

        return notifications_by_feed

    @classmethod
    def push_feed_notifications(cls, feed_id, new_stories, force=False):
        feed = Feed.get_by_id(feed_id)
        notifications = MUserFeedNotification.users_for_feed(feed.pk)
        logging.debug(
            "   ---> [%-30s] ~FCPushing out notifications to ~SB%s users~SN for ~FB~SB%s stories"
            % (feed, len(notifications), new_stories)
        )
        r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)

        latest_story_hashes = r.zrange("zF:%s" % feed.pk, -1 * new_stories, -1)
        mstories = MStory.objects.filter(story_hash__in=latest_story_hashes).order_by("-story_date")
        stories = Feed.format_stories(mstories)
        total_sent_count = 0

        for user_feed_notification in notifications:
            sent_count = 0
            try:
                user = User.objects.get(pk=user_feed_notification.user_id)
            except User.DoesNotExist:
                continue
            months_ago = datetime.datetime.now() - datetime.timedelta(days=90)
            if user.profile.last_seen_on < months_ago:
                logging.user(user, f"~FBSkipping notifications, last seen: ~SB{user.profile.last_seen_on}")
                continue
            last_notification_date = user_feed_notification.last_notification_date
            try:
                usersub = UserSubscription.objects.get(
                    user=user_feed_notification.user_id, feed=user_feed_notification.feed_id
                )
            except UserSubscription.DoesNotExist:
                continue
            classifiers = user_feed_notification.classifiers(usersub)

            if classifiers is None:
                if settings.DEBUG:
                    logging.debug("Has no usersubs")
                continue

            for story in stories:
                if sent_count >= 3:
                    if settings.DEBUG:
                        logging.debug("Sent too many, ignoring...")
                    continue
                if story["story_date"] <= last_notification_date and not force:
                    if settings.DEBUG:
                        logging.debug(
                            "Story date older than last notification date: %s <= %s"
                            % (story["story_date"], last_notification_date)
                        )
                    continue

                if story["story_date"] > user_feed_notification.last_notification_date:
                    user_feed_notification.last_notification_date = story["story_date"]
                    user_feed_notification.save()

                story["story_content"] = html.unescape(story["story_content"])

                sent = user_feed_notification.push_story_notification(story, classifiers, usersub)
                if sent:
                    sent_count += 1
                    total_sent_count += 1
        return total_sent_count, len(notifications)

    def classifiers(self, usersub):
        classifiers = {}
        if usersub.is_trained:
            classifiers["feeds"] = list(
                MClassifierFeed.objects(user_id=self.user_id, feed_id=self.feed_id, social_user_id=0)
            )
            classifiers["authors"] = list(
                MClassifierAuthor.objects(user_id=self.user_id, feed_id=self.feed_id)
            )
            classifiers["titles"] = list(MClassifierTitle.objects(user_id=self.user_id, feed_id=self.feed_id))
            classifiers["tags"] = list(MClassifierTag.objects(user_id=self.user_id, feed_id=self.feed_id))

        return classifiers

    def title_and_body(self, story, usersub, notification_title_only=False):
        def replace_with_newlines(element):
            text = ""
            for elem in element.recursiveChildGenerator():
                if isinstance(elem, (str,)):
                    text += elem
                elif elem.name == "br":
                    text += "\n"
                elif elem.name == "p":
                    text += "\n\n"
            text = re.sub(r" +", " ", text).strip()
            return text

        feed_title = usersub.user_title or usersub.feed.feed_title
        # title = "%s: %s" % (feed_title, story['story_title'])
        title = feed_title
        soup = BeautifulSoup(story["story_content"].strip(), features="lxml")
        # if notification_title_only:
        subtitle = None
        body_title = html.unescape(story["story_title"]).strip()
        body_content = replace_with_newlines(soup)
        if body_content:
            if body_title == body_content[: len(body_title)] or body_content[:100] == body_title[:100]:
                body_content = ""
            else:
                body_content = f"\n※ {body_content}"
        body = f"{body_title}{body_content}"
        # else:
        #     subtitle = html.unescape(story['story_title'])
        #     body = replace_with_newlines(soup)
        body = truncate_chars(body.strip(), 3600)
        if not body:
            body = " "

        if not usersub.user.profile.is_premium:
            body = "Please upgrade to a premium subscription to receive full push notifications."

        return title, subtitle, body

    def push_story_notification(self, story, classifiers, usersub):
        story_score = self.story_score(story, classifiers)
        if self.is_focus and story_score <= 0:
            if settings.DEBUG:
                logging.debug("Is focus, but story is hidden")
            return False
        elif story_score < 0:
            if settings.DEBUG:
                logging.debug("Is unread, but story is hidden")
            return False

        user = User.objects.get(pk=self.user_id)
        logging.user(
            user,
            "~FCSending push notification: %s/%s (score: %s)"
            % (story["story_title"][:40], story["story_hash"], story_score),
        )

        self.send_web(story, user)
        self.send_ios(story, user, usersub)
        self.send_android(story)
        self.send_email(story, usersub)

        return True

    def send_web(self, story, user):
        if not self.is_web:
            return

        r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
        r.publish(user.username, "notification:%s,%s" % (story["story_hash"], story["story_title"]))

    def send_ios(self, story, user, usersub):
        if not self.is_ios:
            return

        tokens = MUserNotificationTokens.get_tokens_for_user(self.user_id)
        # To update APNS:
        # 0. Upgrade to latest openssl: brew install openssl
        # 1. Create certificate signing request in Keychain Access
        # 2. Upload to https://developer.apple.com/account/resources/certificates/list
        # 3. Download to secrets/certificates/ios/aps.cer
        # 4. Open in Keychain Access, Under "My Certificates":
        #    - export "Apple Push Service: com.newsblur.NewsBlur" as aps.p12 (or just use aps.cer in #5)
        #    - export private key as aps_key.p12 WITH A PASSPHRASE (removed later)
        # 5. openssl x509 -in aps.cer -inform DER -out aps.pem -outform PEM
        # 6. openssl pkcs12 -in aps_key.p12 -out aps_key.pem -nodes -legacy
        # 7. openssl rsa -out aps_key.noenc.pem -in aps_key.pem
        # 7. cat aps.pem aps_key.noenc.pem > aps.p12.pem
        # 8. Verify: openssl s_client -connect gateway.push.apple.com:2195 -cert aps.p12.pem
        # 9. Deploy: aps -l work -t apns,repo,celery
        apns = APNsClient("/srv/newsblur/config/certificates/aps.p12.pem", use_sandbox=tokens.use_sandbox)

        notification_title_only = is_true(user.profile.preference_value("notification_title_only"))
        title, subtitle, body = self.title_and_body(story, usersub, notification_title_only)
        image_url = None
        if len(story["image_urls"]):
            image_url = story["image_urls"][0]
            # print image_url

        confirmed_ios_tokens = []
        for token in tokens.ios_tokens:
            logging.user(
                user,
                "~BMStory notification by iOS: ~FY~SB%s~SN~BM~FY/~SB%s"
                % (story["story_title"][:50], usersub.feed.feed_title[:50]),
            )
            payload = Payload(
                alert={"title": title, "subtitle": subtitle, "body": body},
                category="STORY_CATEGORY",
                mutable_content=True,
                custom={
                    "story_hash": story["story_hash"],
                    "story_feed_id": story["story_feed_id"],
                    "image_url": image_url,
                },
            )
            try:
                apns.send_notification(token, payload, topic="com.newsblur.NewsBlur")
            except (BadDeviceToken, Unregistered, DeviceTokenNotForTopic):
                logging.user(user, "~BMiOS token expired: ~FR~SB%s" % (token[:50]))
            else:
                confirmed_ios_tokens.append(token)
                if settings.DEBUG:
                    logging.user(
                        user,
                        "~BMiOS token good: ~FB~SB%s / %s" % (token[:50], len(confirmed_ios_tokens)),
                    )

        if len(confirmed_ios_tokens) < len(tokens.ios_tokens):
            tokens.ios_tokens = confirmed_ios_tokens
            tokens.save()

    def send_android(self, story):
        if not self.is_android:
            return

    def send_email(self, story, usersub):
        if not self.is_email:
            return

        # Increment the daily email counter for this user
        r = redis.Redis(connection_pool=settings.REDIS_STATISTICS_POOL)
        emails_sent_date_key = f"emails_sent:{datetime.datetime.now().strftime('%Y%m%d')}"
        r.hincrby(emails_sent_date_key, usersub.user_id, 1)
        r.expire(emails_sent_date_key, 60 * 60 * 24)  # Keep for a day
        count = int(r.hget(emails_sent_date_key, usersub.user_id) or 0)
        if count > settings.MAX_EMAILS_SENT_PER_DAY_PER_USER:
            logging.user(
                usersub.user,
                "~BMSent too many email Story notifications by email: ~FR~SB%s~SN~FR emails" % (count),
            )
            return

        feed = usersub.feed
        story_content = self.sanitize_story(story["story_content"])

        params = {
            "story": story,
            "story_content": story_content,
            "feed": feed,
            "feed_title": usersub.user_title or feed.feed_title,
            "favicon_border": feed.favicon_color,
        }
        from_address = "notifications@newsblur.com"
        to_address = "%s <%s>" % (usersub.user.username, usersub.user.email)
        text = render_to_string("mail/email_story_notification.txt", params)
        html = render_to_string("mail/email_story_notification.xhtml", params)
        subject = "%s: %s" % (usersub.user_title or usersub.feed.feed_title, story["story_title"])
        subject = subject.replace("\n", " ")
        msg = EmailMultiAlternatives(
            subject, text, from_email="NewsBlur <%s>" % from_address, to=[to_address]
        )
        msg.attach_alternative(html, "text/html")
        # try:
        msg.send()
        # except BotoServerError as e:
        #     logging.user(usersub.user, '~BMStory notification by email error: ~FR%s' % e)
        #     return
        logging.user(
            usersub.user,
            "~BMStory notification by email: ~FY~SB%s~SN~BM~FY/~SB%s"
            % (story["story_title"][:50], usersub.feed.feed_title[:50]),
        )

    def sanitize_story(self, story_content):
        soup = BeautifulSoup(story_content.strip(), features="lxml")
        fqdn = Site.objects.get_current().domain

        # Convert videos in newsletters to images
        for iframe in soup("iframe"):
            url = dict(iframe.attrs).get("src", "")
            youtube_id = self.extract_youtube_id(url)
            if youtube_id:
                a = soup.new_tag("a", href=url)
                img = soup.new_tag(
                    "img",
                    style="display: block; 'background-image': \"url(https://%s/img/reader/youtube_play.png), url(http://img.youtube.com/vi/%s/0.jpg)\""
                    % (fqdn, youtube_id),
                    src="http://img.youtube.com/vi/%s/0.jpg" % youtube_id,
                )
                a.insert(0, img)
                iframe.replaceWith(a)
            else:
                iframe.extract()

        return str(soup)

    def extract_youtube_id(self, url):
        youtube_id = None

        if "youtube.com" in url:
            youtube_parts = urllib.parse.urlparse(url)
            if "/embed/" in youtube_parts.path:
                youtube_id = youtube_parts.path.replace("/embed/", "")

        return youtube_id

    def story_score(self, story, classifiers):
        score = compute_story_score(
            story,
            classifier_titles=classifiers.get("titles", []),
            classifier_authors=classifiers.get("authors", []),
            classifier_tags=classifiers.get("tags", []),
            classifier_feeds=classifiers.get("feeds", []),
        )

        return score