NewsBlur/apps/social/models.py

import datetime
import zlib
import hashlib
import redis
import re
import math
import mongoengine as mongo
import random
from collections import defaultdict
# from mongoengine.queryset import OperationError
from django.conf import settings
from django.contrib.auth.models import User
from django.contrib.sites.models import Site
from django.core.urlresolvers import reverse
from django.template.loader import render_to_string
from django.template.defaultfilters import slugify
from django.core.mail import EmailMultiAlternatives
from apps.reader.models import UserSubscription, MUserStory
from apps.analyzer.models import MClassifierFeed, MClassifierAuthor, MClassifierTag, MClassifierTitle
from apps.analyzer.models import apply_classifier_titles, apply_classifier_feeds, apply_classifier_authors, apply_classifier_tags
from apps.rss_feeds.models import Feed, MStory
from apps.profile.models import Profile, MSentEmail
from vendor import facebook
from vendor import tweepy
from utils import log as logging
from utils.feed_functions import relative_timesince
from utils.story_functions import truncate_chars
from utils import json_functions as json

RECOMMENDATIONS_LIMIT = 5

class MRequestInvite(mongo.Document):
    username = mongo.StringField()
    email_sent = mongo.BooleanField()
    
    meta = {
        'collection': 'social_invites',
        'allow_inheritance': False,
    }
    
    def __unicode__(self):
        return "%s%s" % (self.username, '*' if self.email_sent else '')
    
    @classmethod
    def blast(cls):
        invites = cls.objects.filter(email_sent=None)
        print ' ---> Found %s invites...' % invites.count()
        
        for invite in invites:
            try:
                invite.send_email()
            except:
                print ' ***> Could not send invite to: %s. Deleting.' % invite.username
                invite.delete()
        
    def send_email(self):
        user = User.objects.filter(username__iexact=self.username)
        if not user:
            user = User.objects.filter(email__iexact=self.username)
        if user:
            user = user[0]
            email = user.email or self.username
        else:
            user = {
                'username': self.username,
                'profile': {
                    'autologin_url': '/',
                }
            }
            email = self.username
        params = {
            'user': user,
        }
        text    = render_to_string('mail/email_social_beta.txt', params)
        html    = render_to_string('mail/email_social_beta.xhtml', params)
        subject = "Psst, you're in..."
        msg     = EmailMultiAlternatives(subject, text, 
                                         from_email='NewsBlur <%s>' % settings.HELLO_EMAIL,
                                         to=['<%s>' % (email)])
        msg.attach_alternative(html, "text/html")
        msg.send()
        
        self.email_sent = True
        self.save()
                
        logging.debug(" ---> ~BB~FM~SBSending email for social beta: %s" % self.username)

    
class MSocialProfile(mongo.Document):
    user_id              = mongo.IntField(unique=True)
    username             = mongo.StringField(max_length=30, unique=True)
    email                = mongo.StringField()
    bio                  = mongo.StringField(max_length=160)
    blurblog_title       = mongo.StringField(max_length=256)
    custom_bgcolor       = mongo.StringField(max_length=50)
    custom_css           = mongo.StringField()
    photo_url            = mongo.StringField()
    photo_service        = mongo.StringField()
    location             = mongo.StringField(max_length=40)
    website              = mongo.StringField(max_length=200)
    subscription_count   = mongo.IntField(default=0)
    shared_stories_count = mongo.IntField(default=0)
    following_count      = mongo.IntField(default=0)
    follower_count       = mongo.IntField(default=0)
    following_user_ids   = mongo.ListField(mongo.IntField())
    follower_user_ids    = mongo.ListField(mongo.IntField())
    unfollowed_user_ids  = mongo.ListField(mongo.IntField())
    popular_publishers   = mongo.StringField()
    stories_last_month   = mongo.IntField(default=0)
    average_stories_per_month = mongo.IntField(default=0)
    story_count_history  = mongo.ListField()
    feed_classifier_counts = mongo.DictField()
    favicon_color        = mongo.StringField(max_length=6)
    
    meta = {
        'collection': 'social_profile',
        'indexes': ['user_id', 'following_user_ids', 'follower_user_ids', 'unfollowed_user_ids'],
        'allow_inheritance': False,
        'index_drop_dups': True,
    }
    
    def __unicode__(self):
        return "%s [%s] following %s/%s, shared %s" % (self.username, self.user_id, 
                                  self.following_count, self.follower_count, self.shared_stories_count)
    
    def save(self, *args, **kwargs):
        if not self.username:
            self.import_user_fields()
        if not self.subscription_count:
            self.count_follows(skip_save=True)
        if self.bio and len(self.bio) > MSocialProfile.bio.max_length:
            self.bio = self.bio[:80]
        super(MSocialProfile, self).save(*args, **kwargs)
        if self.user_id not in self.following_user_ids:
            self.follow_user(self.user_id)
            self.count_follows()
            
    @property
    def blurblog_url(self):
        return "http://%s.%s" % (
            self.username_slug,
            Site.objects.get_current().domain.replace('www', 'dev'))
    
    def recommended_users(self):
        r = redis.Redis(connection_pool=settings.REDIS_POOL)
        following_key = "F:%s:F" % (self.user_id)
        social_follow_key = "FF:%s:F" % (self.user_id)
        profile_user_ids = []
        
        # Find potential twitter/fb friends
        services = MSocialServices.objects.get(user_id=self.user_id)
        facebook_user_ids = [u.user_id for u in 
                            MSocialServices.objects.filter(facebook_uid__in=services.facebook_friend_ids).only('user_id')]
        twitter_user_ids = [u.user_id for u in 
                            MSocialServices.objects.filter(twitter_uid__in=services.twitter_friend_ids).only('user_id')]
        social_user_ids = facebook_user_ids + twitter_user_ids
        # Find users not currently followed by this user
        r.delete(social_follow_key)
        nonfriend_user_ids = []
        if social_user_ids:
            r.sadd(social_follow_key, *social_user_ids)
            nonfriend_user_ids = r.sdiff(social_follow_key, following_key)
            profile_user_ids = [int(f) for f in nonfriend_user_ids]
            r.delete(social_follow_key)
        
        # Not enough? Grab popular users.
        if len(nonfriend_user_ids) < RECOMMENDATIONS_LIMIT:
            homepage_user = User.objects.get(username=settings.HOMEPAGE_USERNAME)
            suggested_users_list = r.sdiff("F:%s:F" % homepage_user.pk, following_key)
            suggested_users_list = [int(f) for f in suggested_users_list]
            suggested_user_ids = []
            slots_left = min(len(suggested_users_list), RECOMMENDATIONS_LIMIT - len(nonfriend_user_ids))
            for slot in range(slots_left):
                suggested_user_ids.append(random.choice(suggested_users_list))
            profile_user_ids.extend(suggested_user_ids)
        
        # Sort by shared story count
        profiles = MSocialProfile.profiles(profile_user_ids).order_by('-shared_stories_count')
        
        return profiles[:RECOMMENDATIONS_LIMIT]
    
    @property
    def username_slug(self):
        return slugify(self.username)
        
    def count_stories(self):
        # Popular Publishers
        self.save_popular_publishers()
        
    def save_popular_publishers(self, feed_publishers=None):
        if not feed_publishers:
            publishers = defaultdict(int)
            for story in MSharedStory.objects(user_id=self.user_id).only('story_feed_id')[:500]:
                publishers[story.story_feed_id] += 1
            feed_titles = dict((f.id, f.feed_title) 
                               for f in Feed.objects.filter(pk__in=publishers.keys()).only('id', 'feed_title'))
            feed_publishers = sorted([{'id': k, 'feed_title': feed_titles[k], 'story_count': v} 
                                      for k, v in publishers.items()
                                      if k in feed_titles],
                                     key=lambda f: f['story_count'],
                                     reverse=True)[:20]

        popular_publishers = json.encode(feed_publishers)
        if len(popular_publishers) < 1023:
            self.popular_publishers = popular_publishers
            self.save()
            return

        if len(popular_publishers) > 1:
            self.save_popular_publishers(feed_publishers=feed_publishers[:-1])
        
    @classmethod
    def user_statistics(cls, user):
        try:
            profile = cls.objects.get(user_id=user.pk)
        except cls.DoesNotExist:
            return None
        
        values = {
            'followers': profile.follower_count,
            'following': profile.following_count,
            'shared_stories': profile.shared_stories_count,
        }
        return values
        
    @classmethod
    def profile(cls, user_id):
        try:
            profile = cls.objects.get(user_id=user_id)
        except cls.DoesNotExist:
            return {}
        return profile.to_json(include_follows=True)
        
    @classmethod
    def profiles(cls, user_ids):
        profiles = cls.objects.filter(user_id__in=user_ids)
        return profiles

    @classmethod
    def profile_feeds(cls, user_ids):
        profiles = cls.objects.filter(user_id__in=user_ids)
        profiles = dict((p.user_id, p.feed()) for p in profiles)
        return profiles
        
    @classmethod
    def sync_all_redis(cls):
        for profile in cls.objects.all():
            profile.sync_redis()
    
    def sync_redis(self):
        for user_id in self.following_user_ids:
            self.follow_user(user_id)
        
        self.follow_user(self.user_id)
    
    @property
    def title(self):
        return self.blurblog_title if self.blurblog_title else self.username + "'s blurblog"
        
    def feed(self):
        params = self.to_json(compact=True)
        params.update({
            'feed_title': self.title,
            'page_url': reverse('load-social-page', kwargs={'user_id': self.user_id, 'username': self.username_slug}),
            'shared_stories_count': self.shared_stories_count,
        })
        return params
        
    def page(self):
        params = self.to_json(include_follows=True)
        params.update({
            'feed_title': self.title,
            'custom_css': self.custom_css,
        })
        return params
    
    @property
    def profile_photo_url(self):
        if self.photo_url:
            return self.photo_url
        return settings.MEDIA_URL + 'img/reader/default_profile_photo.png'
    
    @property
    def large_photo_url(self):
        photo_url = self.email_photo_url
        if 'graph.facebook.com' in photo_url:
            return photo_url + '?type=large'
        elif 'twimg' in photo_url:
            return photo_url.replace('_normal', '')
        return photo_url
            
    @property
    def email_photo_url(self):
        if self.photo_url:
            if self.photo_url.startswith('//'):
                self.photo_url = 'http:' + self.photo_url
            return self.photo_url
        domain = Site.objects.get_current().domain.replace('www', 'dev')
        return 'http://' + domain + settings.MEDIA_URL + 'img/reader/default_profile_photo.png'
        
    def to_json(self, compact=False, include_follows=False, common_follows_with_user=None, include_settings=False):
        # domain = Site.objects.get_current().domain
        domain = Site.objects.get_current().domain.replace('www', 'dev')
        params = {
            'id': 'social:%s' % self.user_id,
            'user_id': self.user_id,
            'username': self.username,
            'photo_url': self.email_photo_url,
            'num_subscribers': self.follower_count,
            'feed_title': self.title,
            'feed_address': "http://%s%s" % (domain, reverse('shared-stories-rss-feed', 
                                    kwargs={'user_id': self.user_id, 'username': self.username_slug})),
            'feed_link': self.blurblog_url,
        }
        if not compact:
            params.update({
                'bio': self.bio,
                'location': self.location,
                'website': self.website,
                'subscription_count': self.subscription_count,
                'shared_stories_count': self.shared_stories_count,
                'following_count': self.following_count,
                'follower_count': self.follower_count,
                'popular_publishers': json.decode(self.popular_publishers),
                'stories_last_month': self.stories_last_month,
                'average_stories_per_month': self.average_stories_per_month,
            })
        if include_settings:
            params.update({
                'custom_css': self.custom_css,
                'custom_bgcolor': self.custom_bgcolor,
            })
        if include_follows:
            params.update({
                'photo_service': self.photo_service,
                'following_user_ids': self.following_user_ids_without_self[:50],
                'follower_user_ids': self.follower_user_ids_without_self[:50],
            })
        if common_follows_with_user:
            with_user, _ = MSocialProfile.objects.get_or_create(user_id=common_follows_with_user)
            followers_youknow, followers_everybody = with_user.common_follows(self.user_id, direction='followers')
            following_youknow, following_everybody = with_user.common_follows(self.user_id, direction='following')
            params['followers_youknow'] = followers_youknow[:50]
            params['followers_everybody'] = followers_everybody[:50]
            params['following_youknow'] = following_youknow[:50]
            params['following_everybody'] = following_everybody[:50]

        return params
    
    @property
    def following_user_ids_without_self(self):
        if self.user_id in self.following_user_ids:
            return [u for u in self.following_user_ids if u != self.user_id]
        return self.following_user_ids
        
    @property
    def follower_user_ids_without_self(self):
        if self.user_id in self.follower_user_ids:
            return [u for u in self.follower_user_ids if u != self.user_id]
        return self.follower_user_ids
        
    def import_user_fields(self, skip_save=False):
        user = User.objects.get(pk=self.user_id)
        self.username = user.username
        self.email = user.email

    def count_follows(self, skip_save=False):
        self.subscription_count = UserSubscription.objects.filter(user__pk=self.user_id).count()
        self.shared_stories_count = MSharedStory.objects.filter(user_id=self.user_id).count()
        self.following_count = len(self.following_user_ids_without_self)
        self.follower_count = len(self.follower_user_ids_without_self)
        if not skip_save:
            self.save()
        
    def follow_user(self, user_id, check_unfollowed=False):
        r = redis.Redis(connection_pool=settings.REDIS_POOL)
        
        if check_unfollowed and user_id in self.unfollowed_user_ids:
            return
            
        if user_id not in self.following_user_ids:
            self.following_user_ids.append(user_id)
            if user_id in self.unfollowed_user_ids:
                self.unfollowed_user_ids.remove(user_id)
            self.count_follows()
            self.save()
            
            if self.user_id == user_id:
                followee = self
            else:
                followee, _ = MSocialProfile.objects.get_or_create(user_id=user_id)
            if self.user_id not in followee.follower_user_ids:
                followee.follower_user_ids.append(self.user_id)
                followee.count_follows()
                followee.save()
        
        following_key = "F:%s:F" % (self.user_id)
        r.sadd(following_key, user_id)
        follower_key = "F:%s:f" % (user_id)
        r.sadd(follower_key, self.user_id)
        
        if self.user_id != user_id:
            MInteraction.new_follow(follower_user_id=self.user_id, followee_user_id=user_id)
            MActivity.new_follow(follower_user_id=self.user_id, followee_user_id=user_id)
        socialsub, _ = MSocialSubscription.objects.get_or_create(user_id=self.user_id, 
                                                                 subscription_user_id=user_id)
        socialsub.needs_unread_recalc = True
        socialsub.save()
        
        from apps.social.tasks import EmailNewFollower
        EmailNewFollower.delay(follower_user_id=self.user_id, followee_user_id=user_id)
    
    def is_following_user(self, user_id):
        return user_id in self.following_user_ids
        
    def unfollow_user(self, user_id):
        r = redis.Redis(connection_pool=settings.REDIS_POOL)
        
        if not isinstance(user_id, int):
            user_id = int(user_id)
        
        if user_id == self.user_id:
            # Only unfollow other people, not yourself.
            return

        if user_id in self.following_user_ids:
            self.following_user_ids.remove(user_id)
        if user_id not in self.unfollowed_user_ids:
            self.unfollowed_user_ids.append(user_id)
        self.count_follows()
        self.save()
        
        followee = MSocialProfile.objects.get(user_id=user_id)
        if self.user_id in followee.follower_user_ids:
            followee.follower_user_ids.remove(self.user_id)
            followee.count_follows()
            followee.save()
        
            following_key = "F:%s:F" % (self.user_id)
            r.srem(following_key, user_id)
            follower_key = "F:%s:f" % (user_id)
            r.srem(follower_key, self.user_id)
        
        MSocialSubscription.objects.get(user_id=self.user_id, subscription_user_id=user_id).delete()
    
    def common_follows(self, user_id, direction='followers'):
        r = redis.Redis(connection_pool=settings.REDIS_POOL)
        
        my_followers    = "F:%s:%s" % (self.user_id, 'F' if direction == 'followers' else 'F')
        their_followers = "F:%s:%s" % (user_id, 'f' if direction == 'followers' else 'F')
        follows_inter   = r.sinter(their_followers, my_followers)
        follows_diff    = r.sdiff(their_followers, my_followers)
        follows_inter   = [int(f) for f in follows_inter]
        follows_diff    = [int(f) for f in follows_diff]
        
        if user_id in follows_inter:
            follows_inter.remove(user_id)
        if user_id in follows_diff:
            follows_diff.remove(user_id)
        
        return follows_inter, follows_diff
    
    def send_email_for_new_follower(self, follower_user_id):
        user = User.objects.get(pk=self.user_id)
        if not user.email or not user.profile.send_emails or self.user_id == follower_user_id:
            return
        
        emails_sent = MSentEmail.objects.filter(receiver_user_id=user.pk,
                                                sending_user_id=follower_user_id,
                                                email_type='new_follower')
        day_ago = datetime.datetime.now() - datetime.timedelta(days=1)
        for email in emails_sent:
            if email.date_sent > day_ago:
                logging.user(user, "~BB~SK~FMNot sending new follower email, already sent before. NBD.")
                return
        
        follower_profile = MSocialProfile.objects.get(user_id=follower_user_id)
        common_followers, _ = self.common_follows(follower_user_id, direction='followers')
        common_followings, _ = self.common_follows(follower_user_id, direction='following')
        if self.user_id in common_followers:
            common_followers.remove(self.user_id)
        if self.user_id in common_followings:
            common_followings.remove(self.user_id)
        common_followers = MSocialProfile.profiles(common_followers)
        common_followings = MSocialProfile.profiles(common_followings)
        
        data = {
            'user': user,
            'follower_profile': follower_profile,
            'common_followers': common_followers,
            'common_followings': common_followings,
        }
        
        text    = render_to_string('mail/email_new_follower.txt', data)
        html    = render_to_string('mail/email_new_follower.xhtml', data)
        subject = "%s is now following your Blurblog on NewsBlur!" % follower_profile.username
        msg     = EmailMultiAlternatives(subject, text, 
                                         from_email='NewsBlur <%s>' % settings.HELLO_EMAIL,
                                         to=['%s <%s>' % (user.username, user.email)])
        msg.attach_alternative(html, "text/html")
        msg.send()
        
        MSentEmail.record(receiver_user_id=user.pk, sending_user_id=follower_user_id,
                          email_type='new_follower')
                
        logging.user(user, "~BB~FM~SBSending email for new follower: %s" % follower_profile.username)
            
    def save_feed_story_history_statistics(self):
        """
        Fills in missing months between earlier occurances and now.
        
        Save format: [('YYYY-MM, #), ...]
        Example output: [(2010-12, 123), (2011-01, 146)]
        """
        now = datetime.datetime.utcnow()
        min_year = now.year
        total = 0
        month_count = 0

        # Count stories, aggregate by year and month. Map Reduce!
        map_f = """
            function() {
                var date = (this.shared_date.getFullYear()) + "-" + (this.shared_date.getMonth()+1);
                emit(date, 1);
            }
        """
        reduce_f = """
            function(key, values) {
                var total = 0;
                for (var i=0; i < values.length; i++) {
                    total += values[i];
                }
                return total;
            }
        """
        dates = {}
        res = MSharedStory.objects(user_id=self.user_id).map_reduce(map_f, reduce_f, output='inline')
        for r in res:
            dates[r.key] = r.value
            year = int(re.findall(r"(\d{4})-\d{1,2}", r.key)[0])
            if year < min_year:
                min_year = year
        
        # Assemble a list with 0's filled in for missing months, 
        # trimming left and right 0's.
        months = []
        start = False
        for year in range(min_year, now.year+1):
            for month in range(1, 12+1):
                if datetime.datetime(year, month, 1) < now:
                    key = u'%s-%s' % (year, month)
                    if dates.get(key) or start:
                        start = True
                        months.append((key, dates.get(key, 0)))
                        total += dates.get(key, 0)
                        month_count += 1

        self.story_count_history = months
        self.average_stories_per_month = total / month_count
        self.save()
    
    def save_classifier_counts(self):
        
        def calculate_scores(cls, facet):
            map_f = """
                function() {
                    emit(this["%s"], {
                        pos: this.score>0 ? this.score : 0, 
                        neg: this.score<0 ? Math.abs(this.score) : 0
                    });
                }
            """ % (facet)
            reduce_f = """
                function(key, values) {
                    var result = {pos: 0, neg: 0};
                    values.forEach(function(value) {
                        result.pos += value.pos;
                        result.neg += value.neg;
                    });
                    return result;
                }
            """
            scores = []
            res = cls.objects(social_user_id=self.user_id).map_reduce(map_f, reduce_f, output='inline')
            for r in res:
                facet_values = dict([(k, int(v)) for k,v in r.value.iteritems()])
                facet_values[facet] = r.key
                scores.append(facet_values)
            scores = sorted(scores, key=lambda v: v['neg'] - v['pos'])

            return scores
        
        scores = {}
        for cls, facet in [(MClassifierTitle, 'title'), 
                           (MClassifierAuthor, 'author'), 
                           (MClassifierTag, 'tag'), 
                           (MClassifierFeed, 'feed_id')]:
            scores[facet] = calculate_scores(cls, facet)
            if facet == 'feed_id' and scores[facet]:
                scores['feed'] = scores[facet]
                del scores['feed_id']
            elif not scores[facet]:
                del scores[facet]
                
        if scores:
            self.feed_classifier_counts = scores
            self.save()

class MSocialSubscription(mongo.Document):
    UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta(days=settings.DAYS_OF_UNREAD)

    user_id = mongo.IntField()
    subscription_user_id = mongo.IntField(unique_with='user_id')
    follow_date = mongo.DateTimeField(default=datetime.datetime.utcnow())
    last_read_date = mongo.DateTimeField(default=UNREAD_CUTOFF)
    mark_read_date = mongo.DateTimeField(default=UNREAD_CUTOFF)
    unread_count_neutral = mongo.IntField(default=0)
    unread_count_positive = mongo.IntField(default=0)
    unread_count_negative = mongo.IntField(default=0)
    unread_count_updated = mongo.DateTimeField()
    oldest_unread_story_date = mongo.DateTimeField()
    needs_unread_recalc = mongo.BooleanField(default=False)
    feed_opens = mongo.IntField(default=0)
    is_trained = mongo.BooleanField(default=False)
    
    meta = {
        'collection': 'social_subscription',
        'indexes': [('user_id', 'subscription_user_id')],
        'allow_inheritance': False,
    }

    def __unicode__(self):
        return "%s:%s" % (self.user_id, self.subscription_user_id)
    
    @classmethod
    def feeds(cls, *args, **kwargs):
        user_id = kwargs['user_id']
        params = dict(user_id=user_id)
        if 'subscription_user_id' in kwargs:
            params["subscription_user_id"] = kwargs["subscription_user_id"]
        social_subs = cls.objects.filter(**params)
        # for sub in social_subs:
        #     sub.calculate_feed_scores()
        social_feeds = []
        if social_subs:
            if kwargs.get('calculate_scores'):
                for s in social_subs: s.calculate_feed_scores()
            social_subs = dict((s.subscription_user_id, s.to_json()) for s in social_subs)
            social_user_ids = social_subs.keys()
            
            # Fetch user profiles of subscriptions
            social_profiles = MSocialProfile.profile_feeds(social_user_ids)
            for user_id, social_sub in social_subs.items():
                # Combine subscription read counts with feed/user info
                feed = dict(social_sub.items() + social_profiles[user_id].items())
                social_feeds.append(feed)

        return social_feeds
    
    @classmethod
    def feeds_with_updated_counts(cls, user, social_feed_ids=None):
        feeds = {}
        
        # Get social subscriptions for user
        user_subs = cls.objects.filter(user_id=user.pk)
        if social_feed_ids:
            social_user_ids = [int(f.replace('social:', '')) for f in social_feed_ids]
            user_subs = user_subs.filter(subscription_user_id__in=social_user_ids)
            profiles = MSocialProfile.objects.filter(user_id__in=social_user_ids)
            profiles = dict((p.user_id, p) for p in profiles)
        
        UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta(days=settings.DAYS_OF_UNREAD)

        for i, sub in enumerate(user_subs):
            # Count unreads if subscription is stale.
            if (sub.needs_unread_recalc or 
                (sub.unread_count_updated and
                 sub.unread_count_updated < UNREAD_CUTOFF) or 
                (sub.oldest_unread_story_date and
                 sub.oldest_unread_story_date < UNREAD_CUTOFF)):
                sub = sub.calculate_feed_scores(silent=True)

            feed_id = "social:%s" % sub.subscription_user_id
            feeds[feed_id] = {
                'ps': sub.unread_count_positive,
                'nt': sub.unread_count_neutral,
                'ng': sub.unread_count_negative,
                'id': feed_id,
            }
            if social_feed_ids and sub.subscription_user_id in profiles:
                feeds[feed_id]['shared_stories_count'] = profiles[sub.subscription_user_id].shared_stories_count

        return feeds
        
    def to_json(self):
        return {
            'user_id': self.user_id,
            'subscription_user_id': self.subscription_user_id,
            'nt': self.unread_count_neutral,
            'ps': self.unread_count_positive,
            'ng': self.unread_count_negative,
            'is_trained': self.is_trained,
            'feed_opens': self.feed_opens,
        }
    
    def mark_story_ids_as_read(self, story_ids, feed_id, request=None):
        data = dict(code=0, payload=story_ids)
        r = redis.Redis(connection_pool=settings.REDIS_POOL)
        
        if not request:
            request = self.user
    
        if not self.needs_unread_recalc:
            self.needs_unread_recalc = True
            self.save()
    
        sub_username = MSocialProfile.objects.get(user_id=self.subscription_user_id).username

        if len(story_ids) > 1:
            logging.user(request, "~FYRead %s stories in social subscription: %s" % (len(story_ids), sub_username))
        else:
            logging.user(request, "~FYRead story in social subscription: %s" % (sub_username))
        
        for story_id in set(story_ids):
            story = MSharedStory.objects.get(user_id=self.subscription_user_id, story_guid=story_id)
            now = datetime.datetime.utcnow()
            date = now if now > story.story_date else story.story_date # For handling future stories
            m = MUserStory(user_id=self.user_id, 
                           feed_id=feed_id, read_date=date, 
                           story_id=story.story_guid, story_date=story.story_date)
            m.save()
            
            # Find other social feeds with this story to update their counts
            friend_key = "F:%s:F" % (self.user_id)
            share_key = "S:%s:%s" % (feed_id, story.guid_hash)
            friends_with_shares = [int(f) for f in r.sinter(share_key, friend_key)]
            if self.user_id in friends_with_shares:
                friends_with_shares.remove(self.user_id)
            if friends_with_shares:
                socialsubs = MSocialSubscription.objects.filter(user_id=self.user_id,
                                                                subscription_user_id__in=friends_with_shares)
                for socialsub in socialsubs:
                    if not socialsub.needs_unread_recalc:
                        socialsub.needs_unread_recalc = True
                        socialsub.save()
                    # XXX TODO: Real-time notification, just for this user
            
            # Also count on original subscription
            usersubs = UserSubscription.objects.filter(user=self.user_id, feed=feed_id)
            if usersubs:
                usersub = usersubs[0]
                if not usersub.needs_unread_recalc:
                    usersub.needs_unread_recalc = True
                    usersub.save()
                # XXX TODO: Real-time notification, just for this user
        return data
        
    def mark_feed_read(self):
        latest_story_date = datetime.datetime.utcnow()
        
        # Use the latest story to get last read time.
        if MSharedStory.objects(user_id=self.subscription_user_id).first():
            latest_story_date = MSharedStory.objects(user_id=self.subscription_user_id)\
                                .order_by('-shared_date').only('shared_date')[0]['shared_date']\
                                + datetime.timedelta(seconds=1)

        self.last_read_date = latest_story_date
        self.mark_read_date = latest_story_date
        self.unread_count_negative = 0
        self.unread_count_positive = 0
        self.unread_count_neutral = 0
        self.unread_count_updated = latest_story_date
        self.oldest_unread_story_date = latest_story_date
        self.needs_unread_recalc = False

        # Cannot delete these stories, since the original feed may not be read. 
        # Just go 2 weeks back.
        # UNREAD_CUTOFF = now - datetime.timedelta(days=settings.DAYS_OF_UNREAD)
        # MUserStory.delete_marked_as_read_stories(self.user_id, self.feed_id, mark_read_date=UNREAD_CUTOFF)
                
        self.save()
    
    def calculate_feed_scores(self, silent=False):
        # if not self.needs_unread_recalc:
        #     return
            
        now = datetime.datetime.now()
        UNREAD_CUTOFF = now - datetime.timedelta(days=settings.DAYS_OF_UNREAD)
        user = User.objects.get(pk=self.user_id)

        if user.profile.last_seen_on < UNREAD_CUTOFF:
            # if not silent:
            #     logging.info(' ---> [%s] SKIPPING Computing scores: %s (1 week+)' % (self.user, self.feed))
            return self
            
        feed_scores = dict(negative=0, neutral=0, positive=0)
        
        # Two weeks in age. If mark_read_date is older, mark old stories as read.
        date_delta = UNREAD_CUTOFF
        if date_delta < self.mark_read_date:
            date_delta = self.mark_read_date
        else:
            self.mark_read_date = date_delta

        stories_db = MSharedStory.objects(user_id=self.subscription_user_id,
                                          shared_date__gte=date_delta)
        story_feed_ids = set()
        story_ids = []
        for s in stories_db:
            story_feed_ids.add(s['story_feed_id'])
            story_ids.append(s['story_guid'])
        story_feed_ids = list(story_feed_ids)
        usersubs = UserSubscription.objects.filter(user__pk=self.user_id, feed__pk__in=story_feed_ids)
        usersubs_map = dict((sub.feed_id, sub) for sub in usersubs)

        # usersubs = UserSubscription.objects.filter(user__pk=user.pk, feed__pk__in=story_feed_ids)
        # usersubs_map = dict((sub.feed_id, sub) for sub in usersubs)
        read_stories_ids = []
        if story_feed_ids:
            read_stories = MUserStory.objects(user_id=self.user_id,
                                              feed_id__in=story_feed_ids,
                                              story_id__in=story_ids)
            read_stories_ids = [rs.story_id for rs in read_stories]

        oldest_unread_story_date = now
        unread_stories_db = []
        for story in stories_db:
            if getattr(story, 'story_guid', None) in read_stories_ids:
                continue
            feed_id = story.story_feed_id
            if usersubs_map.get(feed_id) and story.story_date < usersubs_map[feed_id].mark_read_date:
                continue
                
            unread_stories_db.append(story)
            if story.story_date < oldest_unread_story_date:
                oldest_unread_story_date = story.story_date
        stories = Feed.format_stories(unread_stories_db)
        
        classifier_feeds   = list(MClassifierFeed.objects(user_id=self.user_id, social_user_id=self.subscription_user_id))
        classifier_authors = list(MClassifierAuthor.objects(user_id=self.user_id, social_user_id=self.subscription_user_id))
        classifier_titles  = list(MClassifierTitle.objects(user_id=self.user_id, social_user_id=self.subscription_user_id))
        classifier_tags    = list(MClassifierTag.objects(user_id=self.user_id, social_user_id=self.subscription_user_id))
        # Merge with feed specific classifiers
        if story_feed_ids:
            classifier_feeds   = classifier_feeds + list(MClassifierFeed.objects(user_id=self.user_id,
                                                                                 feed_id__in=story_feed_ids))
            classifier_authors = classifier_authors + list(MClassifierAuthor.objects(user_id=self.user_id,
                                                                                     feed_id__in=story_feed_ids))
            classifier_titles  = classifier_titles + list(MClassifierTitle.objects(user_id=self.user_id,
                                                                                   feed_id__in=story_feed_ids))
            classifier_tags    = classifier_tags + list(MClassifierTag.objects(user_id=self.user_id,
                                                                               feed_id__in=story_feed_ids))

        for story in stories:
            scores = {
                'feed'   : apply_classifier_feeds(classifier_feeds, story['story_feed_id'],
                                                  social_user_id=self.subscription_user_id),
                'author' : apply_classifier_authors(classifier_authors, story),
                'tags'   : apply_classifier_tags(classifier_tags, story),
                'title'  : apply_classifier_titles(classifier_titles, story),
            }
            
            max_score = max(scores['author'], scores['tags'], scores['title'])
            min_score = min(scores['author'], scores['tags'], scores['title'])
            if max_score > 0:
                feed_scores['positive'] += 1
            elif min_score < 0:
                feed_scores['negative'] += 1
            else:
                if scores['feed'] > 0:
                    feed_scores['positive'] += 1
                elif scores['feed'] < 0:
                    feed_scores['negative'] += 1
                else:
                    feed_scores['neutral'] += 1
                
        
        self.unread_count_positive = feed_scores['positive']
        self.unread_count_neutral = feed_scores['neutral']
        self.unread_count_negative = feed_scores['negative']
        self.unread_count_updated = datetime.datetime.now()
        self.oldest_unread_story_date = oldest_unread_story_date
        self.needs_unread_recalc = False
        
        self.save()

        if (self.unread_count_positive == 0 and 
            self.unread_count_neutral == 0 and
            self.unread_count_negative == 0):
            self.mark_feed_read()
        
        if not silent:
            logging.info(' ---> [%s] Computing social scores: %s (%s/%s/%s)' % (user.username, self.subscription_user_id, feed_scores['negative'], feed_scores['neutral'], feed_scores['positive']))
            
        return self
    
    @classmethod
    def mark_dirty_sharing_story(cls, user_id, story_feed_id, story_guid_hash):
        r = redis.Redis(connection_pool=settings.REDIS_POOL)
        
        friends_key = "F:%s:F" % (user_id)
        share_key = "S:%s:%s" % (story_feed_id, story_guid_hash)
        following_user_ids = r.sinter(friends_key, share_key)
        following_user_ids = [int(f) for f in following_user_ids]
        if not following_user_ids:
            return None

        social_subs = cls.objects.filter(user_id=user_id, subscription_user_id__in=following_user_ids)
        for social_sub in social_subs:
            social_sub.needs_unread_recalc = True
            social_sub.save()
        return social_subs

class MCommentReply(mongo.EmbeddedDocument):
    user_id                  = mongo.IntField()
    publish_date             = mongo.DateTimeField()
    comments                 = mongo.StringField()
    
    def to_json(self):
        reply = {
            'user_id': self.user_id,
            'publish_date': relative_timesince(self.publish_date),
            'comments': self.comments,
        }
        return reply
        
    meta = {
        'ordering': ['publish_date'],
    }


class MSharedStory(mongo.Document):
    user_id                  = mongo.IntField()
    shared_date              = mongo.DateTimeField()
    comments                 = mongo.StringField()
    has_comments             = mongo.BooleanField(default=False)
    has_replies              = mongo.BooleanField(default=False)
    replies                  = mongo.ListField(mongo.EmbeddedDocumentField(MCommentReply))
    source_user_id           = mongo.IntField()
    story_feed_id            = mongo.IntField()
    story_date               = mongo.DateTimeField()
    story_title              = mongo.StringField(max_length=1024)
    story_content            = mongo.StringField()
    story_content_z          = mongo.BinaryField()
    story_original_content   = mongo.StringField()
    story_original_content_z = mongo.BinaryField()
    story_content_type       = mongo.StringField(max_length=255)
    story_author_name        = mongo.StringField()
    story_permalink          = mongo.StringField()
    story_guid               = mongo.StringField(unique_with=('user_id',))
    story_tags               = mongo.ListField(mongo.StringField(max_length=250))
    posted_to_services       = mongo.ListField(mongo.StringField(max_length=20))
    mute_email_users         = mongo.ListField(mongo.IntField())
    emailed_reshare          = mongo.BooleanField(default=False)
    
    meta = {
        'collection': 'shared_stories',
        'indexes': [('user_id', '-shared_date'), ('user_id', 'story_feed_id'), 
                    'shared_date', 'story_guid', 'story_feed_id'],
        'index_drop_dups': True,
        'ordering': ['shared_date'],
        'allow_inheritance': False,
    }
    
    @property
    def guid_hash(self):
        return hashlib.sha1(self.story_guid).hexdigest()
        
    def save(self, *args, **kwargs):
        if self.story_content:
            self.story_content_z = zlib.compress(self.story_content)
            self.story_content = None
        if self.story_original_content:
            self.story_original_content_z = zlib.compress(self.story_original_content)
            self.story_original_content = None
        
        r = redis.Redis(connection_pool=settings.REDIS_POOL)
        share_key = "S:%s:%s" % (self.story_feed_id, self.guid_hash)
        r.sadd(share_key, self.user_id)
        comment_key = "C:%s:%s" % (self.story_feed_id, self.guid_hash)
        if self.has_comments:
            r.sadd(comment_key, self.user_id)
        else:
            r.srem(comment_key, self.user_id)
        
        self.shared_date = self.shared_date or datetime.datetime.utcnow()
        self.has_replies = bool(len(self.replies))
        
        super(MSharedStory, self).save(*args, **kwargs)
        
        author, _ = MSocialProfile.objects.get_or_create(user_id=self.user_id)
        author.count_follows()
        
        MActivity.new_shared_story(user_id=self.user_id, story_title=self.story_title, 
                                   comments=self.comments, story_feed_id=self.story_feed_id,
                                   story_id=self.story_guid, share_date=self.shared_date)
        
    def delete(self, *args, **kwargs):
        r = redis.Redis(connection_pool=settings.REDIS_POOL)
        share_key = "S:%s:%s" % (self.story_feed_id, self.guid_hash)
        r.srem(share_key, self.user_id)

        comment_key = "C:%s:%s" % (self.story_feed_id, self.guid_hash)
        r.srem(comment_key, self.user_id)
        
        MActivity.remove_shared_story(user_id=self.user_id, story_feed_id=self.story_feed_id,
                                      story_id=self.story_guid)

        super(MSharedStory, self).delete(*args, **kwargs)
    
    def set_source_user_id(self, source_user_id, original_comments=None):
        if source_user_id == self.user_id:
            return
            
        def find_source(source_user_id, seen_user_ids):
            parent_shared_story = MSharedStory.objects.filter(user_id=source_user_id, 
                                                              story_guid=self.story_guid, 
                                                              story_feed_id=self.story_feed_id).limit(1)
            if parent_shared_story and parent_shared_story[0].source_user_id:
                user_id = parent_shared_story[0].source_user_id
                if user_id in seen_user_ids:
                    return source_user_id
                else:
                    seen_user_ids.append(user_id)
                    return find_source(user_id, seen_user_ids)
            else:
                return source_user_id
        
        if source_user_id:
            source_user_id = find_source(source_user_id, [])
            if not self.source_user_id or source_user_id != self.source_user_id or original_comments:
                self.source_user_id = source_user_id
                logging.debug("   ---> Re-share from %s." % source_user_id)
                self.save()
                
                MInteraction.new_reshared_story(user_id=self.source_user_id,
                                                reshare_user_id=self.user_id,
                                                comments=self.comments,
                                                story_title=self.story_title,
                                                story_feed_id=self.story_feed_id,
                                                story_id=self.story_guid,
                                                original_comments=original_comments)
    
    def mute_for_user(self, user_id):
        if user_id not in self.mute_email_users:
            self.mute_email_users.append(user_id)
            self.save()
        
    @classmethod
    def switch_feed(cls, original_feed_id, duplicate_feed_id):
        shared_stories = cls.objects.filter(story_feed_id=duplicate_feed_id)
        logging.info(" ---> %s shared stories" % shared_stories.count())
        for story in shared_stories:
            story.story_feed_id = original_feed_id
            story.save()
        
    @classmethod
    def collect_popular_stories(cls):
        from apps.statistics.models import MStatistics
        shared_stories_count = sum(json.decode(MStatistics.get('stories_shared')))
        cutoff = max(math.floor(.05 * shared_stories_count), 3)
        today = datetime.datetime.now() - datetime.timedelta(days=1)
        
        map_f = """
            function() {
                emit(this.story_guid, {
                    'guid': this.story_guid, 
                    'feed_id': this.story_feed_id, 
                    'count': 1
                });
            }
        """
        reduce_f = """
            function(key, values) {
                var r = {'guid': key, 'count': 0};
                for (var i=0; i < values.length; i++) {
                    r.feed_id = values[i].feed_id;
                    r.count += 1;
                }
                return r;
            }
        """
        finalize_f = """
            function(key, value) {
                if (value.count >= %(cutoff)s) {
                    return value;
                }
            }
        """ % {'cutoff': cutoff}
        res = cls.objects(shared_date__gte=today).map_reduce(map_f, reduce_f, 
                                                             finalize_f=finalize_f, 
                                                             output='inline')
        stories = dict([(r.key, r.value) for r in res if r.value])
        return stories, cutoff
        
    @classmethod
    def share_popular_stories(cls, verbose=True):
        publish_new_stories = False
        popular_profile = MSocialProfile.objects.get(username='popular')
        popular_user = User.objects.get(pk=popular_profile.user_id)
        shared_stories_today, cutoff = cls.collect_popular_stories()
        for guid, story_info in shared_stories_today.items():
            story = MStory.objects(story_feed_id=story_info['feed_id'], 
                                   story_guid=story_info['guid']).limit(1).first()
            if not story:
                logging.user(popular_user, "~FRPopular stories, story not found: %s" % story_info)
                continue

            story_db = dict([(k, v) for k, v in story._data.items() 
                                if k is not None and v is not None])
            story_values = {
                'user_id': popular_profile.user_id,
                'story_guid': story_db['story_guid'],
                'story_feed_id': story_db['story_feed_id'],
                'defaults': story_db,
            }
            shared_story, created = MSharedStory.objects.get_or_create(**story_values)
            if created:
                publish_new_stories = True
            if verbose and created:
                logging.user(popular_user, "~FCSharing: ~SB~FM%s (%s shares, %s min)" % (
                    story.story_title[:50],
                    story_info['count'],
                    cutoff))

        if publish_new_stories:
            socialsubs = MSocialSubscription.objects.filter(subscription_user_id=popular_user.pk)
            for socialsub in socialsubs:
                socialsub.needs_unread_recalc = True
                socialsub.save()
            shared_story.publish_update_to_subscribers()
            
    @classmethod
    def sync_all_redis(cls):
        r = redis.Redis(connection_pool=settings.REDIS_POOL)
        for story in cls.objects.all():
            story.sync_redis(redis_conn=r)
            
    def sync_redis(self, redis_conn=None):
        if not redis_conn:
            redis_conn = redis.Redis(connection_pool=settings.REDIS_POOL)
        
        share_key   = "S:%s:%s" % (self.story_feed_id, self.guid_hash)
        comment_key = "C:%s:%s" % (self.story_feed_id, self.guid_hash)
        redis_conn.sadd(share_key, self.user_id)
        if self.has_comments:
            redis_conn.sadd(comment_key, self.user_id)
        else:
            redis_conn.srem(comment_key, self.user_id)

    def publish_update_to_subscribers(self):
        try:
            r = redis.Redis(connection_pool=settings.REDIS_POOL)
            feed_id = "social:%s" % self.user_id
            listeners_count = r.publish(feed_id, 'story:new')
            if listeners_count:
                logging.debug("   ---> ~FMPublished to %s subscribers" % (listeners_count))
        except redis.ConnectionError:
            logging.debug("   ***> ~BMRedis is unavailable for real-time.")

    def comments_with_author(self):
        comments = {
            'user_id': self.user_id,
            'comments': self.comments,
            'shared_date': relative_timesince(self.shared_date),
            'replies': [reply.to_json() for reply in self.replies],
            'source_user_id': self.source_user_id,
        }
        return comments
    
    @classmethod
    def stories_with_comments_and_profiles(cls, stories, user_id, check_all=False, public=False):
        r = redis.Redis(connection_pool=settings.REDIS_POOL)
        friend_key = "F:%s:F" % (user_id)
        profile_user_ids = set()
        for story in stories: 
            story['friend_comments'] = []
            story['public_comments'] = []
            if check_all or story['comment_count']:
                comment_key = "C:%s:%s" % (story['story_feed_id'], story['guid_hash'])
                if check_all:
                    story['comment_count'] = r.scard(comment_key)
                friends_with_comments = [int(f) for f in r.sinter(comment_key, friend_key)]
                sharer_user_ids = [int(f) for f in r.smembers(comment_key)]
                shared_stories = []
                if sharer_user_ids:
                    params = {
                        'story_guid': story['id'],
                        'story_feed_id': story['story_feed_id'],
                        'user_id__in': sharer_user_ids,
                    }
                    shared_stories = cls.objects.filter(**params)
                for shared_story in shared_stories:
                    comments = shared_story.comments_with_author()
                    if shared_story.user_id in friends_with_comments:
                        story['friend_comments'].append(comments)
                    else:
                        story['public_comments'].append(comments)
                    if comments.get('source_user_id'):
                        profile_user_ids.add(comments['source_user_id'])
                all_comments = story['friend_comments'] + story['public_comments']
                profile_user_ids = profile_user_ids.union([reply['user_id'] 
                                                           for c in all_comments
                                                           for reply in c['replies']])
                if story.get('source_user_id'):
                    profile_user_ids.add(story['source_user_id'])
                story['comment_count_friends'] = len(friends_with_comments)
                story['comment_count_public'] = story['comment_count'] - len(friends_with_comments)
                
            if check_all or story['share_count']:
                share_key = "S:%s:%s" % (story['story_feed_id'], story['guid_hash'])
                if check_all:
                    story['share_count'] = r.scard(share_key)
                friends_with_shares = [int(f) for f in r.sinter(share_key, friend_key)]
                nonfriend_user_ids = [int(f) for f in r.sdiff(share_key, friend_key)]
                profile_user_ids.update(nonfriend_user_ids)
                profile_user_ids.update(friends_with_shares)
                story['shared_by_public']    = nonfriend_user_ids
                story['shared_by_friends']   = friends_with_shares
                story['share_count_public']  = story['share_count'] - len(friends_with_shares)
                story['share_count_friends'] = len(friends_with_shares)
                if story.get('source_user_id'):
                    profile_user_ids.add(story['source_user_id'])
            
        profiles = MSocialProfile.objects.filter(user_id__in=list(profile_user_ids))
        profiles = [profile.to_json(compact=True) for profile in profiles]

        return stories, profiles
    
    @staticmethod
    def attach_users_to_stories(stories, profiles):
        profiles = dict([(p['user_id'], p) for p in profiles])
        for s, story in enumerate(stories):
            for u, user_id in enumerate(story['shared_by_friends']):
                stories[s]['shared_by_friends'][u] = profiles[user_id]
            for u, user_id in enumerate(story['shared_by_public']):
                stories[s]['shared_by_public'][u] = profiles[user_id]
            for comment_set in ['friend_comments', 'public_comments']:
                for c, comment in enumerate(story[comment_set]):
                    stories[s][comment_set][c]['user'] = profiles[comment['user_id']]
                    if comment['source_user_id']:
                        stories[s][comment_set][c]['source_user'] = profiles[comment['source_user_id']]
                    for r, reply in enumerate(comment['replies']):
                        stories[s][comment_set][c]['replies'][r]['user'] = profiles[reply['user_id']]

        return stories
    
    @staticmethod
    def attach_users_to_comment(comment, profiles):
        profiles = dict([(p['user_id'], p) for p in profiles])
        comment['user'] = profiles[comment['user_id']]
        if comment['source_user_id']:
            comment['source_user'] = profiles[comment['source_user_id']]
        for r, reply in enumerate(comment['replies']):
            comment['replies'][r]['user'] = profiles[reply['user_id']]

        return comment
        
        
    def blurblog_permalink(self):
        profile = MSocialProfile.objects.get(user_id=self.user_id)
        return "%s/story/%s" % (
            profile.blurblog_url,
            self.guid_hash[:6]
        )
    
    def generate_post_to_service_message(self):
        message = self.comments
        if not message or len(message) < 1:
            message = self.story_title
        
        message = truncate_chars(message, 116)
        message += " " + self.blurblog_permalink()
        
        return message
        
    def post_to_service(self, service):
        if service in self.posted_to_services:
            return

        message = self.generate_post_to_service_message()
        social_service = MSocialServices.objects.get(user_id=self.user_id)
        user = User.objects.get(pk=self.user_id)
        
        logging.user(user, "~BM~FBPosting to %s (FAKED): ~SB%s" % (service, message))
        return
        
        if service == 'twitter':
            posted = social_service.post_to_twitter(message)
        elif service == 'facebook':
            posted = social_service.post_to_facebook(message)
        
        if posted:
            self.posted_to_services.append(service)
            self.save()
            
    def notify_user_ids(self, include_parent=True):
        user_ids = set()
        for reply in self.replies:
            if reply.user_id not in self.mute_email_users:
                user_ids.add(reply.user_id)
            
        if include_parent and self.user_id not in self.mute_email_users:
            user_ids.add(self.user_id)
        
        return list(user_ids)

    def send_emails_for_new_reply(self, reply_user_id):
        notify_user_ids = self.notify_user_ids()
        if reply_user_id in notify_user_ids:
            notify_user_ids.remove(reply_user_id)
        reply_user = User.objects.get(pk=reply_user_id)
        reply_user_profile = MSocialProfile.objects.get(user_id=reply_user_id)
        sent_emails = 0

        story_feed = Feed.objects.get(pk=self.story_feed_id)
        comment = self.comments_with_author()
        profile_user_ids = set([comment['user_id']])
        reply_user_ids = [reply['user_id'] for reply in comment['replies']]
        profile_user_ids = profile_user_ids.union(reply_user_ids)
        if self.source_user_id:
            profile_user_ids.add(self.source_user_id)
        profiles = MSocialProfile.objects.filter(user_id__in=list(profile_user_ids))
        profiles = [profile.to_json(compact=True) for profile in profiles]
        comment = MSharedStory.attach_users_to_comment(comment, profiles)
        
        for user_id in notify_user_ids:
            user = User.objects.get(pk=user_id)

            if not user.email or not user.profile.send_emails:
                continue
            
            mute_url = "http://%s%s" % (
                Site.objects.get_current().domain.replace('www', 'dev'),
                reverse('social-mute-story', kwargs={
                    'secret_token': user.profile.secret_token,
                    'shared_story_id': self._id,
                })
            )
            data = {
                'reply_user_profile': reply_user_profile,
                'comment': comment,
                'shared_story': self,
                'story_feed': story_feed,
                'mute_url': mute_url,
            }
        
            text    = render_to_string('mail/email_reply.txt', data)
            html    = render_to_string('mail/email_reply.xhtml', data)
            subject = "%s replied to you on \"%s\" on NewsBlur" % (reply_user.username, self.story_title)
            msg     = EmailMultiAlternatives(subject, text, 
                                             from_email='NewsBlur <%s>' % settings.HELLO_EMAIL,
                                             to=['%s <%s>' % (user.username, user.email)])
            msg.attach_alternative(html, "text/html")
            msg.send()
            sent_emails += 1
                
        logging.user(reply_user, "~BB~FM~SBSending %s/%s email%s for new reply: %s" % (
            sent_emails, len(notify_user_ids), 
            '' if len(notify_user_ids) == 1 else 's', 
            self.story_title[:30]))
    
    def send_email_for_reshare(self):
        reshare_user = User.objects.get(pk=self.user_id)
        reshare_user_profile = MSocialProfile.objects.get(user_id=self.user_id)
        original_user = User.objects.get(pk=self.source_user_id)
        original_shared_story = MSharedStory.objects.get(user_id=self.source_user_id,
                                                         story_guid=self.story_guid)
                                                         
        if not original_user.email or not original_user.profile.send_emails:
            return
            
        story_feed = Feed.objects.get(pk=self.story_feed_id)
        comment = self.comments_with_author()
        profile_user_ids = set([comment['user_id']])
        reply_user_ids = [reply['user_id'] for reply in comment['replies']]
        profile_user_ids = profile_user_ids.union(reply_user_ids)
        if self.source_user_id:
            profile_user_ids.add(self.source_user_id)
        profiles = MSocialProfile.objects.filter(user_id__in=list(profile_user_ids))
        profiles = [profile.to_json(compact=True) for profile in profiles]
        comment = MSharedStory.attach_users_to_comment(comment, profiles)
        
        mute_url = "http://%s%s" % (
            Site.objects.get_current().domain.replace('www', 'dev'),
            reverse('social-mute-story', kwargs={
                'secret_token': original_user.profile.secret_token,
                'shared_story_id': original_shared_story._id,
            })
        )
        data = {
            'comment': comment,
            'shared_story': self,
            'reshare_user_profile': reshare_user_profile,
            'original_shared_story': original_shared_story,
            'story_feed': story_feed,
            'mute_url': mute_url,
        }
    
        text    = render_to_string('mail/email_reshare.txt', data)
        html    = render_to_string('mail/email_reshare.xhtml', data)
        subject = "%s re-shared \"%s\" from you on NewsBlur" % (reshare_user.username, self.story_title)
        msg     = EmailMultiAlternatives(subject, text, 
                                         from_email='NewsBlur <%s>' % settings.HELLO_EMAIL,
                                         to=['%s <%s>' % (original_user.username, original_user.email)])
        msg.attach_alternative(html, "text/html")
        msg.send()
        
        self.emailed_reshare = True
        self.save()
            
        logging.user(reshare_user, "~BB~FM~SBSending %s email for story re-share: %s" % (
            original_user.username,
            self.story_title[:30]))
        
        

class MSocialServices(mongo.Document):
    user_id               = mongo.IntField()
    autofollow            = mongo.BooleanField(default=True)
    twitter_uid           = mongo.StringField()
    twitter_access_key    = mongo.StringField()
    twitter_access_secret = mongo.StringField()
    twitter_friend_ids    = mongo.ListField(mongo.StringField())
    twitter_picture_url   = mongo.StringField()
    twitter_username      = mongo.StringField()
    twitter_refresh_date  = mongo.DateTimeField()
    facebook_uid          = mongo.StringField()
    facebook_access_token = mongo.StringField()
    facebook_friend_ids   = mongo.ListField(mongo.StringField())
    facebook_picture_url  = mongo.StringField()
    facebook_refresh_date = mongo.DateTimeField()
    upload_picture_url    = mongo.StringField()
    syncing_twitter       = mongo.BooleanField(default=False)
    syncing_facebook      = mongo.BooleanField(default=False)
    
    meta = {
        'collection': 'social_services',
        'indexes': ['user_id', 'twitter_friend_ids', 'facebook_friend_ids', 'twitter_uid', 'facebook_uid'],
        'allow_inheritance': False,
    }
    
    def __unicode__(self):
        user = User.objects.get(pk=self.user_id)
        return "%s (Twitter: %s, FB: %s)" % (user.username, self.twitter_uid, self.facebook_uid)
        
    def to_json(self):
        user = User.objects.get(pk=self.user_id)
        return {
            'twitter': {
                'twitter_username': self.twitter_username,
                'twitter_picture_url': self.twitter_picture_url,
                'twitter_uid': self.twitter_uid,
                'syncing': self.syncing_twitter,
            },
            'facebook': {
                'facebook_uid': self.facebook_uid,
                'facebook_picture_url': self.facebook_picture_url,
                'syncing': self.syncing_facebook,
            },
            'gravatar': {
                'gravatar_picture_url': "http://www.gravatar.com/avatar/" + \
                                        hashlib.md5(user.email).hexdigest()
            },
            'upload': {
                'upload_picture_url': self.upload_picture_url
            }
        }
    
    @classmethod
    def profile(cls, user_id):
        try:
            profile = cls.objects.get(user_id=user_id)
        except cls.DoesNotExist:
            return {}
        return profile.to_json()

    def twitter_api(self):
        twitter_consumer_key = settings.TWITTER_CONSUMER_KEY
        twitter_consumer_secret = settings.TWITTER_CONSUMER_SECRET
        auth = tweepy.OAuthHandler(twitter_consumer_key, twitter_consumer_secret)
        auth.set_access_token(self.twitter_access_key, self.twitter_access_secret)
        api = tweepy.API(auth)
        return api
    
    def facebook_api(self):
        graph = facebook.GraphAPI(self.facebook_access_token)
        return graph

    def sync_twitter_friends(self):
        api = self.twitter_api()
        if not api:
            return
            
        friend_ids = list(unicode(friend.id) for friend in tweepy.Cursor(api.friends).items())
        if not friend_ids:
            return
        
        twitter_user = api.me()
        self.twitter_picture_url = twitter_user.profile_image_url
        self.twitter_username = twitter_user.screen_name
        self.twitter_friend_ids = friend_ids
        self.twitter_refreshed_date = datetime.datetime.utcnow()
        self.save()
        
        self.follow_twitter_friends()
        
        profile, _ = MSocialProfile.objects.get_or_create(user_id=self.user_id)
        profile.location = profile.location or twitter_user.location
        profile.bio = profile.bio or twitter_user.description
        profile.website = profile.website or twitter_user.url
        profile.save()
        profile.count_follows()
        if not profile.photo_url or not profile.photo_service:
            self.set_photo('twitter')
        
    def sync_facebook_friends(self):
        self.syncing_facebook = False
        self.save()
        
        graph = self.facebook_api()
        if not graph:
            return

        friends = graph.get_connections("me", "friends")
        if not friends:
            return

        facebook_friend_ids = [unicode(friend["id"]) for friend in friends["data"]]
        self.facebook_friend_ids = facebook_friend_ids
        self.facebook_refresh_date = datetime.datetime.utcnow()
        self.facebook_picture_url = "//graph.facebook.com/%s/picture" % self.facebook_uid
        self.save()
        
        self.follow_facebook_friends()
        
        facebook_user = graph.request('me', args={'fields':'website,bio,location'})
        profile, _ = MSocialProfile.objects.get_or_create(user_id=self.user_id)
        profile.location = profile.location or (facebook_user.get('location') and facebook_user['location']['name'])
        profile.bio = profile.bio or facebook_user.get('bio')
        profile.website = profile.website or facebook_user.get('website')
        profile.save()
        profile.count_follows()
        if not profile.photo_url or not profile.photo_service:
            self.set_photo('facebook')
        
    def follow_twitter_friends(self):
        self.syncing_twitter = False
        self.save()
        
        social_profile, _ = MSocialProfile.objects.get_or_create(user_id=self.user_id)
        following = []
        followers = 0
        
        if self.autofollow:
            # Follow any friends already on NewsBlur
            user_social_services = MSocialServices.objects.filter(twitter_uid__in=self.twitter_friend_ids)
            for user_social_service in user_social_services:
                followee_user_id = user_social_service.user_id
                social_profile.follow_user(followee_user_id)
                following.append(followee_user_id)
        
            # Follow any friends already on NewsBlur
            following_users = MSocialServices.objects.filter(twitter_friend_ids__contains=self.twitter_uid)
            for following_user in following_users:
                if following_user.autofollow:
                    following_user_profile = MSocialProfile.objects.get(user_id=following_user.user_id)
                    following_user_profile.follow_user(self.user_id, check_unfollowed=True)
                    followers += 1
        
        user = User.objects.get(pk=self.user_id)
        logging.user(user, "~BB~FRTwitter import: %s users, now following ~SB%s~SN with ~SB%s~SN follower-backs" % (len(self.twitter_friend_ids), len(following), followers))
        
        return following
        
    def follow_facebook_friends(self):
        social_profile, _ = MSocialProfile.objects.get_or_create(user_id=self.user_id)
        following = []
        followers = 0
        
        if self.autofollow:
            # Follow any friends already on NewsBlur
            user_social_services = MSocialServices.objects.filter(facebook_uid__in=self.facebook_friend_ids)
            for user_social_service in user_social_services:
                followee_user_id = user_social_service.user_id
                social_profile.follow_user(followee_user_id)
                following.append(followee_user_id)
        
            # Friends already on NewsBlur should follow back
            following_users = MSocialServices.objects.filter(facebook_friend_ids__contains=self.facebook_uid)
            for following_user in following_users:
                if following_user.autofollow:
                    following_user_profile = MSocialProfile.objects.get(user_id=following_user.user_id)
                    following_user_profile.follow_user(self.user_id, check_unfollowed=True)
                    followers += 1
        
        user = User.objects.get(pk=self.user_id)
        logging.user(user, "~BB~FRFacebook import: %s users, now following ~SB%s~SN with ~SB%s~SN follower-backs" % (len(self.facebook_friend_ids), len(following), followers))
        
        return following
        
    def disconnect_twitter(self):
        self.twitter_uid = None
        self.save()
        
    def disconnect_facebook(self):
        self.facebook_uid = None
        self.save()
        
    def set_photo(self, service):
        profile, _ = MSocialProfile.objects.get_or_create(user_id=self.user_id)
        if service == 'nothing':
            service = None

        profile.photo_service = service
        if not service:
            profile.photo_url = None
        elif service == 'twitter':
            profile.photo_url = self.twitter_picture_url
        elif service == 'facebook':
            profile.photo_url = self.facebook_picture_url
        elif service == 'upload':
            profile.photo_url = self.upload_picture_url
        elif service == 'gravatar':
            user = User.objects.get(pk=self.user_id)
            profile.photo_url = "http://www.gravatar.com/avatar/" + \
                                hashlib.md5(user.email).hexdigest()
        profile.save()
        return profile
    
    def post_to_twitter(self, message):
        try:
            api = self.twitter_api()
            api.update_status(status=message)
        except tweepy.TweepError, e:
            print e
            return

        return True
            
    def post_to_facebook(self, message):
        try:
            api = self.facebook_api()
            api.put_wall_post(message=message)
        except facebook.GraphAPIError, e:
            print e
            return

        return True

class MInteraction(mongo.Document):
    user_id      = mongo.IntField()
    date         = mongo.DateTimeField(default=datetime.datetime.now)
    category     = mongo.StringField()
    title        = mongo.StringField()
    content      = mongo.StringField()
    with_user_id = mongo.IntField()
    feed_id      = mongo.IntField()
    content_id   = mongo.StringField()
    
    meta = {
        'collection': 'interactions',
        'indexes': [('user_id', 'date'), 'category'],
        'allow_inheritance': False,
        'index_drop_dups': True,
        'ordering': ['-date'],
    }
    
    def __unicode__(self):
        user = User.objects.get(pk=self.user_id)
        with_user = self.with_user_id and User.objects.get(pk=self.with_user_id)
        return "<%s> %s on %s: %s - %s" % (user.username, with_user and with_user.username, self.date, 
                                           self.category, self.content and self.content[:20])
    
    def to_json(self):
        return {
            'date': self.date,
            'category': self.category,
            'title': self.title,
            'content': self.content,
            'with_user_id': self.with_user_id,
            'feed_id': self.feed_id,
            'content_id': self.content_id,
        }
        
    @classmethod
    def user(cls, user_id, page=1):
        user_profile = Profile.objects.get(user=user_id)
        dashboard_date = user_profile.dashboard_date or user_profile.last_seen_on
        page = max(1, page)
        limit = 4 # Also set in template
        offset = (page-1) * limit
        interactions_db = cls.objects.filter(user_id=user_id)[offset:offset+limit+1]
        with_user_ids = [i.with_user_id for i in interactions_db if i.with_user_id]
        social_profiles = dict((p.user_id, p) for p in MSocialProfile.objects.filter(user_id__in=with_user_ids))
    
        interactions = []
        for interaction_db in interactions_db:
            interaction = interaction_db.to_json()
            social_profile = social_profiles.get(interaction_db.with_user_id)
            if social_profile:
                interaction['photo_url'] = social_profile.profile_photo_url
            interaction['with_user'] = social_profiles.get(interaction_db.with_user_id)
            interaction['time_since'] = relative_timesince(interaction_db.date)
            interaction['date'] = interaction_db.date
            interaction['is_new'] = interaction_db.date > dashboard_date
            interactions.append(interaction)

        return interactions
        
    @classmethod
    def new_follow(cls, follower_user_id, followee_user_id):
        params = {
            'user_id': followee_user_id, 
            'with_user_id': follower_user_id,
            'category': 'follow',
        }
        try:
            cls.objects.get_or_create(**params)
        except cls.MultipleObjectsReturned:
            dupes = cls.objects.filter(**params).order_by('-date')
            logging.debug(" ---> ~FRDeleting dupe follow interactions. %s found." % dupes.count())
            for dupe in dupes[1:]:
                dupe.delete()
    
    @classmethod
    def new_comment_reply(cls, user_id, reply_user_id, reply_content, social_feed_id, story_id, original_message=None):
        params = {
            'user_id': user_id,
            'with_user_id': reply_user_id,
            'category': 'comment_reply',
            'content': reply_content,
            'feed_id': social_feed_id,
            'content_id': story_id,
        }
        if original_message:
            params['content'] = original_message
            original = cls.objects.filter(**params).limit(1)
            if original:
                original = original[0]
                original.content = reply_content
                original.save()
            else:
                original_message = None

        if not original_message:
            cls.objects.create(**params)

    @classmethod
    def new_reply_reply(cls, user_id, reply_user_id, reply_content, social_feed_id, story_id, original_message=None):
        params = {
            'user_id': user_id,
            'with_user_id': reply_user_id,
            'category': 'reply_reply',
            'content': reply_content,
            'feed_id': social_feed_id,
            'content_id': story_id,
        }
        if original_message:
            params['content'] = original_message
            original = cls.objects.filter(**params).limit(1)
            if original:
                original = original[0]
                original.content = reply_content
                original.save()
            else:
                original_message = None

        if not original_message:
            cls.objects.create(**params)
    
    @classmethod
    def new_reshared_story(cls, user_id, reshare_user_id, comments, story_title, story_feed_id, story_id, original_comments=None):
        params = {
            'user_id': user_id,
            'with_user_id': reshare_user_id,
            'category': 'story_reshare',
            'content': comments,
            'title': story_title,
            'feed_id': story_feed_id,
            'content_id': story_id,
        }
        if original_comments:
            params['content'] = original_comments
            original = cls.objects.filter(**params).limit(1)
            if original:
                original = original[0]
                original.content = comments
                original.save()
            else:
                original_comments = None

        if not original_comments:
            cls.objects.create(**params)

class MActivity(mongo.Document):
    user_id      = mongo.IntField()
    date         = mongo.DateTimeField(default=datetime.datetime.now)
    category     = mongo.StringField()
    title        = mongo.StringField()
    content      = mongo.StringField()
    with_user_id = mongo.IntField()
    feed_id      = mongo.IntField()
    content_id   = mongo.StringField()
    
    meta = {
        'collection': 'activities',
        'indexes': [('user_id', 'date'), 'category'],
        'allow_inheritance': False,
        'index_drop_dups': True,
        'ordering': ['-date'],
    }
    
    def __unicode__(self):
        user = User.objects.get(pk=self.user_id)
        return "<%s> %s - %s" % (user.username, self.category, self.content and self.content[:20])
    
    def to_json(self):
        return {
            'date': self.date,
            'category': self.category,
            'title': self.title,
            'content': self.content,
            'with_user_id': self.with_user_id,
            'feed_id': self.feed_id,
            'content_id': self.content_id,
        }
        
    @classmethod
    def user(cls, user_id, page=1, public=False):
        user_profile = Profile.objects.get(user=user_id)
        dashboard_date = user_profile.dashboard_date or user_profile.last_seen_on
        page = max(1, page)
        limit = 4 # Also set in template
        offset = (page-1) * limit
        
        activities_db = cls.objects.filter(user_id=user_id)
        if public:
            activities_db = activities_db.filter(category__nin=['star', 'feedsub'])
        activities_db = activities_db[offset:offset+limit+1]
        with_user_ids = [a.with_user_id for a in activities_db if a.with_user_id]
        social_profiles = dict((p.user_id, p) for p in MSocialProfile.objects.filter(user_id__in=with_user_ids))
        activities = []
        for activity_db in activities_db:
            activity = activity_db.to_json()
            activity['date'] = activity_db.date
            activity['time_since'] = relative_timesince(activity_db.date)
            social_profile = social_profiles.get(activity_db.with_user_id)
            if social_profile:
                activity['photo_url'] = social_profile.profile_photo_url
            activity['is_new'] = activity_db.date > dashboard_date
            activity['with_user'] = social_profiles.get(activity_db.with_user_id)
            activities.append(activity)
        
        return activities
            
    @classmethod
    def new_starred_story(cls, user_id, story_title, story_feed_id, story_id):
        cls.objects.get_or_create(user_id=user_id,
                                  category='star',
                                  content=story_title,
                                  feed_id=story_feed_id,
                                  content_id=story_id)
                           
    @classmethod
    def new_feed_subscription(cls, user_id, feed_id, feed_title):
        cls.objects.create(user_id=user_id,
                           category='feedsub',
                           content=feed_title,
                           feed_id=feed_id)
                           
    @classmethod
    def new_follow(cls, follower_user_id, followee_user_id):
        params = {
            'user_id': follower_user_id, 
            'with_user_id': followee_user_id,
            'category': 'follow',
        }
        try:
            cls.objects.get_or_create(**params)
        except cls.MultipleObjectsReturned:
            dupes = cls.objects.filter(**params).order_by('-date')
            logging.debug(" ---> ~FRDeleting dupe follow activities. %s found." % dupes.count())
            for dupe in dupes[1:]:
                dupe.delete()
    
    @classmethod
    def new_comment_reply(cls, user_id, comment_user_id, reply_content, story_feed_id, story_id, original_message=None):
        params = {
            'user_id': user_id,
            'with_user_id': comment_user_id,
            'category': 'comment_reply',
            'content': reply_content,
            'feed_id': story_feed_id,
            'content_id': story_id,
        }
        if original_message:
            params['content'] = original_message
            original = cls.objects.filter(**params).limit(1)
            if original:
                original = original[0]
                original.content = reply_content
                original.save()
            else:
                original_message = None

        if not original_message:
            cls.objects.create(**params)
    
    @classmethod
    def new_shared_story(cls, user_id, story_title, comments, story_feed_id, story_id, share_date=None):
        a, _ = cls.objects.get_or_create(user_id=user_id,
                                         with_user_id=user_id,
                                         category='sharedstory',
                                         feed_id=story_feed_id,
                                         content_id=story_id,
                                         defaults={
                                             'title': story_title,
                                             'content': comments,
                                         })
        if a.content != comments:
            a.content = comments
            a.save()
        if share_date:
            a.date = share_date
            a.save()

    @classmethod
    def remove_shared_story(cls, user_id, story_feed_id, story_id):
        try:
            a = cls.objects.get(user_id=user_id,
                                with_user_id=user_id,
                                category='sharedstory',
                                feed_id=story_feed_id,
                                content_id=story_id)
        except cls.DoesNotExist:
            return
        
        a.delete()