import datetime import zlib import hashlib import redis import re import mongoengine as mongo from collections import defaultdict # from mongoengine.queryset import OperationError from django.conf import settings from django.contrib.auth.models import User from django.contrib.sites.models import Site from django.core.urlresolvers import reverse from django.template.loader import render_to_string from django.core.mail import EmailMultiAlternatives from apps.reader.models import UserSubscription, MUserStory from apps.analyzer.models import MClassifierFeed, MClassifierAuthor, MClassifierTag, MClassifierTitle from apps.analyzer.models import apply_classifier_titles, apply_classifier_feeds, apply_classifier_authors, apply_classifier_tags from apps.rss_feeds.models import Feed, MStory from vendor import facebook from vendor import tweepy from utils import log as logging from utils.feed_functions import relative_timesince from utils import json_functions as json class MRequestInvite(mongo.Document): username = mongo.StringField() email_sent = mongo.BooleanField() meta = { 'collection': 'social_invites', 'allow_inheritance': False, } def send_email(self): user = User.objects.filter(username__iexact=self.username) if not user: user = User.objects.filter(email__iexact=self.username) if user: user = user[0] email = user.email or self.username else: user = { 'username': self.username, 'profile': { 'autologin_url': '/', } } email = self.username params = { 'user': user, } text = render_to_string('mail/email_social_beta.txt', params) html = render_to_string('mail/email_social_beta.xhtml', params) subject = "Psst, you're in..." msg = EmailMultiAlternatives(subject, text, from_email='NewsBlur <%s>' % settings.HELLO_EMAIL, to=['<%s>' % (email)]) msg.attach_alternative(html, "text/html") msg.send() self.email_sent = True self.save() logging.debug(" ---> ~BB~FM~SBSending email for social beta: %s" % self.username) class MSocialProfile(mongo.Document): user_id = mongo.IntField(unique=True) username = mongo.StringField(max_length=30, unique=True) email = mongo.StringField() bio = mongo.StringField(max_length=80) blog_title = mongo.StringField(max_length=256) custom_css = mongo.StringField() photo_url = mongo.StringField() photo_service = mongo.StringField() location = mongo.StringField(max_length=40) website = mongo.StringField(max_length=200) subscription_count = mongo.IntField(default=0) shared_stories_count = mongo.IntField(default=0) following_count = mongo.IntField(default=0) follower_count = mongo.IntField(default=0) following_user_ids = mongo.ListField(mongo.IntField()) follower_user_ids = mongo.ListField(mongo.IntField()) unfollowed_user_ids = mongo.ListField(mongo.IntField()) popular_publishers = mongo.StringField() stories_last_month = mongo.IntField(default=0) average_stories_per_month = mongo.IntField(default=0) story_count_history = mongo.ListField() feed_classifier_counts = mongo.DictField() favicon_color = mongo.StringField(max_length=6) meta = { 'collection': 'social_profile', 'indexes': ['user_id', 'following_user_ids', 'follower_user_ids', 'unfollowed_user_ids'], 'allow_inheritance': False, 'index_drop_dups': True, } def __unicode__(self): return "%s [%s] following %s/%s, shared %s" % (self.username, self.user_id, self.following_count, self.follower_count, self.shared_stories_count) def save(self, *args, **kwargs): if not self.username: self.import_user_fields() if not self.subscription_count: self.count(skip_save=True) if self.bio and len(self.bio) > MSocialProfile.bio.max_length: self.bio = self.bio[:80] super(MSocialProfile, self).save(*args, **kwargs) if self.user_id not in self.following_user_ids: self.follow_user(self.user_id) self.count() @classmethod def recommended_users(cls, for_user_id): # profile_count = cls.objects.filter(**{ # 'shared_stories_count__gte': 1, # 'user_id__ne': for_user_id, # }).count() profiles = [] # for i in range(3): # skip = random.randint(0, max(profile_count-2, 0)) # profile = cls.objects.filter(**{ # 'shared_stories_count__gte': 1, # 'user_id__ne': for_user_id, # }) # if profile: # profiles.append(profile[0]) # profiles = sorted(profiles, key=lambda p: p.shared_stories_count) return profiles def count_stories(self): # Popular Publishers self.save_popular_publishers() def save_popular_publishers(self, feed_publishers=None): if not feed_publishers: publishers = defaultdict(int) for story in MSharedStory.objects(user_id=self.user_id).only('story_feed_id')[:500]: publishers[story.story_feed_id] += 1 feed_titles = dict((f.id, f.feed_title) for f in Feed.objects.filter(pk__in=publishers.keys()).only('id', 'feed_title')) feed_publishers = sorted([{'id': k, 'feed_title': feed_titles[k], 'story_count': v} for k, v in publishers.items() if k in feed_titles], key=lambda f: f['story_count'], reverse=True)[:20] popular_publishers = json.encode(feed_publishers) if len(popular_publishers) < 1023: self.popular_publishers = popular_publishers self.save() return if len(popular_publishers) > 1: self.save_popular_publishers(feed_publishers=feed_publishers[:-1]) @classmethod def user_statistics(cls, user): try: profile = cls.objects.get(user_id=user.pk) except cls.DoesNotExist: return None values = { 'followers': profile.follower_count, 'following': profile.following_count, 'shared_stories': profile.shared_stories_count, } return values @classmethod def profile(cls, user_id): try: profile = cls.objects.get(user_id=user_id) except cls.DoesNotExist: return {} return profile.to_json(full=True) @classmethod def profiles(cls, user_ids): profiles = cls.objects.filter(user_id__in=user_ids) return profiles @classmethod def profile_feeds(cls, user_ids): profiles = cls.objects.filter(user_id__in=user_ids, shared_stories_count__gte=1) profiles = dict((p.user_id, p.feed()) for p in profiles) return profiles @classmethod def sync_all_redis(cls): for profile in cls.objects.all(): profile.sync_redis() def sync_redis(self): for user_id in self.following_user_ids: self.follow_user(user_id) self.follow_user(self.user_id) @property def title(self): return self.blog_title if self.blog_title else self.username + "'s blurblog" def feed(self): params = self.to_json(compact=True) params.update({ 'feed_title': self.title, 'page_url': reverse('load-social-page', kwargs={'user_id': self.user_id, 'username': self.username}) }) return params def page(self): params = self.to_json(full=True) params.update({ 'feed_title': self.title, 'custom_css': self.custom_css, }) return params @property def profile_photo_url(self): if self.photo_url: return self.photo_url return settings.MEDIA_URL + 'img/reader/default_profile_photo.png' def to_json(self, compact=False, full=False): # domain = Site.objects.get_current().domain domain = Site.objects.get_current().domain.replace('www', 'dev') params = { 'id': 'social:%s' % self.user_id, 'user_id': self.user_id, 'username': self.username, 'photo_url': self.profile_photo_url, 'num_subscribers': self.follower_count, 'feed_address': "http://%s%s" % (domain, reverse('shared-stories-rss-feed', kwargs={'user_id': self.user_id, 'username': self.username})), 'feed_link': "http://%s%s" % (domain, reverse('load-social-page', kwargs={'user_id': self.user_id, 'username': self.username})), } if not compact: params.update({ 'bio': self.bio, 'location': self.location, 'website': self.website, 'subscription_count': self.subscription_count, 'shared_stories_count': self.shared_stories_count, 'following_count': self.following_count, 'follower_count': self.follower_count, 'popular_publishers': json.decode(self.popular_publishers), 'stories_last_month': self.stories_last_month, 'average_stories_per_month': self.average_stories_per_month, }) if full: params.update({ 'photo_service': self.photo_service, 'following_user_ids': self.following_user_ids, 'follower_user_ids': self.follower_user_ids, }) return params def import_user_fields(self, skip_save=False): user = User.objects.get(pk=self.user_id) self.username = user.username self.email = user.email def count(self, skip_save=False): self.subscription_count = UserSubscription.objects.filter(user__pk=self.user_id).count() self.shared_stories_count = MSharedStory.objects.filter(user_id=self.user_id).count() self.following_count = len(self.following_user_ids) self.follower_count = len(self.follower_user_ids) if not skip_save: self.save() def follow_user(self, user_id, check_unfollowed=False): r = redis.Redis(connection_pool=settings.REDIS_POOL) if check_unfollowed and user_id in self.unfollowed_user_ids: return if user_id not in self.following_user_ids: self.following_user_ids.append(user_id) if user_id in self.unfollowed_user_ids: self.unfollowed_user_ids.remove(user_id) self.count() self.save() if self.user_id == user_id: followee = self else: followee, _ = MSocialProfile.objects.get_or_create(user_id=user_id) if self.user_id not in followee.follower_user_ids: followee.follower_user_ids.append(self.user_id) followee.count() followee.save() following_key = "F:%s:F" % (self.user_id) r.sadd(following_key, user_id) follower_key = "F:%s:f" % (user_id) r.sadd(follower_key, self.user_id) if self.user_id != user_id: MInteraction.new_follow(follower_user_id=self.user_id, followee_user_id=user_id) MActivity.new_follow(follower_user_id=self.user_id, followee_user_id=user_id) MSocialSubscription.objects.get_or_create(user_id=self.user_id, subscription_user_id=user_id) def is_following_user(self, user_id): return user_id in self.following_user_ids def unfollow_user(self, user_id): r = redis.Redis(connection_pool=settings.REDIS_POOL) if not isinstance(user_id, int): user_id = int(user_id) if user_id == self.user_id: # Only unfollow other people, not yourself. return if user_id in self.following_user_ids: self.following_user_ids.remove(user_id) if user_id not in self.unfollowed_user_ids: self.unfollowed_user_ids.append(user_id) self.count() self.save() followee = MSocialProfile.objects.get(user_id=user_id) if self.user_id in followee.follower_user_ids: followee.follower_user_ids.remove(self.user_id) followee.count() followee.save() following_key = "F:%s:F" % (self.user_id) r.srem(following_key, user_id) follower_key = "F:%s:f" % (user_id) r.srem(follower_key, self.user_id) MSocialSubscription.objects.get(user_id=self.user_id, subscription_user_id=user_id).delete() def common_follows(self, user_id, direction='followers'): r = redis.Redis(connection_pool=settings.REDIS_POOL) my_followers = "F:%s:%s" % (self.user_id, 'F' if direction == 'followers' else 'F') their_followers = "F:%s:%s" % (user_id, 'f' if direction == 'followers' else 'F') follows_inter = r.sinter(their_followers, my_followers) follows_diff = r.sdiff(their_followers, my_followers) follows_inter = [int(f) for f in follows_inter] follows_diff = [int(f) for f in follows_diff] return follows_inter, follows_diff def save_feed_story_history_statistics(self): """ Fills in missing months between earlier occurances and now. Save format: [('YYYY-MM, #), ...] Example output: [(2010-12, 123), (2011-01, 146)] """ now = datetime.datetime.utcnow() min_year = now.year total = 0 month_count = 0 # Count stories, aggregate by year and month. Map Reduce! map_f = """ function() { var date = (this.shared_date.getFullYear()) + "-" + (this.shared_date.getMonth()+1); emit(date, 1); } """ reduce_f = """ function(key, values) { var total = 0; for (var i=0; i < values.length; i++) { total += values[i]; } return total; } """ dates = {} res = MSharedStory.objects(user_id=self.user_id).map_reduce(map_f, reduce_f, output='inline') for r in res: dates[r.key] = r.value year = int(re.findall(r"(\d{4})-\d{1,2}", r.key)[0]) if year < min_year: min_year = year # Assemble a list with 0's filled in for missing months, # trimming left and right 0's. months = [] start = False for year in range(min_year, now.year+1): for month in range(1, 12+1): if datetime.datetime(year, month, 1) < now: key = u'%s-%s' % (year, month) if dates.get(key) or start: start = True months.append((key, dates.get(key, 0))) total += dates.get(key, 0) month_count += 1 self.story_count_history = months self.average_stories_per_month = total / month_count self.save() def save_classifier_counts(self): def calculate_scores(cls, facet): map_f = """ function() { emit(this["%s"], { pos: this.score>0 ? this.score : 0, neg: this.score<0 ? Math.abs(this.score) : 0 }); } """ % (facet) reduce_f = """ function(key, values) { var result = {pos: 0, neg: 0}; values.forEach(function(value) { result.pos += value.pos; result.neg += value.neg; }); return result; } """ scores = [] res = cls.objects(social_user_id=self.user_id).map_reduce(map_f, reduce_f, output='inline') for r in res: facet_values = dict([(k, int(v)) for k,v in r.value.iteritems()]) facet_values[facet] = r.key scores.append(facet_values) scores = sorted(scores, key=lambda v: v['neg'] - v['pos']) return scores scores = {} for cls, facet in [(MClassifierTitle, 'title'), (MClassifierAuthor, 'author'), (MClassifierTag, 'tag'), (MClassifierFeed, 'feed_id')]: scores[facet] = calculate_scores(cls, facet) if facet == 'feed_id' and scores[facet]: scores['feed'] = scores[facet] del scores['feed_id'] elif not scores[facet]: del scores[facet] if scores: self.feed_classifier_counts = scores self.save() class MSocialSubscription(mongo.Document): UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta(days=settings.DAYS_OF_UNREAD) user_id = mongo.IntField() subscription_user_id = mongo.IntField(unique_with='user_id') follow_date = mongo.DateTimeField(default=datetime.datetime.utcnow()) last_read_date = mongo.DateTimeField(default=UNREAD_CUTOFF) mark_read_date = mongo.DateTimeField(default=UNREAD_CUTOFF) unread_count_neutral = mongo.IntField(default=0) unread_count_positive = mongo.IntField(default=0) unread_count_negative = mongo.IntField(default=0) unread_count_updated = mongo.DateTimeField() oldest_unread_story_date = mongo.DateTimeField() needs_unread_recalc = mongo.BooleanField(default=False) feed_opens = mongo.IntField(default=0) is_trained = mongo.BooleanField(default=False) meta = { 'collection': 'social_subscription', 'indexes': [('user_id', 'subscription_user_id')], 'allow_inheritance': False, } def __unicode__(self): return "%s:%s" % (self.user_id, self.subscription_user_id) @classmethod def feeds(cls, *args, **kwargs): user_id = kwargs['user_id'] params = dict(user_id=user_id) if 'subscription_user_id' in kwargs: params["subscription_user_id"] = kwargs["subscription_user_id"] social_subs = cls.objects.filter(**params) # for sub in social_subs: # sub.calculate_feed_scores() social_feeds = [] if social_subs: social_subs = dict((s.subscription_user_id, s.to_json()) for s in social_subs) social_user_ids = social_subs.keys() # Fetch user profiles of subscriptions social_profiles = MSocialProfile.profile_feeds(social_user_ids) for user_id, social_sub in social_subs.items(): # Check if the social feed has any stories, otherwise they aren't active. if user_id in social_profiles: # Combine subscription read counts with feed/user info feed = dict(social_sub.items() + social_profiles[user_id].items()) social_feeds.append(feed) return social_feeds @classmethod def feeds_with_updated_counts(cls, user, social_feed_ids=None): feeds = {} # Get social subscriptions for user user_subs = cls.objects.filter(user_id=user.pk) if social_feed_ids: social_user_ids = [int(f.replace('social:', '')) for f in social_feed_ids] user_subs = user_subs.filter(subscription_user_id__in=social_user_ids) UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta(days=settings.DAYS_OF_UNREAD) for i, sub in enumerate(user_subs): # Count unreads if subscription is stale. if (sub.needs_unread_recalc or (sub.unread_count_updated and sub.unread_count_updated < UNREAD_CUTOFF) or (sub.oldest_unread_story_date and sub.oldest_unread_story_date < UNREAD_CUTOFF)): sub = sub.calculate_feed_scores(silent=True) feed_id = "social:%s" % sub.subscription_user_id feeds[feed_id] = { 'ps': sub.unread_count_positive, 'nt': sub.unread_count_neutral, 'ng': sub.unread_count_negative, 'id': feed_id, } return feeds def to_json(self): return { 'user_id': self.user_id, 'subscription_user_id': self.subscription_user_id, 'nt': self.unread_count_neutral, 'ps': self.unread_count_positive, 'ng': self.unread_count_negative, 'is_trained': self.is_trained, } def mark_story_ids_as_read(self, story_ids, feed_id, request=None): data = dict(code=0, payload=story_ids) r = redis.Redis(connection_pool=settings.REDIS_POOL) if not request: request = self.user if not self.needs_unread_recalc: self.needs_unread_recalc = True self.save() sub_username = MSocialProfile.objects.get(user_id=self.subscription_user_id).username if len(story_ids) > 1: logging.user(request, "~FYRead %s stories in social subscription: %s" % (len(story_ids), sub_username)) else: logging.user(request, "~FYRead story in social subscription: %s" % (sub_username)) for story_id in set(story_ids): story = MSharedStory.objects.get(user_id=self.subscription_user_id, story_guid=story_id) now = datetime.datetime.utcnow() date = now if now > story.story_date else story.story_date # For handling future stories m = MUserStory(user_id=self.user_id, feed_id=feed_id, read_date=date, story_id=story.story_guid, story_date=story.story_date) m.save() # Find other social feeds with this story to update their counts friend_key = "F:%s:F" % (self.user_id) share_key = "S:%s:%s" % (feed_id, story.guid_hash) friends_with_shares = [int(f) for f in r.sinter(share_key, friend_key)] if self.user_id in friends_with_shares: friends_with_shares.remove(self.user_id) if friends_with_shares: socialsubs = MSocialSubscription.objects.filter(user_id=self.user_id, subscription_user_id__in=friends_with_shares) for socialsub in socialsubs: if not socialsub.needs_unread_recalc: socialsub.needs_unread_recalc = True socialsub.save() # Also count on original subscription usersubs = UserSubscription.objects.filter(user=self.user_id, feed=feed_id) if usersubs: usersub = usersubs[0] if not usersub.needs_unread_recalc: usersub.needs_unread_recalc = True usersub.save() return data def mark_feed_read(self): latest_story_date = datetime.datetime.utcnow() # Use the latest story to get last read time. if MSharedStory.objects(user_id=self.subscription_user_id).first(): latest_story_date = MSharedStory.objects(user_id=self.subscription_user_id)\ .order_by('-shared_date').only('shared_date')[0]['shared_date']\ + datetime.timedelta(seconds=1) self.last_read_date = latest_story_date self.mark_read_date = latest_story_date self.unread_count_negative = 0 self.unread_count_positive = 0 self.unread_count_neutral = 0 self.unread_count_updated = latest_story_date self.oldest_unread_story_date = latest_story_date self.needs_unread_recalc = False # Cannot delete these stories, since the original feed may not be read. # Just go 2 weeks back. # UNREAD_CUTOFF = now - datetime.timedelta(days=settings.DAYS_OF_UNREAD) # MUserStory.delete_marked_as_read_stories(self.user_id, self.feed_id, mark_read_date=UNREAD_CUTOFF) self.save() def calculate_feed_scores(self, silent=False): # if not self.needs_unread_recalc: # return now = datetime.datetime.now() UNREAD_CUTOFF = now - datetime.timedelta(days=settings.DAYS_OF_UNREAD) user = User.objects.get(pk=self.user_id) if user.profile.last_seen_on < UNREAD_CUTOFF: # if not silent: # logging.info(' ---> [%s] SKIPPING Computing scores: %s (1 week+)' % (self.user, self.feed)) return self feed_scores = dict(negative=0, neutral=0, positive=0) # Two weeks in age. If mark_read_date is older, mark old stories as read. date_delta = UNREAD_CUTOFF if date_delta < self.mark_read_date: date_delta = self.mark_read_date else: self.mark_read_date = date_delta stories_db = MSharedStory.objects(user_id=self.subscription_user_id, shared_date__gte=date_delta) story_feed_ids = set() story_ids = [] for s in stories_db: story_feed_ids.add(s['story_feed_id']) story_ids.append(s['story_guid']) story_feed_ids = list(story_feed_ids) usersubs = UserSubscription.objects.filter(user__pk=self.user_id, feed__pk__in=story_feed_ids) usersubs_map = dict((sub.feed_id, sub) for sub in usersubs) # usersubs = UserSubscription.objects.filter(user__pk=user.pk, feed__pk__in=story_feed_ids) # usersubs_map = dict((sub.feed_id, sub) for sub in usersubs) read_stories_ids = [] if story_feed_ids: read_stories = MUserStory.objects(user_id=self.user_id, feed_id__in=story_feed_ids, story_id__in=story_ids) read_stories_ids = [rs.story_id for rs in read_stories] oldest_unread_story_date = now unread_stories_db = [] for story in stories_db: if getattr(story, 'story_guid', None) in read_stories_ids: continue feed_id = story.story_feed_id if usersubs_map.get(feed_id) and story.story_date < usersubs_map[feed_id].mark_read_date: continue unread_stories_db.append(story) if story.story_date < oldest_unread_story_date: oldest_unread_story_date = story.story_date stories = Feed.format_stories(unread_stories_db) classifier_feeds = list(MClassifierFeed.objects(user_id=self.user_id, social_user_id=self.subscription_user_id)) classifier_authors = list(MClassifierAuthor.objects(user_id=self.user_id, social_user_id=self.subscription_user_id)) classifier_titles = list(MClassifierTitle.objects(user_id=self.user_id, social_user_id=self.subscription_user_id)) classifier_tags = list(MClassifierTag.objects(user_id=self.user_id, social_user_id=self.subscription_user_id)) # Merge with feed specific classifiers if story_feed_ids: classifier_feeds = classifier_feeds + list(MClassifierFeed.objects(user_id=self.user_id, feed_id__in=story_feed_ids)) classifier_authors = classifier_authors + list(MClassifierAuthor.objects(user_id=self.user_id, feed_id__in=story_feed_ids)) classifier_titles = classifier_titles + list(MClassifierTitle.objects(user_id=self.user_id, feed_id__in=story_feed_ids)) classifier_tags = classifier_tags + list(MClassifierTag.objects(user_id=self.user_id, feed_id__in=story_feed_ids)) for story in stories: scores = { 'feed' : apply_classifier_feeds(classifier_feeds, story['story_feed_id'], social_user_id=self.subscription_user_id), 'author' : apply_classifier_authors(classifier_authors, story), 'tags' : apply_classifier_tags(classifier_tags, story), 'title' : apply_classifier_titles(classifier_titles, story), } max_score = max(scores['author'], scores['tags'], scores['title']) min_score = min(scores['author'], scores['tags'], scores['title']) if max_score > 0: feed_scores['positive'] += 1 elif min_score < 0: feed_scores['negative'] += 1 else: if scores['feed'] > 0: feed_scores['positive'] += 1 elif scores['feed'] < 0: feed_scores['negative'] += 1 else: feed_scores['neutral'] += 1 self.unread_count_positive = feed_scores['positive'] self.unread_count_neutral = feed_scores['neutral'] self.unread_count_negative = feed_scores['negative'] self.unread_count_updated = datetime.datetime.now() self.oldest_unread_story_date = oldest_unread_story_date self.needs_unread_recalc = False self.save() if (self.unread_count_positive == 0 and self.unread_count_neutral == 0 and self.unread_count_negative == 0): self.mark_feed_read() if not silent: logging.info(' ---> [%s] Computing social scores: %s (%s/%s/%s)' % (user.username, self.subscription_user_id, feed_scores['negative'], feed_scores['neutral'], feed_scores['positive'])) return self class MCommentReply(mongo.EmbeddedDocument): user_id = mongo.IntField() publish_date = mongo.DateTimeField() comments = mongo.StringField() def to_json(self): reply = { 'user_id': self.user_id, 'publish_date': relative_timesince(self.publish_date), 'comments': self.comments, } return reply meta = { 'ordering': ['publish_date'], } class MSharedStory(mongo.Document): user_id = mongo.IntField() shared_date = mongo.DateTimeField() comments = mongo.StringField() has_comments = mongo.BooleanField(default=False) has_replies = mongo.BooleanField(default=False) replies = mongo.ListField(mongo.EmbeddedDocumentField(MCommentReply)) story_feed_id = mongo.IntField() story_date = mongo.DateTimeField() story_title = mongo.StringField(max_length=1024) story_content = mongo.StringField() story_content_z = mongo.BinaryField() story_original_content = mongo.StringField() story_original_content_z = mongo.BinaryField() story_content_type = mongo.StringField(max_length=255) story_author_name = mongo.StringField() story_permalink = mongo.StringField() story_guid = mongo.StringField(unique_with=('user_id',)) story_tags = mongo.ListField(mongo.StringField(max_length=250)) meta = { 'collection': 'shared_stories', 'indexes': [('user_id', '-shared_date'), ('user_id', 'story_feed_id'), 'shared_date', 'story_guid', 'story_feed_id'], 'index_drop_dups': True, 'ordering': ['shared_date'], 'allow_inheritance': False, } @property def guid_hash(self): return hashlib.sha1(self.story_guid).hexdigest() def save(self, *args, **kwargs): if self.story_content: self.story_content_z = zlib.compress(self.story_content) self.story_content = None if self.story_original_content: self.story_original_content_z = zlib.compress(self.story_original_content) self.story_original_content = None r = redis.Redis(connection_pool=settings.REDIS_POOL) share_key = "S:%s:%s" % (self.story_feed_id, self.guid_hash) r.sadd(share_key, self.user_id) comment_key = "C:%s:%s" % (self.story_feed_id, self.guid_hash) if self.has_comments: r.sadd(comment_key, self.user_id) else: r.srem(comment_key, self.user_id) self.shared_date = self.shared_date or datetime.datetime.utcnow() self.has_replies = bool(len(self.replies)) super(MSharedStory, self).save(*args, **kwargs) author, _ = MSocialProfile.objects.get_or_create(user_id=self.user_id) author.count() MActivity.new_shared_story(user_id=self.user_id, story_title=self.story_title, comments=self.comments, story_feed_id=self.story_feed_id, story_id=self.story_guid, share_date=self.shared_date) def delete(self, *args, **kwargs): r = redis.Redis(connection_pool=settings.REDIS_POOL) share_key = "S:%s:%s" % (self.story_feed_id, self.guid_hash) r.srem(share_key, self.user_id) super(MSharedStory, self).delete(*args, **kwargs) @classmethod def switch_feed(cls, original_feed_id, duplicate_feed_id): shared_stories = cls.objects.filter(story_feed_id=duplicate_feed_id) logging.info(" ---> %s shared stories" % shared_stories.count()) for story in shared_stories: story.story_feed_id = original_feed_id story.save() @classmethod def count_popular_stories(cls, verbose=True): popular_profile = MSocialProfile.objects.get(username='popular') popular_user = User.objects.get(pk=popular_profile.user_id) shared_story = cls.objects.all().order_by('-shared_date')[0] # TODO: Get actual popular stories. story = MStory.objects(story_feed_id=shared_story.story_feed_id, story_guid=shared_story.story_guid).limit(1).first() if not story: logging.user(popular_user, "~FRPopular stories: story not found") return story_db = dict([(k, v) for k, v in story._data.items() if k is not None and v is not None]) story_values = dict(user_id=popular_profile.user_id, has_comments=False, **story_db) MSharedStory.objects.create(**story_values) if verbose: shares = cls.objects.filter(story_guid=story.story_guid).count() logging.user(popular_user, "~FCSharing: ~SB~FM%s (%s shares)" % (story.story_title[:50], shares)) @classmethod def sync_all_redis(cls): r = redis.Redis(connection_pool=settings.REDIS_POOL) for story in cls.objects.all(): story.sync_redis(redis_conn=r) def sync_redis(self, redis_conn=None): if not redis_conn: redis_conn = redis.Redis(connection_pool=settings.REDIS_POOL) share_key = "S:%s:%s" % (self.story_feed_id, self.guid_hash) comment_key = "C:%s:%s" % (self.story_feed_id, self.guid_hash) redis_conn.sadd(share_key, self.user_id) if self.has_comments: redis_conn.sadd(comment_key, self.user_id) else: redis_conn.srem(comment_key, self.user_id) def publish_update_to_subscribers(self): try: r = redis.Redis(connection_pool=settings.REDIS_POOL) feed_id = "social:%s" % self.user_id listeners_count = r.publish(feed_id, 'story:new') if listeners_count: logging.debug(" ---> ~FMPublished to %s subscribers" % (listeners_count)) except redis.ConnectionError: logging.debug(" ***> ~BMRedis is unavailable for real-time.") def comments_with_author(self): comments = { 'user_id': self.user_id, 'comments': self.comments, 'shared_date': relative_timesince(self.shared_date), 'replies': [reply.to_json() for reply in self.replies], } return comments @classmethod def stories_with_comments_and_profiles(cls, stories, user, check_all=False): r = redis.Redis(connection_pool=settings.REDIS_POOL) friend_key = "F:%s:F" % (user.pk) profile_user_ids = set() for story in stories: if check_all or story['comment_count']: comment_key = "C:%s:%s" % (story['story_feed_id'], story['guid_hash']) if check_all: story['comment_count'] = r.scard(comment_key) friends_with_comments = r.sinter(comment_key, friend_key) shared_stories = [] if friends_with_comments: params = { 'story_guid': story['id'], 'story_feed_id': story['story_feed_id'], 'user_id__in': friends_with_comments, } shared_stories = cls.objects.filter(**params) story['comments'] = [] for shared_story in shared_stories: comments = shared_story.comments_with_author() story['comments'].append(comments) profile_user_ids = profile_user_ids.union([reply['user_id'] for c in story['comments'] for reply in c['replies']]) story['comment_count_public'] = story['comment_count'] - len(shared_stories) story['comment_count_friends'] = len(shared_stories) if check_all or story['share_count']: share_key = "S:%s:%s" % (story['story_feed_id'], story['guid_hash']) if check_all: story['share_count'] = r.scard(share_key) friends_with_shares = [int(f) for f in r.sinter(share_key, friend_key)] nonfriend_user_ids = [int(f) for f in r.sdiff(share_key, friend_key)] profile_user_ids.update(nonfriend_user_ids) profile_user_ids.update(friends_with_shares) story['shared_by_public'] = nonfriend_user_ids story['shared_by_friends'] = friends_with_shares story['share_count_public'] = story['share_count'] - len(friends_with_shares) story['share_count_friends'] = len(friends_with_shares) profiles = MSocialProfile.objects.filter(user_id__in=list(profile_user_ids)) profiles = [profile.to_json(compact=True) for profile in profiles] return stories, profiles class MSocialServices(mongo.Document): user_id = mongo.IntField() autofollow = mongo.BooleanField(default=True) twitter_uid = mongo.StringField() twitter_access_key = mongo.StringField() twitter_access_secret = mongo.StringField() twitter_friend_ids = mongo.ListField(mongo.StringField()) twitter_picture_url = mongo.StringField() twitter_username = mongo.StringField() twitter_refresh_date = mongo.DateTimeField() facebook_uid = mongo.StringField() facebook_access_token = mongo.StringField() facebook_friend_ids = mongo.ListField(mongo.StringField()) facebook_picture_url = mongo.StringField() facebook_refresh_date = mongo.DateTimeField() upload_picture_url = mongo.StringField() meta = { 'collection': 'social_services', 'indexes': ['user_id', 'twitter_friend_ids', 'facebook_friend_ids', 'twitter_uid', 'facebook_uid'], 'allow_inheritance': False, } def __unicode__(self): user = User.objects.get(pk=self.user_id) return "%s (Twitter: %s, FB: %s)" % (user.username, self.twitter_uid, self.facebook_uid) def to_json(self): user = User.objects.get(pk=self.user_id) return { 'twitter': { 'twitter_username': self.twitter_username, 'twitter_picture_url': self.twitter_picture_url, 'twitter_uid': self.twitter_uid, }, 'facebook': { 'facebook_uid': self.facebook_uid, 'facebook_picture_url': self.facebook_picture_url, }, 'gravatar': { 'gravatar_picture_url': "http://www.gravatar.com/avatar/" + \ hashlib.md5(user.email).hexdigest() }, 'upload': { 'upload_picture_url': self.upload_picture_url } } def twitter_api(self): twitter_consumer_key = settings.TWITTER_CONSUMER_KEY twitter_consumer_secret = settings.TWITTER_CONSUMER_SECRET auth = tweepy.OAuthHandler(twitter_consumer_key, twitter_consumer_secret) auth.set_access_token(self.twitter_access_key, self.twitter_access_secret) api = tweepy.API(auth) return api def facebook_api(self): graph = facebook.GraphAPI(self.facebook_access_token) return graph def sync_twitter_friends(self): api = self.twitter_api() if not api: return friend_ids = list(unicode(friend.id) for friend in tweepy.Cursor(api.friends).items()) if not friend_ids: return twitter_user = api.me() self.twitter_picture_url = twitter_user.profile_image_url self.twitter_username = twitter_user.screen_name self.twitter_friend_ids = friend_ids self.twitter_refreshed_date = datetime.datetime.utcnow() self.save() self.follow_twitter_friends() profile, _ = MSocialProfile.objects.get_or_create(user_id=self.user_id) profile.location = profile.location or twitter_user.location profile.bio = profile.bio or twitter_user.description profile.website = profile.website or twitter_user.url profile.save() profile.count() if not profile.photo_url or not profile.photo_service: self.set_photo('twitter') def sync_facebook_friends(self): graph = self.facebook_api() if not graph: return friends = graph.get_connections("me", "friends") if not friends: return facebook_friend_ids = [unicode(friend["id"]) for friend in friends["data"]] self.facebook_friend_ids = facebook_friend_ids self.facebook_refresh_date = datetime.datetime.utcnow() self.facebook_picture_url = "//graph.facebook.com/%s/picture" % self.facebook_uid self.save() self.follow_facebook_friends() facebook_user = graph.request('me', args={'fields':'website,bio,location'}) profile, _ = MSocialProfile.objects.get_or_create(user_id=self.user_id) profile.location = profile.location or (facebook_user.get('location') and facebook_user['location']['name']) profile.bio = profile.bio or facebook_user.get('bio') profile.website = profile.website or facebook_user.get('website') profile.save() profile.count() if not profile.photo_url or not profile.photo_service: self.set_photo('facebook') def follow_twitter_friends(self): social_profile, _ = MSocialProfile.objects.get_or_create(user_id=self.user_id) following = [] followers = 0 if self.autofollow: # Follow any friends already on NewsBlur user_social_services = MSocialServices.objects.filter(twitter_uid__in=self.twitter_friend_ids) for user_social_service in user_social_services: followee_user_id = user_social_service.user_id social_profile.follow_user(followee_user_id) following.append(followee_user_id) # Follow any friends already on NewsBlur following_users = MSocialServices.objects.filter(twitter_friend_ids__contains=self.twitter_uid) for following_user in following_users: if following_user.autofollow: following_user_profile = MSocialProfile.objects.get(user_id=following_user.user_id) following_user_profile.follow_user(self.user_id, check_unfollowed=True) followers += 1 user = User.objects.get(pk=self.user_id) logging.user(user, "~BB~FRTwitter import: following ~SB%s~SN with ~SB%s~SN follower-backs" % (following, followers)) return following def follow_facebook_friends(self): social_profile, _ = MSocialProfile.objects.get_or_create(user_id=self.user_id) following = [] followers = 0 if self.autofollow: # Follow any friends already on NewsBlur user_social_services = MSocialServices.objects.filter(facebook_uid__in=self.facebook_friend_ids) for user_social_service in user_social_services: followee_user_id = user_social_service.user_id social_profile.follow_user(followee_user_id) following.append(followee_user_id) # Friends already on NewsBlur should follow back following_users = MSocialServices.objects.filter(facebook_friend_ids__contains=self.facebook_uid) for following_user in following_users: if following_user.autofollow: following_user_profile = MSocialProfile.objects.get(user_id=following_user.user_id) following_user_profile.follow_user(self.user_id, check_unfollowed=True) followers += 1 user = User.objects.get(pk=self.user_id) logging.user(user, "~BB~FRFacebook import: following ~SB%s~SN with ~SB%s~SN follower-backs" % (len(following), followers)) return following def disconnect_twitter(self): self.twitter_uid = None self.save() def disconnect_facebook(self): self.facebook_uid = None self.save() def set_photo(self, service): profile, _ = MSocialProfile.objects.get_or_create(user_id=self.user_id) if service == 'nothing': service = None profile.photo_service = service if not service: profile.photo_url = None elif service == 'twitter': profile.photo_url = self.twitter_picture_url elif service == 'facebook': profile.photo_url = self.facebook_picture_url elif service == 'upload': profile.photo_url = self.upload_picture_url elif service == 'gravatar': user = User.objects.get(pk=self.user_id) profile.photo_url = "http://www.gravatar.com/avatar/" + \ hashlib.md5(user.email).hexdigest() profile.save() return profile class MInteraction(mongo.Document): user_id = mongo.IntField() date = mongo.DateTimeField(default=datetime.datetime.now) category = mongo.StringField() title = mongo.StringField() content = mongo.StringField() with_user_id = mongo.IntField() feed_id = mongo.IntField() content_id = mongo.StringField() meta = { 'collection': 'interactions', 'indexes': [('user_id', 'date'), 'category'], 'allow_inheritance': False, 'index_drop_dups': True, 'ordering': ['-date'], } def __unicode__(self): user = User.objects.get(pk=self.user_id) with_user = self.with_user_id and User.objects.get(pk=self.with_user_id) return "<%s> %s on %s: %s - %s" % (user.username, with_user and with_user.username, self.date, self.category, self.content and self.content[:20]) @classmethod def user(cls, user, page=1): page = max(1, page) limit = 4 # Also set in template offset = (page-1) * limit interactions_db = cls.objects.filter(user_id=user.pk)[offset:offset+limit+1] with_user_ids = [i.with_user_id for i in interactions_db if i.with_user_id] social_profiles = dict((p.user_id, p) for p in MSocialProfile.objects.filter(user_id__in=with_user_ids)) interactions = [] for interaction_db in interactions_db: interaction = interaction_db.to_mongo() social_profile = social_profiles.get(interaction_db.with_user_id) if social_profile: interaction['photo_url'] = social_profile.profile_photo_url interaction['with_user'] = social_profiles.get(interaction_db.with_user_id) interaction['date'] = relative_timesince(interaction_db.date) interactions.append(interaction) return interactions @classmethod def new_follow(cls, follower_user_id, followee_user_id): cls.objects.get_or_create(user_id=followee_user_id, with_user_id=follower_user_id, category='follow') @classmethod def new_comment_reply(cls, user_id, reply_user_id, reply_content, social_feed_id, story_id): cls.objects.create(user_id=user_id, with_user_id=reply_user_id, category='comment_reply', content=reply_content, feed_id=social_feed_id, content_id=story_id) @classmethod def new_reply_reply(cls, user_id, reply_user_id, reply_content, social_feed_id, story_id): cls.objects.create(user_id=user_id, with_user_id=reply_user_id, category='reply_reply', content=reply_content, feed_id=social_feed_id, content_id=story_id) class MActivity(mongo.Document): user_id = mongo.IntField() date = mongo.DateTimeField(default=datetime.datetime.now) category = mongo.StringField() title = mongo.StringField() content = mongo.StringField() with_user_id = mongo.IntField() feed_id = mongo.IntField() content_id = mongo.StringField() meta = { 'collection': 'activities', 'indexes': [('user_id', 'date'), 'category'], 'allow_inheritance': False, 'index_drop_dups': True, 'ordering': ['-date'], } def __unicode__(self): user = User.objects.get(pk=self.user_id) return "<%s> %s - %s" % (user.username, self.category, self.content and self.content[:20]) @classmethod def user(cls, user, page=1): page = max(1, page) limit = 4 # Also set in template offset = (page-1) * limit activities_db = cls.objects.filter(user_id=user.pk)[offset:offset+limit+1] with_user_ids = [a.with_user_id for a in activities_db if a.with_user_id] social_profiles = dict((p.user_id, p) for p in MSocialProfile.objects.filter(user_id__in=with_user_ids)) activities = [] for activity_db in activities_db: activity = activity_db.to_mongo() activity['date'] = relative_timesince(activity_db.date) social_profile = social_profiles.get(activity_db.with_user_id) if social_profile: activity['photo_url'] = social_profile.profile_photo_url activity['with_user'] = social_profiles.get(activity_db.with_user_id) activities.append(activity) return activities @classmethod def new_starred_story(cls, user_id, story_title, story_feed_id, story_id): cls.objects.get_or_create(user_id=user_id, category='star', content=story_title, feed_id=story_feed_id, content_id=story_id) @classmethod def new_feed_subscription(cls, user_id, feed_id, feed_title): cls.objects.create(user_id=user_id, category='feedsub', content=feed_title, feed_id=feed_id) @classmethod def new_follow(cls, follower_user_id, followee_user_id): cls.objects.get_or_create(user_id=follower_user_id, with_user_id=followee_user_id, category='follow') @classmethod def new_comment_reply(cls, user_id, comment_user_id, reply_content, story_feed_id, story_id): cls.objects.create(user_id=user_id, with_user_id=comment_user_id, category='comment_reply', content=reply_content, feed_id=story_feed_id, content_id=story_id) @classmethod def new_shared_story(cls, user_id, story_title, comments, story_feed_id, story_id, share_date=None): a, _ = cls.objects.get_or_create(user_id=user_id, with_user_id=user_id, category='sharedstory', feed_id=story_feed_id, content_id=story_id, defaults={ 'title': story_title, 'content': comments, }) if a.content != comments: a.content = comments a.save() if share_date: a.date = share_date a.save()