Merge remote-tracking branch 'upstream/master'

Conflicts:
	clients/android/NewsBlur/src/com/newsblur/network/APIClient.java
	clients/android/NewsBlur/src/com/newsblur/network/domain/LoginErrors.java
	clients/android/NewsBlur/src/com/newsblur/network/domain/LoginResponse.java
	clients/android/NewsBlur/src/com/newsblur/network/domain/Message.java
	media/android
	media/android/NewsBlur/src/com/newsblur/network/domain/LoginErrors.java
	media/android/NewsBlur/src/com/newsblur/network/domain/ResponseErrors.java

apps/feed_import/models.py

@@ -4,6 +4,7 @@ import httplib2
 import pickle
 import base64
 from StringIO import StringIO
+from oauth2client.client import Error as OAuthError
 from xml.etree.ElementTree import Element, SubElement, Comment, tostring
 from lxml import etree
 from django.db import models
@@ -344,7 +345,10 @@ class GoogleReaderImporter(Importer):
         
     def test(self):
         sub_url = "%s/0/token" % (self.scope)
-        resp = self.send_request(sub_url)
+        try:
+            resp = self.send_request(sub_url)
+        except OAuthError:
+            return False
         return resp
     
     @timelimit(10)

apps/feed_import/views.py

@@ -1,6 +1,7 @@
 import datetime
 import pickle
 import base64
+import httplib2
 from utils import log as logging
 from oauth2client.client import OAuth2WebServerFlow, FlowExchangeError
 from bson.errors import InvalidStringData
@@ -146,6 +147,8 @@ def reader_callback(request):
         )
     FLOW.redirect_uri = STEP2_URI
     
+    http = httplib2.Http()
+    http.disable_ssl_certificate_validation = True
     try:
         credential = FLOW.step2_exchange(request.REQUEST)
     except FlowExchangeError:

apps/profile/middleware.py

@@ -3,10 +3,11 @@ import re
 import random
 import time
 from utils import log as logging
+from django.http import HttpResponse
 from django.conf import settings
 from django.db import connection
 from django.template import Template, Context
-
+from utils import json_functions as json
 
 class LastSeenMiddleware(object):
     def process_response(self, request, response):
@@ -175,3 +176,25 @@ class ServerHostnameMiddleware:
 class TimingMiddleware:
     def process_request(self, request):
         setattr(request, 'start_time', time.time())
+
+BANNED_USER_AGENTS = (
+    'feed reader-background',
+    'feed reader-windows',
+    'missing',
+)
+class UserAgentBanMiddleware:
+    def process_request(self, request):
+        user_agent = request.environ.get('HTTP_USER_AGENT', 'missing').lower()
+        
+        if 'profile' in request.path: return
+        
+        if any(ua in user_agent for ua in BANNED_USER_AGENTS):
+            data = {
+                'error': 'User agent banned: %s' % user_agent,
+                'code': -1
+            }
+            logging.user(request, "~FB~SN~BBBanned UA: ~SB%s" % (user_agent))
+
+            return HttpResponse(json.encode(data), status=403, mimetype='text/json')
+            
+

apps/profile/models.py

@@ -261,7 +261,10 @@ class Profile(models.Model):
             transaction = PayPalIPN.objects.filter(custom=self.user.username,
                                                    txn_type='subscr_payment')[0]
             refund = paypal.refund_transaction(transaction.txn_id)
-            refunded = int(float(refund['raw']['TOTALREFUNDEDAMOUNT'][0]))
+            try:
+                refunded = int(float(refund['raw']['TOTALREFUNDEDAMOUNT'][0]))
+            except KeyError:
+                refunded = int(transaction.amount)
             logging.user(self.user, "~FRRefunding paypal payment: $%s" % refunded)
             self.cancel_premium()
         

apps/push/models.py

@@ -12,11 +12,13 @@ import hashlib
 from apps.push import signals
 from apps.rss_feeds.models import Feed
 from utils import log as logging
+from utils.feed_functions import timelimit
 
 DEFAULT_LEASE_SECONDS = 2592000 # 30 days in seconds
 
 class PushSubscriptionManager(models.Manager):
-
+    
+    @timelimit(5)
     def subscribe(self, topic, feed, hub=None, callback=None,
                   lease_seconds=None, force_retry=False):
         if hub is None:

apps/reader/models.py

@@ -4,6 +4,7 @@ import redis
 from utils import log as logging
 from utils import json_functions as json
 from django.db import models, IntegrityError
+from django.db.models import Q
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.core.cache import cache
@@ -12,11 +13,11 @@ from apps.reader.managers import UserSubscriptionManager
 from apps.rss_feeds.models import Feed, MStory, DuplicateFeed
 from apps.analyzer.models import MClassifierFeed, MClassifierAuthor, MClassifierTag, MClassifierTitle
 from apps.analyzer.models import apply_classifier_titles, apply_classifier_feeds, apply_classifier_authors, apply_classifier_tags
-from utils.feed_functions import add_object_to_folder
+from utils.feed_functions import add_object_to_folder, chunks
 
 class UserSubscription(models.Model):
     """
-    A feed which a user has subscrubed to. Carries all of the cached information
+    A feed which a user has subscribed to. Carries all of the cached information
     about the subscription, including unread counts of the three primary scores.
     
     Also has a dirty flag (needs_unread_recalc) which means that the unread counts
@@ -84,6 +85,85 @@ class UserSubscription(models.Model):
                     break
             else:
                 self.delete()
+    
+    @classmethod
+    def subs_for_feeds(cls, user_id, feed_ids=None, read_filter="unread"):
+        usersubs = cls.objects
+        if read_filter == "unread":
+            usersubs = usersubs.filter(Q(unread_count_neutral__gt=0) |
+                                       Q(unread_count_positive__gt=0))
+        if not feed_ids:
+            usersubs = usersubs.filter(user=user_id, 
+                                       active=True).only('feed', 'mark_read_date', 'is_trained')
+        else:
+            usersubs = usersubs.filter(user=user_id, 
+                                       active=True, 
+                                       feed__in=feed_ids).only('feed', 'mark_read_date', 'is_trained')
+        
+        return usersubs
+        
+    @classmethod
+    def story_hashes(cls, user_id, feed_ids=None, usersubs=None, read_filter="unread", order="newest", 
+                     include_timestamps=False, group_by_feed=True):
+        r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
+        pipeline = r.pipeline()
+        
+        if not usersubs:
+            usersubs = cls.subs_for_feeds(user_id, feed_ids=feed_ids, read_filter=read_filter)
+            feed_ids = [sub.feed_id for sub in usersubs]
+            if not feed_ids:
+                return []
+
+        read_dates = dict((us.feed_id, int(us.mark_read_date.strftime('%s'))) for us in usersubs)
+        current_time = int(time.time() + 60*60*24)
+        unread_interval = datetime.datetime.now() - datetime.timedelta(days=settings.DAYS_OF_UNREAD)
+        unread_timestamp = int(time.mktime(unread_interval.timetuple()))-1000
+        story_hashes = {} if group_by_feed else []
+        feed_counter = 0
+
+        for feed_id_group in chunks(feed_ids, 20):
+            pipeline = r.pipeline()
+            for feed_id in feed_id_group:
+                stories_key               = 'F:%s' % feed_id
+                sorted_stories_key        = 'zF:%s' % feed_id
+                read_stories_key          = 'RS:%s:%s' % (user_id, feed_id)
+                unread_stories_key        = 'U:%s:%s' % (user_id, feed_id)
+                unread_ranked_stories_key = 'zU:%s:%s' % (user_id, feed_id)
+                expire_unread_stories_key = False
+            
+                max_score = current_time
+                if read_filter == 'unread':
+                    # +1 for the intersection b/w zF and F, which carries an implicit score of 1.
+                    min_score = read_dates[feed_id] + 1
+                    pipeline.sdiffstore(unread_stories_key, stories_key, read_stories_key)
+                    expire_unread_stories_key = True
+                else:
+                    min_score = unread_timestamp
+                    unread_stories_key = stories_key
+
+                if order == 'oldest':
+                    byscorefunc = pipeline.zrangebyscore
+                else:
+                    byscorefunc = pipeline.zrevrangebyscore
+                    min_score, max_score = max_score, min_score
+            
+                pipeline.zinterstore(unread_ranked_stories_key, [sorted_stories_key, unread_stories_key])
+                byscorefunc(unread_ranked_stories_key, min_score, max_score, withscores=include_timestamps)
+                pipeline.expire(unread_ranked_stories_key, 60*60)
+                if expire_unread_stories_key:
+                    pipeline.delete(unread_stories_key)
+        
+            results = pipeline.execute()
+        
+            for hashes in results:
+                if not isinstance(hashes, list): continue
+                if group_by_feed:
+                    story_hashes[feed_ids[feed_counter]] = hashes
+                    feed_counter += 1
+                else:
+                    story_hashes.extend(hashes)
+
+        return story_hashes
         
     def get_stories(self, offset=0, limit=6, order='newest', read_filter='all', withscores=False, hashes_only=False):
         r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
@@ -157,8 +237,8 @@ class UserSubscription(models.Model):
             return []
         
     @classmethod
-    def feed_stories(cls, user_id, feed_ids, offset=0, limit=6, order='newest', read_filter='all',
-                     usersubs=None):
+    def feed_stories(cls, user_id, feed_ids=None, offset=0, limit=6, 
+                     order='newest', read_filter='all', usersubs=None):
         r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
         
         if order == 'oldest':
@@ -166,41 +246,46 @@ class UserSubscription(models.Model):
         else:
             range_func = r.zrevrange
             
-        if not isinstance(feed_ids, list):
-            feed_ids = [feed_ids]
-
         ranked_stories_keys  = 'zU:%s:feeds' % (user_id)
         unread_ranked_stories_keys  = 'zhU:%s:feeds' % (user_id)
-        unread_story_hashes = cache.get(unread_ranked_stories_keys)
-        if offset and r.exists(ranked_stories_keys) and unread_story_hashes:
+        stories_cached = r.exists(ranked_stories_keys)
+        unreads_cached = True if read_filter == "unread" else r.exists(unread_ranked_stories_keys)
+        if offset and stories_cached and unreads_cached:
             story_hashes = range_func(ranked_stories_keys, offset, limit)
+            if read_filter == "unread":
+                unread_story_hashes = story_hashes
+            else:
+                unread_story_hashes = range_func(unread_ranked_stories_keys, 0, offset+limit)
             return story_hashes, unread_story_hashes
         else:
             r.delete(ranked_stories_keys)
-            cache.delete(unread_ranked_stories_keys)
+            r.delete(unread_ranked_stories_keys)
         
-        if not usersubs and feed_ids:
-            usersubs = cls.objects.filter(user=user_id, feed__in=feed_ids)
-        if usersubs:
-            usersubs = dict((sub.feed_id, sub) for sub in usersubs)
-            
-        unread_feed_story_hashes = {}
-        for feed_id in feed_ids:
-            if feed_id in usersubs:
-                us = usersubs[feed_id]
-            else:
-                continue
-            story_hashes = us.get_stories(offset=0, limit=200, 
-                                          order=order, read_filter=read_filter, 
-                                          withscores=True)
-            unread_feed_story_hashes[feed_id] = us.get_stories(read_filter='unread', limit=200,
-                                                               hashes_only=True)
-            if story_hashes:
-                r.zadd(ranked_stories_keys, **dict(story_hashes))
-            
+        story_hashes = cls.story_hashes(user_id, feed_ids=feed_ids, 
+                                        read_filter=read_filter, order=order, 
+                                        include_timestamps=True,
+                                        group_by_feed=False, usersubs=usersubs)
+        if not story_hashes:
+            return [], []
+        
+        for story_hash_group in chunks(story_hashes, 100):
+            r.zadd(ranked_stories_keys, **dict(story_hash_group))
         story_hashes = range_func(ranked_stories_keys, offset, limit)
+
+        if read_filter == "unread":
+            unread_feed_story_hashes = story_hashes
+        else:
+            unread_story_hashes = cls.story_hashes(user_id, feed_ids=feed_ids, 
+                                                   read_filter="unread", order=order, 
+                                                   include_timestamps=True,
+                                                   group_by_feed=False)
+            if unread_story_hashes:
+                for unread_story_hash_group in chunks(unread_story_hashes, 100):
+                    r.zadd(unread_ranked_stories_keys, **dict(unread_story_hash_group))
+            unread_feed_story_hashes = range_func(unread_ranked_stories_keys, offset, limit)
+        
         r.expire(ranked_stories_keys, 60*60)
-        cache.set(unread_ranked_stories_keys, unread_feed_story_hashes, 24*60*60)
+        r.expire(unread_ranked_stories_keys, 60*60)
         
         return story_hashes, unread_feed_story_hashes
         
@@ -553,28 +638,44 @@ class UserSubscription(models.Model):
 class RUserStory:
     
     @classmethod
-    def mark_read(cls, user_id, story_feed_id, story_hash, r=None):
+    def mark_read(cls, user_id, story_feed_id, story_hash, r=None, r2=None):
         if not r:
             r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
+        if not r2:
+            r2 = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL2)
         
         story_hash = MStory.ensure_story_hash(story_hash, story_feed_id=story_feed_id)
         
         if not story_hash: return
         
+        now = int(time.time())
         all_read_stories_key = 'RS:%s' % (user_id)
         r.sadd(all_read_stories_key, story_hash)
+        r2.sadd(all_read_stories_key, story_hash)
+        r2.zadd('z' + all_read_stories_key, story_hash, now)
         r.expire(all_read_stories_key, settings.DAYS_OF_UNREAD*24*60*60)
+        r2.expire(all_read_stories_key, settings.DAYS_OF_UNREAD*24*60*60)
+        r2.expire('z' + all_read_stories_key, settings.DAYS_OF_UNREAD*24*60*60)
         
         read_story_key = 'RS:%s:%s' % (user_id, story_feed_id)
         r.sadd(read_story_key, story_hash)
+        r2.sadd(read_story_key, story_hash)
+        r2.zadd('z' + read_story_key, story_hash, now)
         r.expire(read_story_key, settings.DAYS_OF_UNREAD*24*60*60)
+        r2.expire(read_story_key, settings.DAYS_OF_UNREAD*24*60*60)
+        r2.expire('z' + read_story_key, settings.DAYS_OF_UNREAD*24*60*60)
     
     @staticmethod
     def mark_unread(user_id, story_feed_id, story_hash):
         r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
+        r2 = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL2)
         
         r.srem('RS:%s' % user_id, story_hash)
+        r2.srem('RS:%s' % user_id, story_hash)
+        r2.zrem('zRS:%s' % user_id, story_hash)
         r.srem('RS:%s:%s' % (user_id, story_feed_id), story_hash)
+        r2.srem('RS:%s:%s' % (user_id, story_feed_id), story_hash)
+        r2.zrem('zRS:%s:%s' % (user_id, story_feed_id), story_hash)
     
     @staticmethod
     def get_stories(user_id, feed_id, r=None):
@@ -586,15 +687,33 @@ class RUserStory:
     @classmethod
     def switch_feed(cls, user_id, old_feed_id, new_feed_id):
         r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
+        r2 = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL2)
         p = r.pipeline()
+        p2 = r2.pipeline()
         story_hashes = cls.get_stories(user_id, old_feed_id, r=r)
-
+        now = int(time.time())
+        
         for story_hash in story_hashes:
             _, hash_story = MStory.split_story_hash(story_hash)
             new_story_hash = "%s:%s" % (new_feed_id, hash_story)
-            p.sadd("RS:%s:%s" % (user_id, new_feed_id), new_story_hash)
+            read_feed_key = "RS:%s:%s" % (user_id, new_feed_id)
+            p.sadd(read_feed_key, new_story_hash)
+            p2.sadd(read_feed_key, new_story_hash)
+            p2.zadd('z' + read_feed_key, new_story_hash, now)
+            p.expire(read_feed_key, settings.DAYS_OF_UNREAD*24*60*60)
+            p2.expire(read_feed_key, settings.DAYS_OF_UNREAD*24*60*60)
+            p2.expire('z' + read_feed_key, settings.DAYS_OF_UNREAD*24*60*60)
+
+            read_user_key = "RS:%s" % (user_id)
+            p.sadd(read_user_key, new_story_hash)
+            p2.sadd(read_user_key, new_story_hash)
+            p2.zadd('z' + read_user_key, new_story_hash, now)
+            p.expire(read_user_key, settings.DAYS_OF_UNREAD*24*60*60)
+            p2.expire(read_user_key, settings.DAYS_OF_UNREAD*24*60*60)
+            p2.expire('z' + read_user_key, settings.DAYS_OF_UNREAD*24*60*60)
         
         p.execute()
+        p2.execute()
         
         if len(story_hashes) > 0:
             logging.info(" ---> %s read stories" % len(story_hashes))
@@ -602,9 +721,12 @@ class RUserStory:
     @classmethod
     def switch_hash(cls, feed_id, old_hash, new_hash):
         r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
+        r2 = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL2)
         p = r.pipeline()
+        p2 = r2.pipeline()
         UNREAD_CUTOFF = datetime.datetime.now() - datetime.timedelta(days=settings.DAYS_OF_UNREAD)
-
+        now = int(time.time())
+        
         usersubs = UserSubscription.objects.filter(feed_id=feed_id, last_read_date__gte=UNREAD_CUTOFF)
         logging.info(" ---> ~SB%s usersubs~SN to switch read story hashes..." % len(usersubs))
         for sub in usersubs:
@@ -612,10 +734,23 @@ class RUserStory:
             read = r.sismember(rs_key, old_hash)
             if read:
                 p.sadd(rs_key, new_hash)
-                p.sadd("RS:%s" % sub.user.pk, new_hash)
+                p2.sadd(rs_key, new_hash)
+                p2.zadd('z' + rs_key, new_hash, now)
+                p.expire(rs_key, settings.DAYS_OF_UNREAD*24*60*60)
+                p2.expire(rs_key, settings.DAYS_OF_UNREAD*24*60*60)
+                p2.expire('z' + rs_key, settings.DAYS_OF_UNREAD*24*60*60)
+                
+                read_user_key = "RS:%s" % sub.user.pk
+                p.sadd(read_user_key, new_hash)
+                p2.sadd(read_user_key, new_hash)
+                p2.zadd('z' + read_user_key, new_hash, now)
+                p.expire(read_user_key, settings.DAYS_OF_UNREAD*24*60*60)
+                p2.expire(read_user_key, settings.DAYS_OF_UNREAD*24*60*60)
+                p2.expire('z' + read_user_key, settings.DAYS_OF_UNREAD*24*60*60)
         
         p.execute()
-
+        p2.execute()
+    
 
 class UserSubscriptionFolders(models.Model):
     """

apps/reader/views.py

@@ -8,6 +8,7 @@ from django.shortcuts import render
 from django.contrib.auth.decorators import login_required
 from django.template.loader import render_to_string
 from django.db import IntegrityError
+from django.db.models import Q
 from django.views.decorators.cache import never_cache
 from django.core.urlresolvers import reverse
 from django.contrib.auth import login as login_user
@@ -55,7 +56,7 @@ from vendor.timezones.utilities import localtime_for_timezone
 @never_cache
 @render_to('reader/dashboard.xhtml')
 def index(request, **kwargs):
-    if request.method == "GET" and request.subdomain and request.subdomain not in ['dev', 'app10', 'www', 'debug']:
+    if request.method == "GET" and request.subdomain and request.subdomain not in ['dev', 'www', 'debug']:
         username = request.subdomain
         try:
             if '.' in username:
@@ -104,6 +105,7 @@ def dashboard(request, **kwargs):
         'statistics'        : statistics,
         'social_profile'    : social_profile,
         'start_import_from_google_reader': start_import_from_google_reader,
+        'debug'             : settings.DEBUG,
     }, "reader/dashboard.xhtml"
     
 def welcome(request, **kwargs):
@@ -233,7 +235,7 @@ def load_feeds(request):
     scheduled_feeds = []
     for sub in user_subs:
         pk = sub.feed_id
-        if update_counts:
+        if update_counts and sub.needs_unread_recalc:
             sub.calculate_feed_scores(silent=True)
         feeds[pk] = sub.canonical(include_favicon=include_favicons)
         
@@ -653,7 +655,7 @@ def load_single_feed(request, feed_id):
     
     # if page <= 1:
     #     import random
-    #     time.sleep(random.randint(0, 6))
+    #     time.sleep(random.randint(0, 1))
 
     return data
 
@@ -784,33 +786,45 @@ def load_river_stories__redis(request):
     order             = request.REQUEST.get('order', 'newest')
     read_filter       = request.REQUEST.get('read_filter', 'unread')
     now               = localtime_for_timezone(datetime.datetime.now(), user.profile.timezone)
-
-    if not feed_ids and not story_hashes:
-        usersubs = UserSubscription.objects.filter(user=user, active=True)
-        feed_ids = [sub.feed_id for sub in usersubs]
-    else:
-        usersubs = UserSubscription.objects.filter(user=user, active=True, feed__in=feed_ids)
-        
+    usersubs          = []
     offset = (page-1) * limit
     limit = page * limit - 1
     story_date_order = "%sstory_date" % ('' if order == 'oldest' else '-')
     
     if story_hashes:
-        unread_feed_story_hashes = []
+        unread_feed_story_hashes = None
         read_filter = 'unread'
     else:
-        story_hashes, unread_feed_story_hashes = UserSubscription.feed_stories(user.pk, feed_ids,
-                                                                               offset=offset, limit=limit,
-                                                                               order=order,
-                                                                               read_filter=read_filter,
-                                                                               usersubs=usersubs)
+        usersubs = UserSubscription.subs_for_feeds(user.pk, feed_ids=feed_ids,
+                                                   read_filter=read_filter)
+        feed_ids = [sub.feed_id for sub in usersubs]
+        if feed_ids:
+            params = {
+                "user_id": user.pk, 
+                "feed_ids": feed_ids,
+                "offset": offset,
+                "limit": limit,
+                "order": order,
+                "read_filter": read_filter,
+                "usersubs": usersubs,
+            }
+            story_hashes, unread_feed_story_hashes = UserSubscription.feed_stories(**params)
+        else:
+            story_hashes = []
+            unread_feed_story_hashes = []
+    
     mstories = MStory.objects(story_hash__in=story_hashes).order_by(story_date_order)
     stories = Feed.format_stories(mstories)
     found_feed_ids = list(set([story['story_feed_id'] for story in stories]))
     stories, user_profiles = MSharedStory.stories_with_comments_and_profiles(stories, user.pk)
+    
+    if not usersubs:
+        usersubs = UserSubscription.subs_for_feeds(user.pk, feed_ids=found_feed_ids,
+                                                   read_filter=read_filter)
+
     trained_feed_ids = [sub.feed_id for sub in usersubs if sub.is_trained]
     found_trained_feed_ids = list(set(trained_feed_ids) & set(found_feed_ids))
-    
+
     # Find starred stories
     if found_feed_ids:
         starred_stories = MStarredStory.objects(
@@ -825,13 +839,13 @@ def load_river_stories__redis(request):
     # Intelligence classifiers for all feeds involved
     if found_trained_feed_ids:
         classifier_feeds = list(MClassifierFeed.objects(user_id=user.pk,
-                                                   feed_id__in=found_trained_feed_ids))
+                                                        feed_id__in=found_trained_feed_ids))
         classifier_authors = list(MClassifierAuthor.objects(user_id=user.pk, 
-                                                       feed_id__in=found_trained_feed_ids))
+                                                            feed_id__in=found_trained_feed_ids))
         classifier_titles = list(MClassifierTitle.objects(user_id=user.pk, 
-                                                     feed_id__in=found_trained_feed_ids))
+                                                          feed_id__in=found_trained_feed_ids))
         classifier_tags = list(MClassifierTag.objects(user_id=user.pk, 
-                                                 feed_id__in=found_trained_feed_ids))
+                                                      feed_id__in=found_trained_feed_ids))
     else:
         classifier_feeds = []
         classifier_authors = []
@@ -848,7 +862,8 @@ def load_river_stories__redis(request):
     for story in stories:
         story['read_status'] = 0
         if read_filter == 'all':
-            if story['story_hash'] not in unread_feed_story_hashes.get(story['story_feed_id'], []):
+            if (unread_feed_story_hashes is not None and 
+                story['story_hash'] not in unread_feed_story_hashes):
                 story['read_status'] = 1
         story_date = localtime_for_timezone(story['story_date'], user.profile.timezone)
         story['short_parsed_date'] = format_story_link_date__short(story_date, now)
@@ -882,21 +897,30 @@ def load_river_stories__redis(request):
     
 
 @json.json_view
-def unread_story_hashes(request):
+def unread_story_hashes__old(request):
     user              = get_user(request)
     feed_ids          = [int(feed_id) for feed_id in request.REQUEST.getlist('feed_id') if feed_id]
     include_timestamps = is_true(request.REQUEST.get('include_timestamps', False))
+    usersubs = {}
     
     if not feed_ids:
-        usersubs = UserSubscription.objects.filter(user=user, active=True).only('feed')
+        usersubs = UserSubscription.objects.filter(Q(unread_count_neutral__gt=0) |
+                                                   Q(unread_count_positive__gt=0),
+                                                   user=user, active=True)
         feed_ids = [sub.feed_id for sub in usersubs]
+    else:
+        usersubs = UserSubscription.objects.filter(Q(unread_count_neutral__gt=0) |
+                                                   Q(unread_count_positive__gt=0),
+                                                   user=user, active=True, feed__in=feed_ids)
     
     unread_feed_story_hashes = {}
     story_hash_count = 0
+    
+    usersubs = dict((sub.feed_id, sub) for sub in usersubs)
     for feed_id in feed_ids:
-        try:
-            us = UserSubscription.objects.get(user=user.pk, feed=feed_id)
-        except UserSubscription.DoesNotExist:
+        if feed_id in usersubs:
+            us = usersubs[feed_id]
+        else:
             continue
         if not us.unread_count_neutral and not us.unread_count_positive:
             continue
@@ -906,10 +930,27 @@ def unread_story_hashes(request):
         story_hash_count += len(unread_feed_story_hashes[feed_id])
 
     logging.user(request, "~FYLoading ~FCunread story hashes~FY: ~SB%s feeds~SN (%s story hashes)" % 
-                           (len(feed_ids), story_hash_count))
+                           (len(feed_ids), len(story_hash_count)))
 
     return dict(unread_feed_story_hashes=unread_feed_story_hashes)
 
+@json.json_view
+def unread_story_hashes(request):
+    user               = get_user(request)
+    feed_ids           = [int(feed_id) for feed_id in request.REQUEST.getlist('feed_id') if feed_id]
+    include_timestamps = is_true(request.REQUEST.get('include_timestamps', False))
+    order              = request.REQUEST.get('order', 'newest')
+    read_filter        = request.REQUEST.get('read_filter', 'unread')
+    
+    story_hashes = UserSubscription.story_hashes(user.pk, feed_ids=feed_ids, 
+                                                 order=order, read_filter=read_filter,
+                                                 include_timestamps=include_timestamps)
+
+    logging.user(request, "~FYLoading ~FCunread story hashes~FY: ~SB%s feeds~SN (%s story hashes)" % 
+                           (len(feed_ids), len(story_hashes)))
+
+    return dict(unread_feed_story_hashes=story_hashes)
+
 @ajax_login_required
 @json.json_view
 def mark_all_as_read(request):
@@ -940,7 +981,11 @@ def mark_all_as_read(request):
 @json.json_view
 def mark_story_as_read(request):
     story_ids = request.REQUEST.getlist('story_id')
-    feed_id = int(get_argument_or_404(request, 'feed_id'))
+    try:
+        feed_id = int(get_argument_or_404(request, 'feed_id'))
+    except ValueError:
+        return dict(code=-1, errors=["You must pass a valid feed_id: %s" %
+                                     request.REQUEST.get('feed_id')])
     
     try:
         usersub = UserSubscription.objects.select_related('feed').get(user=request.user, feed=feed_id)
@@ -1064,6 +1109,10 @@ def mark_story_as_unread(request):
     
     story, found_original = MStory.find_story(feed_id, story_id)
     
+    if not story:
+        logging.user(request, "~FY~SBUnread~SN story in feed: %s (NOT FOUND)" % (feed))
+        return dict(code=-1, message="Story not found.")
+    
     if usersub and story.story_date < usersub.mark_read_date:
         # Story is outside the mark as read range, so invert all stories before.
         newer_stories = MStory.objects(story_feed_id=story.story_feed_id,

apps/rss_feeds/models.py

@@ -230,12 +230,16 @@ class Feed(models.Model):
     def sync_redis(self):
         return MStory.sync_feed_redis(self.pk)
         
-    def expire_redis(self, r=None):
+    def expire_redis(self, r=None, r2=None):
         if not r:
             r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
+        if not r2:
+            r2 = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL2)
 
         r.expire('F:%s' % self.pk, settings.DAYS_OF_UNREAD*24*60*60)
+        r2.expire('F:%s' % self.pk, settings.DAYS_OF_UNREAD*24*60*60)
         r.expire('zF:%s' % self.pk, settings.DAYS_OF_UNREAD*24*60*60)
+        r2.expire('zF:%s' % self.pk, settings.DAYS_OF_UNREAD*24*60*60)
     
     @classmethod
     def autocomplete(self, prefix, limit=5):
@@ -325,7 +329,7 @@ class Feed(models.Model):
             create_okay = False
             if feedfinder.isFeed(url):
                 create_okay = True
-            elif aggressive:
+            elif fetch:
                 # Could still be a feed. Just check if there are entries
                 fp = feedparser.parse(url)
                 if len(fp.entries):
@@ -903,7 +907,7 @@ class Feed(models.Model):
                        story_guid = story.get('guid'),
                        story_tags = story_tags
                 )
-                s.extract_image_url()
+                s.extract_image_urls()
                 try:
                     s.save()
                     ret_values['new'] += 1
@@ -964,7 +968,7 @@ class Feed(models.Model):
                 # Do not allow publishers to change the story date once a story is published.
                 # Leads to incorrect unread story counts.
                 # existing_story.story_date = story.get('published') # No, don't
-                existing_story.extract_image_url()
+                existing_story.extract_image_urls()
                 
                 try:
                     existing_story.save()
@@ -1151,7 +1155,7 @@ class Feed(models.Model):
         story['story_title']      = story_db.story_title
         story['story_content']    = story_content
         story['story_permalink']  = story_db.story_permalink
-        story['image_url']        = story_db.image_url
+        story['image_urls']       = story_db.image_urls
         story['story_feed_id']    = feed_id or story_db.story_feed_id
         story['comment_count']    = story_db.comment_count if hasattr(story_db, 'comment_count') else 0
         story['comment_user_ids'] = story_db.comment_user_ids if hasattr(story_db, 'comment_user_ids') else []
@@ -1557,7 +1561,7 @@ class MStory(mongo.Document):
     story_permalink          = mongo.StringField()
     story_guid               = mongo.StringField()
     story_hash               = mongo.StringField()
-    image_url                = mongo.StringField(max_length=1024)
+    image_urls               = mongo.ListField(mongo.StringField(max_length=1024))
     story_tags               = mongo.ListField(mongo.StringField(max_length=250))
     comment_count            = mongo.IntField()
     comment_user_ids         = mongo.ListField(mongo.IntField())
@@ -1742,36 +1746,55 @@ class MStory(mongo.Document):
         
         return story_hashes
     
-    def sync_redis(self, r=None):
+    def sync_redis(self, r=None, r2=None):
         if not r:
             r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
+        if not r2:
+            r2 = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL2)
         UNREAD_CUTOFF = datetime.datetime.now() - datetime.timedelta(days=settings.DAYS_OF_UNREAD)
 
         if self.id and self.story_date > UNREAD_CUTOFF:
-            r.sadd('F:%s' % self.story_feed_id, self.story_hash)
-            r.zadd('zF:%s' % self.story_feed_id, self.story_hash, time.mktime(self.story_date.timetuple()))
+            feed_key = 'F:%s' % self.story_feed_id
+            r.sadd(feed_key, self.story_hash)
+            r.expire(feed_key, settings.DAYS_OF_UNREAD*24*60*60)
+            r2.sadd(feed_key, self.story_hash)
+            r2.expire(feed_key, settings.DAYS_OF_UNREAD*24*60*60)
+            
+            r.zadd('z' + feed_key, self.story_hash, time.mktime(self.story_date.timetuple()))
+            r.expire('z' + feed_key, settings.DAYS_OF_UNREAD*24*60*60)
+            r2.zadd('z' + feed_key, self.story_hash, time.mktime(self.story_date.timetuple()))
+            r2.expire('z' + feed_key, settings.DAYS_OF_UNREAD*24*60*60)
     
-    def remove_from_redis(self, r=None):
+    def remove_from_redis(self, r=None, r2=None):
         if not r:
             r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
+        if not r2:
+            r2 = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL2)
         if self.id:
             r.srem('F:%s' % self.story_feed_id, self.story_hash)
+            r2.srem('F:%s' % self.story_feed_id, self.story_hash)
             r.zrem('zF:%s' % self.story_feed_id, self.story_hash)
+            r2.zrem('zF:%s' % self.story_feed_id, self.story_hash)
 
     @classmethod
     def sync_feed_redis(cls, story_feed_id):
         r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
+        r2 = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL2)
         UNREAD_CUTOFF = datetime.datetime.now() - datetime.timedelta(days=settings.DAYS_OF_UNREAD)
         feed = Feed.get_by_id(story_feed_id)
         stories = cls.objects.filter(story_feed_id=story_feed_id, story_date__gte=UNREAD_CUTOFF)
         r.delete('F:%s' % story_feed_id)
+        r2.delete('F:%s' % story_feed_id)
         r.delete('zF:%s' % story_feed_id)
+        r2.delete('zF:%s' % story_feed_id)
 
         logging.info(" ---> [%-30s] ~FMSyncing ~SB%s~SN stories to redis" % (feed and feed.title[:30] or story_feed_id, stories.count()))
         p = r.pipeline()
+        p2 = r2.pipeline()
         for story in stories:
-            story.sync_redis(r=p)
+            story.sync_redis(r=p, r2=p2)
         p.execute()
+        p2.execute()
         
     def count_comments(self):
         from apps.social.models import MSharedStory
@@ -1787,9 +1810,9 @@ class MStory(mongo.Document):
         self.share_user_ids = [s['user_id'] for s in shares]
         self.save()
     
-    def extract_image_url(self, force=False):
-        if self.image_url and not force:
-            return self.image_url
+    def extract_image_urls(self, force=False):
+        if self.image_urls and not force:
+            return self.image_urls
         
         story_content = self.story_content
         if not story_content and self.story_content_z:
@@ -1798,13 +1821,24 @@ class MStory(mongo.Document):
             return
         
         soup = BeautifulSoup(story_content)
-        image = soup.find('img')
-        if image:
+        images = soup.findAll('img')
+        if not images:
+            return
+        
+        image_urls = []
+        for image in images:
             image_url = image.get('src')
+            if not image_url:
+                continue
             if image_url and len(image_url) >= 1024:
-                return
-            self.image_url = image_url
-            return self.image_url
+                continue
+            image_urls.append(image_url)
+
+        if not image_urls:
+            return
+            
+        self.image_urls = image_urls
+        return self.image_urls
 
     def fetch_original_text(self, force=False, request=None):
         original_text_z = self.original_text_z
@@ -1838,7 +1872,7 @@ class MStarredStory(mongo.Document):
     story_guid               = mongo.StringField()
     story_hash               = mongo.StringField()
     story_tags               = mongo.ListField(mongo.StringField(max_length=250))
-    image_url                = mongo.StringField(max_length=1024)
+    image_urls               = mongo.ListField(mongo.StringField(max_length=1024))
 
     meta = {
         'collection': 'starred_stories',
@@ -1945,7 +1979,7 @@ class MFetchHistory(mongo.Document):
         elif fetch_type == 'push':
             history = fetch_history.push_history or []
 
-        history.insert(0, (date, code, message))
+        history = [[date, code, message]] + history
         if code and code >= 400:
             history = history[:50]
         else:
@@ -1960,44 +1994,11 @@ class MFetchHistory(mongo.Document):
         
         fetch_history.save()
         
-        if exception:
-            MFetchExceptionHistory.add(feed_id, date=date, code=code, 
-                                       message=message, exception=exception)
         if fetch_type == 'feed':
             RStats.add('feed_fetch')
         
         return cls.feed(feed_id, fetch_history=fetch_history)
 
-class MFetchExceptionHistory(mongo.Document):
-    feed_id = mongo.IntField(unique=True)
-    date = mongo.DateTimeField()
-    code = mongo.IntField()
-    message = mongo.StringField()
-    exception = mongo.StringField()
-    
-    meta = {
-        'collection': 'fetch_exception_history',
-        'allow_inheritance': False,
-    }
-    
-    @classmethod
-    def add(cls, feed_id, date=None, code=None, message="", exception=""):
-        if not date:
-            date = datetime.datetime.now()
-        if not isinstance(exception, basestring):
-            exception = unicode(exception)
-        
-        try:
-            fetch_exception = cls.objects.read_preference(pymongo.ReadPreference.PRIMARY)\
-                                         .get(feed_id=feed_id)
-        except cls.DoesNotExist:
-            fetch_exception = cls.objects.create(feed_id=feed_id)
-        fetch_exception.date = date
-        fetch_exception.code = code
-        fetch_exception.message = message
-        fetch_exception.exception = exception
-        fetch_exception.save()
-
 
 class DuplicateFeed(models.Model):
     duplicate_address = models.CharField(max_length=764, db_index=True)

apps/rss_feeds/text_importer.py

@@ -35,13 +35,13 @@ class TextImporter:
             if not skip_save:
                 self.story.original_text_z = zlib.compress(content)
                 self.story.save()
-            logging.user(self.request, "~SN~FYFetched ~FGoriginal text~FY: now ~SB%s bytes~SN vs. was ~SB%s bytes" % (
+            logging.user(self.request, ("~SN~FYFetched ~FGoriginal text~FY: now ~SB%s bytes~SN vs. was ~SB%s bytes" % (
                 len(unicode(content)),
                 self.story.story_content_z and len(zlib.decompress(self.story.story_content_z))
-            ))
+            )), warn_color=False)
         else:
-            logging.user(self.request, "~SN~FRFailed~FY to fetch ~FGoriginal text~FY: was ~SB%s bytes" % (
+            logging.user(self.request, ("~SN~FRFailed~FY to fetch ~FGoriginal text~FY: was ~SB%s bytes" % (
                 self.story.story_content_z and len(zlib.decompress(self.story.story_content_z))
-            ))
+            )), warn_color=False)
         
         return content

apps/rss_feeds/views.py

@@ -11,6 +11,7 @@ from django.template import RequestContext
 from apps.rss_feeds.models import Feed, merge_feeds
 from apps.rss_feeds.models import MFetchHistory
 from apps.rss_feeds.models import MFeedIcon
+from apps.push.models import PushSubscription
 from apps.analyzer.models import get_classifiers_for_user
 from apps.reader.models import UserSubscription
 from apps.rss_feeds.models import MStory
@@ -20,7 +21,7 @@ from utils.feed_functions import relative_timeuntil, relative_timesince
 from utils.user_functions import get_user
 from utils.view_functions import get_argument_or_404
 from utils.view_functions import required_params
-
+from vendor.timezones.utilities import localtime_for_timezone
 
 @json.json_view
 def search_feed(request):
@@ -128,6 +129,7 @@ def feed_autocomplete(request):
 @json.json_view
 def load_feed_statistics(request, feed_id):
     user = get_user(request)
+    timezone = user.profile.timezone
     stats = dict()
     feed = get_object_or_404(Feed, pk=feed_id)
     feed.update_all_statistics()
@@ -140,6 +142,14 @@ def load_feed_statistics(request, feed_id):
     stats['last_update'] = relative_timesince(feed.last_update)
     stats['next_update'] = relative_timeuntil(feed.next_scheduled_update)
     stats['push'] = feed.is_push
+    if feed.is_push:
+        try:
+            stats['push_expires'] = localtime_for_timezone(feed.push.lease_expires, 
+                                                           timezone).strftime("%Y-%m-%d %H:%M:%S")
+        except PushSubscription.DoesNotExist:
+            stats['push_expires'] = 'Missing push'
+            feed.is_push = False
+            feed.save()
 
     # Minutes between updates
     update_interval_minutes = feed.get_next_scheduled_update(force=True, verbose=False)
@@ -172,7 +182,6 @@ def load_feed_statistics(request, feed_id):
     stats['classifier_counts'] = json.decode(feed.data.feed_classifier_counts)
     
     # Fetch histories
-    timezone = user.profile.timezone
     fetch_history = MFetchHistory.feed(feed_id, timezone=timezone)
     stats['feed_fetch_history'] = fetch_history['feed_fetch_history']
     stats['page_fetch_history'] = fetch_history['page_fetch_history']

apps/social/models.py

@@ -1228,7 +1228,7 @@ class MSharedStory(mongo.Document):
     story_permalink          = mongo.StringField()
     story_guid               = mongo.StringField(unique_with=('user_id',))
     story_guid_hash          = mongo.StringField(max_length=6)
-    image_url                = mongo.StringField(max_length=1024)
+    image_urls               = mongo.ListField(mongo.StringField(max_length=1024))
     story_tags               = mongo.ListField(mongo.StringField(max_length=250))
     posted_to_services       = mongo.ListField(mongo.StringField(max_length=20))
     mute_email_users         = mongo.ListField(mongo.IntField())
@@ -1522,6 +1522,7 @@ class MSharedStory(mongo.Document):
     def sync_all_redis(cls, drop=False):
         r = redis.Redis(connection_pool=settings.REDIS_POOL)
         h = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
+        h2 = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL2)
         if drop:
             for key_name in ["C", "S"]:
                 keys = r.keys("%s:*" % key_name)
@@ -1530,7 +1531,7 @@ class MSharedStory(mongo.Document):
                     r.delete(key)
         for story in cls.objects.all():
             story.sync_redis_shares(r=r)
-            story.sync_redis_story(r=h)
+            story.sync_redis_story(r=h, r2=h2)
     
     def sync_redis(self):
         self.sync_redis_shares()
@@ -1548,19 +1549,26 @@ class MSharedStory(mongo.Document):
         else:
             r.srem(comment_key, self.user_id)
 
-    def sync_redis_story(self, r=None):
+    def sync_redis_story(self, r=None, r2=None):
         if not r:
             r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
+        if not r2:
+            r2 = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL2)
         
         if not self.story_db_id:
             self.ensure_story_db_id(save=True)
             
         if self.story_db_id:
             r.sadd('B:%s' % self.user_id, self.feed_guid_hash)
+            r2.sadd('B:%s' % self.user_id, self.feed_guid_hash)
             r.zadd('zB:%s' % self.user_id, self.feed_guid_hash,
                    time.mktime(self.shared_date.timetuple()))
+            r2.zadd('zB:%s' % self.user_id, self.feed_guid_hash,
+                   time.mktime(self.shared_date.timetuple()))
             r.expire('B:%s' % self.user_id, settings.DAYS_OF_UNREAD*24*60*60)
+            r2.expire('B:%s' % self.user_id, settings.DAYS_OF_UNREAD*24*60*60)
             r.expire('zB:%s' % self.user_id, settings.DAYS_OF_UNREAD*24*60*60)
+            r2.expire('zB:%s' % self.user_id, settings.DAYS_OF_UNREAD*24*60*60)
     
     def remove_from_redis(self):
         r = redis.Redis(connection_pool=settings.REDIS_POOL)
@@ -1571,8 +1579,11 @@ class MSharedStory(mongo.Document):
         r.srem(comment_key, self.user_id)
 
         h = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
+        h2 = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL2)
         h.srem('B:%s' % self.user_id, self.feed_guid_hash)
+        h2.srem('B:%s' % self.user_id, self.feed_guid_hash)
         h.zrem('zB:%s' % self.user_id, self.feed_guid_hash)
+        h2.zrem('zB:%s' % self.user_id, self.feed_guid_hash)
 
     def publish_update_to_subscribers(self):
         try:

apps/social/views.py

@@ -1078,7 +1078,9 @@ def ignore_follower(request):
 def find_friends(request):
     query = request.GET['query']
     limit = int(request.GET.get('limit', 3))
-    profiles = MSocialProfile.objects.filter(username__icontains=query)[:limit]
+    profiles = MSocialProfile.objects.filter(username__iexact=query)[:limit]
+    if not profiles:
+        profiles = MSocialProfile.objects.filter(username__icontains=query)[:limit]
     if not profiles:
         profiles = MSocialProfile.objects.filter(email__icontains=query)[:limit]
     if not profiles:

apps/statistics/rstats.py

@@ -73,16 +73,19 @@ class RStats:
         keys          = set()
         errors        = set()
         prefixes      = defaultdict(set)
+        sizes         = defaultdict(int)
         prefixes_ttls = defaultdict(lambda: defaultdict(int))
         prefix_re     = re.compile(r"(\w+):(.*)")
 
         p             = r.pipeline()
         [p.randomkey() for _ in range(sample)]
         keys          = set(p.execute())
-        p             = r.pipeline()
 
+        p             = r.pipeline()
         [p.ttl(key) for key in keys]
         ttls          = p.execute()
+
+        dump = [r.execute_command('dump', key) for key in keys]
         
         for k, key in enumerate(keys):
             match = prefix_re.match(key)
@@ -91,8 +94,13 @@ class RStats:
                 continue
             prefix, rest = match.groups()
             prefixes[prefix].add(rest)
+            sizes[prefix] += len(dump[k])
             ttl = ttls[k]
-            if ttl < 60*60: # 1 hour
+            if ttl < 0: # Never expire
+                prefixes_ttls[prefix]['-'] += 1
+            elif ttl == 0:
+                prefixes_ttls[prefix]['X'] += 1
+            elif ttl < 60*60: # 1 hour
                 prefixes_ttls[prefix]['1h'] += 1
             elif ttl < 60*60*12:
                 prefixes_ttls[prefix]['12h'] += 1
@@ -106,9 +114,11 @@ class RStats:
                 prefixes_ttls[prefix]['2w+'] += 1
         
         keys_count = len(keys)
+        total_size = float(sum([k for k in sizes.values()]))
         print " ---> %s total keys" % keys_count
         for prefix, rest in prefixes.items():
-            print " ---> %4s: (%.4s%%) %s keys (%s)" % (prefix, 100. * (len(rest) / float(keys_count)), len(rest), dict(prefixes_ttls[prefix]))
+            total_expiring = sum([k for p, k in dict(prefixes_ttls[prefix]).items() if p != "-"])
+            print " ---> %4s: (%.4s%% keys - %.4s%% space) %s keys (%s expiring: %s)" % (prefix, 100. * (len(rest) / float(keys_count)), 100 * (sizes[prefix] / total_size), len(rest), total_expiring, dict(prefixes_ttls[prefix]))
         print " ---> %s errors: %s" % (len(errors), errors)
 
 def round_time(dt=None, round_to=60):