Merge branch 'master' into recaptcha

* master: (618 commits)
  Build 107
  iOS v9.1.1, still in development.
  #1249 (dark theme colors)
  iOS: #1254 (settings with stories on bottom)
  iOS: #1257 (highlighting issue)
  iOS: #1258 (crash on start)
  Fixing #1251, twitter error should specify the correct dialog.
  Facebook connect url
  Typo
  Hardcoding facebook url
  https for oauth
  add touch intent to widget initialization instead of only refresh
  Handling ImportError
  Handling MemoryError.
  Adding rate limiting logging.
  Handling missing stories from newsletters.
  Handling missing users on clean spammer.
  Upgrading psychopg.
  v9.1 ios.
  Slight change in colors.
  ...

.gitignore

@@ -69,3 +69,4 @@ clients/android/NewsBlur/.gradle
 clients/android/NewsBlur/build.gradle
 clients/android/NewsBlur/gradle*
 clients/android/NewsBlur/settings.gradle
+/docker/volumes/*

README.md

@@ -32,12 +32,15 @@
    used to store stories, read stories, feed/page fetch histories, and proxied sites.
  * [PostgreSQL](http://www.postgresql.com): Relational database, used to store feeds, 
    subscriptions, and user accounts.
-
+ * [Redis](http://redis.io): Programmer's database, used to assemble stories for the river, store story ids, manage feed fetching schedules, and the minuscule bit of caching that NewsBlur uses.
+ * [Elasticsearch](http://elasticsearch.org): Search database, use for searching stories. Optional.
+ 
 ### Client-side and design
 
  * [jQuery](http://www.jquery.com): Cross-browser compliant JavaScript code. IE works without effort.
  * [Underscore.js](http://underscorejs.org/): Functional programming for JavaScript. 
-   Indispensible.
+   Indispensable.
+ * [Backbone.js](http://backbonejs.org/): Framework for the web app. Also indispensable.
  * Miscellaneous jQuery Plugins: Everything from resizable layouts, to progress 
    bars, sortables, date handling, colors, corners, JSON, animations. 
    [See the complete list](https://github.com/samuelclay/NewsBlur/tree/master/media/js).
@@ -191,15 +194,15 @@ these after the installation below.
 If you are on Ubuntu, you can simply use [Fabric](http://docs.fabfile.org/) to install 
 NewsBlur and its many components. NewsBlur is designed to run on three separate servers: 
 an app server, a db server, and assorted task servers. To install everything on a single 
-machine, read through `fabfile.py` and setup all three servers without repeating the 
-`setup_common` steps.
+machine, read through `fabfile.py` and setup all three servers (app, db, and task) without
+repeating the `setup_common` steps.
 
 ### Finishing Installation
 
 You must perform a few tasks to tie all of the various systems together.
 
- 1. First, copy local_settings.py and fill in your OAuth keys, S3 keys, database names (if not `newsblur`),
-task server/broker address (RabbitMQ), and paths:
+ 1. First, copy local_settings.py and fill in your OAuth keys, S3 keys, database names 
+    (if not `newsblur`), task server/broker address (RabbitMQ), and paths:
 
         cp local_settings.py.template local_settings.py
     
@@ -289,18 +292,18 @@ You got the downtime message either through email or SMS. This is the order of o
  1. Check www.newsblur.com to confirm it's down.
     
     If you don't get a 502 page, then NewsBlur isn't even reachable and you just need to contact [the
-    hosting provider](http://cloud.digitalocean.com/support) and yell at them. 
-    
- 2. Check [Sentry](https://app.getsentry.com/newsblur/app/) and see if the answer is at the top of the 
-    list.
- 
-    This will show if a database (redis, mongo, postgres) can't be found.
+    hosting provider](https://cloudsupport.digitalocean.com/s/createticket) and yell at them. 
 
- 3. Check which servers can't be reached on HAProxy stats page. Basic auth can be found in secrets/configs/haproxy.conf.
+ 2. Check which servers can't be reached on HAProxy stats page. Basic auth can be found in secrets/configs/haproxy.conf. Search the secrets repo for "gimmiestats".
  
     Typically it'll be mongo, but any of the redis or postgres servers can be unreachable due to
     acts of god. Otherwise, a frequent cause is lack of disk space. There are monitors on every DB
     server watching for disk space, emailing me when they're running low, but it still happens.
+    
+ 3. Check [Sentry](https://app.getsentry.com/newsblur/app/) and see if the answer is at the top of the 
+    list.
+ 
+    This will show if a database (redis, mongo, postgres) can't be found.
  
  4. Check the various databases:
 
@@ -356,7 +359,7 @@ You got the downtime message either through email or SMS. This is the order of o
     d. Changes should be instant, but you can also bounce every machine with:
     
        ```
-       fab web deploy:fast=True # fast=True just kill -9's processes.
+       fab web deploy
        fab task celery
        ```
       

apps/analyzer/views.py

@@ -15,7 +15,6 @@ from apps.social.models import MSocialSubscription
 from utils import json_functions as json
 from utils.user_functions import get_user
 from utils.user_functions import ajax_login_required
-from utils.view_functions import render_to
 
 def index(requst):
     pass
@@ -84,10 +83,13 @@ def save_classifier(request):
                     try:
                         classifier = ClassifierCls.objects.get(**classifier_dict)
                     except ClassifierCls.DoesNotExist:
-                        classifier_dict.update(dict(score=score))
-                        classifier = ClassifierCls.objects.create(**classifier_dict)
-                    except NotUniqueError:
-                        continue
+                        classifier = None
+                    if not classifier:
+                        try:
+                            classifier_dict.update(dict(score=score))
+                            classifier = ClassifierCls.objects.create(**classifier_dict)
+                        except NotUniqueError:
+                            continue
                     if score == 0:
                         classifier.delete()
                     elif classifier.score != score:

apps/api/urls.py

@@ -7,6 +7,7 @@ urlpatterns = patterns('',
     url(r'^signup', views.signup, name='api-signup'),
     url(r'^add_site_load_script/(?P<token>\w+)', views.add_site_load_script, name='api-add-site-load-script'),
     url(r'^add_site/(?P<token>\w+)', views.add_site, name='api-add-site'),
+    url(r'^add_site/?$', views.add_site_authed, name='api-add-site-authed'),
     url(r'^check_share_on_site/(?P<token>\w+)', views.check_share_on_site, name='api-check-share-on-site'),
     url(r'^share_story/(?P<token>\w+)', views.share_story, name='api-share-story'),
     url(r'^share_story/?$', views.share_story),

apps/api/views.py

@@ -20,6 +20,7 @@ from utils import json_functions as json
 from utils import log as logging
 from utils.feed_functions import relative_timesince
 from utils.view_functions import required_params
+from utils.user_functions import get_user, ajax_login_required
 
 
 @json.json_view
@@ -156,7 +157,42 @@ def add_site(request, token):
         'message': message,
         'usersub': us and us.feed_id,
     }) + ')', mimetype='text/plain')
+
+@ajax_login_required
+def add_site_authed(request):
+    code       = 0
+    url        = request.GET['url']
+    folder     = request.GET['folder']
+    new_folder = request.GET.get('new_folder')
+    callback   = request.GET['callback']
+    user       = get_user(request)
     
+    if not url:
+        code = -1
+    else:
+        if new_folder:
+            usf, _ = UserSubscriptionFolders.objects.get_or_create(user=user)
+            usf.add_folder(folder, new_folder)
+            folder = new_folder
+        code, message, us = UserSubscription.add_subscription(
+            user=user, 
+            feed_address=url,
+            folder=folder,
+            bookmarklet=True
+        )
+    
+    if code > 0:
+        message = 'OK'
+        
+    logging.user(user, "~FRAdding authed URL from site: ~SB%s (in %s)" % (url, folder),
+                 request=request)
+    
+    return HttpResponse(callback + '(' + json.encode({
+        'code':    code,
+        'message': message,
+        'usersub': us and us.feed_id,
+    }) + ')', mimetype='text/plain')
+
 def check_share_on_site(request, token):
     code       = 0
     story_url  = request.GET['story_url']

apps/newsletters/models.py

@@ -24,7 +24,11 @@ class EmailNewsletter:
         sender_name, sender_username, sender_domain = self._split_sender(params['from'])
         feed_address = self._feed_address(user, "%s@%s" % (sender_username, sender_domain))
         
-        usf = UserSubscriptionFolders.objects.get(user=user)
+        try:
+            usf = UserSubscriptionFolders.objects.get(user=user)
+        except UserSubscriptionFolders.DoesNotExist:
+            logging.user(user, "~FRUser does not have a USF, ignoring newsletter.")
+            return
         usf.add_folder('', 'Newsletters')
         
         try:

apps/newsletters/views.py

@@ -53,6 +53,10 @@ def newsletter_receive(request):
     return response
 
 def newsletter_story(request, story_hash):
-    story = MStory.objects.get(story_hash=story_hash)
+    try:
+        story = MStory.objects.get(story_hash=story_hash)
+    except MStory.DoesNotExist:
+        raise Http404
+
     story = Feed.format_story(story)
     return HttpResponse(story['story_content'])

apps/notifications/models.py

@@ -2,7 +2,7 @@ import datetime
 import enum
 import redis
 import mongoengine as mongo
-import boto
+from boto.ses.connection import BotoServerError
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.contrib.sites.models import Site
@@ -193,7 +193,7 @@ class MUserFeedNotification(mongo.Document):
         soup = BeautifulSoup(story['story_content'].strip())
         # print story['story_content']
         body = replace_with_newlines(soup)
-        body = truncate_chars(body.strip(), 1200)
+        body = truncate_chars(body.strip(), 600)
         if not body:
             body = " "
         
@@ -289,8 +289,9 @@ class MUserFeedNotification(mongo.Document):
         msg.attach_alternative(html, "text/html")
         try:
             msg.send()
-        except boto.ses.connection.ResponseError, e:
+        except BotoServerError, e:
             logging.user(usersub.user, '~BMStory notification by email error: ~FR%s' % e)
+            return
         logging.user(usersub.user, '~BMStory notification by email: ~FY~SB%s~SN~BM~FY/~SB%s' % 
                                    (story['story_title'][:50], usersub.feed.feed_title[:50]))
     

apps/oauth/views.py

@@ -1,5 +1,4 @@
 import urllib
-import urlparse
 import datetime
 import lxml.html
 import tweepy
@@ -90,8 +89,8 @@ def facebook_connect(request):
     
     args = {
         "client_id": facebook_app_id,
-        "redirect_uri": "http://" + Site.objects.get_current().domain + reverse('facebook-connect'),
-        "scope": "user_website,user_friends,publish_actions",
+        "redirect_uri": "https://" + Site.objects.get_current().domain + '/oauth/facebook_connect',
+        "scope": "user_friends",
         "display": "popup",
     }
 
@@ -102,13 +101,13 @@ def facebook_connect(request):
         uri = "https://graph.facebook.com/oauth/access_token?" + \
                 urllib.urlencode(args)
         response_text = urllib.urlopen(uri).read()
-        response = urlparse.parse_qs(response_text)
-
+        response = json.decode(response_text)
+        
         if "access_token" not in response:
-            logging.user(request, "~BB~FRFailed Facebook connect")
+            logging.user(request, "~BB~FRFailed Facebook connect, no access_token. (%s): %s" % (args, response))
             return dict(error="Facebook has returned an error. Try connecting again.")
 
-        access_token = response["access_token"][-1]
+        access_token = response["access_token"]
 
         # Get the user's profile.
         graph = facebook.GraphAPI(access_token)
@@ -138,7 +137,7 @@ def facebook_connect(request):
         logging.user(request, "~BB~FRFinishing Facebook connect")
         return {}
     elif request.REQUEST.get('error'):
-        logging.user(request, "~BB~FRFailed Facebook connect")
+        logging.user(request, "~BB~FRFailed Facebook connect, error: %s" % request.REQUEST.get('error'))
         return {'error': '%s... Try connecting again.' % request.REQUEST.get('error')}
     else:
         # Start the OAuth process
@@ -153,7 +152,7 @@ def appdotnet_connect(request):
     args = {
         "client_id": settings.APPDOTNET_CLIENTID,
         "client_secret": settings.APPDOTNET_SECRET,
-        "redirect_uri": "http://" + domain +
+        "redirect_uri": "https://" + domain +
                                     reverse('appdotnet-connect'),
         "scope": ["email", "write_post", "follow"],
     }
@@ -284,7 +283,12 @@ def api_user_info(request):
 @json.json_view
 def api_feed_list(request, trigger_slug=None):
     user = request.user
-    usf = UserSubscriptionFolders.objects.get(user=user)
+    try:
+        usf = UserSubscriptionFolders.objects.get(user=user)
+    except UserSubscriptionFolders.DoesNotExist:
+        return {"errors": [{
+            'message': 'Could not find feeds for user.'
+        }]}
     flat_folders = usf.flatten_folders()
     titles = [dict(label=" - Folder: All Site Stories", value="all")]
     feeds = {}

apps/profile/forms.py

@@ -181,6 +181,10 @@ class AccountSettingsForm(forms.Form):
         if self.user.email != email:
             self.user.email = email
             self.user.save()
+            
+            sp = MSocialProfile.get_user(self.user.pk)
+            sp.email = email
+            sp.save()
         
         if old_password or new_password:
             change_password(self.user, old_password, new_password)

apps/profile/management/commands/reimport_stripe_history.py

@@ -2,63 +2,23 @@ import stripe, datetime, time
 from django.conf import settings
 
 from django.core.management.base import BaseCommand
-from django.contrib.auth.models import User
 from optparse import make_option
 
 from utils import log as logging
-from apps.profile.models import Profile, PaymentHistory
+from apps.profile.models import Profile
 
 class Command(BaseCommand):
     option_list = BaseCommand.option_list + (
         # make_option("-u", "--username", dest="username", nargs=1, help="Specify user id or username"),
         # make_option("-e", "--email", dest="email", nargs=1, help="Specify email if it doesn't exist"),
         make_option("-d", "--days", dest="days", nargs=1, type='int', default=365, help="Number of days to go back"),
-        make_option("-o", "--offset", dest="offset", nargs=1, type='int', default=0, help="Offset customer (in date DESC)"),
+        make_option("-l", "--limit", dest="limit", nargs=1, type='int', default=100, help="Charges per batch"),
+        make_option("-s", "--start", dest="start", nargs=1, type='string', default=None, help="Offset customer_id (starting_after)"),
     )
 
     def handle(self, *args, **options):
-        stripe.api_key = settings.STRIPE_SECRET
-        week = (datetime.datetime.now() - datetime.timedelta(days=int(options.get('days', 365)))).strftime('%s')
-        failed = []
-        limit = 100
-        offset = options.get('offset')
+        limit = options.get('limit')
+        days = int(options.get('days'))
+        starting_after = options.get('start')
         
-        while True:
-            logging.debug(" ---> At %s" % offset)
-            try:
-                data = stripe.Charge.all(created={'gt': week}, count=limit, offset=offset)
-            except stripe.APIConnectionError:
-                time.sleep(10)
-                continue
-            charges = data['data']
-            if not len(charges):
-                logging.debug("At %s, finished" % offset)
-                break
-            offset += limit
-            customers = [c['customer'] for c in charges if 'customer' in c]
-            for customer in customers:
-                try:
-                    profile = Profile.objects.get(stripe_id=customer)
-                    user = profile.user
-                except Profile.DoesNotExist:
-                    logging.debug(" ***> Couldn't find stripe_id=%s" % customer)
-                    failed.append(customer)
-                try:
-                    if not user.profile.is_premium:
-                        user.profile.activate_premium()
-                    elif user.payments.all().count() != 1:
-                        user.profile.setup_premium_history()
-                    elif not user.profile.premium_expire:
-                        user.profile.setup_premium_history()
-                    elif user.profile.premium_expire > datetime.datetime.now() + datetime.timedelta(days=365):
-                        user.profile.setup_premium_history()
-                    else:
-                        logging.debug(" ---> %s is fine" % user.username)
-                except stripe.APIConnectionError:
-                    logging.debug(" ***> Failed: %s" % user.username)
-                    failed.append(username)
-                    time.sleep(2)
-                    continue
-
-        return failed
-
+        Profile.reimport_stripe_history(limit, days, starting_after)

apps/profile/middleware.py

@@ -59,6 +59,9 @@ class DBProfilerMiddleware:
 
     def process_response(self, request, response):
         if hasattr(request, 'sql_times_elapsed'):
+            middleware = SQLLogToConsoleMiddleware()
+            middleware.process_celery(self)
+            # logging.debug(" ---> ~FGProfiling~FB app: %s" % request.sql_times_elapsed)
             self._save_times(request.sql_times_elapsed)
         return response
     
@@ -69,6 +72,13 @@ class DBProfilerMiddleware:
             logging.debug(" ---> ~FGProfiling~FB task: %s" % self.sql_times_elapsed)
             self._save_times(self.sql_times_elapsed, 'task_')
     
+    def process_request_finished(self):
+        middleware = SQLLogToConsoleMiddleware()
+        middleware.process_celery(self)
+        if hasattr(self, 'sql_times_elapsed'):
+            logging.debug(" ---> ~FGProfiling~FB app: %s" % self.sql_times_elapsed)
+            self._save_times(self.sql_times_elapsed, 'app_')
+    
     def _save_times(self, db_times, prefix=""):
         if not db_times: return
         

apps/profile/migrations/0024_auto__add_stripeids.py

@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'StripeIds'
+        db.create_table(u'profile_stripeids', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('user', self.gf('django.db.models.fields.related.ForeignKey')(related_name='stripe_ids', to=orm['auth.User'])),
+            ('stripe_id', self.gf('django.db.models.fields.CharField')(max_length=24, null=True, blank=True)),
+        ))
+        db.send_create_signal(u'profile', ['StripeIds'])
+
+
+    def backwards(self, orm):
+        # Deleting model 'StripeIds'
+        db.delete_table(u'profile_stripeids')
+
+
+    models = {
+        u'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        u'auth.permission': {
+            'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        u'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        u'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        u'profile.paymenthistory': {
+            'Meta': {'ordering': "['-payment_date']", 'object_name': 'PaymentHistory'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'payment_amount': ('django.db.models.fields.IntegerField', [], {}),
+            'payment_date': ('django.db.models.fields.DateTimeField', [], {}),
+            'payment_identifier': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True'}),
+            'payment_provider': ('django.db.models.fields.CharField', [], {'max_length': '20'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'payments'", 'to': u"orm['auth.User']"})
+        },
+        u'profile.profile': {
+            'Meta': {'object_name': 'Profile'},
+            'collapsed_folders': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'dashboard_date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'feed_pane_size': ('django.db.models.fields.IntegerField', [], {'default': '242'}),
+            'has_found_friends': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
+            'has_setup_feeds': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
+            'has_trained_intelligence': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
+            'hide_getting_started': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_premium': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_seen_ip': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'last_seen_on': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'preferences': ('django.db.models.fields.TextField', [], {'default': "'{}'"}),
+            'premium_expire': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'secret_token': ('django.db.models.fields.CharField', [], {'max_length': '12', 'null': 'True', 'blank': 'True'}),
+            'send_emails': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'stripe_4_digits': ('django.db.models.fields.CharField', [], {'max_length': '4', 'null': 'True', 'blank': 'True'}),
+            'stripe_id': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}),
+            'timezone': ('vendor.timezones.fields.TimeZoneField', [], {'default': "'America/New_York'", 'max_length': '100'}),
+            'tutorial_finished': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'user': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'profile'", 'unique': 'True', 'to': u"orm['auth.User']"}),
+            'view_settings': ('django.db.models.fields.TextField', [], {'default': "'{}'"})
+        },
+        u'profile.stripeids': {
+            'Meta': {'object_name': 'StripeIds'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'stripe_id': ('django.db.models.fields.CharField', [], {'max_length': '24', 'null': 'True', 'blank': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'stripe_ids'", 'to': u"orm['auth.User']"})
+        }
+    }
+
+    complete_apps = ['profile']

apps/profile/models.py

@@ -164,20 +164,23 @@ class Profile(models.Model):
         
         EmailNewPremium.delay(user_id=self.user.pk)
         
+        was_premium = self.is_premium
         self.is_premium = True
         self.save()
         self.user.is_active = True
         self.user.save()
         
+        # Only auto-enable every feed if a free user is moving to premium
         subs = UserSubscription.objects.filter(user=self.user)
-        for sub in subs:
-            if sub.active: continue
-            sub.active = True
-            try:
-                sub.save()
-            except (IntegrityError, Feed.DoesNotExist):
-                pass
-        
+        if not was_premium:
+            for sub in subs:
+                if sub.active: continue
+                sub.active = True
+                try:
+                    sub.save()
+                except (IntegrityError, Feed.DoesNotExist):
+                    pass
+    
         try:
             scheduled_feeds = [sub.feed.pk for sub in subs]
         except Feed.DoesNotExist:
@@ -185,8 +188,9 @@ class Profile(models.Model):
         logging.user(self.user, "~SN~FMTasking the scheduling immediate premium setup of ~SB%s~SN feeds..." % 
                      len(scheduled_feeds))
         SchedulePremiumSetup.apply_async(kwargs=dict(feed_ids=scheduled_feeds))
-        
+    
         UserSubscription.queue_new_feeds(self.user)
+        
         self.setup_premium_history()
         
         if never_expire:
@@ -221,9 +225,10 @@ class Profile(models.Model):
         self.user.save()
         self.send_new_user_queue_email()
         
-    def setup_premium_history(self, alt_email=None, check_premium=False, force_expiration=False):
+    def setup_premium_history(self, alt_email=None, set_premium_expire=True, force_expiration=False):
         paypal_payments = []
         stripe_payments = []
+        total_stripe_payments = 0
         existing_history = PaymentHistory.objects.filter(user=self.user, 
                                                          payment_provider__in=['paypal', 'stripe'])
         if existing_history.count():
@@ -257,17 +262,25 @@ class Profile(models.Model):
                 
         # Record Stripe payments
         if self.stripe_id:
-            stripe.api_key = settings.STRIPE_SECRET
-            stripe_customer = stripe.Customer.retrieve(self.stripe_id)
-            stripe_payments = stripe.Charge.all(customer=stripe_customer.id).data
+            self.retrieve_stripe_ids()
             
-            for payment in stripe_payments:
-                created = datetime.datetime.fromtimestamp(payment.created)
-                if payment.status == 'failed': continue
-                PaymentHistory.objects.create(user=self.user,
-                                              payment_date=created,
-                                              payment_amount=payment.amount / 100.0,
-                                              payment_provider='stripe')
+            stripe.api_key = settings.STRIPE_SECRET
+            seen_payments = set()
+            for stripe_id_model in self.user.stripe_ids.all():
+                stripe_id = stripe_id_model.stripe_id
+                stripe_customer = stripe.Customer.retrieve(stripe_id)
+                stripe_payments = stripe.Charge.all(customer=stripe_customer.id).data
+                
+                for payment in stripe_payments:
+                    created = datetime.datetime.fromtimestamp(payment.created)
+                    if payment.status == 'failed': continue
+                    if created in seen_payments: continue
+                    seen_payments.add(created)
+                    total_stripe_payments += 1
+                    PaymentHistory.objects.get_or_create(user=self.user,
+                                                         payment_date=created,
+                                                         payment_amount=payment.amount / 100.0,
+                                                         payment_provider='stripe')
         
         # Calculate payments in last year, then add together
         payment_history = PaymentHistory.objects.filter(user=self.user)
@@ -284,6 +297,7 @@ class Profile(models.Model):
                     oldest_recent_payment_date = payment.payment_date
         
         if free_lifetime_premium:
+            logging.user(self.user, "~BY~SN~FWFree lifetime premium")
             self.premium_expire = None
             self.save()
         elif oldest_recent_payment_date:
@@ -291,18 +305,64 @@ class Profile(models.Model):
                                   datetime.timedelta(days=365*recent_payments_count))
             # Only move premium expire forward, never earlier. Also set expiration if not premium.
             if (force_expiration or 
-                (check_premium and not self.premium_expire) or 
+                (set_premium_expire and not self.premium_expire) or 
                 (self.premium_expire and new_premium_expire > self.premium_expire)):
                 self.premium_expire = new_premium_expire
                 self.save()
 
         logging.user(self.user, "~BY~SN~FWFound ~SB~FB%s paypal~FW~SN and ~SB~FC%s stripe~FW~SN payments (~SB%s payments expire: ~SN~FB%s~FW)" % (
-                     len(paypal_payments), len(stripe_payments), len(payment_history), self.premium_expire))
+                     len(paypal_payments), total_stripe_payments, len(payment_history), self.premium_expire))
 
-        if (check_premium and not self.is_premium and
+        if (set_premium_expire and not self.is_premium and
             (not self.premium_expire or self.premium_expire > datetime.datetime.now())):
             self.activate_premium()
 
+    @classmethod
+    def reimport_stripe_history(cls, limit=10, days=7, starting_after=None):
+        stripe.api_key = settings.STRIPE_SECRET
+        week = (datetime.datetime.now() - datetime.timedelta(days=days)).strftime('%s')
+        failed = []
+        i = 0
+        
+        while True:
+            logging.debug(" ---> At %s / %s" % (i, starting_after))
+            i += 1
+            try:
+                data = stripe.Charge.all(created={'gt': week}, count=limit, starting_after=starting_after)
+            except stripe.APIConnectionError:
+                time.sleep(10)
+                continue
+            charges = data['data']
+            if not len(charges):
+                logging.debug("At %s (%s), finished" % (i, starting_after))
+                break
+            starting_after = charges[-1]["id"]
+            customers = [c['customer'] for c in charges if 'customer' in c]
+            for customer in customers:
+                if not customer:
+                    print " ***> No customer!"
+                    continue
+                try:
+                    profile = Profile.objects.get(stripe_id=customer)
+                    user = profile.user
+                except Profile.DoesNotExist:
+                    logging.debug(" ***> Couldn't find stripe_id=%s" % customer)
+                    failed.append(customer)
+                    continue
+                except Profile.MultipleObjectsReturned:
+                    logging.debug(" ***> Multiple stripe_id=%s" % customer)
+                    failed.append(customer)
+                    continue
+                try:
+                    user.profile.setup_premium_history()
+                except stripe.APIConnectionError:
+                    logging.debug(" ***> Failed: %s" % user.username)
+                    failed.append(user.username)
+                    time.sleep(2)
+                    continue
+
+        return ','.join(failed)
+        
     def refund_premium(self, partial=False):
         refunded = False
         
@@ -407,6 +467,24 @@ class Profile(models.Model):
         
         return True
     
+    def retrieve_stripe_ids(self):
+        if not self.stripe_id:
+            return
+        
+        stripe.api_key = settings.STRIPE_SECRET
+        stripe_customer = stripe.Customer.retrieve(self.stripe_id)
+        stripe_email = stripe_customer.email
+        
+        stripe_ids = set()
+        for email in set([stripe_email, self.user.email]):
+            customers = stripe.Customer.list(email=email)
+            for customer in customers:
+                stripe_ids.add(customer.stripe_id)
+        
+        self.user.stripe_ids.all().delete()
+        for stripe_id in stripe_ids:
+            self.user.stripe_ids.create(stripe_id=stripe_id)
+        
     @property
     def latest_paypal_email(self):
         ipn = PayPalIPN.objects.filter(custom=self.user.username)
@@ -417,9 +495,11 @@ class Profile(models.Model):
     
     def activate_ios_premium(self, product_identifier, transaction_identifier, amount=36):
         payments = PaymentHistory.objects.filter(user=self.user,
-                                                 payment_identifier=transaction_identifier)
+                                                 payment_identifier=transaction_identifier,
+                                                 payment_date__gte=datetime.datetime.now()-datetime.timedelta(days=3))
         if len(payments):
             # Already paid
+            logging.user(self.user, "~FG~BBAlready paid iOS premium subscription: $%s~FW" % transaction_identifier)
             return False
 
         PaymentHistory.objects.create(user=self.user,
@@ -428,7 +508,7 @@ class Profile(models.Model):
                                       payment_provider='ios-subscription',
                                       payment_identifier=transaction_identifier)
         
-        self.setup_premium_history(check_premium=True)
+        self.setup_premium_history()
                                       
         if not self.is_premium:
             self.activate_premium()
@@ -445,17 +525,28 @@ class Profile(models.Model):
             opens = UserSubscription.objects.filter(user=user).aggregate(sum=Sum('feed_opens'))['sum']
             reads = RUserStory.read_story_count(user.pk)
             has_numbers = numerics.search(user.username)
+
+            try:
+                has_profile = user.profile.last_seen_ip
+            except Profile.DoesNotExist:
+                usernames.add(user.username)
+                print " ---> Missing profile: %-20s %-30s %-6s %-6s" % (user.username, user.email, opens, reads)
+                continue
+
             if opens is None and not reads and has_numbers:
                 usernames.add(user.username)
                 print " ---> Numerics: %-20s %-30s %-6s %-6s" % (user.username, user.email, opens, reads)
-            elif not user.profile.last_seen_ip:
+            elif not has_profile:
                 usernames.add(user.username)
                 print " ---> No IP: %-20s %-30s %-6s %-6s" % (user.username, user.email, opens, reads)
         
         if not confirm: return usernames
         
         for username in usernames:
-            u = User.objects.get(username=username)
+            try:
+                u = User.objects.get(username=username)
+            except User.DoesNotExist:
+                continue
             u.profile.delete_user(confirm=True)
 
         RNewUserQueue.user_count()
@@ -956,9 +1047,17 @@ class Profile(models.Model):
             except Profile.DoesNotExist:
                 logging.debug(" ---> ~FRCouldn't find user: ~SB~FC%s" % payment.custom)
                 continue
-            profile.setup_premium_history(check_premium=True)
+            profile.setup_premium_history()
+        
+
+class StripeIds(models.Model):
+    user = models.ForeignKey(User, related_name='stripe_ids')
+    stripe_id = models.CharField(max_length=24, blank=True, null=True)
+
+    def __unicode__(self):
+        return "%s: %s" % (self.user.username, self.stripe_id)
+
         
-            
 def create_profile(sender, instance, created, **kwargs):
     if created:
         Profile.objects.create(user=instance)
@@ -993,7 +1092,7 @@ def paypal_payment_history_sync(sender, **kwargs):
         user = User.objects.get(email__iexact=ipn_obj.payer_email)
     logging.user(user, "~BC~SB~FBPaypal subscription payment")
     try:
-        user.profile.setup_premium_history(check_premium=True)
+        user.profile.setup_premium_history()
     except:
         return {"code": -1, "message": "User doesn't exist."}
 payment_was_successful.connect(paypal_payment_history_sync)
@@ -1006,7 +1105,7 @@ def paypal_payment_was_flagged(sender, **kwargs):
         if ipn_obj.payer_email:
             user = User.objects.get(email__iexact=ipn_obj.payer_email)
     try:
-        user.profile.setup_premium_history(check_premium=True)
+        user.profile.setup_premium_history()
         logging.user(user, "~BC~SB~FBPaypal subscription payment flagged")
     except:
         return {"code": -1, "message": "User doesn't exist."}
@@ -1020,7 +1119,7 @@ def paypal_recurring_payment_history_sync(sender, **kwargs):
         user = User.objects.get(email__iexact=ipn_obj.payer_email)
     logging.user(user, "~BC~SB~FBPaypal subscription recurring payment")
     try:
-        user.profile.setup_premium_history(check_premium=True)
+        user.profile.setup_premium_history()
     except:
         return {"code": -1, "message": "User doesn't exist."}
 recurring_payment.connect(paypal_recurring_payment_history_sync)
@@ -1032,6 +1131,7 @@ def stripe_signup(sender, full_json, **kwargs):
         logging.user(profile.user, "~BC~SB~FBStripe subscription signup")
         profile.activate_premium()
         profile.cancel_premium_paypal()
+        profile.retrieve_stripe_ids()
     except Profile.DoesNotExist:
         return {"code": -1, "message": "User doesn't exist."}
 zebra_webhook_customer_subscription_created.connect(stripe_signup)
@@ -1041,7 +1141,7 @@ def stripe_payment_history_sync(sender, full_json, **kwargs):
     try:
         profile = Profile.objects.get(stripe_id=stripe_id)
         logging.user(profile.user, "~BC~SB~FBStripe subscription payment")
-        profile.setup_premium_history(check_premium=True)
+        profile.setup_premium_history()
     except Profile.DoesNotExist:
         return {"code": -1, "message": "User doesn't exist."}    
 zebra_webhook_charge_succeeded.connect(stripe_payment_history_sync)
@@ -1154,70 +1254,125 @@ class PaymentHistory(models.Model):
     
     @classmethod
     def report(cls, months=26):
-        def _counter(start_date, end_date):
-            payments = PaymentHistory.objects.filter(payment_date__gte=start_date, payment_date__lte=end_date)
-            payments = payments.aggregate(avg=Avg('payment_amount'), 
-                                          sum=Sum('payment_amount'), 
-                                          count=Count('user'))
-            print "%s-%02d-%02d - %s-%02d-%02d:\t$%.2f\t$%-6s\t%-4s" % (
+        output = ""
+        
+        def _counter(start_date, end_date, output, payments=None):
+            if not payments:
+                payments = PaymentHistory.objects.filter(payment_date__gte=start_date, payment_date__lte=end_date)
+                payments = payments.aggregate(avg=Avg('payment_amount'), 
+                                              sum=Sum('payment_amount'), 
+                                              count=Count('user'))
+            output += "%s-%02d-%02d - %s-%02d-%02d:\t$%.2f\t$%-6s\t%-4s\n" % (
                 start_date.year, start_date.month, start_date.day,
                 end_date.year, end_date.month, end_date.day,
                 round(payments['avg'] if payments['avg'] else 0, 2), payments['sum'] if payments['sum'] else 0, payments['count'])
-            return payments['sum']
+            
+            return payments, output
 
-        print "\nMonthly Totals:"
-        month_totals = {}
+        output += "\nMonthly Totals:\n"
         for m in reversed(range(months)):
             now = datetime.datetime.now()
             start_date = datetime.datetime(now.year, now.month, 1) - dateutil.relativedelta.relativedelta(months=m)
             end_time = start_date + datetime.timedelta(days=31)
             end_date = datetime.datetime(end_time.year, end_time.month, 1) - datetime.timedelta(seconds=1)
-            total = _counter(start_date, end_date)
-            month_totals[start_date.strftime("%Y-%m")] = total
+            total, output = _counter(start_date, end_date, output)
+            total = total['sum']
 
-        print "\nCurrent Month Totals:"
-        month_totals = {}
+        output += "\nMTD Totals:\n"
         years = datetime.datetime.now().year - 2009
+        this_mtd_avg = 0
+        last_mtd_avg = 0
+        last_mtd_sum = 0
+        this_mtd_sum = 0
+        last_mtd_count = 0
+        this_mtd_count = 0
+        for y in reversed(range(years)):
+            now = datetime.datetime.now()
+            start_date = datetime.datetime(now.year, now.month, 1) - dateutil.relativedelta.relativedelta(years=y)
+            end_date = now - dateutil.relativedelta.relativedelta(years=y)
+            if end_date > now: end_date = now
+            count, output = _counter(start_date, end_date, output)
+            if end_date.year != now.year:
+                last_mtd_avg = count['avg'] or 0
+                last_mtd_sum = count['sum'] or 0
+                last_mtd_count = count['count']
+            else:
+                this_mtd_avg = count['avg'] or 0
+                this_mtd_sum = count['sum'] or 0
+                this_mtd_count = count['count']
+
+        output += "\nCurrent Month Totals:\n"
+        years = datetime.datetime.now().year - 2009
+        last_month_avg = 0
+        last_month_sum = 0
+        last_month_count = 0
         for y in reversed(range(years)):
             now = datetime.datetime.now()
             start_date = datetime.datetime(now.year, now.month, 1) - dateutil.relativedelta.relativedelta(years=y)
             end_time = start_date + datetime.timedelta(days=31)
             end_date = datetime.datetime(end_time.year, end_time.month, 1) - datetime.timedelta(seconds=1)
-            if end_date > now: end_date = now
-            month_totals[start_date.strftime("%Y-%m")] = _counter(start_date, end_date)
+            if end_date > now:
+                payments = {'avg': this_mtd_avg / (max(1, last_mtd_avg) / float(max(1, last_month_avg))), 
+                            'sum': int(round(this_mtd_sum / (max(1, last_mtd_sum) / float(max(1, last_month_sum))))), 
+                            'count': int(round(this_mtd_count / (max(1, last_mtd_count) / float(max(1, last_month_count)))))}
+                _, output = _counter(start_date, end_date, output, payments=payments)
+            else:
+                count, output = _counter(start_date, end_date, output)
+                last_month_avg = count['avg']
+                last_month_sum = count['sum']
+                last_month_count = count['count']
 
-        print "\nMTD Totals:"
-        month_totals = {}
+        output += "\nYTD Totals:\n"
         years = datetime.datetime.now().year - 2009
+        this_ytd_avg = 0
+        last_ytd_avg = 0
+        this_ytd_sum = 0
+        last_ytd_sum = 0
+        this_ytd_count = 0
+        last_ytd_count = 0
         for y in reversed(range(years)):
             now = datetime.datetime.now()
-            start_date = datetime.datetime(now.year, now.month, 1) - dateutil.relativedelta.relativedelta(years=y)
+            start_date = datetime.datetime(now.year, 1, 1) - dateutil.relativedelta.relativedelta(years=y)
             end_date = now - dateutil.relativedelta.relativedelta(years=y)
-            if end_date > now: end_date = now
-            month_totals[start_date.strftime("%Y-%m")] = _counter(start_date, end_date)
+            count, output = _counter(start_date, end_date, output)
+            if end_date.year != now.year:
+                last_ytd_avg = count['avg'] or 0
+                last_ytd_sum = count['sum'] or 0
+                last_ytd_count = count['count']
+            else:
+                this_ytd_avg = count['avg'] or 0
+                this_ytd_sum = count['sum'] or 0
+                this_ytd_count = count['count']
 
-        print "\nYearly Totals:"
-        year_totals = {}
+        output += "\nYearly Totals:\n"
         years = datetime.datetime.now().year - 2009
+        last_year_avg = 0
+        last_year_sum = 0
+        last_year_count = 0
+        annual = 0
         for y in reversed(range(years)):
             now = datetime.datetime.now()
             start_date = datetime.datetime(now.year, 1, 1) - dateutil.relativedelta.relativedelta(years=y)
             end_date = datetime.datetime(now.year, 1, 1) - dateutil.relativedelta.relativedelta(years=y-1) - datetime.timedelta(seconds=1)
-            if end_date > now: end_date = now
-            year_totals[now.year - y] = _counter(start_date, end_date)
-
-        print "\nYTD Totals:"
-        year_totals = {}
-        years = datetime.datetime.now().year - 2009
-        for y in reversed(range(years)):
-            now = datetime.datetime.now()
-            start_date = datetime.datetime(now.year, 1, 1) - dateutil.relativedelta.relativedelta(years=y)
-            end_date = now - dateutil.relativedelta.relativedelta(years=y)
-            if end_date > now: end_date = now
-            year_totals[now.year - y] = _counter(start_date, end_date)
+            if end_date > now:
+                payments = {'avg': this_ytd_avg / (max(1, last_ytd_avg) / float(max(1, last_year_avg))), 
+                            'sum': int(round(this_ytd_sum / (max(1, last_ytd_sum) / float(max(1, last_year_sum))))), 
+                            'count': int(round(this_ytd_count / (max(1, last_ytd_count) / float(max(1, last_year_count)))))}
+                count, output = _counter(start_date, end_date, output, payments=payments)
+                annual = count['sum']
+            else:
+                count, output = _counter(start_date, end_date, output)
+                last_year_avg = count['avg'] or 0
+                last_year_sum = count['sum'] or 0
+                last_year_count = count['count']
+                
 
         total = cls.objects.all().aggregate(sum=Sum('payment_amount'))
-        print "\nTotal: $%s" % total['sum']
+        output += "\nTotal: $%s\n" % total['sum']
+        
+        print output
+        
+        return {'annual': annual, 'output': output}
 
 
 class MGiftCode(mongo.Document):

apps/profile/tasks.py

@@ -63,6 +63,7 @@ class CleanupUser(Task):
         MInteraction.trim(user_id)
         MActivity.trim(user_id)
         UserSubscriptionFolders.add_missing_feeds_for_user(user_id)
+        UserSubscriptionFolders.compact_for_user(user_id)
         # UserSubscription.refresh_stale_feeds(user_id)
         
         try:
@@ -72,3 +73,18 @@ class CleanupUser(Task):
             return
         ss.sync_twitter_photo()
 
+class CleanSpam(Task):
+    name = 'clean-spam'
+
+    def run(self, **kwargs):
+        logging.debug(" ---> Finding spammers...")
+        Profile.clear_dead_spammers(confirm=True)
+
+class ReimportStripeHistory(Task):
+    name = 'reimport-stripe-history'
+
+    def run(self, **kwargs):
+        logging.debug(" ---> Reimporting Stripe history...")
+        Profile.reimport_stripe_history(limit=10, days=1)
+            
+

apps/profile/urls.py

@@ -29,4 +29,5 @@ urlpatterns = patterns('',
     url(r'^delete_starred_stories/?', views.delete_starred_stories, name='profile-delete-starred-stories'),
     url(r'^delete_all_sites/?', views.delete_all_sites, name='profile-delete-all-sites'),
     url(r'^email_optout/?', views.email_optout, name='profile-email-optout'),
+    url(r'^ios_subscription_status/?', views.ios_subscription_status, name='profile-ios-subscription-status'),
 )

apps/profile/views.py

@@ -27,7 +27,7 @@ from apps.social.models import MSocialServices, MActivity, MSocialProfile
 from apps.analyzer.models import MClassifierTitle, MClassifierAuthor, MClassifierFeed, MClassifierTag
 from utils import json_functions as json
 from utils.user_functions import ajax_login_required
-from utils.view_functions import render_to
+from utils.view_functions import render_to, is_true
 from utils.user_functions import get_user
 from utils import log as logging
 from vendor.paypalapi.exceptions import PayPalAPIResponseError
@@ -332,11 +332,17 @@ def save_ios_receipt(request):
     product_identifier = request.POST.get('product_identifier')
     transaction_identifier = request.POST.get('transaction_identifier')
     
+    logging.user(request, "~BM~FBSaving iOS Receipt: %s %s" % (product_identifier, transaction_identifier))
+    
     paid = request.user.profile.activate_ios_premium(product_identifier, transaction_identifier)
     if paid:
+        logging.user(request, "~BM~FBSending iOS Receipt email: %s %s" % (product_identifier, transaction_identifier))
         subject = "iOS Premium: %s (%s)" % (request.user.profile, product_identifier)
         message = """User: %s (%s) -- Email: %s, product: %s, txn: %s, receipt: %s""" % (request.user.username, request.user.pk, request.user.email, product_identifier, transaction_identifier, receipt)
         mail_admins(subject, message, fail_silently=True)
+    else:
+        logging.user(request, "~BM~FBNot sending iOS Receipt email, already paid: %s %s" % (product_identifier, transaction_identifier))
+        
     
     return request.user.profile
     
@@ -347,6 +353,7 @@ def stripe_form(request):
     stripe.api_key = settings.STRIPE_SECRET
     plan = int(request.GET.get('plan', 2))
     plan = PLANS[plan-1][0]
+    renew = is_true(request.GET.get('renew', False))
     error = None
     
     if request.method == 'POST':
@@ -354,25 +361,29 @@ def stripe_form(request):
         if zebra_form.is_valid():
             user.email = zebra_form.cleaned_data['email']
             user.save()
-            
+            customer = None
             current_premium = (user.profile.is_premium and 
                                user.profile.premium_expire and
                                user.profile.premium_expire > datetime.datetime.now())
+                               
             # Are they changing their existing card?
-            if user.profile.stripe_id and current_premium:
+            if user.profile.stripe_id:
                 customer = stripe.Customer.retrieve(user.profile.stripe_id)
                 try:
-                    card = customer.cards.create(card=zebra_form.cleaned_data['stripe_token'])
+                    card = customer.sources.create(source=zebra_form.cleaned_data['stripe_token'])
                 except stripe.CardError:
                     error = "This card was declined."
                 else:
                     customer.default_card = card.id
                     customer.save()
+                    user.profile.strip_4_digits = zebra_form.cleaned_data['last_4_digits']
+                    user.profile.save()
+                    user.profile.activate_premium() # TODO: Remove, because webhooks are slow
                     success_updating = True
             else:
                 try:
                     customer = stripe.Customer.create(**{
-                        'card': zebra_form.cleaned_data['stripe_token'],
+                        'source': zebra_form.cleaned_data['stripe_token'],
                         'plan': zebra_form.cleaned_data['plan'],
                         'email': user.email,
                         'description': user.username,
@@ -385,6 +396,29 @@ def stripe_form(request):
                     user.profile.save()
                     user.profile.activate_premium() # TODO: Remove, because webhooks are slow
                     success_updating = True
+            
+            # Check subscription to ensure latest plan, otherwise cancel it and subscribe
+            if success_updating and customer and customer.subscriptions.total_count == 1:
+                subscription = customer.subscriptions.data[0]
+                if subscription['plan']['id'] != "newsblur-premium-36":
+                    for sub in customer.subscriptions:
+                        sub.delete()
+                    customer = stripe.Customer.retrieve(user.profile.stripe_id)
+                
+            if success_updating and customer and customer.subscriptions.total_count == 0:
+                params = dict(
+                  customer=customer.id,
+                  items=[
+                    {
+                      "plan": "newsblur-premium-36",
+                    },
+                  ])
+                premium_expire = user.profile.premium_expire
+                if current_premium and premium_expire:
+                    if premium_expire < (datetime.datetime.now() + datetime.timedelta(days=365)):
+                        params['billing_cycle_anchor'] = premium_expire.strftime('%s')
+                        params['trial_end'] = premium_expire.strftime('%s')
+                stripe.Subscription.create(**params)
 
     else:
         zebra_form = StripePlusPaymentForm(email=user.email, plan=plan)
@@ -400,6 +434,10 @@ def stripe_form(request):
         new_user_queue_behind = new_user_queue_count - new_user_queue_position 
         new_user_queue_position -= 1
     
+    immediate_charge = True
+    if user.profile.premium_expire and user.profile.premium_expire > datetime.datetime.now():
+        immediate_charge = False
+    
     logging.user(request, "~BM~FBLoading Stripe form")
 
     return render_to_response('profile/stripe_form.xhtml',
@@ -410,6 +448,8 @@ def stripe_form(request):
           'new_user_queue_count': new_user_queue_count - 1,
           'new_user_queue_position': new_user_queue_position,
           'new_user_queue_behind': new_user_queue_behind,
+          'renew': renew,
+          'immediate_charge': immediate_charge,
           'error': error,
         },
         context_instance=RequestContext(request)
@@ -525,7 +565,7 @@ def never_expire_premium(request):
 def update_payment_history(request):
     user_id = request.REQUEST.get('user_id')
     user = User.objects.get(pk=user_id)
-    user.profile.setup_premium_history(check_premium=False)
+    user.profile.setup_premium_history(set_premium_expire=False)
     
     return {'code': 1}
     
@@ -646,4 +686,15 @@ def email_optout(request):
     return {
         "user": user,
     }
-    
+
+@json.json_view
+def ios_subscription_status(request):
+    logging.debug(" ---> iOS Subscription Status: %s" % request.POST)
+    
+    subject = "iOS Subscription Status"
+    message = """%s""" % (request.POST)
+    mail_admins(subject, message)
+    
+    return {
+        "code": 1
+    }

apps/push/models.py

@@ -60,7 +60,7 @@ class PushSubscriptionManager(models.Manager):
                 'hub.verify_token'  : subscription.generate_token('subscribe'),
                 'hub.lease_seconds' : lease_seconds,
             })
-        except requests.ConnectionError:
+        except (requests.ConnectionError, requests.exceptions.MissingSchema):
             response = None
 
         if response and response.status_code == 204:
@@ -162,7 +162,7 @@ class PushSubscription(models.Model):
                     PushSubscription.objects.subscribe(
                         self_url, feed=self.feed, hub=hub_url,
                         lease_seconds=seconds)
-                except TimeoutError, e:
+                except TimeoutError:
                     logging.debug(u'   ---> [%-30s] ~FR~BKTimed out updating PuSH hub/topic: %s / %s' % (
                                   unicode(self.feed)[:30], hub_url, self_url))
                     

apps/reader/models.py

@@ -283,12 +283,14 @@ class UserSubscription(models.Model):
         unread_ranked_stories_keys  = '%szhU:%s:feeds:%s' % (cache_prefix, user_id, feeds_string)
         stories_cached = rt.exists(ranked_stories_keys)
         unreads_cached = True if read_filter == "unread" else rt.exists(unread_ranked_stories_keys)
-        if offset and stories_cached and unreads_cached:
+        if offset and stories_cached:
             story_hashes = range_func(ranked_stories_keys, offset, offset+limit)
             if read_filter == "unread":
                 unread_story_hashes = story_hashes
-            else:
+            elif unreads_cached:
                 unread_story_hashes = range_func(unread_ranked_stories_keys, 0, offset+limit)
+            else:
+                unread_story_hashes = []
             return story_hashes, unread_story_hashes
         else:
             rt.delete(ranked_stories_keys)
@@ -1297,6 +1299,16 @@ class UserSubscriptionFolders(models.Model):
         verbose_name_plural = "folders"
         verbose_name = "folder"
     
+    @classmethod
+    def compact_for_user(cls, user_id):
+        user = User.objects.get(pk=user_id)
+        try:
+            usf = UserSubscriptionFolders.objects.get(user=user)
+        except UserSubscriptionFolders.DoesNotExist:
+            return
+        
+        usf.compact()
+        
     def compact(self):
         folders = json.decode(self.folders)
         
@@ -1311,11 +1323,13 @@ class UserSubscriptionFolders(models.Model):
             return new_folder
         
         new_folders = _compact(folders)
-        logging.info(" ---> Compacting from %s to %s" % (folders, new_folders))
+        compact_msg = " ---> Compacting from %s to %s" % (folders, new_folders)
         new_folders = json.encode(new_folders)
-        logging.info(" ---> Compacting from %s to %s" % (len(self.folders), len(new_folders)))
-        self.folders = new_folders
-        self.save()
+        if len(self.folders) != len(new_folders):
+            logging.info(compact_msg)
+            logging.info(" ---> Compacting from %s bytes to %s bytes" % (len(self.folders), len(new_folders)))
+            self.folders = new_folders
+            self.save()
         
     def add_folder(self, parent_folder, folder):
         if self.folders:
@@ -1622,12 +1636,25 @@ class UserSubscriptionFolders(models.Model):
             for feed_id in missing_subs:
                 feed = Feed.get_by_id(feed_id)
                 if feed:
+                    if feed_id != feed.pk:
+                        logging.debug(" ---> %s doesn't match %s, rewriting to remove %s..." % (
+                                      feed_id, feed.pk, feed_id))
+                        # Clear out duplicate sub in folders before subscribing to feed
+                        duplicate_feed = Feed.get_by_id(feed_id)
+                        duplicate_feed.pk = feed_id
+                        self.rewrite_feed(feed, duplicate_feed)
                     us, _ = UserSubscription.objects.get_or_create(user=self.user, feed=feed, defaults={
                         'needs_unread_recalc': True
                     })
                     if not us.needs_unread_recalc:
                         us.needs_unread_recalc = True
                         us.save()
+                elif feed_id and not feed:
+                    # No feed found for subscription, remove subscription
+                    logging.debug(" ---> %s: No feed found, removing subscription: %s" % (
+                                  self.user, feed_id))
+                    self.delete_feed(feed_id, None, commit_delete=False)
+                    
 
         missing_folder_feeds = set(subs) - set(all_feeds)
         if missing_folder_feeds:

apps/reader/views.py

@@ -38,7 +38,7 @@ from apps.reader.forms import SignupForm, LoginForm, FeatureForm
 from apps.rss_feeds.models import MFeedIcon, MStarredStoryCounts, MSavedSearch
 from apps.notifications.models import MUserFeedNotification
 from apps.search.models import MUserSearch
-from apps.statistics.models import MStatistics
+from apps.statistics.models import MStatistics, MAnalyticsLoader
 # from apps.search.models import SearchStarredStory
 try:
     from apps.rss_feeds.models import Feed, MFeedPage, DuplicateFeed, MStory, MStarredStory
@@ -415,7 +415,7 @@ def load_feeds_flat(request):
     categories = None
     if not user_subs:
         categories = MCategory.serialize()
-        
+
     logging.user(request, "~FB~SBLoading ~FY%s~FB/~FM%s~FB/~FR%s~FB feeds/socials/inactive ~FMflat~FB%s%s" % (
             len(feeds.keys()), len(social_feeds), len(inactive_feeds), '. ~FCUpdating counts.' if update_counts else '',
             ' ~BB(background fetch)' if background_ios else ''))
@@ -447,6 +447,7 @@ def load_feeds_flat(request):
 @json.json_view
 def refresh_feeds(request):
     start = datetime.datetime.now()
+    start_time = time.time()
     user = get_user(request)
     feed_ids = request.REQUEST.getlist('feed_id') or request.REQUEST.getlist('feed_id[]')
     check_fetch_status = request.REQUEST.get('check_fetch_status')
@@ -506,7 +507,9 @@ def refresh_feeds(request):
             (checkpoint2-start).total_seconds(),
             (end-start).total_seconds(),
             ))
-
+    
+    MAnalyticsLoader.add(page_load=time.time()-start_time)
+    
     return {
         'feeds': feeds, 
         'social_feeds': social_feeds,
@@ -527,6 +530,7 @@ def interactions_count(request):
 @ajax_login_required
 @json.json_view
 def feed_unread_count(request):
+    start = time.time()
     user = request.user
     feed_ids = request.REQUEST.getlist('feed_id') or request.REQUEST.getlist('feed_id[]')
     force = request.REQUEST.get('force', False)
@@ -551,10 +555,12 @@ def feed_unread_count(request):
     else:
         feed_title = "%s feeds" % (len(feeds) + len(social_feeds))
     logging.user(request, "~FBUpdating unread count on: %s" % feed_title)
+    MAnalyticsLoader.add(page_load=time.time()-start)
     
     return {'feeds': feeds, 'social_feeds': social_feeds}
     
 def refresh_feed(request, feed_id):
+    start = time.time()
     user = get_user(request)
     feed = get_object_or_404(Feed, pk=feed_id)
     
@@ -563,6 +569,7 @@ def refresh_feed(request, feed_id):
     usersub.calculate_feed_scores(silent=False)
     
     logging.user(request, "~FBRefreshing feed: %s" % feed)
+    MAnalyticsLoader.add(page_load=time.time()-start)
     
     return load_single_feed(request, feed_id)
     
@@ -575,6 +582,7 @@ def load_single_feed(request, feed_id):
     # limit                   = int(request.REQUEST.get('limit', 6))
     limit                   = 6
     page                    = int(request.REQUEST.get('page', 1))
+    delay                   = int(request.REQUEST.get('delay', 0))
     offset                  = limit * (page-1)
     order                   = request.REQUEST.get('order', 'newest')
     read_filter             = request.REQUEST.get('read_filter', 'all')
@@ -584,7 +592,7 @@ def load_single_feed(request, feed_id):
     include_feeds           = is_true(request.REQUEST.get('include_feeds', False))
     message                 = None
     user_search             = None
-
+        
     dupe_feed_id = None
     user_profiles = []
     now = localtime_for_timezone(datetime.datetime.now(), user.profile.timezone)
@@ -603,7 +611,11 @@ def load_single_feed(request, feed_id):
     if feed.is_newsletter and not usersub:
         # User must be subscribed to a newsletter in order to read it
         raise Http404
-        
+    
+    if page > 200:
+        logging.user(request, "~BR~FK~SBOver page 200 on single feed: %s" % page)
+        raise Http404
+    
     if query:
         if user.profile.is_premium:
             user_search = MUserSearch.get_user(user.pk)
@@ -747,6 +759,7 @@ def load_single_feed(request, feed_id):
     search_log = "~SN~FG(~SB%s~SN) " % query if query else ""
     logging.user(request, "~FYLoading feed: ~SB%s%s (%s/%s) %s%s" % (
         feed.feed_title[:22], ('~SN/p%s' % page) if page > 1 else '', order, read_filter, search_log, time_breakdown))
+    MAnalyticsLoader.add(page_load=timediff)
 
     if not include_hidden:
         hidden_stories_removed = 0
@@ -777,10 +790,11 @@ def load_single_feed(request, feed_id):
     # if not usersub and feed.num_subscribers <= 1:
     #     data = dict(code=-1, message="You must be subscribed to this feed.")
     
-    # if page <= 3:
-    #     import random
-    #     # time.sleep(random.randint(2, 7) / 10.0)
-    #     time.sleep(random.randint(1, 10))
+    if delay and user.is_staff:
+        # import random
+        # time.sleep(random.randint(2, 7) / 10.0)
+        # time.sleep(random.randint(1, 10))
+        time.sleep(delay)
     
     # if page == 2:
     #     assert False
@@ -799,15 +813,18 @@ def load_feed_page(request, feed_id):
                 settings.ORIGINAL_PAGE_SERVER,
                 feed.pk,
             )
-            page_response = requests.get(url)
-            if page_response.status_code == 200:
+            try:
+                page_response = requests.get(url)
+            except requests.ConnectionError:
+                page_response = None
+            if page_response and page_response.status_code == 200:
                 response = HttpResponse(page_response.content, mimetype="text/html; charset=utf-8")
                 response['Content-Encoding'] = 'gzip'
                 response['Last-Modified'] = page_response.headers.get('Last-modified')
                 response['Etag'] = page_response.headers.get('Etag')
                 response['Content-Length'] = str(len(page_response.content))
-                logging.user(request, "~FYLoading original page, proxied from node: ~SB%s bytes" %
-                             (len(page_response.content)))
+                logging.user(request, "~FYLoading original page (%s), proxied from node: ~SB%s bytes" %
+                             (feed_id, len(page_response.content)))
                 return response
         
         if settings.BACKED_BY_AWS['pages_on_s3'] and feed.s3_page:
@@ -874,6 +891,7 @@ def load_starred_stories(request):
             stories = []
             message = "You must be a premium subscriber to read saved stories by tag."
     elif story_hashes:
+        limit = 100
         mstories = MStarredStory.objects(
             user_id=user.pk,
             story_hash__in=story_hashes
@@ -894,6 +912,25 @@ def load_starred_stories(request):
     unsub_feed_ids = list(set(story_feed_ids).difference(set(usersub_ids)))
     unsub_feeds    = Feed.objects.filter(pk__in=unsub_feed_ids)
     unsub_feeds    = dict((feed.pk, feed.canonical(include_favicon=False)) for feed in unsub_feeds)
+    for story in stories:
+        if story['story_feed_id'] in unsub_feeds: continue
+        duplicate_feed = DuplicateFeed.objects.filter(duplicate_feed_id=story['story_feed_id'])
+        if not duplicate_feed: continue
+        feed_id = duplicate_feed[0].feed_id
+        try:
+            saved_story = MStarredStory.objects.get(user_id=user.pk, story_hash=story['story_hash'])
+            saved_story.feed_id = feed_id
+            _, story_hash = MStory.split_story_hash(story['story_hash'])
+            saved_story.story_hash = "%s:%s" % (feed_id, story_hash)
+            saved_story.story_feed_id = feed_id
+            story['story_hash'] = saved_story.story_hash
+            story['story_feed_id'] = saved_story.story_feed_id
+            saved_story.save()
+            logging.user(request, "~FCSaving new feed for starred story: ~SB%s -> %s" % (story['story_hash'], feed_id))
+        except (MStarredStory.DoesNotExist):
+            logging.user(request, "~FCCan't find feed for starred story: ~SB%s" % (story['story_hash']))
+            continue
+    
     shared_story_hashes = MSharedStory.check_shared_story_hashes(user.pk, story_hashes)
     shared_stories = []
     if shared_story_hashes:
@@ -1255,7 +1292,7 @@ def load_river_stories__redis(request):
     user_search       = None
     offset            = (page-1) * limit
     story_date_order  = "%sstory_date" % ('' if order == 'oldest' else '-')
-
+    
     if infrequent:
         feed_ids = Feed.low_volume_feeds(feed_ids, stories_per_month=infrequent)
     
@@ -1336,7 +1373,7 @@ def load_river_stories__redis(request):
             starred_stories = MStarredStory.objects(
                 user_id=user.pk,
                 story_hash__in=story_hashes
-            ).only('story_hash', 'starred_date')
+            ).only('story_hash', 'starred_date', 'user_tags')
         starred_stories = dict([(story.story_hash, dict(starred_date=story.starred_date,
                                                         user_tags=story.user_tags)) 
                                 for story in starred_stories])
@@ -1405,15 +1442,6 @@ def load_river_stories__redis(request):
         #     stories = []
         # else:
         #     stories = stories[:5]
-    diff = time.time() - start
-    timediff = round(float(diff), 2)
-    logging.user(request, "~FYLoading ~FC%sriver stories~FY: ~SBp%s~SN (%s/%s "
-                               "stories, ~SN%s/%s/%s feeds, %s/%s)" % 
-                               ("~FB~SBinfrequent~SN~FC " if infrequent else "",
-                                page, len(stories), len(mstories), len(found_feed_ids), 
-                                len(feed_ids), len(original_feed_ids), order, read_filter))
-
-
     if not include_hidden:
         hidden_stories_removed = 0
         new_stories = []
@@ -1439,6 +1467,16 @@ def load_river_stories__redis(request):
     #     import random
     #     time.sleep(random.randint(3, 6))
     
+    diff = time.time() - start
+    timediff = round(float(diff), 2)
+    logging.user(request, "~FYLoading ~FC%sriver stories~FY: ~SBp%s~SN (%s/%s "
+                               "stories, ~SN%s/%s/%s feeds, %s/%s)" % 
+                               ("~FB~SBinfrequent~SN~FC " if infrequent else "",
+                                page, len(stories), len(mstories), len(found_feed_ids), 
+                                len(feed_ids), len(original_feed_ids), order, read_filter))
+
+    MAnalyticsLoader.add(page_load=diff)
+
     data = dict(code=code,
                 message=message,
                 stories=stories,
@@ -1449,7 +1487,8 @@ def load_river_stories__redis(request):
     
     if include_feeds: data['feeds'] = feeds
     if not include_hidden: data['hidden_stories_removed'] = hidden_stories_removed
-    
+
+
     return data
     
 @json.json_view
@@ -1541,6 +1580,13 @@ def mark_all_as_read(request):
     read_date = datetime.datetime.utcnow() - datetime.timedelta(days=days)
     
     feeds = UserSubscription.objects.filter(user=request.user)
+
+    infrequent        = is_true(request.REQUEST.get('infrequent', False))
+    if infrequent:
+        infrequent = request.REQUEST.get('infrequent')
+        feed_ids = Feed.low_volume_feeds([usersub.feed.pk for usersub in feeds], stories_per_month=infrequent)
+        feeds = UserSubscription.objects.filter(user=request.user, feed_id__in=feed_ids)
+    
     socialsubs = MSocialSubscription.objects.filter(user_id=request.user.pk)
     for subtype in [feeds, socialsubs]:
         for sub in subtype:
@@ -1555,7 +1601,7 @@ def mark_all_as_read(request):
     r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
     r.publish(request.user.username, 'reload:feeds')
     
-    logging.user(request, "~FMMarking all as read: ~SB%s days" % (days,))
+    logging.user(request, "~FMMarking %s as read: ~SB%s days" % (("all" if not infrequent else "infrequent stories"), days,))
     return dict(code=code)
     
 @ajax_login_required
@@ -1594,6 +1640,7 @@ def mark_story_as_read(request):
 @ajax_login_required
 @json.json_view
 def mark_story_hashes_as_read(request):
+    retrying_failed = is_true(request.POST.get('retrying_failed', False))
     r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
     try:
         story_hashes = request.REQUEST.getlist('story_hash') or request.REQUEST.getlist('story_hash[]')
@@ -1626,8 +1673,10 @@ def mark_story_hashes_as_read(request):
             r.publish(request.user.username, 'feed:%s' % feed_id)
     
     hash_count = len(story_hashes)
-    logging.user(request, "~FYRead %s %s in feed/socialsubs: %s/%s" % (
-                 hash_count, 'story' if hash_count == 1 else 'stories', feed_ids, friend_ids))
+    logging.user(request, "~FYRead %s %s in feed/socialsubs: %s/%s: %s %s" % (
+                 hash_count, 'story' if hash_count == 1 else 'stories', feed_ids, friend_ids,
+                 story_hashes,
+                 '(retrying failed)' if retrying_failed else ''))
 
     return dict(code=1, story_hashes=story_hashes, 
                 feed_ids=feed_ids, friend_user_ids=friend_ids)
@@ -1807,11 +1856,18 @@ def mark_feed_as_read(request):
     feed_ids = request.POST.getlist('feed_id') or request.POST.getlist('feed_id[]')
     cutoff_timestamp = int(request.REQUEST.get('cutoff_timestamp', 0))
     direction = request.REQUEST.get('direction', 'older')
+    infrequent        = is_true(request.REQUEST.get('infrequent', False))
+    if infrequent:
+        infrequent = request.REQUEST.get('infrequent')
     multiple = len(feed_ids) > 1
     code = 1
     errors = []
     cutoff_date = datetime.datetime.fromtimestamp(cutoff_timestamp) if cutoff_timestamp else None
     
+    if infrequent:
+        feed_ids = Feed.low_volume_feeds(feed_ids, stories_per_month=infrequent)
+        feed_ids = [unicode(f) for f in feed_ids] # This method expects strings
+    
     if cutoff_date:
         logging.user(request, "~FMMark %s feeds read, %s - cutoff: %s/%s" % 
                      (len(feed_ids), direction, cutoff_timestamp, cutoff_date))
@@ -1904,7 +1960,7 @@ def add_url(request):
                 ss.twitter_api().me()
             except tweepy.TweepError:
                 code = -1
-                message = "Your Twitter connection isn't setup. Go to Manage - Friends and reconnect Twitter."
+                message = "Your Twitter connection isn't setup. Go to Manage - Friends/Followers and reconnect Twitter."
     
     if code == -1:
         return dict(code=code, message=message)
@@ -2427,7 +2483,7 @@ def _mark_story_as_unstarred(request):
                 MStarredStoryCounts.adjust_count(request.user.pk, tag=tag, amount=-1)
             except MStarredStoryCounts.DoesNotExist:
                 pass
-        # MStarredStoryCounts.schedule_count_tags_for_user(request.user.pk)
+        MStarredStoryCounts.schedule_count_tags_for_user(request.user.pk)
         MStarredStoryCounts.count_for_user(request.user.pk, total_only=True)
         starred_counts = MStarredStoryCounts.user_counts(request.user.pk)
         
@@ -2464,7 +2520,7 @@ def send_story_email(request):
     from_address = 'share@newsblur.com'
     share_user_profile = MSocialProfile.get_user(request.user.pk)
     
-    quota = 20 if user.profile.is_premium else 1
+    quota = 32 if user.profile.is_premium else 1
     if share_user_profile.over_story_email_quota(quota=quota):
         code = -1
         if user.profile.is_premium:
@@ -2511,11 +2567,11 @@ def send_story_email(request):
                                          from_email='NewsBlur <%s>' % from_address,
                                          to=to_addresses, 
                                          cc=cc,
-                                         headers={'Reply-To': from_email})
+                                         headers={'Reply-To': "%s <%s>" % (from_name, from_email)})
         msg.attach_alternative(html, "text/html")
         try:
             msg.send()
-        except boto.ses.connection.ResponseError, e:
+        except boto.ses.connection.BotoServerError, e:
             code = -1
             message = "Email error: %s" % str(e)
         

apps/rss_feeds/icon_importer.py

@@ -17,6 +17,7 @@ from boto.s3.key import Key
 from StringIO import StringIO
 from django.conf import settings
 from apps.rss_feeds.models import MFeedPage, MFeedIcon
+from utils.facebook_fetcher import FacebookFetcher
 from utils import log as logging
 from utils.feed_functions import timelimit, TimeoutError
 from OpenSSL.SSL import Error as OpenSSLError
@@ -44,7 +45,10 @@ class IconImporter(object):
         ):
             # print 'Found, but skipping...'
             return
-        image, image_file, icon_url = self.fetch_image_from_page_data()
+        if 'facebook.com' in self.feed.feed_address:
+            image, image_file, icon_url = self.fetch_facebook_image()
+        else:
+            image, image_file, icon_url = self.fetch_image_from_page_data()
         if not image:
             image, image_file, icon_url = self.fetch_image_from_path(force=self.force)
 
@@ -54,6 +58,8 @@ class IconImporter(object):
                 color = self.determine_dominant_color_in_image(image)
             except IndexError:
                 return
+            except MemoryError:
+                return
             try:
                 image_str = self.string_from_image(image)
             except TypeError:
@@ -250,7 +256,17 @@ class IconImporter(object):
             image, image_file = self.get_image_from_url(url)
         # print 'Found: %s - %s' % (url, image)
         return image, image_file, url
-
+    
+    def fetch_facebook_image(self):
+        facebook_fetcher = FacebookFetcher(self.feed)
+        url = facebook_fetcher.favicon_url()
+        image, image_file = self.get_image_from_url(url)
+        if not image:
+            url = urlparse.urljoin(self.feed.feed_link, '/favicon.ico')
+            image, image_file = self.get_image_from_url(url)
+        # print 'Found: %s - %s' % (url, image)
+        return image, image_file, url
+        
     def get_image_from_url(self, url):
         # print 'Requesting: %s' % url
         if not url:

apps/rss_feeds/models.py

@@ -11,6 +11,7 @@ import hashlib
 import redis
 import pymongo
 import HTMLParser
+import urlparse
 from collections import defaultdict
 from operator import itemgetter
 from bson.objectid import ObjectId
@@ -36,6 +37,7 @@ from apps.search.models import SearchStory, SearchFeed
 from apps.statistics.rstats import RStats
 from utils import json_functions as json
 from utils import feedfinder2 as feedfinder
+from utils import feedfinder as feedfinder_old
 from utils import urlnorm
 from utils import log as logging
 from utils.fields import AutoOneToOneField
@@ -45,6 +47,7 @@ from utils.feed_functions import relative_timesince
 from utils.feed_functions import seconds_timesince
 from utils.story_functions import strip_tags, htmldiff, strip_comments, strip_comments__lxml
 from utils.story_functions import prep_for_search
+from utils.story_functions import create_camo_signed_url
 
 ENTRY_NEW, ENTRY_UPDATED, ENTRY_SAME, ENTRY_ERR = range(4)
 
@@ -155,8 +158,8 @@ class Feed(models.Model):
         r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
         current_time = int(time.time() + 60*60*24)
         unread_cutoff = self.unread_cutoff.strftime('%s')
-        print " ---> zrevrangebyscore zF:%s %s %s" % (self.pk, current_time, unread_cutoff)
         story_hashes = r.zrevrangebyscore('zF:%s' % self.pk, current_time, unread_cutoff)
+
         return story_hashes
         
     @classmethod
@@ -399,16 +402,19 @@ class Feed(models.Model):
     @property
     def favicon_fetching(self):
         return bool(not (self.favicon_not_found or self.favicon_color))
-        
+    
     @classmethod
-    def get_feed_from_url(cls, url, create=True, aggressive=False, fetch=True, offset=0, user=None):
+    def get_feed_from_url(cls, url, create=True, aggressive=False, fetch=True, offset=0, user=None, interactive=False):
         feed = None
         without_rss = False
+        original_url = url
         
         if url and url.startswith('newsletter:'):
             return cls.objects.get(feed_address=url)
         if url and re.match('(https?://)?twitter.com/\w+/?$', url):
             without_rss = True
+        if url and re.match(r'(https?://)?(www\.)?facebook.com/\w+/?$', url):
+            without_rss = True
         if url and 'youtube.com/user/' in url:
             username = re.search('youtube.com/user/(\w+)', url).group(1)
             url = "http://gdata.youtube.com/feeds/base/users/%s/uploads" % username
@@ -443,19 +449,44 @@ class Feed(models.Model):
                 
             return feed
         
+        @timelimit(10)
+        def _feedfinder(url):
+            found_feed_urls = feedfinder.find_feeds(url)
+            return found_feed_urls
+
+        @timelimit(10)
+        def _feedfinder_old(url):
+            found_feed_urls = feedfinder_old.feeds(url)
+            return found_feed_urls
+        
         # Normalize and check for feed_address, dupes, and feed_link
         url = urlnorm.normalize(url)
         if not url:
+            logging.debug(" ---> ~FRCouldn't normalize url: ~SB%s" % url)
             return
         
         feed = by_url(url)
         found_feed_urls = []
         
+        if interactive:
+            import pdb; pdb.set_trace()
+        
         # Create if it looks good
         if feed and len(feed) > offset:
             feed = feed[offset]
         else:
-            found_feed_urls = feedfinder.find_feeds(url)
+            try:
+                found_feed_urls = _feedfinder(url)
+            except TimeoutError:
+                logging.debug('   ---> Feed finder timed out...')
+                found_feed_urls = []
+            if not found_feed_urls:
+                try:
+                    found_feed_urls = _feedfinder_old(url)
+                except TimeoutError:
+                    logging.debug('   ---> Feed finder old timed out...')
+                    found_feed_urls = []
+                
             if len(found_feed_urls):
                 feed_finder_url = found_feed_urls[0]
                 logging.debug(" ---> Found feed URLs for %s: %s" % (url, found_feed_urls))
@@ -469,14 +500,17 @@ class Feed(models.Model):
                     feed = cls.objects.create(feed_address=feed_finder_url)
                     feed = feed.update()
             elif without_rss:
-                logging.debug(" ---> Found without_rss feed: %s" % (url))
-                feed = cls.objects.create(feed_address=url)
+                logging.debug(" ---> Found without_rss feed: %s / %s" % (url, original_url))
+                feed = cls.objects.create(feed_address=url, feed_link=original_url)
                 feed = feed.update(requesting_user_id=user.pk if user else None)
                 
         # Check for JSON feed
         if not feed and fetch and create:
-            r = requests.get(url)
-            if 'application/json' in r.headers.get('Content-Type'):
+            try:
+                r = requests.get(url)
+            except (requests.ConnectionError, requests.models.InvalidURL):
+                r = None
+            if r and 'application/json' in r.headers.get('Content-Type'):
                 feed = cls.objects.create(feed_address=url)
                 feed = feed.update()
         
@@ -492,6 +526,7 @@ class Feed(models.Model):
         
         # Not created and not within bounds, so toss results.
         if isinstance(feed, QuerySet):
+            logging.debug(" ---> ~FRNot created and not within bounds, tossing: ~SB%s" % feed)
             return
         
         return feed
@@ -1001,7 +1036,8 @@ class Feed(models.Model):
                         start = True
                         months.append((key, dates.get(key, 0)))
                         total += dates.get(key, 0)
-                        month_count += 1
+                        if dates.get(key, 0) > 0:
+                            month_count += 1 # Only count months that have stories for the average
         original_story_count_history = self.data.story_count_history
         self.data.story_count_history = json.encode({'months': months, 'hours': hours, 'days': days})
         if self.data.story_count_history != original_story_count_history:
@@ -1069,6 +1105,12 @@ class Feed(models.Model):
     
     @property
     def user_agent(self):
+        feed_parts = urlparse.urlparse(self.feed_address)
+        if feed_parts.netloc.find('.tumblr.com') != -1:
+            # Certain tumblr feeds will redirect to tumblr's login page when fetching.
+            # A known workaround is using facebook's user agent.
+            return 'facebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)'
+
         ua = ('NewsBlur Feed Fetcher - %s subscriber%s - %s '
               '(Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) '
               'AppleWebKit/537.36 (KHTML, like Gecko) '
@@ -1096,7 +1138,11 @@ class Feed(models.Model):
         return headers
         
     def update(self, **kwargs):
-        from utils import feed_fetcher
+        try:
+            from utils import feed_fetcher
+        except ImportError, e:
+            logging.info(" ***> ~BR~FRImportError: %s" % e)
+            pass
         r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL)
         original_feed_id = int(self.pk)
 
@@ -1119,7 +1165,6 @@ class Feed(models.Model):
         
         if getattr(settings, 'TEST_DEBUG', False):
             print " ---> Testing feed fetch: %s" % self.log_title
-            options['force'] = False
             # options['force_fp'] = True # No, why would this be needed?
             original_feed_address = self.feed_address
             original_feed_link = self.feed_link
@@ -1207,8 +1252,6 @@ class Feed(models.Model):
                               self.log_title[:30],
                               story.get('title'),
                               story.get('guid')))
-            if not story.get('title'):
-                continue
                 
             story_content = story.get('story_content')
             if error_count:
@@ -1240,7 +1283,6 @@ class Feed(models.Model):
                        story_guid = story.get('guid'),
                        story_tags = story_tags
                 )
-                s.extract_image_urls()
                 try:
                     s.save()
                     ret_values['new'] += 1
@@ -1310,7 +1352,7 @@ class Feed(models.Model):
                 # Leads to incorrect unread story counts.
                 if replace_story_date:
                     existing_story.story_date = story.get('published') # Really shouldn't do this.
-                existing_story.extract_image_urls()                
+                existing_story.extract_image_urls(force=True)
                 try:
                     existing_story.save()
                     ret_values['updated'] += 1
@@ -1473,6 +1515,9 @@ class Feed(models.Model):
                 logging.debug("   ***> [%-30s] Error trimming: %s" % (self.log_title[:30], e))
                 pass
         
+        if getattr(settings, 'OVERRIDE_STORY_COUNT_MAX', None):
+            cutoff = settings.OVERRIDE_STORY_COUNT_MAX
+        
         return cutoff
                 
     def trim_feed(self, verbose=False, cutoff=None):
@@ -1821,17 +1866,31 @@ class Feed(models.Model):
             has_changes = True
         if not show_changes and latest_story_content:
             story_content = latest_story_content
-            
+        
+        story_title = story_db.story_title
+        blank_story_title = False
+        if not story_title:
+            blank_story_title = True
+            if story_content:
+                story_title = strip_tags(story_content)
+            if not story_title and story_db.story_permalink:
+                story_title = story_db.story_permalink
+            if len(story_title) > 80:
+                story_title = story_title[:80] + '...'
+        
         story                     = {}
         story['story_hash']       = getattr(story_db, 'story_hash', None)
         story['story_tags']       = story_db.story_tags or []
         story['story_date']       = story_db.story_date.replace(tzinfo=None)
         story['story_timestamp']  = story_db.story_date.strftime('%s')
         story['story_authors']    = story_db.story_author_name or ""
-        story['story_title']      = story_db.story_title
+        story['story_title']      = story_title
+        if blank_story_title:
+            story['story_title_blank'] = True
         story['story_content']    = story_content
         story['story_permalink']  = story_db.story_permalink
         story['image_urls']       = story_db.image_urls
+        story['secure_image_urls']= cls.secure_image_urls(story_db.image_urls)
         story['story_feed_id']    = feed_id or story_db.story_feed_id
         story['has_modifications']= has_changes
         story['comment_count']    = story_db.comment_count if hasattr(story_db, 'comment_count') else 0
@@ -1863,6 +1922,13 @@ class Feed(models.Model):
         
         return story
     
+    @classmethod
+    def secure_image_urls(cls, urls):
+        signed_urls = [create_camo_signed_url(settings.IMAGES_URL, 
+                                              settings.IMAGES_SECRET_KEY, 
+                                              url) for url in urls]
+        return dict(zip(urls, signed_urls))
+        
     def get_tags(self, entry):
         fcat = []
         if entry.has_key('tags'):
@@ -2029,12 +2095,12 @@ class Feed(models.Model):
         # SpD = 0  Subs > 1:  t = 60 * 3    # 30158  * 1440/(60*3) =  241264
         # SpD = 0  Subs = 1:  t = 60 * 24   # 514131 * 1440/(60*24) = 514131
         if spd >= 1:
-            if subs > 10:
+            if subs >= 10:
                 total = 6
             elif subs > 1:
                 total = 15
             else:
-                total = 60
+                total = 45
         elif spd > 0:
             if subs > 1:
                 total = 60 - (spd * 60)
@@ -2071,9 +2137,9 @@ class Feed(models.Model):
             if len(fetch_history['push_history']):
                 total = total * 12
         
-        # 12 hour max for premiums, 48 hour max for free
+        # 3 hour max for premiums, 48 hour max for free
         if subs >= 1:
-            total = min(total, 60*12*1)
+            total = min(total, 60*4*1)
         else:
             total = min(total, 60*24*2)
         
@@ -2364,6 +2430,8 @@ class MStory(mongo.Document):
         story_content_type_max = MStory._fields['story_content_type'].max_length
         self.story_hash = self.feed_guid_hash
         
+        self.extract_image_urls()
+        
         if self.story_content:
             self.story_content_z = zlib.compress(smart_str(self.story_content))
             self.story_content = None
@@ -2675,7 +2743,10 @@ class MStory(mongo.Document):
             else:
                 return
         
-        image_urls = []
+        image_urls = self.image_urls
+        if not image_urls:
+            image_urls = []
+            
         for image in images:
             image_url = image.get('src')
             if not image_url:
@@ -2683,14 +2754,32 @@ class MStory(mongo.Document):
             if image_url and len(image_url) >= 1024:
                 continue
             image_urls.append(image_url)
-
+                
         if not image_urls:
             if not text:
                 return self.extract_image_urls(force=force, text=True)
             else:
                 return
         
-        self.image_urls = image_urls
+        if text:
+            urls = []
+            for url in image_urls:
+                if 'http://' in url[1:] or 'https://' in url[1:]:
+                    continue
+                urls.append(url)
+            image_urls = urls
+        
+        ordered_image_urls = []
+        for image_url in list(set(image_urls)):
+            if 'feedburner' in image_url:
+                ordered_image_urls.append(image_url)
+            else:
+                ordered_image_urls.insert(0, image_url)
+        image_urls = ordered_image_urls
+        
+        if len(image_urls):
+            self.image_urls = [u for u in image_urls if u]
+        
         return self.image_urls
 
     def fetch_original_text(self, force=False, request=None, debug=False):
@@ -2701,10 +2790,7 @@ class MStory(mongo.Document):
             ti = TextImporter(self, feed=feed, request=request, debug=debug)
             original_doc = ti.fetch(return_document=True)
             original_text = original_doc.get('content') if original_doc else None
-            if original_doc and original_doc.get('image', False):
-                self.image_urls = [original_doc['image']]
-            else:
-                self.extract_image_urls(force=force, text=True)
+            self.extract_image_urls(force=force, text=True)
             self.save()
         else:
             logging.user(request, "~FYFetching ~FGoriginal~FY story text, ~SBfound.")
@@ -2856,6 +2942,9 @@ class MStarredStory(mongo.DynamicDocument):
             original_text = zlib.decompress(original_text_z)
         
         return original_text
+    
+    def fetch_original_page(self, force=False, request=None, debug=False):
+        return None
         
 class MStarredStoryCounts(mongo.Document):
     user_id = mongo.IntField()
@@ -2964,8 +3053,10 @@ class MStarredStoryCounts(mongo.Document):
         if user_feeds.get(0, False):
             user_feeds[-1] = user_feeds.get(0, 0)
             del user_feeds[0]
-
+        
+        too_many_feeds = False if len(user_feeds) < 1000 else True
         for feed_id, count in user_feeds.items():
+            if too_many_feeds and count <= 1: continue
             cls.objects(user_id=user_id, 
                         feed_id=feed_id, 
                         slug="feed:%s" % feed_id).update_one(set__count=count, 

apps/rss_feeds/page_importer.py

@@ -170,7 +170,9 @@ class PageImporter(object):
     def _fetch_story(self):
         html = None
         story_permalink = self.story.story_permalink
-            
+        
+        if not self.feed:
+            return
         if any(story_permalink.startswith(s) for s in BROKEN_PAGES):
             return
         if any(s in story_permalink.lower() for s in BROKEN_PAGE_URLS):
@@ -181,11 +183,12 @@ class PageImporter(object):
         try:
             response = requests.get(story_permalink, headers=self.headers)
             response.connection.close()
-        except requests.exceptions.TooManyRedirects:
-            response = requests.get(story_permalink)
-        except (AttributeError, SocketError, OpenSSLError, PyAsn1Error, requests.exceptions.ConnectionError), e:
-            logging.debug('   ***> [%-30s] Original story fetch failed using requests: %s' % (self.feed.log_title[:30], e))
-            return
+        except (AttributeError, SocketError, OpenSSLError, PyAsn1Error, requests.exceptions.ConnectionError, requests.exceptions.TooManyRedirects), e:
+            try:
+                response = requests.get(story_permalink)
+            except (AttributeError, SocketError, OpenSSLError, PyAsn1Error, requests.exceptions.ConnectionError, requests.exceptions.TooManyRedirects), e:
+                logging.debug('   ***> [%-30s] Original story fetch failed using requests: %s' % (self.feed.log_title[:30], e))
+                return
         try:
             data = response.text
         except (LookupError, TypeError):

apps/rss_feeds/text_importer.py

@@ -1,5 +1,7 @@
 import requests
+import urllib3
 import zlib
+from simplejson.decoder import JSONDecodeError
 from requests.packages.urllib3.exceptions import LocationParseError
 from socket import error as SocketError
 from mongoengine.queryset import NotUniqueError
@@ -12,7 +14,8 @@ from pyasn1.error import PyAsn1Error
 from django.utils.encoding import smart_str
 from django.conf import settings
 from BeautifulSoup import BeautifulSoup
-
+from urlparse import urljoin
+ 
 BROKEN_URLS = [
     "gamespot.com",
 ]
@@ -68,9 +71,12 @@ class TextImporter:
         if not resp:
             return
         
-        doc = resp.json()
-        if doc.get('error', False):
-            logging.user(self.request, "~SN~FRFailed~FY to fetch ~FGoriginal text~FY: %s" % doc.get('messages', "[unknown merucry error]"))
+        try:
+            doc = resp.json()
+        except JSONDecodeError:
+            doc = None
+        if not doc or doc.get('error', False):
+            logging.user(self.request, "~SN~FRFailed~FY to fetch ~FGoriginal text~FY: %s" % (doc and doc.get('messages', None) or "[unknown mercury error]"))
             return
         
         text = doc['content']
@@ -78,6 +84,10 @@ class TextImporter:
         url = doc['url']
         image = doc['lead_image_url']
         
+        if image and ('http://' in image[1:] or 'https://' in image[1:]):
+            logging.user(self.request, "~SN~FRRemoving broken image from text: %s" % image)
+            image = None
+        
         return self.process_content(text, title, url, image, skip_save=skip_save, return_document=return_document)
         
     def fetch_manually(self, skip_save=False, return_document=False):
@@ -126,9 +136,6 @@ class TextImporter:
 
         url = resp.url
         
-        if content:
-            content = self.rewrite_content(content)
-    
         return self.process_content(content, title, url, image=None, skip_save=skip_save, return_document=return_document,
                                     original_text_doc=original_text_doc)
         
@@ -154,6 +161,9 @@ class TextImporter:
             )), warn_color=False)
             return
         
+        if content:
+            content = self.rewrite_content(content)
+        
         if return_document:
             return dict(content=content, title=title, url=url, doc=original_text_doc, image=image)
 
@@ -166,7 +176,14 @@ class TextImporter:
             if len(noscript.contents) > 0:
                 noscript.replaceWith(noscript.contents[0])
         
-        return unicode(soup)
+        content = unicode(soup)
+        
+        images = set([img['src'] for img in soup.findAll('img') if 'src' in img])
+        for image_url in images:
+            abs_image_url = urljoin(self.story.story_permalink, image_url)
+            content = content.replace(image_url, abs_image_url)
+        
+        return content
     
     @timelimit(10)
     def fetch_request(self, use_mercury=True):
@@ -179,7 +196,10 @@ class TextImporter:
             mercury_api_key = getattr(settings, 'MERCURY_PARSER_API_KEY', 'abc123')
             headers["content-type"] = "application/json"
             headers["x-api-key"] = mercury_api_key
-            url = "https://mercury.postlight.com/parser?url=%s" % url
+            if settings.DEBUG:
+                url = "http://nb.local.com:4040/rss_feeds/original_text_fetcher?url=%s" % url
+            else:
+                url = "https://www.newsblur.com/rss_feeds/original_text_fetcher?url=%s" % url
             
         try:
             r = requests.get(url, headers=headers, verify=False)
@@ -190,6 +210,7 @@ class TextImporter:
                 requests.models.InvalidURL,
                 requests.models.ChunkedEncodingError,
                 requests.models.ContentDecodingError,
+                urllib3.exceptions.LocationValueError,
                 LocationParseError, OpenSSLError, PyAsn1Error), e:
             logging.user(self.request, "~SN~FRFailed~FY to fetch ~FGoriginal text~FY: %s" % e)
             return

apps/rss_feeds/views.py

@@ -512,6 +512,8 @@ def original_text(request):
         'feed_id': story.story_feed_id,
         'story_hash': story.story_hash,
         'story_id': story.story_guid,
+        'image_urls': story.image_urls,
+        'secure_image_urls': Feed.secure_image_urls(story.image_urls),
         'original_text': original_text,
         'failed': not original_text or len(original_text) < 100,
     }
@@ -526,7 +528,8 @@ def original_story(request):
 
     if not story:
         logging.user(request, "~FYFetching ~FGoriginal~FY story page: ~FRstory not found")
-        return {'code': -1, 'message': 'Story not found.', 'original_page': None, 'failed': True}
+        # return {'code': -1, 'message': 'Story not found.', 'original_page': None, 'failed': True}
+        raise Http404
     
     original_page = story.fetch_original_page(force=force, request=request, debug=debug)
 

apps/social/models.py

@@ -1,3 +1,5 @@
+import os
+import urlparse
 import datetime
 import time
 import zlib
@@ -347,7 +349,7 @@ class MSocialProfile(mongo.Document):
     def email_photo_url(self):
         if self.photo_url:
             if self.photo_url.startswith('//'):
-                self.photo_url = 'http:' + self.photo_url
+                self.photo_url = 'https:' + self.photo_url
             return self.photo_url
         domain = Site.objects.get_current().domain
         return 'https://' + domain + settings.MEDIA_URL + 'img/reader/default_profile_photo.png'
@@ -2131,8 +2133,8 @@ class MSharedStory(mongo.DynamicDocument):
         
         if service == 'twitter':
             posted = social_service.post_to_twitter(self)
-        elif service == 'facebook':
-            posted = social_service.post_to_facebook(self)
+        # elif service == 'facebook':
+        #     posted = social_service.post_to_facebook(self)
         elif service == 'appdotnet':
             posted = social_service.post_to_appdotnet(self)
         
@@ -2290,6 +2292,19 @@ class MSharedStory(mongo.DynamicDocument):
             original_user.username,
             self.decoded_story_title[:30]))
     
+    def extract_image_urls(self, force=False):
+        if not self.story_content_z:
+            return
+
+        if self.image_urls and not force:
+            return
+            
+        soup = BeautifulSoup(zlib.decompress(self.story_content_z))
+        image_sources = [img.get('src') for img in soup.findAll('img') if img and img.get('src')]
+        if len(image_sources) > 0:
+            self.image_urls = image_sources
+            self.save()
+            
     def calculate_image_sizes(self, force=False):
         if not self.story_content_z:
             return
@@ -2305,11 +2320,11 @@ class MSharedStory(mongo.DynamicDocument):
                 settings.NEWSBLUR_URL
             ),
         }
-        soup = BeautifulSoup(zlib.decompress(self.story_content_z))
-        image_sources = [img.get('src') for img in soup.findAll('img')]
+        
+        self.extract_image_urls()
         image_sizes = []
         
-        for image_source in image_sources[:10]:
+        for image_source in self.image_urls[:10]:
             if any(ignore in image_source for ignore in IGNORE_IMAGE_SOURCES):
                 continue
             req = requests.get(image_source, headers=headers, stream=True)
@@ -2320,8 +2335,8 @@ class MSharedStory(mongo.DynamicDocument):
                 logging.debug(" ***> Couldn't read image: %s / %s" % (e, image_source))
                 datastream = StringIO(req.content[:100])
                 _, width, height = image_size(datastream)
-            if width <= 16 or height <= 16:
-                continue
+            # if width <= 16 or height <= 16:
+            #     continue
             image_sizes.append({'src': image_source, 'size': (width, height)})
         
         if image_sizes:
@@ -2332,7 +2347,7 @@ class MSharedStory(mongo.DynamicDocument):
         self.save()
         
         logging.debug(" ---> ~SN~FGFetched image sizes on shared story: ~SB%s/%s images" % 
-                      (self.image_count, len(image_sources)))
+                      (self.image_count, len(self.image_urls)))
         
         return image_sizes
     
@@ -2473,7 +2488,7 @@ class MSocialServices(mongo.Document):
         return api
     
     def facebook_api(self):
-        graph = facebook.GraphAPI(self.facebook_access_token)
+        graph = facebook.GraphAPI(access_token=self.facebook_access_token, version="3.1")
         return graph
     
     def appdotnet_api(self):
@@ -2581,10 +2596,10 @@ class MSocialServices(mongo.Document):
         self.syncing_facebook = False
         self.save()
         
-        facebook_user = graph.request('me', args={'fields':'website,bio,location'})
+        facebook_user = graph.request('me', args={'fields':'website,about,location'})
         profile = MSocialProfile.get_user(self.user_id)
         profile.location = profile.location or (facebook_user.get('location') and facebook_user['location']['name'])
-        profile.bio = profile.bio or facebook_user.get('bio')
+        profile.bio = profile.bio or facebook_user.get('about')
         if not profile.website and facebook_user.get('website'):
             profile.website = facebook_user.get('website').split()[0]
         profile.save()
@@ -2783,19 +2798,43 @@ class MSocialServices(mongo.Document):
         self.set_photo('twitter')
         
     def post_to_twitter(self, shared_story):
-        message = shared_story.generate_post_to_service_message(truncate=140)
+        message = shared_story.generate_post_to_service_message(truncate=280)
         shared_story.calculate_image_sizes()
         
         try:
             api = self.twitter_api()
-            api.update_status(status=message)
+            filename = self.fetch_image_file_for_twitter(shared_story)
+            if filename:
+                api.update_with_media(filename, status=message)
+                os.remove(filename)
+            else:
+                api.update_status(status=message)
         except tweepy.TweepError, e:
             user = User.objects.get(pk=self.user_id)
             logging.user(user, "~FRTwitter error: ~SB%s" % e)
             return
             
         return True
-            
+    
+    def fetch_image_file_for_twitter(self, shared_story):
+        if not shared_story.image_urls: return
+
+        user = User.objects.get(pk=self.user_id)
+        logging.user(user, "~FCFetching image for twitter: ~SB%s" % shared_story.image_urls[0])
+        
+        url = shared_story.image_urls[0]
+        image_filename = os.path.basename(urlparse.urlparse(url).path)
+        req = requests.get(url, stream=True)
+        filename = "/tmp/%s-%s" % (shared_story.story_hash, image_filename)
+        
+        if req.status_code == 200:
+            f = open(filename, "wb")
+            for chunk in req:
+                f.write(chunk)
+            f.close()
+        
+            return filename
+                
     def post_to_facebook(self, shared_story):
         message = shared_story.generate_post_to_service_message(include_url=False)
         shared_story.calculate_image_sizes()

apps/social/tasks.py

@@ -69,11 +69,11 @@ class SharePopularStories(Task):
         logging.debug(" ---> Sharing popular stories...")
         MSharedStory.share_popular_stories(interactive=False)
             
-class CleanSpam(Task):
-    name = 'clean-spam'
+class CleanSocialSpam(Task):
+    name = 'clean-social-spam'
 
     def run(self, **kwargs):
-        logging.debug(" ---> Finding spammers...")
+        logging.debug(" ---> Finding social spammers...")
         MSharedStory.count_potential_spammers(destroy=True)
             
 

apps/social/views.py

@@ -76,9 +76,11 @@ def load_social_stories(request, user_id, username=None):
         if story_hashes:
             mstories = MSharedStory.objects(user_id=social_user.pk,
                                             story_hash__in=story_hashes).order_by(story_date_order)
+            for story in mstories: story.extract_image_urls()
             stories = Feed.format_stories(mstories)
     else:
         mstories = MSharedStory.objects(user_id=social_user.pk).order_by('-shared_date')[offset:offset+limit]
+        for story in mstories: story.extract_image_urls()
         stories = Feed.format_stories(mstories)
 
     if not stories or False: # False is to force a recount even if 0 stories
@@ -748,9 +750,16 @@ def save_comment_reply(request):
             'message': 'You must be following %s to reply to them.' % commenter_profile.username,
         })
     
-    shared_story = MSharedStory.objects.get(user_id=comment_user_id, 
-                                            story_feed_id=feed_id, 
-                                            story_guid=story_id)
+    try:
+        shared_story = MSharedStory.objects.get(user_id=comment_user_id, 
+                                                story_feed_id=feed_id, 
+                                                story_guid=story_id)
+    except MSharedStory.DoesNotExist:
+        return json.json_response(request, {
+            'code': -1, 
+            'message': 'Shared story cannot be found.',
+        })
+        
     reply = MCommentReply()
     reply.user_id = request.user.pk
     reply.publish_date = datetime.datetime.now()
@@ -980,7 +989,7 @@ def save_user_profile(request):
     profile.private = is_true(data.get('private', False))
     profile.save()
 
-    social_services = MSocialServices.objects.get(user_id=request.user.pk)
+    social_services = MSocialServices.get_user(user_id=request.user.pk)
     profile = social_services.set_photo(data['photo_service'])
     
     logging.user(request, "~BB~FRSaving social profile")

apps/statistics/models.py

@@ -2,10 +2,12 @@ import datetime
 import mongoengine as mongo
 import urllib2
 import redis
+import dateutil
 from django.conf import settings
 from apps.social.models import MSharedStory
 from apps.profile.models import Profile
 from apps.statistics.rstats import RStats, round_time
+from utils.story_functions import relative_date
 from utils import json_functions as json
 from utils import db_functions
 from utils import log as logging
@@ -238,8 +240,8 @@ class MStatistics(mongo.Document):
 
 
 class MFeedback(mongo.Document):
-    date    = mongo.StringField()
-    summary = mongo.StringField()
+    date    = mongo.DateTimeField()
+    date_short = mongo.StringField()
     subject = mongo.StringField()
     url     = mongo.StringField()
     style   = mongo.StringField()
@@ -252,34 +254,46 @@ class MFeedback(mongo.Document):
         'ordering': ['order'],
     }
     
+    CATEGORIES = {
+        5: 'idea',
+        6: 'problem',
+        7: 'praise',
+        8: 'question',
+    }
+    
     def __unicode__(self):
         return "%s: (%s) %s" % (self.style, self.date, self.subject)
         
     @classmethod
     def collect_feedback(cls):
+        seen_posts = set()
         try:
-            data = urllib2.urlopen('https://getsatisfaction.com/newsblur/topics.widget').read()
+            data = urllib2.urlopen('https://forum.newsblur.com/posts.json').read()
         except (urllib2.HTTPError), e:
             logging.debug(" ***> Failed to collect feedback: %s" % e)
             return
-        start = data.index('[')
-        end = data.rfind(']')+1
-        data = json.decode(data[start:end])
-        print data
-        i    = 0
-        if len(data):
-            cls.objects.delete()
-            for feedback in data:
-                feedback['order'] = i
-                i += 1
-                for removal in ['about', 'less than']:
-                    if removal in feedback['date']:
-                        feedback['date'] = feedback['date'].replace(removal, '')
-            for feedback in data:
-                # Convert unicode to strings.
-                fb = dict([(str(k), v) for k, v in feedback.items()])
-                fb['url'] = fb['url'].replace('?utm_medium=widget&utm_source=widget_newsblur', "")
-                cls.objects.create(**fb)
+        data = json.decode(data).get('latest_posts', "")
+
+        if not len(data):
+            print "No data!"
+            return
+            
+        cls.objects.delete()
+        post_count = 0
+        for post in data:
+            if post['topic_id'] in seen_posts: continue
+            seen_posts.add(post['topic_id'])
+            feedback = {}
+            feedback['order'] = post_count
+            post_count += 1
+            feedback['date'] = dateutil.parser.parse(post['created_at']).replace(tzinfo=None)
+            feedback['date_short'] = relative_date(feedback['date'])
+            feedback['subject'] = post['topic_title']
+            feedback['url'] = "https://forum.newsblur.com/t/%s/%s/%s" % (post['topic_slug'], post['topic_id'], post['post_number'])
+            feedback['style'] = cls.CATEGORIES[post['category_id']]
+            cls.objects.create(**feedback)
+            print "%s: %s (%s)" % (feedback['style'], feedback['subject'], feedback['date_short'])
+            if post_count >= 4: break
     
     @classmethod
     def all(cls):
@@ -337,3 +351,30 @@ class MAnalyticsFetcher(mongo.Document):
     @classmethod
     def calculate_stats(cls, stats):
         return cls.aggregate(**stats)
+
+
+class MAnalyticsLoader(mongo.Document):
+    date = mongo.DateTimeField(default=datetime.datetime.now)
+    page_load = mongo.FloatField()
+    server = mongo.StringField()
+    
+    meta = {
+        'db_alias': 'nbanalytics',
+        'collection': 'page_loads',
+        'allow_inheritance': False,
+        'indexes': ['date', 'server'],
+        'ordering': ['date'],
+    }
+    
+    def __unicode__(self):
+        return "%s: %.4ss" % (self.server, self.page_load)
+        
+    @classmethod
+    def add(cls, page_load):
+        server_name = settings.SERVER_NAME
+
+        cls.objects.create(page_load=page_load, server=server_name)
+    
+    @classmethod
+    def calculate_stats(cls, stats):
+        return cls.aggregate(**stats)

apps/statistics/urls.py

@@ -4,4 +4,5 @@ from apps.statistics import views
 urlpatterns = patterns('',
     url(r'^dashboard_graphs', views.dashboard_graphs, name='statistics-graphs'),
     url(r'^feedback_table', views.feedback_table, name='feedback-table'),
+    url(r'^revenue', views.revenue, name='revenue'),
 )

apps/statistics/views.py

@@ -1,6 +1,12 @@
+import datetime
+from django.http import HttpResponse
 from django.template import RequestContext
 from django.shortcuts import render_to_response
+from django.utils import feedgenerator
 from apps.statistics.models import MStatistics, MFeedback
+from apps.profile.models import PaymentHistory
+from utils import log as logging
+
 
 def dashboard_graphs(request):
     statistics = MStatistics.all()
@@ -12,4 +18,32 @@ def feedback_table(request):
     feedbacks = MFeedback.all()
     return render_to_response('statistics/render_feedback_table.xhtml', {
         'feedbacks': feedbacks,
-    }, context_instance=RequestContext(request))
+    }, context_instance=RequestContext(request))
+
+def revenue(request):
+    data = {}
+    data['title'] = "NewsBlur Revenue"
+    data['link'] = "https://www.newsblur.com"
+    data['description'] = "Revenue"
+    data['lastBuildDate'] = datetime.datetime.utcnow()
+    data['generator'] = 'NewsBlur Revenue Writer'
+    data['docs'] = None
+    rss = feedgenerator.Atom1Feed(**data)
+    
+    report = PaymentHistory.report()
+    content = "%s revenue: $%s<br><code>%s</code>" % (datetime.datetime.now().strftime('%Y'), report['annual'], report['output'].replace('\n', '<br>'))
+    
+    story = {
+        'title': "Daily snapshot: %s" % (datetime.datetime.now().strftime('%a %b %-d, %Y')),
+        'link': 'https://www.newsblur.com',
+        'description': content,
+        'unique_id': datetime.datetime.now().strftime('%a %b %-d, %Y'),
+        'pubdate': datetime.datetime.now(),
+    }
+    rss.add_item(**story)
+    
+    logging.user(request, "~FBGenerating Revenue RSS feed: ~FM%s" % (
+        request.META.get('HTTP_USER_AGENT', "")[:24]
+    ))
+    return HttpResponse(rss.writeString('utf-8'), content_type='application/rss+xml')
+    

assets.yml

@@ -25,7 +25,7 @@ compress_assets: on
 
 javascripts:
     common:
-        - media/js/vendor/jquery-2.0.3.js
+        - media/js/vendor/jquery-2.2.4.js
         # - media/js/vendor/jquery.migrate-*.js
         - media/js/vendor/jquery.browser.js
         - media/js/vendor/jquery.json.js
@@ -83,7 +83,7 @@ javascripts:
         - media/js/newsblur/payments/paypal_return.js
         - media/js/newsblur/payments/stripe_form.js
     bookmarklet:
-        - media/js/vendor/jquery-2.0.3.min.js
+        - media/js/vendor/jquery-2.2.4.min.js
         # - media/js/vendor/jquery.migrate-*.js
         - media/js/vendor/jquery.noConflict.js
         - media/js/vendor/jquery.browser.js
@@ -98,7 +98,7 @@ javascripts:
         - media/js/vendor/underscore.noconflict.js
         - media/js/vendor/readability-*.js
     blurblog:
-        - media/js/vendor/jquery-2.0.3.js
+        - media/js/vendor/jquery-2.2.4.js
         # - media/js/vendor/jquery.migrate-*.js
         - media/js/vendor/jquery.browser.js
         - media/js/vendor/jquery.color.js

clients/android/NewsBlur/AndroidManifest.xml

@@ -1,12 +1,12 @@
 <?xml version="1.0" encoding="utf-8"?>
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
     package="com.newsblur"
-    android:versionCode="147"
-    android:versionName="6.3b1" >
+    android:versionCode="163"
+    android:versionName="9.0.1" >
 
     <uses-sdk
-        android:minSdkVersion="19"
-        android:targetSdkVersion="24" />
+        android:minSdkVersion="21"
+        android:targetSdkVersion="28" />
 
     <uses-permission android:name="android.permission.INTERNET" />
     <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
@@ -137,7 +137,17 @@
         <activity
             android:name=".activity.SocialFeedReading"/>
 
-        <service android:name=".service.NBSyncService" />
+        <activity android:name=".widget.ConfigureWidgetActivity">
+            <intent-filter>
+                <action android:name="android.appwidget.action.APPWIDGET_CONFIGURE" />
+            </intent-filter>
+        </activity>
+        <service 
+            android:name=".service.NBSyncService"
+            android:permission="android.permission.BIND_JOB_SERVICE" />
+        <service android:name=".widget.BlurWidgetRemoteViewsService"
+            android:permission="android.permission.BIND_REMOTEVIEWS"
+            />
 
         <receiver android:name=".service.BootReceiver">
             <intent-filter>
@@ -145,12 +155,16 @@
             </intent-filter>
         </receiver>
 
-        <receiver android:name=".service.ServiceScheduleReceiver" />
-        
         <receiver android:name=".util.NotifyDismissReceiver" android:exported="false" />
         <receiver android:name=".util.NotifySaveReceiver" android:exported="false" />
         <receiver android:name=".util.NotifyMarkreadReceiver" android:exported="false" />
-
+        <receiver android:name=".widget.NewsBlurWidgetProvider" >
+            <intent-filter>
+                <action android:name="android.appwidget.action.APPWIDGET_UPDATE" />
+            </intent-filter>
+            <meta-data android:name="android.appwidget.provider"
+                android:resource="@xml/newsblur_appwidget_info" />
+        </receiver>
         <provider
             android:name="android.support.v4.content.FileProvider"
             android:authorities="com.newsblur.fileprovider"

clients/android/NewsBlur/assets/black_reading.css

@@ -0,0 +1,30 @@
+body, span, table, td, p, div, li {
+	background-color: #000000 !important;
+	color: #CCC !important;
+}
+
+a, a span {
+	color: #277D9D !important;
+	background-color: #000000 !important;
+}
+
+a:visited, a:visited span {
+	color: #277D9D !important;
+	background-color: #000000 !important;
+}
+
+pre, blockquote, code {
+    background-color: #1A1A1A;
+}
+
+blockquote *, pre *, code * {
+    background-color: #1A1A1A !important;
+}
+
+blockquote a, blockquote a span {
+    background-color: #1A1A1A !important;
+}
+
+.NB-story > table {
+    background-color: #1A1A1A !important;
+}

clients/android/NewsBlur/assets/light_reading.css

@@ -1,4 +1,4 @@
-body {
+body, html {
 	background-color: #FFF;
 }
 
@@ -17,4 +17,10 @@ code {
 pre, blockquote {
     background-color: #F2F2F2;
     color: rgba(10, 12, 38, .8);
+}
+/* do not adhere to Chromium's color scheme settings */
+@media(prefers-color-scheme: dark){
+    body, html {
+        background-color: #FFF;
+    }
 }

clients/android/NewsBlur/assets/reading.css

@@ -26,7 +26,7 @@ p, a, table, video, embed, object, iframe, div, figure, dl, dt, center {
        can only scroll in vertical. however, do not auto-set height for these, as they tend to ratio down to something
        tiny before dynamic content is done loading. these types will resize well, but exclude images, which distort. */
 	width: auto !important;
-    max-width: none !important;
+    max-width: 100% !important;
     margin: 0px !important;
     min-width: 0px !important;
 }
@@ -34,10 +34,10 @@ p, a, table, video, embed, object, iframe, div, figure, dl, dt, center {
 img {
     /* in addition to the tweaks for other media, set image height to auto, so aspect ratios are correct */
 	width: auto !important;
-    max-width: 100% !important;
+    max-width: 99% !important;
     height: auto !important;
     max-height: none !important;
-    margin: 0px !important;
+    margin: 1px !important;
 }
 
 p {
@@ -62,6 +62,7 @@ pre, blockquote {
     padding: 0.6em;
     margin: 0.6em;
     word-wrap: break-word !important;
+    white-space: pre-wrap !important;
 }
 
 /* these are the diff blocks produced by NewsBlur */

clients/android/NewsBlur/build.gradle

@@ -1,32 +1,49 @@
 buildscript {
     repositories {
         mavenCentral()
+        maven {
+            url 'https://maven.google.com'
+        }
         jcenter()
+        google()
     }
     dependencies {
-        classpath 'com.android.tools.build:gradle:2.3.3'
+        classpath 'com.android.tools.build:gradle:3.1.4'
     }
 }
 
 repositories {
     mavenCentral()
+    maven {
+        url 'https://maven.google.com'
+    }
+    jcenter()
 }
 
 apply plugin: 'com.android.application'
+apply plugin: 'checkstyle'
+apply plugin: 'findbugs'
 
 dependencies {
-    compile 'com.android.support:support-v13:19.1.0'
+    compile 'com.android.support:support-core-utils:27.1.1'
+    compile 'com.android.support:support-fragment:27.1.1'
+    compile 'com.android.support:support-core-ui:27.1.1'
     compile 'com.jakewharton:butterknife:7.0.1'
     compile 'com.squareup.okhttp3:okhttp:3.8.1'
-    compile 'com.google.code.gson:gson:2.7'
+    compile 'com.google.code.gson:gson:2.8.2'
+    compile 'com.android.support:recyclerview-v7:27.1.1'
 }
 
 android {
-    compileSdkVersion 24
-    buildToolsVersion '25.0.2'
+    compileSdkVersion 27
+    buildToolsVersion '27.0.3'
     compileOptions.with {
         sourceCompatibility = JavaVersion.VERSION_1_7
     }
+
+    // force old processing behaviour that butterknife 7 depends on, until we can afford to upgrade
+    android.defaultConfig.javaCompileOptions.annotationProcessorOptions.includeCompileClasspath = true
+
     sourceSets {
         main {
             manifest.srcFile 'AndroidManifest.xml'
@@ -35,9 +52,11 @@ android {
             assets.srcDirs = ['assets']
         }
     }
+
     lintOptions {
         abortOnError false
     }
+
     buildTypes {
         debug {
             minifyEnabled false

clients/android/NewsBlur/res/drawable-nodpi/black_divider_item_header.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<shape xmlns:android="http://schemas.android.com/apk/res/android" android:shape="rectangle">
+    <solid android:color="@color/gray07" />
+    <size android:height="1dp" />
+</shape>

clients/android/NewsBlur/res/drawable-nodpi/black_folder_background_default.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<shape xmlns:android="http://schemas.android.com/apk/res/android"
+       android:shape="rectangle">
+    <gradient
+            android:angle="270"
+            android:type="linear"
+            android:startColor="@color/black_folder_background_start"
+            android:endColor="@color/black_folder_background_end"/>
+</shape>

clients/android/NewsBlur/res/drawable-nodpi/black_gradient_background_default.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="utf-8"?>
+<shape xmlns:android="http://schemas.android.com/apk/res/android" android:shape="rectangle" >
+    <gradient
+        android:angle=     "90"
+        android:type=      "linear"
+        android:startColor="@color/black_folder_background_start"
+        android:endColor=  "@color/black_folder_background_end" />
+</shape>

clients/android/NewsBlur/res/drawable-nodpi/black_row_item_header_background.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="utf-8"?>
+<shape xmlns:android="http://schemas.android.com/apk/res/android" android:shape="rectangle" >
+    <gradient
+            android:angle=     "270"
+            android:type=      "linear"
+            android:startColor="@color/black_item_header_background_start"
+            android:endColor=  "@color/black_item_header_background_end" />
+</shape>

clients/android/NewsBlur/res/drawable-nodpi/black_selector_folder_background.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="utf-8"?>
+<selector xmlns:android="http://schemas.android.com/apk/res/android" >
+    <item android:state_pressed="true" android:drawable="@drawable/dark_folder_background_highlight" />
+    <item android:drawable="@drawable/black_folder_background_default" />
+</selector>

clients/android/NewsBlur/res/drawable-nodpi/black_selector_story_background.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="utf-8"?>
+<selector xmlns:android="http://schemas.android.com/apk/res/android">
+    <item android:state_pressed="true" android:drawable="@drawable/dark_story_background_highlight"/>
+    <item android:drawable="@drawable/black_story_background_default"/>
+</selector>

clients/android/NewsBlur/res/drawable-nodpi/black_story_background_default.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="utf-8"?>
+<shape xmlns:android="http://schemas.android.com/apk/res/android"
+       android:shape="rectangle">
+    <solid android:color="@color/black"/>
+</shape>

clients/android/NewsBlur/res/drawable-nodpi/item_border.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="utf-8"?>
+<shape xmlns:android="http://schemas.android.com/apk/res/android">
+    <!-- on API 14 and 15, the default solid is a black square, not nothing. -->
+    <solid android:color="@android:color/transparent" />
+    <padding
+        android:bottom="1dp" 
+        android:top="1dp" 
+        android:left="1dp" 
+        android:right="1dp" 
+    />
+    <stroke 
+        android:color="@color/gray55" 
+        android:width="1dp" 
+        android:height="1dp" 
+    />
+    <corners android:radius="0dp" />
+</shape>

clients/android/NewsBlur/res/layout/activity_configure_widget.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="utf-8"?>
+<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    android:orientation="vertical" android:layout_width="match_parent"
+    android:layout_height="match_parent">
+</LinearLayout>

clients/android/NewsBlur/res/layout/activity_login.xml

@@ -24,12 +24,18 @@
         android:layout_width="match_parent"
         android:background="@color/gray80" />
     
-    <FrameLayout
-        android:id="@+id/login_container"
+    <ScrollView
         android:layout_width="match_parent"
-        android:layout_height="match_parent"
-        android:gravity="center"
-        android:paddingLeft="30dp"
-        android:paddingRight="30dp" />
+        android:layout_height="match_parent" >
+
+        <FrameLayout
+            android:id="@+id/login_container"
+            android:layout_width="match_parent"
+            android:layout_height="match_parent"
+            android:gravity="center"
+            android:paddingLeft="30dp"
+            android:paddingRight="30dp" />
+    
+    </ScrollView>
 
 </LinearLayout>

clients/android/NewsBlur/res/layout/activity_reading.xml

@@ -14,8 +14,8 @@
         style="?explainerText"
         android:textSize="16sp" />
 
-    <android.support.v4.view.ViewPager
-        android:id="@+id/reading_pager"
+    <FrameLayout
+        android:id="@+id/activity_reading_container"
         android:layout_width="match_parent"
         android:layout_height="match_parent" />
 

clients/android/NewsBlur/res/layout/dialog_rename_feed.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="utf-8"?>
+<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent"
+    android:paddingLeft="10dp"
+    android:paddingRight="10dp" >
+
+    <EditText
+        android:id="@+id/feed_name_field"
+        android:layout_width="match_parent"
+        android:layout_height="wrap_content"
+        android:layout_marginBottom="10dp"
+        android:layout_marginTop="5dp"
+        android:singleLine="true"
+    />
+
+</RelativeLayout>

clients/android/NewsBlur/res/layout/dialog_trainstory.xml

@@ -24,8 +24,8 @@
         android:layout_below="@id/intel_title_header"
         android:layout_alignParentRight="true"
         android:layout_margin="4dp"
-        android:layout_width="28dp"
-        android:layout_height="28dp"
+        android:layout_width="32dp"
+        android:layout_height="32dp"
         android:background="@drawable/ic_clear_gray55"
     />
 
@@ -37,8 +37,8 @@
         android:layout_marginBottom="4dp"
         android:layout_marginRight="4dp"
         android:layout_marginLeft="6dp"
-        android:layout_width="28dp"
-        android:layout_height="28dp"
+        android:layout_width="32dp"
+        android:layout_height="32dp"
         android:background="@drawable/ic_dislike_gray55"
     />
 
@@ -47,8 +47,8 @@
         android:layout_below="@id/intel_title_header"
         android:layout_toLeftOf="@id/intel_title_dislike"
         android:layout_margin="4dp"
-        android:layout_width="28dp"
-        android:layout_height="28dp"
+        android:layout_width="32dp"
+        android:layout_height="32dp"
         android:background="@drawable/ic_like_gray55"
     />
         

clients/android/NewsBlur/res/layout/fragment_itemgrid.xml

@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="utf-8"?>
+<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent"
+    style="?itemBackground"
+    android:orientation="vertical" >
+
+    <RelativeLayout
+        android:id="@+id/empty_view"
+        android:layout_width="match_parent"
+        android:layout_height="match_parent"
+        >
+
+        <TextView
+            android:id="@+id/empty_view_text"
+            android:layout_width="match_parent"
+            android:layout_height="wrap_content"
+            android:layout_alignParentTop="true"
+            android:layout_marginTop="50dp"
+            android:gravity="center_horizontal"
+            android:text="@string/empty_list_view_loading"
+            android:textSize="16sp"
+        />
+
+        <ImageView
+            android:id="@+id/empty_view_image"
+            android:layout_width="80dp"
+            android:layout_height="80dp"
+            android:contentDescription="@string/description_empty_list_image"
+            android:src="@drawable/world_big"
+            android:layout_below="@id/empty_view_text"
+            android:layout_marginTop="15dp"
+            android:layout_centerHorizontal="true"
+            android:scaleType="fitCenter"
+            android:visibility="invisible"
+        />
+
+    </RelativeLayout>
+
+    <com.newsblur.view.ProgressThrobber
+        android:id="@+id/top_loading_throb"
+        android:layout_width="fill_parent"
+        android:layout_alignParentTop="true"
+        android:layout_height="6dp"
+    />
+    
+    <android.support.v7.widget.RecyclerView
+        android:id="@+id/itemgridfragment_grid"
+        android:layout_width="match_parent"
+        android:layout_height="match_parent"
+        android:scrollbars="vertical"
+    />
+    
+</RelativeLayout>

clients/android/NewsBlur/res/layout/fragment_itemlist.xml

@@ -1,49 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
-    android:layout_width="match_parent"
-    android:layout_height="match_parent"
-    style="?itemBackground"
-    android:orientation="vertical" >
-
-    <RelativeLayout
-        android:id="@+id/empty_view"
-        android:layout_width="match_parent"
-        android:layout_height="match_parent" >
-
-        <TextView
-            android:id="@+id/empty_view_text"
-            android:layout_width="match_parent"
-            android:layout_height="wrap_content"
-            android:layout_alignParentTop="true"
-            android:layout_marginTop="50dp"
-            android:gravity="center_horizontal"
-            android:text="@string/empty_list_view_loading"
-            android:textSize="16sp" />
-
-        <ImageView
-            android:id="@+id/empty_view_image"
-            android:layout_width="80dp"
-            android:layout_height="80dp"
-            android:contentDescription="@string/description_empty_list_image"
-            android:src="@drawable/world_big"
-            android:layout_below="@id/empty_view_text"
-            android:layout_marginTop="15dp"
-            android:layout_centerHorizontal="true"
-            android:scaleType="fitCenter"
-            android:visibility="invisible" />
-
-        <com.newsblur.view.ProgressThrobber
-            android:id="@+id/empty_view_loading_throb"
-            android:layout_width="fill_parent"
-            android:layout_alignParentTop="true"
-            android:layout_height="6dp" />
-    
-    </RelativeLayout>
-
-    <ListView
-        android:id="@+id/itemlistfragment_list"
-        android:layout_width="match_parent"
-        android:layout_height="match_parent"
-        android:divider="@null" />
-    
-</RelativeLayout>

clients/android/NewsBlur/res/layout/fragment_readingitem.xml

@@ -10,7 +10,7 @@
         android:layout_width="match_parent"
         android:layout_height="match_parent"
         style="?readingBackground"
-     >
+        >
 
         <LinearLayout
             android:layout_width="match_parent"
@@ -18,9 +18,105 @@
             android:orientation="vertical"
             android:focusable="true"
             android:focusableInTouchMode="true"
-        >
+            >
 
-            <include layout="@layout/include_reading_item_metadata" />
+            <RelativeLayout
+                android:layout_width="match_parent"
+                android:layout_height="wrap_content"
+                android:paddingBottom="8dp"
+                style="?rowItemHeaderBackground">
+
+                <RelativeLayout
+                    android:id="@+id/row_item_feed_header"
+                    android:layout_height="wrap_content"
+                    android:layout_width="match_parent"
+                    android:minHeight="6dp"
+                    android:paddingTop="4dp"
+                    android:paddingBottom="4dp" >
+
+                    <ImageView
+                        android:id="@+id/reading_feed_icon"
+                        android:layout_width="17dp"
+                        android:layout_height="17dp"
+                        android:layout_alignParentLeft="true"
+                        android:layout_marginLeft="16dp"
+                        android:layout_marginRight="5dp"
+                        android:contentDescription="@string/description_row_folder_icon" />
+
+                    <TextView
+                        android:id="@+id/reading_feed_title"
+                        android:layout_width="match_parent"
+                        android:layout_height="match_parent"
+                        android:layout_toRightOf="@id/reading_feed_icon"
+                        android:layout_centerVertical="true"
+                        android:ellipsize="end"
+                        android:lines="1"
+                        android:textSize="12sp"
+                        android:textStyle="bold" />
+
+                </RelativeLayout>
+
+                <View
+                    android:id="@+id/item_feed_border"
+                    android:layout_width="match_parent"
+                    android:layout_height="1dp"
+                    android:layout_below="@id/row_item_feed_header"
+                    android:background="@color/gray55"/>
+
+                <Button
+                    android:id="@+id/story_context_menu_button"
+                    android:layout_width="34dp"
+                    android:layout_height="34dp"
+                    android:layout_margin="5dp"
+                    android:background="@drawable/ic_menu_moreoverflow"
+                    android:contentDescription="@string/description_menu"
+                    android:layout_below="@id/item_feed_border"
+                    android:layout_alignParentRight="true" />
+
+                <TextView
+                    android:id="@+id/reading_item_title"
+                    android:layout_width="match_parent"
+                    android:layout_height="wrap_content"
+                    android:layout_marginRight="16dp"
+                    android:layout_marginTop="10dp"
+                    android:layout_marginLeft="16dp"
+                    android:layout_below="@id/item_feed_border"
+                    android:layout_toLeftOf="@id/story_context_menu_button"
+                    android:textSize="17sp" />
+
+                <TextView
+                    android:id="@+id/reading_item_date"
+                    android:layout_width="wrap_content"
+                    android:layout_height="wrap_content"
+                    android:layout_below="@id/reading_item_title"
+                    android:layout_alignLeft="@id/reading_item_title"
+                    android:layout_marginRight="12dp"
+                    android:layout_marginTop="8dp"
+                    style="?readingItemMetadata"
+                    android:textSize="12sp" />
+
+                <TextView
+                    android:id="@+id/reading_item_authors"
+                    android:layout_width="wrap_content"
+                    android:layout_height="wrap_content"
+                    android:layout_alignTop="@id/reading_item_date"
+                    android:layout_toRightOf="@id/reading_item_date"
+                    android:maxLines="1"
+                    android:minWidth="80dp"
+                    style="?readingItemMetadata"
+                    android:textSize="12sp"
+                    android:textStyle="bold"/>
+
+                <com.newsblur.view.FlowLayout
+                    android:id="@+id/reading_item_tags"
+                    android:layout_width="match_parent"
+                    android:layout_height="17dp"
+                    android:layout_alignLeft="@id/reading_item_title"
+                    android:layout_below="@id/reading_item_date"
+                    android:layout_marginBottom="8dp"
+                    android:layout_marginTop="8dp" />
+
+            </RelativeLayout>
 
             <View
                 android:layout_width="match_parent"
@@ -37,6 +133,34 @@
                 style="?divider"
                 android:visibility="gone" />
 
+            <TextView
+                android:id="@+id/reading_textloading"
+                android:layout_width="match_parent"
+                android:layout_height="wrap_content"
+                android:paddingTop="5dp"
+                android:paddingBottom="5dp"
+                style="?readingBackground"
+                android:gravity="center_horizontal"
+                android:text="@string/orig_text_loading"
+                android:textSize="16sp"
+                android:textStyle="italic"
+                android:visibility="gone"
+            />
+
+            <TextView
+                android:id="@+id/reading_textmodefailed"
+                android:layout_width="match_parent"
+                android:layout_height="wrap_content"
+                android:paddingTop="5dp"
+                android:paddingBottom="5dp"
+                style="?readingBackground"
+                android:gravity="center_horizontal"
+                android:text="@string/orig_text_failed"
+                android:textSize="16sp"
+                android:textStyle="italic"
+                android:visibility="gone"
+            />
+
             <com.newsblur.view.NewsblurWebview
                 android:id="@+id/reading_webview"
                 android:layout_width="match_parent"

clients/android/NewsBlur/res/layout/fragment_readingpager.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="utf-8"?>
+<FrameLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent"
+    >
+
+    <android.support.v4.view.ViewPager
+        android:id="@+id/reading_pager"
+        android:layout_width="match_parent"
+        android:layout_height="match_parent" 
+    />
+
+</FrameLayout>

clients/android/NewsBlur/res/layout/include_intel_row.xml

@@ -9,8 +9,8 @@
         android:layout_alignParentTop="true"
         android:layout_alignParentRight="true"
         android:layout_margin="4dp"
-        android:layout_width="28dp"
-        android:layout_height="28dp"
+        android:layout_width="32dp"
+        android:layout_height="32dp"
         android:background="@drawable/ic_clear_gray55"
     />
 
@@ -22,8 +22,8 @@
         android:layout_marginBottom="4dp"
         android:layout_marginRight="4dp"
         android:layout_marginLeft="6dp"
-        android:layout_width="28dp"
-        android:layout_height="28dp"
+        android:layout_width="32dp"
+        android:layout_height="32dp"
         android:background="@drawable/ic_dislike_gray55"
     />
 
@@ -32,8 +32,8 @@
         android:layout_alignParentTop="true"
         android:layout_toLeftOf="@id/intel_row_dislike"
         android:layout_margin="4dp"
-        android:layout_width="28dp"
-        android:layout_height="28dp"
+        android:layout_width="32dp"
+        android:layout_height="32dp"
         android:background="@drawable/ic_like_gray55"
     />
         

clients/android/NewsBlur/res/layout/include_reading_item_metadata.xml

@@ -1,96 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
-    android:layout_width="match_parent"
-    android:layout_height="wrap_content"
-    android:orientation="horizontal" >
-
-    <RelativeLayout
-        android:layout_width="match_parent"
-        android:layout_height="wrap_content"
-        android:paddingBottom="8dp"
-        style="?rowItemHeaderBackground">
-
-        <RelativeLayout
-            android:id="@+id/row_item_feed_header"
-            android:orientation="horizontal"
-            android:layout_height="wrap_content"
-            android:layout_width="match_parent"
-            android:minHeight="6dp"
-            android:paddingTop="4dp"
-            android:paddingBottom="4dp" >
-
-            <ImageView
-                android:id="@+id/reading_feed_icon"
-                android:layout_width="17dp"
-                android:layout_height="17dp"
-                android:layout_alignParentLeft="true"
-                android:layout_marginLeft="16dp"
-                android:layout_marginRight="5dp"
-                android:contentDescription="@string/description_row_folder_icon" />
-
-            <TextView
-                android:id="@+id/reading_feed_title"
-                android:layout_width="match_parent"
-                android:layout_height="match_parent"
-                android:layout_toRightOf="@id/reading_feed_icon"
-                android:layout_centerVertical="true"
-                android:ellipsize="end"
-                android:lines="1"
-                android:textColor="@color/newsblur_blue"
-                android:textSize="12sp"
-                android:textStyle="bold" />
-
-        </RelativeLayout>
-
-        <View
-            android:id="@+id/item_feed_border"
-            android:layout_width="match_parent"
-            android:layout_height="1dp"
-            android:layout_below="@id/row_item_feed_header"
-            android:background="@color/gray55"/>
-
-        <TextView
-            android:id="@+id/reading_item_title"
-            android:layout_width="match_parent"
-            android:layout_height="wrap_content"
-            android:layout_marginRight="16dp"
-            android:layout_marginTop="10dp"
-            android:layout_marginLeft="16dp"
-            android:layout_below="@id/item_feed_border"
-            android:textSize="17sp" />
-
-        <TextView
-            android:id="@+id/reading_item_date"
-            android:layout_width="wrap_content"
-            android:layout_height="wrap_content"
-            android:layout_below="@id/reading_item_title"
-            android:layout_alignLeft="@id/reading_item_title"
-            android:layout_marginRight="12dp"
-            android:layout_marginTop="8dp"
-            style="?readingItemMetadata"
-            android:textSize="12sp" />
-
-        <TextView
-            android:id="@+id/reading_item_authors"
-            android:layout_width="wrap_content"
-            android:layout_height="wrap_content"
-            android:layout_alignTop="@id/reading_item_date"
-            android:layout_toRightOf="@id/reading_item_date"
-            android:maxLines="1"
-            android:minWidth="80dp"
-            style="?readingItemMetadata"
-            android:textSize="12sp"
-            android:textStyle="bold"/>
-
-        <com.newsblur.view.FlowLayout
-            android:id="@+id/reading_item_tags"
-            android:layout_width="match_parent"
-            android:layout_height="17dp"
-            android:layout_alignLeft="@id/reading_item_title"
-            android:layout_below="@id/reading_item_date"
-            android:layout_marginBottom="8dp"
-            android:layout_marginTop="8dp" />
-
-    </RelativeLayout>
-
-</LinearLayout>

clients/android/NewsBlur/res/layout/newsblur_widget.xml

@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="utf-8"?>
+<FrameLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:tools="http://schemas.android.com/tools"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent">
+
+    <LinearLayout
+        android:background="@color/white"
+        android:orientation="vertical"
+        android:layout_width="match_parent"
+        android:layout_height="match_parent">
+
+        <TextView
+            android:id="@+id/txt_feed_name"
+            style="@style/rowItemHeaderBackground"
+            tools:text="Coding Horror"
+            android:paddingTop="9dp"
+            android:paddingBottom="9dp"
+            android:paddingStart="8dp"
+            android:paddingEnd="8dp"
+            android:shadowDy="1"
+            android:textStyle="bold"
+            android:lines="1"
+            android:layout_width="match_parent"
+            android:layout_height="wrap_content" />
+        <ListView
+            tools:listitem="@layout/newsblur_widget_item"
+            android:id="@+id/widget_list"
+            android:layout_width="match_parent"
+            android:layout_height="match_parent" />
+    </LinearLayout>
+
+<!--    Note that empty views must be siblings of the collection view
+        for which the empty view represents empty state.-->
+
+
+</FrameLayout>

clients/android/NewsBlur/res/layout/newsblur_widget_item.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="utf-8"?>
+<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:tools="http://schemas.android.com/tools"
+    android:id="@+id/newsblur_widget_item"
+    android:orientation="horizontal"
+    android:gravity="center_vertical"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent">
+    <TextView
+        android:id="@+id/widget_item_title"
+        android:paddingBottom="7dp"
+        android:paddingTop="7dp"
+        android:paddingStart="8dp"
+        android:paddingEnd="8dp"
+        android:ellipsize="end"
+        tools:text="The Next CEO of Stackoverflow"
+        android:lines="1"
+        android:textSize="14sp"
+        android:textStyle="bold"
+        android:layout_weight="1"
+        android:layout_width="0dp"
+        android:layout_height="wrap_content"/>
+
+    <TextView
+        android:paddingEnd="8dp"
+        android:id="@+id/widget_item_time"
+        tools:text="30 min ago"
+        android:textSize="12sp"
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content" />
+</LinearLayout>

clients/android/NewsBlur/res/layout/row_all_shared_stories.xml

@@ -77,7 +77,7 @@
         android:layout_toRightOf="@id/row_everything_icon"
         android:paddingBottom="9dp"
         android:paddingTop="9dp"
-        android:text="@string/all_shared_stories"
+        android:text="@string/all_shared_stories_row_title"
         android:textStyle="bold" />
 
     <View

clients/android/NewsBlur/res/layout/row_fleuron.xml

@@ -2,11 +2,12 @@
 <LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
     android:layout_width="match_parent"
     android:layout_height="wrap_content"
-    android:orientation="horizontal"
+    android:orientation="vertical"
     android:gravity="center" 
 >
 
     <ImageView
+        android:id="@+id/fleuron"
         android:src="@drawable/fleuron"
         android:layout_height="32dp"
         android:layout_width="wrap_content"

clients/android/NewsBlur/res/layout/row_global_shared_stories.xml

@@ -23,7 +23,7 @@
         android:layout_toRightOf="@id/row_everything_icon"
         android:paddingBottom="9dp"
         android:paddingTop="9dp"
-        android:text="@string/global_shared_stories"
+        android:text="@string/global_shared_stories_row_title"
         android:textStyle="bold" />
 
     <View

clients/android/NewsBlur/res/layout/row_story.xml

@@ -1,172 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
-    android:layout_width="match_parent"
-    android:layout_height="wrap_content"
-    style="?selectorStoryBackground" >
-
-<!--
-    This RelativeLayout really should be the top-most parent with a full width and wraped height. However,
-    due to this being used in a scrollable ListView, the height of the favicon_borderbar would never be
-    set, since the height of each row is calculated very lazily.  Wrapping the whole thing in an otherwise
-    useless LinearLayout forces the heights to be calculated correctly every time.  If the lazy layout
-    bug in scrolling ListViews can ever be fixed, the extra layout can be removed.
--->
-<RelativeLayout
-    android:layout_width="match_parent"
-    android:layout_height="match_parent" >
-
-    <View
-        android:id="@+id/row_item_favicon_borderbar_1"
-        android:layout_width="5dp"
-        android:layout_height="match_parent" />
-
-    <View
-        android:id="@+id/row_item_favicon_borderbar_2"
-        android:layout_width="5dp"
-        android:layout_height="match_parent"
-        android:layout_toRightOf="@id/row_item_favicon_borderbar_1" />
-
-    <!--
-        The next item to the right is actually the intel dot, but it has vertical bounds set relative to
-        the story title height, so it cannot be declared until after the title row and associated bits.
-        To prevent a cyclic layout dependency, we hard-code the left margin of everything to the right
-        of the dot by assuming the width of it. In fact, much of the remaining layout is done from the
-        right to the left.
-    -->
-
-    <ImageView
-        android:id="@+id/row_item_thumbnail"
-        android:layout_width="90dp"
-        android:layout_height="90dp"
-        android:layout_marginTop="1dp"
-        android:layout_marginBottom="1dp"
-        android:layout_alignParentRight="true"
-        android:visibility="gone" />
-
-    <ImageView
-        android:id="@+id/row_item_feedicon"
-        android:layout_width="18dp"
-        android:layout_height="18dp"
-        android:layout_marginTop="4dp"
-        android:layout_marginLeft="33dp"
-        android:layout_alignParentLeft="true" />
-
-    <TextView
-        android:id="@+id/row_item_feedtitle"
-        android:layout_width="match_parent"
-        android:layout_height="wrap_content"
-        android:layout_marginLeft="6dp"
-        android:layout_marginTop="4dp"
-        android:layout_toRightOf="@id/row_item_feedicon"
-        android:layout_toLeftOf="@id/row_item_thumbnail"
-        android:ellipsize="end"
-        android:singleLine="true"
-        style="?storyFeedTitleText" />
-
-    <ImageView
-        android:id="@+id/row_item_saved_icon"
-        android:src="@drawable/clock"
-        android:layout_width="20dp"
-        android:layout_height="20dp"
-        android:layout_below="@id/row_item_feedtitle"
-        android:layout_toLeftOf="@id/row_item_thumbnail"
-        android:layout_alignWithParentIfMissing="true"
-        android:layout_marginTop="2dp"
-        android:layout_marginLeft="2dp"
-        android:layout_marginRight="8dp"
-        android:visibility="gone" />
-
-    <ImageView
-        android:id="@+id/row_item_shared_icon"
-        android:src="@drawable/ic_shared"
-        android:layout_width="20dp"
-        android:layout_height="20dp"
-        android:layout_below="@id/row_item_feedtitle"
-        android:layout_toLeftOf="@id/row_item_saved_icon"
-        android:layout_alignWithParentIfMissing="true"
-        android:layout_marginTop="2dp"
-        android:layout_marginLeft="2dp"
-        android:layout_marginRight="8dp"
-        android:visibility="gone" />
-
-    <TextView
-        android:id="@+id/row_item_title"
-        android:layout_width="match_parent"
-        android:layout_height="wrap_content"
-        android:layout_toLeftOf="@id/row_item_shared_icon"
-        android:layout_below="@id/row_item_feedicon"
-        android:layout_marginLeft="33dp"
-        android:paddingTop="6dp"
-        android:paddingBottom="4dp"
-        android:paddingRight="4dp"
-        android:maxLines="2"
-        android:ellipsize="end" />
-
-    <TextView
-        android:id="@+id/row_item_content"
-        android:layout_width="wrap_content"
-        android:layout_height="wrap_content"
-        android:layout_toLeftOf="@id/row_item_thumbnail"
-        android:layout_below="@id/row_item_title"
-        android:layout_marginLeft="33dp"
-        android:paddingBottom="4dp"
-        android:paddingRight="4dp"
-        android:maxLines="2"
-        android:ellipsize="end"
-        style="?storySnippetText" />
-
-    <TextView
-        android:id="@+id/row_item_date"
-        android:layout_width="wrap_content"
-        android:layout_height="wrap_content"
-        android:layout_toLeftOf="@id/row_item_thumbnail"
-        android:layout_alignWithParentIfMissing="true"
-        android:layout_below="@id/row_item_content"
-        android:paddingBottom="2dp"
-        android:paddingRight="8dp"
-        style="?storyDateText" />
-
-    <TextView
-        android:id="@+id/row_item_author"
-        android:layout_width="wrap_content"
-        android:layout_height="wrap_content"
-        android:layout_below="@id/row_item_content"
-        android:layout_alignParentLeft="true"
-        android:layout_marginLeft="33dp"
-        android:layout_toLeftOf="@id/row_item_date"
-        android:paddingBottom="2dp"
-        android:ellipsize="end"
-        android:singleLine="true"
-        android:textColor="@color/story_author_text" />
-
-    <RelativeLayout
-        android:id="@+id/row_item_inteldot_container"
-        android:layout_width="wrap_content"
-        android:layout_height="match_parent"
-        android:layout_alignTop="@id/row_item_title"
-        android:layout_alignBottom="@id/row_item_title"
-        android:layout_toRightOf="@id/row_item_favicon_borderbar_2" >
-        
-        <ImageView
-            android:id="@+id/row_item_inteldot"
-            android:layout_width="9dp"
-            android:layout_height="9dp"
-            android:layout_marginLeft="6dp"
-            android:layout_centerVertical="true" />
-
-    </RelativeLayout>
-
-    <View 
-        android:layout_height="0.5dp"
-        android:layout_width="match_parent"
-        style="?rowBorderTop"
-        android:layout_alignParentTop="true" />
-    <View 
-        android:layout_height="0.5dp"
-        android:layout_width="match_parent"
-        style="?rowBorderBottom"
-        android:layout_alignParentBottom="true" />
-
-</RelativeLayout>
-
-</LinearLayout>

clients/android/NewsBlur/res/layout/view_footer_tile.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="utf-8"?>
+<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    android:layout_width="match_parent"
+    android:layout_height="wrap_content"
+    >
+
+    <FrameLayout
+        android:id="@+id/footer_view_inner"
+        android:layout_width="match_parent"
+        android:layout_height="wrap_content"
+    />
+
+</RelativeLayout>

clients/android/NewsBlur/res/layout/view_story_row.xml

@@ -0,0 +1,173 @@
+<?xml version="1.0" encoding="utf-8"?>
+<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    android:layout_width="match_parent"
+    android:layout_height="wrap_content"
+    style="?selectorStoryBackground" >
+
+<!--
+    This RelativeLayout really should be the top-most parent with a full width and wraped height. However,
+    due to this being used in a scrollable ListView, the height of the favicon_borderbar would never be
+    set, since the height of each row is calculated very lazily.  Wrapping the whole thing in an otherwise
+    useless LinearLayout forces the heights to be calculated correctly every time.  If the lazy layout
+    bug in scrolling ListViews can ever be fixed, the extra layout can be removed.
+-->
+<RelativeLayout
+    android:layout_width="match_parent"
+    android:layout_height="match_parent" >
+
+    <View
+        android:id="@+id/story_item_favicon_borderbar_1"
+        android:layout_width="5dp"
+        android:layout_height="match_parent" />
+
+    <View
+        android:id="@+id/story_item_favicon_borderbar_2"
+        android:layout_width="5dp"
+        android:layout_height="match_parent"
+        android:layout_toRightOf="@id/story_item_favicon_borderbar_1" />
+
+    <!--
+        The next item to the right is actually the intel dot, but it has vertical bounds set relative to
+        the story title height, so it cannot be declared until after the title row and associated bits.
+        To prevent a cyclic layout dependency, we hard-code the left margin of everything to the right
+        of the dot by assuming the width of it. In fact, much of the remaining layout is done from the
+        right to the left.
+    -->
+
+    <ImageView
+        android:id="@+id/story_item_thumbnail"
+        android:layout_width="90dp"
+        android:layout_height="90dp"
+        android:layout_marginTop="1dp"
+        android:layout_marginBottom="1dp"
+        android:layout_alignParentRight="true"
+        android:layout_centerVertical="true"
+        android:visibility="gone" />
+
+    <ImageView
+        android:id="@+id/story_item_feedicon"
+        android:layout_width="18dp"
+        android:layout_height="18dp"
+        android:layout_marginTop="4dp"
+        android:layout_marginLeft="33dp"
+        android:layout_alignParentLeft="true" />
+
+    <TextView
+        android:id="@+id/story_item_feedtitle"
+        android:layout_width="match_parent"
+        android:layout_height="wrap_content"
+        android:layout_marginLeft="6dp"
+        android:layout_marginTop="4dp"
+        android:layout_toRightOf="@id/story_item_feedicon"
+        android:layout_toLeftOf="@id/story_item_thumbnail"
+        android:ellipsize="end"
+        android:singleLine="true"
+        style="?storyFeedTitleText" />
+
+    <ImageView
+        android:id="@+id/story_item_saved_icon"
+        android:src="@drawable/clock"
+        android:layout_width="20dp"
+        android:layout_height="20dp"
+        android:layout_below="@id/story_item_feedtitle"
+        android:layout_toLeftOf="@id/story_item_thumbnail"
+        android:layout_alignWithParentIfMissing="true"
+        android:layout_marginTop="2dp"
+        android:layout_marginLeft="2dp"
+        android:layout_marginRight="8dp"
+        android:visibility="gone" />
+
+    <ImageView
+        android:id="@+id/story_item_shared_icon"
+        android:src="@drawable/ic_shared"
+        android:layout_width="20dp"
+        android:layout_height="20dp"
+        android:layout_below="@id/story_item_feedtitle"
+        android:layout_toLeftOf="@id/story_item_saved_icon"
+        android:layout_alignWithParentIfMissing="true"
+        android:layout_marginTop="2dp"
+        android:layout_marginLeft="2dp"
+        android:layout_marginRight="8dp"
+        android:visibility="gone" />
+
+    <TextView
+        android:id="@+id/story_item_title"
+        android:layout_width="match_parent"
+        android:layout_height="wrap_content"
+        android:layout_toLeftOf="@id/story_item_shared_icon"
+        android:layout_below="@id/story_item_feedicon"
+        android:layout_marginLeft="33dp"
+        android:paddingTop="6dp"
+        android:paddingBottom="4dp"
+        android:paddingRight="4dp"
+        android:maxLines="2"
+        android:ellipsize="end" />
+
+    <TextView
+        android:id="@+id/story_item_content"
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:layout_toLeftOf="@id/story_item_thumbnail"
+        android:layout_below="@id/story_item_title"
+        android:layout_marginLeft="33dp"
+        android:paddingBottom="4dp"
+        android:paddingRight="4dp"
+        android:maxLines="2"
+        android:ellipsize="end"
+        style="?storySnippetText" />
+
+    <TextView
+        android:id="@+id/story_item_date"
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:layout_toLeftOf="@id/story_item_thumbnail"
+        android:layout_alignWithParentIfMissing="true"
+        android:layout_below="@id/story_item_content"
+        android:paddingBottom="2dp"
+        android:paddingRight="8dp"
+        style="?storyDateText" />
+
+    <TextView
+        android:id="@+id/story_item_author"
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:layout_below="@id/story_item_content"
+        android:layout_alignParentLeft="true"
+        android:layout_marginLeft="33dp"
+        android:layout_toLeftOf="@id/story_item_date"
+        android:paddingBottom="2dp"
+        android:ellipsize="end"
+        android:singleLine="true"
+        android:textColor="@color/story_author_text" />
+
+    <RelativeLayout
+        android:id="@+id/story_item_inteldot_container"
+        android:layout_width="wrap_content"
+        android:layout_height="match_parent"
+        android:layout_alignTop="@id/story_item_title"
+        android:layout_alignBottom="@id/story_item_title"
+        android:layout_toRightOf="@id/story_item_favicon_borderbar_2" >
+        
+        <ImageView
+            android:id="@+id/story_item_inteldot"
+            android:layout_width="9dp"
+            android:layout_height="9dp"
+            android:layout_marginLeft="6dp"
+            android:layout_centerVertical="true" />
+
+    </RelativeLayout>
+
+    <View 
+        android:layout_height="0.5dp"
+        android:layout_width="match_parent"
+        style="?rowBorderTop"
+        android:layout_alignParentTop="true" />
+    <View 
+        android:layout_height="0.5dp"
+        android:layout_width="match_parent"
+        style="?rowBorderBottom"
+        android:layout_alignParentBottom="true" />
+
+</RelativeLayout>
+
+</LinearLayout>

clients/android/NewsBlur/res/layout/view_story_tile.xml

@@ -0,0 +1,122 @@
+<?xml version="1.0" encoding="utf-8"?>
+<com.newsblur.view.SquaredRelativeLayout
+    xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:newsblur="http://schemas.android.com/apk/res-auto"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent"
+    newsblur:addedHeight="40"
+    android:background="@drawable/item_border"
+    >
+
+    <View
+        android:id="@+id/story_item_favicon_borderbar_1"
+        android:layout_width="4dp"
+        android:layout_height="match_parent" 
+        />
+
+    <View
+        android:id="@+id/story_item_favicon_borderbar_2"
+        android:layout_width="4dp"
+        android:layout_height="match_parent"
+        android:layout_toRightOf="@id/story_item_favicon_borderbar_1" 
+        />
+
+    <TextView
+        android:id="@+id/story_item_date"
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:layout_alignParentRight="true"
+        android:layout_alignParentBottom="true"
+        android:layout_marginBottom="2dp"
+        android:layout_marginRight="4dp"
+        android:singleLine="true"
+        style="?storyDateText" 
+        />
+
+    <ImageView
+        android:id="@+id/story_item_feedicon"
+        android:layout_width="15dp"
+        android:layout_height="15dp"
+        android:layout_toRightOf="@id/story_item_favicon_borderbar_2"
+        android:layout_above="@id/story_item_date"
+        android:layout_marginTop="5dp"
+        android:layout_marginLeft="18dp"
+        android:layout_marginBottom="2dp"
+        />
+
+    <TextView
+        android:id="@+id/story_item_feedtitle"
+        android:layout_width="match_parent"
+        android:layout_height="wrap_content"
+        android:layout_alignParentRight="true"
+        android:layout_toRightOf="@id/story_item_feedicon"
+        android:layout_above="@id/story_item_date"
+        android:layout_marginLeft="6dp"
+        android:layout_marginRight="2dp"
+        android:layout_marginBottom="2dp"
+        android:ellipsize="end"
+        android:singleLine="true"
+        style="?storyFeedTitleText"
+        />
+
+    <ImageView
+        android:id="@+id/story_item_saved_icon"
+        android:src="@drawable/clock"
+        android:layout_width="20dp"
+        android:layout_height="20dp"
+        android:layout_above="@id/story_item_feedtitle"
+        android:layout_alignParentRight="true"
+        android:layout_marginTop="2dp"
+        android:layout_marginLeft="2dp"
+        android:layout_marginRight="2dp"
+        android:visibility="gone" 
+        />
+
+    <ImageView
+        android:id="@+id/story_item_shared_icon"
+        android:src="@drawable/ic_shared"
+        android:layout_width="20dp"
+        android:layout_height="20dp"
+        android:layout_above="@id/story_item_feedtitle"
+        android:layout_toLeftOf="@id/story_item_saved_icon"
+        android:layout_alignWithParentIfMissing="true"
+        android:layout_marginTop="2dp"
+        android:layout_marginLeft="2dp"
+        android:layout_marginRight="2dp"
+        android:visibility="gone" 
+        />
+
+    <TextView
+        android:id="@+id/story_item_title"
+        android:layout_width="match_parent"
+        android:layout_height="wrap_content"
+        android:layout_above="@id/story_item_feedtitle"
+        android:layout_toLeftOf="@id/story_item_shared_icon"
+        android:layout_marginLeft="26dp"
+        android:layout_marginRight="2dp"
+        android:layout_marginBottom="2dp"
+        android:maxLines="2"
+        android:ellipsize="end" 
+        />
+
+    <ImageView
+        android:id="@+id/story_item_inteldot"
+        android:layout_width="9dp"
+        android:layout_height="match_parent"
+        android:layout_alignTop="@id/story_item_title"
+        android:layout_alignBottom="@id/story_item_title"
+        android:layout_toRightOf="@id/story_item_favicon_borderbar_2"
+        android:layout_marginLeft="4dp"
+        />
+
+    <ImageView
+        android:id="@+id/story_item_thumbnail"
+        android:layout_width="match_parent"
+        android:layout_height="match_parent"
+        android:layout_above="@id/story_item_title"
+        android:layout_marginBottom="2dp"
+        android:layout_marginLeft="10dp"
+        android:scaleType="centerCrop"
+        />
+
+</com.newsblur.view.SquaredRelativeLayout>

clients/android/NewsBlur/res/menu/allsocialstories_itemslist.xml

@@ -1,21 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<menu xmlns:android="http://schemas.android.com/apk/res/android" >
-    <item android:id="@+id/menu_read_filter"
-          android:title="@string/menu_read_filter"
-          android:showAsAction="never" />
-    <item
-        android:id="@+id/menu_textsize"
-        android:showAsAction="never"
-        android:title="@string/menu_textsize"/>
-    <item android:id="@+id/menu_theme"
-          android:title="@string/menu_theme_choose" >
-        <menu>
-            <group android:checkableBehavior="single">
-                <item android:id="@+id/menu_theme_light"
-                      android:title="@string/light" />
-                <item android:id="@+id/menu_theme_dark"
-                      android:title="@string/dark" />
-            </group>
-        </menu>
-    </item>
-</menu>

clients/android/NewsBlur/res/menu/allstories_itemslist.xml

@@ -1,29 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<menu xmlns:android="http://schemas.android.com/apk/res/android" >
-    <item android:id="@+id/menu_mark_all_as_read"
-          android:title="@string/menu_mark_all_as_read"
-          android:showAsAction="ifRoom" android:icon="@drawable/ic_menu_markread_gray55" />
-    <item android:id="@+id/menu_story_order"
-          android:title="@string/menu_story_order"
-          android:showAsAction="never" />
-    <item android:id="@+id/menu_read_filter"
-          android:title="@string/menu_read_filter"
-          android:showAsAction="never" />
-    <item android:id="@+id/menu_textsize"
-          android:showAsAction="never"
-          android:title="@string/menu_textsize"/>
-    <item android:id="@+id/menu_search_stories"
-          android:title="@string/menu_search_stories"
-          android:showAsAction="ifRoom" android:icon="@drawable/ic_menu_search_gray55" />
-    <item android:id="@+id/menu_theme"
-          android:title="@string/menu_theme_choose" >
-        <menu>
-            <group android:checkableBehavior="single">
-                <item android:id="@+id/menu_theme_light"
-                      android:title="@string/light" />
-                <item android:id="@+id/menu_theme_dark"
-                      android:title="@string/dark" />
-            </group>
-        </menu>
-    </item>
-</menu>

clients/android/NewsBlur/res/menu/context_feed.xml

@@ -13,6 +13,9 @@
     <item android:id="@+id/menu_choose_folders"
           android:title="@string/menu_choose_folders" />
 
+    <item android:id="@+id/menu_rename_feed"
+          android:title="@string/menu_rename_feed" />
+
     <item android:id="@+id/menu_notifications"
           android:title="@string/menu_notifications_choose" >
         <menu>

clients/android/NewsBlur/res/menu/context_story_newest.xml

@@ -25,6 +25,9 @@
     <item android:id="@+id/menu_unsave_story"
         android:title="@string/menu_unsave_story" />
 
+    <item android:id="@+id/menu_go_to_feed"
+        android:title="@string/go_to_feed"/>
+
     <item android:id="@+id/menu_intel"
         android:title="@string/menu_intel" />
 

clients/android/NewsBlur/res/menu/context_story_oldest.xml

@@ -25,4 +25,7 @@
     <item android:id="@+id/menu_unsave_story"
         android:title="@string/menu_unsave_story" />
 
+    <item android:id="@+id/menu_go_to_feed"
+        android:title="@string/go_to_feed"/>
+
 </menu>

clients/android/NewsBlur/res/menu/feed_itemslist.xml

@@ -1,52 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<menu xmlns:android="http://schemas.android.com/apk/res/android" >
-    <item android:id="@+id/menu_mark_all_as_read"
-          android:title="@string/menu_mark_all_as_read"
-          android:showAsAction="ifRoom" android:icon="@drawable/ic_menu_markread_gray55"/>
-    <item android:id="@+id/menu_delete_feed"
-          android:title="@string/menu_delete_feed"
-          android:showAsAction="never" />
-    <item android:id="@+id/menu_story_order"
-          android:title="@string/menu_story_order"
-          android:showAsAction="never" />
-    <item android:id="@+id/menu_read_filter"
-          android:title="@string/menu_read_filter"
-          android:showAsAction="never" />
-    <item android:id="@+id/menu_textsize"
-          android:showAsAction="never"
-          android:title="@string/menu_textsize"/>
-    <item android:id="@+id/menu_notifications"
-          android:title="@string/menu_notifications_choose" >
-        <menu>
-            <group android:checkableBehavior="single">
-                <item android:id="@+id/menu_notifications_disable"
-                      android:title="@string/menu_notifications_disable"
-                      android:checkable="true" />
-                <item android:id="@+id/menu_notifications_focus"
-                      android:title="@string/menu_notifications_focus"
-                      android:checkable="true" />
-                <item android:id="@+id/menu_notifications_unread"
-                      android:title="@string/menu_notifications_unread"
-                      android:checkable="true" />
-            </group>
-        </menu>
-    </item>
-    <item android:id="@+id/menu_instafetch_feed"
-          android:title="@string/menu_instafetch_feed" />
-    <item android:id="@+id/menu_intel"
-          android:title="@string/menu_intel" />
-    <item android:id="@+id/menu_search_stories"
-          android:title="@string/menu_search_stories"
-          android:showAsAction="ifRoom" android:icon="@drawable/ic_menu_search_gray55" />
-    <item android:id="@+id/menu_theme"
-          android:title="@string/menu_theme_choose" >
-        <menu>
-            <group android:checkableBehavior="single">
-                <item android:id="@+id/menu_theme_light"
-                      android:title="@string/light" />
-                <item android:id="@+id/menu_theme_dark"
-                      android:title="@string/dark" />
-            </group>
-        </menu>
-    </item>
-</menu>

clients/android/NewsBlur/res/menu/infrequent_itemslist.xml

@@ -1,29 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<menu xmlns:android="http://schemas.android.com/apk/res/android" >
-    <item android:id="@+id/menu_infrequent_cutoff"
-          android:title="@string/menu_infrequent_cutoff"
-          android:showAsAction="never" />
-    <item android:id="@+id/menu_story_order"
-          android:title="@string/menu_story_order"
-          android:showAsAction="never" />
-    <item android:id="@+id/menu_read_filter"
-          android:title="@string/menu_read_filter"
-          android:showAsAction="never" />
-    <item android:id="@+id/menu_textsize"
-          android:showAsAction="never"
-          android:title="@string/menu_textsize"/>
-    <item android:id="@+id/menu_search_stories"
-          android:title="@string/menu_search_stories"
-          android:showAsAction="ifRoom" android:icon="@drawable/ic_menu_search_gray55" />
-    <item android:id="@+id/menu_theme"
-          android:title="@string/menu_theme_choose" >
-        <menu>
-            <group android:checkableBehavior="single">
-                <item android:id="@+id/menu_theme_light"
-                      android:title="@string/light" />
-                <item android:id="@+id/menu_theme_dark"
-                      android:title="@string/dark" />
-            </group>
-        </menu>
-    </item>
-</menu>

clients/android/NewsBlur/res/menu/itemslist.xml

@@ -3,18 +3,33 @@
     <item android:id="@+id/menu_mark_all_as_read"
           android:title="@string/menu_mark_all_as_read"
           android:showAsAction="ifRoom" android:icon="@drawable/ic_menu_markread_gray55" />
+    <item android:id="@+id/menu_search_stories"
+          android:title="@string/menu_search_stories"
+          android:showAsAction="ifRoom" android:icon="@drawable/ic_menu_search_gray55" />
     <item android:id="@+id/menu_story_order"
           android:title="@string/menu_story_order"
           android:showAsAction="never" />
     <item android:id="@+id/menu_read_filter"
           android:title="@string/menu_read_filter"
           android:showAsAction="never" />
+    <item android:id="@+id/menu_story_list_style"
+          android:title="@string/menu_story_list_style_choose" >
+        <menu>
+            <group android:checkableBehavior="single">
+                <item android:id="@+id/menu_list_style_list"
+                      android:title="@string/list_style_list" />
+                <item android:id="@+id/menu_list_style_grid_c"
+                      android:title="@string/list_style_grid_c" />
+                <item android:id="@+id/menu_list_style_grid_m"
+                      android:title="@string/list_style_grid_m" />
+                <item android:id="@+id/menu_list_style_grid_f"
+                      android:title="@string/list_style_grid_f" />
+            </group>
+        </menu>
+    </item>
     <item android:id="@+id/menu_textsize"
           android:showAsAction="never"
           android:title="@string/menu_textsize"/>
-    <item android:id="@+id/menu_search_stories"
-          android:title="@string/menu_search_stories"
-          android:showAsAction="ifRoom" android:icon="@drawable/ic_menu_search_gray55" />
     <item android:id="@+id/menu_theme"
           android:title="@string/menu_theme_choose" >
         <menu>
@@ -23,7 +38,38 @@
                       android:title="@string/light" />
                 <item android:id="@+id/menu_theme_dark"
                       android:title="@string/dark" />
+                <item android:id="@+id/menu_theme_black"
+                      android:title="@string/black" />
             </group>
         </menu>
     </item>
+    <item android:id="@+id/menu_intel"
+          android:title="@string/menu_intel" />
+    <item android:id="@+id/menu_notifications"
+          android:title="@string/menu_notifications_choose" >
+        <menu>
+            <group android:checkableBehavior="single">
+                <item android:id="@+id/menu_notifications_disable"
+                      android:title="@string/menu_notifications_disable"
+                      android:checkable="true" />
+                <item android:id="@+id/menu_notifications_focus"
+                      android:title="@string/menu_notifications_focus"
+                      android:checkable="true" />
+                <item android:id="@+id/menu_notifications_unread"
+                      android:title="@string/menu_notifications_unread"
+                      android:checkable="true" />
+            </group>
+        </menu>
+    </item>
+    <item android:id="@+id/menu_delete_feed"
+          android:title="@string/menu_delete_feed"
+          android:showAsAction="never" />
+    <item android:id="@+id/menu_rename_feed"
+          android:title="@string/menu_rename_feed"
+          android:showAsAction="never" />
+    <item android:id="@+id/menu_instafetch_feed"
+          android:title="@string/menu_instafetch_feed" />
+    <item android:id="@+id/menu_infrequent_cutoff"
+          android:title="@string/menu_infrequent_cutoff"
+          android:showAsAction="never" />
 </menu>

clients/android/NewsBlur/res/menu/main.xml

@@ -52,6 +52,8 @@
                       android:title="@string/light" />
                 <item android:id="@+id/menu_theme_dark"
                       android:title="@string/dark" />
+                <item android:id="@+id/menu_theme_black"
+                      android:title="@string/black" />
             </group>
         </menu>
     </item>

clients/android/NewsBlur/res/menu/reading.xml

@@ -5,64 +5,6 @@
         android:id="@+id/menu_reading_fullscreen"
         android:icon="@drawable/ic_menu_fullscreen_gray55"
         android:showAsAction="always"
-        android:title="@string/menu_fullscreen"
-        android:visible="false"/>
-
-    <item
-        android:id="@+id/menu_reading_sharenewsblur"
-        android:showAsAction="never"
-        android:title="@string/menu_sharenewsblur"/>
-
-    <item
-        android:id="@+id/menu_reading_original"
-        android:showAsAction="never"
-        android:title="@string/menu_original"/>
-
-    <item
-        android:id="@+id/menu_send_story"
-        android:showAsAction="never"
-        android:title="@string/menu_send_story"/>
-
-    <item
-        android:id="@+id/menu_send_story_full"
-        android:showAsAction="never"
-        android:title="@string/menu_send_story_full"/>
-
-    <item
-        android:id="@+id/menu_textsize"
-        android:showAsAction="never"
-        android:title="@string/menu_textsize"/>
-
-    <item
-        android:id="@+id/menu_font"
-        android:showAsAction="never"
-        android:title="@string/menu_font"/>
-        
-    <item
-        android:id="@+id/menu_reading_save"
-        android:showAsAction="never"
-        android:title="@string/menu_save_story"/>
-
-    <item
-        android:id="@+id/menu_reading_markunread"
-        android:showAsAction="never"
-        android:title="@string/menu_mark_unread"/>
-
-    <item
-        android:id="@+id/menu_intel"
-        android:showAsAction="never"
-        android:title="@string/menu_intel"/>
-
-    <item android:id="@+id/menu_theme"
-          android:title="@string/menu_theme_choose" >
-        <menu>
-            <group android:checkableBehavior="single">
-                <item android:id="@+id/menu_theme_light"
-                      android:title="@string/light" />
-                <item android:id="@+id/menu_theme_dark"
-                      android:title="@string/dark" />
-            </group>
-        </menu>
-    </item>
+        android:title="@string/menu_fullscreen"/>
 
 </menu>

clients/android/NewsBlur/res/menu/readstories_itemslist.xml

@@ -1,18 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<menu xmlns:android="http://schemas.android.com/apk/res/android" >
-    <item
-        android:id="@+id/menu_textsize"
-        android:showAsAction="never"
-        android:title="@string/menu_textsize"/>
-    <item android:id="@+id/menu_theme"
-          android:title="@string/menu_theme_choose" >
-        <menu>
-            <group android:checkableBehavior="single">
-                <item android:id="@+id/menu_theme_light"
-                      android:title="@string/light" />
-                <item android:id="@+id/menu_theme_dark"
-                      android:title="@string/dark" />
-            </group>
-        </menu>
-    </item>
-</menu>

clients/android/NewsBlur/res/menu/savedstories_itemslist.xml

@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<menu xmlns:android="http://schemas.android.com/apk/res/android" >
-    <item android:id="@+id/menu_story_order"
-          android:title="@string/menu_story_order"
-          android:showAsAction="never" />
-    <item android:id="@+id/menu_textsize"
-          android:showAsAction="never"
-          android:title="@string/menu_textsize"/>
-    <item android:id="@+id/menu_search_stories"
-          android:title="@string/menu_search_stories"
-          android:showAsAction="ifRoom" android:icon="@drawable/ic_menu_search_gray55" />
-    <item android:id="@+id/menu_theme"
-          android:title="@string/menu_theme_choose" >
-        <menu>
-            <group android:checkableBehavior="single">
-                <item android:id="@+id/menu_theme_light"
-                      android:title="@string/light" />
-                <item android:id="@+id/menu_theme_dark"
-                      android:title="@string/dark" />
-            </group>
-        </menu>
-    </item>
-</menu>

clients/android/NewsBlur/res/menu/story_context.xml

@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="utf-8"?>
+<menu xmlns:android="http://schemas.android.com/apk/res/android" >
+
+    <item
+        android:id="@+id/menu_reading_sharenewsblur"
+        android:showAsAction="never"
+        android:title="@string/menu_sharenewsblur"/>
+
+    <item
+        android:id="@+id/menu_reading_original"
+        android:showAsAction="never"
+        android:title="@string/menu_original"/>
+
+    <item
+        android:id="@+id/menu_send_story"
+        android:showAsAction="never"
+        android:title="@string/menu_send_story"/>
+
+    <item
+        android:id="@+id/menu_send_story_full"
+        android:showAsAction="never"
+        android:title="@string/menu_send_story_full"/>
+
+    <item
+        android:id="@+id/menu_textsize"
+        android:showAsAction="never"
+        android:title="@string/menu_textsize"/>
+
+    <item
+        android:id="@+id/menu_font"
+        android:showAsAction="never"
+        android:title="@string/menu_font"/>
+        
+    <item
+        android:id="@+id/menu_reading_save"
+        android:showAsAction="never"
+        android:title="@string/menu_save_story"/>
+
+    <item
+        android:id="@+id/menu_reading_markunread"
+        android:showAsAction="never"
+        android:title="@string/menu_mark_unread"/>
+
+    <item android:id="@+id/menu_go_to_feed"
+        android:title="@string/go_to_feed"/>
+    <item
+        android:id="@+id/menu_intel"
+        android:showAsAction="never"
+        android:title="@string/menu_intel"/>
+
+    <item android:id="@+id/menu_theme"
+          android:title="@string/menu_theme_choose" >
+        <menu>
+            <group android:checkableBehavior="single">
+                <item android:id="@+id/menu_theme_light"
+                      android:title="@string/light" />
+                <item android:id="@+id/menu_theme_dark"
+                      android:title="@string/dark" />
+                <item android:id="@+id/menu_theme_black"
+                      android:title="@string/black" />
+            </group>
+        </menu>
+    </item>
+
+</menu>

clients/android/NewsBlur/res/values/attrs.xml

@@ -1,7 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
 <resources>
-    <attr name="flow" format="string" />
-    <attr name="imageViewSize" format="integer" />
     <attr name="selectorFolderBackground" format="string" />
     <attr name="selectorFeedBackground" format="string" />
     <attr name="feedRowNeutCountText" format="string" />
@@ -41,8 +39,15 @@
     <attr name="selectorOverlayBackgroundText" format="string" />
     <attr name="muteicon" format="string" />
     
+    <attr name="flow" format="string" />
+    <attr name="imageViewSize" format="integer" />
     <declare-styleable name="FlowLayout">
         <attr name="flow" />
         <attr name="imageViewSize" />
     </declare-styleable>
+
+    <attr name="addedHeight" format="integer" />
+    <declare-styleable name="SquaredRelativeLayout">
+        <attr name="addedHeight" />
+    </declare-styleable>
 </resources>

clients/android/NewsBlur/res/values/colors.xml

@@ -37,6 +37,8 @@
     <color name="folder_background_start">#DDE0D7</color>
     <color name="dark_folder_background_end">@color/gray07</color>
     <color name="dark_folder_background_start">@color/gray13</color>
+    <color name="black_folder_background_end">@color/black</color>
+    <color name="black_folder_background_start">@color/gray07</color>
     <color name="folder_background_selected_end">#D9DBD4</color>
     <color name="folder_background_selected_start">#CDD0C7</color>
     <color name="dark_folder_background_selected_end">#4C4C4C</color>
@@ -45,6 +47,8 @@
     <color name="row_border_bottom">@color/gray80</color>
     <color name="dark_row_border_top">@color/gray20</color>
     <color name="dark_row_border_bottom">@color/gray13</color>
+    <color name="black_row_border_top">@color/gray10</color>
+    <color name="black_row_border_bottom">@color/gray07</color>
     <color name="feed_background_selected_end">#FFFFD2</color>
     <color name="feed_background_selected_start">#E3D0AE</color>
     <color name="dark_feed_background_selected_end">#4C4C4C</color>
@@ -55,7 +59,7 @@
     <color name="dark_story_background_selected">#4C4C4C</color>
 
     <color name="story_feed_title_text">#606060</color>
-    <color name="dark_story_feed_title_text">#F7F8F5</color>
+    <color name="dark_story_feed_title_text">@color/gray55</color>
     <color name="story_date_text">#424242</color>
     <color name="dark_story_date_text">#BDBDBD</color>
     <color name="story_content_text">#404040</color>
@@ -72,6 +76,8 @@
     <color name="item_header_background_end">#F3F4EF</color>
     <color name="dark_item_header_background_start">#505050</color>
     <color name="dark_item_header_background_end">#303030</color>
+    <color name="black_item_header_background_start">@color/gray07</color>
+    <color name="black_item_header_background_end">@color/black</color>
     <color name="item_header_border">#C2C5BE</color>
     <color name="dark_item_header_border">#42453E</color>
     <color name="item_background">#F4F4F4</color>

clients/android/NewsBlur/res/values/dimens.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+    <dimen name="thumbnails_small_size">50dp</dimen>
+    <dimen name="thumbnails_size">90dp</dimen>
+</resources>

clients/android/NewsBlur/res/values/strings.xml

@@ -19,7 +19,7 @@
     <string name="add_feed_message">Add \"%s\" to your feeds?</string>
     
     <string name="loading">Loading…</string>
-    <string name="orig_text_loading">Fetching text…</string>
+    <string name="orig_text_loading">Fetching story text…</string>
     
     <string name="follow_error">There was a problem following the user. Check your internet connection and try again.</string>
     <string name="unfollow_error">There was a problem unfollowing the user. Check your internet connection and try again.</string>
@@ -35,15 +35,21 @@
     <string name="description_menu">Menu</string>
     
     <string name="title_choose_folders">Choose Folders for Feed %s</string>
+    <string name="title_rename_feed">Rename Feed %s</string>
 	
 	<string name="all_stories_row_title">ALL STORIES</string>
+	<string name="all_stories_title">All Stories</string>
 	<string name="infrequent_row_title">INFREQUENT SITE STORIES</string>
-	<string name="all_shared_stories">ALL SHARED STORIES</string>
-    <string name="global_shared_stories">GLOBAL SHARED STORIES</string>
+	<string name="infrequent_title">Infrequent Site Stories</string>
+	<string name="all_shared_stories_row_title">ALL SHARED STORIES</string>
+	<string name="all_shared_stories_title">All Shared Stories</string>
+    <string name="global_shared_stories_row_title">GLOBAL SHARED STORIES</string>
+    <string name="global_shared_stories_title">Global Shared Stories</string>
 	<string name="read_stories_row_title">READ STORIES</string>
-	<string name="saved_stories_row_title">SAVED STORIES</string>
 	<string name="read_stories_title">Read Stories</string>
+	<string name="saved_stories_row_title">SAVED STORIES</string>
 	<string name="saved_stories_title">Saved Stories</string>
+
     <string name="top_level">TOP LEVEL</string>
 	
 	<string name="send_brief">%1$s %2$s</string>
@@ -55,6 +61,7 @@
     <string name="share_this_story">SHARE</string>
     <string name="unshare">DELETE SHARE</string>
     <string name="update_shared">UPDATE COMMENT</string>
+    <string name="feed_name_save">RENAME FEED</string>
 
 	<string name="save_this">SAVE</string>
 	<string name="unsave_this">REMOVE FROM SAVED</string>
@@ -65,7 +72,7 @@
     <string name="overlay_count_toast_1">1 unread story</string>
     <string name="overlay_text">TEXT</string>
     <string name="overlay_story">STORY</string>
-    <string name="text_mode_unavailable">Sorry, the story\'s text could not be extracted.</string>
+    <string name="orig_text_failed">Sorry, the story\'s text could not be extracted.</string>
 
     <string name="state_all">ALL</string>
     <string name="state_unread">UNREAD</string>
@@ -80,6 +87,7 @@
 	<string name="alert_dialog_ok">OKAY</string>
 	<string name="alert_dialog_cancel">CANCEL</string>
 	<string name="alert_dialog_done">DONE</string>
+	<string name="alert_dialog_openlink">OPEN LINK</string>
 	<string name="alert_dialog_openimage">OPEN IMAGE</string>
     <string name="dialog_folders_save">SAVE FOLDERS</string>
     <string name="dialog_story_intel_save">SAVE TRAINING</string>
@@ -149,9 +157,14 @@
     <string name="menu_unmute_folder">Unmute folder</string>
     <string name="menu_instafetch_feed">Insta-fetch stories</string>
     <string name="menu_infrequent_cutoff">Infrequent stories per month</string>
-    <string name="menu_intel">Intellegence trainer</string>
+    <string name="menu_intel">Intelligence trainer</string>
+    <string name="menu_rename_feed">Rename feed</string>
+    <string name="menu_story_list_style_choose">List style…</string>
+    <string name="list_style_list">List</string>
+    <string name="list_style_grid_f">Grid (fine)</string>
+    <string name="list_style_grid_m">Grid (medium)</string>
+    <string name="list_style_grid_c">Grid (coarse)</string>
     
-    <string name="toast_story_unread">Story marked as unread</string>
     <string name="toast_hold_to_select">Press and hold to select text</string>
 
     <string name="logout_warning">Are you sure you want to log out?</string>
@@ -231,6 +244,33 @@
     </string-array>
     <string name="default_network_select_value">NOMONONME</string>
 
+    <string name="menu_cache_age_select">Maximum Cache Age</string>
+    <string name="menu_cache_age_select_sum">Clean up cached story content after…</string>
+    <string name="menu_cache_age_select_opt_2d">2 days (reduce storage use)</string>
+    <string name="menu_cache_age_select_opt_7d">7 days</string>
+    <string name="menu_cache_age_select_opt_14d">14 days</string>
+    <string name="menu_cache_age_select_opt_30d">30 days (reduce network use)</string>
+    <string-array name="cache_age_select_entries">
+        <item>@string/menu_cache_age_select_opt_2d</item>
+        <item>@string/menu_cache_age_select_opt_7d</item>
+        <item>@string/menu_cache_age_select_opt_14d</item>
+        <item>@string/menu_cache_age_select_opt_30d</item>
+    </string-array>
+    <string-array name="cache_age_select_values">
+        <item>CACHE_AGE_2D</item>
+        <item>CACHE_AGE_7D</item>
+        <item>CACHE_AGE_14D</item>
+        <item>CACHE_AGE_30D</item>
+    </string-array>
+    <string name="default_cache_age_select_value">CACHE_AGE_30D</string>
+
+    <string name="settings_cat_feed_list">Feed List</string>
+
+    <string name="settings_enable_row_global_shared">Show Global Shared Stories</string>
+    <string name="settings_enable_row_global_shared_sum">Show the Global Shared Stories folder</string>
+    <string name="settings_enable_row_infrequent_stories">Show Infrequent Stories</string>
+    <string name="settings_enable_row_infrequent_stories_sum">Show the Infrequent Site Stories folder</string>
+
     <string name="settings_cat_story_list">Story List</string>
 
     <string name="oldest">Oldest</string>
@@ -284,9 +324,26 @@
     <string name="settings_immersive_enter_single_tap">Immersive Mode Via Single Tap</string>
     <string name="settings_show_content_preview">Show Content Preview Text</string>
     <string name="settings_show_thumbnails">Show Image Preview Thumbnails</string>
+    <string name="settings_thumbnails_style">Image Preview Thumbnails</string>
     <string name="settings_notifications">Notifications</string>
     <string name="settings_enable_notifications">Enable Notifications</string>
 
+    <string name="large">Large</string>
+    <string name="small">Small</string>
+    <string-array name="thumbnails_style_entries">
+        <item>@string/large</item>
+        <item>@string/small</item>
+        <item>@string/off</item>
+    </string-array>
+
+    <string-array name="thumbnails_style_values">
+        <item>LARGE</item>
+        <item>SMALL</item>
+        <item>OFF</item>
+    </string-array>
+
+    <string name="thumbnails_style_value">LARGE</string>
+
     <string name="infrequent_choice_title">Stories from sites with</string>
     <string name="infrequent_5">&lt; 5 STORIES/MONTH</string>
     <string name="infrequent_15">&lt; 15 STORIES/MONTH</string>
@@ -304,18 +361,9 @@
     <string name="feed_intel_author_header">AUTHORS</string>
     <string name="intel_feed_header">EVERYTHING BY PUBLISHER</string>
 
-    <string name="theme">Theme</string>
     <string name="light">Light</string>
     <string name="dark">Dark</string>
-    <string-array name="default_theme_entries">
-        <item>@string/light</item>
-        <item>@string/dark</item>
-    </string-array>
-    <string-array name="default_theme_values">
-        <item>light</item>
-        <item>dark</item>
-    </string-array>
-    <string name="default_theme_value">light</string>
+    <string name="black">Black</string>
 
     <string name="settings_gestures">Gestures</string>
 
@@ -441,4 +489,10 @@
         <item>@string/whitney_font_prefvalue</item>
     </string-array>
     <string name="default_font_value">DEFAULT</string>
+
+    <string name="story_notification_channel_id">story_notification_channel</string>
+    <string name="story_notification_channel_name">New Stories</string>
+    <string name="save_widget">Save Widget</string>
+    <string name="select_feed">Select Feed</string>
+    <string name="go_to_feed">Go to feed</string>
 </resources>

clients/android/NewsBlur/res/values/styles.xml

@@ -5,32 +5,19 @@
         <item name="android:windowBackground">@color/transparent</item>
     </style>
 
-    <style name="classifyDialog" parent="@android:style/Theme.Dialog">
-        <item name="android:background">@color/bar_background</item>
-        <item name="android:textColor">@color/text</item>
-        <item name="android:textSize">15sp</item>
-        <item name="android:padding">10dp</item>
-    </style>
-
-    <style name="darkClassifyDialog" parent="@android:style/Theme.Dialog">
-        <item name="android:background">@color/dark_bar_background</item>
-        <item name="android:textColor">@color/dark_text</item>
-        <item name="android:textSize">15sp</item>
-        <item name="android:padding">10dp</item>
-    </style>
-
     <style name="actionbar" parent="@android:style/Widget.Holo.Light.ActionBar">
         <item name="android:background">@color/bar_background</item>
     </style>
-
     <style name="actionbar.dark" parent="@android:style/Widget.Holo.ActionBar">
         <item name="android:background">@color/dark_bar_background</item>
     </style>
+    <style name="actionbar.black" parent="@android:style/Widget.Holo.ActionBar">
+        <item name="android:background">@color/black</item>
+    </style>
 
     <style name="expandableListView" parent="@android:style/Widget.Holo.Light.ExpandableListView">
         <item name="android:background">@drawable/list_background</item>
     </style>
-
     <style name="expandableListView.dark" parent="@android:style/Widget.Holo.ExpandableListView">
         <item name="android:background">@drawable/dark_list_background</item>
     </style>
@@ -41,6 +28,9 @@
     <style name="selectorFolderBackground.dark">
         <item name="android:background">@drawable/dark_selector_folder_background</item>
     </style>
+    <style name="selectorFolderBackground.black">
+        <item name="android:background">@drawable/black_selector_folder_background</item>
+    </style>
 
     <style name="selectorFeedBackground">
         <item name="android:background">@drawable/selector_feed_background</item>
@@ -62,6 +52,9 @@
     <style name="actionbarBackground.dark">
         <item name="android:background">@color/dark_bar_background</item>
     </style>
+    <style name="actionbarBackground.black">
+        <item name="android:background">@color/black</item>
+    </style>
 
     <style name="listBackground">
         <item name="android:background">@drawable/list_background</item>
@@ -76,6 +69,9 @@
     <style name="itemBackground.dark">
         <item name="android:background">@color/dark_item_background</item>
     </style>
+    <style name="itemBackground.black">
+        <item name="android:background">@color/black</item>
+    </style>
 
     <style name="readingBackground">
         <item name="android:background">@color/white</item>
@@ -83,6 +79,9 @@
     <style name="readingBackground.dark">
         <item name="android:background">@color/dark_item_background</item>
     </style>
+    <style name="readingBackground.black">
+        <item name="android:background">@color/black</item>
+    </style>
 
     <style name="defaultText">
         <item name="android:textColorLink">@color/linkblue</item>
@@ -129,6 +128,9 @@
     <style name="selectorStoryBackground.dark">
         <item name="android:background">@drawable/dark_selector_story_background</item>
     </style>
+    <style name="selectorStoryBackground.black">
+        <item name="android:background">@drawable/black_selector_story_background</item>
+    </style>
 
     <style name="rowItemHeaderBackground">
         <item name="android:background">@drawable/row_item_header_background</item>
@@ -136,6 +138,9 @@
     <style name="rowItemHeaderBackground.dark">
         <item name="android:background">@drawable/dark_row_item_header_background</item>
     </style>
+    <style name="rowItemHeaderBackground.black">
+        <item name="android:background">@drawable/black_row_item_header_background</item>
+    </style>
 
     <style name="readingItemMetadata">
         <item name="android:textColor">@color/half_darkgray</item>
@@ -187,6 +192,9 @@
     <style name="shareBarBackground.dark">
         <item name="android:background">@color/dark_share_bar_background</item>
     </style>
+    <style name="shareBarBackground.black">
+        <item name="android:background">@color/black</item>
+    </style>
 
     <style name="shareBarText">
         <item name="android:textColor">@color/gray55</item>
@@ -210,6 +218,10 @@
         <item name="android:background">@drawable/dark_gradient_background_default</item>
         <item name="android:textColor">@color/gray55</item>
     </style>
+    <style name="commentsHeader.black">
+        <item name="android:background">@drawable/black_gradient_background_default</item>
+        <item name="android:textColor">@color/gray55</item>
+    </style>
 
     <style name="activityDetailsPager">
         <item name="android:background">@drawable/gradient_background_default</item>
@@ -219,6 +231,10 @@
         <item name="android:background">@drawable/dark_gradient_background_default</item>
         <item name="android:textColor">@color/dark_text</item>
     </style>
+    <style name="activityDetailsPager.black">
+        <item name="android:background">@drawable/black_gradient_background_default</item>
+        <item name="android:textColor">@color/dark_text</item>
+    </style>
 
     <style name="rowBorderTop">
         <item name="android:background">@color/row_border_top</item>
@@ -226,6 +242,9 @@
     <style name="rowBorderTop.dark">
         <item name="android:background">@color/dark_row_border_top</item>
     </style>
+    <style name="rowBorderTop.black">
+        <item name="android:background">@color/black_row_border_top</item>
+    </style>
 
     <style name="rowBorderBottom">
         <item name="android:background">@color/row_border_bottom</item>
@@ -233,6 +252,9 @@
     <style name="rowBorderBottom.dark">
         <item name="android:background">@color/dark_row_border_bottom</item>
     </style>
+    <style name="rowBorderBottom.black">
+        <item name="android:background">@color/black_row_border_bottom</item>
+    </style>
 
     <style name="profileCount">
         <item name="android:background">@color/item_background</item>
@@ -242,6 +264,10 @@
         <item name="android:background">@color/dark_item_background</item>
         <item name="android:textColor">@color/white</item>
     </style>
+    <style name="profileCount.black">
+        <item name="android:background">@color/black</item>
+        <item name="android:textColor">@color/white</item>
+    </style>
 
     <style name="profileActivityList">
         <item name="android:background">@color/item_background</item>
@@ -251,6 +277,10 @@
         <item name="android:background">@color/dark_item_background</item>
         <item name="android:divider">@drawable/divider_dark</item>
     </style>
+    <style name="profileActivityList.black">
+        <item name="android:background">@color/black</item>
+        <item name="android:divider">@drawable/divider_dark</item>
+    </style>
 
     <style name="itemHeaderDivider">
         <item name="android:background">@drawable/divider_item_header</item>
@@ -258,6 +288,9 @@
     <style name="itemHeaderDivider.dark">
         <item name="android:background">@drawable/dark_divider_item_header</item>
     </style>
+    <style name="itemHeaderDivider.black">
+        <item name="android:background">@drawable/black_divider_item_header</item>
+    </style>
 
     <style name="storyCommentDivider">
         <item name="android:background">@color/story_comment_divider</item>
@@ -265,6 +298,9 @@
     <style name="storyCommentDivider.dark">
         <item name="android:background">@color/dark_story_comment_divider</item>
     </style>
+    <style name="storyCommentDivider.black">
+        <item name="android:background">@color/gray07</item>
+    </style>
 
     <style name="explainerText">
         <item name="android:textColor">@color/gray55</item>
@@ -350,4 +386,12 @@
     <style name="contextPopupStyle.dark" parent="@android:style/Widget.Holo.PopupMenu">
         <item name="android:overlapAnchor">true</item>
     </style>
+
+    <!-- Fixes menu text being too dark on Moto devices -->
+    <style name="itemTextAppearance">
+        <item name="android:textColor">@color/black</item>
+    </style>
+    <style name="itemTextAppearance.dark">
+        <item name="android:textColor">@color/white</item>
+    </style>
 </resources>

clients/android/NewsBlur/res/values/theme.xml

@@ -41,6 +41,7 @@
         <item name="selectorOverlayBackgroundText">@style/selectorOverlayBackgroundText</item>
         <item name="muteicon">@style/muteicon</item>
         <item name="android:contextPopupMenuStyle">@style/contextPopupStyle</item>
+        <item name="android:textColorPrimary">@color/black</item>
 	</style>
 
     <style name="NewsBlurDarkTheme" parent="@android:style/Theme.Holo" >
@@ -83,6 +84,55 @@
         <item name="selectorOverlayBackgroundStory">@style/selectorOverlayBackgroundStory.dark</item>
         <item name="selectorOverlayBackgroundText">@style/selectorOverlayBackgroundText.dark</item>
         <item name="muteicon">@style/muteicon.dark</item>
+
+        <!--requires API 24-->
         <item name="android:contextPopupMenuStyle">@style/contextPopupStyle.dark</item>
+        <item name="android:textColorPrimary">@color/white</item>
+        <item name="android:itemTextAppearance">@style/itemTextAppearance.dark</item>
+    </style>
+
+    <style name="NewsBlurBlackTheme" parent="@android:style/Theme.Holo" >
+        <item name="android:actionBarStyle">@style/actionbar.black</item>
+        <item name="android:expandableListViewStyle">@style/expandableListView.dark</item>
+        <item name="selectorFolderBackground">@style/selectorFolderBackground.black</item>
+        <item name="selectorFeedBackground">@style/selectorFeedBackground.dark</item>
+        <item name="feedRowNeutCountText">@style/feedRowNeutCountText.dark</item>
+        <item name="actionbarBackground">@style/actionbarBackground.black</item>
+        <item name="listBackground">@style/listBackground.dark</item>
+        <item name="itemBackground">@style/itemBackground.black</item>
+        <item name="readingBackground">@style/readingBackground.black</item>
+        <item name="defaultText">@style/defaultText.dark</item>
+        <item name="linkText">@style/linkText.dark</item>
+        <item name="storySnippetText">@style/storySnippetText.dark</item>
+        <item name="storyDateText">@style/storyDateText.dark</item>
+        <item name="storyFeedTitleText">@style/storyFeedTitleText.dark</item>
+        <item name="selectorStoryBackground">@style/selectorStoryBackground.black</item>
+        <item name="rowItemHeaderBackground">@style/rowItemHeaderBackground.black</item>
+        <item name="readingItemMetadata">@style/readingItemMetadata.dark</item>
+        <item name="tag">@style/tag.dark</item>
+        <item name="actionButtons">@style/actionButtons.dark</item>
+        <item name="storyButtons">@style/storyButtons.dark</item>
+        <item name="shareBarBackground">@style/shareBarBackground.black</item>
+        <item name="shareBarText">@style/shareBarText.dark</item>
+        <item name="commentsHeader">@style/commentsHeader.black</item>
+        <item name="rowBorderTop">@style/rowBorderTop.black</item>
+        <item name="rowBorderBottom">@style/rowBorderBottom.black</item>
+        <item name="profileCount">@style/profileCount.black</item>
+        <item name="profileActivityList">@style/profileActivityList.black</item>
+        <item name="itemHeaderDivider">@style/itemHeaderDivider.black</item>
+        <item name="storyCommentDivider">@style/storyCommentDivider.black</item>
+        <item name="activityDetailsPager">@style/activityDetailsPager.black</item>
+        <item name="explainerText">@style/explainerText.dark</item>
+        <item name="toggleText">@style/toggleText.dark</item>
+        <item name="selectorOverlayBackgroundLeft">@style/selectorOverlayBackgroundLeft.dark</item>
+        <item name="selectorOverlayBackgroundRight">@style/selectorOverlayBackgroundRight.dark</item>
+        <item name="selectorOverlayBackgroundRightDone">@style/selectorOverlayBackgroundRightDone.dark</item>
+        <item name="selectorOverlayBackgroundSend">@style/selectorOverlayBackgroundSend.dark</item>
+        <item name="selectorOverlayBackgroundStory">@style/selectorOverlayBackgroundStory.dark</item>
+        <item name="selectorOverlayBackgroundText">@style/selectorOverlayBackgroundText.dark</item>
+        <item name="muteicon">@style/muteicon.dark</item>
+        <item name="android:contextPopupMenuStyle">@style/contextPopupStyle.dark</item>
+        <item name="android:textColorPrimary">@color/white</item>
+        <item name="android:itemTextAppearance">@style/itemTextAppearance.dark</item>
     </style>
 </resources>

clients/android/NewsBlur/res/xml/activity_settings.xml

@@ -26,6 +26,30 @@
             android:key="keep_old_stories"
             android:title="@string/settings_keep_old_stories" 
             android:summary="@string/settings_keep_old_stories_sum" />
+        <ListPreference
+            android:key="cache_age_select"
+            android:title="@string/menu_cache_age_select"
+            android:dialogTitle="@string/menu_cache_age_select"
+            android:summary="@string/menu_cache_age_select_sum"
+            android:entries="@array/cache_age_select_entries"
+            android:entryValues="@array/cache_age_select_values"
+            android:defaultValue="@string/default_cache_age_select_value" />
+    </PreferenceCategory>
+
+    <PreferenceCategory
+        android:title="@string/settings_cat_feed_list">
+        <CheckBoxPreference
+            android:defaultValue="true"
+            android:key="enable_row_global_shared"
+            android:title="@string/settings_enable_row_global_shared" 
+            android:summary="@string/settings_enable_row_global_shared_sum"
+            />
+        <CheckBoxPreference
+            android:defaultValue="true"
+            android:key="enable_row_infrequent_stories"
+            android:title="@string/settings_enable_row_infrequent_stories" 
+            android:summary="@string/settings_enable_row_infrequent_stories_sum"
+            />
     </PreferenceCategory>
 
     <PreferenceCategory 
@@ -64,10 +88,13 @@
             android:defaultValue="true"
             android:key="pref_show_content_preview"
             android:title="@string/settings_show_content_preview" />
-        <CheckBoxPreference
-            android:defaultValue="true"
-            android:key="pref_show_thumbnails"
-            android:title="@string/settings_show_thumbnails" />
+        <ListPreference
+            android:key="pref_thumbnails_style"
+            android:title="@string/settings_thumbnails_style"
+            android:dialogTitle="@string/settings_thumbnails_style"
+            android:entries="@array/thumbnails_style_entries"
+            android:entryValues="@array/thumbnails_style_values"
+            android:defaultValue="@string/thumbnails_style_value" />
         <CheckBoxPreference
             android:defaultValue="false"
             android:key="pref_mark_read_on_scroll"
@@ -108,17 +135,6 @@
         </CheckBoxPreference>
     </PreferenceCategory>
 
-    <PreferenceCategory
-        android:title="@string/theme">
-        <ListPreference
-            android:key="theme"
-            android:title="@string/theme"
-            android:dialogTitle="@string/theme"
-            android:entries="@array/default_theme_entries"
-            android:entryValues="@array/default_theme_values"
-            android:defaultValue="@string/default_theme_value" />
-    </PreferenceCategory>
-
     <PreferenceCategory
         android:title="@string/settings_gestures">
         <ListPreference

clients/android/NewsBlur/res/xml/newsblur_appwidget_info.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="utf-8"?>
+<appwidget-provider xmlns:android="http://schemas.android.com/apk/res/android"
+    android:minWidth="180dp"
+    android:minHeight="100dp"
+    android:minResizeWidth="100dp"
+    android:minResizeHeight="60dp"
+    android:updatePeriodMillis="14400000"
+    android:initialLayout="@layout/newsblur_widget"
+    android:configure="com.newsblur.widget.ConfigureWidgetActivity"
+    android:resizeMode="horizontal|vertical"
+    android:widgetCategory="home_screen">
+</appwidget-provider>
+<!--    android:previewImage="@drawable/preview"-->

clients/android/NewsBlur/src/com/newsblur/activity/ActivityDetailsPagerAdapter.java

@@ -1,9 +1,9 @@
 package com.newsblur.activity;
 
-import android.app.Fragment;
-import android.app.FragmentManager;
 import android.content.res.Resources;
-import android.support.v13.app.FragmentPagerAdapter;
+import android.support.v4.app.Fragment;
+import android.support.v4.app.FragmentManager;
+import android.support.v4.app.FragmentPagerAdapter;
 
 import com.newsblur.R;
 import com.newsblur.domain.UserDetails;

clients/android/NewsBlur/src/com/newsblur/activity/AddFeedExternal.java

@@ -1,9 +1,9 @@
 package com.newsblur.activity;
 
-import android.app.DialogFragment;
 import android.content.Intent;
 import android.net.Uri;
 import android.os.Bundle;
+import android.support.v4.app.DialogFragment;
 import android.view.View;
 import android.widget.TextView;
 
@@ -41,7 +41,7 @@ public class AddFeedExternal extends NbActivity implements AddFeedFragment.AddFe
         com.newsblur.util.Log.d(this, "intent filter caught feed-like URI: " + uri);
 
 		DialogFragment addFeedFragment = AddFeedFragment.newInstance(uri.toString(), uri.toString());
-		addFeedFragment.show(getFragmentManager(), "dialog");
+		addFeedFragment.show(getSupportFragmentManager(), "dialog");
     }
 
     @Override

clients/android/NewsBlur/src/com/newsblur/activity/AddSocial.java

@@ -2,8 +2,8 @@ package com.newsblur.activity;
 
 import android.content.Intent;
 import android.os.Bundle;
-import android.app.FragmentManager;
-import android.app.FragmentTransaction;
+import android.support.v4.app.FragmentManager;
+import android.support.v4.app.FragmentTransaction;
 import android.view.View;
 import android.view.View.OnClickListener;
 import android.widget.Button;
@@ -22,7 +22,7 @@ public class AddSocial extends NbActivity {
 		super.onCreate(bundle);
 		setContentView(R.layout.activity_addsocial);
 		
-		fragmentManager = getFragmentManager();
+		fragmentManager = getSupportFragmentManager();
 
 		if (fragmentManager.findFragmentByTag(currentTag) == null) {
 			FragmentTransaction transaction = fragmentManager.beginTransaction();

clients/android/NewsBlur/src/com/newsblur/activity/AllSharedStoriesItemsList.java

@@ -1,12 +1,8 @@
 package com.newsblur.activity;
 
 import android.os.Bundle;
-import android.app.FragmentTransaction;
-import android.view.Menu;
-import android.view.MenuInflater;
 
 import com.newsblur.R;
-import com.newsblur.fragment.AllSharedStoriesItemListFragment;
 import com.newsblur.util.UIUtils;
 
 public class AllSharedStoriesItemsList extends ItemsList {
@@ -15,23 +11,7 @@ public class AllSharedStoriesItemsList extends ItemsList {
 	protected void onCreate(Bundle bundle) {
 		super.onCreate(bundle);
 
-        UIUtils.setCustomActionBar(this, R.drawable.ak_icon_blurblogs, getResources().getString(R.string.all_shared_stories));
-
-		itemListFragment = (AllSharedStoriesItemListFragment) fragmentManager.findFragmentByTag(AllSharedStoriesItemListFragment.class.getName());
-		if (itemListFragment == null) {
-			itemListFragment = AllSharedStoriesItemListFragment.newInstance();
-			itemListFragment.setRetainInstance(true);
-			FragmentTransaction listTransaction = fragmentManager.beginTransaction();
-			listTransaction.add(R.id.activity_itemlist_container, itemListFragment, AllSharedStoriesItemListFragment.class.getName());
-			listTransaction.commit();
-		}
+        UIUtils.setCustomActionBar(this, R.drawable.ak_icon_blurblogs, getResources().getString(R.string.all_shared_stories_title));
 	}
 
-    @Override
-    public boolean onCreateOptionsMenu(Menu menu) {
-        MenuInflater inflater = getMenuInflater();
-        inflater.inflate(R.menu.allsocialstories_itemslist, menu);
-        return true;
-    }
-
 }

clients/android/NewsBlur/src/com/newsblur/activity/AllSharedStoriesReading.java

@@ -3,7 +3,6 @@ package com.newsblur.activity;
 import android.os.Bundle;
 
 import com.newsblur.R;
-import com.newsblur.database.MixedFeedsReadingAdapter;
 import com.newsblur.util.UIUtils;
 
 public class AllSharedStoriesReading extends Reading {
@@ -12,12 +11,7 @@ public class AllSharedStoriesReading extends Reading {
     protected void onCreate(Bundle savedInstanceBundle) {
         super.onCreate(savedInstanceBundle);
 
-        UIUtils.setCustomActionBar(this, R.drawable.ak_icon_blurblogs, getResources().getString(R.string.all_shared_stories));
-
-        // No sourceUserId since this is all shared stories. The sourceUsedId for each story will be used.
-        readingAdapter = new MixedFeedsReadingAdapter(getFragmentManager(), null);
-
-        getLoaderManager().initLoader(0, null, this);
+        UIUtils.setCustomActionBar(this, R.drawable.ak_icon_blurblogs, getResources().getString(R.string.all_shared_stories_title));
     }
 
 }

clients/android/NewsBlur/src/com/newsblur/activity/AllStoriesItemsList.java

@@ -1,12 +1,8 @@
 package com.newsblur.activity;
 
 import android.os.Bundle;
-import android.app.FragmentTransaction;
-import android.view.Menu;
-import android.view.MenuInflater;
 
 import com.newsblur.R;
-import com.newsblur.fragment.AllStoriesItemListFragment;
 import com.newsblur.util.UIUtils;
 
 public class AllStoriesItemsList extends ItemsList {
@@ -15,24 +11,7 @@ public class AllStoriesItemsList extends ItemsList {
 	protected void onCreate(Bundle bundle) {
 		super.onCreate(bundle);
 
-        UIUtils.setCustomActionBar(this, R.drawable.ak_icon_allstories, getResources().getString(R.string.all_stories_row_title));
-
-		itemListFragment = (AllStoriesItemListFragment) fragmentManager.findFragmentByTag(AllStoriesItemListFragment.class.getName());
-		if (itemListFragment == null) {
-			itemListFragment = AllStoriesItemListFragment.newInstance();
-			itemListFragment.setRetainInstance(true);
-			FragmentTransaction listTransaction = fragmentManager.beginTransaction();
-			listTransaction.add(R.id.activity_itemlist_container, itemListFragment, AllStoriesItemListFragment.class.getName());
-			listTransaction.commit();
-		}
-	}
-
-	@Override
-	public boolean onCreateOptionsMenu(Menu menu) {
-		super.onCreateOptionsMenu(menu);
-		MenuInflater inflater = getMenuInflater();
-		inflater.inflate(R.menu.allstories_itemslist, menu);
-		return true;
+        UIUtils.setCustomActionBar(this, R.drawable.ak_icon_allstories, getResources().getString(R.string.all_stories_title));
 	}
 
 }

clients/android/NewsBlur/src/com/newsblur/activity/AllStoriesReading.java

@@ -3,7 +3,6 @@ package com.newsblur.activity;
 import android.os.Bundle;
 
 import com.newsblur.R;
-import com.newsblur.database.MixedFeedsReadingAdapter;
 import com.newsblur.util.UIUtils;
 
 public class AllStoriesReading extends Reading {
@@ -12,10 +11,8 @@ public class AllStoriesReading extends Reading {
     protected void onCreate(Bundle savedInstanceBundle) {
         super.onCreate(savedInstanceBundle);
 
-        UIUtils.setCustomActionBar(this, R.drawable.ak_icon_allstories, getResources().getString(R.string.all_stories_row_title));
+        UIUtils.setCustomActionBar(this, R.drawable.ak_icon_allstories, getResources().getString(R.string.all_stories_title));
         setTitle(getResources().getString(R.string.all_stories_row_title));
-        readingAdapter = new MixedFeedsReadingAdapter(getFragmentManager(), null);
-        getLoaderManager().initLoader(0, null, this);
     }
 
 }

clients/android/NewsBlur/src/com/newsblur/activity/FeedItemsList.java

@@ -1,17 +1,18 @@
 package com.newsblur.activity;
 
+import android.content.Context;
+import android.content.Intent;
 import android.os.Bundle;
-import android.app.DialogFragment;
-import android.app.FragmentTransaction;
+import android.support.v4.app.DialogFragment;
 import android.view.Menu;
-import android.view.MenuInflater;
 import android.view.MenuItem;
 
 import com.newsblur.R;
 import com.newsblur.domain.Feed;
 import com.newsblur.fragment.DeleteFeedFragment;
 import com.newsblur.fragment.FeedIntelTrainerFragment;
-import com.newsblur.fragment.FeedItemListFragment;
+import com.newsblur.fragment.RenameFeedFragment;
+import com.newsblur.util.FeedSet;
 import com.newsblur.util.FeedUtils;
 import com.newsblur.util.UIUtils;
 
@@ -22,6 +23,15 @@ public class FeedItemsList extends ItemsList {
 	private Feed feed;
 	private String folderName;
 
+    public static void startActivity(Context context, FeedSet feedSet,
+                                     Feed feed, String folderName) {
+        Intent intent = new Intent(context, FeedItemsList.class);
+        intent.putExtra(ItemsList.EXTRA_FEED_SET, feedSet);
+        intent.putExtra(FeedItemsList.EXTRA_FEED, feed);
+        intent.putExtra(FeedItemsList.EXTRA_FOLDER_NAME, folderName);
+        context.startActivity(intent);
+    }
+
 	@Override
 	protected void onCreate(Bundle bundle) {
 		feed = (Feed) getIntent().getSerializableExtra(EXTRA_FEED);
@@ -30,20 +40,11 @@ public class FeedItemsList extends ItemsList {
 		super.onCreate(bundle);
 
         UIUtils.setCustomActionBar(this, feed.faviconUrl, feed.title);
-
-		itemListFragment = (FeedItemListFragment) fragmentManager.findFragmentByTag(FeedItemListFragment.class.getName());
-		if (itemListFragment == null) {
-			itemListFragment = FeedItemListFragment.newInstance(feed);
-			itemListFragment.setRetainInstance(true);
-			FragmentTransaction listTransaction = fragmentManager.beginTransaction();
-			listTransaction.add(R.id.activity_itemlist_container, itemListFragment, FeedItemListFragment.class.getName());
-			listTransaction.commit();
-		}
 	}
 
 	public void deleteFeed() {
 		DialogFragment deleteFeedFragment = DeleteFeedFragment.newInstance(feed, folderName);
-		deleteFeedFragment.show(fragmentManager, "dialog");
+		deleteFeedFragment.show(getSupportFragmentManager(), "dialog");
 	}
 
 	@Override
@@ -74,35 +75,27 @@ public class FeedItemsList extends ItemsList {
         }
         if (item.getItemId() == R.id.menu_intel) {
             FeedIntelTrainerFragment intelFrag = FeedIntelTrainerFragment.newInstance(feed, fs);
-            intelFrag.show(getFragmentManager(), FeedIntelTrainerFragment.class.getName());
+            intelFrag.show(getSupportFragmentManager(), FeedIntelTrainerFragment.class.getName());
             return true;
         }
+        if (item.getItemId() == R.id.menu_rename_feed) {
+            RenameFeedFragment frag = RenameFeedFragment.newInstance(feed);
+            frag.show(getSupportFragmentManager(), RenameFeedFragment.class.getName());
+            return true;
+            // TODO: since this activity uses a feed object passed as an extra and doesn't query the DB,
+            // the name change won't be reflected until the activity finishes.
+        }
         return false;
 	}
 	
 	@Override
 	public boolean onCreateOptionsMenu(Menu menu) {
-		super.onCreateOptionsMenu(menu);
         if (!feed.active) {
             // there is currently no way for a feed to be un-muted while in this activity, so
             // don't bother creating the menu, which contains no valid options for a muted feed
             return false;
         }
-		MenuInflater inflater = getMenuInflater();
-		inflater.inflate(R.menu.feed_itemslist, menu);
-        if (feed.isNotifyUnread()) {
-            menu.findItem(R.id.menu_notifications_disable).setChecked(false);
-            menu.findItem(R.id.menu_notifications_unread).setChecked(true);
-            menu.findItem(R.id.menu_notifications_focus).setChecked(false);
-        } else if (feed.isNotifyFocus()) {
-            menu.findItem(R.id.menu_notifications_disable).setChecked(false);
-            menu.findItem(R.id.menu_notifications_unread).setChecked(false);
-            menu.findItem(R.id.menu_notifications_focus).setChecked(true);
-        } else {
-            menu.findItem(R.id.menu_notifications_disable).setChecked(true);
-            menu.findItem(R.id.menu_notifications_unread).setChecked(false);
-            menu.findItem(R.id.menu_notifications_focus).setChecked(false);
-        }
+		super.onCreateOptionsMenu(menu);
 		return true;
 	}
 

clients/android/NewsBlur/src/com/newsblur/activity/FeedReading.java

@@ -2,8 +2,6 @@ package com.newsblur.activity;
 
 import android.os.Bundle;
 
-import com.newsblur.database.FeedReadingAdapter;
-import com.newsblur.domain.Classifier;
 import com.newsblur.domain.Feed;
 import com.newsblur.util.FeedUtils;
 import com.newsblur.util.UIUtils;
@@ -25,13 +23,7 @@ public class FeedReading extends Reading {
             return;
         }
 
-        Classifier classifier = FeedUtils.dbHelper.getClassifierForFeed(feed.feedId);
-
         UIUtils.setCustomActionBar(this, feed.faviconUrl, feed.title);
-
-        readingAdapter = new FeedReadingAdapter(fragmentManager, feed, classifier);
-
-        getLoaderManager().initLoader(0, null, this);
     }
 
 }