Merge branch 'master' into circular

* master:
  Don't collide stories that are too short to be collided.
  Fixing the long broken last update display on story titles header.
  Fixing assortment of small bugs.
  Allowing anonymous users to use statistics.
  Handling previously read stories gracefully on unread count update.
  Fixing updating of folder title when renaming. Thanks @afita.
  Adding Metroblur to logging.
  The 'm' key should only find oldest story, not also mark it read/unread. Typo? Thanks @donmelton.

apps/analyzer/views.py

@@ -3,7 +3,7 @@ from utils import log as logging
 from django.shortcuts import get_object_or_404
 from django.views.decorators.http import require_POST
 from django.conf import settings
-# from mongoengine.queryset import OperationError
+from mongoengine.queryset import NotUniqueError
 from apps.rss_feeds.models import Feed
 from apps.reader.models import UserSubscription
 from apps.analyzer.models import MClassifierTitle, MClassifierAuthor, MClassifierFeed, MClassifierTag
@@ -80,10 +80,10 @@ def save_classifier(request):
                     if content_type == 'feed':
                         if not post_content.startswith('social:'):
                             classifier_dict['feed_id'] = post_content
-                    # try:
-                    classifier, created = ClassifierCls.objects.get_or_create(**classifier_dict)
-                    # except OperationError:
-                    #     continue
+                    try:
+                        classifier, created = ClassifierCls.objects.get_or_create(**classifier_dict)
+                    except NotUniqueError:
+                        continue
                     if score == 0:
                         classifier.delete()
                     elif classifier.score != score:

apps/reader/views.py

@@ -589,7 +589,7 @@ def load_single_feed(request, feed_id):
                 feed_tags=feed_tags, 
                 feed_authors=feed_authors, 
                 classifiers=classifiers,
-                last_update=last_update,
+                updated=last_update,
                 feed_id=feed.pk,
                 elapsed_time=round(float(timediff), 2))
     

apps/rss_feeds/models.py

@@ -791,7 +791,7 @@ class Feed(models.Model):
             logging.debug("   ---> [%-30s] ~FBChecking ~SB%s~SN new/updated against ~SB%s~SN stories" % (
                           self.title[:30],
                           len(stories),
-                          len(existing_stories)))
+                          len(existing_stories.keys())))
         
         for story in stories:
             if not story.get('title'):
@@ -1112,12 +1112,13 @@ class Feed(models.Model):
         story_in_system = None
         story_has_changed = False
         story_link = self.get_permalink(story)
+        existing_stories_guids = existing_stories.keys()
         # story_pub_date = story.get('published')
         # story_published_now = story.get('published_now', False)
         # start_date = story_pub_date - datetime.timedelta(hours=8)
         # end_date = story_pub_date + datetime.timedelta(hours=8)
         
-        for existing_story in existing_stories:
+        for existing_story in existing_stories.values():
             content_ratio = 0
             # existing_story_pub_date = existing_story.story_date
             # print 'Story pub date: %s %s' % (story_published_now, story_pub_date)
@@ -1135,7 +1136,10 @@ class Feed(models.Model):
                 
             if isinstance(existing_story.id, unicode):
                 existing_story.story_guid = existing_story.id
-            if story.get('guid') and story.get('guid') == existing_story.story_guid:
+            if (story.get('guid') in existing_stories_guids and 
+                story.get('guid') != existing_story.story_guid):
+                continue
+            elif story.get('guid') == existing_story.story_guid:
                 story_in_system = existing_story
             
             # Title distance + content distance, checking if story changed
@@ -1167,8 +1171,10 @@ class Feed(models.Model):
                 
             if story_in_system and not story_has_changed:
                 if story_content != existing_story_content:
+                    # print "Content difference - %s/%s" % (story_content, existing_story_content)
                     story_has_changed = True
                 if story_link != existing_story.story_permalink:
+                    # print "Permalink difference - %s/%s" % (story_link, existing_story.story_permalink)
                     story_has_changed = True
                 # if story_pub_date != existing_story.story_date:
                 #     story_has_changed = True
@@ -1573,6 +1579,7 @@ class MStarredStory(mongo.Document):
     story_content_z          = mongo.BinaryField()
     story_original_content   = mongo.StringField()
     story_original_content_z = mongo.BinaryField()
+    original_text_z          = mongo.BinaryField()
     story_content_type       = mongo.StringField(max_length=255)
     story_author_name        = mongo.StringField()
     story_permalink          = mongo.StringField()
@@ -1611,7 +1618,19 @@ class MStarredStory(mongo.Document):
     @property
     def guid_hash(self):
         return hashlib.sha1(self.story_guid).hexdigest()[:6]
-    
+
+    def fetch_original_text(self, force=False, request=None):
+        original_text_z = self.original_text_z
+        
+        if not original_text_z or force:
+            ti = TextImporter(self, request=request)
+            original_text = ti.fetch()
+        else:
+            logging.user(request, "~FYFetching ~FGoriginal~FY story text, ~SBfound.")
+            original_text = zlib.decompress(original_text_z)
+        
+        return original_text
+        
 
 class MFeedFetchHistory(mongo.Document):
     feed_id = mongo.IntField()

apps/rss_feeds/text_importer.py

@@ -23,13 +23,18 @@ class TextImporter:
             'Connection': 'close',
         }
     
-    def fetch(self):
-        html = requests.get(self.story.story_permalink, headers=self.headers)
-        original_text_doc = readability.Document(html.text, url=html.url, debug=settings.DEBUG)
-        content = original_text_doc.summary(html_partial=True)
+    def fetch(self, skip_save=False):
+        try:
+            html = requests.get(self.story.story_permalink, headers=self.headers)
+            original_text_doc = readability.Document(html.text, url=html.url, debug=settings.DEBUG)
+            content = original_text_doc.summary(html_partial=True)
+        except:
+            content = None
+        
         if content:
-            self.story.original_text_z = zlib.compress(content)
-            self.story.save()
+            if not skip_save:
+                self.story.original_text_z = zlib.compress(content)
+                self.story.save()
             logging.user(self.request, "~SN~FYFetched ~FGoriginal text~FY: now ~SB%s bytes~SN vs. was ~SB%s bytes" % (
                 len(unicode(content)),
                 self.story.story_content_z and len(zlib.decompress(self.story.story_content_z))

apps/rss_feeds/views.py

@@ -123,6 +123,7 @@ def feed_autocomplete(request):
     
 @json.json_view
 def load_feed_statistics(request, feed_id):
+    user = get_user(request)
     stats = dict()
     feed = get_object_or_404(Feed, pk=feed_id)
     feed.save_feed_story_history_statistics()
@@ -171,7 +172,7 @@ def load_feed_statistics(request, feed_id):
     stats['classifier_counts'] = json.decode(feed.data.feed_classifier_counts)
     
     # Fetch histories
-    timezone = request.user.profile.timezone
+    timezone = user.profile.timezone
     stats['feed_fetch_history'] = MFeedFetchHistory.feed_history(feed_id, timezone=timezone)
     stats['page_fetch_history'] = MPageFetchHistory.feed_history(feed_id, timezone=timezone)
     stats['feed_push_history'] = MFeedPushHistory.feed_history(feed_id, timezone=timezone)
@@ -184,7 +185,8 @@ def load_feed_statistics(request, feed_id):
 def load_feed_settings(request, feed_id):
     stats = dict()
     feed = get_object_or_404(Feed, pk=feed_id)
-    timezone = request.user.profile.timezone
+    user = get_user(request)
+    timezone = user.profile.timezone
     
     stats['duplicate_addresses'] = feed.duplicate_addresses.all()
     stats['feed_fetch_history'] = MFeedFetchHistory.feed_history(feed_id, timezone=timezone)

apps/social/models.py

@@ -10,7 +10,7 @@ import random
 import requests
 from collections import defaultdict
 from BeautifulSoup import BeautifulSoup
-# from mongoengine.queryset import OperationError
+from mongoengine.queryset import NotUniqueError
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.contrib.sites.models import Site
@@ -963,10 +963,10 @@ class MSocialSubscription(mongo.Document):
                                                             "read_date": date,
                                                             "story_date": story.shared_date,
                                                         })
-            # except OperationError:
-            #     if not mark_all_read:
-            #         logging.user(request, "~FRAlready saved read story: %s" % story.story_guid)
-            #     continue
+            except NotUniqueError:
+                if not mark_all_read:
+                    logging.user(request, "~FRAlready saved read story: %s" % story.story_guid)
+                continue
             except MUserStory.MultipleObjectsReturned:
                 if not mark_all_read:
                     logging.user(request, "~BR~FW~SKMultiple read stories: %s" % story.story_guid)
@@ -1965,11 +1965,17 @@ class MSharedStory(mongo.Document):
         
         return image_sizes
     
-    def fetch_original_text(self):
-        ti = TextImporter(self)
-        original_text_doc = ti.fetch()
+    def fetch_original_text(self, force=False, request=None):
+        original_text_z = self.original_text_z
         
-        return original_text_doc
+        if not original_text_z or force:
+            ti = TextImporter(self, request=request)
+            original_text = ti.fetch()
+        else:
+            logging.user(request, "~FYFetching ~FGoriginal~FY story text, ~SBfound.")
+            original_text = zlib.decompress(original_text_z)
+        
+        return original_text
 
 class MSocialServices(mongo.Document):
     user_id               = mongo.IntField()

media/js/newsblur/common/assetmodel.js

@@ -450,6 +450,11 @@ NEWSBLUR.AssetModel = Backbone.Router.extend({
                 this.user_profiles.add(profiles);
             }
             
+            if (data.updated) {
+                var feed = this.get_feed(feed_id);
+                feed.set('updated', data.updated);
+            }
+            
             if (data.stories && first_load) {
                 this.feed_tags = data.feed_tags || {};
                 this.feed_authors = data.feed_authors || {};

media/js/newsblur/reader/reader.js

@@ -3924,7 +3924,10 @@
         },
         
         feed_unread_count: function(feed_id) {
-            this.model.feed_unread_count(feed_id || this.active_feed);
+            feed_id = feed_id || this.active_feed;
+            if (!feed_id) return;
+            
+            this.model.feed_unread_count(feed_id);
         },
         
         // ===================
@@ -5442,10 +5445,6 @@
                 e.preventDefault();
                 self.mark_active_story_read();
             });
-            $document.bind('keydown', 'm', function(e) {
-                e.preventDefault();
-                self.mark_active_story_read();
-            });
             $document.bind('keydown', 'g', function(e) {
                 e.preventDefault();
                 NEWSBLUR.app.feed_selector.toggle();

media/js/newsblur/views/folder_view.js

@@ -144,7 +144,7 @@ NEWSBLUR.Views.Folder = Backbone.View.extend({
     },
     
     update_title: function() {
-        this.$('.folder_title_text').eq(0).html(this.options.folder_title);
+        this.$('.folder_title_text').eq(0).html(this.model.get('folder_title'));
     },
     
     update_selected: function() {

utils/feed_fetcher.py

@@ -216,7 +216,7 @@ class ProcessFeed:
             stories.append(story)
             story_guids.append(story.get('guid'))
 
-        existing_stories = list(MStory.objects(
+        existing_stories = dict((s.story_guid, s) for s in MStory.objects(
             # story_guid__in=story_guids,
             story_date__gte=start_date,
             story_feed_id=self.feed.pk

utils/user_functions.py

@@ -83,6 +83,8 @@ def extract_user_agent(request):
         platform = 'Blar'
     elif 'Android' in user_agent:
         platform = 'Androd'
+    elif 'Metroblur' in user_agent:
+        platform = 'Metrob'
     elif 'MSIE' in user_agent:
         platform = 'IE'
         if 'MSIE 9' in user_agent: