From 51f9dbd36f693dd847a564fd53f7b0b1b3b9e726 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Fri, 25 Jan 2013 15:30:56 -0800 Subject: [PATCH 1/8] The 'm' key should only find oldest story, not also mark it read/unread. Typo? Thanks @donmelton. --- media/js/newsblur/reader/reader.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/media/js/newsblur/reader/reader.js b/media/js/newsblur/reader/reader.js index a7ef24f4e..1be7277df 100644 --- a/media/js/newsblur/reader/reader.js +++ b/media/js/newsblur/reader/reader.js @@ -5570,10 +5570,6 @@ e.preventDefault(); self.mark_active_story_read(); }); - $document.bind('keydown', 'm', function(e) { - e.preventDefault(); - self.mark_active_story_read(); - }); $document.bind('keydown', 'g', function(e) { e.preventDefault(); NEWSBLUR.app.feed_selector.toggle(); From 4aacfe64cb24d1ce2cb0382ac09f91fcae63dbff Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Sun, 27 Jan 2013 15:35:41 -0800 Subject: [PATCH 2/8] Adding Metroblur to logging. --- utils/user_functions.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/user_functions.py b/utils/user_functions.py index 910ded25e..2e44ba2c1 100644 --- a/utils/user_functions.py +++ b/utils/user_functions.py @@ -83,6 +83,8 @@ def extract_user_agent(request): platform = 'Blar' elif 'Android' in user_agent: platform = 'Androd' + elif 'Metroblur' in user_agent: + platform = 'Metrob' elif 'MSIE' in user_agent: platform = 'IE' if 'MSIE 9' in user_agent: From a56d50f1641ebb5cb612a376ce3a504bcb166d75 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Mon, 28 Jan 2013 14:45:04 -0800 Subject: [PATCH 3/8] Fixing updating of folder title when renaming. Thanks @afita. --- media/js/newsblur/views/folder_view.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/media/js/newsblur/views/folder_view.js b/media/js/newsblur/views/folder_view.js index 085626792..6d316642e 100644 --- a/media/js/newsblur/views/folder_view.js +++ b/media/js/newsblur/views/folder_view.js @@ -138,7 +138,7 @@ NEWSBLUR.Views.Folder = Backbone.View.extend({ }, update_title: function() { - this.$('.folder_title_text').eq(0).html(this.options.folder_title); + this.$('.folder_title_text').eq(0).html(this.model.get('folder_title')); }, update_selected: function() { From cb5beb73a414c7e7020e62da3a0b5e93fee5e571 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Mon, 28 Jan 2013 15:02:52 -0800 Subject: [PATCH 4/8] Handling previously read stories gracefully on unread count update. --- apps/social/models.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/social/models.py b/apps/social/models.py index fb3873f11..d42b5c450 100644 --- a/apps/social/models.py +++ b/apps/social/models.py @@ -10,7 +10,7 @@ import random import requests from collections import defaultdict from BeautifulSoup import BeautifulSoup -# from mongoengine.queryset import OperationError +from mongoengine.queryset import NotUniqueError from django.conf import settings from django.contrib.auth.models import User from django.contrib.sites.models import Site @@ -903,10 +903,10 @@ class MSocialSubscription(mongo.Document): "read_date": date, "story_date": story.shared_date, }) - # except OperationError: - # if not mark_all_read: - # logging.user(request, "~FRAlready saved read story: %s" % story.story_guid) - # continue + except NotUniqueError: + if not mark_all_read: + logging.user(request, "~FRAlready saved read story: %s" % story.story_guid) + continue except MUserStory.MultipleObjectsReturned: if not mark_all_read: logging.user(request, "~BR~FW~SKMultiple read stories: %s" % story.story_guid) From 8fbc9f5174cf27d8bc0a7f55e63df8bdde74afb0 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Mon, 28 Jan 2013 15:23:19 -0800 Subject: [PATCH 5/8] Allowing anonymous users to use statistics. --- apps/rss_feeds/views.py | 6 ++++-- media/js/newsblur/reader/reader.js | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/apps/rss_feeds/views.py b/apps/rss_feeds/views.py index 8030bfbff..d2d35b7fd 100644 --- a/apps/rss_feeds/views.py +++ b/apps/rss_feeds/views.py @@ -123,6 +123,7 @@ def feed_autocomplete(request): @json.json_view def load_feed_statistics(request, feed_id): + user = get_user(request) stats = dict() feed = get_object_or_404(Feed, pk=feed_id) feed.save_feed_story_history_statistics() @@ -171,7 +172,7 @@ def load_feed_statistics(request, feed_id): stats['classifier_counts'] = json.decode(feed.data.feed_classifier_counts) # Fetch histories - timezone = request.user.profile.timezone + timezone = user.profile.timezone stats['feed_fetch_history'] = MFeedFetchHistory.feed_history(feed_id, timezone=timezone) stats['page_fetch_history'] = MPageFetchHistory.feed_history(feed_id, timezone=timezone) stats['feed_push_history'] = MFeedPushHistory.feed_history(feed_id, timezone=timezone) @@ -184,7 +185,8 @@ def load_feed_statistics(request, feed_id): def load_feed_settings(request, feed_id): stats = dict() feed = get_object_or_404(Feed, pk=feed_id) - timezone = request.user.profile.timezone + user = get_user(request) + timezone = user.profile.timezone stats['duplicate_addresses'] = feed.duplicate_addresses.all() stats['feed_fetch_history'] = MFeedFetchHistory.feed_history(feed_id, timezone=timezone) diff --git a/media/js/newsblur/reader/reader.js b/media/js/newsblur/reader/reader.js index 1be7277df..b93642311 100644 --- a/media/js/newsblur/reader/reader.js +++ b/media/js/newsblur/reader/reader.js @@ -4012,7 +4012,10 @@ }, feed_unread_count: function(feed_id) { - this.model.feed_unread_count(feed_id || this.active_feed); + feed_id = feed_id || this.active_feed; + if (!feed_id) return; + + this.model.feed_unread_count(feed_id); }, // =================== From 7e52fc37efef9cdc98671bcb82e5c6408f54a65c Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Mon, 28 Jan 2013 15:43:00 -0800 Subject: [PATCH 6/8] Fixing assortment of small bugs. --- apps/analyzer/views.py | 10 +++++----- apps/rss_feeds/models.py | 15 ++++++++++++++- apps/rss_feeds/text_importer.py | 17 +++++++++++------ apps/social/models.py | 14 ++++++++++---- 4 files changed, 40 insertions(+), 16 deletions(-) diff --git a/apps/analyzer/views.py b/apps/analyzer/views.py index fb309d047..664389b50 100644 --- a/apps/analyzer/views.py +++ b/apps/analyzer/views.py @@ -3,7 +3,7 @@ from utils import log as logging from django.shortcuts import get_object_or_404 from django.views.decorators.http import require_POST from django.conf import settings -# from mongoengine.queryset import OperationError +from mongoengine.queryset import NotUniqueError from apps.rss_feeds.models import Feed from apps.reader.models import UserSubscription from apps.analyzer.models import MClassifierTitle, MClassifierAuthor, MClassifierFeed, MClassifierTag @@ -80,10 +80,10 @@ def save_classifier(request): if content_type == 'feed': if not post_content.startswith('social:'): classifier_dict['feed_id'] = post_content - # try: - classifier, created = ClassifierCls.objects.get_or_create(**classifier_dict) - # except OperationError: - # continue + try: + classifier, created = ClassifierCls.objects.get_or_create(**classifier_dict) + except NotUniqueError: + continue if score == 0: classifier.delete() elif classifier.score != score: diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index 2d1d879a3..6f4cff10c 100644 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -1573,6 +1573,7 @@ class MStarredStory(mongo.Document): story_content_z = mongo.BinaryField() story_original_content = mongo.StringField() story_original_content_z = mongo.BinaryField() + original_text_z = mongo.BinaryField() story_content_type = mongo.StringField(max_length=255) story_author_name = mongo.StringField() story_permalink = mongo.StringField() @@ -1611,7 +1612,19 @@ class MStarredStory(mongo.Document): @property def guid_hash(self): return hashlib.sha1(self.story_guid).hexdigest()[:6] - + + def fetch_original_text(self, force=False, request=None): + original_text_z = self.original_text_z + + if not original_text_z or force: + ti = TextImporter(self, request=request) + original_text = ti.fetch() + else: + logging.user(request, "~FYFetching ~FGoriginal~FY story text, ~SBfound.") + original_text = zlib.decompress(original_text_z) + + return original_text + class MFeedFetchHistory(mongo.Document): feed_id = mongo.IntField() diff --git a/apps/rss_feeds/text_importer.py b/apps/rss_feeds/text_importer.py index 8c33b31eb..727afde2d 100644 --- a/apps/rss_feeds/text_importer.py +++ b/apps/rss_feeds/text_importer.py @@ -23,13 +23,18 @@ class TextImporter: 'Connection': 'close', } - def fetch(self): - html = requests.get(self.story.story_permalink, headers=self.headers) - original_text_doc = readability.Document(html.text, url=html.url, debug=settings.DEBUG) - content = original_text_doc.summary(html_partial=True) + def fetch(self, skip_save=False): + try: + html = requests.get(self.story.story_permalink, headers=self.headers) + original_text_doc = readability.Document(html.text, url=html.url, debug=settings.DEBUG) + content = original_text_doc.summary(html_partial=True) + except: + content = None + if content: - self.story.original_text_z = zlib.compress(content) - self.story.save() + if not skip_save: + self.story.original_text_z = zlib.compress(content) + self.story.save() logging.user(self.request, "~SN~FYFetched ~FGoriginal text~FY: now ~SB%s bytes~SN vs. was ~SB%s bytes" % ( len(unicode(content)), self.story.story_content_z and len(zlib.decompress(self.story.story_content_z)) diff --git a/apps/social/models.py b/apps/social/models.py index d42b5c450..f9597f6b8 100644 --- a/apps/social/models.py +++ b/apps/social/models.py @@ -1899,11 +1899,17 @@ class MSharedStory(mongo.Document): return image_sizes - def fetch_original_text(self): - ti = TextImporter(self) - original_text_doc = ti.fetch() + def fetch_original_text(self, force=False, request=None): + original_text_z = self.original_text_z - return original_text_doc + if not original_text_z or force: + ti = TextImporter(self, request=request) + original_text = ti.fetch() + else: + logging.user(request, "~FYFetching ~FGoriginal~FY story text, ~SBfound.") + original_text = zlib.decompress(original_text_z) + + return original_text class MSocialServices(mongo.Document): user_id = mongo.IntField() From 4f6edc2acf39c2f3193c8cfc4e8b396d3e9ba610 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Mon, 28 Jan 2013 16:13:12 -0800 Subject: [PATCH 7/8] Fixing the long broken last update display on story titles header. --- apps/reader/views.py | 2 +- media/js/newsblur/common/assetmodel.js | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/reader/views.py b/apps/reader/views.py index 80cb1b1a0..e067ba85f 100644 --- a/apps/reader/views.py +++ b/apps/reader/views.py @@ -589,7 +589,7 @@ def load_single_feed(request, feed_id): feed_tags=feed_tags, feed_authors=feed_authors, classifiers=classifiers, - last_update=last_update, + updated=last_update, feed_id=feed.pk, elapsed_time=round(float(timediff), 2)) diff --git a/media/js/newsblur/common/assetmodel.js b/media/js/newsblur/common/assetmodel.js index 502a79be0..a719ee4e2 100644 --- a/media/js/newsblur/common/assetmodel.js +++ b/media/js/newsblur/common/assetmodel.js @@ -450,6 +450,11 @@ NEWSBLUR.AssetModel = Backbone.Router.extend({ this.user_profiles.add(profiles); } + if (data.updated) { + var feed = this.get_feed(feed_id); + feed.set('updated', data.updated); + } + if (data.stories && first_load) { this.feed_tags = data.feed_tags || {}; this.feed_authors = data.feed_authors || {}; From 79c6ea09cd37a46abc00707dd6864db931e7a442 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Mon, 28 Jan 2013 16:45:48 -0800 Subject: [PATCH 8/8] Don't collide stories that are too short to be collided. --- apps/rss_feeds/models.py | 12 +++++++++--- utils/feed_fetcher.py | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index 6f4cff10c..d3d558e59 100644 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -791,7 +791,7 @@ class Feed(models.Model): logging.debug(" ---> [%-30s] ~FBChecking ~SB%s~SN new/updated against ~SB%s~SN stories" % ( self.title[:30], len(stories), - len(existing_stories))) + len(existing_stories.keys()))) for story in stories: if not story.get('title'): @@ -1112,12 +1112,13 @@ class Feed(models.Model): story_in_system = None story_has_changed = False story_link = self.get_permalink(story) + existing_stories_guids = existing_stories.keys() # story_pub_date = story.get('published') # story_published_now = story.get('published_now', False) # start_date = story_pub_date - datetime.timedelta(hours=8) # end_date = story_pub_date + datetime.timedelta(hours=8) - for existing_story in existing_stories: + for existing_story in existing_stories.values(): content_ratio = 0 # existing_story_pub_date = existing_story.story_date # print 'Story pub date: %s %s' % (story_published_now, story_pub_date) @@ -1135,7 +1136,10 @@ class Feed(models.Model): if isinstance(existing_story.id, unicode): existing_story.story_guid = existing_story.id - if story.get('guid') and story.get('guid') == existing_story.story_guid: + if (story.get('guid') in existing_stories_guids and + story.get('guid') != existing_story.story_guid): + continue + elif story.get('guid') == existing_story.story_guid: story_in_system = existing_story # Title distance + content distance, checking if story changed @@ -1167,8 +1171,10 @@ class Feed(models.Model): if story_in_system and not story_has_changed: if story_content != existing_story_content: + # print "Content difference - %s/%s" % (story_content, existing_story_content) story_has_changed = True if story_link != existing_story.story_permalink: + # print "Permalink difference - %s/%s" % (story_link, existing_story.story_permalink) story_has_changed = True # if story_pub_date != existing_story.story_date: # story_has_changed = True diff --git a/utils/feed_fetcher.py b/utils/feed_fetcher.py index 102c9c89b..ca21dc4c6 100644 --- a/utils/feed_fetcher.py +++ b/utils/feed_fetcher.py @@ -216,7 +216,7 @@ class ProcessFeed: stories.append(story) story_guids.append(story.get('guid')) - existing_stories = list(MStory.objects( + existing_stories = dict((s.story_guid, s) for s in MStory.objects( # story_guid__in=story_guids, story_date__gte=start_date, story_feed_id=self.feed.pk