From b66080ca4da3505da87d1d70f6d02fe8bc50516c Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Tue, 9 Nov 2010 09:55:44 -0500 Subject: [PATCH] Fixing how duplicate feeds are handled and found. When a subscription is not found, check for duplicate feeds and then switch accordingly. When merging feeds, don't forget to bring along the dupe feed's dupe feeds. --- apps/reader/models.py | 23 ++- apps/reader/views.py | 16 +- .../rss_feeds/migrations/0030_dupe_feed_id.py | 141 ++++++++++++++++++ apps/rss_feeds/models.py | 10 +- 4 files changed, 182 insertions(+), 8 deletions(-) create mode 100644 apps/rss_feeds/migrations/0030_dupe_feed_id.py diff --git a/apps/reader/models.py b/apps/reader/models.py index bb7aa5160..cd6624db4 100644 --- a/apps/reader/models.py +++ b/apps/reader/models.py @@ -2,11 +2,11 @@ import datetime import mongoengine as mongo from utils import log as logging from utils import json_functions as json -from django.db import models +from django.db import models, IntegrityError from django.conf import settings from django.contrib.auth.models import User from django.core.cache import cache -from apps.rss_feeds.models import Feed, Story, MStory +from apps.rss_feeds.models import Feed, Story, MStory, DuplicateFeed from apps.analyzer.models import MClassifierFeed, MClassifierAuthor, MClassifierTag, MClassifierTitle from apps.analyzer.models import apply_classifier_titles, apply_classifier_feeds, apply_classifier_authors, apply_classifier_tags @@ -36,8 +36,14 @@ class UserSubscription(models.Model): return '[' + self.feed.feed_title + '] ' def save(self, *args, **kwargs): - if self.feed: + try: super(UserSubscription, self).save(*args, **kwargs) + except IntegrityError: + duplicate_feed = DuplicateFeed.objects.filter(duplicate_feed_id=self.feed.pk) + if duplicate_feed: + self.feed = duplicate_feed[0].feed + super(UserSubscription, self).save(*args, **kwargs) + def mark_feed_read(self): now = datetime.datetime.utcnow() @@ -256,7 +262,16 @@ class UserSubscriptionFolders(models.Model): self.save() if not multiples_found and deleted: - user_sub = UserSubscription.objects.get(user=self.user, feed=feed_id) + try: + user_sub = UserSubscription.objects.get(user=self.user, feed=feed_id) + except Feed.DoesNotExist: + duplicate_feed = DuplicateFeed.objects.filter(duplicate_feed_id=feed_id) + if duplicate_feed: + try: + user_sub = UserSubscription.objects.get(user=self.user, + feed=duplicate_feed[0].feed) + except Feed.DoesNotExist: + return user_sub.delete() MUserStory.objects(user_id=self.user.pk, feed_id=feed_id).delete() diff --git a/apps/reader/views.py b/apps/reader/views.py index a3c82b093..46a915391 100644 --- a/apps/reader/views.py +++ b/apps/reader/views.py @@ -375,8 +375,18 @@ def mark_all_as_read(request): def mark_story_as_read(request): story_ids = request.REQUEST.getlist('story_id') feed_id = int(request.REQUEST['feed_id']) - - usersub = UserSubscription.objects.select_related('feed').get(user=request.user, feed=feed_id) + + try: + usersub = UserSubscription.objects.select_related('feed').get(user=request.user, feed=feed_id) + except Feed.DoesNotExist: + duplicate_feed = DuplicateFeed.objects.filter(duplicate_feed_id=feed_id) + if duplicate_feed: + try: + usersub = UserSubscription.objects.get(user=request.user, + feed=duplicate_feed[0].feed) + except Feed.DoesNotExist: + return dict(code=-1) + if not usersub.needs_unread_recalc: usersub.needs_unread_recalc = True usersub.save() @@ -632,7 +642,7 @@ def save_feed_chooser(request): activated = 0 usersubs = UserSubscription.objects.filter(user=request.user) for sub in usersubs: - if sub.feed and sub.feed.pk in approved_feeds: + if sub.feed.pk in approved_feeds: sub.active = True activated += 1 sub.save() diff --git a/apps/rss_feeds/migrations/0030_dupe_feed_id.py b/apps/rss_feeds/migrations/0030_dupe_feed_id.py new file mode 100644 index 000000000..8868ef275 --- /dev/null +++ b/apps/rss_feeds/migrations/0030_dupe_feed_id.py @@ -0,0 +1,141 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Changing field 'Feed.premium_subscribers' + db.alter_column('feeds', 'premium_subscribers', self.gf('django.db.models.fields.IntegerField')()) + + # Adding field 'DuplicateFeed.duplicate_feed_id' + db.add_column('rss_feeds_duplicatefeed', 'duplicate_feed_id', self.gf('django.db.models.fields.CharField')(max_length=255, null=True), keep_default=False) + + + def backwards(self, orm): + + # Changing field 'Feed.premium_subscribers' + db.alter_column('feeds', 'premium_subscribers', self.gf('django.db.models.fields.IntegerField')(null=True)) + + # Deleting field 'DuplicateFeed.duplicate_feed_id' + db.delete_column('rss_feeds_duplicatefeed', 'duplicate_feed_id') + + + models = { + 'rss_feeds.duplicatefeed': { + 'Meta': {'object_name': 'DuplicateFeed'}, + 'duplicate_address': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'duplicate_feed_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}), + 'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'duplicate_addresses'", 'to': "orm['rss_feeds.Feed']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}) + }, + 'rss_feeds.feed': { + 'Meta': {'ordering': "['feed_title']", 'object_name': 'Feed', 'db_table': "'feeds'"}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'active_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'average_stories_per_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'creation': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'days_to_trim': ('django.db.models.fields.IntegerField', [], {'default': '90'}), + 'etag': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'exception_code': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'feed_address': ('django.db.models.fields.URLField', [], {'unique': 'True', 'max_length': '255'}), + 'feed_link': ('django.db.models.fields.URLField', [], {'default': "''", 'max_length': '1000', 'null': 'True', 'blank': 'True'}), + 'feed_tagline': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1024', 'null': 'True', 'blank': 'True'}), + 'feed_title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'fetched_once': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'has_feed_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'has_page_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_load_time': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'last_modified': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'last_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'min_to_decay': ('django.db.models.fields.IntegerField', [], {'default': '15'}), + 'next_scheduled_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'num_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'popular_authors': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}), + 'popular_tags': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}), + 'premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'queued_date': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'stories_last_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'story_count_history': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) + }, + 'rss_feeds.feedfetchhistory': { + 'Meta': {'object_name': 'FeedFetchHistory'}, + 'exception': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'feed_fetch_history'", 'to': "orm['rss_feeds.Feed']"}), + 'fetch_date': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'status_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}) + }, + 'rss_feeds.feedloadtime': { + 'Meta': {'object_name': 'FeedLoadtime'}, + 'date_accessed': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'loadtime': ('django.db.models.fields.FloatField', [], {}) + }, + 'rss_feeds.feedpage': { + 'Meta': {'object_name': 'FeedPage'}, + 'feed': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'feed_page'", 'unique': 'True', 'to': "orm['rss_feeds.Feed']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'page_data': ('utils.compressed_textfield.StoryField', [], {'null': 'True', 'blank': 'True'}) + }, + 'rss_feeds.feedupdatehistory': { + 'Meta': {'object_name': 'FeedUpdateHistory'}, + 'average_per_feed': ('django.db.models.fields.DecimalField', [], {'max_digits': '4', 'decimal_places': '1'}), + 'fetch_date': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'number_of_feeds': ('django.db.models.fields.IntegerField', [], {}), + 'seconds_taken': ('django.db.models.fields.IntegerField', [], {}) + }, + 'rss_feeds.feedxml': { + 'Meta': {'object_name': 'FeedXML'}, + 'feed': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'feed_xml'", 'unique': 'True', 'to': "orm['rss_feeds.Feed']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'rss_xml': ('utils.compressed_textfield.StoryField', [], {'null': 'True', 'blank': 'True'}) + }, + 'rss_feeds.pagefetchhistory': { + 'Meta': {'object_name': 'PageFetchHistory'}, + 'exception': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'page_fetch_history'", 'to': "orm['rss_feeds.Feed']"}), + 'fetch_date': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'status_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}) + }, + 'rss_feeds.story': { + 'Meta': {'ordering': "['-story_date']", 'unique_together': "(('story_feed', 'story_guid_hash'),)", 'object_name': 'Story', 'db_table': "'stories'"}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'story_author': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.StoryAuthor']"}), + 'story_author_name': ('django.db.models.fields.CharField', [], {'max_length': '500', 'null': 'True', 'blank': 'True'}), + 'story_content': ('utils.compressed_textfield.StoryField', [], {'null': 'True', 'blank': 'True'}), + 'story_content_type': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'story_date': ('django.db.models.fields.DateTimeField', [], {}), + 'story_feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'stories'", 'to': "orm['rss_feeds.Feed']"}), + 'story_guid': ('django.db.models.fields.CharField', [], {'max_length': '1000'}), + 'story_guid_hash': ('django.db.models.fields.CharField', [], {'max_length': '40'}), + 'story_original_content': ('utils.compressed_textfield.StoryField', [], {'null': 'True', 'blank': 'True'}), + 'story_past_trim_date': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'story_permalink': ('django.db.models.fields.CharField', [], {'max_length': '1000'}), + 'story_tags': ('django.db.models.fields.CharField', [], {'max_length': '2000', 'null': 'True', 'blank': 'True'}), + 'story_title': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + }, + 'rss_feeds.storyauthor': { + 'Meta': {'object_name': 'StoryAuthor'}, + 'author_name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}) + }, + 'rss_feeds.tag': { + 'Meta': {'object_name': 'Tag'}, + 'feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + } + } + + complete_apps = ['rss_feeds'] diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index 3c00204fb..32a1d9747 100644 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -886,8 +886,8 @@ class FeedLoadtime(models.Model): class DuplicateFeed(models.Model): duplicate_address = models.CharField(max_length=255, unique=True) + duplicate_feed_id = models.CharField(max_length=255, null=True) feed = models.ForeignKey(Feed, related_name='duplicate_addresses') - def merge_feeds(original_feed_id, duplicate_feed_id): from apps.reader.models import UserSubscription, UserSubscriptionFolders, MUserStory @@ -974,11 +974,19 @@ def merge_feeds(original_feed_id, duplicate_feed_id): try: DuplicateFeed.objects.create( duplicate_address=duplicate_feed.feed_address, + duplicate_feed_id=duplicate_feed.pk, feed=original_feed ) except (IntegrityError, OperationError), e: logging.info(" ***> Could not save DuplicateFeed: %s" % e) + # Switch this dupe feed's dupe feeds over to the new original. + duplicate_feeds_duplicate_feeds = DuplicateFeed.objects.filter(feed=duplicate_feed) + for dupe_feed in duplicate_feeds_duplicate_feeds: + dupe_feed.feed = original_feed + dupe_feed.duplicate_feed_id = duplicate_feed.pk + dupe_feed.save() + duplicate_feed.delete()