From 517248beade7e2fe897969e77bc7cdb5f3dca448 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Tue, 15 Nov 2011 22:14:21 -0800 Subject: [PATCH] Adding two new columns, necessary for the feed_settings branch: hash of address+link, and locking address. --- .../0049_lock_feed_address_and_hash.py | 86 +++++++++++++++++++ apps/rss_feeds/models.py | 13 ++- 2 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 apps/rss_feeds/migrations/0049_lock_feed_address_and_hash.py diff --git a/apps/rss_feeds/migrations/0049_lock_feed_address_and_hash.py b/apps/rss_feeds/migrations/0049_lock_feed_address_and_hash.py new file mode 100644 index 000000000..bcd6ab949 --- /dev/null +++ b/apps/rss_feeds/migrations/0049_lock_feed_address_and_hash.py @@ -0,0 +1,86 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Adding field 'Feed.feed_address_locked' + db.add_column('feeds', 'feed_address_locked', self.gf('django.db.models.fields.NullBooleanField')(default=False, null=True, blank=True), keep_default=False) + + # Adding field 'Feed.hash_address_and_link' + db.add_column('feeds', 'hash_address_and_link', self.gf('django.db.models.fields.CharField')(max_length=64, null=True, blank=True), keep_default=False) + + + def backwards(self, orm): + + # Deleting field 'Feed.feed_address_locked' + db.delete_column('feeds', 'feed_address_locked') + + # Deleting field 'Feed.hash_address_and_link' + db.delete_column('feeds', 'hash_address_and_link') + + + models = { + 'rss_feeds.duplicatefeed': { + 'Meta': {'object_name': 'DuplicateFeed'}, + 'duplicate_address': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'duplicate_feed_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}), + 'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'duplicate_addresses'", 'to': "orm['rss_feeds.Feed']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}) + }, + 'rss_feeds.feed': { + 'Meta': {'ordering': "['feed_title']", 'object_name': 'Feed', 'db_table': "'feeds'"}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}), + 'active_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}), + 'average_stories_per_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'creation': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'days_to_trim': ('django.db.models.fields.IntegerField', [], {'default': '90'}), + 'etag': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'exception_code': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'favicon_color': ('django.db.models.fields.CharField', [], {'max_length': '6', 'null': 'True', 'blank': 'True'}), + 'favicon_not_found': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'feed_address': ('django.db.models.fields.URLField', [], {'unique': 'True', 'max_length': '255'}), + 'feed_address_locked': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}), + 'feed_link': ('django.db.models.fields.URLField', [], {'default': "''", 'max_length': '1000', 'null': 'True', 'blank': 'True'}), + 'feed_link_locked': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'feed_title': ('django.db.models.fields.CharField', [], {'default': "'[Untitled]'", 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'fetched_once': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'has_feed_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'has_page': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_page_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'hash_address_and_link': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_load_time': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'last_modified': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'last_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'min_to_decay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'next_scheduled_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'num_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'queued_date': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'stories_last_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}) + }, + 'rss_feeds.feeddata': { + 'Meta': {'object_name': 'FeedData'}, + 'feed': ('utils.fields.AutoOneToOneField', [], {'related_name': "'data'", 'unique': 'True', 'to': "orm['rss_feeds.Feed']"}), + 'feed_classifier_counts': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'feed_tagline': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'popular_authors': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}), + 'popular_tags': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}), + 'story_count_history': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) + }, + 'rss_feeds.feedloadtime': { + 'Meta': {'object_name': 'FeedLoadtime'}, + 'date_accessed': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'loadtime': ('django.db.models.fields.FloatField', [], {}) + } + } + + complete_apps = ['rss_feeds'] diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index 56ca5486a..6272d91ba 100644 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -7,6 +7,7 @@ import mongoengine as mongo import redis import zlib import urllib +import hashlib from collections import defaultdict from operator import itemgetter # from nltk.collocations import TrigramCollocationFinder, BigramCollocationFinder, TrigramAssocMeasures, BigramAssocMeasures @@ -36,8 +37,10 @@ ENTRY_NEW, ENTRY_UPDATED, ENTRY_SAME, ENTRY_ERR = range(4) class Feed(models.Model): feed_address = models.URLField(max_length=255, verify_exists=True, unique=True) + feed_address_locked = models.NullBooleanField(default=False, blank=True, null=True) feed_link = models.URLField(max_length=1000, default="", blank=True, null=True) feed_link_locked = models.BooleanField(default=False) + hash_address_and_link = models.CharField(max_length=64, blank=True, null=True) feed_title = models.CharField(max_length=255, default="[Untitled]", blank=True, null=True) active = models.BooleanField(default=True, db_index=True) num_subscribers = models.IntegerField(default=-1) @@ -61,6 +64,11 @@ class Feed(models.Model): last_load_time = models.IntegerField(default=0) favicon_color = models.CharField(max_length=6, null=True, blank=True) favicon_not_found = models.BooleanField(default=False) + + class Meta: + db_table="feeds" + ordering=["feed_title"] + # unique_together=[('feed_address', 'feed_link')] def __unicode__(self): if not self.feed_title: @@ -116,6 +124,7 @@ class Feed(models.Model): self.next_scheduled_update = datetime.datetime.utcnow() if not self.queued_date: self.queued_date = datetime.datetime.utcnow() + self.hash_address_and_link = hashlib.sha1(self.feed_address+self.feed_link).hexdigest() max_feed_title = Feed._meta.get_field('feed_title').max_length if len(self.feed_title) > max_feed_title: @@ -992,10 +1001,6 @@ class Feed(models.Model): # phrases = [' '.join(phrase) for phrase in best] # # return phrases - - class Meta: - db_table="feeds" - ordering=["feed_title"] # class FeedCollocations(models.Model): # feed = models.ForeignKey(Feed)