From cb30fe2ae8fddd40a26d536f536b11d323e9e38e Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Mon, 7 Jan 2013 16:35:29 -0800 Subject: [PATCH] Bumping up duplicate address length for feeds. --- .../migrations/0062_dupe_feed_maxlen.py | 86 +++++++++++++++++++ apps/rss_feeds/models.py | 4 +- utils/feed_fetcher.py | 6 +- utils/rtail.py | 6 +- 4 files changed, 96 insertions(+), 6 deletions(-) create mode 100644 apps/rss_feeds/migrations/0062_dupe_feed_maxlen.py diff --git a/apps/rss_feeds/migrations/0062_dupe_feed_maxlen.py b/apps/rss_feeds/migrations/0062_dupe_feed_maxlen.py new file mode 100644 index 000000000..b7abe5d61 --- /dev/null +++ b/apps/rss_feeds/migrations/0062_dupe_feed_maxlen.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Changing field 'DuplicateFeed.duplicate_link' + db.alter_column('rss_feeds_duplicatefeed', 'duplicate_link', self.gf('django.db.models.fields.CharField')(max_length=764, null=True)) + + # Changing field 'DuplicateFeed.duplicate_address' + db.alter_column('rss_feeds_duplicatefeed', 'duplicate_address', self.gf('django.db.models.fields.CharField')(max_length=764)) + + def backwards(self, orm): + + # Changing field 'DuplicateFeed.duplicate_link' + db.alter_column('rss_feeds_duplicatefeed', 'duplicate_link', self.gf('django.db.models.fields.CharField')(max_length=255, null=True)) + + # Changing field 'DuplicateFeed.duplicate_address' + db.alter_column('rss_feeds_duplicatefeed', 'duplicate_address', self.gf('django.db.models.fields.CharField')(max_length=255)) + + models = { + 'rss_feeds.duplicatefeed': { + 'Meta': {'object_name': 'DuplicateFeed'}, + 'duplicate_address': ('django.db.models.fields.CharField', [], {'max_length': '764', 'db_index': 'True'}), + 'duplicate_feed_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'db_index': 'True'}), + 'duplicate_link': ('django.db.models.fields.CharField', [], {'max_length': '764', 'null': 'True', 'db_index': 'True'}), + 'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'duplicate_addresses'", 'to': "orm['rss_feeds.Feed']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}) + }, + 'rss_feeds.feed': { + 'Meta': {'ordering': "['feed_title']", 'object_name': 'Feed', 'db_table': "'feeds'"}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}), + 'active_premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}), + 'active_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}), + 'average_stories_per_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'branch_from_feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']", 'null': 'True', 'blank': 'True'}), + 'creation': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'days_to_trim': ('django.db.models.fields.IntegerField', [], {'default': '90'}), + 'errors_since_good': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'etag': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'exception_code': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'favicon_color': ('django.db.models.fields.CharField', [], {'max_length': '6', 'null': 'True', 'blank': 'True'}), + 'favicon_not_found': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'feed_address': ('django.db.models.fields.URLField', [], {'max_length': '764', 'db_index': 'True'}), + 'feed_address_locked': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}), + 'feed_link': ('django.db.models.fields.URLField', [], {'default': "''", 'max_length': '1000', 'null': 'True', 'blank': 'True'}), + 'feed_link_locked': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'feed_title': ('django.db.models.fields.CharField', [], {'default': "'[Untitled]'", 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'fetched_once': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'has_feed_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'has_page': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_page_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'hash_address_and_link': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_push': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}), + 'known_good': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'last_load_time': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'last_modified': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'last_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'min_to_decay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'next_scheduled_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'num_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'queued_date': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 's3_icon': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}), + 's3_page': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}), + 'stories_last_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}) + }, + 'rss_feeds.feeddata': { + 'Meta': {'object_name': 'FeedData'}, + 'feed': ('utils.fields.AutoOneToOneField', [], {'related_name': "'data'", 'unique': 'True', 'to': "orm['rss_feeds.Feed']"}), + 'feed_classifier_counts': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'feed_tagline': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'popular_authors': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}), + 'popular_tags': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}), + 'story_count_history': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) + } + } + + complete_apps = ['rss_feeds'] \ No newline at end of file diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py index 7b088f22a..aa7a7d423 100644 --- a/apps/rss_feeds/models.py +++ b/apps/rss_feeds/models.py @@ -1713,8 +1713,8 @@ class MFeedPushHistory(mongo.Document): class DuplicateFeed(models.Model): - duplicate_address = models.CharField(max_length=255, db_index=True) - duplicate_link = models.CharField(max_length=255, null=True, db_index=True) + duplicate_address = models.CharField(max_length=764, db_index=True) + duplicate_link = models.CharField(max_length=764, null=True, db_index=True) duplicate_feed_id = models.CharField(max_length=255, null=True, db_index=True) feed = models.ForeignKey(Feed, related_name='duplicate_addresses') diff --git a/utils/feed_fetcher.py b/utils/feed_fetcher.py index 7340c675d..5f4a2e755 100644 --- a/utils/feed_fetcher.py +++ b/utils/feed_fetcher.py @@ -381,7 +381,7 @@ class Dispatcher: mail_feed_error_to_admin(feed, e, local_vars=locals()) if (not settings.DEBUG and hasattr(settings, 'RAVEN_CLIENT') and settings.RAVEN_CLIENT): - settings.RAVEN_CLIENT.captureException(e) + settings.RAVEN_CLIENT.captureException() if not feed_code: if ret_feed == FEED_OK: @@ -424,7 +424,7 @@ class Dispatcher: fetched_feed = None page_data = None mail_feed_error_to_admin(feed, e, local_vars=locals()) - settings.RAVEN_CLIENT.captureException(e) + settings.RAVEN_CLIENT.captureException() feed = self.refresh_feed(feed.pk) logging.debug(u' ---> [%-30s] ~FYFetching icon: %s' % (feed.title[:30], feed.feed_link)) @@ -442,7 +442,7 @@ class Dispatcher: logging.debug('[%d] ! -------------------------' % (feed_id,)) # feed.save_feed_history(560, "Icon Error", tb) mail_feed_error_to_admin(feed, e, local_vars=locals()) - settings.RAVEN_CLIENT.captureException(e) + settings.RAVEN_CLIENT.captureException() else: logging.debug(u' ---> [%-30s] ~FBSkipping page fetch: (%s on %s stories) %s' % (feed.title[:30], self.feed_trans[ret_feed], feed.stories_last_month, '' if feed.has_page else ' [HAS NO PAGE]')) diff --git a/utils/rtail.py b/utils/rtail.py index eb21f45b6..237463c0a 100755 --- a/utils/rtail.py +++ b/utils/rtail.py @@ -16,13 +16,17 @@ import sys def main(): op = optparse.OptionParser() + op.add_option("-i", "--identity", dest="identity") options, args = op.parse_args() streams = list() for arg in args: if re.match(r"^(.+@)?[a-zA-Z0-9.-]+:.+", arg): # this is a remote location hostname, path = arg.split(":", 1) - s = subprocess.Popen(["ssh", hostname, "tail -f " + path], stdout=subprocess.PIPE) + if options.identity: + s = subprocess.Popen(["ssh", "-i", options.identity, hostname, "tail -f " + path], stdout=subprocess.PIPE) + else: + s = subprocess.Popen(["ssh", hostname, "tail -f " + path], stdout=subprocess.PIPE) s.name = arg streams.append(s) else: