Merge branch 'master' into circular

* master: Fixing story hash migration to run by user. Adding story_hash to starred stories, user stories, and shared stories. Adding redis backup script.
2025-09-18 21:50:56 +00:00 · 2013-04-29 18:00:55 -07:00 · 2013-04-29 18:00:55 -07:00 · 97041dece7
commit 97041dece7
parent 3aed74e44f bbd430ec59
6 changed files with 145 additions and 12 deletions
--- a/apps/reader/models.py
+++ b/apps/reader/models.py
@ -591,6 +591,7 @@ class MUserStory(mongo.Document):
    feed_id = mongo.IntField()
    read_date = mongo.DateTimeField()
    story_id = mongo.StringField()
+    story_hash = mongo.StringField()
    story_date = mongo.DateTimeField()
    story = mongo.ReferenceField(MStory, dbref=True)
    found_story = mongo.GenericReferenceField()
@ -610,6 +611,8 @@ class MUserStory(mongo.Document):
    }
    
    def save(self, *args, **kwargs):
+        self.story_hash = self.feed_guid_hash
+        
        self.sync_redis()
        
        super(MUserStory, self).save(*args, **kwargs)
@ -625,7 +628,7 @@ class MUserStory(mongo.Document):
    
    @property
    def feed_guid_hash(self):
-        return "%s:%s" % (self.feed_id, self.guid_hash)
+        return "%s:%s" % (self.feed_id or "0", self.guid_hash)
    
    @classmethod
    def delete_old_stories(cls, feed_id):
--- a/apps/rss_feeds/migrations/0069_story_hash_fields.py
+++ b/apps/rss_feeds/migrations/0069_story_hash_fields.py
@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+from south.v2 import DataMigration
+from django.contrib.auth.models import User
+from django.conf import settings
+import pymongo
+
+class Migration(DataMigration):
+
+    def forwards(self, orm):
+        from apps.rss_feeds.models import MStarredStory
+        from apps.social.models import MSharedStory
+        db = settings.MONGODB
+
+        starred_count = MStarredStory.objects.count()
+        print " ---> Saving %s starred stories..." % starred_count
+        shared_count = MSharedStory.objects.count()
+        print " ---> Saving %s shared stories..." % shared_count
+
+        start = 0
+        user_count = User.objects.latest('pk').pk
+        for user_id in xrange(start, user_count):
+            if user_id % 1000 == 0:
+                print " ---> %s/%s" % (user_id, user_count)
+
+            stories = MStarredStory.objects(user_id=user_id, story_hash__exists=False)\
+                                .only('id', 'story_feed_id', 'story_guid')\
+                                .read_preference(
+                                    pymongo.ReadPreference.SECONDARY
+                                )
+            for i, story in enumerate(stories):
+                
+                db.prod_newsblur.starred_stories.update({"_id": story.id}, {"$set": {
+                    "story_hash": story.feed_guid_hash
+                }})
+            stories = MSharedStory.objects(user_id=user_id, story_hash__exists=False)\
+                                .only('id', 'user_id', 'story_feed_id', 'story_guid')\
+                                .read_preference(
+                                    pymongo.ReadPreference.SECONDARY
+                                )
+            for i, story in enumerate(stories):
+                db.prod_newsblur.shared_stories.update({"_id": story.id}, {"$set": {
+                    "story_hash": story.feed_guid_hash
+                }})
+        
+
+    def backwards(self, orm):
+        "Write your backwards methods here."
+
+    models = {
+        u'rss_feeds.duplicatefeed': {
+            'Meta': {'object_name': 'DuplicateFeed'},
+            'duplicate_address': ('django.db.models.fields.CharField', [], {'max_length': '764', 'db_index': 'True'}),
+            'duplicate_feed_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'db_index': 'True'}),
+            'duplicate_link': ('django.db.models.fields.CharField', [], {'max_length': '764', 'null': 'True', 'db_index': 'True'}),
+            'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'duplicate_addresses'", 'to': u"orm['rss_feeds.Feed']"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        },
+        u'rss_feeds.feed': {
+            'Meta': {'ordering': "['feed_title']", 'object_name': 'Feed', 'db_table': "'feeds'"},
+            'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
+            'active_premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+            'active_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}),
+            'average_stories_per_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'branch_from_feed': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['rss_feeds.Feed']", 'null': 'True', 'blank': 'True'}),
+            'creation': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'days_to_trim': ('django.db.models.fields.IntegerField', [], {'default': '90'}),
+            'errors_since_good': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'etag': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'exception_code': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'favicon_color': ('django.db.models.fields.CharField', [], {'max_length': '6', 'null': 'True', 'blank': 'True'}),
+            'favicon_not_found': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'feed_address': ('django.db.models.fields.URLField', [], {'max_length': '764', 'db_index': 'True'}),
+            'feed_address_locked': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
+            'feed_link': ('django.db.models.fields.URLField', [], {'default': "''", 'max_length': '1000', 'null': 'True', 'blank': 'True'}),
+            'feed_link_locked': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'feed_title': ('django.db.models.fields.CharField', [], {'default': "'[Untitled]'", 'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'fetched_once': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'has_feed_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'has_page': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'has_page_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'hash_address_and_link': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_push': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
+            'known_good': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_load_time': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'last_modified': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'last_story_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'last_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+            'min_to_decay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'next_scheduled_update': ('django.db.models.fields.DateTimeField', [], {}),
+            'num_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+            'premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+            's3_icon': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
+            's3_page': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
+            'stories_last_month': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
+        u'rss_feeds.feeddata': {
+            'Meta': {'object_name': 'FeedData'},
+            'feed': ('utils.fields.AutoOneToOneField', [], {'related_name': "'data'", 'unique': 'True', 'to': u"orm['rss_feeds.Feed']"}),
+            'feed_classifier_counts': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'feed_tagline': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'popular_authors': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}),
+            'popular_tags': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
+            'story_count_history': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['rss_feeds']
+    symmetrical = True
--- a/apps/rss_feeds/models.py
+++ b/apps/rss_feeds/models.py
@ -1723,6 +1723,7 @@ class MStarredStory(mongo.Document):
    story_author_name        = mongo.StringField()
    story_permalink          = mongo.StringField()
    story_guid               = mongo.StringField()
+    story_hash               = mongo.StringField()
    story_tags               = mongo.ListField(mongo.StringField(max_length=250))

    meta = {
@ -1740,6 +1741,8 @@ class MStarredStory(mongo.Document):
        if self.story_original_content:
            self.story_original_content_z = zlib.compress(self.story_original_content)
            self.story_original_content = None
+        self.story_hash = self.feed_guid_hash
+        
        super(MStarredStory, self).save(*args, **kwargs)

        # self.index_for_search()
@ -1758,6 +1761,10 @@ class MStarredStory(mongo.Document):
    def guid_hash(self):
        return hashlib.sha1(self.story_guid).hexdigest()[:6]

+    @property
+    def feed_guid_hash(self):
+        return "%s:%s" % (self.story_feed_id or "0", self.guid_hash)
+    
    def fetch_original_text(self, force=False, request=None):
        original_text_z = self.original_text_z
        
--- a/apps/social/models.py
+++ b/apps/social/models.py
@ -1250,6 +1250,7 @@ class MSharedStory(mongo.Document):
    replies                  = mongo.ListField(mongo.EmbeddedDocumentField(MCommentReply))
    source_user_id           = mongo.IntField()
    story_db_id              = mongo.ObjectIdField()
+    story_hash               = mongo.StringField()
    story_feed_id            = mongo.IntField()
    story_date               = mongo.DateTimeField()
    story_title              = mongo.StringField(max_length=1024)
@ -1292,17 +1293,11 @@ class MSharedStory(mongo.Document):

    @property
    def guid_hash(self):
-        if self.story_guid_hash:
-            return self.story_guid_hash
-        
-        self.story_guid_hash = hashlib.sha1(self.story_guid).hexdigest()[:6]
-        self.save()
-        
-        return self.story_guid_hash
+        return hashlib.sha1(self.story_guid).hexdigest()[:6]
    
    @property
    def feed_guid_hash(self):
-        return "%s:%s" % (self.story_feed_id, self.guid_hash)
+        return "%s:%s" % (self.story_feed_id or "0", self.guid_hash)
    
    def to_json(self):
        return {
@ -1326,7 +1321,8 @@ class MSharedStory(mongo.Document):

        self.story_guid_hash = hashlib.sha1(self.story_guid).hexdigest()[:6]
        self.story_title = strip_tags(self.story_title)
-        
+        self.story_hash = self.feed_guid_hash
+
        self.comments = linkify(strip_tags(self.comments))
        for reply in self.replies:
            reply.comments = linkify(strip_tags(reply.comments))
--- a/utils/backups/backup_redis.py
+++ b/utils/backups/backup_redis.py
@ -0,0 +1,17 @@
+import os
+import sys
+
+CURRENT_DIR  = os.path.dirname(__file__)
+NEWSBLUR_DIR = ''.join([CURRENT_DIR, '/../../'])
+sys.path.insert(0, NEWSBLUR_DIR)
+os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
+
+import time
+import s3
+from django.conf import settings
+
+filename = 'backup_redis_%s.rdb.gz' % time.strftime('%Y-%m-%d-%H-%M')
+path = '/var/lib/redis/dump.rdb'
+print 'Uploading %s / %s to S3...' % (filename, path)
+s3.save_file_in_s3(path, name=filename)
+os.remove(filename)
--- a/utils/backups/s3.py
+++ b/utils/backups/s3.py
@ -13,11 +13,11 @@ ACCESS_KEY  = settings.S3_ACCESS_KEY
 SECRET      = settings.S3_SECRET
 BUCKET_NAME = settings.S3_BACKUP_BUCKET  # Note that you need to create this bucket first

-def save_file_in_s3(filename):
+def save_file_in_s3(filename, name=None):
    conn   = S3Connection(ACCESS_KEY, SECRET)
    bucket = conn.get_bucket(BUCKET_NAME)
    k      = Key(bucket)
-    k.key  = filename
+    k.key  = name or filename

    k.set_contents_from_filename(filename)