Adding mongo backup scripts to handle db lock, exporting data, compressing, and uploading to s3.

requirements.txt

@@ -13,3 +13,4 @@ nltk
 lxml
 oauth2
 pytz
+boto

utils/backups/fsync_lock.js

@@ -0,0 +1,9 @@
+function freeze_db() {
+        rc = db.runCommand({fsync: 1, lock: 1});
+        if (rc.ok == 1){
+            return 1;
+        } else {
+            return 0;
+        }
+    }
+freeze_db();

utils/backups/fsync_unlock.js

@@ -0,0 +1 @@
+db.$cmd.sys.unlock.findOne();

utils/backups/mongo_backupo.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+
+MONGODB_SHELL = '/usr/bin/mongo'
+DUMP_UTILITY  = '/usr/bin/mongodump'
+DB_NAME       = 'newsblur'
+COLLECTIONS   = "classifier_tag classifier_author classifier_feed classifier_title"
+
+date_now      = `date +%Y_%m_%d_%H_%M_%S`
+dir_name      = 'mongo_backup_'${date_now}
+file_name     = 'mongo_backup_'${date_now}'.bz2'
+
+log() {
+    echo $1
+}
+
+do_cleanup(){
+    rm -rf db_backup_2010* 
+    log 'cleaning up....'
+}
+
+do_backup(){
+    log 'snapshotting the db and creating archive' && \
+    ${MONGODB_SHELL} admin fsync_lock.js && \
+    for collection in $COLLECTIONS ;
+    do
+        ${DUMP_UTILITY} -d ${DB_NAME} -o ${dir_name} -c $collection
+    done &&
+    tar -jcf $file_name ${dir_name}
+    ${MONGODB_SHELL} admin unlock.js && \
+    log 'data backd up and created snapshot'
+}
+
+save_in_s3(){
+    log 'saving the backup archive in amazon S3' && \
+    python aws_s3.py set ${file_name} && \
+    log 'data backup saved in amazon s3'
+}
+
+do_backup #&& save_in_s3 && do_cleanup

utils/backups/s3.py

@@ -0,0 +1,55 @@
+from boto.s3.connection import S3Connection
+from boto.s3.key import Key
+from django.conf import settings
+
+ACCESS_KEY  = settings.S3_ACCESS_KEY
+SECRET      = settings.S3_SECRET
+BUCKET_NAME = settings.S3_BACKUP_BUCKET  # Note that you need to create this bucket first
+
+def save_file_in_s3(filename):
+    conn   = S3Connection(ACCESS_KEY, SECRET)
+    bucket = conn.get_bucket(BUCKET_NAME)
+    k      = Key(bucket)
+    k.key  = filename
+
+    k.set_contents_from_filename(filename)
+
+def get_file_from_s3(filename):
+    conn   = S3Connection(ACCESS_KEY, SECRET)
+    bucket = conn.get_bucket(BUCKET_NAME)
+    k      = Key(bucket)
+    k.key  = filename
+
+    k.get_contents_to_filename(filename)
+
+def list_backup_in_s3():
+    conn   = S3Connection(ACCESS_KEY, SECRET)
+    bucket = conn.get_bucket(BUCKET_NAME)
+
+    for i, key in enumerate(bucket.get_all_keys()):
+        print "[%s] %s" % (i, key.name)
+
+def delete_all_backups():
+    #FIXME: validate filename exists
+    conn   = S3Connection(ACCESS_KEY, SECRET)
+    bucket = conn.get_bucket(BUCKET_NAME)
+
+    for i, key in enumerate(bucket.get_all_keys()):
+        print "deleting %s" % (key.name)
+        key.delete()
+
+if __name__ == '__main__':
+    import sys
+    if len(sys.argv) < 3:
+        print 'Usage: %s <get/set/list/delete> <backup_filename>' % (sys.argv[0])
+    else:
+        if sys.argv[1] == 'set':
+            save_file_in_s3(sys.argv[2])
+        elif sys.argv[1] == 'get':
+            get_file_from_s3(sys.argv[2])
+        elif sys.argv[1] == 'list':
+            list_backup_in_s3()
+        elif sys.argv[1] == 'delete':
+            delete_all_backups()
+        else:
+            print 'Usage: %s <get/set/list/delete> <backup_filename>' % (sys.argv[0])