From afee20edca42d7277041791ed2f63255c7b54e31 Mon Sep 17 00:00:00 2001
From: Samuel Clay <samuel@ofbrooklyn.com>
Date: Wed, 11 Feb 2015 17:30:15 -0800
Subject: [PATCH] Adding disk usage monitor to all database servers.

---
 fabfile.py                  | 10 +++++++++-
 utils/monitor_disk_usage.py | 29 +++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+), 1 deletion(-)
 create mode 100755 utils/monitor_disk_usage.py

diff --git a/fabfile.py b/fabfile.py
index fba292b3b..943f82d27 100644
--- a/fabfile.py
+++ b/fabfile.py
@@ -583,7 +583,7 @@ def setup_ulimit():
     sudo('sysctl -p')
     sudo('ulimit -n 100000')
     connections.connect(env.host_string)
-
+    
     # run('touch /home/ubuntu/.bash_profile')
     # run('echo "ulimit -n $FILEMAX" >> /home/ubuntu/.bash_profile')
 
@@ -881,6 +881,9 @@ def setup_mongo():
     sudo('/etc/init.d/mongodb restart')
     put('config/logrotate.mongo.conf', '/etc/logrotate.d/mongodb', use_sudo=True)
 
+    # Reclaim 5% disk space used for root logs. Set to 1%.
+    sudo('tune2fs -m 1 /dev/vda')
+
 def setup_mongo_configsvr():
     sudo('mkdir -p /var/lib/mongodb_configsvr')
     sudo('chown mongodb.mongodb /var/lib/mongodb_configsvr')
@@ -1039,6 +1042,11 @@ def setup_db_search():
     put('config/supervisor_celeryd_search_indexer_tasker.conf', '/etc/supervisor/conf.d/celeryd_search_indexer_tasker.conf', use_sudo=True)
     sudo('supervisorctl reread')
     sudo('supervisorctl update')
+
+def setup_usage_monitor():
+    sudo('ln -s %s/utils/monitor_disk_usage.py /etc/cron.daily/monitor_disk_usage' % env.NEWSBLUR_PATH)
+    with settings(warn_only=True):
+        sudo('/etc/cron.daily/monitor_disk_usage')
     
 # ================
 # = Setup - Task =
diff --git a/utils/monitor_disk_usage.py b/utils/monitor_disk_usage.py
new file mode 100755
index 000000000..63e794bc0
--- /dev/null
+++ b/utils/monitor_disk_usage.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+
+import sys
+
+if '/srv/newsblur' not in ' '.join(sys.path):
+    sys.path.append('/srv/newsblur')
+
+import psutil
+import requests
+import settings
+import socket
+
+def main():
+    usage = psutil.disk_usage('/')
+    hostname = socket.gethostname()
+
+    if usage.percent > 95:
+        requests.post(
+                "https://api.mailgun.net/v2/%s/messages" % settings.MAILGUN_SERVER_NAME,
+                auth=("api", settings.MAILGUN_ACCESS_KEY),
+                data={"from": "NewsBlur Monitor: %s <admin@%s.newsblur.com>" % (hostname, hostname),
+                      "to": [settings.ADMINS[0][1]],
+                      "subject": "%s hit %s%% disk usage!" % (hostname, usage.percent),
+                      "text": "Usage on %s: %s" % (hostname, usage)})
+    else:
+        print " ---> Disk usage is fine: %s / %s%% used" % (hostname, usage.percent)
+        
+if __name__ == '__main__':
+    main()