Adding disk usage monitor to all database servers.

fabfile.py

@@ -583,7 +583,7 @@ def setup_ulimit():
     sudo('sysctl -p')
     sudo('ulimit -n 100000')
     connections.connect(env.host_string)
-
+    
     # run('touch /home/ubuntu/.bash_profile')
     # run('echo "ulimit -n $FILEMAX" >> /home/ubuntu/.bash_profile')
 
@@ -881,6 +881,9 @@ def setup_mongo():
     sudo('/etc/init.d/mongodb restart')
     put('config/logrotate.mongo.conf', '/etc/logrotate.d/mongodb', use_sudo=True)
 
+    # Reclaim 5% disk space used for root logs. Set to 1%.
+    sudo('tune2fs -m 1 /dev/vda')
+
 def setup_mongo_configsvr():
     sudo('mkdir -p /var/lib/mongodb_configsvr')
     sudo('chown mongodb.mongodb /var/lib/mongodb_configsvr')
@@ -1039,6 +1042,11 @@ def setup_db_search():
     put('config/supervisor_celeryd_search_indexer_tasker.conf', '/etc/supervisor/conf.d/celeryd_search_indexer_tasker.conf', use_sudo=True)
     sudo('supervisorctl reread')
     sudo('supervisorctl update')
+
+def setup_usage_monitor():
+    sudo('ln -s %s/utils/monitor_disk_usage.py /etc/cron.daily/monitor_disk_usage' % env.NEWSBLUR_PATH)
+    with settings(warn_only=True):
+        sudo('/etc/cron.daily/monitor_disk_usage')
     
 # ================
 # = Setup - Task =

utils/monitor_disk_usage.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+
+import sys
+
+if '/srv/newsblur' not in ' '.join(sys.path):
+    sys.path.append('/srv/newsblur')
+
+import psutil
+import requests
+import settings
+import socket
+
+def main():
+    usage = psutil.disk_usage('/')
+    hostname = socket.gethostname()
+
+    if usage.percent > 95:
+        requests.post(
+                "https://api.mailgun.net/v2/%s/messages" % settings.MAILGUN_SERVER_NAME,
+                auth=("api", settings.MAILGUN_ACCESS_KEY),
+                data={"from": "NewsBlur Monitor: %s <admin@%s.newsblur.com>" % (hostname, hostname),
+                      "to": [settings.ADMINS[0][1]],
+                      "subject": "%s hit %s%% disk usage!" % (hostname, usage.percent),
+                      "text": "Usage on %s: %s" % (hostname, usage)})
+    else:
+        print " ---> Disk usage is fine: %s / %s%% used" % (hostname, usage.percent)
+        
+if __name__ == '__main__':
+    main()