diff --git a/config/munin/mongo_btree b/config/munin/mongo_btree new file mode 100755 index 000000000..33d56a2e4 --- /dev/null +++ b/config/munin/mongo_btree @@ -0,0 +1,64 @@ +#!/usr/bin/env python + +## GENERATED FILE - DO NOT EDIT + +import urllib2 +import sys +import os + +try: + import json +except ImportError: + import simplejson as json + + +def getServerStatus(): + host = os.environ.get("host", "127.0.0.1") + port = 28017 + url = "http://%s:%d/_status" % (host, port) + req = urllib2.Request(url) + user = os.environ.get("user") + password = os.environ.get("password") + if user and password: + print user,password + passwdmngr = urllib2.HTTPPasswordMgrWithDefaultRealm() + passwdmngr.add_password(None, 'http://%s:%d' % (host, port), user, password) + authhandler = urllib2.HTTPDigestAuthHandler(passwdmngr) + opener = urllib2.build_opener(authhandler) + urllib2.install_opener(opener) + raw = urllib2.urlopen(req).read() + return json.loads( raw )["serverStatus"] + +def get(): + return getServerStatus()["indexCounters"]["btree"] + +def doData(): + for k,v in get().iteritems(): + print( str(k) + ".value " + str(int(v)) ) + +def doConfig(): + + print "graph_title MongoDB btree stats" + print "graph_args --base 1000 -l 0" + print "graph_vlabel mb ${graph_period}" + print "graph_category MongoDB" + + for k in get(): + print k + ".label " + k + print k + ".min 0" + print k + ".type COUNTER" + print k + ".max 500000" + print k + ".draw LINE1" + + + + + + +if __name__ == "__main__": + if len(sys.argv) > 1 and sys.argv[1] == "config": + doConfig() + else: + doData() + + diff --git a/config/munin/mongo_conn b/config/munin/mongo_conn new file mode 100755 index 000000000..de5e6b550 --- /dev/null +++ b/config/munin/mongo_conn @@ -0,0 +1,57 @@ +#!/usr/bin/env python + +## GENERATED FILE - DO NOT EDIT + +import urllib2 +import sys +import os + +try: + import json +except ImportError: + import simplejson as json + + +def getServerStatus(): + host = os.environ.get("host", "127.0.0.1") + port = 28017 + url = "http://%s:%d/_status" % (host, port) + req = urllib2.Request(url) + user = os.environ.get("user") + password = os.environ.get("password") + if user and password: + passwdmngr = urllib2.HTTPPasswordMgrWithDefaultRealm() + passwdmngr.add_password(None, 'http://%s:%d' % (host, port), user, password) + authhandler = urllib2.HTTPDigestAuthHandler(passwdmngr) + opener = urllib2.build_opener(authhandler) + urllib2.install_opener(opener) + raw = urllib2.urlopen(req).read() + return json.loads( raw )["serverStatus"] + +name = "connections" + + +def doData(): + print name + ".value " + str( getServerStatus()["connections"]["current"] ) + +def doConfig(): + + print "graph_title MongoDB current connections" + print "graph_args --base 1000 -l 0" + print "graph_vlabel connections" + print "graph_category MongoDB" + + print name + ".label " + name + + + + + + +if __name__ == "__main__": + if len(sys.argv) > 1 and sys.argv[1] == "config": + doConfig() + else: + doData() + + diff --git a/config/munin/mongo_lock b/config/munin/mongo_lock new file mode 100755 index 000000000..eeb53b471 --- /dev/null +++ b/config/munin/mongo_lock @@ -0,0 +1,56 @@ +#!/usr/bin/env python + +## GENERATED FILE - DO NOT EDIT + +import urllib2 +import sys +import os + +try: + import json +except ImportError: + import simplejson as json + + +def getServerStatus(): + host = os.environ.get("host", "127.0.0.1") + port = 28017 + url = "http://%s:%d/_status" % (host, port) + req = urllib2.Request(url) + user = os.environ.get("user") + password = os.environ.get("password") + if user and password: + passwdmngr = urllib2.HTTPPasswordMgrWithDefaultRealm() + passwdmngr.add_password(None, 'http://%s:%d' % (host, port), user, password) + authhandler = urllib2.HTTPDigestAuthHandler(passwdmngr) + opener = urllib2.build_opener(authhandler) + urllib2.install_opener(opener) + raw = urllib2.urlopen(req).read() + return json.loads( raw )["serverStatus"] + +name = "locked" + +def doData(): + print name + ".value " + str( 100 * getServerStatus()["globalLock"]["ratio"] ) + +def doConfig(): + + print "graph_title MongoDB write lock percentage" + print "graph_args --base 1000 -l 0 " + print "graph_vlabel percentage" + print "graph_category MongoDB" + + print name + ".label " + name + + + + + + +if __name__ == "__main__": + if len(sys.argv) > 1 and sys.argv[1] == "config": + doConfig() + else: + doData() + + diff --git a/config/munin/mongo_mem b/config/munin/mongo_mem new file mode 100755 index 000000000..d1ef449d0 --- /dev/null +++ b/config/munin/mongo_mem @@ -0,0 +1,62 @@ +#!/usr/bin/env python + +## GENERATED FILE - DO NOT EDIT + +import urllib2 +import sys +import os + +try: + import json +except ImportError: + import simplejson as json + + +def getServerStatus(): + host = os.environ.get("host", "127.0.0.1") + port = 28017 + url = "http://%s:%d/_status" % (host, port) + req = urllib2.Request(url) + user = os.environ.get("user") + password = os.environ.get("password") + if user and password: + passwdmngr = urllib2.HTTPPasswordMgrWithDefaultRealm() + passwdmngr.add_password(None, 'http://%s:%d' % (host, port), user, password) + authhandler = urllib2.HTTPDigestAuthHandler(passwdmngr) + opener = urllib2.build_opener(authhandler) + urllib2.install_opener(opener) + raw = urllib2.urlopen(req).read() + return json.loads( raw )["serverStatus"] + +def ok(s): + return s == "resident" or s == "virtual" or s == "mapped" + +def doData(): + for k,v in getServerStatus()["mem"].iteritems(): + if ok(k): + print( str(k) + ".value " + str(v * 1024 * 1024) ) + +def doConfig(): + + print "graph_title MongoDB memory usage" + print "graph_args --base 1024 -l 0 --vertical-label Bytes" + print "graph_category MongoDB" + + for k in getServerStatus()["mem"]: + if ok( k ): + print k + ".label " + k + print k + ".draw LINE1" + + + + + + + +if __name__ == "__main__": + if len(sys.argv) > 1 and sys.argv[1] == "config": + doConfig() + else: + doData() + + diff --git a/config/munin/mongo_ops b/config/munin/mongo_ops new file mode 100755 index 000000000..0c9f63d96 --- /dev/null +++ b/config/munin/mongo_ops @@ -0,0 +1,58 @@ +#!/usr/bin/env python + +## GENERATED FILE - DO NOT EDIT + +import urllib2 +import sys +import os + +try: + import json +except ImportError: + import simplejson as json + + +def getServerStatus(): + host = os.environ.get("host", "127.0.0.1") + port = 28017 + url = "http://%s:%d/_status" % (host, port) + req = urllib2.Request(url) + user = os.environ.get("user") + password = os.environ.get("password") + if user and password: + passwdmngr = urllib2.HTTPPasswordMgrWithDefaultRealm() + passwdmngr.add_password(None, 'http://%s:%d' % (host, port), user, password) + authhandler = urllib2.HTTPDigestAuthHandler(passwdmngr) + opener = urllib2.build_opener(authhandler) + urllib2.install_opener(opener) + raw = urllib2.urlopen(req).read() + return json.loads( raw )["serverStatus"] + + +def doData(): + ss = getServerStatus() + for k,v in ss["opcounters"].iteritems(): + print( str(k) + ".value " + str(v) ) + +def doConfig(): + + print "graph_title MongoDB ops" + print "graph_args --base 1000 -l 0" + print "graph_vlabel ops / ${graph_period}" + print "graph_category MongoDB" + print "graph_total total" + + for k in getServerStatus()["opcounters"]: + print k + ".label " + k + print k + ".min 0" + print k + ".type COUNTER" + print k + ".max 500000" + print k + ".draw LINE1" + +if __name__ == "__main__": + if len(sys.argv) > 1 and sys.argv[1] == "config": + doConfig() + else: + doData() + + diff --git a/config/munin/mongodb_flush_avg b/config/munin/mongodb_flush_avg deleted file mode 100755 index ea9bbec61..000000000 --- a/config/munin/mongodb_flush_avg +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from munin.mongodb import MuninMongoDBPlugin - -class MongoDBFlushAvg(MuninMongoDBPlugin): - args = "-l 0 --base 1000" - vlabel = "seconds" - title = "MongoDB background flush interval" - info = "The average time between background flushes" - fields = ( - ('total_ms', dict( - label = "Flush interval", - info = "The time interval for background flushes", - type = "DERIVE", - min = "0", - )), - ) - - def execute(self): - status = self.connection.admin.command('serverStatus') - try: - value = float(status["backgroundFlushing"]["total_ms"])/1000 - except KeyError: - value = "U" - return dict(total_ms=value) - -if __name__ == "__main__": - MongoDBFlushAvg().run() diff --git a/config/munin/mongodb_lock_ratio b/config/munin/mongodb_lock_ratio deleted file mode 100755 index 2da8d3e6c..000000000 --- a/config/munin/mongodb_lock_ratio +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from munin.mongodb import MuninMongoDBPlugin - -class MongoDBLockRatio(MuninMongoDBPlugin): - args = "-l 0 --base 1000" - vlabel = "ratio" - title = "MongoDB global lock time ratio" - info = "How long the global lock has been held compared to the global execution time" - fields = ( - ('lockratio', dict( - label = "Global lock time ratio", - info = "How long the global lock has been held compared to the global execution time", - type = "GAUGE", - min = "0", - )), - ) - - def execute(self): - status = self.connection.admin.command('serverStatus') - try: - value = status["globalLock"]["ratio"] - except KeyError: - value = "U" - return dict(lockratio=value) - -if __name__ == "__main__": - MongoDBLockRatio().run() diff --git a/config/munin/mongodb_memory b/config/munin/mongodb_memory deleted file mode 100755 index 713289def..000000000 --- a/config/munin/mongodb_memory +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from munin.mongodb import MuninMongoDBPlugin - -class MongoDBMemoryPlugin(MuninMongoDBPlugin): - args = "-l 0 --base 1024" - vlabel = "bytes" - title = "MongoDB memory usage" - info = "Memory usage" - fields = ( - ('virtual', dict( - label = "virtual", - info = "Bytes of virtual memory", - type = "GAUGE", - min = "0", - )), - ('resident', dict( - label = "resident", - info = "Bytes of resident memory", - type = "GAUGE", - min = "0", - )), - ('mapped', dict( - label = "mapped", - info = "Bytes of mapped memory", - type = "GAUGE", - min = "0", - )), - ) - - def execute(self): - status = self.connection.admin.command('serverStatus') - values = {} - for k in ("virtual", "resident", "mapped"): - try: - value = int(status["mem"][k]) * 1024 * 1024 - except KeyError: - value = "U" - values[k] = value - return values - -if __name__ == "__main__": - MongoDBMemoryPlugin().run() diff --git a/config/munin/mongodb_objects_ b/config/munin/mongodb_objects_newsblur similarity index 100% rename from config/munin/mongodb_objects_ rename to config/munin/mongodb_objects_newsblur diff --git a/config/munin/mongodb_ops b/config/munin/mongodb_ops deleted file mode 100755 index f8aff2827..000000000 --- a/config/munin/mongodb_ops +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from munin.mongodb import MuninMongoDBPlugin - -class MongoDBOpsPlugin(MuninMongoDBPlugin): - args = "-l 0 --base 1000" - vlabel = "ops/sec" - title = "MongoDB operations" - info = "Operations" - ops = ("query", "update", "insert", "delete", "command", "getmore") - - @property - def fields(self): - return [ - (op, dict( - label = "%s operations" % op, - info = "%s operations" % op, - type = "DERIVE", - min = "0", - )) for op in self.ops - ] - - def execute(self): - status = self.connection.admin.command('serverStatus') - return dict( - (op, status["opcounters"].get(op, 0)) - for op in self.ops - ) - -if __name__ == "__main__": - MongoDBOpsPlugin().run() diff --git a/config/munin/mongodb_size_ b/config/munin/mongodb_size_newsblur similarity index 100% rename from config/munin/mongodb_size_ rename to config/munin/mongodb_size_newsblur diff --git a/fabfile.py b/fabfile.py index 32145ac74..e1b974aa2 100644 --- a/fabfile.py +++ b/fabfile.py @@ -453,7 +453,6 @@ def setup_node(): sudo('ufw allow 8888') put('config/supervisor_node.conf', '/etc/supervisor/conf.d/node.conf', use_sudo=True) - # ============== # = Setup - DB = @@ -510,6 +509,10 @@ def setup_redis(): sudo('mkdir -p /var/lib/redis') sudo('update-rc.d redis defaults') sudo('/etc/init.d/redis start') + +def setup_db_munin(): + sudo('ln -s %s/config/munin/mongo* /etc/munin/plugins/' % env.NEWSBLUR_PATH) + # ================ # = Setup - Task =