mirror of
https://github.com/samuelclay/NewsBlur.git
synced 2025-04-13 09:42:01 +00:00
Removing unused mongo mms utils and configs.
This commit is contained in:
Samuel Clay
parent
30898df3e5
commit
5dba6549ef
34 changed files with 1 additions and 2111 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -41,7 +41,6 @@ media/css/circular
|
|||
config/settings
|
||||
config/secrets
|
||||
templates/maintenance_on.html
|
||||
vendor/mms-agent/settings.py
|
||||
apps/social/spam.py
|
||||
venv*
|
||||
backup
|
||||
|
|
|
|||
|
|
@ -86,7 +86,7 @@ pyasn1==0.4.8
|
|||
pycparser==2.20
|
||||
pyflakes==2.2.0
|
||||
PyJWT==1.7.1
|
||||
pymongo>=3,<3.9
|
||||
pymongo>=3,<4
|
||||
PyMySQL==0.10.1
|
||||
pynliner==0.8.0
|
||||
pyOpenSSL==20.0.1
|
||||
|
|
|
|||
196
vendor/mms-agent/README
vendored
196
vendor/mms-agent/README
vendored
|
|
@ -1,196 +0,0 @@
|
|||
(C) Copyright 2012, 10gen
|
||||
|
||||
10gen MongoDB Monitoring Service (MMS) - Agent
|
||||
=============
|
||||
|
||||
For the complete MMS documentation, see: https://mms.10gen.com/help/
|
||||
|
||||
The MMS agent polls your MongoDB instance(s) once per minute and relays statistics to 10gen.
|
||||
|
||||
New hosts are added by entering them on this page (click the plus/+ button on the top of the page) - https://mms.10gen.com/host/list
|
||||
|
||||
If you have a replica set, you only need to enter one host and the agent will discover the rest of the nodes in the group/cluster.
|
||||
|
||||
If you have a shard cluster, you only need to enter one mongos host and the agent will discover the rest of the nodes in the cluster.
|
||||
|
||||
Only one MMS agent is active at a time, the others are simply hot-standys.
|
||||
|
||||
If you restart your agent or fail over to a secondary agent, there is a five minute timeout before data is sent to the central MMS servers.
|
||||
|
||||
The MMS agent only supports MongoDB 1.6.x+
|
||||
|
||||
The MMS agent currently does not support Python 3.x
|
||||
|
||||
Important
|
||||
------------
|
||||
|
||||
If you have thousands of databases or collections, you should disable "DB Stats" on the "Settings" page before starting your agent.
|
||||
|
||||
Collecting db stats on thousands of databases or collections can adversely impact performance of your system.
|
||||
|
||||
* https://mms.10gen.com/settings
|
||||
|
||||
Installation
|
||||
------------
|
||||
|
||||
* Install Python (2.4+) and pymongo (1.9+) - http://api.mongodb.org/python/current/
|
||||
|
||||
* For most people (on Linux) this is as easy as running: sudo easy_install pymongo
|
||||
|
||||
* If you do not have easy_install, typically you can run: sudo yum install python-setuptools
|
||||
|
||||
* For more information, see: http://pypi.python.org/pypi/setuptools
|
||||
|
||||
* The hashlib package must be installed if running Python 2.4
|
||||
|
||||
* Try (as root): easy_install hashlib - if that does not work, use: http://pypi.python.org/pypi/hashlib/20081119
|
||||
|
||||
* Download and unzip and then run (requires root/sudo permission to install):
|
||||
|
||||
* python setup.py build
|
||||
|
||||
* python setup.py install
|
||||
|
||||
* The backported hmac package must be installed if running Python 2.4
|
||||
|
||||
* Do *not* use pip for installing this package. Pip is not pulling the correct files.
|
||||
|
||||
* Try (as root): easy_install hmac - if that does not work, use: http://pypi.python.org/pypi/hmac
|
||||
|
||||
* Download and unzip and then run (requires root/sudo permission to install):
|
||||
|
||||
* python setup.py build
|
||||
|
||||
* python setup.py install
|
||||
|
||||
* If you need to install any libraries then make sure you close your session/terminal after installing and open a new one, before starting the agent.
|
||||
|
||||
* The agent will securely update itself as new releases are published by 10gen. The agent directory must be writable by the user the agent is running under.
|
||||
|
||||
* Start the agent:
|
||||
|
||||
nohup python agent.py > /REPLACE_SOME_DIRECTORY/agent.log 2>&1 &
|
||||
|
||||
Note: If you are running Python 2.4, you should reduce the default stack size for the user account the MMS agent runs under. In Python 2.5+, the agent reduces the stack size programmatically.
|
||||
|
||||
Hardware Monitoring - Munin Setup
|
||||
------------
|
||||
|
||||
MMS also supports hardware monitoring on database servers. To accomplish this, MMS collects data from the ubiquitous Munin Node daemon.
|
||||
|
||||
Note munin-node, and hardware monitoring is only available for MongoDB instances running on Linux hosts.
|
||||
|
||||
The following are a list of steps to install and configure the Munin Node daemon for MMS.
|
||||
|
||||
* Install the munin-node module
|
||||
|
||||
* On the majority of OSs, this can be accomplished by using apt-get or yum - e.g., sudo yum install munin-node)
|
||||
|
||||
* If you install from source you must also install the plugins (described in the links below under "Additional Documentation")
|
||||
|
||||
* MMS currently uses the cpu, iostat and iostat_ios plugins
|
||||
|
||||
* Configure munin-node.conf to allow connections from the server(s) running your MMS agent(s).
|
||||
|
||||
* Typically, this file is located in /etc/munin/munin-node.conf
|
||||
|
||||
* Modify the "allow" property - See the munin-node.conf documentation for more information: http://munin-monitoring.org/wiki/munin-node.conf
|
||||
|
||||
* If your disk devices are numbered (e.g., /dev/sda1) then you need to configure the munin iostat plugin to support numbered disks (disabled by default).
|
||||
|
||||
* For a typical installation, the configuration file is: /etc/munin/plugin-conf.d/munin-node
|
||||
|
||||
* If this file does not exist, create and add the following:
|
||||
[iostat]
|
||||
env.SHOW_NUMBERED 1
|
||||
|
||||
* Ensure your Munin Node process is registered to start on server boot
|
||||
|
||||
* E.g. sudo ln -s /etc/init.d/munin-node /etc/rc3.d/S99munin-node
|
||||
|
||||
* Restart your Munin Node process to update the configuration:
|
||||
|
||||
* sudo /etc/init.d/munin-node restart
|
||||
|
||||
* You can verify your munin-node setup by running the following from the agent(s) nodes:
|
||||
|
||||
* telnet HOSTNAME 4949 (the default and required port for munin-node)
|
||||
|
||||
* fetch iostat
|
||||
|
||||
* fetch iostat_ios
|
||||
|
||||
* fetch cpu
|
||||
|
||||
(each command should be on a separate line - i.e., hit return after fetch COMMAND - if the plugins are not installed you will see '# Unknown service')
|
||||
|
||||
* Additional Documentation
|
||||
|
||||
* http://munin-monitoring.org/wiki/munin-node
|
||||
|
||||
* http://munin-monitoring.org/wiki/LinuxInstallation
|
||||
|
||||
* http://munin-monitoring.org/wiki/SolarisInstallation
|
||||
|
||||
* http://munin-monitoring.org/wiki/HowToMonitorWindows
|
||||
|
||||
* http://munin-monitoring.org
|
||||
|
||||
Firewall
|
||||
-------------
|
||||
|
||||
If your datacenter is secured by a firewall, please configure your firewall to allow outbound connections on port 443 (SSL) to mms.10gen.com
|
||||
|
||||
Proxy
|
||||
-------------
|
||||
|
||||
If your datacenter requires outbound connections to route through a proxy, you can add the following environment variable:
|
||||
|
||||
export https_proxy='http://someProxyServer:port';
|
||||
|
||||
(on Windows use HTTPS_PROXY)
|
||||
|
||||
Note: This requires Python 2.6.1+
|
||||
|
||||
|
||||
Data Collected
|
||||
-------------
|
||||
|
||||
Different commands are executed based on the host type. Below is an overall list of commands/queries executed.
|
||||
|
||||
* Default
|
||||
* serverStatus
|
||||
* buildinfo
|
||||
* getCmdLineOpts
|
||||
* connPoolStats
|
||||
* _isSelf
|
||||
* getParameter
|
||||
* ismaster
|
||||
* getShardVersion
|
||||
* netstat
|
||||
* replSetGetStatus
|
||||
* shards.find
|
||||
* mongos.find
|
||||
* config.chunks.group
|
||||
* oplog.find
|
||||
* collstats - oplog.rs
|
||||
* sources.find (slave)
|
||||
* config.settings.find
|
||||
* config.collections.find
|
||||
* config.databases.find
|
||||
* config.lockpings.find
|
||||
* dbstats
|
||||
* db.locks
|
||||
|
||||
* Enabled By User
|
||||
* profiler.find
|
||||
|
||||
For more information see:
|
||||
|
||||
* https://groups.google.com/forum/#!forum/10gen-mms
|
||||
|
||||
To submit a support request:
|
||||
|
||||
* https://jira.mongodb.org/secure/CreateIssue.jspa?pid=10582&issuetype=6
|
||||
|
||||
|
||||
57
vendor/mms-agent/WINDOWS.txt
vendored
57
vendor/mms-agent/WINDOWS.txt
vendored
|
|
@ -1,57 +0,0 @@
|
|||
Contents
|
||||
------------
|
||||
* README.txt
|
||||
* mongommsinstall.bat
|
||||
* mongommstinstall.ps1
|
||||
* srvany.exe
|
||||
|
||||
Prerequisites
|
||||
------------
|
||||
* Expects a MMS/PyMongo compatible python version to be installed
|
||||
* Expects pymongo to be installed
|
||||
* Powershell version 2.0 (for windows service installation)
|
||||
* Powershell script execution to be turned on
|
||||
|
||||
Installation
|
||||
------------
|
||||
* The following instructions are to run MMS as a windows service
|
||||
* More information can be found at https://mms.10gen.com/help/install.html#installing-mms-on-windows
|
||||
|
||||
Install Prerequisites
|
||||
---------------------
|
||||
* Install python 2.7.x using the Windows Installer. 64 bit installer is preferred.
|
||||
e.g. http://www.python.org/ftp/python/2.7.2/python-2.7.2.amd64.msi
|
||||
* Install latest version of pymongo using the Windows Installer from pypi. Ensure
|
||||
you pick the installer that corresponds to your python version and the type (32 bit or 64 bit).
|
||||
http://pypi.python.org/pypi/pymongo/
|
||||
|
||||
Install Powershell 2.0
|
||||
----------------------
|
||||
* If you do not have powershell 2.0 installed you would need to install it from http://support.microsoft.com/kb/968929.
|
||||
|
||||
Enable Powershell Script Execution
|
||||
----------------------------------
|
||||
* Open the Windows Powershell by right-clicking its icon in the Start bar and selecting Run as Administrator
|
||||
* Type `Set-ExecutionPolicy -ExecutionPolicy RemoteSigned` and agree when prompted about changing the policy
|
||||
|
||||
Specifying a Proxy Server
|
||||
-------------------------
|
||||
If the MMS agent needs a proxy server to connect outside, specify it in the HTTPS_PROXY environment variable
|
||||
as http://proxyservername:port
|
||||
|
||||
Sevice Installation
|
||||
-------------------
|
||||
* The script reads python path from windows registry if not specified on command line
|
||||
* If multiple python installations are present edit the bat file to specify the path of the python install to work
|
||||
* Run mongommsinstall.bat from an administrator command window
|
||||
* This will install the windows service and start it
|
||||
* If you stop the windows service, the agent will stop
|
||||
|
||||
Uninstall
|
||||
---------
|
||||
- Stop the "MongoDB MMS" windows service
|
||||
- Run "sc delete mongomms"
|
||||
|
||||
More Information
|
||||
----------------
|
||||
- MMS Help https://mms.10gen.com/help/
|
||||
336
vendor/mms-agent/agent.py
vendored
336
vendor/mms-agent/agent.py
vendored
|
|
@ -1,336 +0,0 @@
|
|||
"""
|
||||
(C) Copyright 2011, 10gen
|
||||
|
||||
This is a label on a mattress. Do not modify this file!
|
||||
"""
|
||||
|
||||
# App
|
||||
import settings as _settings
|
||||
import logConfig
|
||||
|
||||
# Python
|
||||
import sys, socket, time, os, hmac, urllib.request, urllib.error, urllib.parse, threading, subprocess, traceback
|
||||
|
||||
try:
|
||||
import hashlib
|
||||
except ImportError:
|
||||
sys.exit( 'ERROR - you must have hashlib installed - see README for more info' )
|
||||
|
||||
_logger = logConfig.initLogger()
|
||||
|
||||
socket.setdefaulttimeout( _settings.socket_timeout )
|
||||
|
||||
_pymongoVersion = None
|
||||
|
||||
_processPid = os.getpid()
|
||||
|
||||
# Try and reduce the stack size.
|
||||
try:
|
||||
threading.stack_size( 409600 )
|
||||
except:
|
||||
pass
|
||||
|
||||
if _settings.mms_key == '@API_KEY@':
|
||||
sys.exit( 'ERROR - you must set your @API_KEY@ - see https://mms.10gen.com/settings' )
|
||||
|
||||
if _settings.secret_key == '@SECRET_KEY@':
|
||||
sys.exit( 'ERROR - you must set your @SECRET_KEY@ - see https://mms.10gen.com/settings' )
|
||||
|
||||
if sys.version_info < ( 2, 4 ):
|
||||
sys.exit( 'ERROR - old Python - the MMS agent requires Python 2.4 or higher' )
|
||||
|
||||
# Make sure pymongo is installed
|
||||
try:
|
||||
import pymongo
|
||||
import bson
|
||||
except ImportError:
|
||||
sys.exit( 'ERROR - pymongo not installed - see: http://api.mongodb.org/python/ - run: easy_install pymongo' )
|
||||
|
||||
# Check the version of pymongo.
|
||||
pyv = pymongo.version
|
||||
if "partition" in dir( pyv ):
|
||||
pyv = pyv.partition( "+" )[0]
|
||||
_pymongoVersion = pyv
|
||||
if list(map( int, pyv.split('.') )) < [ 1, 9]:
|
||||
sys.exit( 'ERROR - The MMS agent requires pymongo 1.9 or higher: easy_install -U pymongo' )
|
||||
|
||||
if _settings.useSslForAllConnections:
|
||||
if list(map( int, pyv.split('.') )) < [ 2, 1, 1]:
|
||||
sys.exit( 'ERROR - The MMS agent requires pymongo 2.1.1 or higher to use SSL: easy_install -U pymongo' )
|
||||
|
||||
_pymongoVersion = pymongo.version
|
||||
|
||||
class AgentProcessContainer( object ):
|
||||
""" Store the handle and lock to the agent process. """
|
||||
|
||||
def __init__( self ):
|
||||
""" Init the lock and init process to none """
|
||||
self.lock = threading.Lock()
|
||||
self.agent = None
|
||||
|
||||
def pingAgentProcess( self ):
|
||||
""" Ping the agent process """
|
||||
try:
|
||||
self.lock.acquire()
|
||||
|
||||
if self.agent is None or self.agent.poll() is not None:
|
||||
return
|
||||
|
||||
self.agent.stdin.write( 'hello\n' )
|
||||
self.agent.stdin.flush()
|
||||
finally:
|
||||
self.lock.release()
|
||||
|
||||
def stopAgentProcess( self ):
|
||||
""" Send the stop message to the agent process """
|
||||
try:
|
||||
self.lock.acquire()
|
||||
|
||||
if self.agent is None or self.agent.poll() is not None:
|
||||
return
|
||||
|
||||
self.agent.stdin.write( 'seeya\n' )
|
||||
self.agent.stdin.flush()
|
||||
|
||||
time.sleep( 1 )
|
||||
self.agent = None
|
||||
|
||||
finally:
|
||||
self.lock.release()
|
||||
|
||||
class AgentShutdownListenerThread( threading.Thread ):
|
||||
""" Disabled by default. When enabled listens for shutdown messages. """
|
||||
|
||||
def __init__( self, loggerObj, settingsObj ):
|
||||
""" Initialize the object """
|
||||
self.logger = loggerObj
|
||||
self.settings = settingsObj
|
||||
threading.Thread.__init__( self )
|
||||
|
||||
def run( self ):
|
||||
""" Listen for the shutdown messages """
|
||||
try:
|
||||
sock = socket.socket( socket.AF_INET, socket.SOCK_DGRAM )
|
||||
sock.bind( ( self.settings.shutdownAgentBindAddr, self.settings.shutdownAgentBindPort ) )
|
||||
except Exception as e:
|
||||
self.logger.error( traceback.format_exc( e ) )
|
||||
|
||||
self.logger.info( 'Shutdown listener bound to address %s on port %d' % ( self.settings.shutdownAgentBindAddr, self.settings.shutdownAgentBindPort ) )
|
||||
|
||||
while True:
|
||||
try:
|
||||
time.sleep( 5 )
|
||||
data, addr = sock.recvfrom( 1024 )
|
||||
if data != self.settings.shutdownAgentBindChallenge:
|
||||
self.logger.error( 'received bad shutdown message from: %s' % addr[0] )
|
||||
else:
|
||||
self.logger.info( 'received valid shutdown message from: %s - exiting' % addr[0] )
|
||||
os._exit( 0 )
|
||||
except:
|
||||
pass
|
||||
|
||||
class AgentProcessMonitorThread( threading.Thread ):
|
||||
""" Make sure the agent process is running """
|
||||
|
||||
def __init__( self, logger, agentDir, processContainerObj ):
|
||||
""" Initialize the object """
|
||||
self.logger = logger
|
||||
self.agentDir = agentDir
|
||||
self.processContainer = processContainerObj
|
||||
threading.Thread.__init__( self )
|
||||
|
||||
def _launchAgentProcess( self ):
|
||||
""" Execute the agent process and keep a handle to it. """
|
||||
return subprocess.Popen( [ sys.executable, os.path.join( sys.path[0], 'agentProcess.py' ), str( _processPid ) ], stdin=subprocess.PIPE, stdout=subprocess.PIPE )
|
||||
|
||||
def run( self ):
|
||||
""" If the agent process is not alive, start the process """
|
||||
while True:
|
||||
try:
|
||||
time.sleep( 5 )
|
||||
self._monitorProcess()
|
||||
except Exception as e:
|
||||
self.logger.error( traceback.format_exc( e ) )
|
||||
|
||||
def _monitorProcess( self ):
|
||||
""" Monitor the child process """
|
||||
self.processContainer.lock.acquire()
|
||||
try:
|
||||
try:
|
||||
if self.processContainer.agent is None or self.processContainer.agent.poll() is not None:
|
||||
self.processContainer.agent = self._launchAgentProcess()
|
||||
except Exception as e:
|
||||
self.logger.error( traceback.format_exc( e ) )
|
||||
finally:
|
||||
self.processContainer.lock.release()
|
||||
|
||||
class AgentUpdateThread( threading.Thread ):
|
||||
""" Check to see if updates are available - if so download and restart agent process """
|
||||
|
||||
def __init__( self, logger, agentDir, settingsObj, processContainerObj ):
|
||||
""" Initialize the object """
|
||||
self.logger = logger
|
||||
self.agentDir = agentDir
|
||||
self.settings = settingsObj
|
||||
self.processContainer = processContainerObj
|
||||
threading.Thread.__init__( self )
|
||||
|
||||
def run( self ):
|
||||
""" Update the agent if possible """
|
||||
while True:
|
||||
try:
|
||||
time.sleep( 300 )
|
||||
self._checkForUpdate()
|
||||
except Exception as e:
|
||||
self.logger.error( 'Problem with upgrade check: ' + traceback.format_exc( e ) )
|
||||
|
||||
def _checkForUpdate( self ):
|
||||
""" Update the agent if possible """
|
||||
|
||||
res = urllib.request.urlopen( self.settings.version_url % { 'key' : self.settings.mms_key } )
|
||||
|
||||
resBson = None
|
||||
try:
|
||||
resBson = bson.decode_all( res.read() )
|
||||
finally:
|
||||
if res is not None:
|
||||
res.close()
|
||||
res = None
|
||||
|
||||
if len(resBson) != 1:
|
||||
return
|
||||
|
||||
versionResponse = resBson[0]
|
||||
|
||||
if 'status' not in versionResponse or versionResponse['status'] != 'ok':
|
||||
return
|
||||
|
||||
if 'agentVersion' not in versionResponse or 'authCode' not in versionResponse:
|
||||
return
|
||||
|
||||
remoteAgentVersion = versionResponse['agentVersion']
|
||||
authCode = versionResponse['authCode']
|
||||
|
||||
if authCode != hmac.new( self.settings.secret_key, remoteAgentVersion, digestmod=hashlib.sha1 ).hexdigest():
|
||||
self.logger.error( 'Invalid auth code - please confirm your secret key (defined on Settings page) is correct and hmac is properly installed - http://mms.10gen.com/help/' )
|
||||
return
|
||||
|
||||
if self._shouldUpgradeAgent( self.settings.settingsAgentVersion, remoteAgentVersion ):
|
||||
self._upgradeAgent( remoteAgentVersion )
|
||||
|
||||
def _shouldUpgradeAgent( self, localVersion, remoteVersion ):
|
||||
""" Returns true if the agent should upgrade itself. """
|
||||
try:
|
||||
for l, r in zip( localVersion.split('.'), remoteVersion.split('.') ):
|
||||
if int( l ) < int( r ):
|
||||
return True
|
||||
if int( l ) > int( r ):
|
||||
return False
|
||||
except Exception:
|
||||
self.logger.error( "Upgrade problem with versions - local: '%s' - remote: '%s'" % ( localVersion, remoteVersion ) )
|
||||
|
||||
return False
|
||||
|
||||
def _upgradeAgent( self, newAgentVersion ):
|
||||
""" Pull down the files, verify and then stop the current process """
|
||||
|
||||
res = urllib.request.urlopen( self.settings.upgrade_url % { 'key' : self.settings.mms_key } )
|
||||
|
||||
resBson = None
|
||||
try:
|
||||
resBson = bson.decode_all( res.read() )
|
||||
finally:
|
||||
if res is not None:
|
||||
res.close()
|
||||
res = None
|
||||
|
||||
if len(resBson) != 1:
|
||||
return
|
||||
|
||||
upgradeResponse = resBson[0]
|
||||
|
||||
if 'status' not in upgradeResponse or upgradeResponse['status'] != 'ok' or 'files' not in upgradeResponse:
|
||||
return
|
||||
|
||||
# Verify the auth codes for all files and names first.
|
||||
for fileInfo in upgradeResponse['files']:
|
||||
if fileInfo['fileAuthCode'] != hmac.new( self.settings.secret_key, fileInfo['file'], digestmod=hashlib.sha1 ).hexdigest():
|
||||
self.logger.error( 'Invalid file auth code for upgrade - cancelling' )
|
||||
return
|
||||
|
||||
if fileInfo['fileNameAuthCode'] != hmac.new( self.settings.secret_key, fileInfo['fileName'], digestmod=hashlib.sha1 ).hexdigest():
|
||||
self.logger.error( 'Invalid file name auth code for upgrade - cancelling' )
|
||||
return
|
||||
|
||||
# Write the files.
|
||||
for fileInfo in upgradeResponse['files']:
|
||||
|
||||
fileContent = fileInfo['file']
|
||||
fileName = fileInfo['fileName']
|
||||
|
||||
# If the user has a global username/password defined, make sure it is set in the new settings.py file.
|
||||
if fileName == 'settings.py' and getattr( self.settings, 'globalAuthUsername', None ) is not None and getattr( self.settings, 'globalAuthPassword', None ) is not None:
|
||||
fileContent = fileContent.replace( 'globalAuthPassword = None', 'globalAuthPassword=%r' % self.settings.globalAuthPassword )
|
||||
fileContent = fileContent.replace( 'globalAuthUsername = None', 'globalAuthUsername=%r' % self.settings.globalAuthUsername )
|
||||
|
||||
fileSystemName = os.path.join( self.agentDir, fileName )
|
||||
newFile = open( fileSystemName, 'w' )
|
||||
|
||||
try:
|
||||
newFile.write( fileContent )
|
||||
finally:
|
||||
if newFile is not None:
|
||||
newFile.close()
|
||||
|
||||
# Stop the current agent process
|
||||
try:
|
||||
self.processContainer.stopAgentProcess()
|
||||
self.settings.settingsAgentVersion = newAgentVersion
|
||||
self.logger.info( 'Agent upgraded to version: ' + newAgentVersion + ' - there is up to a five minute timeout before data will be sent again' )
|
||||
except Exception as e:
|
||||
self.logger.error( 'Problem restarting agent process: ' + traceback.format_exc( e ) )
|
||||
|
||||
#
|
||||
# Run the process monitor and update threads.
|
||||
#
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
_logger.info( 'Starting agent parent process - version: %s' % ( _settings.settingsAgentVersion ) )
|
||||
_logger.info( 'Note: If you have hundreds or thousands of databases, disable dbstats on the settings page before running the MMS agent.' )
|
||||
|
||||
processContainer = AgentProcessContainer()
|
||||
|
||||
# Star the agent monitor thread.
|
||||
monitorThread = AgentProcessMonitorThread( _logger, sys.path[0], processContainer )
|
||||
monitorThread.setName( 'AgentProcessMonitorThread' )
|
||||
monitorThread.setDaemon( True )
|
||||
monitorThread.start()
|
||||
|
||||
# If enabled, start the shutdown listener thread (disabled by default).
|
||||
if _settings.shutdownAgentBindAddr is not None:
|
||||
shutdownListenerThread = AgentShutdownListenerThread( _logger, _settings )
|
||||
shutdownListenerThread.setName( 'AgentShutdownListenerThread' )
|
||||
shutdownListenerThread.setDaemon( True )
|
||||
shutdownListenerThread.start()
|
||||
|
||||
if _settings.autoUpdateEnabled:
|
||||
updateThread = AgentUpdateThread( _logger, sys.path[0], _settings, processContainer )
|
||||
updateThread.setName( 'AgentUpdateThread' )
|
||||
updateThread.setDaemon( True )
|
||||
updateThread.start()
|
||||
|
||||
_logger.info( 'Started agent parent process - version: %s' % ( _settings.settingsAgentVersion ) )
|
||||
|
||||
# The parent process will let the child process know it's alive.
|
||||
while True:
|
||||
try:
|
||||
time.sleep( 2 )
|
||||
processContainer.pingAgentProcess()
|
||||
except Exception as exc:
|
||||
_logger.error( traceback.format_exc( exc ) )
|
||||
|
||||
except KeyboardInterrupt:
|
||||
processContainer.stopAgentProcess()
|
||||
except Exception as ex:
|
||||
_logger.error( traceback.format_exc( ex ) )
|
||||
|
||||
289
vendor/mms-agent/agentProcess.py
vendored
289
vendor/mms-agent/agentProcess.py
vendored
|
|
@ -1,289 +0,0 @@
|
|||
"""
|
||||
(C) Copyright 2011, 10gen
|
||||
|
||||
This is a label on a mattress. Do not modify this file!
|
||||
"""
|
||||
|
||||
# Mongo
|
||||
import pymongo, bson
|
||||
|
||||
if '_closed' in dir( pymongo.connection ):
|
||||
pymongo.connection._closed = lambda sock: False
|
||||
|
||||
# App
|
||||
import settings as _settings
|
||||
from mmsAgent import MmsAgent
|
||||
from confPull import ConfPullThread
|
||||
import logConfig
|
||||
|
||||
# Python
|
||||
import os, sys, platform, time, threading, socket, traceback, random, hashlib, tempfile
|
||||
|
||||
socket.setdefaulttimeout( _settings.socket_timeout )
|
||||
|
||||
_agentVersion = "1.5.7"
|
||||
|
||||
_pymongoVersion = pymongo.version
|
||||
|
||||
_pymongoHasC = False
|
||||
|
||||
try:
|
||||
_pymongoHasC = pymongo.has_c()
|
||||
except:
|
||||
pass
|
||||
|
||||
# Try and reduce the stack size.
|
||||
try:
|
||||
threading.stack_size(409600)
|
||||
except:
|
||||
pass
|
||||
|
||||
class AgentProcess( threading.Thread ):
|
||||
""" The parent process - monitors agent process and checks for updates etc """
|
||||
|
||||
def __init__( self, loggerObj, agentDir, existingSessionKey ):
|
||||
""" Construct the object """
|
||||
self.logger = loggerObj
|
||||
self.agentDir = agentDir
|
||||
self.mmsAgent = MmsAgent( _settings, _agentVersion, platform.python_version(), _pymongoVersion, _pymongoHasC, platform.uname()[1], self.logger, existingSessionKey )
|
||||
threading.Thread.__init__( self )
|
||||
|
||||
def stop( self ):
|
||||
""" Stop the agent process """
|
||||
try:
|
||||
self.mmsAgent.done = True
|
||||
self.mmsAgent.stopAll()
|
||||
|
||||
except Exception as fe:
|
||||
self.logger.error( traceback.format_exc( fe ) )
|
||||
|
||||
def run( self ):
|
||||
""" The agent process """
|
||||
try:
|
||||
# Start the configuration request
|
||||
confThread = ConfPullThread( _settings, self.mmsAgent )
|
||||
confThread.setName( 'ConfPullThread' )
|
||||
confThread.start()
|
||||
|
||||
hostStateMonitorThread = MonitorHostState( self.logger, self.mmsAgent )
|
||||
hostStateMonitorThread.setName( 'MonitorHostState' )
|
||||
hostStateMonitorThread.start()
|
||||
|
||||
# Loop through and send data back to the MMS servers.
|
||||
while not self.mmsAgent.done:
|
||||
try:
|
||||
try:
|
||||
self.mmsAgent.sendDataToMms()
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error( traceback.format_exc( e ) )
|
||||
finally:
|
||||
try:
|
||||
time.sleep( self.mmsAgent.collectionInterval )
|
||||
except:
|
||||
pass
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error( traceback.format_exc( e ) )
|
||||
|
||||
class ParentProcessMonitor( threading.Thread ):
|
||||
""" The parent process monitor - monitors to see if the parent process is sending heartbeats """
|
||||
|
||||
def __init__( self, agentProcessObj ):
|
||||
""" Construct the object """
|
||||
self.mmsAgent = agentProcessObj.mmsAgent
|
||||
self.logger = self.mmsAgent.logger
|
||||
self.agentProcess = agentProcessObj
|
||||
self.lock = threading.Lock()
|
||||
self.lastHeartbeat = time.time()
|
||||
self.running = True
|
||||
threading.Thread.__init__( self )
|
||||
|
||||
def run( self ):
|
||||
""" Verify the parent process is sending pings """
|
||||
while self.running:
|
||||
time.sleep( 2 )
|
||||
try:
|
||||
self._check()
|
||||
except Exception as ex:
|
||||
self.logger.error( traceback.format_exc( ex ) )
|
||||
raise
|
||||
|
||||
def _check( self ):
|
||||
""" Verify the heartbeat """
|
||||
try:
|
||||
self.lock.acquire()
|
||||
|
||||
secsSinceHeartbeat = ( time.time() - self.lastHeartbeat )
|
||||
|
||||
if secsSinceHeartbeat > 10:
|
||||
self.agentProcess.stop()
|
||||
os._exit( 0 )
|
||||
finally:
|
||||
self.lock.release()
|
||||
|
||||
def stop( self ):
|
||||
""" Stop the process """
|
||||
self.running = False
|
||||
|
||||
def heartbeat( self ):
|
||||
""" Update the last time a message was sent from the parent. """
|
||||
try:
|
||||
self.lock.acquire()
|
||||
self.lastHeartbeat = time.time()
|
||||
finally:
|
||||
self.lock.release()
|
||||
|
||||
class ParentMsgReader( threading.Thread ):
|
||||
""" The parent process message reader """
|
||||
|
||||
def __init__( self, loggerObj, agentProcessObj, parentMonitorObj ):
|
||||
""" Construct the object """
|
||||
self.logger = loggerObj
|
||||
self.parentMonitor = parentMonitorObj
|
||||
self.agentProcess = agentProcessObj
|
||||
threading.Thread.__init__( self )
|
||||
|
||||
def run( self ):
|
||||
""" Read the data from stdin and process """
|
||||
while True:
|
||||
try:
|
||||
time.sleep( 1 )
|
||||
self._readParentMessage()
|
||||
|
||||
except Exception as exc:
|
||||
self.logger.error( traceback.format_exc( exc ) )
|
||||
|
||||
def _readParentMessage( self ):
|
||||
""" Read the heartbeat or stop message from the parent process """
|
||||
line = sys.stdin.readline()
|
||||
|
||||
if not line:
|
||||
return
|
||||
|
||||
if line == 'hello\n':
|
||||
self.parentMonitor.heartbeat()
|
||||
return
|
||||
|
||||
if line == 'seeya\n':
|
||||
try:
|
||||
self.agentProcess.stop()
|
||||
self.parentMonitor.stop()
|
||||
finally:
|
||||
os._exit( 0 )
|
||||
|
||||
class MonitorHostState( threading.Thread ):
|
||||
""" Check to see if we're not getting updates to host state - if not, flush """
|
||||
|
||||
def __init__( self, loggerObj, mmsAgentObj ):
|
||||
""" Construct the object """
|
||||
self.logger = loggerObj
|
||||
self.mmsAgent = mmsAgentObj
|
||||
threading.Thread.__init__( self )
|
||||
|
||||
def run( self ):
|
||||
""" Make sure the data is current, if not remove """
|
||||
while True:
|
||||
try:
|
||||
self.mmsAgent.cleanHostState()
|
||||
except Exception as e:
|
||||
self.logger.error( traceback.format_exc( e ) )
|
||||
|
||||
time.sleep( 10 )
|
||||
|
||||
def generateSessionKey( *args ):
|
||||
""" Generate a session key """
|
||||
t = time.time() * 1000
|
||||
r = random.random()*100000000000000000
|
||||
try:
|
||||
a = platform.uname()[1]
|
||||
except:
|
||||
a = random.random()*100000000000000000
|
||||
|
||||
return hashlib.md5( ( '%(time)s %(random)s %(host)s %(args)s' % {'time' : t, 'random': r, 'host' : a, 'args' : str( args ) } ).encode('utf-8') ).hexdigest()
|
||||
|
||||
def readTmpFile( processPid ):
|
||||
""" Read the temp file """
|
||||
fileName = os.path.join( tempfile.gettempdir(), 'mms-' + str( processPid ) )
|
||||
|
||||
if not os.path.isfile( fileName ):
|
||||
return None
|
||||
|
||||
f = open( fileName )
|
||||
|
||||
try:
|
||||
fileContent = f.read()
|
||||
|
||||
# Handle the legacy json files
|
||||
if fileContent.startswith( '{' ):
|
||||
os.remove( fileName )
|
||||
return None
|
||||
|
||||
resBson = bson.decode_all( fileContent )
|
||||
|
||||
if len(resBson) != 1:
|
||||
return None
|
||||
|
||||
return resBson[0]
|
||||
|
||||
finally:
|
||||
f.close()
|
||||
|
||||
def writeTmpFile( processPid, content ):
|
||||
""" Write the temp file """
|
||||
fileName = os.path.join( tempfile.gettempdir(), 'mms-' + str( processPid ) )
|
||||
|
||||
f = open( fileName, 'wb', 0 )
|
||||
|
||||
try:
|
||||
f.write( bson.BSON.encode( content) )
|
||||
finally:
|
||||
f.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
logger = logConfig.initLogger()
|
||||
|
||||
logger.info( 'Starting agent process - version %s' % ( _agentVersion ) )
|
||||
|
||||
sessionKey = generateSessionKey()
|
||||
|
||||
parentPid = None
|
||||
|
||||
try:
|
||||
if len( sys.argv ) > 1:
|
||||
parentPid = sys.argv[1]
|
||||
currentState = readTmpFile( parentPid )
|
||||
if currentState is not None:
|
||||
if 'sessionKey' in currentState:
|
||||
sessionKey = currentState['sessionKey']
|
||||
else:
|
||||
currentState['sessionKey'] = sessionKey
|
||||
else:
|
||||
currentState = { }
|
||||
currentState['sessionKey'] = sessionKey
|
||||
|
||||
writeTmpFile( parentPid, currentState )
|
||||
|
||||
except Exception as ec:
|
||||
logger.error( traceback.format_exc( ec ) )
|
||||
|
||||
try:
|
||||
# Star the agent monitor thread.
|
||||
agentProcess = AgentProcess( logger, sys.path[0], sessionKey )
|
||||
agentProcess.setName( 'AgentProcess' )
|
||||
agentProcess.start()
|
||||
|
||||
parentMonitor = ParentProcessMonitor( agentProcess )
|
||||
parentMonitor.setName( 'ParentProcessMonitor' )
|
||||
parentMonitor.start()
|
||||
|
||||
msgReader = ParentMsgReader( logger, agentProcess, parentMonitor)
|
||||
msgReader.setName( 'ParentMsgReader' )
|
||||
msgReader.start()
|
||||
|
||||
logger.info( 'Started agent process - parent pid: %s - version: %s' % ( str( parentPid ), _agentVersion ) )
|
||||
|
||||
except Exception as ec:
|
||||
logger.error( traceback.format_exc( ec ) )
|
||||
|
||||
304
vendor/mms-agent/blockingStats.py
vendored
304
vendor/mms-agent/blockingStats.py
vendored
|
|
@ -1,304 +0,0 @@
|
|||
"""
|
||||
(C) Copyright 2011, 10gen
|
||||
|
||||
This is a label on a mattress. Do not modify this file!
|
||||
"""
|
||||
|
||||
import threading, time, datetime, pymongo, traceback, socket, warnings
|
||||
|
||||
warnings.simplefilter( 'ignore', DeprecationWarning )
|
||||
|
||||
blockingStatsAgentVersion = "1.5.7"
|
||||
|
||||
class BlockingMongoStatsThread( threading.Thread ):
|
||||
""" Pull the blocking data from the various hosts. """
|
||||
|
||||
def __init__( self, hostKey, mmsAgent):
|
||||
""" Initialize the object """
|
||||
|
||||
self.hostKey = hostKey
|
||||
self.mmsAgent = mmsAgent
|
||||
self.logger = mmsAgent.logger
|
||||
self.slowDbStats = False
|
||||
self.lastDbStatsCheck = time.time()
|
||||
self.running = True
|
||||
self.host = mmsAgent.extractHostname( hostKey )
|
||||
self.port = mmsAgent.extractPort( hostKey )
|
||||
|
||||
threading.Thread.__init__( self )
|
||||
|
||||
def stopThread( self ):
|
||||
""" Stop the thread """
|
||||
self.running = False
|
||||
|
||||
def run( self ):
|
||||
""" Pull the data from the various hosts. """
|
||||
|
||||
self.logger.info( 'starting blocking stats monitoring: ' + self.hostKey )
|
||||
|
||||
sleepTime = ( self.mmsAgent.collectionInterval / 2 ) - 1
|
||||
|
||||
if ( sleepTime < 1 ):
|
||||
sleepTime = 1
|
||||
|
||||
monitorConn = None
|
||||
|
||||
passes = 0
|
||||
|
||||
while not self.mmsAgent.done and self.running:
|
||||
try:
|
||||
time.sleep( sleepTime )
|
||||
passes = passes + 1
|
||||
|
||||
if passes % 60 == 0:
|
||||
monitorConn = self.mmsAgent.closeDbConnection( self.hostKey, monitorConn )
|
||||
|
||||
if not monitorConn:
|
||||
monitorConn = self.mmsAgent.getDbConnection( self.hostKey )
|
||||
|
||||
if not self._collectBlockingStats( passes, monitorConn ):
|
||||
monitorConn = self.mmsAgent.closeDbConnection( self.hostKey, monitorConn )
|
||||
|
||||
except Exception as e:
|
||||
monitorConn = self.mmsAgent.closeDbConnection( self.hostKey, monitorConn )
|
||||
self.logger.error( 'Problem collecting blocking data from: ' + self.hostKey + " - exception: " + traceback.format_exc( e ) )
|
||||
|
||||
self.logger.info( 'stopping blocking stats monitoring: ' + self.hostKey )
|
||||
self.mmsAgent.closeDbConnection( self.hostKey, monitorConn )
|
||||
|
||||
def _collectBlockingStats( self, passes, monitorConn ):
|
||||
""" Collect the blocking stats from the hosts """
|
||||
|
||||
try:
|
||||
if not self.mmsAgent.haveHostDef( self.hostKey ):
|
||||
return False
|
||||
|
||||
# Close the connection once per hour
|
||||
# Verify the connection.
|
||||
if not self.mmsAgent.isValidMonitorConn( self.hostKey, monitorConn ):
|
||||
return False
|
||||
|
||||
stats = self._collectStats( passes, monitorConn )
|
||||
|
||||
try:
|
||||
stats['hostIpAddr'] = socket.gethostbyname(self.hostKey[0 : self.hostKey.find( ':' )])
|
||||
except Exception as e:
|
||||
self.mmsAgent.handleOperationFailure( self.hostKey, 'hostIpAddr', e )
|
||||
|
||||
stats['host'] = self.host
|
||||
stats['port'] = self.port
|
||||
|
||||
# Make sure we ended up with the same connection.
|
||||
if not self.mmsAgent.isValidMonitorConn( self.hostKey, monitorConn ):
|
||||
return False
|
||||
|
||||
self.mmsAgent.setHostState( self.hostKey, 'mongoBlocking', stats )
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error( 'Problem collecting blocking data from (check if it is up and DNS): ' + self.hostKey + " - exception: " + traceback.format_exc( e ) )
|
||||
return False
|
||||
|
||||
def _collectStats( self, passes, monitorConn ):
|
||||
""" Make the call to mongo host and collect the blocking data """
|
||||
root = {}
|
||||
|
||||
# Set the agent version and hostname.
|
||||
root['agentVersion'] = self.mmsAgent.agentVersion
|
||||
root['agentHostname'] = self.mmsAgent.agentHostname
|
||||
|
||||
isMaster = monitorConn.admin.command( 'ismaster' )
|
||||
root['isMaster'] = isMaster
|
||||
|
||||
isMongos = ( 'msg' in isMaster and isMaster['msg'] == 'isdbgrid' )
|
||||
|
||||
# Check to see if this is a mongod host
|
||||
try:
|
||||
if isMaster['ismaster'] == True and isMongos:
|
||||
# Look at the shards
|
||||
root['shards'] = list( monitorConn.config.shards.find() )
|
||||
|
||||
# Pull from config.locks
|
||||
try:
|
||||
root['locks'] = list( monitorConn.config.locks.find( limit=200, sort=[ ( "$natural" , pymongo.DESCENDING ) ]) )
|
||||
except Exception as e:
|
||||
self.mmsAgent.handleOperationFailure( self.hostKey, 'config.locks.find', e )
|
||||
|
||||
# Pull from config.collections if enabled
|
||||
try:
|
||||
if self.mmsAgent.settings.configCollectionsEnabled:
|
||||
root['configCollections'] = list( monitorConn.config.collections.find( limit=200, sort=[ ( "$natural" , pymongo.DESCENDING ) ] ) )
|
||||
except Exception as e:
|
||||
self.mmsAgent.handleOperationFailure( self.hostKey, 'config.collections.find', e )
|
||||
|
||||
# Pull from config.databases if enabled
|
||||
try:
|
||||
if self.mmsAgent.settings.configDatabasesEnabled:
|
||||
root['configDatabases'] = list( monitorConn.config.databases.find( limit=200, sort=[ ( "$natural" , pymongo.DESCENDING ) ] ) )
|
||||
except Exception as e:
|
||||
self.mmsAgent.handleOperationFailure( self.hostKey, 'config.databases.find', e )
|
||||
|
||||
try:
|
||||
root['configLockpings'] = list( monitorConn.config.lockpings.find( limit=200, sort=[ ( "$natural" , pymongo.DESCENDING ) ] ) )
|
||||
except Exception as e:
|
||||
self.mmsAgent.handleOperationFailure( self.hostKey, 'config.lockpings.find', e )
|
||||
|
||||
# Look at the mongos instances - only pull hosts that have a ping time
|
||||
# updated in the last twenty minutes (and max 1k).
|
||||
queryTime = datetime.datetime.utcnow() - datetime.timedelta( seconds=1200 )
|
||||
root['mongoses'] = list( monitorConn.config.mongos.find( { 'ping' : { '$gte' : queryTime } } ).limit( 1000 ) )
|
||||
|
||||
# Get the shard chunk counts.
|
||||
shardChunkCounts = []
|
||||
positions = { }
|
||||
counter = 0
|
||||
|
||||
if passes % 10 == 0:
|
||||
for chunk in monitorConn.config.chunks.find():
|
||||
key = chunk['ns'] + chunk['shard']
|
||||
if key not in positions:
|
||||
count = {}
|
||||
positions[key] = counter
|
||||
shardChunkCounts.append( count )
|
||||
counter = counter + 1
|
||||
count['count'] = 0
|
||||
count['ns'] = chunk['ns']
|
||||
count['shard'] = chunk['shard']
|
||||
|
||||
shardChunkCounts[positions[key]]['count'] = shardChunkCounts[positions[key]]['count'] + 1
|
||||
|
||||
root['shardChunkCounts'] = shardChunkCounts
|
||||
except pymongo.errors.OperationFailure:
|
||||
pass
|
||||
|
||||
root['serverStatus'] = monitorConn.admin.command( 'serverStatus' )
|
||||
|
||||
isReplSet = False
|
||||
isArbiter = False
|
||||
|
||||
if 'repl' in root['serverStatus']:
|
||||
# Check to see if this is a replica set
|
||||
try:
|
||||
root['replStatus'] = monitorConn.admin.command( 'replSetGetStatus' )
|
||||
if root['replStatus']['myState'] == 7:
|
||||
isArbiter = True
|
||||
|
||||
isReplSet = True
|
||||
except pymongo.errors.OperationFailure:
|
||||
pass
|
||||
|
||||
if isReplSet:
|
||||
oplog = "oplog.rs"
|
||||
else:
|
||||
oplog = "oplog.$main"
|
||||
|
||||
try:
|
||||
root['localSystemReplSet'] = monitorConn.local.system.replset.find_one()
|
||||
except pymongo.errors.OperationFailure as e:
|
||||
self.mmsAgent.handleOperationFailure( self.hostKey, 'local.system.replset.findOne', e )
|
||||
|
||||
localConn = monitorConn.local
|
||||
|
||||
oplogStats = {}
|
||||
|
||||
# Get oplog status
|
||||
if isArbiter:
|
||||
# Do nothing for the time being
|
||||
pass
|
||||
elif isMaster['ismaster'] == True or isReplSet:
|
||||
try:
|
||||
oplogStats["start"] = localConn[oplog].find( limit=1, sort=[ ( "$natural" , pymongo.ASCENDING ) ], fields={ 'ts' : 1 } )[0]["ts"]
|
||||
|
||||
oplogStats["end"] = localConn[oplog].find( limit=1, sort=[ ( "$natural" , pymongo.DESCENDING ) ], fields={ 'ts' : 1} )[0]["ts"]
|
||||
|
||||
oplogStats['rsStats'] = localConn.command( {'collstats' : 'oplog.rs' } )
|
||||
|
||||
except pymongo.errors.OperationFailure as e:
|
||||
self.mmsAgent.handleOperationFailure( self.hostKey, 'local.' + oplog + '.find', e )
|
||||
else:
|
||||
# Slave
|
||||
try:
|
||||
oplogStats["sources"] = {}
|
||||
for s in localConn.sources.find():
|
||||
oplogStats["sources"][s["host"]] = s
|
||||
except pymongo.errors.OperationFailure as e:
|
||||
self.mmsAgent.handleOperationFailure( self.hostKey, 'local.sources.find', e )
|
||||
|
||||
root["oplog"] = oplogStats
|
||||
|
||||
# Load the config.gettings collection (balancer info etc.)
|
||||
if not isArbiter:
|
||||
try:
|
||||
root['configSettings'] = list( monitorConn.config.settings.find() )
|
||||
except Exception as e:
|
||||
self.mmsAgent.handleOperationFailure( self.hostKey, 'config.settings.find', e )
|
||||
|
||||
# per db info - mongos doesn't allow calls to local
|
||||
root['databases'] = { }
|
||||
root['dbProfiling'] = { }
|
||||
root['dbProfileData'] = { }
|
||||
|
||||
profilerEnabled = self.mmsAgent.hostDefValue( self.hostKey, 'profiler' )
|
||||
|
||||
if ( passes % 20 == 0 or profilerEnabled ) and not isArbiter and not isMongos:
|
||||
|
||||
count = 0
|
||||
for x in monitorConn.database_names():
|
||||
try:
|
||||
if passes % 20 == 0:
|
||||
if self.slowDbStats and ( ( time.time() - self.lastDbStatsCheck ) < 7200 ):
|
||||
continue
|
||||
|
||||
if not self.mmsAgent.disableDbstats:
|
||||
|
||||
count += 1
|
||||
|
||||
if count > 100:
|
||||
break
|
||||
|
||||
startTime = time.time()
|
||||
|
||||
temp = monitorConn[x].command( 'dbstats' )
|
||||
# work around Python 2.4 and older bug
|
||||
for f in temp:
|
||||
# this is super hacky b/c of Python 2.4
|
||||
if isinstance( temp[f] , (int, float, complex)) and str(temp[f]) == "-inf":
|
||||
temp[f] = 0
|
||||
root['databases'][x] = temp
|
||||
|
||||
if ( time.time() - startTime ) > 6:
|
||||
self.slowDbStats = True
|
||||
else:
|
||||
self.slowDbStats = False
|
||||
|
||||
self.lastDbStatsCheck = time.time()
|
||||
|
||||
# If the profiler is enabled in MMS, collect data
|
||||
if profilerEnabled and not isArbiter and not self.mmsAgent.settings.disableProfileDataCollection:
|
||||
try:
|
||||
# Get the most recent entries.
|
||||
profileData = list( monitorConn[x].system.profile.find( spec=None, fields=None, skip=0, limit=20, sort=[ ( "$natural", pymongo.DESCENDING ) ] ) )
|
||||
|
||||
if len( profileData ) > 0:
|
||||
root['dbProfileData'][x] = profileData
|
||||
except Exception as e:
|
||||
self.mmsAgent.handleOperationFailure( self.hostKey, 'system.profile.find-' + x, e )
|
||||
|
||||
# Check to see if the profiler is enabled
|
||||
try:
|
||||
profiling = monitorConn[x].command( { 'profile' : -1 } )
|
||||
if profiling is not None and 'ok' in profiling:
|
||||
del profiling['ok']
|
||||
root['dbProfiling'][x] = profiling
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
except:
|
||||
continue
|
||||
|
||||
if 'serverStatus' in root:
|
||||
del root['serverStatus']
|
||||
|
||||
return root
|
||||
|
||||
109
vendor/mms-agent/confPull.py
vendored
109
vendor/mms-agent/confPull.py
vendored
|
|
@ -1,109 +0,0 @@
|
|||
"""
|
||||
(C) Copyright 2011, 10gen
|
||||
|
||||
This is a label on a mattress. Do not modify this file!
|
||||
"""
|
||||
|
||||
import threading, time, urllib.request, urllib.error, urllib.parse, traceback
|
||||
|
||||
import bson
|
||||
|
||||
confPullAgentVersion = "1.5.7"
|
||||
|
||||
class ConfPullThread( threading.Thread ):
|
||||
""" The remote configuration pull thread object """
|
||||
|
||||
def __init__( self, settings, mmsAgent):
|
||||
""" Initialize the object """
|
||||
self.settings = settings
|
||||
self.logger = mmsAgent.logger
|
||||
self.mmsAgent = mmsAgent
|
||||
|
||||
self.confUrl = self.settings.config_url % {
|
||||
'key' : self.settings.mms_key,
|
||||
'hostname' : self.mmsAgent.agentHostname,
|
||||
'sessionKey' : self.mmsAgent.sessionKey,
|
||||
'agentVersion' : self.mmsAgent.agentVersion,
|
||||
'srcVersion' : self.mmsAgent.srcVersion
|
||||
}
|
||||
|
||||
threading.Thread.__init__( self )
|
||||
|
||||
def run( self ):
|
||||
""" Pull the configuration from the cloud (if enabled) """
|
||||
|
||||
while not self.mmsAgent.done:
|
||||
self._pullRemoteConf()
|
||||
time.sleep( self.mmsAgent.confInterval )
|
||||
|
||||
def _pullRemoteConf( self ):
|
||||
""" Pull the remote configuration data """
|
||||
|
||||
uniqueHostnames = []
|
||||
|
||||
res = None
|
||||
|
||||
try:
|
||||
|
||||
res = urllib.request.urlopen( self.confUrl )
|
||||
|
||||
resBson = None
|
||||
try:
|
||||
resBson = bson.decode_all( res.read() )
|
||||
finally:
|
||||
if res is not None:
|
||||
res.close()
|
||||
res = None
|
||||
|
||||
if len(resBson) != 1:
|
||||
return
|
||||
|
||||
confResponse = resBson[0]
|
||||
|
||||
if 'hosts' not in confResponse:
|
||||
self.mmsAgent.stopAll()
|
||||
return
|
||||
|
||||
if 'disableDbstats' in confResponse:
|
||||
self.mmsAgent.disableDbstats = confResponse['disableDbstats']
|
||||
else:
|
||||
self.mmsAgent.disableDbstats = False
|
||||
|
||||
hosts = confResponse['hosts']
|
||||
|
||||
self.mmsAgent.serverHostDefsLock.acquire()
|
||||
try:
|
||||
# Extract the host information
|
||||
if hosts is not None:
|
||||
for host in hosts:
|
||||
|
||||
hostDef, hostDefLast = self.mmsAgent.extractHostDef( host )
|
||||
|
||||
hostKey = hostDef['hostKey']
|
||||
uniqueHostnames.append( hostKey )
|
||||
|
||||
if hostKey not in self.mmsAgent.serverHostDefs:
|
||||
self.mmsAgent.startMonitoringThreads( hostDef )
|
||||
else:
|
||||
self.mmsAgent.checkChangedHostDef( hostDef, hostDefLast )
|
||||
|
||||
hostDef = None
|
||||
hostDefLast = None
|
||||
|
||||
# Check to see if anything was removed
|
||||
for hostDef in list(self.mmsAgent.serverHostDefs.values()):
|
||||
if hostDef['hostKey'] not in uniqueHostnames:
|
||||
self.mmsAgent.stopAndClearHost( hostDef['hostKey'] )
|
||||
finally:
|
||||
self.mmsAgent.serverHostDefsLock.release()
|
||||
|
||||
except Exception as e:
|
||||
if res is not None:
|
||||
try:
|
||||
res.close()
|
||||
res = None
|
||||
except:
|
||||
pass
|
||||
|
||||
self.logger.warning( "Problem pulling configuration data from MMS (check firewall and network): " + traceback.format_exc( e ) )
|
||||
|
||||
130
vendor/mms-agent/getLogs.py
vendored
130
vendor/mms-agent/getLogs.py
vendored
|
|
@ -1,130 +0,0 @@
|
|||
"""
|
||||
(C) Copyright 2012, 10gen
|
||||
|
||||
This is a label on a mattress. Do not modify this file!
|
||||
"""
|
||||
|
||||
import threading, time, traceback, warnings, sets
|
||||
|
||||
warnings.simplefilter( 'ignore', DeprecationWarning )
|
||||
|
||||
getLogsAgentVersion = "1.5.7"
|
||||
|
||||
class GetLogsThread( threading.Thread ):
|
||||
""" When enabled pull log data from hosts. """
|
||||
|
||||
def __init__( self, hostKey, mmsAgent):
|
||||
""" Initialize the object """
|
||||
threading.Thread.__init__( self )
|
||||
self.hostKey = hostKey
|
||||
self.mmsAgent = mmsAgent
|
||||
self.logger = mmsAgent.logger
|
||||
self.running = True
|
||||
self.lastLogEntries = sets.Set()
|
||||
|
||||
def stopThread( self ):
|
||||
""" Stop the thread """
|
||||
self.running = False
|
||||
|
||||
def run( self ):
|
||||
""" Pull the data from the various hosts. """
|
||||
|
||||
self.logger.info( 'starting log data collection: ' + self.hostKey )
|
||||
|
||||
sleepTime = self.mmsAgent.logCollectionInterval
|
||||
|
||||
if ( sleepTime < 1 ):
|
||||
sleepTime = 5
|
||||
|
||||
logConn = None
|
||||
passes = 0
|
||||
|
||||
while not self.mmsAgent.done and self.running:
|
||||
try:
|
||||
enabled = self.mmsAgent.hostDefValue( self.hostKey, 'getLogs' )
|
||||
|
||||
if not enabled:
|
||||
return
|
||||
|
||||
passes = passes + 1
|
||||
|
||||
# Close the connection every so often.
|
||||
if passes % 120 == 0:
|
||||
logConn = self.mmsAgent.closeDbConnection( self.hostKey, logConn )
|
||||
|
||||
if not logConn:
|
||||
logConn = self.mmsAgent.getDbConnection( self.hostKey )
|
||||
|
||||
time.sleep( sleepTime )
|
||||
|
||||
if not self._processLogs( logConn ):
|
||||
logConn = self.mmsAgent.closeDbConnection( self.hostKey, logConn )
|
||||
|
||||
except Exception as e:
|
||||
logConn = self.mmsAgent.closeDbConnection( self.hostKey, logConn )
|
||||
self.logger.error( 'Problem collecting log data from: ' + self.hostKey + " - exception: " + traceback.format_exc( e ) )
|
||||
|
||||
self.logger.info( 'stopping log data collection: ' + self.hostKey )
|
||||
self.mmsAgent.closeDbConnection( self.hostKey, logConn )
|
||||
|
||||
|
||||
def _processLogs( self, logConn ):
|
||||
""" Process the logs """
|
||||
|
||||
try:
|
||||
if not self.mmsAgent.haveHostDef( self.hostKey ):
|
||||
return False
|
||||
|
||||
# Verify the connection.
|
||||
if not self.mmsAgent.isValidMonitorConn( self.hostKey, logConn ):
|
||||
return False
|
||||
|
||||
if not self.mmsAgent.hasCommand( 'getLog', self.hostKey, logConn ):
|
||||
self.logger.error( 'This version of MongoDB does not support the getLog command' )
|
||||
time.sleep( 60 )
|
||||
return False
|
||||
|
||||
logs = self._collectLogs( logConn )
|
||||
|
||||
# Make sure we ended up with the same connection.
|
||||
if not self.mmsAgent.isValidMonitorConn( self.hostKey, logConn ):
|
||||
return False
|
||||
|
||||
self.mmsAgent.setHostState( self.hostKey, 'logs', logs )
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error( 'Problem collecting log data from: ' + self.hostKey + " - exception: " + traceback.format_exc( e ) )
|
||||
return False
|
||||
|
||||
def _collectLogs( self, logConn ):
|
||||
""" Make the call to mongo host and pull the log data """
|
||||
root = { }
|
||||
|
||||
logEntries = []
|
||||
|
||||
entries = logConn.admin.command( { 'getLog' : 'global' } )
|
||||
|
||||
if not entries or not entries.get('log'):
|
||||
self.lastLogEntries.clear()
|
||||
return root
|
||||
|
||||
for logEntry in entries['log']:
|
||||
if logEntry in self.lastLogEntries:
|
||||
continue
|
||||
|
||||
logEntries.append( logEntry )
|
||||
|
||||
if len ( logEntries ) == 0:
|
||||
return root
|
||||
|
||||
self.lastLogEntries.clear()
|
||||
|
||||
for logEntry in entries['log']:
|
||||
self.lastLogEntries.add( logEntry )
|
||||
|
||||
root['entries'] = logEntries
|
||||
|
||||
return root
|
||||
|
||||
125
vendor/mms-agent/logConfig.py
vendored
125
vendor/mms-agent/logConfig.py
vendored
|
|
@ -1,125 +0,0 @@
|
|||
"""
|
||||
(C) Copyright 2011, 10gen
|
||||
|
||||
This is a label on a mattress. Do not modify this file!
|
||||
"""
|
||||
|
||||
# App
|
||||
import settings as _settings
|
||||
|
||||
# Mongo
|
||||
import pymongo, bson
|
||||
|
||||
# Python
|
||||
import logging, threading, time, logging.handlers, urllib.request, urllib.error, urllib.parse, platform, socket, queue
|
||||
|
||||
socket.setdefaulttimeout( _settings.socket_timeout )
|
||||
|
||||
class LogRelayThread( threading.Thread ):
|
||||
""" The log relay thread - batch messages """
|
||||
|
||||
def __init__( self, recordQueue ):
|
||||
""" Construct the object """
|
||||
self.recordQueue = recordQueue
|
||||
self.logUrl = _settings.logging_url % { 'key' : _settings.mms_key }
|
||||
self.pythonVersion = platform.python_version()
|
||||
|
||||
try:
|
||||
self.hostname = platform.uname()[1]
|
||||
except:
|
||||
self.hostname = 'UNKNOWN'
|
||||
|
||||
self.pymongoVersion = pymongo.version
|
||||
threading.Thread.__init__( self )
|
||||
|
||||
def run( self ):
|
||||
""" The agent process """
|
||||
while True:
|
||||
try:
|
||||
# Let the records batch for five seconds
|
||||
time.sleep( 5 )
|
||||
|
||||
self._processRecords()
|
||||
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _processRecords( self ):
|
||||
""" Process the log records """
|
||||
records = []
|
||||
|
||||
try:
|
||||
while not self.recordQueue.empty():
|
||||
|
||||
record = self.recordQueue.get( True, 1 )
|
||||
|
||||
if record is None:
|
||||
break
|
||||
|
||||
data = { }
|
||||
data['levelname'] = record.levelname
|
||||
data['msg'] = record.msg
|
||||
data['filename'] = record.filename
|
||||
data['threadName'] = record.threadName
|
||||
|
||||
# This is to deal with older versions of python.
|
||||
try:
|
||||
data['funcName'] = record.funcName
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
data['process'] = record.process
|
||||
data['lineno'] = record.lineno
|
||||
data['pymongoVersion'] = self.pymongoVersion
|
||||
data['pythonVersion'] = self.pythonVersion
|
||||
data['hostname'] = self.hostname
|
||||
|
||||
records.append( data )
|
||||
|
||||
if len( records ) >= 10:
|
||||
break
|
||||
|
||||
except queue.Empty:
|
||||
pass
|
||||
|
||||
if len( records ) == 0:
|
||||
return
|
||||
|
||||
# Send the data back to mms.
|
||||
res = None
|
||||
try:
|
||||
res = urllib.request.urlopen( self.logUrl, bson.BSON.encode( { 'records' : records }, check_keys=False ) )
|
||||
res.read()
|
||||
finally:
|
||||
if res is not None:
|
||||
res.close()
|
||||
|
||||
class MmsRemoteHandler( logging.Handler ):
|
||||
""" The mms remote log handler """
|
||||
def __init__( self ):
|
||||
""" Construct a new object """
|
||||
logging.Handler.__init__( self )
|
||||
self.recordQueue = queue.Queue( 250 )
|
||||
self.logRelay = LogRelayThread( self.recordQueue )
|
||||
self.logRelay.setName( 'LogRelay' )
|
||||
self.logRelay.start()
|
||||
|
||||
def emit( self, record ):
|
||||
""" Send the record to the remote servers """
|
||||
try:
|
||||
if record is not None:
|
||||
self.recordQueue.put_nowait( record )
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def initLogger( ):
|
||||
""" Initialize the logger """
|
||||
logger = logging.getLogger('MMS')
|
||||
streamHandler = logging.StreamHandler()
|
||||
streamHandler.setFormatter( logging.Formatter('%(asctime)s %(levelname)s %(message)s') )
|
||||
logger.addHandler( streamHandler )
|
||||
logging.handlers.MmsRemoteHandler = MmsRemoteHandler
|
||||
logger.addHandler( logging.handlers.MmsRemoteHandler() )
|
||||
logger.setLevel( logging.INFO )
|
||||
return logger
|
||||
|
||||
1
vendor/mms-agent/mongommsinstall.bat
vendored
1
vendor/mms-agent/mongommsinstall.bat
vendored
|
|
@ -1 +0,0 @@
|
|||
powershell .\mongommsinstall.ps1
|
||||
195
vendor/mms-agent/mongommsinstall.ps1
vendored
195
vendor/mms-agent/mongommsinstall.ps1
vendored
|
|
@ -1,195 +0,0 @@
|
|||
$python64RegKey = "HKLM:\Software\Python\PythonCore\*"
|
||||
$python32RegKey = "HKLM:\Software\Wow6432Node\Python\PythonCore\*"
|
||||
$pythonExe = "python.exe"
|
||||
$srvanyExe = "srvany.exe"
|
||||
$scriptName = "agent.py"
|
||||
$serviceName = "MongoMMS"
|
||||
$serviceDisplayName = "MongoDB MMS"
|
||||
$serviceDesc = "Service that runs the MongoDB Python MMS Script"
|
||||
$serviceRegPath = Join-Path "HKLM:\SYSTEM\CurrentControlSet\Services\" $serviceName
|
||||
$pyMongoTest = "pymongotest.py"
|
||||
$pythonTest = "pythonversiontest.py"
|
||||
|
||||
function Test-PyMongo {
|
||||
Param($pythonPath)
|
||||
if ($pythonPath -eq $null) {
|
||||
return -1
|
||||
}
|
||||
$res = (Start-Process -FilePath $pythonPath `
|
||||
-ArgumentList $pyMongoTest -Wait -PassThru -NoNewWindow).ExitCode
|
||||
return $res
|
||||
}
|
||||
|
||||
function Test-Python-Version {
|
||||
Param($pythonPath)
|
||||
if ($pythonPath -eq $null) {
|
||||
return -1
|
||||
}
|
||||
$res = (Start-Process -FilePath $pythonPath `
|
||||
-ArgumentList $pythonTest -Wait -PassThru -NoNewWindow).ExitCode
|
||||
return $res
|
||||
}
|
||||
|
||||
function Test-ValidPython {
|
||||
Param($pythonPath)
|
||||
if ($pythonPath -eq $null) {
|
||||
return -1
|
||||
}
|
||||
$res = Test-Python-Version($pythonPath)
|
||||
if (!($res -eq 0)) {
|
||||
return $res
|
||||
}
|
||||
$res = Test-PyMongo($pythonPath)
|
||||
return $res
|
||||
}
|
||||
|
||||
|
||||
|
||||
function Get-PythonPath {
|
||||
Param($location)
|
||||
if ($location -eq $null) {
|
||||
return Get-PythonRegistryPath
|
||||
}
|
||||
else {
|
||||
$pythonExePath = $null
|
||||
if (Test-Path -LiteralPath $location -PathType Leaf) {
|
||||
$pythonExePath = $location
|
||||
}
|
||||
else {
|
||||
if (Test-Path -Path $location -PathType Container) {
|
||||
$pythonExePath = Join-Path $location $pythonExe
|
||||
if (!(Test-Path -Path $pythonExePath -PathType Leaf)) {
|
||||
$pythonExePath = $null
|
||||
}
|
||||
}
|
||||
else {
|
||||
Write-Error "Error: Python installation cannot be found. Service not installed"
|
||||
break
|
||||
}
|
||||
}
|
||||
$res = Test-ValidPython($pythonExePath)
|
||||
if ($res -eq 0) {
|
||||
return $pythonExePath
|
||||
}
|
||||
}
|
||||
return $null
|
||||
}
|
||||
|
||||
function Get-PythonRegistryPath {
|
||||
$pythonRegPath = Get-ActualPythonRegistryPath($python64RegKey)
|
||||
if ($pythonRegPath -eq $null) {
|
||||
$pythonRegPath = Get-ActualPythonRegistryPath($python32RegKey)
|
||||
}
|
||||
return $pythonRegPath
|
||||
}
|
||||
|
||||
function Get-ActualPythonRegistryPath {
|
||||
Param($registryBasePath)
|
||||
if (Test-Path -Path $registryBasePath -PathType Container) {
|
||||
$list = Get-Item $registryBasePath | select PSPath
|
||||
foreach ($regPath in $list) {
|
||||
$installPath = Join-Path $regPath.PSPath "InstallPath"
|
||||
if (Test-Path -Path $installPath) {
|
||||
$installPathValue = (Get-ItemProperty $installPath "(default)")."(default)"
|
||||
$installPathValue = Join-Path $installPathValue $pythonExe
|
||||
if (Test-Path -LiteralPath $installPathValue -PathType Leaf) {
|
||||
$res = Test-ValidPython($installPathValue)
|
||||
if ($res -eq 0) {
|
||||
return $installPathValue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-SrvanyPath {
|
||||
$currentPath = Resolve-Path .
|
||||
$exePath = Join-Path $currentPath $srvanyExe
|
||||
|
||||
if (Test-Path -LiteralPath $exePath -PathType Leaf) {
|
||||
return $exePath
|
||||
}
|
||||
|
||||
$progFilesPath = [Environment]::GetEnvironmentVariable("ProgramFiles(x86)")
|
||||
if ($progFilesPath -eq $null) {
|
||||
$progFilesPath = [Environment]::GetEnvironmentVariable("ProgramFiles")
|
||||
}
|
||||
$exePath = Join-Path $progFilesPath (Join-Path "\Windows Resource Kits\Tools" $srvanyExe)
|
||||
if (Test-Path -LiteralPath $exePath -PathType Leaf) {
|
||||
return $exePath
|
||||
}
|
||||
else {
|
||||
Write-Error "Error: Windows Resource Toolkit srvany not found"
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
function Create-MMSService {
|
||||
Param($execName)
|
||||
$svcList = Get-Service -Name $serviceName -ErrorAction SilentlyContinue
|
||||
if ($svcList -eq $null -or $svcList.Count -eq 0) {
|
||||
New-Service -Name $serviceName -BinaryPathName $execName `
|
||||
-Description $serviceDesc -DisplayName $serviceDisplayName `
|
||||
-StartupType Automatic | Out-Null
|
||||
}
|
||||
else {
|
||||
Write-Error "Service already exists"
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
function Get-MMSScriptPath {
|
||||
$currentPath = Resolve-Path .
|
||||
$scriptPath = Join-Path $currentPath $scriptName
|
||||
if (Test-Path -LiteralPath $scriptPath -PathType Leaf) {
|
||||
return $scriptPath
|
||||
}
|
||||
else {
|
||||
Write-Error "Error: MMS Script not found"
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
function Modify-MMSService {
|
||||
$appDir = Get-Item $mmsScriptPath | Split-Path -parent
|
||||
$quotedAppDir = Quote-String($appDir)
|
||||
$quotedPythonPath = Quote-String($pythonPath)
|
||||
$quotedScriptPath = Quote-String($mmsScriptPath)
|
||||
$application = $quotedPythonPath+" "+$quotedScriptPath
|
||||
$parameterKey = Join-Path $serviceRegPath "Parameters"
|
||||
New-Item -Path $parameterKey | Out-Null
|
||||
New-ItemProperty -Path $parameterKey -Name "Application" -Value $application | Out-Null
|
||||
New-ItemProperty -Path $parameterKey -Name "AppDir" -Value $quotedAppDir | Out-Null
|
||||
}
|
||||
|
||||
function Start-MMSService {
|
||||
Start-Service -Name $serviceName
|
||||
}
|
||||
|
||||
function Quote-String {
|
||||
Param($unquotedString)
|
||||
return "`"" + $unquotedString + "`""
|
||||
}
|
||||
|
||||
if ($args.Length -gt 0) {
|
||||
$pythonPath = Get-PythonPath($args[0])
|
||||
}
|
||||
else {
|
||||
$pythonPath = Get-PythonPath
|
||||
}
|
||||
|
||||
if ($pythonPath -eq $null) {
|
||||
Write-Error -Message "Valid python install not found. Service not installed" `
|
||||
-Category NotInstalled
|
||||
Exit
|
||||
}
|
||||
|
||||
$srvAnyPath = Get-SrvanyPath
|
||||
$mmsScriptPath = Get-MMSScriptPath
|
||||
|
||||
Create-MMSService(Quote-String($srvAnyPath))
|
||||
Modify-MMSService
|
||||
Write-Host "Service succesfully created"
|
||||
Start-MMSService
|
||||
Write-Host "Service started"
|
||||
170
vendor/mms-agent/munin.py
vendored
170
vendor/mms-agent/munin.py
vendored
|
|
@ -1,170 +0,0 @@
|
|||
"""
|
||||
(C) Copyright 2011, 10gen
|
||||
|
||||
This is a label on a mattress. Do not modify this file!
|
||||
"""
|
||||
|
||||
# Python
|
||||
import socket, threading, time
|
||||
|
||||
muninAgentVersion = "1.5.7"
|
||||
|
||||
def containsStr( val, query ):
|
||||
""" Returns true if the value is contained in the string """
|
||||
return val.find( query ) > -1
|
||||
|
||||
class MuninThread( threading.Thread ):
|
||||
""" Pull them munin data from the various hosts. """
|
||||
|
||||
def __init__( self, hostname, mmsAgent ):
|
||||
""" Initialize the object """
|
||||
self.hostname = hostname
|
||||
self.mmsAgent = mmsAgent
|
||||
self.logger = mmsAgent.logger
|
||||
self.muninNode = MuninNode( self.hostname )
|
||||
self.running = True
|
||||
threading.Thread.__init__( self )
|
||||
|
||||
def run( self ):
|
||||
""" Pull the munin data from the various hosts. """
|
||||
|
||||
self.logger.info( 'starting munin monitoring: ' + self.hostname + ':4949' )
|
||||
|
||||
sleepTime = ( self.mmsAgent.collectionInterval / 2 ) - 1
|
||||
|
||||
if ( sleepTime < 1 ):
|
||||
sleepTime = 1
|
||||
|
||||
while not self.mmsAgent.done and self.mmsAgent.hasUniqueServer( self.hostname ) and self.running:
|
||||
try:
|
||||
time.sleep( sleepTime )
|
||||
self._collectAndSetState()
|
||||
except:
|
||||
pass
|
||||
|
||||
self.logger.info( 'stopping munin monitoring: ' + self.hostname + ':4949' )
|
||||
|
||||
def stopThread( self ):
|
||||
""" Stop the thread. This sets a running flag to false """
|
||||
self.running = False
|
||||
|
||||
def _collectAndSetState( self ):
|
||||
""" Collect the data and set the state """
|
||||
muninStats = self._collectStats()
|
||||
|
||||
if muninStats is None:
|
||||
return
|
||||
|
||||
muninStats['host'] = self.hostname
|
||||
|
||||
self.mmsAgent.setMuninHostState( self.hostname, muninStats )
|
||||
|
||||
def _collectStats( self ):
|
||||
""" Collect the data from the munin host """
|
||||
try:
|
||||
return self.muninNode.fetchAndConfigMany( [ "cpu" , "iostat" , "iostat_ios" ] )
|
||||
except:
|
||||
return None
|
||||
|
||||
class MuninNode( object ):
|
||||
""" The Munin node collection object """
|
||||
|
||||
def __init__( self, host='127.0.0.1', port=4949 ):
|
||||
""" Constructor """
|
||||
self.host = host
|
||||
self.port = port
|
||||
|
||||
def _send( self, cmd, sock ):
|
||||
""" Send a command to Munin """
|
||||
sock.send( cmd + "\r\n" )
|
||||
|
||||
def _readline( self, f ):
|
||||
""" Read data from vendor.munin """
|
||||
return f.readline().split("\n")[0]
|
||||
|
||||
def list( self, sock, f ):
|
||||
""" Run a list operation """
|
||||
self._send( 'list', sock )
|
||||
s = self._readline( f )
|
||||
return s.split( ' ' )
|
||||
|
||||
def config( self, cmd, sock, f ):
|
||||
""" Run a config operation """
|
||||
return self._data( 'config', cmd, sock, f )
|
||||
|
||||
def fetch( self, cmd, sock, f ):
|
||||
""" Run a fetch operation """
|
||||
return self._data( 'fetch', cmd, sock, f )
|
||||
|
||||
def _data( self, cmdType, cmd, sock, f ):
|
||||
""" Collect data """
|
||||
self._send( cmdType + ' ' + cmd, sock )
|
||||
data = []
|
||||
while True:
|
||||
s = self._readline( f )
|
||||
if s == ".":
|
||||
break
|
||||
|
||||
if cmdType == 'config':
|
||||
if containsStr( s, '.label' ) == False:
|
||||
continue
|
||||
|
||||
data.append( s )
|
||||
return data
|
||||
|
||||
def connect( self ):
|
||||
""" Connect to the Munin node """
|
||||
sock = socket.socket( socket.AF_INET, socket.SOCK_STREAM )
|
||||
sock.connect( ( self.host, self.port ) )
|
||||
|
||||
f = sock.makefile()
|
||||
|
||||
if not f:
|
||||
try:
|
||||
sock.close()
|
||||
except:
|
||||
pass
|
||||
|
||||
raise Exception( 'Error reading data from socket' )
|
||||
|
||||
banner = f.readline() # banner
|
||||
|
||||
if len( banner ) == 0:
|
||||
raise Exception( 'Unable to connect to Munin' )
|
||||
|
||||
return ( sock, f )
|
||||
|
||||
def disconnect( self, sock, f ):
|
||||
""" Disconnect from vendor.munin """
|
||||
try:
|
||||
try:
|
||||
self._send( 'quit', sock )
|
||||
finally:
|
||||
sock.close()
|
||||
finally:
|
||||
if f is not None:
|
||||
f.close()
|
||||
|
||||
def fetchAndConfigMany( self, cmdTypes ):
|
||||
""" The fetch and config many cmds - opens and closes the connection """
|
||||
sock = None
|
||||
f = None
|
||||
try:
|
||||
sock, f = self.connect()
|
||||
fetch = {}
|
||||
config = {}
|
||||
for t in cmdTypes:
|
||||
fetch[t] = self.fetch( t, sock, f )
|
||||
|
||||
if ( t == 'cpu' ):
|
||||
config[t] = { }
|
||||
else:
|
||||
config[t] = self.config( t, sock, f )
|
||||
|
||||
return { 'fetch' : fetch, 'config' : config }
|
||||
finally:
|
||||
try:
|
||||
self.disconnect( sock, f )
|
||||
except:
|
||||
pass
|
||||
|
||||
184
vendor/mms-agent/nonBlockingStats.py
vendored
184
vendor/mms-agent/nonBlockingStats.py
vendored
|
|
@ -1,184 +0,0 @@
|
|||
"""
|
||||
(C) Copyright 2011, 10gen
|
||||
|
||||
This is a label on a mattress. Do not modify this file!
|
||||
"""
|
||||
|
||||
import threading, time, pymongo, traceback, warnings, datetime
|
||||
|
||||
warnings.simplefilter( 'ignore', DeprecationWarning )
|
||||
|
||||
nonBlockingStatsAgentVersion = "1.5.7"
|
||||
|
||||
class NonBlockingMongoStatsThread( threading.Thread ):
|
||||
""" Pull the non-blocking data from the various hosts. """
|
||||
|
||||
def __init__( self, hostKey, mmsAgent ):
|
||||
""" Initialize the object """
|
||||
self.hostKey = hostKey
|
||||
self.mmsAgent = mmsAgent
|
||||
self.logger = mmsAgent.logger
|
||||
self.running = True
|
||||
self.host = mmsAgent.extractHostname( hostKey )
|
||||
self.port = mmsAgent.extractPort( hostKey )
|
||||
|
||||
threading.Thread.__init__( self )
|
||||
|
||||
def stopThread( self ):
|
||||
""" Stop the thread """
|
||||
self.running = False
|
||||
|
||||
|
||||
def run( self ):
|
||||
""" The thread to collect stats """
|
||||
|
||||
self.logger.info( 'starting non-blocking stats monitoring: ' + self.hostKey )
|
||||
|
||||
sleepTime = ( self.mmsAgent.collectionInterval / 2 ) - 1
|
||||
|
||||
if ( sleepTime < 1 ):
|
||||
sleepTime = 1
|
||||
|
||||
monitorConn = None
|
||||
|
||||
passes = 0
|
||||
|
||||
while not self.mmsAgent.done and self.running:
|
||||
try:
|
||||
time.sleep( sleepTime )
|
||||
passes = passes + 1
|
||||
|
||||
# Close the connection periodically
|
||||
if passes % 60 == 0:
|
||||
monitorConn = self.mmsAgent.closeDbConnection( self.hostKey, monitorConn )
|
||||
|
||||
if not monitorConn:
|
||||
monitorConn = self.mmsAgent.getDbConnection( self.hostKey )
|
||||
|
||||
if not self._collectNonBlockingStats( monitorConn ):
|
||||
monitorConn = self.mmsAgent.closeDbConnection( self.hostKey, monitorConn )
|
||||
|
||||
except Exception as e:
|
||||
monitorConn = self.mmsAgent.closeDbConnection( self.hostKey, monitorConn )
|
||||
self.logger.error( 'Problem collecting non-blocking data from: ' + self.hostKey + " - exception: " + traceback.format_exc( e ) )
|
||||
|
||||
self.logger.info( 'stopping non-blocking stats monitoring: ' + self.hostKey )
|
||||
self.mmsAgent.closeDbConnection( self.hostKey, monitorConn )
|
||||
|
||||
def _collectNonBlockingStats( self, monitorConn ):
|
||||
""" Collect the non-blocking stats """
|
||||
try:
|
||||
if not self.mmsAgent.haveHostDef( self.hostKey ):
|
||||
return False
|
||||
|
||||
# Verify the connection.
|
||||
if not self.mmsAgent.isValidMonitorConn( self.hostKey, monitorConn ):
|
||||
return False
|
||||
|
||||
stats = self._collectStats( monitorConn )
|
||||
|
||||
stats['host'] = self.host
|
||||
stats['port'] = self.port
|
||||
|
||||
# Make sure we ended up with the same connection.
|
||||
if not self.mmsAgent.isValidMonitorConn( self.hostKey, monitorConn ):
|
||||
return False
|
||||
|
||||
self.mmsAgent.setHostState( self.hostKey, 'mongoNonBlocking', stats )
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
self.logger.warning( 'Problem collecting non-blocking data from (check if it is up and DNS): ' + self.hostKey + ' - ' + traceback.format_exc( e ) )
|
||||
return False
|
||||
|
||||
def _collectStats( self, monitorConn ):
|
||||
""" Make the call to mongo host and collect the data """
|
||||
root = {}
|
||||
|
||||
# Set the agent version and hostname.
|
||||
root['agentVersion'] = self.mmsAgent.agentVersion
|
||||
root['agentHostname'] = self.mmsAgent.agentHostname
|
||||
root['agentSessionKey'] = self.mmsAgent.sessionKey
|
||||
|
||||
cmdStartTime = datetime.datetime.now()
|
||||
root['serverStatus'] = monitorConn.admin.command( 'serverStatus' )
|
||||
cmdExecTime = datetime.datetime.now() - cmdStartTime
|
||||
root['serverStatusExecTimeMs'] = ( cmdExecTime.days * 24 * 60 * 60 + cmdExecTime.seconds ) * 1000 + cmdExecTime.microseconds / 1000.0
|
||||
|
||||
# The server build info
|
||||
root['buildInfo'] = monitorConn.admin.command( 'buildinfo' )
|
||||
|
||||
# Try and get the command line operations
|
||||
try:
|
||||
root['cmdLineOpts'] = monitorConn.admin.command( 'getCmdLineOpts' )
|
||||
except Exception as e:
|
||||
self.mmsAgent.handleOperationFailure( self.hostKey, 'getCmdLineOpts', e )
|
||||
|
||||
# Get the connection pool stats.
|
||||
try:
|
||||
root['connPoolStats'] = monitorConn.admin.command( 'connPoolStats' )
|
||||
except Exception as e:
|
||||
self.mmsAgent.handleOperationFailure( self.hostKey, 'connPoolStats', e )
|
||||
|
||||
# Look for any startup warnings if this is a valid version of mongo.
|
||||
if self.mmsAgent.hasCommand( 'getLog', self.hostKey, monitorConn ):
|
||||
try:
|
||||
root['startupWarnings'] = monitorConn.admin.command( { 'getLog' : 'startupWarnings' } )
|
||||
except Exception as e:
|
||||
self.mmsAgent.handleOperationFailure( self.hostKey, 'getLog.startupWarnings', e )
|
||||
|
||||
|
||||
# See if we can get hostInfo.
|
||||
if self.mmsAgent.hasCommand( 'hostInfo', self.hostKey, monitorConn ):
|
||||
try:
|
||||
root['hostInfo'] = monitorConn.admin.command( { 'hostInfo' : 1 } )
|
||||
except Exception as e:
|
||||
self.mmsAgent.handleOperationFailure( self.hostKey, 'hostInfo', e )
|
||||
|
||||
# Try and get the isSelf data
|
||||
try:
|
||||
root['isSelf'] = monitorConn.admin.command( '_isSelf' )
|
||||
except Exception as e:
|
||||
self.mmsAgent.handleOperationFailure( self.hostKey, '_isSelf', e )
|
||||
|
||||
# Get the params.
|
||||
try:
|
||||
root['getParameterAll'] = monitorConn.admin.command( { 'getParameter' : '*' } )
|
||||
except Exception as e:
|
||||
self.mmsAgent.handleOperationFailure( self.hostKey, 'getParameter.*', e )
|
||||
|
||||
# Check occasionally to see if we can discover nodes
|
||||
isMaster = monitorConn.admin.command( 'ismaster' )
|
||||
root['isMaster'] = isMaster
|
||||
|
||||
# Try and get the shard version
|
||||
if isMaster['ismaster'] and 'msg' in isMaster:
|
||||
if isMaster['msg'] != 'isdbgrid':
|
||||
|
||||
try:
|
||||
root['shardVersion'] = monitorConn.admin.command( { 'getShardVersion' : 'mdbfoo.foo' } )
|
||||
except Exception as e:
|
||||
self.mmsAgent.handleOperationFailure( self.hostKey, 'getShardVersion.mdbfoo.foo', e )
|
||||
|
||||
elif isMaster['ismaster']:
|
||||
try:
|
||||
root['shardVersion'] = monitorConn.admin.command( { 'getShardVersion' : 'mdbfoo.foo' } )
|
||||
except Exception as e:
|
||||
self.mmsAgent.handleOperationFailure( self.hostKey, 'getShardVersion.mdbfoo.foo', e )
|
||||
|
||||
# Check to see if this is a mongod host
|
||||
try:
|
||||
if isMaster['ismaster'] and isMaster.get('msg', '') == 'isdbgrid':
|
||||
root['netstat'] = monitorConn.admin.command( 'netstat' )
|
||||
except pymongo.errors.OperationFailure as e:
|
||||
self.mmsAgent.handleOperationFailure( self.hostKey, 'netstat', e )
|
||||
|
||||
if 'repl' in root['serverStatus']:
|
||||
try:
|
||||
root['replStatus'] = monitorConn.admin.command( 'replSetGetStatus' )
|
||||
except pymongo.errors.OperationFailure as e:
|
||||
self.mmsAgent.handleOperationFailure( self.hostKey, 'replSetGetStatus', e )
|
||||
|
||||
return root
|
||||
|
||||
7
vendor/mms-agent/pymongotest.py
vendored
7
vendor/mms-agent/pymongotest.py
vendored
|
|
@ -1,7 +0,0 @@
|
|||
import sys
|
||||
try:
|
||||
import pymongo
|
||||
sys.exit(0)
|
||||
except Exception as exc:
|
||||
sys.exit(-1)
|
||||
|
||||
6
vendor/mms-agent/pythonversiontest.py
vendored
6
vendor/mms-agent/pythonversiontest.py
vendored
|
|
@ -1,6 +0,0 @@
|
|||
import sys
|
||||
major_version=sys.version_info[0]
|
||||
if major_version == 2:
|
||||
sys.exit(0)
|
||||
else:
|
||||
sys.exit(-1)
|
||||
BIN
vendor/mms-agent/srvany.exe
vendored
BIN
vendor/mms-agent/srvany.exe
vendored
Binary file not shown.
Loading…
Add table
Reference in a new issue