Several fixes to configuration, dependencies, and support tools.
root [Fri, 13 May 2011 19:00:19 +0000 (19:00 +0000)]
   Added missing dependencies to .spec
   Added configurable dbuser, dbname, and dbhost  >= myplc-5.0-18
   Added external commands to fetch.py
   Added googlevis java script template

Monitor.spec
monitor-server.init
monitor/wrapper/plccache.py
tools/automate/fetch.py
tools/getsshkeys.py
tools/setup-agent.sh
web/MonitorWeb/monitorweb/static/javascript/googlevis.js [new file with mode: 0644]
web/monitorweb.conf [moved from web/monitorweb-httpd.conf with 100% similarity]

index 68ed180..3bb70c4 100644 (file)
@@ -96,10 +96,13 @@ Requires: openssh-clients
 Requires: perl-libwww-perl
 Requires: perl-IO-Socket-SSL 
 Requires: MySQL-python
+Requires: perl-DBD-mysql
 Requires: nmap
+Requires: mailx
 Requires: nc
 Requires: rt3
 Requires: traceroute
+Requires: sendmail
 
 Requires: plewww-plekit
 Requires: pcucontrol
@@ -173,7 +176,7 @@ install -D -m 644 nagios/monitor-nagios.cron $RPM_BUILD_ROOT/%{_sysconfdir}/cron
 install -D -m 644 monitor-server.cron $RPM_BUILD_ROOT/%{_sysconfdir}/cron.d/monitor-server.cron
 
 # apache configuration
-install -D -m 644 web/monitorweb-httpd.conf $RPM_BUILD_ROOT/etc/httpd/conf.d/
+install -D -m 644 web/monitorweb.conf $RPM_BUILD_ROOT/etc/httpd/conf.d/
 
 # we'll install monitor in site-packages install rest to
 # /usr/share/monitor
@@ -283,6 +286,8 @@ export TMPDIR=/var/tmp/
 #easy_install -UZ http://files.turbogears.org/eggs/TurboGears-1.0.7-py2.5.egg
 #easy_install -UZ http://pypi.python.org/packages/source/S/SQLAlchemy/SQLAlchemy-0.5.3.tar.gz
 #easy_install -UZ Elixir
+chkconfig sendmail on
+chkconfig crond on
 
 # crazy openssl libs for racadm binary
 ln -s /lib/libssl.so.0.9.8b /usr/lib/libssl.so.2
index 32d1209..544b47e 100644 (file)
@@ -33,8 +33,8 @@ pghba_conf=$PGDATA/pg_hba.conf
 export PGPORT=$PLC_DB_PORT
 
 
-MONITOR_DB_USER="monitoruser"
-MONITOR_DB_NAME="monitor"
+MONITOR_DB_USER=$PLC_MONITOR_DBUSER
+MONITOR_DB_NAME=$PLC_MONITOR_DBNAME
 
 WROTE_PG_CONFIG=
 
@@ -123,14 +123,14 @@ MONITOR_IP=${PLC_MONITOR_IP}
 PLC_WWW_HOSTNAME=${PLC_WWW_HOST}
 PLC_NAME=${PLC_NAME}
 
-# used for debug mode
-email=${PLC_MONITOR_CC_EMAIL}
+email=${PLC_MONITOR_EMAIL}
 # all messages will appear to be from this address
 from_email=${PLC_MONITOR_EMAIL}
 # a separate address for support messages
 support_email=${PLC_MAIL_SUPPORT_ADDRESS}
 # mailing list copied on all out-going messages
 cc_email=${PLC_MONITOR_CC_EMAIL}
+exception_email=${PLC_MONITOR_CC_EMAIL}
 
 # these are reserved values
 RT_DB_HOST=${PLC_RT_HOST}
@@ -139,8 +139,7 @@ RT_DB_PASSWORD=
 RT_DB_NAME=
 
 [monitordatabase]
-monitor_dburi=postgres://${MONITOR_DB_USER}:${PLC_MONITOR_DBPASSWORD}@localhost:5432/${MONITOR_DB_NAME}
-zabbix_dburi=postgres://${ZABBIX_DB_USER}:${PLC_MONITOR_DBPASSWORD}@localhost:5432/${ZABBIX_DB_NAME}
+monitor_dburi=postgres://${MONITOR_DB_USER}:${PLC_MONITOR_DBPASSWORD}@${PLC_MONITOR_DBHOST}:5432/${MONITOR_DB_NAME}
 
 cachetime=60
 
@@ -209,7 +208,7 @@ function start_tg_server ()
 {
        stop_tg_server
        pushd ${MONITORPATH}/web/MonitorWeb/
-       mkdir -p /var/log/monitor/monitorweb/
+       mkdir -p /var/lib/monitor/monitorweb/
        cp /var/log/monitorweb.log /var/lib/monitor/monitorweb/`date +%Y-%m-%d-%H-%M`-monitorweb.log
        ./start-monitorweb.py ${MONITORPATH}/web/MonitorWeb/prod.cfg &> /var/log/monitorweb.log &
        popd
index 4778a7d..d818e97 100755 (executable)
@@ -163,7 +163,8 @@ def sync():
        l_nodes = plc.api.GetNodes({'peer_id':None}, 
                                                ['hostname', 'node_id', 'ports', 'site_id', 'boot_state', 'run_level',
                                                 'version', 'last_updated', 'date_created', 'key',
-                                                'last_contact', 'pcu_ids', 'interface_ids'])
+                                                'last_contact', 'pcu_ids', 'interface_ids',
+                                                'last_boot', 'last_download', 'last_pcu_reboot', 'last_pcu_confirmation'])
        l_pcus = plc.api.GetPCUs()
 
        print >>sys.stderr, "sync sites"
index 35cb9da..01fb399 100755 (executable)
@@ -10,72 +10,110 @@ from monitor import parser as parsermodule
 from monitor import common
 from automate import *
 
+def build_vx_args_external(shell_cmd):
+    args = shell_cmd.split()
+    return args
+
+def vx_start_external(filelist,outdir,cmd, timeout=0, threadcount=20):
+    args = build_vx_args_external(cmd)
+    vxargs.start(None, threadcount, filelist, outdir, False, args, timeout)
+
 def build_vx_args(shell_cmd):
-       ssh_options="-q -o UserKnownHostsFile=junkssh -o StrictHostKeyChecking=no"
-       cmd="""ssh %s root@{} """  % ssh_options
-       args = cmd.split()
-       args.append(shell_cmd)
-       return args
+    ssh_options="-q -o UserKnownHostsFile=junkssh -o StrictHostKeyChecking=no"
+    cmd="""ssh %s root@{} """  % ssh_options
+    args = cmd.split()
+    args.append(shell_cmd)
+    return args
 
 def vx_start(filelist,outdir,cmd, timeout=0):
-       args = build_vx_args(cmd)
-       vxargs.start(None, 20, filelist, outdir, False, args, timeout)
+    args = build_vx_args(cmd)
+    vxargs.start(None, 20, filelist, outdir, False, args, timeout)
+
+def get_hostlist_from_myops(filter):
+    ret = []
+    curl_url = "curl -s 'http://myops.planet-lab.org:5984/query"
+    curl_params = "fields=hostname&skip_header&filter=%s'" % filter
+    curl_cmd = "%s?%s" % (curl_url, curl_params)
+    print curl_cmd
+    f = os.popen(curl_cmd, 'r')
+    for h in f.read().split():
+        ret.append((h, ''))
+    return ret
 
 if __name__ == "__main__":
 
-       parser = parsermodule.getParser(['nodesets'])
-       parser.set_defaults(outdir=None,
-                                               timeout=0,
-                                               simple=False,
-                                               run=False,
-                                               cmdfile=None,)
-
-       parser.add_option("", "--timeout", dest="timeout", metavar="seconds",
-                                               help="Number of seconds to wait before timing out on host.")
-       parser.add_option("", "--outdir", dest="outdir", metavar="dirname",
-                                               help="Name of directory to place output")
-       parser.add_option("", "--cmd", dest="cmdfile", metavar="filename",
-                                               help="Name of file that contains a unix-to-csv command " + \
-                                                        "to run on the hosts.")
-
-       config = parsermodule.parse_args(parser)
-
-       if config.outdir == None: 
-               outdir="checkhosts"
-       else: 
-               outdir=config.outdir
-
-       if not os.path.exists(outdir):
-               os.system('mkdir -p %s' % outdir)
-
-       #if config.site is not None or \
-       #   config.nodeselect is not None or \
-       #   config.nodegroup is not None:
-       #       print "TODO: implement support for nodeselect and site queries."
-       #       print "%s %s %s" % (config.site, config.nodeselect, config.nodegroup)
-       #       sys.exit(1)
-
-       nodelist = common.get_nodeset(config)
-
-       if len(nodelist) > 0:
-               filelist = [ (x, '') for x in nodelist ]
-       elif os.path.exists(str(config.nodelist)) and os.path.isfile(config.nodelist):
-               filelist = vxargs.getListFromFile(open(config.nodelist,'r'))
-       elif os.path.exists(str(config.nodelist)) and os.path.isdir(config.nodelist):
-               filelist = get_hostlist_from_dir(config.nodelist)
-       elif config.node is not None:
-               filelist = [(config.node, '')]
-       else:
-               # probably no such file.
-               raise Exception("No such file %s" % config.nodelist)
-
-       if config.cmdfile == None:
-               f = open("command.txt",'r')
-               cmd = f.read()
-       else:
-               f = open(config.cmdfile,'r')
-               cmd = f.read()
-
-       print filelist
-
-       vx_start(filelist, outdir, cmd, int(config.timeout))
+    parser = parsermodule.getParser(['nodesets'])
+    parser.set_defaults(outdir=None,
+                        timeout=0,
+                        simple=False,
+                        threadcount=20,
+                        external=False,
+                        myopsfilter=None,
+                        run=False,
+                        template=None,
+                        cmdfile=None,)
+
+    parser.add_option("", "--timeout", dest="timeout", metavar="seconds",
+                        help="Number of seconds to wait before timing out on host.")
+    parser.add_option("", "--myopsfilter", dest="myopsfilter", metavar="",
+                        help="filter string to pass directly to myops query")
+    parser.add_option("", "--outdir", dest="outdir", metavar="dirname",
+                        help="Name of directory to place output")
+    parser.add_option("", "--cmd", dest="cmdfile", metavar="filename",
+                        help="Name of file that contains a unix-to-csv command " + \
+                             "to run on the hosts.")
+    parser.add_option("", "--external", dest="external",  action="store_true",
+                        help="Run commands external to the server. The default is internal.")
+    parser.add_option("", "--template", dest="template", 
+                        help="Command template for external commands; substitutes {} with hostname.")
+    parser.add_option("", "--threadcount", dest="threadcount", 
+                        help="Number of simultaneous threads: default 20.")
+
+    config = parsermodule.parse_args(parser)
+
+    if config.outdir == None: 
+        outdir="checkhosts"
+    else: 
+        outdir=config.outdir
+
+    if not os.path.exists(outdir):
+        os.system('mkdir -p %s' % outdir)
+
+    #if config.site is not None or \
+    #   config.nodeselect is not None or \
+    #   config.nodegroup is not None:
+    #    print "TODO: implement support for nodeselect and site queries."
+    #    print "%s %s %s" % (config.site, config.nodeselect, config.nodegroup)
+    #    sys.exit(1)
+    if config.myopsfilter is not None:
+        filelist = get_hostlist_from_myops(config.myopsfilter)
+    else:
+        nodelist = common.get_nodeset(config)
+        if len(nodelist) > 0:
+            filelist = [ (x, '') for x in nodelist ]
+        elif os.path.exists(str(config.nodelist)) and os.path.isfile(config.nodelist):
+            filelist = vxargs.getListFromFile(open(config.nodelist,'r'))
+        elif os.path.exists(str(config.nodelist)) and os.path.isdir(config.nodelist):
+            filelist = get_hostlist_from_dir(config.nodelist)
+        elif config.node is not None:
+            filelist = [(config.node, '')]
+        else:
+            # probably no such file.
+            raise Exception("No such file %s" % config.nodelist)
+
+    if config.cmdfile == None and config.template == None:
+        f = open("command.txt",'r')
+        cmd = f.read()
+    elif config.template is not None and config.external:
+        cmd = config.template
+    else:
+        f = open(config.cmdfile,'r')
+        cmd = f.read()
+
+    print filelist
+
+    if config.external or config.template is not None:
+        vx_start_external(filelist, outdir, cmd, int(config.timeout), int(config.threadcount))
+    else:
+        vx_start(filelist, outdir, cmd, int(config.timeout))
+        
index 0104deb..6d04ce2 100755 (executable)
@@ -4,7 +4,7 @@ import sys
 sys.path.append('.')
 sys.path.append('..')
 
-from monitor.util.file import *
+from monitor.util.sshknownhosts import *
 
 def main(hosts):
        k = SSHKnownHosts()
index b78dcf0..9cae55b 100755 (executable)
@@ -2,7 +2,7 @@
 
 # NOTE: Must be an absolute path to guarantee it is read.
 INSTALLPATH=/usr/share/monitor/
-$INSTALLPATH/shconfig.py >  $INSTALLPATH/monitorconfig.sh
+$INSTALLPATH/commands/shconfig.py >  $INSTALLPATH/monitorconfig.sh
 source $INSTALLPATH/monitorconfig.sh
 cd ${MONITOR_SCRIPT_ROOT}
 set -e
diff --git a/web/MonitorWeb/monitorweb/static/javascript/googlevis.js b/web/MonitorWeb/monitorweb/static/javascript/googlevis.js
new file mode 100644 (file)
index 0000000..f9c98db
--- /dev/null
@@ -0,0 +1,43 @@
+/*
+ *  NOTE: requires the Google JSAPI using a key for your domain.
+    <script type='text/javascript' src='https://www.google.com/jsapi?key=KEY-GOES-HERE'></script>
+ */
+google.load('visualization', '1', {'packages':['annotatedtimeline']});
+google.load("jquery", "1");
+
+function getData2() {
+    // To see the data that this visualization uses, browse to
+    var link = 'http://spreadsheets.google.com/tq?key=0AubvfWUvLDrpdGs2eGdwNndDUEZSV2hfNFprTkQxUkE&hl=en&authkey=CO3y2JoO&sheet=NodeHistory';
+    var query = new google.visualization.Query(link);
+    
+    // Apply query language.
+    //console.log("run select");
+    query.setQuery('SELECT A,B+E,I,J,G+H,K,L');
+    // Send the query with a callback function.
+    query.send(handleQueryResponse);
+}
+function handleQueryResponse(response) {
+    if (response.isError()) {
+        alert('Error in query: ' + response.getMessage() + ' ' + response.getDetailedMessage());
+        return;
+    }
+    var data = response.getDataTable();
+    //console.log("loaded data");
+    //console.log(data);
+    if ( data ) {
+        drawChart2(data);
+    }
+}
+function drawChart2(data) {
+      //data.insertColumn(2, "string", "title1");
+      //data.insertColumn(3, "string", "text1");
+      //data.insertColumn(5, "string", "title1");
+      //data.insertColumn(6, "string", "text1");
+      data.setColumnLabel(1, "Production Nodes");
+      data.setColumnLabel(4, "Failing Nodes");
+      console.log(data);
+      var chart = new google.visualization.AnnotatedTimeLine(document.getElementById('chart_div'));
+      var st = new Date();
+      st.setDate(st.getDate()-60);
+      chart.draw(data, {displayAnnotations: true, zoomStartTime: st, zoomEndTime: new Date()});
+}