* new interactive option
Thierry Parmentelat [Tue, 18 Nov 2008 12:24:25 +0000 (12:24 +0000)]
 * nodes IP addresses saved
 * config display now a separate step
 * default config has 2 nodes again
 * onelab config reviewed, using more testboxes

system/TestMain.py
system/TestMapper.py
system/TestNode.py
system/TestPlc.py
system/config_1testbox.py
system/config_1vnodes.py
system/config_1vservers.py
system/config_default.py
system/config_main.py
system/config_wifilab.py
system/utils.py

index c99fe85..b0db61f 100755 (executable)
@@ -5,6 +5,7 @@ import sys, os, os.path
 from optparse import OptionParser
 import traceback
 from time import strftime
+import readline
 
 import utils
 from TestPlc import TestPlc
@@ -45,7 +46,7 @@ build-url defaults to the last value used, as stored in arg-build-url,
    or %s
 config defaults to the last value used, as stored in arg-config,
    or %r
-node-ips and plc-ips defaults to the last value used, as stored in arg-node-ips and arg-plc-ips,
+node-ips and plc-ips defaults to the last value used, as stored in arg-ips-node and arg-ips-plc,
    default is to use IP scanning
 steps refer to a method in TestPlc or to a step_* module
 ===
@@ -72,10 +73,10 @@ steps refer to a method in TestPlc or to a step_* module
                           help="Run all default steps")
         parser.add_option("-l","--list",action="store_true",dest="list_steps", default=False,
                           help="List known steps")
-        parser.add_option("-N","--nodes",action="callback", callback=TestMain.optparse_list, dest="node_ips",
+        parser.add_option("-N","--nodes",action="callback", callback=TestMain.optparse_list, dest="ips_node",
                           nargs=1,type="string",
                           help="Specify the set of IP addresses to use for nodes (scanning disabled)")
-        parser.add_option("-P","--plcs",action="callback", callback=TestMain.optparse_list, dest="plc_ips",
+        parser.add_option("-P","--plcs",action="callback", callback=TestMain.optparse_list, dest="ips_plc",
                           nargs=1,type="string",
                           help="Specify the set of IP addresses to use for plcs (scanning disabled)")
         parser.add_option("-1","--small",action="store_true",dest="small_test",default=False,
@@ -86,6 +87,8 @@ steps refer to a method in TestPlc or to a step_* module
                           help="Run in verbose mode")
         parser.add_option("-q","--quiet", action="store_true", dest="quiet", default=False, 
                           help="Run in quiet mode")
+        parser.add_option("-i","--interactive",action="store_true",dest="interactive",default=False,
+                          help="prompts before each step")
         parser.add_option("-n","--dry-run", action="store_true", dest="dry_run", default=False,
                           help="Show environment and exits")
         parser.add_option("-r","--restart-nm", action="store_true", dest="forcenm", default=False, 
@@ -113,8 +116,8 @@ steps refer to a method in TestPlc or to a step_* module
         # handle defaults and option persistence
         for (recname,filename,default) in (
             ('build_url','arg-build-url',TestMain.default_build_url) ,
-            ('node_ips','arg-node-ips',[]) , 
-            ('plc_ips','arg-plc-ips',[]) , 
+            ('ips_node','arg-ips-node',[]) , 
+            ('ips_plc','arg-ips-plc',[]) , 
             ('config','arg-config',TestMain.default_config) , 
             ('arch_rpms_url','arg-arch-rpms-url',"") , 
             ('personality','arg-personality',"linux32"),
@@ -195,16 +198,22 @@ steps refer to a method in TestPlc or to a step_* module
                 traceback.print_exc()
                 print 'Cannot load config %s -- ignored'%modulename
                 raise
-        # show config
-        if not self.options.quiet:
-            utils.show_test_spec("Test specifications",all_plc_specs)
         # remember plc IP address(es) if not specified
-        current=file('arg-plc-ips').read()
+        current=file('arg-ips-plc').read()
         if not current:
-            plc_ips_file=open('arg-plc-ips','w')
+            ips_plc_file=open('arg-ips-plc','w')
             for plc_spec in all_plc_specs:
-                plc_ips_file.write("%s\n"%plc_spec['PLC_API_HOST'])
-            plc_ips_file.close()
+                ips_plc_file.write("%s\n"%plc_spec['PLC_API_HOST'])
+            ips_plc_file.close()
+        # ditto for nodes
+        current=file('arg-ips-node').read()
+        if not current:
+            ips_node_file=open('arg-ips-node','w')
+            for plc_spec in all_plc_specs:
+                for site_spec in plc_spec['sites']:
+                    for node_spec in site_spec['nodes']:
+                        ips_node_file.write("%s\n"%node_spec['node_fields']['hostname'])
+            ips_node_file.close()
         # build a TestPlc object from the result, passing options
         for spec in all_plc_specs:
             spec['disabled'] = False
@@ -268,7 +277,17 @@ steps refer to a method in TestPlc or to a step_* module
 
                 # run the step
                 time=strftime("%Y-%m-%d-%H-%M")
-                if not spec['disabled'] or force:
+                if not spec['disabled'] or force or self.options.interactive:
+                    if self.options.interactive:
+                        msg="Run step %s on %s [y]/n/q ? "%(stepname,plcname)
+                        answer=raw_input(msg).strip().lower() or "y"
+                        answer=answer[0]
+                        if answer in ['n']:
+                            print '%s on %s skipped'%(stepname,plcname)
+                            continue
+                        elif answer in ['q','b']:
+                            print 'Exiting'
+                            return
                     try:
                         force_msg=""
                         if force: force_msg=" (forced)"
index d60cf05..13a0780 100644 (file)
@@ -26,6 +26,8 @@ class TestMapper:
 
     def apply_first_map (self, type, name, obj, maplist):
         for (map_pattern,rename_dict) in maplist:
+            if self.options.verbose:
+                print 'dbg: TestMapper/',type,'name=',name,'& pattern=',map_pattern
             if utils.match (name,map_pattern):
                 utils.header("TestMapper/%s : applying rules '%s' on %s"%(type,map_pattern,name))
                 for (k,v) in rename_dict.iteritems():
@@ -62,14 +64,8 @@ class TestMapper:
 
     def map (self,mapper):
 
-        try:
-            plc_maps = mapper['plc']
-        except:
-            plc_maps = []
-        try:
-            node_maps = mapper['node']
-        except:
-            node_maps = []
+        plc_maps  = mapper.get('plc',[])
+        node_maps = mapper.get('node',[])
 
         for plc in self.plcs:
             name=TestMapper.plc_name(plc)
index 7f682c0..08dd17c 100644 (file)
@@ -71,16 +71,16 @@ class TestNode:
         server.UpdateNode(userauth, self.name(), {'boot_state':'reinstall'})
         # populate network interfaces - primary
         server.AddInterface(userauth,self.name(),
-                                            self.node_spec['network_fields'])
+                                            self.node_spec['interface_fields'])
         # populate network interfaces - others
         if self.node_spec.has_key('extra_interfaces'):
             for interface in self.node_spec['extra_interfaces']:
                 server.AddInterface(userauth,self.name(),
-                                                    interface['network_fields'])
+                                                    interface['interface_fields'])
                 if interface.has_key('settings'):
                     for (attribute,value) in interface['settings'].iteritems():
                         # locate node network
-                        nn = server.GetInterfaces(userauth,{'ip':interface['network_fields']['ip']})[0]
+                        nn = server.GetInterfaces(userauth,{'ip':interface['interface_fields']['ip']})[0]
                         nnid=nn['interface_id']
                         # locate or create node network attribute type
                         try:
@@ -137,9 +137,9 @@ class TestNode:
     def configure_qemu(self):
         if not self.is_qemu():
             return
-        mac=self.node_spec['network_fields']['mac']
+        mac=self.node_spec['interface_fields']['mac']
         hostname=self.node_spec['node_fields']['hostname']
-        ip=self.node_spec['network_fields']['ip']
+        ip=self.node_spec['interface_fields']['ip']
         auth=self.test_plc.auth_root()
         target_arch=self.test_plc.apiserver.GetPlcRelease(auth)['build']['target-arch']
         conf_filename="%s/qemu.conf"%(self.nodedir())
index 734ee7c..f5ea38e 100644 (file)
@@ -62,7 +62,7 @@ SEP='<sep>'
 
 class TestPlc:
 
-    default_steps = ['uninstall','install','install_rpm', 
+    default_steps = ['display','uninstall','install','install_rpm', 
                      'configure', 'start', 'fetch_keys', SEP,
                      'store_keys', 'clear_known_hosts', 'initscripts', SEP,
                      'sites', 'nodes', 'slices', 'nodegroups', SEP,
@@ -267,6 +267,9 @@ class TestPlc:
                 node.kill_qemu()
         return True
 
+    def display (self):
+        utils.show_plc_spec (self.plc_spec)
+        return True
 
     ### utility methods for handling the pool of IP addresses allocated to plcs
     # Logic
index aa44e7d..1a7f37e 100644 (file)
@@ -7,9 +7,15 @@ from TestMapper import TestMapper
 def config (plcs, options):
 
     if options.arch == "i386":
-        target = 'testbox32.onelab.eu'
+        testbox1 = 'testbox32.onelab.eu'
+        testbox2 = 'testbox32.onelab.eu'
+        target=testbox1
     elif options.arch == "x86_64":
-        target = 'testbox64.onelab.eu'
+#        testbox1 = 'testbox64-1.onelab.eu'
+#        testbox2 = 'testbox64-2.onelab.eu'
+        testbox1 = 'testbox64.onelab.eu'
+        testbox2 = 'estran.inria.fr'
+        target=testbox1
     else:
         print 'Unsupported arch %s'%options.arch
         sys.exit(1)
@@ -19,8 +25,11 @@ def config (plcs, options):
                                'PLC_API_HOST':target,
                                'PLC_BOOT_HOST':target,
                                'PLC_WWW_HOST':target,
-                               'name':'%s-'+options.arch } ) ],
-              'node': [ ('*' , {'host_box': target } ) ],
+                               'name':'%s-'+options.arch } ) 
+                       ],
+              'node': [ ('deferred01' , {'host_box': testbox1 } ),
+                        ('deferred02' , {'host_box': testbox2 } ),
+                        ],
               }
     
     return TestMapper(plcs,options).map(mapper)
index 2474a75..48862a4 100644 (file)
@@ -6,12 +6,12 @@ from TestPool import TestPool
 onelab_plcs_pool = [ 
     ( 'vnode%02d.inria.fr'%i, '138.96.255.%d'%(220+i), '02:34:56:00:00:%02d'%i) for i in range(1,10) ]
 site_dict = {
-    'network_fields:gateway':'138.96.248.250',
-    'network_fields:network':'138.96.0.0',
-    'network_fields:broadcast':'138.96.255.255',
-    'network_fields:netmask':'255.255.0.0',
-    'network_fields:dns1': '138.96.0.10',
-    'network_fields:dns2': '138.96.0.11',
+    'interface_fields:gateway':'138.96.248.250',
+    'interface_fields:network':'138.96.0.0',
+    'interface_fields:broadcast':'138.96.255.255',
+    'interface_fields:netmask':'255.255.0.0',
+    'interface_fields:dns1': '138.96.0.10',
+    'interface_fields:dns2': '138.96.0.11',
 }
 
 def config (plcs, options):
@@ -22,21 +22,21 @@ def config (plcs, options):
     all_nodenames = test_mapper.node_names()
     maps = []
     for nodename in all_nodenames:
-        if len(options.node_ips) != 0:
-            ip=options.node_ips[0]
-            options.node_ips=options.node_ips[1:]
+        if len(options.ips_node) != 0:
+            ip=options.ips_node[0]
+            options.ips_node=options.ips_node[1:]
             (hostname,ip,mac)=test_pool.locate(ip)
         else:
             (hostname,ip,mac) = test_pool.next_free()
         node_dict= {'node_fields:hostname':hostname,
-                    'network_fields:ip':ip, 
-                    'network_fields:mac':mac,
+                    'interface_fields:ip':ip, 
+                    'interface_fields:mac':mac,
                     }
     
         node_dict.update(site_dict)
         maps.append ( ( nodename, node_dict) )
 
-    plc_map = [ ( '*' , { 'PLC_NET_DNS1' : site_dict [ 'network_fields:dns1' ],
-                        'PLC_NET_DNS2' : site_dict [ 'network_fields:dns2' ], } ) ]
+    plc_map = [ ( '*' , { 'PLC_NET_DNS1' : site_dict [ 'interface_fields:dns1' ],
+                        'PLC_NET_DNS2' : site_dict [ 'interface_fields:dns2' ], } ) ]
 
     return test_mapper.map ({'node': maps, 'plc' : plc_map } )
index 8713096..6e63fe3 100644 (file)
@@ -15,16 +15,16 @@ def config (plcs,options):
 
     test_pool = TestPool (onelab_plcs_pool,options)
 
-    if len(options.plc_ips) != 0:
-        utils.header('Using user-provided IPS:\nplc_ips=%r'%options.plc_ips)
-        options.plc_ips.reverse()
+    if len(options.ips_plc) != 0:
+        utils.header('Using user-provided IPS:\nips_plc=%r'%options.ips_plc)
+        options.ips_plc.reverse()
 
     plc_counter=0
     for plc in plcs:
         try:
-            if len(options.plc_ips) != 0:
-                ip=options.plc_ips[0]
-                options.plc_ips=options.plc_ips[1:]
+            if len(options.ips_plc) != 0:
+                ip=options.ips_plc[0]
+                options.ips_plc=options.ips_plc[1:]
                 (hostname,ip,mac)=test_pool.locate(ip)
                 utils.header("Using user-provided %s %s for plc %s"%(
                         hostname,ip,plc['name']))
index e9082b4..08051e9 100644 (file)
@@ -3,16 +3,12 @@ import utils
 
 def config (plc_specs, options):
 
-# tmp : force small test 
-    utils.header("XXX WARNING : forcing small tests in config_default")
-    options.small_test = True
-
     import config_main
     plcs = config_main.config([],options)
-    import config_1vnodes
-    plcs = config_1vnodes.config(plcs,options)
     import config_1testbox
     plcs = config_1testbox.config (plcs,options)
+    import config_1vnodes
+    plcs = config_1vnodes.config(plcs,options)
 
     import config_1vservers
     plcs = config_1vservers.config (plcs,options)
index b4e9aa2..41aeac4 100644 (file)
@@ -11,10 +11,10 @@ def nodes(options):
     nodes= [{'name':'node1',
              'node_fields': {'hostname': 'deferred01',
                              'model':'qemu/minhw', } ,
-             'host_box': 'testbox1.onelab.eu',
+             'host_box': 'deferred-testbox01',
              'owner' : 'pi',
              'nodegroups' : 'mynodegroup',
-             'network_fields': { 'method':'static',
+             'interface_fields': { 'method':'static',
                                  'type':'ipv4',
                                  'ip':'xxx-deferred-xxx',
                                  'gateway':'xxx-deferred-xxx',
@@ -28,9 +28,9 @@ def nodes(options):
             {'name':'node2',
              'node_fields': {'hostname': 'deferred02',
                              'model':'qemu/minhw', } ,
-             'host_box': 'testbox1.onelab.eu',
+             'host_box': 'deferred-testbox02',
              'owner' : 'pi',
-             'network_fields': { 'method':'static',
+             'interface_fields': { 'method':'static',
                                  'type':'ipv4',
                                  'ip':'xxx-deferred-xxx',
                                  'gateway':'xxx-deferred-xxx',
index 3fe7e0d..0aafdcf 100644 (file)
@@ -38,8 +38,8 @@ def nodes():
               'node_fields': {'hostname': 'wlab02.inria.fr', 'model':'Dell Latitude 830'},
               'owner' : 'pi',
               'nodegroups' : 'wifi',
-              'network_fields': { 'method':'dhcp', 'type' : 'ipv4', 'ip':'138.96.250.162',},
-              'extra_interfaces' : [ { 'network_fields' : { 'method' : 'dhcp',
+              'interface_fields': { 'method':'dhcp', 'type' : 'ipv4', 'ip':'138.96.250.162',},
+              'extra_interfaces' : [ { 'interface_fields' : { 'method' : 'dhcp',
                                                             'type' : 'ipv4',
                                                             'mac' : '00:1B:77:70:F4:C6',
                                                             'ip' : '138.96.250.192', },
@@ -52,8 +52,8 @@ def nodes():
               'node_fields': {'hostname': 'wlab17.inria.fr', 'model':'Dell Latitude 830'},
               'owner' : 'pi',
               'nodegroups' : ['wifi','x86_64'] ,
-              'network_fields': { 'method':'dhcp', 'type' : 'ipv4', 'ip':'138.96.250.177',},
-              'extra_interfaces' : [ { 'network_fields' : { 'method' : 'dhcp',
+              'interface_fields': { 'method':'dhcp', 'type' : 'ipv4', 'ip':'138.96.250.177',},
+              'extra_interfaces' : [ { 'interface_fields' : { 'method' : 'dhcp',
                                                             'type' : 'ipv4',
                                                             'mac' : '00:1c:bf:51:3c:19',
                                                             'ip' : '138.96.250.207',},
@@ -66,8 +66,8 @@ def nodes():
               'node_fields': {'hostname': 'wlab05.inria.fr', 'model':'Dell Latitude 830'},
               'owner' : 'pi',
               'nodegroups' : 'wifi',
-              'network_fields': { 'method':'dhcp', 'type' : 'ipv4', 'ip':'138.96.250.165',},
-              'extra_interfaces' : [ { 'network_fields' : { 'method' : 'static',
+              'interface_fields': { 'method':'dhcp', 'type' : 'ipv4', 'ip':'138.96.250.165',},
+              'extra_interfaces' : [ { 'interface_fields' : { 'method' : 'static',
                                                             'type' : 'ipv4',
                                                             'mac' : '00:1B:77:70:FC:84',
                                                             'ip' : '138.96.250.215',
@@ -80,7 +80,7 @@ def nodes():
                                        'settings' : { 'essid' : 'guest-inria-sophia',
                                                       'ifname' : 'wlan0',},
                                        },
-                                     { 'network_fields' : { 'method' : 'dhcp',
+                                     { 'interface_fields' : { 'method' : 'dhcp',
                                                             'type' : 'ipv4',
                                                             'mac' : '00:20:A6:4E:FF:E6',
                                                             'ip' : '138.96.250.50',
index 42eb93b..75250ef 100644 (file)
@@ -18,6 +18,43 @@ def pprint(message,spec,depth=2):
     print ">",now,"--",message
     PrettyPrinter(indent=8,depth=depth).pprint(spec)
 
+
+
+def system(command,background=False):
+    if background: command += " &"
+    if options.dry_run:
+        print 'dry_run:',command
+        return 0
+    else:
+        return os.system("set -x; " + command)
+
+### WARNING : this ALWAYS does its job, even in dry_run mode
+def output_of (command):
+    import commands
+#    if options.dry_run:
+#        print 'dry_run',command
+#        return (0,'[[dry-run - fake output]]')
+#    else:
+    (code,string) = commands.getstatusoutput(command)
+    return (code,string)
+
+
+
+# convenience: translating shell-like pattern into regexp
+def match (string, pattern):
+    # tmp - there's probably much simpler
+    # rewrite * into .*, ? into .
+    pattern=pattern.replace("*",".*")
+    pattern=pattern.replace("?",".")
+    return re.compile(pattern).match(string)
+    
+def locate_sanity_scripts (message,path,extensions):
+    print message,'searching',path,'for extensions',extensions
+    scripts=[]
+    for ext in extensions:
+        scripts += glob.glob (path+'/*.'+ext)
+    return scripts
+    
 # quick & dirty - should probably use the parseroption object instead
 # and move to TestMain as well
 exclude_options_keys = [ 'ensure_value' , 'read_file', 'read_module' ]
@@ -29,27 +66,56 @@ def show_options (message,options):
         if k in exclude_options_keys: continue
         print "    ",k,":",getattr(options,k)
 
+
+
+#################### display config
+# entry point
+def show_plc_spec (plc_spec):
+    show_plc_spec_pass (plc_spec,1)
+    show_plc_spec_pass (plc_spec,2)
+
+def show_plc_spec_pass (plc_spec,passno):
+    for (key,val) in plc_spec.iteritems():
+        if passno == 2:
+            if key == 'sites':
+                for site in val:
+                    show_site_spec(site)
+                    for node in site['nodes']:
+                        show_node_spec(node)
+            elif key=='initscripts':
+                for initscript in val:
+                    show_initscript_spec (initscript)
+            elif key=='slices':
+                for slice in val:
+                    show_slice_spec (slice)
+            elif key=='keys':
+                for key in val:
+                    show_key_spec (key)
+        elif passno == 1:
+            if key not in ['sites','initscripts','slices','keys']:
+                print '*   ',key,':',val
+
 def show_site_spec (site):
     print '* ======== site',site['site_fields']['name']
     for (k,v) in site.iteritems():
         if k=='nodes':
             if v: 
-                print '* \t\t','nodes : ',
+                print '*       ','nodes : ',
                 for node in v:  
                     print node['node_fields']['hostname'],'',
                 print ''
         elif k=='users':
             if v: 
-                print '* \t\tusers : ',
+                print '*       users : ',
                 for user in v:  
                     print user['name'],'',
                 print ''
         elif k == 'site_fields':
-            print '* \t\tlogin_base',':',v['login_base']
+            print '*       login_base',':',v['login_base']
         elif k == 'address_fields':
             pass
         else:
-            print '* \t\t',k,
+            print '*       ',k,
             PrettyPrinter(indent=8,depth=2).pprint(v)
         
 def show_initscript_spec (initscript):
@@ -63,79 +129,26 @@ def show_slice_spec (slice):
     for (k,v) in slice.iteritems():
         if k=='nodenames':
             if v: 
-                print '* \t\tnodes : ',
+                print '*       nodes : ',
                 for nodename in v:  
                     print nodename,'',
                 print ''
         elif k=='usernames':
             if v: 
-                print '* \t\tusers : ',
+                print '*       users : ',
                 for username in v:  
                     print username,'',
                 print ''
         elif k=='slice_fields':
-            print '* \t\tfields',':',
+            print '*       fields',':',
             print 'max_nodes=',v['max_nodes'],
             print ''
         else:
-            print '* \t\t',k,v
-
-def show_test_spec (message,all_plc_specs):
-    now=time.strftime("%H:%M:%S", time.localtime())
-    print "*",now,"--",message
-    for plc_spec in all_plc_specs:
-        show_test_spec_pass (plc_spec,1)
-        show_test_spec_pass (plc_spec,2)
-
-def show_test_spec_pass (plc_spec,passno):
-    for (key,val) in plc_spec.iteritems():
-        if passno == 2:
-            if key == 'sites':
-                for site in val:
-                    show_site_spec(site)
-            elif key=='initscripts':
-                for initscript in val:
-                    show_initscript_spec (initscript)
-            elif key=='slices':
-                for slice in val:
-                    show_slice_spec (slice)
-            elif key=='keys':
-                for key in val:
-                    show_key_spec (key)
-        elif passno == 1:
-            if key not in ['sites','initscripts','slices','keys']:
-                print '* \t',key,':',val
-
-def system(command,background=False):
-    if background: command += " &"
-    if options.dry_run:
-        print 'dry_run:',command
-        return 0
-    else:
-        return os.system("set -x; " + command)
-
-### WARNING : this ALWAYS does its job, even in dry_run mode
-def output_of (command):
-    import commands
-#    if options.dry_run:
-#        print 'dry_run',command
-#        return (0,'[[dry-run - fake output]]')
-#    else:
-    (code,string) = commands.getstatusoutput(command)
-    return (code,string)
+            print '*       ',k,v
 
-# convenience: translating shell-like pattern into regexp
-def match (string, pattern):
-    # tmp - there's probably much simpler
-    # rewrite * into .*, ? into .
-    pattern=pattern.replace("*",".*")
-    pattern=pattern.replace("?",".")
-    return re.compile(pattern).match(string)
-    
-def locate_sanity_scripts (message,path,extensions):
-    print message,'searching',path,'for extensions',extensions
-    scripts=[]
-    for ext in extensions:
-        scripts += glob.glob (path+'/*.'+ext)
-    return scripts
+def show_node_spec (node):
+    print "*           node",node['name'],"host_box=",node['host_box'],
+    print "hostname=",node['node_fields']['hostname'],
+    print "ip=",node['interface_fields']['ip']
     
+