bonds: use rung <=> run -G that hurts my pinky
Thierry Parmentelat [Mon, 13 Apr 2015 13:00:04 +0000 (15:00 +0200)]
fix IP pool allocation
installed_bonds
install_syslinux6
store nodefamily in qemu workdir (step qemu_nodeflavour)

system/AA-bonding.md [new file with mode: 0644]
system/Substrate.py
system/TestApiserver.py
system/TestBonding.py
system/TestMain.py
system/TestNode.py
system/TestPlc.py
system/macros.py
system/rung [new symlink]

diff --git a/system/AA-bonding.md b/system/AA-bonding.md
new file mode 100644 (file)
index 0000000..a37c9d3
--- /dev/null
@@ -0,0 +1,65 @@
+# bonding
+
+Say you have 
+
+* one build `2015.04.13--f14` under test
+* another build `2015.04.13--f18` that you would like to use to create f18 nodes inside        `2015.04.13--f14`
+
+general idea is to use this command
+
+       root@testmaster ~/2015.04.13--f14 # run -g 2015.04.13--f18 ...
+       
+or in an equivalent manner
+
+       export bonding= 2015.04.13--f18
+       run -G
+       
+# requirements
+
+* the build to test (f14) must be running
+* the build to bond with does not need to run but needs to be have run under `../2015.04.13--f18` so that its arg-* files reflect its flavour
+
+# convenience
+
+* rung can be used as a symlink to `TestMain.py`
+* in this case it is like running `run -G`
+
+* plus, `comp-testmaster` offers the convenience function
+               
+               bond ../$(t)*18
+               
+* or, even simpler
+
+               bond18          
+
+# initialisation
+
+       bond18
+       rung
+    
+will define the f18 node flavour to the f14 build
+# creating bonding node
+
+       rung nodes
+
+will create an additional node in tested myplc. 
+
+**NOTE** for efficiency the IP and hostname of that node are stored in `arg-bonding-$bonding`
+
+# starting node
+
+as a matter of fact when doing `run -g $bonding` one can invoke most of the usual targets, including for starting the node
+
+       rung start-node wait-node
+       
+# usual sequence
+
+       alias rung='run -g $bonding'
+       bonding=2015.04.13--f21
+       rung 
+       rung -n bonding-node
+       cat arg-bonding-$bonding
+       <<visual check>>
+       rung bonding-node
+
index 098ff51..a6d02e2 100644 (file)
@@ -151,10 +151,10 @@ class PoolItem:
         elif self.status == 'starting': return 'S'
 
     def get_ip(self):
-        if self.ip: return self.ip
-        ip=socket.gethostbyname(self.hostname)
-        self.ip=ip
-        return ip
+        if self.ip:
+            return self.ip
+        self.ip = socket.gethostbyname(self.hostname)
+        return self.ip
 
 class Pool:
 
@@ -165,9 +165,10 @@ class Pool:
         self.substrate = substrate
 
     def __repr__(self):
-        return "<Pool {} {}>".format(self.message, self.tuples)
+        return "<Pool {} : {} .. {}>".format(self.message, self.pool_items[0], self.pool_items[-1])
 
     def list (self, verbose=False):
+        print(self)
         for i in self.pool_items: print(i.line())
 
     def line (self):
@@ -198,7 +199,7 @@ class Pool:
     def next_free (self):
         for i in self.pool_items:
             if i.status == 'free':
-                i.status='mine'
+                i.status = 'mine'
                 return (i.hostname, i.userdata)
         return None
 
@@ -243,13 +244,14 @@ class Pool:
             else:
                 item.status = 'free'
                 print('.', end=' ')
+            sys.stdout.flush()
     
     def sense (self):
         print('Sensing IP pool', self.message, end=' ')
         sys.stdout.flush()
         self._sense()
         print('Done')
-        for (vname,bname) in self.load_starting():
+        for vname, bname in self.load_starting():
             self.substrate.add_starting_dummy(bname, vname)
         print("After having loaded 'starting': IP pool")
         print(self.line())
@@ -257,6 +259,8 @@ class Pool:
     ping_timeout_option = None
     # returns True when a given hostname/ip responds to ping
     def check_ping (self, hostname):
+        if '.' not in hostname:
+            hostname = self.substrate.fqdn(hostname)
         if not Pool.ping_timeout_option:
             (status, osname) = subprocess.getstatusoutput("uname -s")
             if status != 0:
@@ -266,7 +270,7 @@ class Pool:
             elif osname == "Darwin":
                 Pool.ping_timeout_option = "-t"
 
-        command="ping -c 1 {} 1 {}".format(Pool.ping_timeout_option, hostname)
+        command = "ping -c 1 {} 1 {}".format(Pool.ping_timeout_option, hostname)
         (status, output) = subprocess.getstatusoutput(command)
         return status == 0
 
index a87fc36..00a73ac 100644 (file)
@@ -52,6 +52,7 @@ server_methods = [ ('GetNodes' ,  []),
                    ('GetConfFiles',[]),
                    ('AddConfFile','True'),
                    ('GetSliceTags',[]),
+                   ('GetNodeFlavour','dry-run-nodeflavour'),
                    ('system.listMethods',[]),
                    ]
 
index 9f4c432..d9d8c1d 100644 (file)
@@ -107,7 +107,7 @@ class TestBonding(object):
         if self.persistent_load():
             print("Re-using bonding nodes attributes from {}".format(self.persistent_name()))
         else:
-            print("Could not load bonding nodes attributes from {}".format(self.persistent_name()))
+            print("Sensing for an avail. IP (Could not load from {})".format(self.persistent_name()))
             vnode_pool = self.substrate.vnode_pool
             vnode_pool.sense()
             try:
@@ -119,6 +119,8 @@ class TestBonding(object):
             except:
                 raise Exception("Cannot provision bonding node")
 
+        print("Bonding on node {} - {}".format(self.vnode_hostname, self.vnode_ip))
+
         # implement the node on another IP
         node_spec['node_fields']['hostname'] = self.vnode_hostname
         node_spec['interface_fields']['ip'] = self.vnode_ip
index 1d88af4..19a934d 100755 (executable)
@@ -108,7 +108,7 @@ class TestMain:
 
     def init_steps(self):
         self.steps_message  = ""
-        if not self.options.bonding:
+        if not self.options.bonding_build:
             self.steps_message += 20*'x' + " Defaut steps are\n" + \
                                   TestPlc.printable_steps(TestPlc.default_steps)
             self.steps_message += 20*'x' + " Other useful steps are\n" + \
@@ -208,11 +208,29 @@ run with -l to see a list of available steps
                             help="Show environment and exits")
         parser.add_argument("-t", "--trace", action="store", dest="trace_file", default=None,
                             help="Trace file location")
-        parser.add_argument("-g", "--bonding", action='store', dest='bonding', default=None,
+        parser.add_argument("-g", "--bonding", action='store', dest='bonding_build', default=None,
                             help="specify build to bond with")
+        # if we call symlink 'rung' instead of just run this is equivalent to run -G
+        bonding_default = 'rung' in sys.argv[0]
+        parser.add_argument("-G", "--bonding-env", action='store_true', dest='bonding_env', default=bonding_default,
+                            help="get bonding build from env. variable $bonding")
         parser.add_argument("steps", nargs='*')
         self.options = parser.parse_args()
 
+        # handle -G/-g options
+        if self.options.bonding_env:
+            if 'bonding' not in os.environ:
+                print("env. variable $bonding must be set with --bonding-env")
+                sys.exit(1)
+            self.options.bonding_build = os.environ['bonding']
+
+        if self.options.bonding_build:
+            ## allow to pass -g ../2015.03.15--f18 so we can use bash completion
+            self.options.bonding_build = os.path.basename(self.options.bonding_build)
+            if not os.path.isdir("../{}".format(self.options.bonding_build)):
+                print("could not find test dir for bonding build {}".format(self.options.bonding_build))
+                sys.exit(1)
+
         # allow things like "run -c 'c1 c2' -c c3"
         def flatten (x):
             result = []
@@ -293,7 +311,7 @@ run with -l to see a list of available steps
         # initialize steps
         if not self.options.steps:
             # defaults, depends on using bonding or not
-            if self.options.bonding:
+            if self.options.bonding_build:
                 self.options.steps = TestPlc.default_bonding_steps
             else:
                 self.options.steps = TestPlc.default_steps
@@ -380,13 +398,11 @@ run with -l to see a list of available steps
 
         # populate TestBonding objects
         # need to wait until here as we need all_plcs
-        if self.options.bonding:
-            ## allow to pass -g ../2015.03.15--f18 so we can use bash completion
-            self.options.bonding = os.path.basename(self.options.bonding)
-            # this will fail if ../{bonding} has not the right arg- files
+        if self.options.bonding_build:
+            # this will fail if ../{bonding_build} has not the right arg- files
             for spec, test_plc in all_plcs:
                 test_plc.test_bonding = TestBonding (test_plc,
-                                                     onelab_bonding_spec(self.options.bonding),
+                                                     onelab_bonding_spec(self.options.bonding_build),
                                                      LocalSubstrate.local_substrate,
                                                      self.options)
         
index 59a7924..ab876d2 100644 (file)
@@ -274,6 +274,17 @@ class TestNode:
         return test_box.run_in_buildname("echo {:d} > {}/timestamp"\
                                          .format(now, self.nodedir()), dry_run=self.dry_run()) == 0
 
+    def qemu_nodeflavour(self):
+        auth = self.test_plc.auth_root()
+        hostname = self.node_spec['node_fields']['hostname']
+        nodeflavour = self.test_plc.apiserver.GetNodeFlavour(auth, hostname)
+        if self.dry_run():
+            return True
+        nodedir = self.nodedir()
+        nodefamily = nodeflavour['nodefamily']
+        self.test_box().run_in_buildname("echo {nodefamily} > {nodedir}/nodefamily".format(**locals()))
+        return True
+
     def start_qemu(self):
         test_box = self.test_box()
         utils.header("Starting qemu node {} on {}".format(self.name(), test_box.hostname()))
index 177cad6..b24aa62 100644 (file)
@@ -162,7 +162,7 @@ class TestPlc:
         'check_vsys_defaults_ignore', SEP,
 # run this first off so it's easier to re-run on another qemu box        
         'qemu_kill_mine', 'nodestate_reinstall', 'qemu_local_init','bootcd', 'qemu_local_config', SEP,
-        'qemu_clean_mine', 'qemu_export', 'qemu_start', 'qemu_timestamp', SEP,
+        'qemu_clean_mine', 'qemu_export', 'qemu_start', 'qemu_timestamp', 'qemu_nodeflavour', SEP,
         'sfa_install_all', 'sfa_configure', 'cross_sfa_configure', 'sfa_start', 'sfa_import', SEPSFA,
         'sfi_configure@1', 'sfa_register_site@1','sfa_register_pi@1', SEPSFA,
         'sfa_register_user@1', 'sfa_update_user@1', 'sfa_register_slice@1', 'sfa_renew_slice@1', SEPSFA,
@@ -204,6 +204,7 @@ class TestPlc:
         'check_netflow','check_drl', SEP,
         'debug_nodemanager', 'slice_fs_present', SEP,
         'standby_1_through_20','yes','no',SEP,
+        'install_syslinux6', 'installed_bonds', SEP,
         ]
     default_bonding_steps = [
         'bonding_init_partial',
@@ -726,6 +727,28 @@ class TestPlc:
         pkgs_string=" ".join(pkgs_list)
         return self.yum_install(pkgs_list)
 
+    def install_syslinux6(self):
+        """
+        install syslinux6 from the fedora21 release
+        """
+        key = 'http://mirror.onelab.eu/keys/RPM-GPG-KEY-fedora-21-primary'
+
+        rpms = [ 
+            'http://mirror.onelab.eu/fedora/releases/21/Everything/x86_64/os/Packages/s/syslinux-6.03-1.fc21.x86_64.rpm',
+            'http://mirror.onelab.eu/fedora/releases/21/Everything/x86_64/os/Packages/s/syslinux-nonlinux-6.03-1.fc21.noarch.rpm',
+            'http://mirror.onelab.eu/fedora/releases/21/Everything/x86_64/os/Packages/s/syslinux-perl-6.03-1.fc21.x86_64.rpm',
+        ]
+        # this can be done several times
+        self.run_in_guest("rpm --import {key}".format(**locals()))
+        return self.run_in_guest("yum -y localinstall {}".format(" ".join(rpms))) == 0
+
+    def installed_bonds(self):
+        """
+        list /etc/yum.repos.d on the myplc side
+        """
+        self.run_in_guest("ls /etc/yum.repos.d/*partial.repo")
+        return True
+        
     ###
     def mod_python(self):
         """yum install mod_python, useful on f18 and above so as to avoid broken wsgi"""
@@ -1330,6 +1353,9 @@ class TestPlc:
     @node_mapper
     def qemu_timestamp(self) : pass
 
+    @node_mapper
+    def qemu_nodeflavour(self): pass
+
     # when a spec refers to a node possibly on another plc
     def locate_sliver_obj_cross(self, nodename, slicename, other_plcs):
         for plc in [ self ] + other_plcs:
index 85a570b..af65174 100644 (file)
@@ -185,7 +185,7 @@ sequences['node'] = [ 'nodes' ]
 
 sequences['restart_node'] = sequences['start_node'] = """
 qemu_kill_mine nodestate_reinstall qemu_local_init bootcd qemu_local_config 
-qemu_clean_mine qemu_export qemu_start qemu_timestamp
+qemu_clean_mine qemu_export qemu_start qemu_timestamp qemu_nodeflavour
 """.split()
 
 sequences['bonding_node'] = 'node start-node'.split()
diff --git a/system/rung b/system/rung
new file mode 120000 (symlink)
index 0000000..23360cc
--- /dev/null
@@ -0,0 +1 @@
+TestMain.py
\ No newline at end of file