a first rough implementation for testing 'bonding' myplc's
Thierry Parmentelat [Tue, 17 Mar 2015 09:49:39 +0000 (10:49 +0100)]
typicallly, you would have 2 complete builds b1 and b2

then on testmaster in b1 you can do
$ run --bonding b2
or simply
$ run -g b2

in order to expose the nodeflavour and slicefamily provided by b2 to build b1

for now it only does the partial-rpms & yum install bootcd nodeimage noderepo & packages reindexing

need some more work to bring a actual nodes and slices online

system/TestBonding.py [new file with mode: 0644]
system/TestMain.py
system/TestPlc.py

diff --git a/system/TestBonding.py b/system/TestBonding.py
new file mode 100644 (file)
index 0000000..4066e44
--- /dev/null
@@ -0,0 +1,182 @@
+"""
+Utilities to create a setup made of 2 different builds
+read : 2 different node flavours 
+so that each myplc knows about the nodeflavour/slicefamily supported
+by the other one
+---
+This would be the basics for running tests on multi-node myplc, 
+in particular for node upgrades
+"""
+
+#################### WARNING
+
+# this feature relies on a few assumptions that need to be taken care of
+# more or less manually; this is based on the onelab.eu setup
+
+# (*) the build host is expected to have /root/git-build.sh reasonably up-to-date
+# with our build module, so we can locate partial-repo.sh
+# this utility needs to be run on the build host so we can point at a PARTIAL-RPMS
+# sub-repo that exposes the
+# bootcd/bootstraps/ and the like rpms from one flavour to another
+
+# a utility to create a bonding_plc_spec from
+# a plc_spec and just a buildname
+
+def onelab_bonding_spec (buildname):
+
+    # essentially generic ..
+    buildname      = buildname
+    
+    with open ("../{}/arg-fcdistro".format(buildname)) as input:
+        fcdistro   = input.read().strip()
+    with open ("../{}/arg-pldistro".format(buildname)) as input:
+        pldistro   = input.read().strip()
+    with open ("../{}/arg-ips-bplc".format(buildname)) as input:
+        plc_box    = input.read().strip().split()[0]
+    # e.g. http://build.onelab.eu/onelab//2015.03.15--f14/RPMS/x86_64
+    with open ("../{}/arg-arch-rpms-url".format(buildname)) as input:
+        arch_rpms_url = input.read().strip()
+    arch           = arch_rpms_url.split('/')[-1]
+    build_www_host = arch_rpms_url.split('/')[2]
+    base_url       = arch_rpms_url.replace("RPMS/{}".format(arch), "PARTIAL-RPMS")
+        
+    # onelab specifics
+    build_www_git = '/root/git-build/'
+    build_www_dir  = '/build/{}/{}'.format(pldistro, buildname)
+    
+    return locals()
+
+####################
+import os, os.path
+
+import utils
+from TestSsh import TestSsh
+
+####################
+class TestBonding(object):
+
+    """
+    Holds details about a 'bonding' build
+    so we can configure the local myplc (test_plc)
+    for multi-flavour nodes and slices
+    options is a TestMain options
+    """
+    
+    def __init__(self, test_plc, bonding_spec, options):
+        """
+        test_plc is one local TestPlc instance
+        bonding_spec is a dictionary that gives details on
+        the build we want to be bonding with
+        """
+        self.test_plc = test_plc
+        self.bonding_spec = bonding_spec
+        self.options = options
+        # the local build & plc is described in options
+        # the bonding build is described in bonding_spec
+
+    def nodefamily(self):
+        return "{pldistro}-{fcdistro}-{arch}".format(**self.bonding_spec)
+        
+    def init_partial(self):
+        """
+        runs partial-repo.sh for the bonding build
+        this action takes place on the build host
+        """
+        test_ssh = TestSsh (self.bonding_spec['build_www_host'])
+        command = "{build_www_git}/partial-repo.sh -i {build_www_dir}".\
+                  format(**self.bonding_spec)
+                         
+        return test_ssh.run (command, dry_run = self.options.dry_run) == 0
+        
+
+    def add_yum(self):
+        """
+        creates a separate yum.repo file in the myplc box
+        where our own build runs, and that points at the partial
+        repo for the bonding build
+        """
+
+        # create a .repo file locally
+        yumrepo_contents = """
+[{buildname}]
+name=Partial repo from bonding build {buildname}
+baseurl={base_url}
+enabled=1
+gpgcheck=0
+""".format(**self.bonding_spec)
+
+        yumrepo_local = '{buildname}-partial.repo'.\
+                        format(**self.bonding_spec)
+        with open(yumrepo_local, 'w') as yumrepo_file:
+            yumrepo_file.write(yumrepo_contents)
+        utils.header("(Over)wrote {}".format(yumrepo_local))
+
+        # push onto our myplc instance
+        test_ssh = TestSsh (self.test_plc.vserverip)
+
+        yumrepo_remote = '/etc/yum.repos.d/{bonding_buildname}-partial.repo'.\
+                         format(bonding_buildname = self.bonding_spec['buildname'])
+
+        if test_ssh.copy_abs (yumrepo_local, yumrepo_remote,
+                              dry_run=self.options.dry_run) != 0:
+            return False
+
+        # xxx TODO looks like drupal also needs to be excluded
+        # from the 2 entries in building.repo
+        # otherwise subsequent yum update calls will fail
+
+        return True
+        
+    def install_rpms(self):
+        """
+        once the 2 operations above have been performed, we can 
+        actually install the various rpms that provide support for the 
+        nodeflavour/slicefamily offered byt the bonding build to our own build
+        """
+
+        test_ssh = TestSsh (self.test_plc.vserverip)
+        
+        command1 = "yum -y update"
+        if test_ssh.run (command1, dry_run = self.options.dry_run) != 0:
+            return False
+
+        nodefamily = self.nodefamily()
+        extra_list = [ 'bootcd', 'nodeimage', 'noderepo' ]
+
+        extra_rpms = [ "{}-{}".format(rpm, nodefamily) for rpm in extra_list]
+
+        command2 = "yum -y install " + " ".join(extra_rpms) 
+        if test_ssh.run (command2, dry_run = self.options.dry_run) != 0:
+            return False
+
+        command3 = "/etc/plc.d/packages force"
+        if test_ssh.run (command3, dry_run = self.options.dry_run) != 0:
+            return False
+
+        return True
+
+### probably obsolete already    
+if __name__ == '__main__':
+
+    from TestPlc import TestPlc    
+
+    from config_default import sample_test_plc_spec
+    test_plc_spec = sample_test_plc_spec()
+    test_plc = TestPlc (test_plc_spec)
+    test_plc.show()
+
+    print test_plc.host_box
+
+    from argparse import ArgumentParser
+    parser = ArgumentParser()
+    parser.add_argument ("-n", "--dry-run", dest='dry_run', default=False,
+                         action='store_true', help="dry run")
+    parser.add_argument ("build_name")
+    args = parser.parse_args()
+
+    test_bonding = TestBonding (test_plc,
+                                onelab_bonding_spec(args.build_name),
+                                dry_run = args.dry_run)
+
+    test_bonding.bond ()
+    
index 1f470f6..01bf6cd 100755 (executable)
@@ -12,6 +12,7 @@ from datetime import datetime
 
 import utils
 from TestPlc import TestPlc, Ignored
+from TestBonding import TestBonding, onelab_bonding_spec
 from TestSite import TestSite
 from TestNode import TestNode
 from macros import sequences
@@ -106,12 +107,17 @@ class TestMain:
             utils.show_options("main options", options)
 
     def init_steps(self):
-        self.steps_message  = 20*'x' + " Defaut steps are\n" + \
-                              TestPlc.printable_steps(TestPlc.default_steps)
-        self.steps_message += 20*'x' + " Other useful steps are\n" + \
-                              TestPlc.printable_steps(TestPlc.other_steps)
-        self.steps_message += 20*'x' + " Macro steps are\n" + \
-                              " ".join(Step.list_macros())
+        self.steps_message  = ""
+        if not self.options.bonding:
+            self.steps_message += 20*'x' + " Defaut steps are\n" + \
+                                  TestPlc.printable_steps(TestPlc.default_steps)
+            self.steps_message += 20*'x' + " Other useful steps are\n" + \
+                                  TestPlc.printable_steps(TestPlc.other_steps)
+            self.steps_message += 20*'x' + " Macro steps are\n" + \
+                                  " ".join(Step.list_macros())
+        else:
+            self.steps_message += 20*'x' + " Default steps with bonding are\n" + \
+                                  TestPlc.printable_steps(TestPlc.bonding_steps)
 
     def list_steps(self):
         if not self.options.verbose:
@@ -139,7 +145,6 @@ class TestMain:
                     Step(stepname).print_doc()
 
     def run (self):
-        self.init_steps()
         usage = """usage: %%prog [options] steps
 arch-rpms-url defaults to the last value used, as stored in arg-arch-rpms-url,
    no default
@@ -148,9 +153,10 @@ config defaults to the last value used, as stored in arg-config,
 ips_vnode, ips_vplc and ips_qemu defaults to the last value used, as stored in arg-ips-{bplc,vplc,bnode,vnode},
    default is to use IP scanning
 steps refer to a method in TestPlc or to a step_* module
+
+run with -l to see a list of available steps
 ===
 """%(TestMain.default_config)
-        usage += self.steps_message
 
         parser = ArgumentParser(usage = usage)
         parser.add_argument("-u", "--url", action="store",  dest="arch_rpms_url", 
@@ -201,8 +207,8 @@ steps refer to a method in TestPlc or to a step_* module
                             help="Show environment and exits")
         parser.add_argument("-t", "--trace", action="store", dest="trace_file", default=None,
                             help="Trace file location")
-#        parser.add_argument("-g", "--bonding", action='store', dest='bonding', default=None,
-#                            help="specify build to bond with")
+        parser.add_argument("-g", "--bonding", action='store', dest='bonding', default=None,
+                            help="specify build to bond with")
         parser.add_argument("steps", nargs='*')
         self.options = parser.parse_args()
 
@@ -282,21 +288,19 @@ steps refer to a method in TestPlc or to a step_* module
         # hack : if sfa is not among the published rpms, skip these tests
         TestPlc.check_whether_build_has_sfa(self.options.arch_rpms_url)
 
-        # use the default list of steps if unspecified
-        if len(self.options.steps) == 0:
-            self.options.steps = TestPlc.default_steps
+        # initialize steps
+        if not self.options.steps:
+            # defaults, depends on using bonding or not
+            if self.options.bonding:
+                self.options.steps = TestPlc.bonding_steps
+            else:
+                self.options.steps = TestPlc.default_steps
 
         if self.options.list_steps:
             self.init_steps()
             self.list_steps()
             return 'SUCCESS'
 
-        # steps
-        if not self.options.steps:
-            #default (all) steps
-            #self.options.steps=['dump','clean','install','populate']
-            self.options.steps = TestPlc.default_steps
-
         # rewrite '-' into '_' in step names
         self.options.steps   = [ step.replace('-', '_') for step in self.options.steps ]
         self.options.exclude = [ step.replace('-', '_') for step in self.options.exclude ]
@@ -376,6 +380,17 @@ steps refer to a method in TestPlc or to a step_* module
         # pass options to utils as well
         utils.init_options(self.options)
 
+        # populate TestBonding objects
+        # need to wait until here as we need all_plcs
+        if self.options.bonding:
+            ## allow to pass -g ../2015.03.15--f18 so we can use bash completion
+            self.options.bonding = os.path.basename(self.options.bonding)
+            # this will fail if ../{bonding} has not the right arg- files
+            for spec, test_plc in all_plcs:
+                test_plc.test_bonding = TestBonding (test_plc,
+                                                     onelab_bonding_spec(self.options.bonding),
+                                                     self.options)
+        
         overall_result = 'SUCCESS'
         all_step_infos = []
         for step in self.options.steps:
index 61ae1df..a60f619 100644 (file)
@@ -23,6 +23,8 @@ from TestApiserver import TestApiserver
 from TestAuthSfa import TestAuthSfa
 from PlcapiUrlScanner import PlcapiUrlScanner
 
+from TestBonding import TestBonding
+
 has_sfa_cache_filename="sfa-cache"
 
 # step methods must take (self) and return a boolean (options is a member of the class)
@@ -72,6 +74,17 @@ def slice_mapper(method):
     map_on_slices.__doc__ = TestSlice.__dict__[method.__name__].__doc__
     return map_on_slices
 
+def bonding_redirector(method):
+    bonding_name = method.__name__.replace('bonding_', '')
+    def redirect(self):
+        bonding_method = TestBonding.__dict__[bonding_name]
+        return bonding_method(self.test_bonding)
+    # maintain __name__ for ignore_result
+    redirect.__name__ = method.__name__
+    # restore the doc text
+    redirect.__doc__ = TestBonding.__dict__[bonding_name].__doc__
+    return redirect
+
 # run a step but return True so that we can go on
 def ignore_result(method):
     def ignoring(self):
@@ -194,6 +207,11 @@ class TestPlc:
         'debug_nodemanager', 'slice_fs_present', SEP,
         'standby_1_through_20','yes','no',SEP,
         ]
+    bonding_steps = [
+        'bonding_init_partial',
+        'bonding_add_yum',
+        'bonding_install_rpms', SEP,
+        ]
 
     @staticmethod
     def printable_steps(list):
@@ -507,20 +525,20 @@ class TestPlc:
             return True
         TestPlc.exported_id += 1
         domain = socket.gethostname().split('.',1)[1]
-        fqdn   = "%s.%s" % (self.plc_spec['host_box'],domain)
+        fqdn   = "%s.%s" % (self.plc_spec['host_box'], domain)
         print "export BUILD=%s" % self.options.buildname
         print "export PLCHOSTLXC=%s" % fqdn
         print "export GUESTNAME=%s" % self.plc_spec['vservername']
         vplcname = self.plc_spec['vservername'].split('-')[-1]
-        print "export GUESTHOSTNAME=%s.%s"%(vplcname,domain)
+        print "export GUESTHOSTNAME=%s.%s"%(vplcname, domain)
         # find hostname of first node
-        hostname,qemubox = self.all_node_infos()[0]
-        print "export KVMHOST=%s.%s" % (qemubox,domain)
+        hostname, qemubox = self.all_node_infos()[0]
+        print "export KVMHOST=%s.%s" % (qemubox, domain)
         print "export NODE=%s" % (hostname)
         return True
 
     # entry point
-    always_display_keys=['PLC_WWW_HOST','nodes','sites',]
+    always_display_keys=['PLC_WWW_HOST', 'nodes', 'sites']
     def show_pass(self, passno):
         for (key,val) in self.plc_spec.iteritems():
             if not self.options.verbose and key not in TestPlc.always_display_keys:
@@ -531,18 +549,18 @@ class TestPlc:
                         self.display_site_spec(site)
                         for node in site['nodes']:
                             self.display_node_spec(node)
-                elif key=='initscripts':
+                elif key == 'initscripts':
                     for initscript in val:
                         self.display_initscript_spec(initscript)
-                elif key=='slices':
+                elif key == 'slices':
                     for slice in val:
                         self.display_slice_spec(slice)
-                elif key=='keys':
+                elif key == 'keys':
                     for key in val:
                         self.display_key_spec(key)
             elif passno == 1:
                 if key not in ['sites', 'initscripts', 'slices', 'keys']:
-                    print '+   ',key,':',val
+                    print '+   ', key, ':', val
 
     def display_site_spec(self, site):
         print '+ ======== site', site['site_fields']['name']
@@ -562,7 +580,7 @@ class TestPlc:
                         print user['name'],'',
                     print ''
             elif k == 'site_fields':
-                print '+       login_base',':',v['login_base']
+                print '+       login_base', ':', v['login_base']
             elif k == 'address_fields':
                 pass
             else:
@@ -1743,6 +1761,19 @@ class TestPlc:
         remote = (self.run_in_guest(command) == 0);
         return local and remote
 
+
+    ####################
+    @bonding_redirector
+    def bonding_init_partial(self): pass
+
+    @bonding_redirector
+    def bonding_add_yum(self): pass
+
+    @bonding_redirector
+    def bonding_install_rpms(self): pass
+
+    ####################
+
     def gather_logs(self):
         "gets all possible logs from plc's/qemu node's/slice's for future reference"
         # (1.a) get the plc's /var/log/ and store it locally in logs/myplc.var-log.<plcname>/*