import repository from arizona
[raven.git] / apps / ravenpublish / raven.py
1 #! /usr/bin/env python
2
3 from sfa.trust.certificate import Keypair
4 from sfa.trust.credential import Credential
5
6 import getpass
7 import hashlib
8 import logging
9 from optparse import OptionParser
10 import os
11 import shutil
12 import sys
13 import traceback
14
15 import builder
16 import container
17 import kongclient
18 import ravenlib.package.storkpackage
19 import ravenlib.crypto
20 from ravenlib.ravenlog import RavenLog
21 import slicerun
22 from ravenlib.files.tpfile import tpfile
23 from ravenlib.files.pacpackages import pacpackages
24 from ravenlib.files.pacgroups import pacgroups
25 from ravenlib.client.repoclient import RepoClient
26 import ravenversion
27
28 from ravenlib.modpythonapi.BaseClient import EnableVerboseExceptions
29 EnableVerboseExceptions(True)
30
31 #REPO_URL="https://198.0.0.136/REPOAPI/"
32 DEFAULT_REPO_URL="https://stork-repository.cs.arizona.edu/REPOAPI/"
33
34 RAVENCONFIG_RPM_SUFFIX = "_ravenconfig"
35
36 def sorteddir(dir):
37    """ returns a sorted directory listing
38        also, differs from os.listdir() in that a non-existent dir returns [] instead of throwing exception
39    """
40    if not os.path.exists(dir):
41        return []
42
43    items = os.listdir(dir)
44    items.sort()
45
46    return items
47
48 # passphrase callback for SFA
49 glo_passphrase = None
50 def ask_passphrase(kp, pkstring, x):
51    try:
52        from sfa.trust.certificate import test_passphrase
53        global glo_passphrase\r
54 \r
55        while True:\r
56           # if our currently cached passphrase is good, use it.\r
57           if (glo_passphrase) and (test_passphrase(pkstring, glo_passphrase)):\r
58               break\r
59 \r
60           glo_passphrase = getpass.getpass("Enter passphrase:")\r
61 \r
62           # inform user of the error of his ways\r
63           if not test_passphrase(pkstring, glo_passphrase):\r
64               print "Passphrase incorrect. Please try again."\r
65 \r
66        return glo_passphrase
67    except:
68        # pyOpenSSL tends to hide these. we want to see them.
69        print "Exception inside of ask_passphrase:"
70        traceback.print_exc()
71        print "-----------------------------------"
72        raise
73
74 class Raven(RavenLog):
75     def __init__(self):
76         # RavenLog object is mixed in to give us a configurable Print statement
77         RavenLog.__init__(self)
78         self.do_tempest_rpm = True
79
80     def create_cmd_parser(self, command, additional_cmdargs = None):
81        cmdargs = {"buildrpm": "",
82                   "create": "",
83                   "config": "",
84                   "build": "",
85                   "publish": "",
86                   "upload": "",
87                   "info": "",
88                   "upgrade": "",
89                   "testconnect": "",
90                   "experiment": "",
91                   "slice": "",
92                  }
93
94        if additional_cmdargs:
95           cmdargs.update(additional_cmdargs)
96
97        if command not in cmdargs:
98           self.error("Invalid command: %s", command)
99           self.error("Commands: " + ",".join(cmdargs.keys()))
100           sys.exit(2)
101
102        parser = OptionParser(usage="raven [raven_options] %s [options] %s" \
103           % (command, cmdargs[command]))
104
105        if command in ["publish", "upload", "testconnect"]:
106           parser.add_option("-u", "--url", dest="url", default=DEFAULT_REPO_URL,
107             help="url of repository")
108
109        if command in ["create"]:
110           parser.add_option("-i", "--import", dest="importconfig", default="",
111             help="name of existing config file to import")
112           parser.add_option("", "--noninteractive", dest="noninteractive", default=False, action="store_true",
113             help="do not ask questions")
114           parser.add_option("", "--keyfn", dest="keyfn", default=None,
115             help="private key filename for create")
116           parser.add_option("", "--credfn", dest="credfn", default=None,
117             help="credential filename for create")
118           parser.add_option("", "--expname", dest="expname", default=None,
119             help="experiment name")
120           parser.add_option("", "--slices", dest="slices", default=None,
121             help="slices (comma-separated)")
122
123        if command in ["experiment"]:
124            parser.add_option("-n", "--clients", dest="numclients", default=0,
125              help="number of clients")
126            parser.add_option("", "--html", dest="html", default=False, action="store_true",
127              help="display tabular results in html")
128            parser.add_option("", "--csv", dest="csv", default=False, action="store_true",
129              help="display tabular results in csv")
130
131        if command in ["slice"]:
132            slicerun.add_parser_options(parser)
133
134        return parser
135
136     USAGE="""
137     raven [options] command [command_options] [command_args]
138
139     Commands: create, config, build, buildrpm, experiment, publish, upload, upgrade,
140               slice
141
142     Create
143         Create a new experiment container in the current directory, or the directory
144         specified by the -D option.
145
146     Config
147         Configure experiment-related options.
148
149     Build
150         Build the current experiment. The build directory is scanned for source
151         files and used to build RPMS, which are in turn placed in the packages
152         directory. The packages directory is scanned for packages
153         and the users directory is scanned for public keys. These are combined to
154         create a tpfile in the tpfiles directory, and optionally a packages.pacman
155         file in the tempest directory. Files are signed and copied to the upload
156         directory.
157
158     Buildrpm
159         Build any RPMs in the build/ directory.
160
161     Experiment
162         Experiment control subsystem. Subcommands:
163             raven experiment -n <number_of_clients> run
164             raven experiment results [-csv | -html]
165             raven experiment setstate [-n <num_of_clients] <state_name>
166             raven experiment status
167
168     Publish
169         Publish executes both the build and upload commands.
170
171     Upload
172         Uploads the files in the upload directory to the repository.
173
174     Slice
175         Slice control subsytem. Subcommands:
176             raven slice --slicename <slicename> install <rspec_fn>
177             raven slice check <rspec_fn>
178             raven slice exec <rspec_fn> <cmd> <arg> <arg> ...
179             raven slice dump <rspec_fn>
180     """
181
182
183     def create_parser(self):
184        # Generate command line parser
185        parser = OptionParser(usage="raven [options] command [command_options] [command_args]",
186             description="Commands: create, config, build, buildrpm, publish, upload",
187             version="raven "+ravenversion.VERREL)
188
189        parser.add_option("-D", "--directory", dest="directory",
190             help="directory of raven experiment", metavar="DIR", default=None)
191        parser.add_option("", "--no_tempest_rpm", dest="no_tempest_rpm",
192             help="do the new tempest rpm thing", action="store_true", default=False)
193        parser.add_option("-v", "--verbose", dest="verbose",
194             help="verbose mode", action="store_true", default=False)
195
196        parser.disable_interspersed_args()
197
198        return parser
199
200     def yesno(self, c, optname, text):
201        optval = getattr(c, optname)
202        while True:
203           if optval:
204              opttext = "y"
205           else:
206              opttext = "n"
207           name = raw_input( text + " (y/n): [" + opttext + "] ? " )
208           if (not name):
209               break;
210           elif (name == "Y") or (name == "y"):
211               setattr(c, optname, True)
212               break;
213           elif (name == "N") or (name == "n"):
214               setattr(c, optname, False)
215               break
216
217     def test_privatekey(self, pkey_fn):
218        try:
219            pkey = Keypair(filename = pkey_fn)
220        except Exception, e:
221            self.warning("Exception when reading key: %s", str(e))
222            return False
223
224        return True
225
226     def test_cred(self, cred_fn):
227        try:
228            cred = Credential(filename = cred_fn)
229        except Exception, e:
230            self.warning("Exception while reading cred: %s", str(e))
231            return False
232
233        return True
234
235     def test_privatekey_cred(self, pkey_fn, cred_fn):
236        pkey = Keypair(filename = pkey_fn)
237        cred = Credential(filename = cred_fn)
238
239        test_string = "this is a test"
240        signature = pkey.sign_string(test_string)
241
242        return cred.get_gid_caller().get_pubkey().verify_string(test_string, signature)
243
244     def sanity_check(self, container):
245        # these are some checks mainly intended for non-interactive create mode
246
247        name = container.get_name()
248        if (not name):
249            self.warning("Warning: experiment name cannot be blank")
250        elif " " in name:
251            self.warning("Warning: experiment name cannot contain spaces")
252        elif "." in name:
253            self.warning("Warning: experiment name cannot contain periods (.)")
254
255        privateKeyName = container.get_privateKeyName()
256        if (privateKeyName==None) or (privateKeyName=="None"):
257            self.warning("Warning: private key name cannot be blank")
258        elif not os.path.exists(privateKeyName):
259            self.warning("Warning: " + privateKeyName + " does not exist")
260        elif not self.test_privatekey(privateKeyName):
261            self.warning("Warning: failed to read private key")
262
263        credName = container.get_credName()
264        if (not credName) or (credName=="None"):
265            self.warning("Warning: no credential -- uploading will be unavailable")
266        elif not os.path.exists(credName):
267            self.warning("Warning: " + credName+ " does not exist")
268        elif not self.test_cred(credName):
269            self.warning("Warning: failed to read cred")
270        elif (not privateKeyName) or (privateKeyName=="None"):
271            # do nothing; can't do the final checks
272            pass
273        elif (not self.test_privatekey_cred(privateKeyName, credName)):
274            self.warning("Warning: the private key you specified doesn't seem to match the public key in the credential")
275
276     def config_questions(self, container):
277        self.Print()
278        while True:
279           name = raw_input( "Experiment name: [" + str(container.get_name()) + "] ? " )
280
281           if (not name) and (not container.get_name()):
282              self.warning("Error: experiment name cannot be blank")
283           elif not name:
284              break
285           elif " " in name:
286              self.warning("Error: experiment name cannot contain spaces")
287           elif "." in name:
288              self.warning("Error: experiment name cannot contain periods (.)")
289           else:
290              container.set_name(name)
291              break
292
293        self.Print()
294        while True:
295           privateKeyName = raw_input( "Location of your private key: [" + str(container.get_privateKeyName()) + "] ? " )
296
297           privateKeyName = os.path.expanduser(privateKeyName)
298
299           if (not privateKeyName):
300               privateKeyName = container.get_privateKeyName()
301
302           if (privateKeyName==None) or (privateKeyName=="None"):
303               self.warning("Error: private key name cannot be blank")
304           elif not os.path.exists(privateKeyName):
305               self.warning("Error: " + privateKeyName + " does not exist")
306           elif not self.test_privatekey(privateKeyName):
307               self.warning("Error: failed to read private key")
308           else:
309               container.set_privateKeyName(privateKeyName)
310               break
311
312        self.Print()
313        self.Print( "A GENI Credential file may be used to automatically upload files to the Raven" )
314        self.Print( "repository. This file is optional, but without it you will be responsible" )
315        self.Print( "for manually uploading the files." )
316        self.Print()
317
318        while True:
319           credName = raw_input( "Location of GENI cred file: [" + str(container.get_credName()) + "] ? " )
320
321           credName = os.path.expanduser(credName)
322
323           if (not credName):
324               credName = container.get_credName()
325
326           if (credName==None) or (credName == "None"):
327               container.set_credName("None")
328               break
329           elif not os.path.exists(credName):
330               self.warning("Error: " + credName + " does not exist")
331           elif not self.test_cred(credName):
332               self.warning("Error: failed to read cred")
333           elif not self.test_privatekey_cred(container.get_privateKeyName(), credName):
334               self.warning("Error: the private key you specified doesn't seem to match the public key in the credential")
335           else:
336               container.set_credName(credName)
337               break
338
339        self.Print()
340        self.Print( "Raven may be configured to manage the config files on your slices for you." )
341        self.Print( "You may enter multiple slice names separated by commas. Enterning no" )
342        self.Print( "slice names will cause packages and tpfiles to be uploaded, but not" )
343        self.Print( "slice configuration files." )
344        self.Print()
345        slices = raw_input( "Slices that should be managed by this experiment: [" +
346                            ",".join(container.get_slices()) +
347                            "] ? ")
348        if slices:
349            slices = [slice.strip() for slice in slices.split(",")]
350            container.set_slices(slices)
351
352        self.Print()
353        self.Print( "The packages.pacman file controls which packages will be installed" )
354        self.Print( "on your nodes. This tool can be configured to automatically manage" )
355        self.Print( "this file, by installing all of your packages on all of your nodes." )
356        self.Print()
357
358        self.yesno(container, "manage_packages", "Automatically manage your packages.pacman")
359
360        if (container.manage_packages):
361            self.yesno(container, "upgrade_stork", "Automatically install and upgrade stork")
362            self.yesno(container, "upgrade_owld", "Automatically install and upgrade the owl slice monitoring service")
363            self.yesno(container, "upgrade_kong", "Automatically install and upgrade the kong experiment management service")
364
365            self.Print()
366            self.Print( "By default, raven will attempt to install/upgrade all packages that" )
367            self.Print( "you place in the packages directory. You can choose to not automatically" )
368            self.Print( "install some packages, in which case the packages will be made" )
369            self.Print( "available for dependency resolution, but not automatic installation." )
370            self.Print()
371
372            while True:
373                noinstall = raw_input( "Package names to not install: [" +
374                                       ",".join(container.get_noinstall()) +
375                                       "] ? ")
376                if not noinstall:
377                    break
378
379                noinstall = [packname.strip() for packname in noinstall.split(",")]
380                packnames_with_dots = [packname for packname in noinstall if "." in packname]
381                if packnames_with_dots:
382                    self.warning( "Error: package names should not contain version numbers or extensions" )
383                else:
384                    container.set_noinstall(noinstall)
385                    break
386
387        container.check_update_id()
388
389     def isPackage(self, filename):
390        """
391        xxx hijacked from repoclassify.py
392        """
393        return filename.endswith(".rpm") or filename.endswith(".tar") or \
394               filename.endswith(".tar.gz") or filename.endswith("tar.bz2") or \
395               filename.endswith(".tgz")
396
397     def isNameFile(self, filename):
398        return filename.endswith(".name")
399
400     def isUninstallFile(self, filename):
401        return filename.endswith(".uninstall")
402
403     def build_tpfiles(self, c, save=True):
404        # start with a blank tpfile
405        # for each package in packageDir, add to tpfile
406        # for each user in userDir, add to tpfile
407
408        # packages tpfile
409        tp = tpfile()
410
411        self.build_tpfile_packages(c, tp, c.get_packagedir(), [])
412
413        self.build_tpfile_users(c, tp, c.get_userdir())
414
415        tp.save(os.path.join(c.get_tpfiledir(), c.get_name() + ".tpfile"))
416
417        return tp
418
419     def build_tpfile_users(self, c, tp, dir):
420        for filename in sorteddir(dir):
421           pathname = os.path.join(dir, filename)
422
423           if os.path.isdir(pathname):
424               continue
425
426           parts = filename.split('.')
427           ext = parts[-1]
428           if ext in ["publickey"]:
429               username = parts[-2]
430               self.Print( "  trusting user:", username )
431               tp.add_user_from_file(username, pathname)
432
433     def build_tpfile_packages(self, c, tp, dir, tags):
434        for filename in sorteddir(dir):
435           pathname = os.path.join(dir, filename)
436
437           if os.path.isdir(pathname):
438               continue
439
440           if self.isPackage(filename):
441               self.Print( "  trusting:", filename )
442               tp.add_package_from_file(pathname, tags=tags)
443           elif self.isNameFile(filename):
444               tp.add_name_from_file(filename)
445           elif self.isUninstallFile(filename):
446               tp.add_uninstall_from_file(filename)
447           else:
448               self.Print( "  unknown file in package dir:", filename )
449
450        # If subdirectories are present, they will be used to tag packages. For
451        # example, packages/foo/foobar.rpm would trust a package called foobar.rpm
452        # and apply the tag "foo" to it.
453
454        for filename in sorteddir(dir):
455            pathname = os.path.join(dir, filename)
456
457            if filename.startswith("."):
458                self.Print( "  skipping subdirectory:", filename )
459                continue
460
461            if os.path.isdir(pathname) and (not filename.startswith(".")):
462                self.Print( "  processing subdirectory:", pathname )
463                self.build_tpfile_packages(c, tp, pathname, tags + [filename])
464
465     def build_groups_pacman(self, c):
466        pg = pacgroups()
467
468        pg.add("MANAGED", "INCLUDE", "SELF")
469
470        pg.save(os.path.join(c.get_tempestgroupsdir(), "managed.groups.pacman"))
471
472     def apply_management_option(self, dir, enable, fn):
473        enabled_fn = os.path.join(dir, fn)
474        disabled_fn = os.path.join(dir, "DISABLED-" + fn)
475
476        if enable:
477           if os.path.exists(disabled_fn):
478               if os.path.exists(enabled_fn):
479                   # somehow we ended up in a situation with both files existing,
480                   # so delete the extra one.
481                   os.remove(disabled_fn)
482               else:
483                   os.rename(disabled_fn, enabled_fn)
484           elif not os.path.exists(enabled_fn):
485               self.Print( "ERROR:", enabled_fn, "is missing" )
486        else:
487           if os.path.exists(enabled_fn):
488               if os.path.exists(disabled_fn):
489                   # somehow we ended up in a situation with both files existing,
490                   # so delete the extra one.
491                   os.remove(enabled_fn)
492               else:
493                   os.rename(enabled_fn, disabled_fn)
494           elif not os.path.exists(disabled_fn):
495               self.Print( "ERROR:", enabled_fn, "or", disenabled_fn, "is missing" )
496
497     def apply_management_options(self, c):
498        dir = c.get_tempestactionsdir()
499
500        if (not os.path.exists(dir)):
501            os.makedirs(dir)
502
503        self.apply_management_option(dir, c.upgrade_owld, "owl.packages.pacman")
504        self.apply_management_option(dir, c.upgrade_stork, "stork.packages.pacman")
505        self.apply_management_option(dir, c.upgrade_kong, "kong.packages.pacman")
506
507     def build_packages_pacman(self, c, tp):
508        pp = pacpackages()
509
510        if not self.do_tempest_rpm:
511            # Deprecated -- this is the old way
512            if c.upgrade_owld:
513                pp.add("GROUP", "sfa", "UPDATE", "sfa")
514                pp.add("ALL",   None,  "UPDATE", "owl")
515
516            if c.upgrade_stork:
517                pp.add("ALL", None, "UPDATE", "stork-client")
518
519        for item in tp.get_contents():
520            if (item["type"] == "package") or (item["type"] == "name"):
521                if not ("packagename" in item):
522                    self.warning("   skipping " + str(item["pattern"]) + " from pacpackages generation because packagename is not known")
523                elif item["packagename"] == c.get_name() + RAVENCONFIG_RPM_SUFFIX:
524                    # the _ravenconfig rpm will be installed by default by tempest;
525                    # there's no reason to put it in our pacman.packages file
526                    pass
527                elif item["packagename"] in c.get_noinstall():
528                    self.info("  skipping " + str(item["packagename"]) + "from packages.pacman (on noinstall list)")
529                elif item["packagename"] in tp.uninstall_names:
530                    self.info("  skipping " + item["packagename"] + " because it is on the uninstall list")
531                else:
532                    self.info("  adding to packages.pacman:" + str(item["packagename"]))
533                    pp.add("ALL", None, "UPDATE", item["packagename"])
534
535        for item in tp.uninstall_names:
536            pp.add("ALL", None, "REMOVE", item)
537
538        # actions.d was added to ravenbuilder; older containers might not have it
539        if not os.path.exists(c.get_tempestactionsdir()):
540            os.makedirs(c.get_tempestactionsdir())
541
542        pp.save(os.path.join(c.get_tempestactionsdir(), c.get_name() + ".packages.pacman"))
543
544        return pp
545
546     def build_ravenconfig_rpm(self, c, tp):
547        """ returns True if a new RPM was built """
548
549        builder.Builder().check_rpmbuild()
550
551        spec = builder.specfile()
552        spec.cloneStdoutConfig(self)
553        spec.name = c.get_name() + RAVENCONFIG_RPM_SUFFIX
554        spec.packageRoot = c.get_tempestdir()
555        spec.load_meta()
556
557        spec.add_dir(None, "/etc", empty=False)
558        spec.add_dir(None, "/etc/tempest", empty=False)
559        spec.add_dir(None, "/etc/tempest/groups.d", empty=False)
560        spec.add_dir(None, "/etc/tempest/actions.d", empty=False)
561        spec.add_dir(None, "/etc/kong.d")
562
563        # tempest actions.d
564        for filename in sorteddir(c.get_tempestactionsdir()):
565            if os.path.isdir(filename):
566                continue
567
568            ext = filename.split('.')[-1]
569            if ext in ["pacman"]:
570                pathname = os.path.join(c.get_tempestactionsdir(), filename)
571                spec.add_file(os.path.abspath(pathname), os.path.join("/etc/tempest/actions.d", filename))
572
573        # tempest groups.d
574        for filename in sorteddir(c.get_tempestgroupsdir()):
575            if os.path.isdir(filename):
576                continue
577
578            ext = filename.split('.')[-1]
579            if ext in ["pacman"]:
580                pathname = os.path.join(c.get_tempestgroupsdir(), filename)
581                spec.add_file(os.path.abspath(pathname), os.path.join("/etc/tempest/groups.d", filename))
582
583        for filename in sorteddir(c.get_kongdir()):
584            if os.path.isdir(filename):
585                continue
586
587            # nuisance files...
588            if (filename.startswith(".")) or (filename == "__history") or (filename.endswith("~")):
589                continue
590
591            pathname = os.path.join(c.get_kongdir(), filename)
592            spec.add_file(os.path.abspath(pathname), os.path.join("/etc/kong.d", filename))
593
594        self.Print("  building:", spec.name, "RPM")
595
596        spec.update_version(None)
597        if (spec.check_current(c.get_packagedir())):
598            # no need to re-build
599            self.Print("    already up-to-date")
600            c.set_ravenconfig_rpm(spec.name + "-" + spec.version + "-" + spec.release)
601            c.save()
602            return False
603
604        result = spec.buildrpm(c.get_packagedir())
605        # result==None if build failed; pathname of build rpm otherwise
606        if not result:
607            self.error("Failed to make rpm: " + spec.name)
608            sys.exit(-1)
609
610        spec.save_meta()
611
612        c.set_ravenconfig_rpm(spec.name + "-" + spec.version + "-" + spec.release)
613        c.save()
614
615        return True
616
617     def process_template(self, c, srcFn, destFn, addlines=[]):
618        contents = open(srcFn, "r").read()
619        contents = contents.replace("%EXPERIMENT%", c.get_name())
620
621        # "addlines" specifies additional lines to add to the end of the file
622        if addlines!=[]:
623            if not contents.endswith("\n"):
624                contents = contents + "\n"
625            for line in addlines:
626                contents = contents + line + "\n"
627
628        open(destFn, "w").write(contents)
629
630     def build_config(self, c):
631        # Kind of an ugly hack here; add --tempestnew to the config files if we're
632        # going to use the new %experimentname%_ravenconfig rpm files. This will
633        # tell tempest to use actions.d/groups.d instead of the legacy files.
634        addlines=[]
635        if self.do_tempest_rpm:
636            addlines.append("tempestnew")
637
638        for slice in c.get_slices():
639            self.process_template(c,
640                             os.path.join(c.get_templatedir(), "stork.conf.template"),
641                             os.path.join(c.get_configdir(), slice + ".stork.conf"),
642                             addlines)
643
644     def build_kong(self, c, tp):
645        f = open(os.path.join(c.get_kongdir(), "kong-raven.conf"), "w")
646        f.write("# This file is maintained by the raven tool. Do not edit it.\n")
647        f.write("[kong]\n")
648        f.write("channel = " + c.get_id() + "\n")
649
650        # If c.manage_packages isn't specified, then we're not auto-generating
651        # packages.pacman files. Thus, we better not try to auto-generate the
652        # kong package list either.
653
654        if c.manage_packages:
655            # Note: We assume that everything that is in 'tp' is going to also be in
656            # the packages.pacman
657
658            f.write("\n[packages]\n")
659            for item in tp.get_contents():
660                if (not ("packagename" in item)):
661                    continue
662
663                # we're duplicating some logic of build_pacman_packages
664
665                # We'll handle the ravenconfig rpm by putting it's version directly
666                # in the digdug packet.
667                if item["packagename"] == c.get_name() + RAVENCONFIG_RPM_SUFFIX:
668                    continue
669
670                # Don't require things we're uninstalling
671                if (item["packagename"] in c.get_noinstall()) or (item["packagename"] in tp.uninstall_names):
672                    continue
673
674                if (item["type"] == "package"):
675                    metadata = item.get("metadata",{})
676                    if (not ("version" in metadata)) or (not ("release" in metadata)):
677                        continue
678
679                    f.write(item["packagename"] + " = " + item["packagename"] + "-" + metadata["version"] + "-" + metadata["release"] + "\n")
680
681                elif (item["type"] == "name"):
682                    f.write(item["packagename"] + " = " + item["packagename"] + "\n")
683
684     def sign_everything(self, c):
685        # tpfiles
686        for filename in sorteddir(c.get_tpfiledir()):
687            pathname = os.path.join(c.get_tpfiledir(), filename)
688            if os.path.isdir(pathname):
689                continue
690
691            ext = filename.split('.')[-1]
692            if ext in ["tpfile"]:
693                self.Print("  signing:", filename)
694                ravenlib.crypto.xmlsignfile_to_dir(c.get_privateKey(), pathname, c.get_uploaddir())
695
696        if not self.do_tempest_rpm:
697            # tempest actions.d
698            for filename in sorteddir(c.get_tempestactionsdir()):
699                if os.path.isdir(filename):
700                    continue
701
702                ext = filename.split('.')[-1]
703                if ext in ["pacman"]:
704                    pathname = os.path.join(c.get_tempestactionsdir(), filename)
705                    self.Print("  signing:", filename)
706                    ravenlib.crypto.xmlsignfile_to_dir(c.get_privateKey(), pathname, c.get_uploaddir())
707
708            # tempest groups.d
709            for filename in sorteddir(c.get_tempestgroupsdir()):
710                if os.path.isdir(filename):
711                    continue
712
713                ext = filename.split('.')[-1]
714                if ext in ["pacman"]:
715                    pathname = os.path.join(c.get_tempestgroupsdir(), filename)
716                    self.Print("  signing:", filename)
717                    ravenlib.crypto.xmlsignfile_to_dir(c.get_privateKey(), pathname, c.get_uploaddir())
718
719        # config files
720        for filename in sorteddir(c.get_configdir()):
721            if os.path.isdir(filename):
722                continue
723
724            ext = filename.split('.')[-1]
725            if ext in ["conf"]:
726                pathname = os.path.join(c.get_configdir(), filename)
727                self.Print("  signing:", filename)
728                ravenlib.crypto.xmlsignfile_to_dir(c.get_privateKey(), pathname, c.get_uploaddir())
729
730     def clean_upload_symlinks(self, c):
731         for filename in os.listdir(c.get_uploaddir()):
732             pathname = os.path.join(c.get_uploaddir(), filename)
733             if os.path.islink(pathname) and (not filename.startswith(".")):
734                 os.remove(pathname)
735
736     def copy_packages(self, c, tp):
737         for item in tp.contents:
738             if item["type"] != "package":
739                 continue
740
741             if not ("metadata" in item):
742                 continue
743
744             metadata = item['metadata']
745
746             # for packages where a URL was specified, we want to create a metadata
747             # file, and the metadata file will be uploaded instead of the package.
748             if "URL" in metadata:
749                 fn = metadata['name'] + '-' + metadata['version'] + '-' + metadata['release'] + ".metadata"
750                 self.Print("  writing:", fn)
751                 ravenlib.package.storkpackage.package_metadata_dict_to_fn(metadata, c.get_uploaddir(), fn)
752
753             # otherwise, copy the full package to the upload directory.
754             elif "srcFileName" in item:
755                 srcname = item["srcFileName"]
756                 basename = os.path.basename(srcname)
757                 if item.get("tags", None):
758                     # when tags are used, it's possible we could have a filename
759                     # collision, since we might have the same named package in two
760                     # different tagged subdirectories.
761                     basename = "".join(item["tags"]) + "_" + basename
762
763                 destname = os.path.join(c.get_uploaddir(), basename)
764
765                 if os.path.lexists(destname):
766                     os.remove(destname)
767
768                 self.Print("  linking:", os.path.basename(srcname))
769                 os.symlink(os.path.abspath(srcname), destname)
770
771             else:
772                 self.warning("  unsure how to process for upload: " + str(metadata['name']))
773
774
775     def upload_everything(self, c, url):
776        self.Print("repository:", url)
777
778        if c.get_credName() == "None":
779            self.Print("skipping upload because no GENI credential configured")
780            return
781
782        client = RepoClient(url, c.get_privateKeyName())
783        for filename in os.listdir(c.get_uploaddir()):
784            pathname = os.path.join(c.get_uploaddir(), filename)
785
786            if os.path.isdir(pathname):
787                continue
788
789            # If it's a symlink, upload using the linked name. This is used as
790            # part of the workaround for the same package with multiple tags (see
791            # copy_packages, where the symlink is done)
792            if os.path.islink(pathname):
793                pathname = os.readlink(pathname)
794                filename = os.path.basename(pathname)
795
796            fileData = file(pathname).read()
797
798            # Compute the hash of the file to be uploaded, see if a file exists in
799            # the cache directory by that name. If it does, then the file has already
800            # been uploaded.
801            hash = hashlib.sha1(filename + fileData).hexdigest()
802            hashFileName = os.path.join(c.get_cachedir(), hash + ".uploaded")
803            if os.path.exists(hashFileName):
804                self.Print("  already-uploaded:", filename)
805                continue
806
807            self.Print("  uploading:", filename)
808            result = client.upload_file(c.get_cred(), pathname, fileData)
809            self.Print("   ", result)
810
811            if result == True:
812                open(hashFileName,"w").write("filename: "+pathname+"\n")
813
814     def do_create(self, c, opts, args):
815        if c.exists():
816            self.error("Error: experiement container already exists")
817            sys.exit(-1)
818
819        if opts.importconfig:
820            c.read_config(opts.importconfig)
821
822        if opts.keyfn:
823            c.set_privateKeyName(opts.keyfn)
824        if opts.credfn:
825            c.set_credName(opts.credfn)
826        if opts.expname:
827            c.set_name(opts.expname)
828        if opts.slices:
829            slices = [slice.strip() for slice in opts.slices.split(",")]
830            c.set_slices(slices)
831
832        if not opts.noninteractive:
833           self.config_questions(c)
834
835        self.sanity_check(c)
836
837        c.makedirs()
838
839        c.copy_initial_files()
840
841        c.save()
842
843     def do_config(self, c, opts, args):
844        self.config_questions(c)
845        c.save()
846
847     def do_info(self, c, opts, args):
848        self.Print("Experiment:", c.get_name())
849        self.Print("Private Key:", c.get_privateKeyName())
850        self.Print("Credential:", c.get_credName())
851        self.Print("Slices:", ",".join(c.get_slices()))
852        self.Print("Manage Packages:", c.manage_packages)
853        if c.manage_packages:
854            self.Print("Keep stork upgraded:", c.upgrade_stork)
855            self.Print("Keep owl upgraded:", c.upgrade_owld)
856            self.Print("Packages to not install:", ",".join(c.get_noinstall()))
857
858
859     def do_buildrpm(self, c, opts, args):
860        b = builder.Builder()
861        b.cloneStdoutConfig(self)
862        b.build_root(c.get_builderdir(), c.get_packagedir())
863
864     def do_build(self, c, opts, args):
865        self.do_buildrpm(c, opts, args)
866
867        something_changed = True
868
869        while something_changed:
870            tp = self.build_tpfiles(c)
871            self.build_groups_pacman(c)
872            if c.manage_packages:
873                self.build_packages_pacman(c, tp)
874                self.apply_management_options(c)
875            self.build_config(c)
876            self.build_kong(c, tp)
877
878            if self.do_tempest_rpm:
879               something_changed = self.build_ravenconfig_rpm(c, tp)
880               if something_changed:
881                   # if we modified the tempest rpm, then we have to go back and compute
882                   # tpfiles again
883                   self.Print("    re-building tpfiles due to change in tempest rpm")
884            else:
885               something_changed = False
886
887        self.clean_upload_symlinks(c)
888        self.copy_packages(c, tp)
889        self.sign_everything(c)
890
891     def do_upload(self, c, opts, args):
892        self.upload_everything(c, opts.url)
893
894     def do_testconnect(self, c, opts, args):
895        client = RepoClient(opts.url, c.get_privateKeyName())
896        client.credNoop(value="hello, world!", cred=c.get_cred())
897
898     def do_publish(self, c, opts, args):
899        self.do_build(c, opts, args)
900        self.do_upload(c, opts, args)
901
902     def do_upgrade(self, c):
903        if c.version < 1:
904            self.Print("  upgrading experiment container to version 1")
905
906            if not os.path.exists(c.get_tempestactionsdir()):
907                os.makedirs(c.get_tempestactionsdir())
908
909            if not os.path.exists(c.get_tempestgroupsdir()):
910                os.makedirs(c.get_tempestgroupsdir())
911
912            shutil.copyfile("/usr/local/raven/etc/stork.packages.pacman",
913                            os.path.join(c.get_tempestactionsdir(), "stork.packages.pacman"))
914
915            shutil.copyfile("/usr/local/raven/etc/owl.packages.pacman",
916                            os.path.join(c.get_tempestactionsdir(), "owl.packages.pacman"))
917
918            c.version = 1
919            c.save()
920
921        if c.version < 2:
922            self.Print("  upgrading experiment container to version 2")
923
924            if not os.path.exists(c.get_kongdir()):
925                os.makedirs(c.get_kongdir())
926
927            for fn in ["prepare", "start", "stop", "abort", "kong-server.conf",
928                       "prepared", "complete", "opt_out"]:
929                shutil.copyfile("/usr/local/raven/etc/" + fn,
930                                os.path.join(c.get_kongdir(), fn))
931
932            c.version = 2
933            c.save()
934
935        if c.version < 3:
936            self.Print("  upgrading experiment container to version 3")
937
938            shutil.copyfile("/usr/local/raven/etc/kong.packages.pacman",
939                    os.path.join(c.get_tempestactionsdir(), "kong.packages.pacman"))
940
941            c.version = 3
942            c.save()
943
944        if c.version < 4:
945            self.Print("  upgrading experiment container to version 4")
946
947            c.check_update_id()
948
949            c.version = 4
950            c.save()
951
952     def do_kong(self, c, opts, args):
953        if len(args) < 1:
954           self.error("syntax: raven experiment <command>")
955           self.error("   command = [prepare|run|reset|status|results]")
956           sys.exit(-1)
957
958        command = args[0]
959        args = args[1:]
960
961        client = kongclient.KongClient(configdir = c.get_kongdir())
962
963        if opts.numclients > 0:
964            client.set_minClients(int(opts.numclients))
965            client.set_maxClients(int(opts.numclients))
966
967        if opts.html:
968            client.set_table_output("html")
969        elif opts.csv:
970            client.set_table_output("csv")
971
972        ravenconfig_rpm = c.get_ravenconfig_rpm()
973        if ravenconfig_rpm:
974            client.set_variable("packages", [("ravenconfig_rpm", ravenconfig_rpm)])
975
976        if command == "prepare":
977            client.prepare()
978        elif command == "run":
979            client.run()
980        elif command == "reset":
981            client.reset()
982        elif command == "status":
983            client.status()
984        elif command == "results":
985            client.results()
986        elif command == "setstate":
987            if len(args)<1:
988               self.error("syntax: raven experiment setstate <state-name>")
989               sys.exit(-1)
990            client.setstate(args[0])
991
992     def do_slice(self, c, opts, args):
993        if (not "SSH_AGENT_PID" in os.environ):
994            self.Print("NOTE: Make sure you use ssh-agent/ssh-add before using this command to")
995            self.Print("      install Stork if you use a passphrase. (see wiki documentation)")
996            self.Print()
997        if (not opts.slicename) and (len(args)>0) and (args[0] == "install"):
998            self.Print("NOTE: For ProtoGENI slices, make sure to use a --slicename option")
999            self.Print("   example: raven slice --slicename myslice install myspec.xml")
1000            self.Print()
1001
1002        if (len(args)>0) and (args[0] == "install"):
1003            cred_fn = c.get_credName()
1004            if (os.path.exists(cred_fn)):
1005               cred = Credential(filename = cred_fn)
1006               if opts.sslkey==None:
1007                   opts.sslkey = cred.get_gid_caller().get_pubkey().get_pubkey_string()
1008
1009        slicerun.main("raven slice", opts, args)
1010
1011
1012     def verify_error(self, err_msg):
1013        self.error("Using the Publish or Upload commands requires a valid GENI private key")
1014        self.error("and credential.")
1015        self.error("")
1016        self.error(err_msg)
1017        self.error("")
1018        self.error("Please verify the files are correct and use 'raven config' if necessary")
1019        self.error("to change the setting")
1020        sys.exit(-1)
1021
1022     def verify_uploadable(self, c):
1023         pk = c.get_privateKeyName()
1024         cred = c.get_credName()
1025
1026         if (not pk) or (not cred):
1027             self.verify_error("Error: Private key or credential is not configured.")
1028
1029         if not os.path.isfile(pk):
1030             self.verify_error("Error: private key " + pk + " does not exist.")
1031
1032         if not os.path.isfile(cred):
1033             self.verify_error("Error: credential " + cred + " does not exist.")
1034
1035         if not self.test_privatekey(pk):
1036             self.verify_error("Error: private key " + pk + " cannot be loaded.")
1037
1038         if not self.test_cred(cred):
1039             self.verify_error("Error: credential " + cred + " cannoy be loaded.")
1040
1041         if not self.test_privatekey_cred(pk, cred):
1042             self.verify_error("Error: private key does not match credential.")
1043
1044     # break main() into some subsections so they can be easily overridable.
1045
1046     def main_parse(self, args):
1047        """ parse the command line arguments """
1048        parser = self.create_parser()
1049        (options, args) = parser.parse_args(args)
1050        self.options = options
1051        self.main_cmd_parse(args)
1052
1053     def main_cmd_parse(self, args):
1054        """ parse options that occur after the command """
1055        if len(args) <= 0:
1056             self.error("No command given. Use -h for help.")
1057             sys.exit(-1)
1058
1059        command = args[0]
1060        (cmd_opts, cmd_args) = self.create_cmd_parser(command).parse_args(args[1:])
1061        self.command = command
1062        self.cmd_opts = cmd_opts
1063        self.cmd_args = cmd_args
1064
1065     def main_initlogger(self):
1066        """ initialize logging (where is this used?) """
1067        handler = logging.StreamHandler()
1068        handler.setFormatter(logging.Formatter("%(levelname)s:%(filename)s:%(message)s"))
1069        logging.getLogger().addHandler(handler)
1070        if self.options.verbose:
1071            logging.getLogger().setLevel(logging.DEBUG)
1072            handler.setLevel(logging.DEBUG)
1073
1074     def main_init(self):
1075        """ initialize various stork/raven modules """
1076        ravenlib.package.storkpackage.initialize()
1077
1078     def main_init_passphrase_handler(self):
1079        """ initialize the passphrase handler """
1080        try:
1081            from sfa.trust.certificate import set_passphrase_callback\r
1082            set_passphrase_callback(ask_passphrase)
1083        except ImportError:
1084            # older SFA do not support this
1085            pass
1086
1087     def main_init_container(self):
1088        """ create and/or open the container """
1089        c = container.container()
1090        if self.options.directory:
1091            c.set_dir(self.options.directory)
1092
1093        if self.options.no_tempest_rpm:
1094            self.do_tempest_rpm = False
1095
1096        if self.command in ["config", "publish", "build", "upload", "buildrpm", "info", "upgrade", "testconnect", "experiment", "slice"]:
1097            if not c.exists():
1098                self.error("Error: experiment container does not exist")
1099                sys.exit(-1)
1100            c.load()
1101            if (self.command != "upgrade") and (c.version < container.CONTAINER_VERSION):
1102                self.error("Error: experiment container is old version. Use 'raven upgrade' to upgrade.")
1103                sys.exit(-1)
1104
1105        if self.command in ["publish", "upload", "testconnect"]:
1106            self.verify_uploadable(c)
1107
1108        return c
1109
1110     def main_dispatch(self, c):
1111        """ execute the command """
1112        if self.command == "create":
1113            self.do_create(c, self.cmd_opts, self.cmd_args)
1114        elif self.command == "upgrade":
1115            self.do_upgrade(c)
1116        elif self.command == "config":
1117            self.do_config(c, self.cmd_opts, self.cmd_args)
1118        elif self.command == "publish":
1119            self.do_publish(c, self.cmd_opts, self.cmd_args)
1120        elif self.command == "build":
1121            self.do_build(c, self.cmd_opts, self.cmd_args)
1122        elif self.command == "upload":
1123            self.do_upload(c, self.cmd_opts, self.cmd_args)
1124        elif self.command == "buildrpm":
1125            self.do_buildrpm(c, self.cmd_opts, self.cmd_args)
1126        elif self.command == "info":
1127            self.do_info(c, self.cmd_opts, self.cmd_args)
1128        elif self.command == "testconnect":
1129            self.do_testconnect(c, self.cmd_opts, self.cmd_args)
1130        elif self.command == "experiment":
1131            self.do_kong(c, self.cmd_opts, self.cmd_args)
1132        elif self.command == "slice":
1133            self.do_slice(c, self.cmd_opts, self.cmd_args)
1134
1135     def main(self, args):
1136        args = self.main_parse(args)
1137
1138        self.main_initlogger()
1139
1140        self.main_init()
1141
1142        self.main_init_passphrase_handler()
1143
1144        c = self.main_init_container()
1145
1146        self.main_dispatch(c)
1147
1148 if __name__=="__main__":
1149    r = Raven()
1150    r.main(sys.argv[1:])
1151