import repository from arizona
[raven.git] / apps / tempest / tempest.py
1 #!/usr/bin/env python
2
3 # John H. Hartman
4 # pacman -- package manager
5 # Installs, updates, and removes packages based on command-line options and
6 # the contents of the packages.xml and groups.xml files.
7
8 #           [option,    long option,            variable,               action,         data,           default,                                metavar,        description]
9 """arizonaconfig
10     options=[["",       "--tempestsummaryfile",  "tempestsummaryfile",    "store",      "string",       "/var/log/tempest.log",  None,       "tempest log file"],
11              ["",       "--tempestconfigfile",   "tempestconfigfile",   "store",        "string",       "/usr/local/stork/etc/stork.conf",      "FILENAME",     "configuration file"],
12              ["",       "--template-install",   "templateInstall",      "store",        "string",       "stork ( %a) (%p)(=%v)",        "TEMPLATE",     "template for install"],
13              ["",       "--template-remove",    "templateRemove",       "store",        "string",       "stork --remove( %a) (%p)(=%v)",                "TEMPLATE",     "template for remove"],
14              ["",       "--template-update",    "templateUpdate",       "store",        "string",       "stork --upgrade( %a) (%p)(=%v)",               "TEMPLATE",     "template for update"],
15              ["",       "--no-act",             "noAct",                "store_true",   None,           False,                                  None,           "Don't execute any actions"],
16              ["",       "--lockdir",            "lockdir",              "store",        "string",       "/var/lock",                        "dir",   "use the specified mutex lock directory (default /var/lock)"],
17              ["",       "--mergeactions",       "mergeactions",         "store_true",   None,           False,   None,   "Merge actions when calling stork"],
18              ["",       "--node",               "node",                 "store",        "string",       None,                                   "HOSTNAME",    "node name. uses current hostname by default"],
19              ["",       "--tempestnew",         "tempestnew",           "store_true",   None,           False,   None,   "Use new packages.d/actions.d directory; disable legacy file processing"],
20              ]
21     includes=[]
22 """
23
24 import sys, os, string, re, socket
25 import arizonaconfig
26 import arizonareport
27 import storkusername
28 import arizonacrypt
29 import arizonageneral
30 import time
31 import ravenlib.dirchecker
32 import storkrepolist
33 import storkstatuscodes
34 import subprocess
35 import tempestversion
36 import ravenlib.platform.all
37
38 TEMPEST_ETC_DIR = "/etc/tempest"
39
40 usagemsg="""
41 tempest [OPTIONS]
42
43 Options:
44 --tempestconfigfile CONFIGFILE       specify a configuration file
45                                     (default: stork.conf)
46 --template-install  COMMAND         change the command for an "install" call
47 --template-remove   COMMAND         change the command for a "remove" call
48 --template-update   COMMAND         change the command for an "update" call
49 --no-act                            tempest does nothing
50 """
51
52 class Action(object):
53     def __init__(self, tag, args, package, version):
54         self.tag = tag
55         self.args = args
56
57         package = self.fixup_packagename(package)
58
59         # smbaker: combined version and package name because it makes it
60         #    easier to merge related actions.
61
62         if version:
63             self.package = package + '=' + version
64         else:
65             self.package = package
66         self.version = None
67
68     def fixup_packagename(self, package):
69         if "%storkusername%" in package:
70             # NOTE: this only returns the first user name -- could there be more?
71             package = package.replace("%storkusername%", storkusername.get_usernames()[0])
72
73         return package
74
75     def __cmp__(self, other):
76         if (self.tag == other.tag) and (self.args == other.args) and \
77                 (self.package == other.package) and (self.version == other.version):
78             return 0
79         else:
80             return 1
81
82     def canMerge(self, other):
83         if self.tag != other.tag:
84             return 0
85         if self.args != other.args:
86             return 0
87         # let's only try to merge installs and updates for now
88         if (self.tag != "INSTALL") and (self.tag != "UPDATE"):
89             return 0
90         return 1
91
92     def merge(self, other):
93         if not self.canMerge(other):
94             return 0
95         self.package = self.package + ' ' + other.package
96         return 1
97
98 def pacmansummary_create():
99     """
100     <Purpose>
101        Creates a pacman summary log file.
102
103     <Arguments>
104        None
105
106     <Returns>
107        None
108     """
109     fn = arizonaconfig.get_option("tempestsummaryfile")
110     if fn:
111         try:
112             # append _current to the filename. This will prevent us from
113             # overwriting the existing pacman summary file. We will rename it
114             # in pacmansummary_complete
115             file = open(fn, "a+")
116             if file:
117                 file.write("***tempest session started " + time.ctime() + "\n")
118                 file.close()
119         except IOError:         #changed from general exception
120             arizonareport.send_error(0, "failed to open tempest log file")
121
122
123 def pacmansummary_complete():
124     """
125     <Purpose>
126        Finalize the pacman summary file by writing the current date and time.
127
128     <Arguments>
129        None
130
131     <Returns>
132        None
133     """
134     fn = arizonaconfig.get_option("tempestsummaryfile")
135     if fn:
136         try:
137             file = open(fn, "a+")
138             if file:
139                 file.write("***tempest session completed " + time.ctime() + "\n")
140                 file.close()
141         except (IOError, OSError):   #changed from general exception
142             arizonareport.send_error(0, "failed to complete tempest log file")
143
144
145 def pacmansummary_append(s):
146     """
147     <Purpose>
148        Append a line to the pacman summary file.
149
150     <Arguments>
151        None
152
153     <Returns>
154        None
155     """
156     fn = arizonaconfig.get_option("tempestsummaryfile")
157     if fn:
158         try:
159             file = open(fn, "a")
160             if file:
161                 file.write(s + "\n")
162                 file.close()
163         except IOError:   #changed from general exception
164             arizonareport.send_error(0, "failed to append tempest log file");
165
166 def pacmansummary_append_msg(s):
167     pacmansummary_append("   " + s)
168
169 def pacmansummary_append_status(rc):
170     """
171     <Purpose>
172        Append status info to a pacman summary file.
173
174     <Arguments>
175        rc
176            The exit status of the stork.py program. It is a bitmask of possible
177            error codes.
178
179     <Returns>
180        None
181     """
182     rc = rc >> 8
183     msg = "   status: "+str(rc)
184     if rc & storkstatuscodes.STATUS_ERROR != 0:
185         msg = msg + " ERROR"
186     if rc & storkstatuscodes.STATUS_ALREADY_RUNNING != 0:
187         msg = msg + " ALREADY-RUNNING"
188     if rc & storkstatuscodes.STATUS_BAD_OPTION != 0:
189         msg = msg + " BAD-OPTION"
190     if rc & storkstatuscodes.STATUS_ALREADY_DONE != 0:
191         msg = msg + " ALREADY-DONE"
192     if rc & storkstatuscodes.STATUS_PACKAGES_INSTALLED != 0:
193         msg = msg + " SOME-PACKAGES-INSTALLED"
194     if rc & storkstatuscodes.STATUS_PACKAGES_REMOVED != 0:
195         msg = msg + " SOME-PACKAGES-REMOVED"
196     if rc & storkstatuscodes.STATUS_NOT_FOUND != 0:
197         msg = msg + " PACKAGES-NOT-FOUND"
198
199     pacmansummary_append(msg)
200
201
202
203
204
205 def MergeActions(actions):
206     """
207     <Purpose>
208        Merge similar actions
209
210     <Arguments>
211        actions
212            list of actions to merge
213
214     <Side Effects>
215        Elements of 'actions' may be modified
216     """
217     newActions = []
218     lastAction = None
219
220     for action in actions:
221         if lastAction and lastAction.canMerge(action):
222             lastAction.merge(action)
223         else:
224             if lastAction:
225                 newActions.append(lastAction)
226             lastAction = action
227
228     if lastAction:
229         newActions.append(lastAction)
230
231     return newActions
232
233 def GroupFileParse(group_file_name, node_names, signed = False):
234     pacmansummary_append_msg("groups file = " + str(group_file_name))
235     arizonareport.send_out(1, "reading groups file " + str(group_file_name))
236     temp_filename = None
237     src_filename = None
238     try:
239         if signed:
240             try:
241                # SMB - storkpackagelist.find_file() already checked the signature for\r
242                # us, so we can assume the file is valid and extract the contents\r
243                temp_filename = arizonacrypt.XML_retrieve_originalfile_from_signedfile(group_file_name)\r
244                src_filename = temp_filename\r
245             except TypeError, e:\r
246                arizonareport.send_error(0, str(e))\r
247                sys.exit(1)
248         else:
249             src_filename = group_file_name
250
251         lines = file(src_filename).readlines()
252
253         if len(lines)>0 and lines[0].startswith("#!"):
254            script = src_filename
255            os.system("chmod +x " + src_filename)
256         else:
257            script = os.path.join(os.path.dirname(os.path.realpath(__file__)), "xmlgroupparse.py")
258
259         command = script + " " + src_filename + " " + ",".join(node_names)
260         arizonareport.send_out(2, "Running: " + command)
261         (stdout, stderr, status) = arizonageneral.popen5(command)
262         if stderr:
263             arizonareport.send_out(2, "".join(stderr))
264
265         groups = []
266         for line in stdout:
267             if line.startswith("#"):
268                 continue
269             line = line.strip("\n")
270             if line:
271                 groups.append(line)
272
273         return groups
274     finally:
275         if temp_filename:
276             os.remove(temp_filename)
277
278 def PackageFileParse(package_file_name, node_names, slice_names, group_list, signed = False):
279     pacmansummary_append_msg("package file = " + str(package_file_name))
280     arizonareport.send_out(1, "reading package file " + str(package_file_name))
281     temp_filename = None
282     src_filename = None
283     try:
284         if signed:
285             try:
286                # SMB - storkpackagelist.find_file() already checked the signature for
287                # us, so we can assume the file is valid and extract the contents\r
288                temp_filename = arizonacrypt.XML_retrieve_originalfile_from_signedfile(package_file_name)\r
289                src_filename = temp_filename\r
290             except TypeError, e:\r
291                arizonareport.send_error(0, str(e))\r
292                sys.exit(1)
293         else:
294             src_filename = package_file_name
295
296         lines = file(src_filename).readlines()
297
298         if len(lines)>0 and lines[0].startswith("#!"):
299            script = src_filename
300            os.system("chmod +x " + src_filename)
301         else:
302            script = os.path.join(os.path.dirname(os.path.realpath(__file__)), "xmlpackageparse.py")
303
304         command = script + " " + src_filename + " " + ",".join(node_names) + " " + ",".join(slice_names) + " " + " ".join(group_list)
305         arizonareport.send_out(2, "Running: " + command)
306         (stdout, stderr, status) = arizonageneral.popen5(command)
307         if stderr:
308            arizonareport.send_out(2, "".join(stderr))
309
310         actions = []
311         for line in stdout:
312             if line.startswith("#"):
313                 continue
314             parts = line.strip("\n").split(" ")
315             if parts:
316                 tag = parts[0]
317                 packagever = parts[1]
318                 if "=" in packagever:
319                    package = packagever.split("=")[0]
320                    version = packagever.split("=")[1]
321                 else:
322                    package = packagever
323                    version = None
324                 args = []
325                 action = Action(tag, args, package, version)
326                 actions.append(action)
327
328         return actions
329     finally:
330         if temp_filename:
331             os.remove(temp_filename)
332
333 def Find_Local_Files(kind):
334     """ get a list of files in /etc/tempest/groups.d or /etc/tempest/actions.d
335         ignore obviously wrong things (".svn", ".cvs", editor temp files, directories, etc)
336         sort the list
337     """
338     path = os.path.join(TEMPEST_ETC_DIR, kind+".d")
339
340     # if the directory does not exist, or is not a dir, then return
341     if not os.path.isdir(path):
342         return []
343
344     items = os.listdir(path)
345     items.sort()
346     result = []
347     for item in items:
348         if item.startswith("."):
349             # things that start with "." are ignored
350             continue
351
352         if item.endswith("~"):
353             # things that end with "~" (editor backup files) are ignored
354             continue
355
356         if item.startswith("DISABLED-") or item.startswith("disabled-"):
357             # things that start with DISABLED- are disabled by the raven tool
358             continue
359
360         pathname = os.path.join(path, item)
361         if not os.path.isfile(pathname):
362             # things that are not regular files are ignored
363             continue
364
365         result.append(pathname)
366
367     return result
368
369 glo_last_exec_log_lines = []
370
371 def exec_and_log(c):
372     """ Create a pipe to handle log messages, and then execute stork. """
373
374     global glo_last_exec_log_lines
375
376     (r,w) = os.pipe()
377
378     c = c + " --storklogdescriptor " + str(w)
379
380     p = subprocess.Popen(c, shell=True)
381     os.close(w)
382
383     f = os.fdopen(r, "r")
384     log_lines = f.readlines()
385     f.close()
386
387     status = os.waitpid(p.pid, 0)[1]
388
389     for line in log_lines:
390         if line in glo_last_exec_log_lines:
391             # If the prior invocation of stork produced the same output, then
392             # don't print it again.
393             pass
394         else:
395             pacmansummary_append_msg(line.strip())
396
397     glo_last_exec_log_lines = log_lines
398
399     return status
400
401 def Main(path):
402     args = arizonaconfig.init_options('tempest.py', usage=usagemsg, configfile_optvar='tempestconfigfile', version=tempestversion.VERREL)
403
404     dircheck = None
405
406     if os.geteuid() > 0:
407         arizonareport.send_error(0, "You must be root to run this program...")
408         sys.exit(1)
409
410     # XXX quick and dirty way to prevent pacman from running on a node.
411     # XXX remember to disable when I'm done with this
412     if os.path.exists("/usr/local/stork/disable_pacman"):
413         arizonareport.send_out(0, "tempest disabled by /usr/local/stork/disable_pacman")
414         sys.exit(0)
415
416     # grab the pacman mutex. This prevents multiple copies of pacman from
417     # running at the same time. It is held for the duration that pacman is
418     # active.
419     pacmanLock = arizonageneral.mutex_lock("pacman", arizonaconfig.get_option("lockdir"))
420     if not pacmanLock:
421         arizonareport.send_error(0, "Another copy of tempest is already running...")
422         sys.exit(0)
423
424     pacmansummary_create()
425
426     # TODO: do we need to wait for the 'stork' mutex like we do in stork.py?
427
428     # grab the stork mutex. This ensure that another copy of stork is not
429     # downloading config files or repository. This mutex is released before
430     # pacman calls stork to execute actions (or else deadlock would result)
431     storkLock = arizonageneral.mutex_lock("stork", arizonaconfig.get_option("lockdir"))
432     if not storkLock:
433         arizonareport.send_error(0, "Another copy of stork is already running...")
434         pacmansummary_append_msg("exited due to stork already running")
435         pacmansummary_complete()
436         sys.exit(0)
437
438     act = (arizonaconfig.get_option("noAct") == False)
439
440     templates = {}
441     for key,var in (("INSTALL","templateInstall"),("REMOVE","templateRemove"), ("UPDATE","templateUpdate")):
442         templates[key] = arizonaconfig.get_option(var)
443
444     ravenlib.platform.all.init()
445
446     if arizonaconfig.get_option("node"):
447         node_names = [arizonaconfig.get_option("node")]
448     else:
449         node_names = ravenlib.platform.all.get_node_names()
450
451     slice_names = ravenlib.platform.all.get_slice_names()
452
453     pacmansummary_append_msg("slice = " + ",".join(slice_names))
454
455     # start out with platform-dependant group names
456     myGroups = ravenlib.platform.all.get_group_names()
457
458     # start out with no actions
459     actions = []
460
461     if not arizonaconfig.get_option("tempestnew"):
462         # so we can find the pacman files from the repo (LEGACY)
463         storkrepolist.init()
464
465         # find the groupFile in the repositories (LEGACY)
466         (groupFile, junk1, junk2) = \
467             storkrepolist.find_file_kind("pacman", "groups.pacman")
468         if groupFile:
469             myGroups = myGroups + GroupFileParse(groupFile, node_names, signed=True)
470         else:
471             arizonareport.send_error(0, "WARNING: no groups file from repository")
472
473         pacmansummary_append_msg("groups = " + ",".join(myGroups))
474
475         # find the packageFile in the repositories (LEGACY)
476         (packageFile, junk1, junk2) = \
477             storkrepolist.find_file_kind("pacman", "packages.pacman")
478         if packageFile:
479             actions = actions + PackageFileParse(packageFile, node_names, slice_names, myGroups, signed=True)
480         else:
481             arizonareport.send_error(0, "WARNING: no packages file from repository")
482
483     else:
484         # create a dirchecker object so we can tell if the actions we execute
485         # modify the groups.d/actions.d directories.
486         dircheck = ravenlib.dirchecker.dirchecker(TEMPEST_ETC_DIR)
487
488         # find groupFiles in directories (NEW)
489         groupFiles = Find_Local_Files("groups")
490         for groupFile in groupFiles:
491             myGroups = myGroups + GroupFileParse(groupFile, node_names, signed=False)
492
493         pacmansummary_append_msg("groups = " + ",".join(myGroups))
494
495         # find packageFiles in directories (NEW)
496         packageFiles = Find_Local_Files("actions")
497         for packageFile in packageFiles:
498             actions = actions + PackageFileParse(packageFile, node_names, slice_names, myGroups, signed=False)
499
500     if not actions:
501        arizonareport.send_error(0, "ERROR: no actions to perform.")
502        pacmansummary_append_msg("exiting due to failure to find packages.pacman file")
503        pacmansummary_complete();
504        sys.exit(1)
505
506     arizonareport.send_out(2, "Slice: " + ",".join(slice_names))
507     arizonareport.send_out(2, "Node: " + " ,".join(node_names))
508     arizonareport.send_out(2, "Groups: " + ",".join(myGroups))
509
510     if arizonaconfig.get_option("mergeactions"):
511         actions = MergeActions(actions)
512
513     #
514     # Create the commands to run based on the command templates.
515     #
516     commands = []
517     for action in actions:
518         cmd = templates[action.tag]
519         fmt = r'\((?P<foo>[^\(]*)\%%%s(?P<bar>[^\)]*)\)'
520         for k,value in (("p",action.package),("v",action.version),
521                 ("a",action.args)):
522             r = re.compile(fmt % k)
523             if value:
524                 s = r'\g<foo>' + value + r'\g<bar>'
525                 cmd = r.sub(s, cmd)
526             else:
527                 cmd = r.sub("", cmd)
528         commands.append(cmd)
529
530     arizonareport.send_out(1, "Actions:")
531     for c in commands:
532         arizonareport.send_out(1, "  " + c)
533
534     if act:
535         # release the stork lock, so stork can run
536         arizonageneral.mutex_unlock(storkLock)
537
538         for c in commands:
539             pacmansummary_append("executing: " + c)
540             rc = exec_and_log(c) #os.system(c)
541             pacmansummary_append_status(rc)
542             if (rc != 0) and (rc != storkstatuscodes.STATUS_ALREADY_DONE):
543                 print "Command '" + c + "' failed"
544     pacmansummary_complete()
545
546     if (dircheck!=None) and (dircheck.has_changed()):
547         print "\n-------------------------------------"
548         print "Changes detected in", TEMPEST_ETC_DIR
549         print "Re-running tempest to process changes"
550         print "-------------------------------------\n"
551         # drop mutexes so we don't immediately block
552         arizonageneral.mutex_unlock(storkLock)
553         arizonageneral.mutex_unlock(pacmanLock)
554         # restart
555         os.execv(sys.argv[0], sys.argv);
556
557 if __name__ == "__main__":
558     Main(None)