import repository from arizona
[raven.git] / apps / ravenpublish / builder.py
1 import fnmatch
2 import hashlib
3 import os
4 import shutil
5 import subprocess
6 import stat
7 import sys
8 import tempfile
9
10 from ravenlib.ravenlog import RavenLog
11
12 def shafile(fn):
13     return hashlib.sha1(open(fn, "r").read()).hexdigest()
14
15 def replace_symlink(src, dest):
16    """ create a symlink. if the symlink already exists, remove it first. """
17    if (os.path.lexists(dest)):
18        os.remove(dest)
19    os.symlink(src, dest)
20
21 def find(dir, pattern):
22     """ Find a wildcard pattern in a directory. Return the pathname of the file
23         or None if no file exists.
24     """
25     for filename in os.listdir(dir):
26         pathname = os.path.join(dir, filename)
27         if fnmatch.fnmatch(os.path.basename(pathname), pattern):
28             return pathname
29         if os.path.isdir(pathname):
30             result = find(pathname, pattern)
31             if result:
32                 return result
33     return None
34
35 def is_empty(dir):
36     """ return True if a directory has no files other than those that start
37         with .
38     """
39     for filename in os.listdir(dir):
40         if not filename.startswith("."):
41              return False
42     return True
43
44 class specfile(RavenLog):
45     def __init__(self):
46         # for Print() ...
47         RavenLog.__init__(self)
48         self.reset()
49
50     def reset(self):
51         self.summary = "built by raven packagebuilder"
52         self.group = "built by raven packagebuilder"
53         self.license = "built by raven packagebuilder"
54         self.description = "built by raven packagebuilder"
55         self.arch = "noarch"
56         self.contents = ""
57         self.packageRoot = ""
58         self.name = "package"
59         self.hash = ""
60         self.lastRpmFilename = ""
61         self.lastRpmHash = ""
62         self.version = "0.0"
63         self.release = "0"
64         self.owneruidstr = "root"
65         self.ownergidstr = "root"
66         self.files = []
67         self.dirs = []
68         self.buildroot = "%{_tmppath}/%{name}-root"
69         self.postinstall = []
70         self.requires = []
71         self.provides = []
72         self.verboseRpmOutput = False
73
74     def meta_filename(self, kind):
75        return os.path.join(self.packageRoot, kind)
76
77     def add_dir(self, srcname, destname, empty=False):
78         dict = {"type": "dir", "destname": destname, "empty": empty}
79         if srcname:
80             if is_empty(srcname):
81                 dict["empty"] = True
82         self.dirs.append(dict)
83
84     def add_file(self, srcname, destname, mode=0755):
85         dict = {"type": "file", "destname": destname, "srcname": srcname, "mode": mode}
86         self.files.append(dict)
87
88     def add_postinstall(self, srcname, destname):
89         dict = {"type": "postinstall", "destname": destname, "srcname": srcname}
90         self.postinstall.append(dict)
91
92     def add_provides(self, list):
93         self.provides.extend(list)
94
95     def add_requires(self, list):
96         self.requires.extend(list)
97
98     def add_contents(self, line):
99         self.contents = self.contents + line + "\n"
100
101     def generate(self):
102         self.contents = ""
103
104         self.add_contents("Name: " + self.name)
105         self.add_contents("Version: " + self.version)
106         self.add_contents("Release: " + self.release)
107         self.add_contents("Summary: " + self.summary)
108         self.add_contents("Group: " + self.group)
109         self.add_contents("License: " + self.license)
110         self.add_contents("BuildRoot: " + self.buildroot)
111
112         if self.provides:
113             self.add_contents("Provides: " + " ".join(self.provides))
114
115         if self.requires:
116             self.add_contents("Requires: " + " ".join(self.requires))
117
118         self.add_contents("%description")
119         self.add_contents(self.description)
120
121         # XXX do we need %defattr ??
122
123         self.add_contents("%install")
124         for dir in self.dirs:
125             self.add_contents("install -d %{buildroot}" + dir["destname"])
126             #self.add_contents("install -d -o " + self.owneruidstr + " -g " + self.ownergidstr + " %{buildroot}" + dir["destname"])
127
128         for file in self.files:
129             self.add_contents("install " + file["srcname"] + " %{buildroot}" + file["destname"])
130
131         self.add_contents("%post")
132
133         # similar variables were supposed to have been created by rpmbuild,
134         # but they don't seem to be there.
135         self.add_contents("RPMBUILDER_PACKAGE_VERSION=" + self.version)
136         self.add_contents("RPMBUILDER_PACKAGE_RELEASE=" + self.release)
137
138         # This assures that any empty directories are created. Unfortunately,
139         # it also litters them behind after an uninstall.
140         #
141         # As an alternative, we could list the dirs in the %files section, but
142         # that has the negative property of changing the attributes of any
143         # directories, such as /tmp.
144
145         for dir in self.dirs:
146             if dir["empty"]:
147                 self.add_contents("mkdir -p " + dir["destname"])
148
149         for script in self.postinstall:
150             # suck the contents of the script into the postinstall section
151
152             self.add_contents("cd " + os.path.dirname(script["destname"]))
153             for line in open(script["srcname"], "r").readlines():
154                 self.add_contents(line.strip())
155
156             # XXX -- if I had instead wanted to run the script
157             #self.add_contents("chmod +x " + script["destname"])
158             #self.add_contents(script["destname"])
159
160         self.add_contents("%files")
161
162         for file in self.files:
163             self.add_contents("%%attr(0%o,%s,%s) %s" % (file["mode"], self.owneruidstr, self.ownergidstr, file["destname"]) )
164
165         return self.contents
166
167     def update_version(self, force_version = None):
168         """ if force_version != None, use that as the version number
169             otherwise, check the hash and increment as necessary
170             set self.modified to True if the package has been modified since last build
171         """
172         self.modified = False
173
174         if force_version:
175             self.modified = True
176             self.version = force_version
177         else:
178             # check the hash and increment version number if necessary
179             if (self.compute_hash() != self.hash):
180                 self.modified = True
181                 self.increment_version()
182                 self.Print("    version incremented to", self.version)
183
184     def check_current(self, destdir):
185         """ return True if the RPM already exists and is current """
186         if (self.lastRpmFilename and
187               os.path.isfile(self.lastRpmFilename) and
188               (os.path.abspath(os.path.dirname(self.lastRpmFilename)) == os.path.abspath(destdir)) and
189               (shafile(self.lastRpmFilename)==self.lastRpmHash) and
190               (not self.modified)):
191             self.builtRpmFilename = self.lastRpmFilename
192             return True
193         else:
194             return False
195
196     def save(self, specFileName=None):
197         if specFileName == None:
198             specFileName = os.path.join(self.packageRoot, ".spec")
199         file(specFileName, "w").write(self.generate())
200         return specFileName
201
202     def load_meta(self):
203        """ load the .version and .hash files """
204        v_filename = self.meta_filename(".version")
205        if os.path.exists(v_filename):
206            self.version = file(v_filename, "r").readline()
207
208        h_filename = self.meta_filename(".hash")
209        if os.path.exists(h_filename):
210            f = file(h_filename, "r")
211            self.hash = f.readline().strip()
212            self.lastRpmFilename = f.readline().strip()
213            self.lastRpmHash = f.readline().strip()
214
215     def save_meta(self, createDestSymlink=False):
216        """ save the .version and .hash files """
217        absRpmFileName = os.path.abspath(self.builtRpmFilename)
218        file(self.meta_filename(".hash"), "w").write(self.compute_hash() + "\n" + self.builtRpmFilename + "\n" + shafile(self.builtRpmFilename))
219        file(self.meta_filename(".version"), "w").write(self.version)
220        replace_symlink(absRpmFileName, self.meta_filename(".symlink"))
221        if createDestSymlink:
222            replace_symlink(os.path.basename(absRpmFileName), os.path.join(os.path.dirname(absRpmFileName), self.name + ".rpm"))
223
224
225     def buildrpm(self, destdir):
226         builddir = None
227         rcname = None
228         macrosname = None
229         result = None
230
231         # remove old packages
232         # this is necessary because we use find() to locate the rpm that was built
233         os.system("rm -f " + os.path.join(destdir, self.name + "-*.rpm"))
234
235         try:
236             builddir = tempfile.mkdtemp()
237             (rcfile, rcname) = tempfile.mkstemp()
238             (macrosfile, macrosname) = tempfile.mkstemp()
239
240             self.buildroot = os.path.join(builddir, "buildroot")
241
242             # create a suitable build environment for the package
243             os.mkdir(self.buildroot)
244             os.mkdir(os.path.join(builddir, "BUILD"))
245             os.mkdir(os.path.join(builddir, "RPMS"))
246
247             macros = "/usr/lib/rpm/macros:" +\
248                      "/usr/lib/rpm/%{_target}/macros:" +\
249                      "/etc/rpm/macros.*:" +\
250                      "/etc/rpm/macros:" +\
251                      "/etc/rpm/%{_target}/macros:" +\
252                      "~/.rpmmacros:" +\
253                      macrosname
254
255             # change the topdir to something that is user-writable, so that
256             # we don't need root access to build the rpm
257             os.write(macrosfile, "%_topdir " + builddir + "\n")
258             os.write(macrosfile, "%__os_install_post %{nil}\n")
259             os.write(rcfile, "macrofiles: " + macros)
260
261             os.close(rcfile)
262             os.close(macrosfile)
263
264             specFileName = self.save()
265
266             # rpmbuild version 4.4.2.2 (fedora 8) pays attention to --rcname,
267             # but ignores --macros
268
269             # rpmbuild version 4.7.2 (debian 503) ignores --rcname, but pays
270             # attention to --macros.
271
272             # thus, we supply both and hope for the best...
273
274             args = ["rpmbuild", "--rcfile", rcname, "--macros", macros, "-bb", specFileName]
275             if self.arch:
276                 args.extend(["--target", "noarch"])
277             sub = subprocess.Popen(args,
278                                    stderr = subprocess.STDOUT, stdout = subprocess.PIPE)
279             sub.wait()
280             if (sub.returncode != 0):
281                 self.error("    rpmbuild failed with error " + str(sub.returncode))
282                 self.error(sub.stdout.read())
283                 sys.exit(-1)
284
285             if (self.verboseRpmOutput):
286                 self.Print("---- output of rpmbuild ----")
287                 self.Print(sub.stdout.read())
288                 self.Print("---- end output of rpmbuild ----")
289
290             # XXX what we had before subprocess.Popen....
291             # os.system("rpmbuild --rcfile " + rcname + " -bb " + specFileName)
292
293             mask = self.name + "*.rpm"
294             rpm_pathname = find(builddir, mask)
295             if rpm_pathname:
296                 self.builtRpmFilename = os.path.join(destdir, os.path.basename(rpm_pathname))
297                 shutil.move(rpm_pathname, self.builtRpmFilename)
298                 result = self.builtRpmFilename
299             else:
300                 self.error("   failed to find built rpm " + mask + " in " + builddir)
301
302         finally:
303             # cleanup temp files and directories
304             if rcname and os.path.exists(rcname):
305                 os.remove(rcname)
306             if macrosname and os.path.exists(macrosname):
307                 os.remove(macrosname)
308             if builddir and os.path.exists(builddir):
309                 shutil.rmtree(builddir)
310
311         return result
312
313     def compute_hash(self):
314         s = hashlib.sha1()
315         s.update(str(self.name))
316         s.update(str(self.arch))
317         s.update(str(self.files))
318         s.update(str(self.dirs))
319         s.update(str(self.postinstall))
320         s.update(str(self.provides))
321         s.update(str(self.requires))
322         for file in self.files:
323             s.update(open(file["srcname"], "r").read())
324         for script in self.postinstall:
325             s.update(open(script["srcname"], "r").read())
326         return s.hexdigest()
327
328     def increment_version(self):
329         parts = self.version.split(".")
330         try:
331             i = int(parts[-1])
332         except ValueError:
333             i = 0
334
335         self.version = ".".join(parts[:-1] + [str(i+1)])
336
337 class Builder(RavenLog):
338     def __init__(self):
339         # for Print()...
340         RavenLog.__init__(self)
341
342     def check_rpmbuild(self, exitOnError=True):
343        """ Check to see that rpmbuild is installed. Exit if unavailable. """
344        result = os.system("which rpmbuild > /dev/null")
345
346        if (result != 0):
347            self.error("Could not find 'rpmbuild' in the system path. Please ensure")
348            self.error("that the rpm-build package is installed on your computer in")
349            self.error("order to use the raven rpm builder.")
350            self.error()
351            self.error("rpm-build can be installed on most fedora/redhat machines by")
352            self.error("typing \"yum install rpm-build\"")
353
354            if exitOnError:
355                sys.exit(-1)
356
357     def read_provides(self,filename):
358        """ Read the contents of a .provides or a .requires file. It's assumed that
359            multiple names can be specified on a single line by separating them with
360            commas, or on multple lines by separating them with newlines, or some
361            combination of the two
362        """
363        lines = open(filename).readlines()
364        provides = []
365        for line in lines:
366            line_provides = line.strip().split(",")
367            for item in line_provides:
368                item = item.strip()
369                if item:
370                    provides.append(item)
371        return provides
372
373     def build_root(self, dir, destdir, verbose=False):
374        """ Entrypoint for raven's built-in rpmbuilder. It assumes 'dir' to be
375            a directory containing one or more subdirectories, each subdirectory
376            containing a package to be built. 'destdir' is the destination director
377            to place the created RPM files.
378        """
379
380        if not os.path.exists(dir):
381            return
382
383        for filename in os.listdir(dir):
384            if filename.startswith("."):
385                # skip .svn, .cvs, etc
386                continue
387
388            if filename == "__history":
389                # skip Borland's history files, from Scott's development machine
390                continue
391
392            pathname = os.path.join(dir, filename)
393            if os.path.isdir(pathname):
394                packageName = os.path.basename(pathname)
395                self.build_package(packageName, pathname, destdir, verbose=verbose)
396
397     def build_package(self, packageName, packageRoot, destdir, version=None, verbose=False, createDestSymlink=False):
398        """ Build a single package from the directory 'packageRoot', and storing
399            the RPM file in the directory 'destdir'.
400        """
401
402        self.check_rpmbuild()
403
404        packageRoot = os.path.abspath(packageRoot)
405
406        while packageRoot.endswith("/"):
407            packageRoot = packageRoot[:-1]
408
409        self.Print("  building:", packageName)
410
411        modified = False
412
413        spec = specfile()
414        spec.cloneStdoutConfig(self)
415        spec.name = packageName
416        spec.packageRoot = packageRoot
417        spec.verboseRpmOutput = verbose
418
419        # read the .version, .hash, etc
420        spec.load_meta()
421
422        self.build_spec_dir(spec, packageRoot)
423
424        # if version==None, it'll automatically check the hash to see if version
425        # should be updated.
426        spec.update_version(version)
427
428        if (spec.check_current(destdir)):
429            self.Print("    already current: ", spec.lastRpmFilename)
430            return
431
432        result = spec.buildrpm(destdir)
433
434        if result:
435            spec.save_meta(createDestSymlink)
436            self.Print("    built: ", result)
437        else:
438            self.Print("    failed")
439
440     def build_spec_dir(self, spec, dir):
441        """ Build a spec file from a directory, recursing as necessary """
442
443        for filename in os.listdir(dir):
444            pathname = os.path.join(dir, filename)
445            assert(pathname.startswith(spec.packageRoot))
446            destname = pathname[len(spec.packageRoot):]
447
448            # don't put svn directories into the rpm
449            if (filename == ".svn") or (filename == ".cvs"):
450                continue
451
452            if filename == "__history":
453                # skip Borland's history files, from Scott's development machine
454                continue
455
456            if filename == ".spec":
457                continue
458            elif filename == ".symlink":
459                continue
460            elif filename == ".version":
461                continue
462            elif filename == ".hash":
463                continue
464            elif filename == ".provides":
465                spec.add_provides( self.read_provides(pathname) )
466                continue
467            elif filename == ".requires":
468                spec.add_requires( self.read_provides(pathname) )
469                continue
470            elif filename == "autorun.sh":
471                spec.add_postinstall(pathname, destname)
472                continue
473
474            if os.path.isdir(pathname):
475                spec.add_dir(pathname, destname)
476                self.build_spec_dir(spec, pathname)
477            else:
478                mode = os.stat(pathname)[stat.ST_MODE] & 0777
479                spec.add_file(pathname, destname, mode)
480