X-Git-Url: https://git.sur5r.net/?a=blobdiff_plain;f=tools%2Fbuildman%2Fbuilderthread.py;h=c84ba6acf11a64a611ae2e7a8d209391da38ccd3;hb=0ddc510ea37aa578b8cb197840a5125409978bec;hp=bc4541cb3eb90b357e4608aca5ceae583eee1c05;hpb=ce267335c31e95d69d42abf886ce7f3df1b5b2a4;p=u-boot diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py index bc4541cb3e..c84ba6acf1 100644 --- a/tools/buildman/builderthread.py +++ b/tools/buildman/builderthread.py @@ -1,17 +1,19 @@ +# SPDX-License-Identifier: GPL-2.0+ # Copyright (c) 2014 Google, Inc # -# SPDX-License-Identifier: GPL-2.0+ -# import errno import glob import os import shutil +import sys import threading import command import gitutil +RETURN_CODE_RETRY = -1 + def Mkdir(dirname, parents = False): """Make a directory if it doesn't already exist. @@ -25,6 +27,9 @@ def Mkdir(dirname, parents = False): os.mkdir(dirname) except OSError as err: if err.errno == errno.EEXIST: + if os.path.realpath('.') == os.path.realpath(dirname): + print "Cannot create the current working directory '%s'!" % dirname + sys.exit(1) pass else: raise @@ -78,11 +83,13 @@ class BuilderThread(threading.Thread): thread_num: Our thread number (0-n-1), used to decide on a temporary directory """ - def __init__(self, builder, thread_num): + def __init__(self, builder, thread_num, incremental, per_board_out_dir): """Set up a new builder thread""" threading.Thread.__init__(self) self.builder = builder self.thread_num = thread_num + self.incremental = incremental + self.per_board_out_dir = per_board_out_dir def Make(self, commit, brd, stage, cwd, *args, **kwargs): """Run 'make' on a particular commit and board. @@ -106,8 +113,8 @@ class BuilderThread(threading.Thread): return self.builder.do_make(commit, brd, stage, cwd, *args, **kwargs) - def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build, - force_build_failures): + def RunCommit(self, commit_upto, brd, work_dir, do_config, config_only, + force_build, force_build_failures): """Build a particular commit. If the build is already done, and we are not forcing a build, we skip @@ -118,6 +125,7 @@ class BuilderThread(threading.Thread): brd: Board object to build work_dir: Directory to which the source will be checked out do_config: True to run a make _defconfig on the source + config_only: Only configure the source, do not build it force_build: Force a build even if one was previously done force_build_failures: Force a bulid if the previous result showed failure @@ -134,7 +142,11 @@ class BuilderThread(threading.Thread): if self.builder.in_tree: out_dir = work_dir else: - out_dir = os.path.join(work_dir, 'build') + if self.per_board_out_dir: + out_rel_dir = os.path.join('..', brd.target) + else: + out_rel_dir = 'build' + out_dir = os.path.join(work_dir, out_rel_dir) # Check if the job was already completed last time done_file = self.builder.GetDoneFile(commit_upto, brd.target) @@ -145,7 +157,11 @@ class BuilderThread(threading.Thread): # Get the return code from that build and use it with open(done_file, 'r') as fd: result.return_code = int(fd.readline()) - if will_build: + + # Check the signal that the build needs to be retried + if result.return_code == RETURN_CODE_RETRY: + will_build = True + elif will_build: err_file = self.builder.GetErrFile(commit_upto, brd.target) if os.path.exists(err_file) and os.stat(err_file).st_size: result.stderr = 'bad' @@ -177,7 +193,7 @@ class BuilderThread(threading.Thread): commit = 'current' # Set up the environment and command line - env = self.toolchain.MakeEnvironment() + env = self.toolchain.MakeEnvironment(self.builder.full_path) Mkdir(out_dir) args = [] cwd = work_dir @@ -191,31 +207,43 @@ class BuilderThread(threading.Thread): # # Symlinks can confuse U-Boot's Makefile since # we may use '..' in our path, so remove them. - work_dir = os.path.realpath(work_dir) - args.append('O=%s/build' % work_dir) + out_dir = os.path.realpath(out_dir) + args.append('O=%s' % out_dir) cwd = None src_dir = os.getcwd() else: - args.append('O=build') - args.append('-s') + args.append('O=%s' % out_rel_dir) + if self.builder.verbose_build: + args.append('V=1') + else: + args.append('-s') if self.builder.num_jobs is not None: args.extend(['-j', str(self.builder.num_jobs)]) + if self.builder.warnings_as_errors: + args.append('KCFLAGS=-Werror') config_args = ['%s_defconfig' % brd.target] config_out = '' args.extend(self.builder.toolchains.GetMakeArguments(brd)) # If we need to reconfigure, do that now if do_config: - result = self.Make(commit, brd, 'mrproper', cwd, - 'mrproper', *args, env=env) + config_out = '' + if not self.incremental: + result = self.Make(commit, brd, 'mrproper', cwd, + 'mrproper', *args, env=env) + config_out += result.combined result = self.Make(commit, brd, 'config', cwd, *(args + config_args), env=env) - config_out = result.combined + config_out += result.combined do_config = False # No need to configure next time if result.return_code == 0: + if config_only: + args.append('cfg') result = self.Make(commit, brd, 'build', cwd, *args, env=env) result.stderr = result.stderr.replace(src_dir + '/', '') + if self.builder.verbose_build: + result.stdout = config_out + result.stdout else: result.return_code = 1 result.stderr = 'No tool chain for %s\n' % brd.arch @@ -239,9 +267,10 @@ class BuilderThread(threading.Thread): if result.return_code < 0: return - # Aborted? - if result.stderr and 'No child processes' in result.stderr: - return + # If we think this might have been aborted with Ctrl-C, record the + # failure but not that we are 'done' with this board. A retry may fix + # it. + maybe_aborted = result.stderr and 'No child processes' in result.stderr if result.already_done: return @@ -256,13 +285,15 @@ class BuilderThread(threading.Thread): outfile = os.path.join(build_dir, 'log') with open(outfile, 'w') as fd: if result.stdout: - fd.write(result.stdout) + # We don't want unicode characters in log files + fd.write(result.stdout.decode('UTF-8').encode('ASCII', 'replace')) errfile = self.builder.GetErrFile(result.commit_upto, result.brd.target) if result.stderr: with open(errfile, 'w') as fd: - fd.write(result.stderr) + # We don't want unicode characters in log files + fd.write(result.stderr.decode('UTF-8').encode('ASCII', 'replace')) elif os.path.exists(errfile): os.remove(errfile) @@ -271,7 +302,11 @@ class BuilderThread(threading.Thread): done_file = self.builder.GetDoneFile(result.commit_upto, result.brd.target) with open(done_file, 'w') as fd: - fd.write('%s' % result.return_code) + if maybe_aborted: + # Special code to indicate we need to retry + fd.write('%s' % RETURN_CODE_RETRY) + else: + fd.write('%s' % result.return_code) with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: print >>fd, 'gcc', result.toolchain.gcc print >>fd, 'path', result.toolchain.path @@ -279,12 +314,8 @@ class BuilderThread(threading.Thread): print >>fd, 'arch', result.toolchain.arch fd.write('%s' % result.return_code) - with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: - print >>fd, 'gcc', result.toolchain.gcc - print >>fd, 'path', result.toolchain.path - # Write out the image and function size information and an objdump - env = result.toolchain.MakeEnvironment() + env = result.toolchain.MakeEnvironment(self.builder.full_path) lines = [] for fname in ['u-boot', 'spl/u-boot-spl']: cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname] @@ -320,6 +351,16 @@ class BuilderThread(threading.Thread): lines.append(size_result.stdout.splitlines()[1] + ' ' + rodata_size) + # Extract the environment from U-Boot and dump it out + cmd = ['%sobjcopy' % self.toolchain.cross, '-O', 'binary', + '-j', '.rodata.default_environment', + 'env/built-in.o', 'uboot.env'] + command.RunPipe([cmd], capture=True, + capture_stderr=True, cwd=result.out_dir, + raise_on_error=False, env=env) + ubootenv = os.path.join(result.out_dir, 'uboot.env') + self.CopyFiles(result.out_dir, build_dir, '', ['uboot.env']) + # Write out the image sizes file. This is similar to the output # of binutil's 'size' utility, but it omits the header line and # adds an additional hex value at the end of each line for the @@ -330,16 +371,37 @@ class BuilderThread(threading.Thread): with open(sizes, 'w') as fd: print >>fd, '\n'.join(lines) + # Write out the configuration files, with a special case for SPL + for dirname in ['', 'spl', 'tpl']: + self.CopyFiles(result.out_dir, build_dir, dirname, ['u-boot.cfg', + 'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg', '.config', + 'include/autoconf.mk', 'include/generated/autoconf.h']) + # Now write the actual build output if keep_outputs: - patterns = ['u-boot', '*.bin', 'u-boot.dtb', '*.map', '*.img', - 'include/autoconf.mk', 'spl/u-boot-spl', - 'spl/u-boot-spl.bin'] - for pattern in patterns: - file_list = glob.glob(os.path.join(result.out_dir, pattern)) - for fname in file_list: - shutil.copy(fname, build_dir) + self.CopyFiles(result.out_dir, build_dir, '', ['u-boot*', '*.bin', + '*.map', '*.img', 'MLO', 'SPL', 'include/autoconf.mk', + 'spl/u-boot-spl*']) + def CopyFiles(self, out_dir, build_dir, dirname, patterns): + """Copy files from the build directory to the output. + + Args: + out_dir: Path to output directory containing the files + build_dir: Place to copy the files + dirname: Source directory, '' for normal U-Boot, 'spl' for SPL + patterns: A list of filenames (strings) to copy, each relative + to the build directory + """ + for pattern in patterns: + file_list = glob.glob(os.path.join(out_dir, dirname, pattern)) + for fname in file_list: + target = os.path.basename(fname) + if dirname: + base, ext = os.path.splitext(target) + if ext: + target = '%s-%s%s' % (base, dirname, ext) + shutil.copy(fname, os.path.join(build_dir, target)) def RunJob(self, job): """Run a single job @@ -359,7 +421,7 @@ class BuilderThread(threading.Thread): force_build = False for commit_upto in range(0, len(job.commits), job.step): result, request_config = self.RunCommit(commit_upto, brd, - work_dir, do_config, + work_dir, do_config, self.builder.config_only, force_build or self.builder.force_build, self.builder.force_build_failures) failed = result.return_code or result.stderr @@ -369,7 +431,7 @@ class BuilderThread(threading.Thread): # with a reconfig. if self.builder.force_config_on_failure: result, request_config = self.RunCommit(commit_upto, - brd, work_dir, True, True, False) + brd, work_dir, True, False, True, False) did_config = True if not self.builder.force_reconfig: do_config = request_config @@ -413,7 +475,8 @@ class BuilderThread(threading.Thread): else: # Just build the currently checked-out build result, request_config = self.RunCommit(None, brd, work_dir, True, - True, self.builder.force_build_failures) + self.builder.config_only, True, + self.builder.force_build_failures) result.commit_upto = 0 self._WriteResult(result, job.keep_outputs) self.builder.out_queue.put(result) @@ -424,17 +487,7 @@ class BuilderThread(threading.Thread): This thread picks a job from the queue, runs it, and then goes to the next job. """ - alive = True while True: job = self.builder.queue.get() - if self.builder.active and alive: - self.RunJob(job) - ''' - try: - if self.builder.active and alive: - self.RunJob(job) - except Exception as err: - alive = False - print err - ''' + self.RunJob(job) self.builder.queue.task_done()