1 # SPDX-License-Identifier: GPL-2.0+
2 # Copyright (c) 2014 Google, Inc
15 RETURN_CODE_RETRY = -1
17 def Mkdir(dirname, parents = False):
18 """Make a directory if it doesn't already exist.
21 dirname: Directory to create
28 except OSError as err:
29 if err.errno == errno.EEXIST:
30 if os.path.realpath('.') == os.path.realpath(dirname):
31 print "Cannot create the current working directory '%s'!" % dirname
38 """Holds information about a job to be performed by a thread
41 board: Board object to build
42 commits: List of commit options to build.
49 class ResultThread(threading.Thread):
50 """This thread processes results from builder threads.
52 It simply passes the results on to the builder. There is only one
53 result thread, and this helps to serialise the build output.
55 def __init__(self, builder):
56 """Set up a new result thread
59 builder: Builder which will be sent each result
61 threading.Thread.__init__(self)
62 self.builder = builder
65 """Called to start up the result thread.
67 We collect the next result job and pass it on to the build.
70 result = self.builder.out_queue.get()
71 self.builder.ProcessResult(result)
72 self.builder.out_queue.task_done()
75 class BuilderThread(threading.Thread):
76 """This thread builds U-Boot for a particular board.
78 An input queue provides each new job. We run 'make' to build U-Boot
79 and then pass the results on to the output queue.
82 builder: The builder which contains information we might need
83 thread_num: Our thread number (0-n-1), used to decide on a
86 def __init__(self, builder, thread_num, incremental, per_board_out_dir):
87 """Set up a new builder thread"""
88 threading.Thread.__init__(self)
89 self.builder = builder
90 self.thread_num = thread_num
91 self.incremental = incremental
92 self.per_board_out_dir = per_board_out_dir
94 def Make(self, commit, brd, stage, cwd, *args, **kwargs):
95 """Run 'make' on a particular commit and board.
97 The source code will already be checked out, so the 'commit'
98 argument is only for information.
101 commit: Commit object that is being built
102 brd: Board object that is being built
103 stage: Stage of the build. Valid stages are:
104 mrproper - can be called to clean source
105 config - called to configure for a board
106 build - the main make invocation - it does the build
107 args: A list of arguments to pass to 'make'
108 kwargs: A list of keyword arguments to pass to command.RunPipe()
113 return self.builder.do_make(commit, brd, stage, cwd, *args,
116 def RunCommit(self, commit_upto, brd, work_dir, do_config, config_only,
117 force_build, force_build_failures):
118 """Build a particular commit.
120 If the build is already done, and we are not forcing a build, we skip
121 the build and just return the previously-saved results.
124 commit_upto: Commit number to build (0...n-1)
125 brd: Board object to build
126 work_dir: Directory to which the source will be checked out
127 do_config: True to run a make <board>_defconfig on the source
128 config_only: Only configure the source, do not build it
129 force_build: Force a build even if one was previously done
130 force_build_failures: Force a bulid if the previous result showed
135 - CommandResult object containing the results of the build
136 - boolean indicating whether 'make config' is still needed
138 # Create a default result - it will be overwritte by the call to
139 # self.Make() below, in the event that we do a build.
140 result = command.CommandResult()
141 result.return_code = 0
142 if self.builder.in_tree:
145 if self.per_board_out_dir:
146 out_rel_dir = os.path.join('..', brd.target)
148 out_rel_dir = 'build'
149 out_dir = os.path.join(work_dir, out_rel_dir)
151 # Check if the job was already completed last time
152 done_file = self.builder.GetDoneFile(commit_upto, brd.target)
153 result.already_done = os.path.exists(done_file)
154 will_build = (force_build or force_build_failures or
155 not result.already_done)
156 if result.already_done:
157 # Get the return code from that build and use it
158 with open(done_file, 'r') as fd:
159 result.return_code = int(fd.readline())
161 # Check the signal that the build needs to be retried
162 if result.return_code == RETURN_CODE_RETRY:
165 err_file = self.builder.GetErrFile(commit_upto, brd.target)
166 if os.path.exists(err_file) and os.stat(err_file).st_size:
167 result.stderr = 'bad'
168 elif not force_build:
169 # The build passed, so no need to build it again
173 # We are going to have to build it. First, get a toolchain
174 if not self.toolchain:
176 self.toolchain = self.builder.toolchains.Select(brd.arch)
177 except ValueError as err:
178 result.return_code = 10
180 result.stderr = str(err)
181 # TODO(sjg@chromium.org): This gets swallowed, but needs
185 # Checkout the right commit
186 if self.builder.commits:
187 commit = self.builder.commits[commit_upto]
188 if self.builder.checkout:
189 git_dir = os.path.join(work_dir, '.git')
190 gitutil.Checkout(commit.hash, git_dir, work_dir,
195 # Set up the environment and command line
196 env = self.toolchain.MakeEnvironment(self.builder.full_path)
200 src_dir = os.path.realpath(work_dir)
201 if not self.builder.in_tree:
202 if commit_upto is None:
203 # In this case we are building in the original source
204 # directory (i.e. the current directory where buildman
205 # is invoked. The output directory is set to this
206 # thread's selected work directory.
208 # Symlinks can confuse U-Boot's Makefile since
209 # we may use '..' in our path, so remove them.
210 out_dir = os.path.realpath(out_dir)
211 args.append('O=%s' % out_dir)
213 src_dir = os.getcwd()
215 args.append('O=%s' % out_rel_dir)
216 if self.builder.verbose_build:
220 if self.builder.num_jobs is not None:
221 args.extend(['-j', str(self.builder.num_jobs)])
222 if self.builder.warnings_as_errors:
223 args.append('KCFLAGS=-Werror')
224 config_args = ['%s_defconfig' % brd.target]
226 args.extend(self.builder.toolchains.GetMakeArguments(brd))
228 # If we need to reconfigure, do that now
231 if not self.incremental:
232 result = self.Make(commit, brd, 'mrproper', cwd,
233 'mrproper', *args, env=env)
234 config_out += result.combined
235 result = self.Make(commit, brd, 'config', cwd,
236 *(args + config_args), env=env)
237 config_out += result.combined
238 do_config = False # No need to configure next time
239 if result.return_code == 0:
242 result = self.Make(commit, brd, 'build', cwd, *args,
244 result.stderr = result.stderr.replace(src_dir + '/', '')
245 if self.builder.verbose_build:
246 result.stdout = config_out + result.stdout
248 result.return_code = 1
249 result.stderr = 'No tool chain for %s\n' % brd.arch
250 result.already_done = False
252 result.toolchain = self.toolchain
254 result.commit_upto = commit_upto
255 result.out_dir = out_dir
256 return result, do_config
258 def _WriteResult(self, result, keep_outputs):
259 """Write a built result to the output directory.
262 result: CommandResult object containing result to write
263 keep_outputs: True to store the output binaries, False
267 if result.return_code < 0:
270 # If we think this might have been aborted with Ctrl-C, record the
271 # failure but not that we are 'done' with this board. A retry may fix
273 maybe_aborted = result.stderr and 'No child processes' in result.stderr
275 if result.already_done:
278 # Write the output and stderr
279 output_dir = self.builder._GetOutputDir(result.commit_upto)
281 build_dir = self.builder.GetBuildDir(result.commit_upto,
285 outfile = os.path.join(build_dir, 'log')
286 with open(outfile, 'w') as fd:
288 # We don't want unicode characters in log files
289 fd.write(result.stdout.decode('UTF-8').encode('ASCII', 'replace'))
291 errfile = self.builder.GetErrFile(result.commit_upto,
294 with open(errfile, 'w') as fd:
295 # We don't want unicode characters in log files
296 fd.write(result.stderr.decode('UTF-8').encode('ASCII', 'replace'))
297 elif os.path.exists(errfile):
301 # Write the build result and toolchain information.
302 done_file = self.builder.GetDoneFile(result.commit_upto,
304 with open(done_file, 'w') as fd:
306 # Special code to indicate we need to retry
307 fd.write('%s' % RETURN_CODE_RETRY)
309 fd.write('%s' % result.return_code)
310 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
311 print >>fd, 'gcc', result.toolchain.gcc
312 print >>fd, 'path', result.toolchain.path
313 print >>fd, 'cross', result.toolchain.cross
314 print >>fd, 'arch', result.toolchain.arch
315 fd.write('%s' % result.return_code)
317 # Write out the image and function size information and an objdump
318 env = result.toolchain.MakeEnvironment(self.builder.full_path)
320 for fname in ['u-boot', 'spl/u-boot-spl']:
321 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
322 nm_result = command.RunPipe([cmd], capture=True,
323 capture_stderr=True, cwd=result.out_dir,
324 raise_on_error=False, env=env)
326 nm = self.builder.GetFuncSizesFile(result.commit_upto,
327 result.brd.target, fname)
328 with open(nm, 'w') as fd:
329 print >>fd, nm_result.stdout,
331 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
332 dump_result = command.RunPipe([cmd], capture=True,
333 capture_stderr=True, cwd=result.out_dir,
334 raise_on_error=False, env=env)
336 if dump_result.stdout:
337 objdump = self.builder.GetObjdumpFile(result.commit_upto,
338 result.brd.target, fname)
339 with open(objdump, 'w') as fd:
340 print >>fd, dump_result.stdout,
341 for line in dump_result.stdout.splitlines():
342 fields = line.split()
343 if len(fields) > 5 and fields[1] == '.rodata':
344 rodata_size = fields[2]
346 cmd = ['%ssize' % self.toolchain.cross, fname]
347 size_result = command.RunPipe([cmd], capture=True,
348 capture_stderr=True, cwd=result.out_dir,
349 raise_on_error=False, env=env)
350 if size_result.stdout:
351 lines.append(size_result.stdout.splitlines()[1] + ' ' +
354 # Write out the image sizes file. This is similar to the output
355 # of binutil's 'size' utility, but it omits the header line and
356 # adds an additional hex value at the end of each line for the
359 sizes = self.builder.GetSizesFile(result.commit_upto,
361 with open(sizes, 'w') as fd:
362 print >>fd, '\n'.join(lines)
364 # Write out the configuration files, with a special case for SPL
365 for dirname in ['', 'spl', 'tpl']:
366 self.CopyFiles(result.out_dir, build_dir, dirname, ['u-boot.cfg',
367 'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg', '.config',
368 'include/autoconf.mk', 'include/generated/autoconf.h'])
370 # Now write the actual build output
372 self.CopyFiles(result.out_dir, build_dir, '', ['u-boot*', '*.bin',
373 '*.map', '*.img', 'MLO', 'SPL', 'include/autoconf.mk',
376 def CopyFiles(self, out_dir, build_dir, dirname, patterns):
377 """Copy files from the build directory to the output.
380 out_dir: Path to output directory containing the files
381 build_dir: Place to copy the files
382 dirname: Source directory, '' for normal U-Boot, 'spl' for SPL
383 patterns: A list of filenames (strings) to copy, each relative
384 to the build directory
386 for pattern in patterns:
387 file_list = glob.glob(os.path.join(out_dir, dirname, pattern))
388 for fname in file_list:
389 target = os.path.basename(fname)
391 base, ext = os.path.splitext(target)
393 target = '%s-%s%s' % (base, dirname, ext)
394 shutil.copy(fname, os.path.join(build_dir, target))
396 def RunJob(self, job):
399 A job consists of a building a list of commits for a particular board.
405 work_dir = self.builder.GetThreadDir(self.thread_num)
406 self.toolchain = None
408 # Run 'make board_defconfig' on the first commit
412 for commit_upto in range(0, len(job.commits), job.step):
413 result, request_config = self.RunCommit(commit_upto, brd,
414 work_dir, do_config, self.builder.config_only,
415 force_build or self.builder.force_build,
416 self.builder.force_build_failures)
417 failed = result.return_code or result.stderr
418 did_config = do_config
419 if failed and not do_config:
420 # If our incremental build failed, try building again
422 if self.builder.force_config_on_failure:
423 result, request_config = self.RunCommit(commit_upto,
424 brd, work_dir, True, False, True, False)
426 if not self.builder.force_reconfig:
427 do_config = request_config
429 # If we built that commit, then config is done. But if we got
430 # an warning, reconfig next time to force it to build the same
431 # files that created warnings this time. Otherwise an
432 # incremental build may not build the same file, and we will
433 # think that the warning has gone away.
434 # We could avoid this by using -Werror everywhere...
435 # For errors, the problem doesn't happen, since presumably
436 # the build stopped and didn't generate output, so will retry
437 # that file next time. So we could detect warnings and deal
438 # with them specially here. For now, we just reconfigure if
439 # anything goes work.
440 # Of course this is substantially slower if there are build
441 # errors/warnings (e.g. 2-3x slower even if only 10% of builds
443 if (failed and not result.already_done and not did_config and
444 self.builder.force_config_on_failure):
445 # If this build failed, try the next one with a
447 # Sometimes if the board_config.h file changes it can mess
448 # with dependencies, and we get:
449 # make: *** No rule to make target `include/autoconf.mk',
450 # needed by `depend'.
455 if self.builder.force_config_on_failure:
458 result.commit_upto = commit_upto
459 if result.return_code < 0:
460 raise ValueError('Interrupt')
462 # We have the build results, so output the result
463 self._WriteResult(result, job.keep_outputs)
464 self.builder.out_queue.put(result)
466 # Just build the currently checked-out build
467 result, request_config = self.RunCommit(None, brd, work_dir, True,
468 self.builder.config_only, True,
469 self.builder.force_build_failures)
470 result.commit_upto = 0
471 self._WriteResult(result, job.keep_outputs)
472 self.builder.out_queue.put(result)
475 """Our thread's run function
477 This thread picks a job from the queue, runs it, and then goes to the
481 job = self.builder.queue.get()
483 self.builder.queue.task_done()