1 # SPDX-License-Identifier: GPL-2.0+
2 # Copyright (c) 2014 Google, Inc
14 RETURN_CODE_RETRY = -1
16 def Mkdir(dirname, parents = False):
17 """Make a directory if it doesn't already exist.
20 dirname: Directory to create
27 except OSError as err:
28 if err.errno == errno.EEXIST:
34 """Holds information about a job to be performed by a thread
37 board: Board object to build
38 commits: List of commit options to build.
45 class ResultThread(threading.Thread):
46 """This thread processes results from builder threads.
48 It simply passes the results on to the builder. There is only one
49 result thread, and this helps to serialise the build output.
51 def __init__(self, builder):
52 """Set up a new result thread
55 builder: Builder which will be sent each result
57 threading.Thread.__init__(self)
58 self.builder = builder
61 """Called to start up the result thread.
63 We collect the next result job and pass it on to the build.
66 result = self.builder.out_queue.get()
67 self.builder.ProcessResult(result)
68 self.builder.out_queue.task_done()
71 class BuilderThread(threading.Thread):
72 """This thread builds U-Boot for a particular board.
74 An input queue provides each new job. We run 'make' to build U-Boot
75 and then pass the results on to the output queue.
78 builder: The builder which contains information we might need
79 thread_num: Our thread number (0-n-1), used to decide on a
82 def __init__(self, builder, thread_num, incremental, per_board_out_dir):
83 """Set up a new builder thread"""
84 threading.Thread.__init__(self)
85 self.builder = builder
86 self.thread_num = thread_num
87 self.incremental = incremental
88 self.per_board_out_dir = per_board_out_dir
90 def Make(self, commit, brd, stage, cwd, *args, **kwargs):
91 """Run 'make' on a particular commit and board.
93 The source code will already be checked out, so the 'commit'
94 argument is only for information.
97 commit: Commit object that is being built
98 brd: Board object that is being built
99 stage: Stage of the build. Valid stages are:
100 mrproper - can be called to clean source
101 config - called to configure for a board
102 build - the main make invocation - it does the build
103 args: A list of arguments to pass to 'make'
104 kwargs: A list of keyword arguments to pass to command.RunPipe()
109 return self.builder.do_make(commit, brd, stage, cwd, *args,
112 def RunCommit(self, commit_upto, brd, work_dir, do_config, config_only,
113 force_build, force_build_failures):
114 """Build a particular commit.
116 If the build is already done, and we are not forcing a build, we skip
117 the build and just return the previously-saved results.
120 commit_upto: Commit number to build (0...n-1)
121 brd: Board object to build
122 work_dir: Directory to which the source will be checked out
123 do_config: True to run a make <board>_defconfig on the source
124 config_only: Only configure the source, do not build it
125 force_build: Force a build even if one was previously done
126 force_build_failures: Force a bulid if the previous result showed
131 - CommandResult object containing the results of the build
132 - boolean indicating whether 'make config' is still needed
134 # Create a default result - it will be overwritte by the call to
135 # self.Make() below, in the event that we do a build.
136 result = command.CommandResult()
137 result.return_code = 0
138 if self.builder.in_tree:
141 if self.per_board_out_dir:
142 out_rel_dir = os.path.join('..', brd.target)
144 out_rel_dir = 'build'
145 out_dir = os.path.join(work_dir, out_rel_dir)
147 # Check if the job was already completed last time
148 done_file = self.builder.GetDoneFile(commit_upto, brd.target)
149 result.already_done = os.path.exists(done_file)
150 will_build = (force_build or force_build_failures or
151 not result.already_done)
152 if result.already_done:
153 # Get the return code from that build and use it
154 with open(done_file, 'r') as fd:
155 result.return_code = int(fd.readline())
157 # Check the signal that the build needs to be retried
158 if result.return_code == RETURN_CODE_RETRY:
161 err_file = self.builder.GetErrFile(commit_upto, brd.target)
162 if os.path.exists(err_file) and os.stat(err_file).st_size:
163 result.stderr = 'bad'
164 elif not force_build:
165 # The build passed, so no need to build it again
169 # We are going to have to build it. First, get a toolchain
170 if not self.toolchain:
172 self.toolchain = self.builder.toolchains.Select(brd.arch)
173 except ValueError as err:
174 result.return_code = 10
176 result.stderr = str(err)
177 # TODO(sjg@chromium.org): This gets swallowed, but needs
181 # Checkout the right commit
182 if self.builder.commits:
183 commit = self.builder.commits[commit_upto]
184 if self.builder.checkout:
185 git_dir = os.path.join(work_dir, '.git')
186 gitutil.Checkout(commit.hash, git_dir, work_dir,
191 # Set up the environment and command line
192 env = self.toolchain.MakeEnvironment(self.builder.full_path)
196 src_dir = os.path.realpath(work_dir)
197 if not self.builder.in_tree:
198 if commit_upto is None:
199 # In this case we are building in the original source
200 # directory (i.e. the current directory where buildman
201 # is invoked. The output directory is set to this
202 # thread's selected work directory.
204 # Symlinks can confuse U-Boot's Makefile since
205 # we may use '..' in our path, so remove them.
206 out_dir = os.path.realpath(out_dir)
207 args.append('O=%s' % out_dir)
209 src_dir = os.getcwd()
211 args.append('O=%s' % out_rel_dir)
212 if self.builder.verbose_build:
216 if self.builder.num_jobs is not None:
217 args.extend(['-j', str(self.builder.num_jobs)])
218 if self.builder.warnings_as_errors:
219 args.append('KCFLAGS=-Werror')
220 config_args = ['%s_defconfig' % brd.target]
222 args.extend(self.builder.toolchains.GetMakeArguments(brd))
224 # If we need to reconfigure, do that now
227 if not self.incremental:
228 result = self.Make(commit, brd, 'mrproper', cwd,
229 'mrproper', *args, env=env)
230 config_out += result.combined
231 result = self.Make(commit, brd, 'config', cwd,
232 *(args + config_args), env=env)
233 config_out += result.combined
234 do_config = False # No need to configure next time
235 if result.return_code == 0:
238 result = self.Make(commit, brd, 'build', cwd, *args,
240 result.stderr = result.stderr.replace(src_dir + '/', '')
241 if self.builder.verbose_build:
242 result.stdout = config_out + result.stdout
244 result.return_code = 1
245 result.stderr = 'No tool chain for %s\n' % brd.arch
246 result.already_done = False
248 result.toolchain = self.toolchain
250 result.commit_upto = commit_upto
251 result.out_dir = out_dir
252 return result, do_config
254 def _WriteResult(self, result, keep_outputs):
255 """Write a built result to the output directory.
258 result: CommandResult object containing result to write
259 keep_outputs: True to store the output binaries, False
263 if result.return_code < 0:
266 # If we think this might have been aborted with Ctrl-C, record the
267 # failure but not that we are 'done' with this board. A retry may fix
269 maybe_aborted = result.stderr and 'No child processes' in result.stderr
271 if result.already_done:
274 # Write the output and stderr
275 output_dir = self.builder._GetOutputDir(result.commit_upto)
277 build_dir = self.builder.GetBuildDir(result.commit_upto,
281 outfile = os.path.join(build_dir, 'log')
282 with open(outfile, 'w') as fd:
284 # We don't want unicode characters in log files
285 fd.write(result.stdout.decode('UTF-8').encode('ASCII', 'replace'))
287 errfile = self.builder.GetErrFile(result.commit_upto,
290 with open(errfile, 'w') as fd:
291 # We don't want unicode characters in log files
292 fd.write(result.stderr.decode('UTF-8').encode('ASCII', 'replace'))
293 elif os.path.exists(errfile):
297 # Write the build result and toolchain information.
298 done_file = self.builder.GetDoneFile(result.commit_upto,
300 with open(done_file, 'w') as fd:
302 # Special code to indicate we need to retry
303 fd.write('%s' % RETURN_CODE_RETRY)
305 fd.write('%s' % result.return_code)
306 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
307 print >>fd, 'gcc', result.toolchain.gcc
308 print >>fd, 'path', result.toolchain.path
309 print >>fd, 'cross', result.toolchain.cross
310 print >>fd, 'arch', result.toolchain.arch
311 fd.write('%s' % result.return_code)
313 # Write out the image and function size information and an objdump
314 env = result.toolchain.MakeEnvironment(self.builder.full_path)
316 for fname in ['u-boot', 'spl/u-boot-spl']:
317 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
318 nm_result = command.RunPipe([cmd], capture=True,
319 capture_stderr=True, cwd=result.out_dir,
320 raise_on_error=False, env=env)
322 nm = self.builder.GetFuncSizesFile(result.commit_upto,
323 result.brd.target, fname)
324 with open(nm, 'w') as fd:
325 print >>fd, nm_result.stdout,
327 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
328 dump_result = command.RunPipe([cmd], capture=True,
329 capture_stderr=True, cwd=result.out_dir,
330 raise_on_error=False, env=env)
332 if dump_result.stdout:
333 objdump = self.builder.GetObjdumpFile(result.commit_upto,
334 result.brd.target, fname)
335 with open(objdump, 'w') as fd:
336 print >>fd, dump_result.stdout,
337 for line in dump_result.stdout.splitlines():
338 fields = line.split()
339 if len(fields) > 5 and fields[1] == '.rodata':
340 rodata_size = fields[2]
342 cmd = ['%ssize' % self.toolchain.cross, fname]
343 size_result = command.RunPipe([cmd], capture=True,
344 capture_stderr=True, cwd=result.out_dir,
345 raise_on_error=False, env=env)
346 if size_result.stdout:
347 lines.append(size_result.stdout.splitlines()[1] + ' ' +
350 # Write out the image sizes file. This is similar to the output
351 # of binutil's 'size' utility, but it omits the header line and
352 # adds an additional hex value at the end of each line for the
355 sizes = self.builder.GetSizesFile(result.commit_upto,
357 with open(sizes, 'w') as fd:
358 print >>fd, '\n'.join(lines)
360 # Write out the configuration files, with a special case for SPL
361 for dirname in ['', 'spl', 'tpl']:
362 self.CopyFiles(result.out_dir, build_dir, dirname, ['u-boot.cfg',
363 'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg', '.config',
364 'include/autoconf.mk', 'include/generated/autoconf.h'])
366 # Now write the actual build output
368 self.CopyFiles(result.out_dir, build_dir, '', ['u-boot*', '*.bin',
369 '*.map', '*.img', 'MLO', 'SPL', 'include/autoconf.mk',
372 def CopyFiles(self, out_dir, build_dir, dirname, patterns):
373 """Copy files from the build directory to the output.
376 out_dir: Path to output directory containing the files
377 build_dir: Place to copy the files
378 dirname: Source directory, '' for normal U-Boot, 'spl' for SPL
379 patterns: A list of filenames (strings) to copy, each relative
380 to the build directory
382 for pattern in patterns:
383 file_list = glob.glob(os.path.join(out_dir, dirname, pattern))
384 for fname in file_list:
385 target = os.path.basename(fname)
387 base, ext = os.path.splitext(target)
389 target = '%s-%s%s' % (base, dirname, ext)
390 shutil.copy(fname, os.path.join(build_dir, target))
392 def RunJob(self, job):
395 A job consists of a building a list of commits for a particular board.
401 work_dir = self.builder.GetThreadDir(self.thread_num)
402 self.toolchain = None
404 # Run 'make board_defconfig' on the first commit
408 for commit_upto in range(0, len(job.commits), job.step):
409 result, request_config = self.RunCommit(commit_upto, brd,
410 work_dir, do_config, self.builder.config_only,
411 force_build or self.builder.force_build,
412 self.builder.force_build_failures)
413 failed = result.return_code or result.stderr
414 did_config = do_config
415 if failed and not do_config:
416 # If our incremental build failed, try building again
418 if self.builder.force_config_on_failure:
419 result, request_config = self.RunCommit(commit_upto,
420 brd, work_dir, True, False, True, False)
422 if not self.builder.force_reconfig:
423 do_config = request_config
425 # If we built that commit, then config is done. But if we got
426 # an warning, reconfig next time to force it to build the same
427 # files that created warnings this time. Otherwise an
428 # incremental build may not build the same file, and we will
429 # think that the warning has gone away.
430 # We could avoid this by using -Werror everywhere...
431 # For errors, the problem doesn't happen, since presumably
432 # the build stopped and didn't generate output, so will retry
433 # that file next time. So we could detect warnings and deal
434 # with them specially here. For now, we just reconfigure if
435 # anything goes work.
436 # Of course this is substantially slower if there are build
437 # errors/warnings (e.g. 2-3x slower even if only 10% of builds
439 if (failed and not result.already_done and not did_config and
440 self.builder.force_config_on_failure):
441 # If this build failed, try the next one with a
443 # Sometimes if the board_config.h file changes it can mess
444 # with dependencies, and we get:
445 # make: *** No rule to make target `include/autoconf.mk',
446 # needed by `depend'.
451 if self.builder.force_config_on_failure:
454 result.commit_upto = commit_upto
455 if result.return_code < 0:
456 raise ValueError('Interrupt')
458 # We have the build results, so output the result
459 self._WriteResult(result, job.keep_outputs)
460 self.builder.out_queue.put(result)
462 # Just build the currently checked-out build
463 result, request_config = self.RunCommit(None, brd, work_dir, True,
464 self.builder.config_only, True,
465 self.builder.force_build_failures)
466 result.commit_upto = 0
467 self._WriteResult(result, job.keep_outputs)
468 self.builder.out_queue.put(result)
471 """Our thread's run function
473 This thread picks a job from the queue, runs it, and then goes to the
477 job = self.builder.queue.get()
479 self.builder.queue.task_done()