git.sur5r.net Git - u-boot/blob - tools/buildman/builderthread.py

   1 # Copyright (c) 2014 Google, Inc
   2 #
   3 # SPDX-License-Identifier:      GPL-2.0+
   4 #
   5
   6 import errno
   7 import glob
   8 import os
   9 import shutil
  10 import threading
  11
  12 import command
  13 import gitutil
  14
  15 RETURN_CODE_RETRY = -1
  16
  17 def Mkdir(dirname, parents = False):
  18     """Make a directory if it doesn't already exist.
  19
  20     Args:
  21         dirname: Directory to create
  22     """
  23     try:
  24         if parents:
  25             os.makedirs(dirname)
  26         else:
  27             os.mkdir(dirname)
  28     except OSError as err:
  29         if err.errno == errno.EEXIST:
  30             pass
  31         else:
  32             raise
  33
  34 class BuilderJob:
  35     """Holds information about a job to be performed by a thread
  36
  37     Members:
  38         board: Board object to build
  39         commits: List of commit options to build.
  40     """
  41     def __init__(self):
  42         self.board = None
  43         self.commits = []
  44
  45
  46 class ResultThread(threading.Thread):
  47     """This thread processes results from builder threads.
  48
  49     It simply passes the results on to the builder. There is only one
  50     result thread, and this helps to serialise the build output.
  51     """
  52     def __init__(self, builder):
  53         """Set up a new result thread
  54
  55         Args:
  56             builder: Builder which will be sent each result
  57         """
  58         threading.Thread.__init__(self)
  59         self.builder = builder
  60
  61     def run(self):
  62         """Called to start up the result thread.
  63
  64         We collect the next result job and pass it on to the build.
  65         """
  66         while True:
  67             result = self.builder.out_queue.get()
  68             self.builder.ProcessResult(result)
  69             self.builder.out_queue.task_done()
  70
  71
  72 class BuilderThread(threading.Thread):
  73     """This thread builds U-Boot for a particular board.
  74
  75     An input queue provides each new job. We run 'make' to build U-Boot
  76     and then pass the results on to the output queue.
  77
  78     Members:
  79         builder: The builder which contains information we might need
  80         thread_num: Our thread number (0-n-1), used to decide on a
  81                 temporary directory
  82     """
  83     def __init__(self, builder, thread_num, incremental, per_board_out_dir):
  84         """Set up a new builder thread"""
  85         threading.Thread.__init__(self)
  86         self.builder = builder
  87         self.thread_num = thread_num
  88         self.incremental = incremental
  89         self.per_board_out_dir = per_board_out_dir
  90
  91     def Make(self, commit, brd, stage, cwd, *args, **kwargs):
  92         """Run 'make' on a particular commit and board.
  93
  94         The source code will already be checked out, so the 'commit'
  95         argument is only for information.
  96
  97         Args:
  98             commit: Commit object that is being built
  99             brd: Board object that is being built
 100             stage: Stage of the build. Valid stages are:
 101                         mrproper - can be called to clean source
 102                         config - called to configure for a board
 103                         build - the main make invocation - it does the build
 104             args: A list of arguments to pass to 'make'
 105             kwargs: A list of keyword arguments to pass to command.RunPipe()
 106
 107         Returns:
 108             CommandResult object
 109         """
 110         return self.builder.do_make(commit, brd, stage, cwd, *args,
 111                 **kwargs)
 112
 113     def RunCommit(self, commit_upto, brd, work_dir, do_config, config_only,
 114                   force_build, force_build_failures):
 115         """Build a particular commit.
 116
 117         If the build is already done, and we are not forcing a build, we skip
 118         the build and just return the previously-saved results.
 119
 120         Args:
 121             commit_upto: Commit number to build (0...n-1)
 122             brd: Board object to build
 123             work_dir: Directory to which the source will be checked out
 124             do_config: True to run a make <board>_defconfig on the source
 125             config_only: Only configure the source, do not build it
 126             force_build: Force a build even if one was previously done
 127             force_build_failures: Force a bulid if the previous result showed
 128                 failure
 129
 130         Returns:
 131             tuple containing:
 132                 - CommandResult object containing the results of the build
 133                 - boolean indicating whether 'make config' is still needed
 134         """
 135         # Create a default result - it will be overwritte by the call to
 136         # self.Make() below, in the event that we do a build.
 137         result = command.CommandResult()
 138         result.return_code = 0
 139         if self.builder.in_tree:
 140             out_dir = work_dir
 141         else:
 142             if self.per_board_out_dir:
 143                 out_rel_dir = os.path.join('..', brd.target)
 144             else:
 145                 out_rel_dir = 'build'
 146             out_dir = os.path.join(work_dir, out_rel_dir)
 147
 148         # Check if the job was already completed last time
 149         done_file = self.builder.GetDoneFile(commit_upto, brd.target)
 150         result.already_done = os.path.exists(done_file)
 151         will_build = (force_build or force_build_failures or
 152             not result.already_done)
 153         if result.already_done:
 154             # Get the return code from that build and use it
 155             with open(done_file, 'r') as fd:
 156                 result.return_code = int(fd.readline())
 157
 158             # Check the signal that the build needs to be retried
 159             if result.return_code == RETURN_CODE_RETRY:
 160                 will_build = True
 161             elif will_build:
 162                 err_file = self.builder.GetErrFile(commit_upto, brd.target)
 163                 if os.path.exists(err_file) and os.stat(err_file).st_size:
 164                     result.stderr = 'bad'
 165                 elif not force_build:
 166                     # The build passed, so no need to build it again
 167                     will_build = False
 168
 169         if will_build:
 170             # We are going to have to build it. First, get a toolchain
 171             if not self.toolchain:
 172                 try:
 173                     self.toolchain = self.builder.toolchains.Select(brd.arch)
 174                 except ValueError as err:
 175                     result.return_code = 10
 176                     result.stdout = ''
 177                     result.stderr = str(err)
 178                     # TODO(sjg@chromium.org): This gets swallowed, but needs
 179                     # to be reported.
 180
 181             if self.toolchain:
 182                 # Checkout the right commit
 183                 if self.builder.commits:
 184                     commit = self.builder.commits[commit_upto]
 185                     if self.builder.checkout:
 186                         git_dir = os.path.join(work_dir, '.git')
 187                         gitutil.Checkout(commit.hash, git_dir, work_dir,
 188                                          force=True)
 189                 else:
 190                     commit = 'current'
 191
 192                 # Set up the environment and command line
 193                 env = self.toolchain.MakeEnvironment(self.builder.full_path)
 194                 Mkdir(out_dir)
 195                 args = []
 196                 cwd = work_dir
 197                 src_dir = os.path.realpath(work_dir)
 198                 if not self.builder.in_tree:
 199                     if commit_upto is None:
 200                         # In this case we are building in the original source
 201                         # directory (i.e. the current directory where buildman
 202                         # is invoked. The output directory is set to this
 203                         # thread's selected work directory.
 204                         #
 205                         # Symlinks can confuse U-Boot's Makefile since
 206                         # we may use '..' in our path, so remove them.
 207                         out_dir = os.path.realpath(out_dir)
 208                         args.append('O=%s' % out_dir)
 209                         cwd = None
 210                         src_dir = os.getcwd()
 211                     else:
 212                         args.append('O=%s' % out_rel_dir)
 213                 if self.builder.verbose_build:
 214                     args.append('V=1')
 215                 else:
 216                     args.append('-s')
 217                 if self.builder.num_jobs is not None:
 218                     args.extend(['-j', str(self.builder.num_jobs)])
 219                 if self.builder.warnings_as_errors:
 220                     args.append('KCFLAGS=-Werror')
 221                 config_args = ['%s_defconfig' % brd.target]
 222                 config_out = ''
 223                 args.extend(self.builder.toolchains.GetMakeArguments(brd))
 224
 225                 # If we need to reconfigure, do that now
 226                 if do_config:
 227                     config_out = ''
 228                     if not self.incremental:
 229                         result = self.Make(commit, brd, 'mrproper', cwd,
 230                                 'mrproper', *args, env=env)
 231                         config_out += result.combined
 232                     result = self.Make(commit, brd, 'config', cwd,
 233                             *(args + config_args), env=env)
 234                     config_out += result.combined
 235                     do_config = False   # No need to configure next time
 236                 if result.return_code == 0:
 237                     if config_only:
 238                         args.append('cfg')
 239                     result = self.Make(commit, brd, 'build', cwd, *args,
 240                             env=env)
 241                 result.stderr = result.stderr.replace(src_dir + '/', '')
 242                 if self.builder.verbose_build:
 243                     result.stdout = config_out + result.stdout
 244             else:
 245                 result.return_code = 1
 246                 result.stderr = 'No tool chain for %s\n' % brd.arch
 247             result.already_done = False
 248
 249         result.toolchain = self.toolchain
 250         result.brd = brd
 251         result.commit_upto = commit_upto
 252         result.out_dir = out_dir
 253         return result, do_config
 254
 255     def _WriteResult(self, result, keep_outputs):
 256         """Write a built result to the output directory.
 257
 258         Args:
 259             result: CommandResult object containing result to write
 260             keep_outputs: True to store the output binaries, False
 261                 to delete them
 262         """
 263         # Fatal error
 264         if result.return_code < 0:
 265             return
 266
 267         # If we think this might have been aborted with Ctrl-C, record the
 268         # failure but not that we are 'done' with this board. A retry may fix
 269         # it.
 270         maybe_aborted =  result.stderr and 'No child processes' in result.stderr
 271
 272         if result.already_done:
 273             return
 274
 275         # Write the output and stderr
 276         output_dir = self.builder._GetOutputDir(result.commit_upto)
 277         Mkdir(output_dir)
 278         build_dir = self.builder.GetBuildDir(result.commit_upto,
 279                 result.brd.target)
 280         Mkdir(build_dir)
 281
 282         outfile = os.path.join(build_dir, 'log')
 283         with open(outfile, 'w') as fd:
 284             if result.stdout:
 285                 # We don't want unicode characters in log files
 286                 fd.write(result.stdout.decode('UTF-8').encode('ASCII', 'replace'))
 287
 288         errfile = self.builder.GetErrFile(result.commit_upto,
 289                 result.brd.target)
 290         if result.stderr:
 291             with open(errfile, 'w') as fd:
 292                 # We don't want unicode characters in log files
 293                 fd.write(result.stderr.decode('UTF-8').encode('ASCII', 'replace'))
 294         elif os.path.exists(errfile):
 295             os.remove(errfile)
 296
 297         if result.toolchain:
 298             # Write the build result and toolchain information.
 299             done_file = self.builder.GetDoneFile(result.commit_upto,
 300                     result.brd.target)
 301             with open(done_file, 'w') as fd:
 302                 if maybe_aborted:
 303                     # Special code to indicate we need to retry
 304                     fd.write('%s' % RETURN_CODE_RETRY)
 305                 else:
 306                     fd.write('%s' % result.return_code)
 307             with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
 308                 print >>fd, 'gcc', result.toolchain.gcc
 309                 print >>fd, 'path', result.toolchain.path
 310                 print >>fd, 'cross', result.toolchain.cross
 311                 print >>fd, 'arch', result.toolchain.arch
 312                 fd.write('%s' % result.return_code)
 313
 314             # Write out the image and function size information and an objdump
 315             env = result.toolchain.MakeEnvironment(self.builder.full_path)
 316             lines = []
 317             for fname in ['u-boot', 'spl/u-boot-spl']:
 318                 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
 319                 nm_result = command.RunPipe([cmd], capture=True,
 320                         capture_stderr=True, cwd=result.out_dir,
 321                         raise_on_error=False, env=env)
 322                 if nm_result.stdout:
 323                     nm = self.builder.GetFuncSizesFile(result.commit_upto,
 324                                     result.brd.target, fname)
 325                     with open(nm, 'w') as fd:
 326                         print >>fd, nm_result.stdout,
 327
 328                 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
 329                 dump_result = command.RunPipe([cmd], capture=True,
 330                         capture_stderr=True, cwd=result.out_dir,
 331                         raise_on_error=False, env=env)
 332                 rodata_size = ''
 333                 if dump_result.stdout:
 334                     objdump = self.builder.GetObjdumpFile(result.commit_upto,
 335                                     result.brd.target, fname)
 336                     with open(objdump, 'w') as fd:
 337                         print >>fd, dump_result.stdout,
 338                     for line in dump_result.stdout.splitlines():
 339                         fields = line.split()
 340                         if len(fields) > 5 and fields[1] == '.rodata':
 341                             rodata_size = fields[2]
 342
 343                 cmd = ['%ssize' % self.toolchain.cross, fname]
 344                 size_result = command.RunPipe([cmd], capture=True,
 345                         capture_stderr=True, cwd=result.out_dir,
 346                         raise_on_error=False, env=env)
 347                 if size_result.stdout:
 348                     lines.append(size_result.stdout.splitlines()[1] + ' ' +
 349                                  rodata_size)
 350
 351             # Write out the image sizes file. This is similar to the output
 352             # of binutil's 'size' utility, but it omits the header line and
 353             # adds an additional hex value at the end of each line for the
 354             # rodata size
 355             if len(lines):
 356                 sizes = self.builder.GetSizesFile(result.commit_upto,
 357                                 result.brd.target)
 358                 with open(sizes, 'w') as fd:
 359                     print >>fd, '\n'.join(lines)
 360
 361         # Write out the configuration files, with a special case for SPL
 362         for dirname in ['', 'spl', 'tpl']:
 363             self.CopyFiles(result.out_dir, build_dir, dirname, ['u-boot.cfg',
 364                 'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg', '.config',
 365                 'include/autoconf.mk', 'include/generated/autoconf.h'])
 366
 367         # Now write the actual build output
 368         if keep_outputs:
 369             self.CopyFiles(result.out_dir, build_dir, '', ['u-boot*', '*.bin',
 370                 '*.map', '*.img', 'MLO', 'SPL', 'include/autoconf.mk',
 371                 'spl/u-boot-spl*'])
 372
 373     def CopyFiles(self, out_dir, build_dir, dirname, patterns):
 374         """Copy files from the build directory to the output.
 375
 376         Args:
 377             out_dir: Path to output directory containing the files
 378             build_dir: Place to copy the files
 379             dirname: Source directory, '' for normal U-Boot, 'spl' for SPL
 380             patterns: A list of filenames (strings) to copy, each relative
 381                to the build directory
 382         """
 383         for pattern in patterns:
 384             file_list = glob.glob(os.path.join(out_dir, dirname, pattern))
 385             for fname in file_list:
 386                 target = os.path.basename(fname)
 387                 if dirname:
 388                     base, ext = os.path.splitext(target)
 389                     if ext:
 390                         target = '%s-%s%s' % (base, dirname, ext)
 391                 shutil.copy(fname, os.path.join(build_dir, target))
 392
 393     def RunJob(self, job):
 394         """Run a single job
 395
 396         A job consists of a building a list of commits for a particular board.
 397
 398         Args:
 399             job: Job to build
 400         """
 401         brd = job.board
 402         work_dir = self.builder.GetThreadDir(self.thread_num)
 403         self.toolchain = None
 404         if job.commits:
 405             # Run 'make board_defconfig' on the first commit
 406             do_config = True
 407             commit_upto  = 0
 408             force_build = False
 409             for commit_upto in range(0, len(job.commits), job.step):
 410                 result, request_config = self.RunCommit(commit_upto, brd,
 411                         work_dir, do_config, self.builder.config_only,
 412                         force_build or self.builder.force_build,
 413                         self.builder.force_build_failures)
 414                 failed = result.return_code or result.stderr
 415                 did_config = do_config
 416                 if failed and not do_config:
 417                     # If our incremental build failed, try building again
 418                     # with a reconfig.
 419                     if self.builder.force_config_on_failure:
 420                         result, request_config = self.RunCommit(commit_upto,
 421                             brd, work_dir, True, False, True, False)
 422                         did_config = True
 423                 if not self.builder.force_reconfig:
 424                     do_config = request_config
 425
 426                 # If we built that commit, then config is done. But if we got
 427                 # an warning, reconfig next time to force it to build the same
 428                 # files that created warnings this time. Otherwise an
 429                 # incremental build may not build the same file, and we will
 430                 # think that the warning has gone away.
 431                 # We could avoid this by using -Werror everywhere...
 432                 # For errors, the problem doesn't happen, since presumably
 433                 # the build stopped and didn't generate output, so will retry
 434                 # that file next time. So we could detect warnings and deal
 435                 # with them specially here. For now, we just reconfigure if
 436                 # anything goes work.
 437                 # Of course this is substantially slower if there are build
 438                 # errors/warnings (e.g. 2-3x slower even if only 10% of builds
 439                 # have problems).
 440                 if (failed and not result.already_done and not did_config and
 441                         self.builder.force_config_on_failure):
 442                     # If this build failed, try the next one with a
 443                     # reconfigure.
 444                     # Sometimes if the board_config.h file changes it can mess
 445                     # with dependencies, and we get:
 446                     # make: *** No rule to make target `include/autoconf.mk',
 447                     #     needed by `depend'.
 448                     do_config = True
 449                     force_build = True
 450                 else:
 451                     force_build = False
 452                     if self.builder.force_config_on_failure:
 453                         if failed:
 454                             do_config = True
 455                     result.commit_upto = commit_upto
 456                     if result.return_code < 0:
 457                         raise ValueError('Interrupt')
 458
 459                 # We have the build results, so output the result
 460                 self._WriteResult(result, job.keep_outputs)
 461                 self.builder.out_queue.put(result)
 462         else:
 463             # Just build the currently checked-out build
 464             result, request_config = self.RunCommit(None, brd, work_dir, True,
 465                         self.builder.config_only, True,
 466                         self.builder.force_build_failures)
 467             result.commit_upto = 0
 468             self._WriteResult(result, job.keep_outputs)
 469             self.builder.out_queue.put(result)
 470
 471     def run(self):
 472         """Our thread's run function
 473
 474         This thread picks a job from the queue, runs it, and then goes to the
 475         next job.
 476         """
 477         while True:
 478             job = self.builder.queue.get()
 479             self.RunJob(job)
 480             self.builder.queue.task_done()