git.sur5r.net Git - u-boot/blob - tools/buildman/control.py

   1 # Copyright (c) 2013 The Chromium OS Authors.
   2 #
   3 # SPDX-License-Identifier:      GPL-2.0+
   4 #
   5
   6 import multiprocessing
   7 import os
   8 import shutil
   9 import sys
  10
  11 import board
  12 import bsettings
  13 from builder import Builder
  14 import gitutil
  15 import patchstream
  16 import terminal
  17 from terminal import Print
  18 import toolchain
  19 import command
  20 import subprocess
  21
  22 def GetPlural(count):
  23     """Returns a plural 's' if count is not 1"""
  24     return 's' if count != 1 else ''
  25
  26 def GetActionSummary(is_summary, commits, selected, options):
  27     """Return a string summarising the intended action.
  28
  29     Returns:
  30         Summary string.
  31     """
  32     if commits:
  33         count = len(commits)
  34         count = (count + options.step - 1) / options.step
  35         commit_str = '%d commit%s' % (count, GetPlural(count))
  36     else:
  37         commit_str = 'current source'
  38     str = '%s %s for %d boards' % (
  39         'Summary of' if is_summary else 'Building', commit_str,
  40         len(selected))
  41     str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
  42             GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
  43     return str
  44
  45 def ShowActions(series, why_selected, boards_selected, builder, options):
  46     """Display a list of actions that we would take, if not a dry run.
  47
  48     Args:
  49         series: Series object
  50         why_selected: Dictionary where each key is a buildman argument
  51                 provided by the user, and the value is the boards brought
  52                 in by that argument. For example, 'arm' might bring in
  53                 400 boards, so in this case the key would be 'arm' and
  54                 the value would be a list of board names.
  55         boards_selected: Dict of selected boards, key is target name,
  56                 value is Board object
  57         builder: The builder that will be used to build the commits
  58         options: Command line options object
  59     """
  60     col = terminal.Color()
  61     print 'Dry run, so not doing much. But I would do this:'
  62     print
  63     if series:
  64         commits = series.commits
  65     else:
  66         commits = None
  67     print GetActionSummary(False, commits, boards_selected,
  68             options)
  69     print 'Build directory: %s' % builder.base_dir
  70     if commits:
  71         for upto in range(0, len(series.commits), options.step):
  72             commit = series.commits[upto]
  73             print '   ', col.Color(col.YELLOW, commit.hash[:8], bright=False),
  74             print commit.subject
  75     print
  76     for arg in why_selected:
  77         if arg != 'all':
  78             print arg, ': %d boards' % why_selected[arg]
  79     print ('Total boards to build for each commit: %d\n' %
  80             why_selected['all'])
  81
  82 def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
  83                clean_dir=False):
  84     """The main control code for buildman
  85
  86     Args:
  87         options: Command line options object
  88         args: Command line arguments (list of strings)
  89         toolchains: Toolchains to use - this should be a Toolchains()
  90                 object. If None, then it will be created and scanned
  91         make_func: Make function to use for the builder. This is called
  92                 to execute 'make'. If this is None, the normal function
  93                 will be used, which calls the 'make' tool with suitable
  94                 arguments. This setting is useful for tests.
  95         board: Boards() object to use, containing a list of available
  96                 boards. If this is None it will be created and scanned.
  97     """
  98     global builder
  99
 100     if options.full_help:
 101         pager = os.getenv('PAGER')
 102         if not pager:
 103             pager = 'more'
 104         fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
 105                              'README')
 106         command.Run(pager, fname)
 107         return 0
 108
 109     gitutil.Setup()
 110
 111     options.git_dir = os.path.join(options.git, '.git')
 112
 113     if not toolchains:
 114         toolchains = toolchain.Toolchains()
 115         toolchains.GetSettings()
 116         toolchains.Scan(options.list_tool_chains)
 117     if options.list_tool_chains:
 118         toolchains.List()
 119         print
 120         return 0
 121
 122     if options.fetch_arch:
 123         if options.fetch_arch == 'list':
 124             sorted_list = toolchains.ListArchs()
 125             print 'Available architectures: %s\n' % ' '.join(sorted_list)
 126             return 0
 127         else:
 128             fetch_arch = options.fetch_arch
 129             if fetch_arch == 'all':
 130                 fetch_arch = ','.join(toolchains.ListArchs())
 131                 print 'Downloading toolchains: %s\n' % fetch_arch
 132             for arch in fetch_arch.split(','):
 133                 ret = toolchains.FetchAndInstall(arch)
 134                 if ret:
 135                     return ret
 136             return 0
 137
 138     # Work out how many commits to build. We want to build everything on the
 139     # branch. We also build the upstream commit as a control so we can see
 140     # problems introduced by the first commit on the branch.
 141     col = terminal.Color()
 142     count = options.count
 143     has_range = options.branch and '..' in options.branch
 144     if count == -1:
 145         if not options.branch:
 146             count = 1
 147         else:
 148             if has_range:
 149                 count, msg = gitutil.CountCommitsInRange(options.git_dir,
 150                                                          options.branch)
 151             else:
 152                 count, msg = gitutil.CountCommitsInBranch(options.git_dir,
 153                                                           options.branch)
 154             if count is None:
 155                 sys.exit(col.Color(col.RED, msg))
 156             elif count == 0:
 157                 sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
 158                                    options.branch))
 159             if msg:
 160                 print col.Color(col.YELLOW, msg)
 161             count += 1   # Build upstream commit also
 162
 163     if not count:
 164         str = ("No commits found to process in branch '%s': "
 165                "set branch's upstream or use -c flag" % options.branch)
 166         sys.exit(col.Color(col.RED, str))
 167
 168     # Work out what subset of the boards we are building
 169     if not boards:
 170         board_file = os.path.join(options.git, 'boards.cfg')
 171         status = subprocess.call([os.path.join(options.git,
 172                                                 'tools/genboardscfg.py')])
 173         if status != 0:
 174                 sys.exit("Failed to generate boards.cfg")
 175
 176         boards = board.Boards()
 177         boards.ReadBoards(os.path.join(options.git, 'boards.cfg'))
 178
 179     exclude = []
 180     if options.exclude:
 181         for arg in options.exclude:
 182             exclude += arg.split(',')
 183
 184     why_selected = boards.SelectBoards(args, exclude)
 185     selected = boards.GetSelected()
 186     if not len(selected):
 187         sys.exit(col.Color(col.RED, 'No matching boards found'))
 188
 189     # Read the metadata from the commits. First look at the upstream commit,
 190     # then the ones in the branch. We would like to do something like
 191     # upstream/master~..branch but that isn't possible if upstream/master is
 192     # a merge commit (it will list all the commits that form part of the
 193     # merge)
 194     # Conflicting tags are not a problem for buildman, since it does not use
 195     # them. For example, Series-version is not useful for buildman. On the
 196     # other hand conflicting tags will cause an error. So allow later tags
 197     # to overwrite earlier ones by setting allow_overwrite=True
 198     if options.branch:
 199         if count == -1:
 200             if has_range:
 201                 range_expr = options.branch
 202             else:
 203                 range_expr = gitutil.GetRangeInBranch(options.git_dir,
 204                                                       options.branch)
 205             upstream_commit = gitutil.GetUpstream(options.git_dir,
 206                                                   options.branch)
 207             series = patchstream.GetMetaDataForList(upstream_commit,
 208                 options.git_dir, 1, series=None, allow_overwrite=True)
 209
 210             series = patchstream.GetMetaDataForList(range_expr,
 211                     options.git_dir, None, series, allow_overwrite=True)
 212         else:
 213             # Honour the count
 214             series = patchstream.GetMetaDataForList(options.branch,
 215                     options.git_dir, count, series=None, allow_overwrite=True)
 216     else:
 217         series = None
 218         options.verbose = True
 219         if not options.summary:
 220             options.show_errors = True
 221
 222     # By default we have one thread per CPU. But if there are not enough jobs
 223     # we can have fewer threads and use a high '-j' value for make.
 224     if not options.threads:
 225         options.threads = min(multiprocessing.cpu_count(), len(selected))
 226     if not options.jobs:
 227         options.jobs = max(1, (multiprocessing.cpu_count() +
 228                 len(selected) - 1) / len(selected))
 229
 230     if not options.step:
 231         options.step = len(series.commits) - 1
 232
 233     gnu_make = command.Output(os.path.join(options.git,
 234                                            'scripts/show-gnu-make')).rstrip()
 235     if not gnu_make:
 236         sys.exit('GNU Make not found')
 237
 238     # Create a new builder with the selected options.
 239     output_dir = options.output_dir
 240     if options.branch:
 241         dirname = options.branch.replace('/', '_')
 242         # As a special case allow the board directory to be placed in the
 243         # output directory itself rather than any subdirectory.
 244         if not options.no_subdirs:
 245             output_dir = os.path.join(options.output_dir, dirname)
 246     if (clean_dir and output_dir != options.output_dir and
 247             os.path.exists(output_dir)):
 248         shutil.rmtree(output_dir)
 249     builder = Builder(toolchains, output_dir, options.git_dir,
 250             options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
 251             show_unknown=options.show_unknown, step=options.step,
 252             no_subdirs=options.no_subdirs, full_path=options.full_path,
 253             verbose_build=options.verbose_build,
 254             incremental=options.incremental,
 255             per_board_out_dir=options.per_board_out_dir,)
 256     builder.force_config_on_failure = not options.quick
 257     if make_func:
 258         builder.do_make = make_func
 259
 260     # For a dry run, just show our actions as a sanity check
 261     if options.dry_run:
 262         ShowActions(series, why_selected, selected, builder, options)
 263     else:
 264         builder.force_build = options.force_build
 265         builder.force_build_failures = options.force_build_failures
 266         builder.force_reconfig = options.force_reconfig
 267         builder.in_tree = options.in_tree
 268
 269         # Work out which boards to build
 270         board_selected = boards.GetSelectedDict()
 271
 272         if series:
 273             commits = series.commits
 274             # Number the commits for test purposes
 275             for commit in range(len(commits)):
 276                 commits[commit].sequence = commit
 277         else:
 278             commits = None
 279
 280         Print(GetActionSummary(options.summary, commits, board_selected,
 281                                 options))
 282
 283         # We can't show function sizes without board details at present
 284         if options.show_bloat:
 285             options.show_detail = True
 286         builder.SetDisplayOptions(options.show_errors, options.show_sizes,
 287                                   options.show_detail, options.show_bloat,
 288                                   options.list_error_boards,
 289                                   options.show_config)
 290         if options.summary:
 291             builder.ShowSummary(commits, board_selected)
 292         else:
 293             fail, warned = builder.BuildBoards(commits, board_selected,
 294                                 options.keep_outputs, options.verbose)
 295             if fail:
 296                 return 128
 297             elif warned:
 298                 return 129
 299     return 0