1 # Copyright (c) 2012 The Chromium OS Authors.
3 # SPDX-License-Identifier: GPL-2.0+
8 from HTMLParser import HTMLParser
18 (PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
19 PRIORITY_CALC) = range(4)
21 # Simple class to collect links from a page
22 class MyHTMLParser(HTMLParser):
23 def __init__(self, arch):
24 """Create a new parser
26 After the parser runs, self.links will be set to a list of the links
27 to .xz archives found in the page, and self.arch_link will be set to
28 the one for the given architecture (or None if not found).
31 arch: Architecture to search for
33 HTMLParser.__init__(self)
36 self._match = '_%s-' % arch
38 def handle_starttag(self, tag, attrs):
40 for tag, value in attrs:
42 if value and value.endswith('.xz'):
43 self.links.append(value)
44 if self._match in value:
45 self.arch_link = value
52 gcc: Full path to C compiler
53 path: Directory path containing C compiler
54 cross: Cross compile string, e.g. 'arm-linux-'
55 arch: Architecture of toolchain as determined from the first
56 component of the filename. E.g. arm-linux-gcc becomes arm
57 priority: Toolchain priority (0=highest, 20=lowest)
59 def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,
61 """Create a new toolchain object.
64 fname: Filename of the gcc component
65 test: True to run the toolchain to test it
66 verbose: True to print out the information
67 priority: Priority to use for this toolchain, or PRIORITY_CALC to
71 self.path = os.path.dirname(fname)
73 # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
74 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
75 basename = os.path.basename(fname)
76 pos = basename.rfind('-')
77 self.cross = basename[:pos + 1] if pos != -1 else ''
79 # The architecture is the first part of the name
80 pos = self.cross.find('-')
84 self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
86 env = self.MakeEnvironment(False)
88 # As a basic sanity check, run the C compiler with --version
89 cmd = [fname, '--version']
90 if priority == PRIORITY_CALC:
91 self.priority = self.GetPriority(fname)
93 self.priority = priority
95 result = command.RunPipe([cmd], capture=True, env=env,
97 self.ok = result.return_code == 0
99 print 'Tool chain test: ',
101 print "OK, arch='%s', priority %d" % (self.arch,
105 print 'Command: ', cmd
111 def GetPriority(self, fname):
112 """Return the priority of the toolchain.
114 Toolchains are ranked according to their suitability by their
118 fname: Filename of toolchain
120 Priority of toolchain, PRIORITY_CALC=highest, 20=lowest.
122 priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
123 '-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux',
124 '-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi',
125 '-linux-gnueabihf', '-le-linux', '-uclinux']
126 for prio in range(len(priority_list)):
127 if priority_list[prio] in fname:
128 return PRIORITY_CALC + prio
129 return PRIORITY_CALC + prio
131 def GetWrapper(self, show_warning=True):
132 """Get toolchain wrapper from the setting file.
135 for name, value in bsettings.GetItems('toolchain-wrapper'):
137 print "Warning: Wrapper not found"
143 def MakeEnvironment(self, full_path):
144 """Returns an environment for using the toolchain.
146 Thie takes the current environment and adds CROSS_COMPILE so that
147 the tool chain will operate correctly. This also disables localized
148 output and possibly unicode encoded output of all build tools by
152 full_path: Return the full path in CROSS_COMPILE and don't set
155 env = dict(os.environ)
156 wrapper = self.GetWrapper()
159 env['CROSS_COMPILE'] = wrapper + os.path.join(self.path, self.cross)
161 env['CROSS_COMPILE'] = wrapper + self.cross
162 env['PATH'] = self.path + ':' + env['PATH']
170 """Manage a list of toolchains for building U-Boot
172 We select one toolchain for each architecture type
175 toolchains: Dict of Toolchain objects, keyed by architecture name
176 prefixes: Dict of prefixes to check, keyed by architecture. This can
177 be a full path and toolchain prefix, for example
178 {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of
179 something on the search path, for example
180 {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported.
181 paths: List of paths to check for toolchains (may contain wildcards)
188 self._make_flags = dict(bsettings.GetItems('make-flags'))
190 def GetPathList(self, show_warning=True):
191 """Get a list of available toolchain paths
194 show_warning: True to show a warning if there are no tool chains.
197 List of strings, each a path to a toolchain mentioned in the
198 [toolchain] section of the settings file.
200 toolchains = bsettings.GetItems('toolchain')
201 if show_warning and not toolchains:
202 print ("Warning: No tool chains. Please run 'buildman "
203 "--fetch-arch all' to download all available toolchains, or "
204 "add a [toolchain] section to your buildman config file "
205 "%s. See README for details" %
206 bsettings.config_fname)
209 for name, value in toolchains:
211 paths += glob.glob(value)
216 def GetSettings(self, show_warning=True):
217 """Get toolchain settings from the settings file.
220 show_warning: True to show a warning if there are no tool chains.
222 self.prefixes = bsettings.GetItems('toolchain-prefix')
223 self.paths += self.GetPathList(show_warning)
225 def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC,
227 """Add a toolchain to our list
229 We select the given toolchain as our preferred one for its
230 architecture if it is a higher priority than the others.
233 fname: Filename of toolchain's gcc driver
234 test: True to run the toolchain to test it
235 priority: Priority to use for this toolchain
236 arch: Toolchain architecture, or None if not known
238 toolchain = Toolchain(fname, test, verbose, priority, arch)
239 add_it = toolchain.ok
240 if toolchain.arch in self.toolchains:
241 add_it = (toolchain.priority <
242 self.toolchains[toolchain.arch].priority)
244 self.toolchains[toolchain.arch] = toolchain
246 print ("Toolchain '%s' at priority %d will be ignored because "
247 "another toolchain for arch '%s' has priority %d" %
248 (toolchain.gcc, toolchain.priority, toolchain.arch,
249 self.toolchains[toolchain.arch].priority))
251 def ScanPath(self, path, verbose):
252 """Scan a path for a valid toolchain
256 verbose: True to print out progress information
258 Filename of C compiler if found, else None
261 for subdir in ['.', 'bin', 'usr/bin']:
262 dirname = os.path.join(path, subdir)
263 if verbose: print " - looking in '%s'" % dirname
264 for fname in glob.glob(dirname + '/*gcc'):
265 if verbose: print " - found '%s'" % fname
269 def ScanPathEnv(self, fname):
270 """Scan the PATH environment variable for a given filename.
273 fname: Filename to scan for
275 List of matching pathanames, or [] if none
278 for path in os.environ["PATH"].split(os.pathsep):
279 path = path.strip('"')
280 pathname = os.path.join(path, fname)
281 if os.path.exists(pathname):
282 pathname_list.append(pathname)
285 def Scan(self, verbose):
286 """Scan for available toolchains and select the best for each arch.
288 We look for all the toolchains we can file, figure out the
289 architecture for each, and whether it works. Then we select the
290 highest priority toolchain for each arch.
293 verbose: True to print out progress information
295 if verbose: print 'Scanning for tool chains'
296 for name, value in self.prefixes:
297 if verbose: print " - scanning prefix '%s'" % value
298 if os.path.exists(value):
299 self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name)
301 fname = value + 'gcc'
302 if os.path.exists(fname):
303 self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name)
305 fname_list = self.ScanPathEnv(fname)
307 self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
309 raise ValueError, ("No tool chain found for prefix '%s'" %
311 for path in self.paths:
312 if verbose: print " - scanning path '%s'" % path
313 fnames = self.ScanPath(path, verbose)
315 self.Add(fname, True, verbose)
318 """List out the selected toolchains for each architecture"""
319 col = terminal.Color()
320 print col.Color(col.BLUE, 'List of available toolchains (%d):' %
321 len(self.toolchains))
322 if len(self.toolchains):
323 for key, value in sorted(self.toolchains.iteritems()):
324 print '%-10s: %s' % (key, value.gcc)
328 def Select(self, arch):
329 """Returns the toolchain for a given architecture
332 args: Name of architecture (e.g. 'arm', 'ppc_8xx')
335 toolchain object, or None if none found
337 for tag, value in bsettings.GetItems('toolchain-alias'):
339 for alias in value.split():
340 if alias in self.toolchains:
341 return self.toolchains[alias]
343 if not arch in self.toolchains:
344 raise ValueError, ("No tool chain found for arch '%s'" % arch)
345 return self.toolchains[arch]
347 def ResolveReferences(self, var_dict, args):
348 """Resolve variable references in a string
350 This converts ${blah} within the string to the value of blah.
351 This function works recursively.
354 var_dict: Dictionary containing variables and their values
355 args: String containing make arguments
359 >>> bsettings.Setup()
360 >>> tcs = Toolchains()
361 >>> tcs.Add('fred', False)
362 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
364 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
366 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
367 'this=OBLIQUE_setfi2ndrstnd'
369 re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
372 m = re_var.search(args)
375 lookup = m.group(0)[2:-1]
376 value = var_dict.get(lookup, '')
377 args = args[:m.start(0)] + value + args[m.end(0):]
380 def GetMakeArguments(self, board):
381 """Returns 'make' arguments for a given board
383 The flags are in a section called 'make-flags'. Flags are named
384 after the target they represent, for example snapper9260=TESTING=1
385 will pass TESTING=1 to make when building the snapper9260 board.
387 References to other boards can be added in the string also. For
391 at91-boards=ENABLE_AT91_TEST=1
392 snapper9260=${at91-boards} BUILD_TAG=442
393 snapper9g45=${at91-boards} BUILD_TAG=443
395 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
396 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
398 A special 'target' variable is set to the board target.
401 board: Board object for the board to check.
403 'make' flags for that board, or '' if none
405 self._make_flags['target'] = board.target
406 arg_str = self.ResolveReferences(self._make_flags,
407 self._make_flags.get(board.target, ''))
408 args = arg_str.split(' ')
417 def LocateArchUrl(self, fetch_arch):
418 """Find a toolchain available online
420 Look in standard places for available toolchains. At present the
421 only standard place is at kernel.org.
424 arch: Architecture to look for, or 'list' for all
426 If fetch_arch is 'list', a tuple:
427 Machine architecture (e.g. x86_64)
430 URL containing this toolchain, if avaialble, else None
432 arch = command.OutputOneLine('uname', '-m')
433 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
434 versions = ['4.9.0', '4.6.3', '4.6.2', '4.5.1', '4.2.4']
436 for version in versions:
437 url = '%s/%s/%s/' % (base, arch, version)
438 print 'Checking: %s' % url
439 response = urllib2.urlopen(url)
440 html = response.read()
441 parser = MyHTMLParser(fetch_arch)
443 if fetch_arch == 'list':
444 links += parser.links
445 elif parser.arch_link:
446 return url + parser.arch_link
447 if fetch_arch == 'list':
451 def Download(self, url):
452 """Download a file to a temporary directory
458 Temporary directory name
459 Full path to the downloaded archive file in that directory,
460 or None if there was an error while downloading
462 print 'Downloading: %s' % url
463 leaf = url.split('/')[-1]
464 tmpdir = tempfile.mkdtemp('.buildman')
465 response = urllib2.urlopen(url)
466 fname = os.path.join(tmpdir, leaf)
467 fd = open(fname, 'wb')
468 meta = response.info()
469 size = int(meta.getheaders('Content-Length')[0])
474 # Read the file in chunks and show progress as we go
476 buffer = response.read(block_size)
478 print chr(8) * (len(status) + 1), '\r',
483 status = r'%10d MiB [%3d%%]' % (done / 1024 / 1024,
485 status = status + chr(8) * (len(status) + 1)
490 print 'Error, failed to download'
495 def Unpack(self, fname, dest):
499 fname: Filename to unpack
500 dest: Destination directory
502 Directory name of the first entry in the archive, without the
505 stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)
506 return stdout.splitlines()[0][:-1]
508 def TestSettingsHasPath(self, path):
509 """Check if buildman will find this toolchain
512 True if the path is in settings, False if not
514 paths = self.GetPathList(False)
518 """List architectures with available toolchains to download"""
519 host_arch, archives = self.LocateArchUrl('list')
520 re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*')
522 for archive in archives:
523 # Remove the host architecture from the start
524 arch = re_arch.match(archive[len(host_arch):])
526 arch_set.add(arch.group(1))
527 return sorted(arch_set)
529 def FetchAndInstall(self, arch):
530 """Fetch and install a new toolchain
533 Architecture to fetch, or 'list' to list
535 # Fist get the URL for this architecture
536 col = terminal.Color()
537 print col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch)
538 url = self.LocateArchUrl(arch)
540 print ("Cannot find toolchain for arch '%s' - use 'list' to list" %
543 home = os.environ['HOME']
544 dest = os.path.join(home, '.buildman-toolchains')
545 if not os.path.exists(dest):
548 # Download the tar file for this toolchain and unpack it
549 tmpdir, tarfile = self.Download(url)
552 print col.Color(col.GREEN, 'Unpacking to: %s' % dest),
554 path = self.Unpack(tarfile, dest)
559 # Check that the toolchain works
560 print col.Color(col.GREEN, 'Testing')
561 dirpath = os.path.join(dest, path)
562 compiler_fname_list = self.ScanPath(dirpath, True)
563 if not compiler_fname_list:
564 print 'Could not locate C compiler - fetch failed.'
566 if len(compiler_fname_list) != 1:
567 print col.Color(col.RED, 'Warning, ambiguous toolchains: %s' %
568 ', '.join(compiler_fname_list))
569 toolchain = Toolchain(compiler_fname_list[0], True, True)
571 # Make sure that it will be found by buildman
572 if not self.TestSettingsHasPath(dirpath):
573 print ("Adding 'download' to config file '%s'" %
574 bsettings.config_fname)
575 bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest)