1 # Copyright (c) 2012 The Chromium OS Authors.
3 # SPDX-License-Identifier: GPL-2.0+
8 from HTMLParser import HTMLParser
18 (PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
19 PRIORITY_CALC) = range(4)
21 # Simple class to collect links from a page
22 class MyHTMLParser(HTMLParser):
23 def __init__(self, arch):
24 """Create a new parser
26 After the parser runs, self.links will be set to a list of the links
27 to .xz archives found in the page, and self.arch_link will be set to
28 the one for the given architecture (or None if not found).
31 arch: Architecture to search for
33 HTMLParser.__init__(self)
36 self._match = '_%s-' % arch
38 def handle_starttag(self, tag, attrs):
40 for tag, value in attrs:
42 if value and value.endswith('.xz'):
43 self.links.append(value)
44 if self._match in value:
45 self.arch_link = value
52 gcc: Full path to C compiler
53 path: Directory path containing C compiler
54 cross: Cross compile string, e.g. 'arm-linux-'
55 arch: Architecture of toolchain as determined from the first
56 component of the filename. E.g. arm-linux-gcc becomes arm
57 priority: Toolchain priority (0=highest, 20=lowest)
59 def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,
61 """Create a new toolchain object.
64 fname: Filename of the gcc component
65 test: True to run the toolchain to test it
66 verbose: True to print out the information
67 priority: Priority to use for this toolchain, or PRIORITY_CALC to
71 self.path = os.path.dirname(fname)
73 # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
74 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
75 basename = os.path.basename(fname)
76 pos = basename.rfind('-')
77 self.cross = basename[:pos + 1] if pos != -1 else ''
79 # The architecture is the first part of the name
80 pos = self.cross.find('-')
84 self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
86 env = self.MakeEnvironment(False)
88 # As a basic sanity check, run the C compiler with --version
89 cmd = [fname, '--version']
90 if priority == PRIORITY_CALC:
91 self.priority = self.GetPriority(fname)
93 self.priority = priority
95 result = command.RunPipe([cmd], capture=True, env=env,
97 self.ok = result.return_code == 0
99 print 'Tool chain test: ',
101 print "OK, arch='%s', priority %d" % (self.arch,
105 print 'Command: ', cmd
111 def GetPriority(self, fname):
112 """Return the priority of the toolchain.
114 Toolchains are ranked according to their suitability by their
118 fname: Filename of toolchain
120 Priority of toolchain, PRIORITY_CALC=highest, 20=lowest.
122 priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
123 '-none-linux-gnueabi', '-uclinux', '-none-eabi',
124 '-gentoo-linux-gnu', '-linux-gnueabi', '-le-linux', '-uclinux']
125 for prio in range(len(priority_list)):
126 if priority_list[prio] in fname:
127 return PRIORITY_CALC + prio
128 return PRIORITY_CALC + prio
130 def GetWrapper(self, show_warning=True):
131 """Get toolchain wrapper from the setting file.
134 for name, value in bsettings.GetItems('toolchain-wrapper'):
136 print "Warning: Wrapper not found"
142 def MakeEnvironment(self, full_path):
143 """Returns an environment for using the toolchain.
145 Thie takes the current environment and adds CROSS_COMPILE so that
146 the tool chain will operate correctly.
149 full_path: Return the full path in CROSS_COMPILE and don't set
152 env = dict(os.environ)
153 wrapper = self.GetWrapper()
156 env['CROSS_COMPILE'] = wrapper + os.path.join(self.path, self.cross)
158 env['CROSS_COMPILE'] = wrapper + self.cross
159 env['PATH'] = self.path + ':' + env['PATH']
165 """Manage a list of toolchains for building U-Boot
167 We select one toolchain for each architecture type
170 toolchains: Dict of Toolchain objects, keyed by architecture name
171 prefixes: Dict of prefixes to check, keyed by architecture. This can
172 be a full path and toolchain prefix, for example
173 {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of
174 something on the search path, for example
175 {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported.
176 paths: List of paths to check for toolchains (may contain wildcards)
183 self._make_flags = dict(bsettings.GetItems('make-flags'))
185 def GetPathList(self, show_warning=True):
186 """Get a list of available toolchain paths
189 show_warning: True to show a warning if there are no tool chains.
192 List of strings, each a path to a toolchain mentioned in the
193 [toolchain] section of the settings file.
195 toolchains = bsettings.GetItems('toolchain')
196 if show_warning and not toolchains:
197 print ("Warning: No tool chains. Please run 'buildman "
198 "--fetch-arch all' to download all available toolchains, or "
199 "add a [toolchain] section to your buildman config file "
200 "%s. See README for details" %
201 bsettings.config_fname)
204 for name, value in toolchains:
206 paths += glob.glob(value)
211 def GetSettings(self, show_warning=True):
212 """Get toolchain settings from the settings file.
215 show_warning: True to show a warning if there are no tool chains.
217 self.prefixes = bsettings.GetItems('toolchain-prefix')
218 self.paths += self.GetPathList(show_warning)
220 def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC,
222 """Add a toolchain to our list
224 We select the given toolchain as our preferred one for its
225 architecture if it is a higher priority than the others.
228 fname: Filename of toolchain's gcc driver
229 test: True to run the toolchain to test it
230 priority: Priority to use for this toolchain
231 arch: Toolchain architecture, or None if not known
233 toolchain = Toolchain(fname, test, verbose, priority, arch)
234 add_it = toolchain.ok
235 if toolchain.arch in self.toolchains:
236 add_it = (toolchain.priority <
237 self.toolchains[toolchain.arch].priority)
239 self.toolchains[toolchain.arch] = toolchain
241 print ("Toolchain '%s' at priority %d will be ignored because "
242 "another toolchain for arch '%s' has priority %d" %
243 (toolchain.gcc, toolchain.priority, toolchain.arch,
244 self.toolchains[toolchain.arch].priority))
246 def ScanPath(self, path, verbose):
247 """Scan a path for a valid toolchain
251 verbose: True to print out progress information
253 Filename of C compiler if found, else None
256 for subdir in ['.', 'bin', 'usr/bin']:
257 dirname = os.path.join(path, subdir)
258 if verbose: print " - looking in '%s'" % dirname
259 for fname in glob.glob(dirname + '/*gcc'):
260 if verbose: print " - found '%s'" % fname
264 def ScanPathEnv(self, fname):
265 """Scan the PATH environment variable for a given filename.
268 fname: Filename to scan for
270 List of matching pathanames, or [] if none
273 for path in os.environ["PATH"].split(os.pathsep):
274 path = path.strip('"')
275 pathname = os.path.join(path, fname)
276 if os.path.exists(pathname):
277 pathname_list.append(pathname)
280 def Scan(self, verbose):
281 """Scan for available toolchains and select the best for each arch.
283 We look for all the toolchains we can file, figure out the
284 architecture for each, and whether it works. Then we select the
285 highest priority toolchain for each arch.
288 verbose: True to print out progress information
290 if verbose: print 'Scanning for tool chains'
291 for name, value in self.prefixes:
292 if verbose: print " - scanning prefix '%s'" % value
293 if os.path.exists(value):
294 self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name)
296 fname = value + 'gcc'
297 if os.path.exists(fname):
298 self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name)
300 fname_list = self.ScanPathEnv(fname)
302 self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
304 raise ValueError, ("No tool chain found for prefix '%s'" %
306 for path in self.paths:
307 if verbose: print " - scanning path '%s'" % path
308 fnames = self.ScanPath(path, verbose)
310 self.Add(fname, True, verbose)
313 """List out the selected toolchains for each architecture"""
314 col = terminal.Color()
315 print col.Color(col.BLUE, 'List of available toolchains (%d):' %
316 len(self.toolchains))
317 if len(self.toolchains):
318 for key, value in sorted(self.toolchains.iteritems()):
319 print '%-10s: %s' % (key, value.gcc)
323 def Select(self, arch):
324 """Returns the toolchain for a given architecture
327 args: Name of architecture (e.g. 'arm', 'ppc_8xx')
330 toolchain object, or None if none found
332 for tag, value in bsettings.GetItems('toolchain-alias'):
334 for alias in value.split():
335 if alias in self.toolchains:
336 return self.toolchains[alias]
338 if not arch in self.toolchains:
339 raise ValueError, ("No tool chain found for arch '%s'" % arch)
340 return self.toolchains[arch]
342 def ResolveReferences(self, var_dict, args):
343 """Resolve variable references in a string
345 This converts ${blah} within the string to the value of blah.
346 This function works recursively.
349 var_dict: Dictionary containing variables and their values
350 args: String containing make arguments
354 >>> bsettings.Setup()
355 >>> tcs = Toolchains()
356 >>> tcs.Add('fred', False)
357 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
359 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
361 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
362 'this=OBLIQUE_setfi2ndrstnd'
364 re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
367 m = re_var.search(args)
370 lookup = m.group(0)[2:-1]
371 value = var_dict.get(lookup, '')
372 args = args[:m.start(0)] + value + args[m.end(0):]
375 def GetMakeArguments(self, board):
376 """Returns 'make' arguments for a given board
378 The flags are in a section called 'make-flags'. Flags are named
379 after the target they represent, for example snapper9260=TESTING=1
380 will pass TESTING=1 to make when building the snapper9260 board.
382 References to other boards can be added in the string also. For
386 at91-boards=ENABLE_AT91_TEST=1
387 snapper9260=${at91-boards} BUILD_TAG=442
388 snapper9g45=${at91-boards} BUILD_TAG=443
390 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
391 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
393 A special 'target' variable is set to the board target.
396 board: Board object for the board to check.
398 'make' flags for that board, or '' if none
400 self._make_flags['target'] = board.target
401 arg_str = self.ResolveReferences(self._make_flags,
402 self._make_flags.get(board.target, ''))
403 args = arg_str.split(' ')
412 def LocateArchUrl(self, fetch_arch):
413 """Find a toolchain available online
415 Look in standard places for available toolchains. At present the
416 only standard place is at kernel.org.
419 arch: Architecture to look for, or 'list' for all
421 If fetch_arch is 'list', a tuple:
422 Machine architecture (e.g. x86_64)
425 URL containing this toolchain, if avaialble, else None
427 arch = command.OutputOneLine('uname', '-m')
428 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
429 versions = ['4.9.0', '4.6.3', '4.6.2', '4.5.1', '4.2.4']
431 for version in versions:
432 url = '%s/%s/%s/' % (base, arch, version)
433 print 'Checking: %s' % url
434 response = urllib2.urlopen(url)
435 html = response.read()
436 parser = MyHTMLParser(fetch_arch)
438 if fetch_arch == 'list':
439 links += parser.links
440 elif parser.arch_link:
441 return url + parser.arch_link
442 if fetch_arch == 'list':
446 def Download(self, url):
447 """Download a file to a temporary directory
453 Temporary directory name
454 Full path to the downloaded archive file in that directory,
455 or None if there was an error while downloading
457 print 'Downloading: %s' % url
458 leaf = url.split('/')[-1]
459 tmpdir = tempfile.mkdtemp('.buildman')
460 response = urllib2.urlopen(url)
461 fname = os.path.join(tmpdir, leaf)
462 fd = open(fname, 'wb')
463 meta = response.info()
464 size = int(meta.getheaders('Content-Length')[0])
469 # Read the file in chunks and show progress as we go
471 buffer = response.read(block_size)
473 print chr(8) * (len(status) + 1), '\r',
478 status = r'%10d MiB [%3d%%]' % (done / 1024 / 1024,
480 status = status + chr(8) * (len(status) + 1)
485 print 'Error, failed to download'
490 def Unpack(self, fname, dest):
494 fname: Filename to unpack
495 dest: Destination directory
497 Directory name of the first entry in the archive, without the
500 stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)
501 return stdout.splitlines()[0][:-1]
503 def TestSettingsHasPath(self, path):
504 """Check if buildman will find this toolchain
507 True if the path is in settings, False if not
509 paths = self.GetPathList(False)
513 """List architectures with available toolchains to download"""
514 host_arch, archives = self.LocateArchUrl('list')
515 re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*')
517 for archive in archives:
518 # Remove the host architecture from the start
519 arch = re_arch.match(archive[len(host_arch):])
521 arch_set.add(arch.group(1))
522 return sorted(arch_set)
524 def FetchAndInstall(self, arch):
525 """Fetch and install a new toolchain
528 Architecture to fetch, or 'list' to list
530 # Fist get the URL for this architecture
531 col = terminal.Color()
532 print col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch)
533 url = self.LocateArchUrl(arch)
535 print ("Cannot find toolchain for arch '%s' - use 'list' to list" %
538 home = os.environ['HOME']
539 dest = os.path.join(home, '.buildman-toolchains')
540 if not os.path.exists(dest):
543 # Download the tar file for this toolchain and unpack it
544 tmpdir, tarfile = self.Download(url)
547 print col.Color(col.GREEN, 'Unpacking to: %s' % dest),
549 path = self.Unpack(tarfile, dest)
554 # Check that the toolchain works
555 print col.Color(col.GREEN, 'Testing')
556 dirpath = os.path.join(dest, path)
557 compiler_fname_list = self.ScanPath(dirpath, True)
558 if not compiler_fname_list:
559 print 'Could not locate C compiler - fetch failed.'
561 if len(compiler_fname_list) != 1:
562 print col.Color(col.RED, 'Warning, ambiguous toolchains: %s' %
563 ', '.join(compiler_fname_list))
564 toolchain = Toolchain(compiler_fname_list[0], True, True)
566 # Make sure that it will be found by buildman
567 if not self.TestSettingsHasPath(dirpath):
568 print ("Adding 'download' to config file '%s'" %
569 bsettings.config_fname)
570 bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest)