1 # SPDX-License-Identifier: GPL-2.0+
2 # Copyright (c) 2012 The Chromium OS Authors.
7 from HTMLParser import HTMLParser
17 (PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
18 PRIORITY_CALC) = range(4)
20 # Simple class to collect links from a page
21 class MyHTMLParser(HTMLParser):
22 def __init__(self, arch):
23 """Create a new parser
25 After the parser runs, self.links will be set to a list of the links
26 to .xz archives found in the page, and self.arch_link will be set to
27 the one for the given architecture (or None if not found).
30 arch: Architecture to search for
32 HTMLParser.__init__(self)
35 self.re_arch = re.compile('[-_]%s-' % arch)
37 def handle_starttag(self, tag, attrs):
39 for tag, value in attrs:
41 if value and value.endswith('.xz'):
42 self.links.append(value)
43 if self.re_arch.search(value):
44 self.arch_link = value
51 gcc: Full path to C compiler
52 path: Directory path containing C compiler
53 cross: Cross compile string, e.g. 'arm-linux-'
54 arch: Architecture of toolchain as determined from the first
55 component of the filename. E.g. arm-linux-gcc becomes arm
56 priority: Toolchain priority (0=highest, 20=lowest)
58 def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,
60 """Create a new toolchain object.
63 fname: Filename of the gcc component
64 test: True to run the toolchain to test it
65 verbose: True to print out the information
66 priority: Priority to use for this toolchain, or PRIORITY_CALC to
70 self.path = os.path.dirname(fname)
72 # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
73 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
74 basename = os.path.basename(fname)
75 pos = basename.rfind('-')
76 self.cross = basename[:pos + 1] if pos != -1 else ''
78 # The architecture is the first part of the name
79 pos = self.cross.find('-')
83 self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
85 env = self.MakeEnvironment(False)
87 # As a basic sanity check, run the C compiler with --version
88 cmd = [fname, '--version']
89 if priority == PRIORITY_CALC:
90 self.priority = self.GetPriority(fname)
92 self.priority = priority
94 result = command.RunPipe([cmd], capture=True, env=env,
96 self.ok = result.return_code == 0
98 print 'Tool chain test: ',
100 print "OK, arch='%s', priority %d" % (self.arch,
104 print 'Command: ', cmd
110 def GetPriority(self, fname):
111 """Return the priority of the toolchain.
113 Toolchains are ranked according to their suitability by their
117 fname: Filename of toolchain
119 Priority of toolchain, PRIORITY_CALC=highest, 20=lowest.
121 priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
122 '-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux',
123 '-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi',
124 '-linux-gnueabihf', '-le-linux', '-uclinux']
125 for prio in range(len(priority_list)):
126 if priority_list[prio] in fname:
127 return PRIORITY_CALC + prio
128 return PRIORITY_CALC + prio
130 def GetWrapper(self, show_warning=True):
131 """Get toolchain wrapper from the setting file.
134 for name, value in bsettings.GetItems('toolchain-wrapper'):
136 print "Warning: Wrapper not found"
142 def MakeEnvironment(self, full_path):
143 """Returns an environment for using the toolchain.
145 Thie takes the current environment and adds CROSS_COMPILE so that
146 the tool chain will operate correctly. This also disables localized
147 output and possibly unicode encoded output of all build tools by
151 full_path: Return the full path in CROSS_COMPILE and don't set
154 env = dict(os.environ)
155 wrapper = self.GetWrapper()
158 env['CROSS_COMPILE'] = wrapper + os.path.join(self.path, self.cross)
160 env['CROSS_COMPILE'] = wrapper + self.cross
161 env['PATH'] = self.path + ':' + env['PATH']
169 """Manage a list of toolchains for building U-Boot
171 We select one toolchain for each architecture type
174 toolchains: Dict of Toolchain objects, keyed by architecture name
175 prefixes: Dict of prefixes to check, keyed by architecture. This can
176 be a full path and toolchain prefix, for example
177 {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of
178 something on the search path, for example
179 {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported.
180 paths: List of paths to check for toolchains (may contain wildcards)
187 self._make_flags = dict(bsettings.GetItems('make-flags'))
189 def GetPathList(self, show_warning=True):
190 """Get a list of available toolchain paths
193 show_warning: True to show a warning if there are no tool chains.
196 List of strings, each a path to a toolchain mentioned in the
197 [toolchain] section of the settings file.
199 toolchains = bsettings.GetItems('toolchain')
200 if show_warning and not toolchains:
201 print ("Warning: No tool chains. Please run 'buildman "
202 "--fetch-arch all' to download all available toolchains, or "
203 "add a [toolchain] section to your buildman config file "
204 "%s. See README for details" %
205 bsettings.config_fname)
208 for name, value in toolchains:
210 paths += glob.glob(value)
215 def GetSettings(self, show_warning=True):
216 """Get toolchain settings from the settings file.
219 show_warning: True to show a warning if there are no tool chains.
221 self.prefixes = bsettings.GetItems('toolchain-prefix')
222 self.paths += self.GetPathList(show_warning)
224 def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC,
226 """Add a toolchain to our list
228 We select the given toolchain as our preferred one for its
229 architecture if it is a higher priority than the others.
232 fname: Filename of toolchain's gcc driver
233 test: True to run the toolchain to test it
234 priority: Priority to use for this toolchain
235 arch: Toolchain architecture, or None if not known
237 toolchain = Toolchain(fname, test, verbose, priority, arch)
238 add_it = toolchain.ok
239 if toolchain.arch in self.toolchains:
240 add_it = (toolchain.priority <
241 self.toolchains[toolchain.arch].priority)
243 self.toolchains[toolchain.arch] = toolchain
245 print ("Toolchain '%s' at priority %d will be ignored because "
246 "another toolchain for arch '%s' has priority %d" %
247 (toolchain.gcc, toolchain.priority, toolchain.arch,
248 self.toolchains[toolchain.arch].priority))
250 def ScanPath(self, path, verbose):
251 """Scan a path for a valid toolchain
255 verbose: True to print out progress information
257 Filename of C compiler if found, else None
260 for subdir in ['.', 'bin', 'usr/bin']:
261 dirname = os.path.join(path, subdir)
262 if verbose: print " - looking in '%s'" % dirname
263 for fname in glob.glob(dirname + '/*gcc'):
264 if verbose: print " - found '%s'" % fname
268 def ScanPathEnv(self, fname):
269 """Scan the PATH environment variable for a given filename.
272 fname: Filename to scan for
274 List of matching pathanames, or [] if none
277 for path in os.environ["PATH"].split(os.pathsep):
278 path = path.strip('"')
279 pathname = os.path.join(path, fname)
280 if os.path.exists(pathname):
281 pathname_list.append(pathname)
284 def Scan(self, verbose):
285 """Scan for available toolchains and select the best for each arch.
287 We look for all the toolchains we can file, figure out the
288 architecture for each, and whether it works. Then we select the
289 highest priority toolchain for each arch.
292 verbose: True to print out progress information
294 if verbose: print 'Scanning for tool chains'
295 for name, value in self.prefixes:
296 if verbose: print " - scanning prefix '%s'" % value
297 if os.path.exists(value):
298 self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name)
300 fname = value + 'gcc'
301 if os.path.exists(fname):
302 self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name)
304 fname_list = self.ScanPathEnv(fname)
306 self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
308 raise ValueError, ("No tool chain found for prefix '%s'" %
310 for path in self.paths:
311 if verbose: print " - scanning path '%s'" % path
312 fnames = self.ScanPath(path, verbose)
314 self.Add(fname, True, verbose)
317 """List out the selected toolchains for each architecture"""
318 col = terminal.Color()
319 print col.Color(col.BLUE, 'List of available toolchains (%d):' %
320 len(self.toolchains))
321 if len(self.toolchains):
322 for key, value in sorted(self.toolchains.iteritems()):
323 print '%-10s: %s' % (key, value.gcc)
327 def Select(self, arch):
328 """Returns the toolchain for a given architecture
331 args: Name of architecture (e.g. 'arm', 'ppc_8xx')
334 toolchain object, or None if none found
336 for tag, value in bsettings.GetItems('toolchain-alias'):
338 for alias in value.split():
339 if alias in self.toolchains:
340 return self.toolchains[alias]
342 if not arch in self.toolchains:
343 raise ValueError, ("No tool chain found for arch '%s'" % arch)
344 return self.toolchains[arch]
346 def ResolveReferences(self, var_dict, args):
347 """Resolve variable references in a string
349 This converts ${blah} within the string to the value of blah.
350 This function works recursively.
353 var_dict: Dictionary containing variables and their values
354 args: String containing make arguments
358 >>> bsettings.Setup()
359 >>> tcs = Toolchains()
360 >>> tcs.Add('fred', False)
361 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
363 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
365 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
366 'this=OBLIQUE_setfi2ndrstnd'
368 re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
371 m = re_var.search(args)
374 lookup = m.group(0)[2:-1]
375 value = var_dict.get(lookup, '')
376 args = args[:m.start(0)] + value + args[m.end(0):]
379 def GetMakeArguments(self, board):
380 """Returns 'make' arguments for a given board
382 The flags are in a section called 'make-flags'. Flags are named
383 after the target they represent, for example snapper9260=TESTING=1
384 will pass TESTING=1 to make when building the snapper9260 board.
386 References to other boards can be added in the string also. For
390 at91-boards=ENABLE_AT91_TEST=1
391 snapper9260=${at91-boards} BUILD_TAG=442
392 snapper9g45=${at91-boards} BUILD_TAG=443
394 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
395 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
397 A special 'target' variable is set to the board target.
400 board: Board object for the board to check.
402 'make' flags for that board, or '' if none
404 self._make_flags['target'] = board.target
405 arg_str = self.ResolveReferences(self._make_flags,
406 self._make_flags.get(board.target, ''))
407 args = arg_str.split(' ')
416 def LocateArchUrl(self, fetch_arch):
417 """Find a toolchain available online
419 Look in standard places for available toolchains. At present the
420 only standard place is at kernel.org.
423 arch: Architecture to look for, or 'list' for all
425 If fetch_arch is 'list', a tuple:
426 Machine architecture (e.g. x86_64)
429 URL containing this toolchain, if avaialble, else None
431 arch = command.OutputOneLine('uname', '-m')
432 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
433 versions = ['7.3.0', '6.4.0', '4.9.4']
435 for version in versions:
436 url = '%s/%s/%s/' % (base, arch, version)
437 print 'Checking: %s' % url
438 response = urllib2.urlopen(url)
439 html = response.read()
440 parser = MyHTMLParser(fetch_arch)
442 if fetch_arch == 'list':
443 links += parser.links
444 elif parser.arch_link:
445 return url + parser.arch_link
446 if fetch_arch == 'list':
450 def Download(self, url):
451 """Download a file to a temporary directory
457 Temporary directory name
458 Full path to the downloaded archive file in that directory,
459 or None if there was an error while downloading
461 print 'Downloading: %s' % url
462 leaf = url.split('/')[-1]
463 tmpdir = tempfile.mkdtemp('.buildman')
464 response = urllib2.urlopen(url)
465 fname = os.path.join(tmpdir, leaf)
466 fd = open(fname, 'wb')
467 meta = response.info()
468 size = int(meta.getheaders('Content-Length')[0])
473 # Read the file in chunks and show progress as we go
475 buffer = response.read(block_size)
477 print chr(8) * (len(status) + 1), '\r',
482 status = r'%10d MiB [%3d%%]' % (done / 1024 / 1024,
484 status = status + chr(8) * (len(status) + 1)
489 print 'Error, failed to download'
494 def Unpack(self, fname, dest):
498 fname: Filename to unpack
499 dest: Destination directory
501 Directory name of the first entry in the archive, without the
504 stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)
505 return stdout.splitlines()[0][:-1]
507 def TestSettingsHasPath(self, path):
508 """Check if buildman will find this toolchain
511 True if the path is in settings, False if not
513 paths = self.GetPathList(False)
517 """List architectures with available toolchains to download"""
518 host_arch, archives = self.LocateArchUrl('list')
519 re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*')
521 for archive in archives:
522 # Remove the host architecture from the start
523 arch = re_arch.match(archive[len(host_arch):])
525 arch_set.add(arch.group(1))
526 return sorted(arch_set)
528 def FetchAndInstall(self, arch):
529 """Fetch and install a new toolchain
532 Architecture to fetch, or 'list' to list
534 # Fist get the URL for this architecture
535 col = terminal.Color()
536 print col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch)
537 url = self.LocateArchUrl(arch)
539 print ("Cannot find toolchain for arch '%s' - use 'list' to list" %
542 home = os.environ['HOME']
543 dest = os.path.join(home, '.buildman-toolchains')
544 if not os.path.exists(dest):
547 # Download the tar file for this toolchain and unpack it
548 tmpdir, tarfile = self.Download(url)
551 print col.Color(col.GREEN, 'Unpacking to: %s' % dest),
553 path = self.Unpack(tarfile, dest)
558 # Check that the toolchain works
559 print col.Color(col.GREEN, 'Testing')
560 dirpath = os.path.join(dest, path)
561 compiler_fname_list = self.ScanPath(dirpath, True)
562 if not compiler_fname_list:
563 print 'Could not locate C compiler - fetch failed.'
565 if len(compiler_fname_list) != 1:
566 print col.Color(col.RED, 'Warning, ambiguous toolchains: %s' %
567 ', '.join(compiler_fname_list))
568 toolchain = Toolchain(compiler_fname_list[0], True, True)
570 # Make sure that it will be found by buildman
571 if not self.TestSettingsHasPath(dirpath):
572 print ("Adding 'download' to config file '%s'" %
573 bsettings.config_fname)
574 bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest)