1 # Copyright (c) 2012 The Chromium OS Authors.
3 # SPDX-License-Identifier: GPL-2.0+
8 from HTMLParser import HTMLParser
18 (PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
19 PRIORITY_CALC) = range(4)
21 # Simple class to collect links from a page
22 class MyHTMLParser(HTMLParser):
23 def __init__(self, arch):
24 """Create a new parser
26 After the parser runs, self.links will be set to a list of the links
27 to .xz archives found in the page, and self.arch_link will be set to
28 the one for the given architecture (or None if not found).
31 arch: Architecture to search for
33 HTMLParser.__init__(self)
36 self._match = '_%s-' % arch
38 def handle_starttag(self, tag, attrs):
40 for tag, value in attrs:
42 if value and value.endswith('.xz'):
43 self.links.append(value)
44 if self._match in value:
45 self.arch_link = value
52 gcc: Full path to C compiler
53 path: Directory path containing C compiler
54 cross: Cross compile string, e.g. 'arm-linux-'
55 arch: Architecture of toolchain as determined from the first
56 component of the filename. E.g. arm-linux-gcc becomes arm
57 priority: Toolchain priority (0=highest, 20=lowest)
59 def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,
61 """Create a new toolchain object.
64 fname: Filename of the gcc component
65 test: True to run the toolchain to test it
66 verbose: True to print out the information
67 priority: Priority to use for this toolchain, or PRIORITY_CALC to
71 self.path = os.path.dirname(fname)
73 # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
74 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
75 basename = os.path.basename(fname)
76 pos = basename.rfind('-')
77 self.cross = basename[:pos + 1] if pos != -1 else ''
79 # The architecture is the first part of the name
80 pos = self.cross.find('-')
84 self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
86 env = self.MakeEnvironment(False)
88 # As a basic sanity check, run the C compiler with --version
89 cmd = [fname, '--version']
90 if priority == PRIORITY_CALC:
91 self.priority = self.GetPriority(fname)
93 self.priority = priority
95 result = command.RunPipe([cmd], capture=True, env=env,
97 self.ok = result.return_code == 0
99 print 'Tool chain test: ',
101 print "OK, arch='%s', priority %d" % (self.arch,
105 print 'Command: ', cmd
111 def GetPriority(self, fname):
112 """Return the priority of the toolchain.
114 Toolchains are ranked according to their suitability by their
118 fname: Filename of toolchain
120 Priority of toolchain, PRIORITY_CALC=highest, 20=lowest.
122 priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
123 '-none-linux-gnueabi', '-uclinux', '-none-eabi',
124 '-gentoo-linux-gnu', '-linux-gnueabi', '-le-linux', '-uclinux']
125 for prio in range(len(priority_list)):
126 if priority_list[prio] in fname:
127 return PRIORITY_CALC + prio
128 return PRIORITY_CALC + prio
130 def MakeEnvironment(self, full_path):
131 """Returns an environment for using the toolchain.
133 Thie takes the current environment and adds CROSS_COMPILE so that
134 the tool chain will operate correctly.
137 full_path: Return the full path in CROSS_COMPILE and don't set
140 env = dict(os.environ)
142 env['CROSS_COMPILE'] = os.path.join(self.path, self.cross)
144 env['CROSS_COMPILE'] = self.cross
145 env['PATH'] = self.path + ':' + env['PATH']
151 """Manage a list of toolchains for building U-Boot
153 We select one toolchain for each architecture type
156 toolchains: Dict of Toolchain objects, keyed by architecture name
157 prefixes: Dict of prefixes to check, keyed by architecture. This can
158 be a full path and toolchain prefix, for example
159 {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of
160 something on the search path, for example
161 {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported.
162 paths: List of paths to check for toolchains (may contain wildcards)
169 self._make_flags = dict(bsettings.GetItems('make-flags'))
171 def GetPathList(self, show_warning=True):
172 """Get a list of available toolchain paths
175 show_warning: True to show a warning if there are no tool chains.
178 List of strings, each a path to a toolchain mentioned in the
179 [toolchain] section of the settings file.
181 toolchains = bsettings.GetItems('toolchain')
182 if show_warning and not toolchains:
183 print ("Warning: No tool chains. Please run 'buildman "
184 "--fetch-arch all' to download all available toolchains, or "
185 "add a [toolchain] section to your buildman config file "
186 "%s. See README for details" %
187 bsettings.config_fname)
190 for name, value in toolchains:
192 paths += glob.glob(value)
197 def GetSettings(self, show_warning=True):
198 """Get toolchain settings from the settings file.
201 show_warning: True to show a warning if there are no tool chains.
203 self.prefixes = bsettings.GetItems('toolchain-prefix')
204 self.paths += self.GetPathList(show_warning)
206 def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC,
208 """Add a toolchain to our list
210 We select the given toolchain as our preferred one for its
211 architecture if it is a higher priority than the others.
214 fname: Filename of toolchain's gcc driver
215 test: True to run the toolchain to test it
216 priority: Priority to use for this toolchain
217 arch: Toolchain architecture, or None if not known
219 toolchain = Toolchain(fname, test, verbose, priority, arch)
220 add_it = toolchain.ok
221 if toolchain.arch in self.toolchains:
222 add_it = (toolchain.priority <
223 self.toolchains[toolchain.arch].priority)
225 self.toolchains[toolchain.arch] = toolchain
227 print ("Toolchain '%s' at priority %d will be ignored because "
228 "another toolchain for arch '%s' has priority %d" %
229 (toolchain.gcc, toolchain.priority, toolchain.arch,
230 self.toolchains[toolchain.arch].priority))
232 def ScanPath(self, path, verbose):
233 """Scan a path for a valid toolchain
237 verbose: True to print out progress information
239 Filename of C compiler if found, else None
242 for subdir in ['.', 'bin', 'usr/bin']:
243 dirname = os.path.join(path, subdir)
244 if verbose: print " - looking in '%s'" % dirname
245 for fname in glob.glob(dirname + '/*gcc'):
246 if verbose: print " - found '%s'" % fname
250 def ScanPathEnv(self, fname):
251 """Scan the PATH environment variable for a given filename.
254 fname: Filename to scan for
256 List of matching pathanames, or [] if none
259 for path in os.environ["PATH"].split(os.pathsep):
260 path = path.strip('"')
261 pathname = os.path.join(path, fname)
262 if os.path.exists(pathname):
263 pathname_list.append(pathname)
266 def Scan(self, verbose):
267 """Scan for available toolchains and select the best for each arch.
269 We look for all the toolchains we can file, figure out the
270 architecture for each, and whether it works. Then we select the
271 highest priority toolchain for each arch.
274 verbose: True to print out progress information
276 if verbose: print 'Scanning for tool chains'
277 for name, value in self.prefixes:
278 if verbose: print " - scanning prefix '%s'" % value
279 if os.path.exists(value):
280 self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name)
282 fname = value + 'gcc'
283 if os.path.exists(fname):
284 self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name)
286 fname_list = self.ScanPathEnv(fname)
288 self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
290 raise ValueError, ("No tool chain found for prefix '%s'" %
292 for path in self.paths:
293 if verbose: print " - scanning path '%s'" % path
294 fnames = self.ScanPath(path, verbose)
296 self.Add(fname, True, verbose)
299 """List out the selected toolchains for each architecture"""
300 col = terminal.Color()
301 print col.Color(col.BLUE, 'List of available toolchains (%d):' %
302 len(self.toolchains))
303 if len(self.toolchains):
304 for key, value in sorted(self.toolchains.iteritems()):
305 print '%-10s: %s' % (key, value.gcc)
309 def Select(self, arch):
310 """Returns the toolchain for a given architecture
313 args: Name of architecture (e.g. 'arm', 'ppc_8xx')
316 toolchain object, or None if none found
318 for tag, value in bsettings.GetItems('toolchain-alias'):
320 for alias in value.split():
321 if alias in self.toolchains:
322 return self.toolchains[alias]
324 if not arch in self.toolchains:
325 raise ValueError, ("No tool chain found for arch '%s'" % arch)
326 return self.toolchains[arch]
328 def ResolveReferences(self, var_dict, args):
329 """Resolve variable references in a string
331 This converts ${blah} within the string to the value of blah.
332 This function works recursively.
335 var_dict: Dictionary containing variables and their values
336 args: String containing make arguments
340 >>> bsettings.Setup()
341 >>> tcs = Toolchains()
342 >>> tcs.Add('fred', False)
343 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
345 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
347 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
348 'this=OBLIQUE_setfi2ndrstnd'
350 re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
353 m = re_var.search(args)
356 lookup = m.group(0)[2:-1]
357 value = var_dict.get(lookup, '')
358 args = args[:m.start(0)] + value + args[m.end(0):]
361 def GetMakeArguments(self, board):
362 """Returns 'make' arguments for a given board
364 The flags are in a section called 'make-flags'. Flags are named
365 after the target they represent, for example snapper9260=TESTING=1
366 will pass TESTING=1 to make when building the snapper9260 board.
368 References to other boards can be added in the string also. For
372 at91-boards=ENABLE_AT91_TEST=1
373 snapper9260=${at91-boards} BUILD_TAG=442
374 snapper9g45=${at91-boards} BUILD_TAG=443
376 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
377 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
379 A special 'target' variable is set to the board target.
382 board: Board object for the board to check.
384 'make' flags for that board, or '' if none
386 self._make_flags['target'] = board.target
387 arg_str = self.ResolveReferences(self._make_flags,
388 self._make_flags.get(board.target, ''))
389 args = arg_str.split(' ')
398 def LocateArchUrl(self, fetch_arch):
399 """Find a toolchain available online
401 Look in standard places for available toolchains. At present the
402 only standard place is at kernel.org.
405 arch: Architecture to look for, or 'list' for all
407 If fetch_arch is 'list', a tuple:
408 Machine architecture (e.g. x86_64)
411 URL containing this toolchain, if avaialble, else None
413 arch = command.OutputOneLine('uname', '-m')
414 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
415 versions = ['4.9.0', '4.6.3', '4.6.2', '4.5.1', '4.2.4']
417 for version in versions:
418 url = '%s/%s/%s/' % (base, arch, version)
419 print 'Checking: %s' % url
420 response = urllib2.urlopen(url)
421 html = response.read()
422 parser = MyHTMLParser(fetch_arch)
424 if fetch_arch == 'list':
425 links += parser.links
426 elif parser.arch_link:
427 return url + parser.arch_link
428 if fetch_arch == 'list':
432 def Download(self, url):
433 """Download a file to a temporary directory
439 Temporary directory name
440 Full path to the downloaded archive file in that directory,
441 or None if there was an error while downloading
443 print 'Downloading: %s' % url
444 leaf = url.split('/')[-1]
445 tmpdir = tempfile.mkdtemp('.buildman')
446 response = urllib2.urlopen(url)
447 fname = os.path.join(tmpdir, leaf)
448 fd = open(fname, 'wb')
449 meta = response.info()
450 size = int(meta.getheaders('Content-Length')[0])
455 # Read the file in chunks and show progress as we go
457 buffer = response.read(block_size)
459 print chr(8) * (len(status) + 1), '\r',
464 status = r'%10d MiB [%3d%%]' % (done / 1024 / 1024,
466 status = status + chr(8) * (len(status) + 1)
471 print 'Error, failed to download'
476 def Unpack(self, fname, dest):
480 fname: Filename to unpack
481 dest: Destination directory
483 Directory name of the first entry in the archive, without the
486 stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)
487 return stdout.splitlines()[0][:-1]
489 def TestSettingsHasPath(self, path):
490 """Check if buildman will find this toolchain
493 True if the path is in settings, False if not
495 paths = self.GetPathList(False)
499 """List architectures with available toolchains to download"""
500 host_arch, archives = self.LocateArchUrl('list')
501 re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*')
503 for archive in archives:
504 # Remove the host architecture from the start
505 arch = re_arch.match(archive[len(host_arch):])
507 arch_set.add(arch.group(1))
508 return sorted(arch_set)
510 def FetchAndInstall(self, arch):
511 """Fetch and install a new toolchain
514 Architecture to fetch, or 'list' to list
516 # Fist get the URL for this architecture
517 col = terminal.Color()
518 print col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch)
519 url = self.LocateArchUrl(arch)
521 print ("Cannot find toolchain for arch '%s' - use 'list' to list" %
524 home = os.environ['HOME']
525 dest = os.path.join(home, '.buildman-toolchains')
526 if not os.path.exists(dest):
529 # Download the tar file for this toolchain and unpack it
530 tmpdir, tarfile = self.Download(url)
533 print col.Color(col.GREEN, 'Unpacking to: %s' % dest),
535 path = self.Unpack(tarfile, dest)
540 # Check that the toolchain works
541 print col.Color(col.GREEN, 'Testing')
542 dirpath = os.path.join(dest, path)
543 compiler_fname_list = self.ScanPath(dirpath, True)
544 if not compiler_fname_list:
545 print 'Could not locate C compiler - fetch failed.'
547 if len(compiler_fname_list) != 1:
548 print col.Color(col.RED, 'Warning, ambiguous toolchains: %s' %
549 ', '.join(compiler_fname_list))
550 toolchain = Toolchain(compiler_fname_list[0], True, True)
552 # Make sure that it will be found by buildman
553 if not self.TestSettingsHasPath(dirpath):
554 print ("Adding 'download' to config file '%s'" %
555 bsettings.config_fname)
556 bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest)