1 # Copyright (c) 2012 The Chromium OS Authors.
3 # SPDX-License-Identifier: GPL-2.0+
8 from HTMLParser import HTMLParser
17 (PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
18 PRIORITY_CALC) = range(4)
20 # Simple class to collect links from a page
21 class MyHTMLParser(HTMLParser):
22 def __init__(self, arch):
23 """Create a new parser
25 After the parser runs, self.links will be set to a list of the links
26 to .xz archives found in the page, and self.arch_link will be set to
27 the one for the given architecture (or None if not found).
30 arch: Architecture to search for
32 HTMLParser.__init__(self)
35 self._match = '_%s-' % arch
37 def handle_starttag(self, tag, attrs):
39 for tag, value in attrs:
41 if value and value.endswith('.xz'):
42 self.links.append(value)
43 if self._match in value:
44 self.arch_link = value
51 gcc: Full path to C compiler
52 path: Directory path containing C compiler
53 cross: Cross compile string, e.g. 'arm-linux-'
54 arch: Architecture of toolchain as determined from the first
55 component of the filename. E.g. arm-linux-gcc becomes arm
56 priority: Toolchain priority (0=highest, 20=lowest)
58 def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,
60 """Create a new toolchain object.
63 fname: Filename of the gcc component
64 test: True to run the toolchain to test it
65 verbose: True to print out the information
66 priority: Priority to use for this toolchain, or PRIORITY_CALC to
70 self.path = os.path.dirname(fname)
72 # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
73 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
74 basename = os.path.basename(fname)
75 pos = basename.rfind('-')
76 self.cross = basename[:pos + 1] if pos != -1 else ''
78 # The architecture is the first part of the name
79 pos = self.cross.find('-')
83 self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
85 env = self.MakeEnvironment(False)
87 # As a basic sanity check, run the C compiler with --version
88 cmd = [fname, '--version']
89 if priority == PRIORITY_CALC:
90 self.priority = self.GetPriority(fname)
92 self.priority = priority
94 result = command.RunPipe([cmd], capture=True, env=env,
96 self.ok = result.return_code == 0
98 print 'Tool chain test: ',
100 print "OK, arch='%s', priority %d" % (self.arch,
104 print 'Command: ', cmd
110 def GetPriority(self, fname):
111 """Return the priority of the toolchain.
113 Toolchains are ranked according to their suitability by their
117 fname: Filename of toolchain
119 Priority of toolchain, PRIORITY_CALC=highest, 20=lowest.
121 priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
122 '-none-linux-gnueabi', '-uclinux', '-none-eabi',
123 '-gentoo-linux-gnu', '-linux-gnueabi', '-le-linux', '-uclinux']
124 for prio in range(len(priority_list)):
125 if priority_list[prio] in fname:
126 return PRIORITY_CALC + prio
127 return PRIORITY_CALC + prio
129 def MakeEnvironment(self, full_path):
130 """Returns an environment for using the toolchain.
132 Thie takes the current environment and adds CROSS_COMPILE so that
133 the tool chain will operate correctly.
136 full_path: Return the full path in CROSS_COMPILE and don't set
139 env = dict(os.environ)
141 env['CROSS_COMPILE'] = os.path.join(self.path, self.cross)
143 env['CROSS_COMPILE'] = self.cross
144 env['PATH'] = self.path + ':' + env['PATH']
150 """Manage a list of toolchains for building U-Boot
152 We select one toolchain for each architecture type
155 toolchains: Dict of Toolchain objects, keyed by architecture name
156 prefixes: Dict of prefixes to check, keyed by architecture. This can
157 be a full path and toolchain prefix, for example
158 {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of
159 something on the search path, for example
160 {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported.
161 paths: List of paths to check for toolchains (may contain wildcards)
168 self._make_flags = dict(bsettings.GetItems('make-flags'))
170 def GetPathList(self):
171 """Get a list of available toolchain paths
174 List of strings, each a path to a toolchain mentioned in the
175 [toolchain] section of the settings file.
177 toolchains = bsettings.GetItems('toolchain')
179 print ('Warning: No tool chains - please add a [toolchain] section'
180 ' to your buildman config file %s. See README for details' %
181 bsettings.config_fname)
184 for name, value in toolchains:
186 paths += glob.glob(value)
191 def GetSettings(self):
192 self.prefixes = bsettings.GetItems('toolchain-prefix')
193 self.paths += self.GetPathList()
195 def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC,
197 """Add a toolchain to our list
199 We select the given toolchain as our preferred one for its
200 architecture if it is a higher priority than the others.
203 fname: Filename of toolchain's gcc driver
204 test: True to run the toolchain to test it
205 priority: Priority to use for this toolchain
206 arch: Toolchain architecture, or None if not known
208 toolchain = Toolchain(fname, test, verbose, priority, arch)
209 add_it = toolchain.ok
210 if toolchain.arch in self.toolchains:
211 add_it = (toolchain.priority <
212 self.toolchains[toolchain.arch].priority)
214 self.toolchains[toolchain.arch] = toolchain
216 print ("Toolchain '%s' at priority %d will be ignored because "
217 "another toolchain for arch '%s' has priority %d" %
218 (toolchain.gcc, toolchain.priority, toolchain.arch,
219 self.toolchains[toolchain.arch].priority))
221 def ScanPath(self, path, verbose):
222 """Scan a path for a valid toolchain
226 verbose: True to print out progress information
228 Filename of C compiler if found, else None
231 for subdir in ['.', 'bin', 'usr/bin']:
232 dirname = os.path.join(path, subdir)
233 if verbose: print " - looking in '%s'" % dirname
234 for fname in glob.glob(dirname + '/*gcc'):
235 if verbose: print " - found '%s'" % fname
239 def ScanPathEnv(self, fname):
240 """Scan the PATH environment variable for a given filename.
243 fname: Filename to scan for
245 List of matching pathanames, or [] if none
248 for path in os.environ["PATH"].split(os.pathsep):
249 path = path.strip('"')
250 pathname = os.path.join(path, fname)
251 if os.path.exists(pathname):
252 pathname_list.append(pathname)
255 def Scan(self, verbose):
256 """Scan for available toolchains and select the best for each arch.
258 We look for all the toolchains we can file, figure out the
259 architecture for each, and whether it works. Then we select the
260 highest priority toolchain for each arch.
263 verbose: True to print out progress information
265 if verbose: print 'Scanning for tool chains'
266 for name, value in self.prefixes:
267 if verbose: print " - scanning prefix '%s'" % value
268 if os.path.exists(value):
269 self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name)
271 fname = value + 'gcc'
272 if os.path.exists(fname):
273 self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name)
275 fname_list = self.ScanPathEnv(fname)
277 self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
279 raise ValueError, ("No tool chain found for prefix '%s'" %
281 for path in self.paths:
282 if verbose: print " - scanning path '%s'" % path
283 fnames = self.ScanPath(path, verbose)
285 self.Add(fname, True, verbose)
288 """List out the selected toolchains for each architecture"""
289 print 'List of available toolchains (%d):' % len(self.toolchains)
290 if len(self.toolchains):
291 for key, value in sorted(self.toolchains.iteritems()):
292 print '%-10s: %s' % (key, value.gcc)
296 def Select(self, arch):
297 """Returns the toolchain for a given architecture
300 args: Name of architecture (e.g. 'arm', 'ppc_8xx')
303 toolchain object, or None if none found
305 for tag, value in bsettings.GetItems('toolchain-alias'):
307 for alias in value.split():
308 if alias in self.toolchains:
309 return self.toolchains[alias]
311 if not arch in self.toolchains:
312 raise ValueError, ("No tool chain found for arch '%s'" % arch)
313 return self.toolchains[arch]
315 def ResolveReferences(self, var_dict, args):
316 """Resolve variable references in a string
318 This converts ${blah} within the string to the value of blah.
319 This function works recursively.
322 var_dict: Dictionary containing variables and their values
323 args: String containing make arguments
327 >>> bsettings.Setup()
328 >>> tcs = Toolchains()
329 >>> tcs.Add('fred', False)
330 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
332 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
334 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
335 'this=OBLIQUE_setfi2ndrstnd'
337 re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
340 m = re_var.search(args)
343 lookup = m.group(0)[2:-1]
344 value = var_dict.get(lookup, '')
345 args = args[:m.start(0)] + value + args[m.end(0):]
348 def GetMakeArguments(self, board):
349 """Returns 'make' arguments for a given board
351 The flags are in a section called 'make-flags'. Flags are named
352 after the target they represent, for example snapper9260=TESTING=1
353 will pass TESTING=1 to make when building the snapper9260 board.
355 References to other boards can be added in the string also. For
359 at91-boards=ENABLE_AT91_TEST=1
360 snapper9260=${at91-boards} BUILD_TAG=442
361 snapper9g45=${at91-boards} BUILD_TAG=443
363 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
364 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
366 A special 'target' variable is set to the board target.
369 board: Board object for the board to check.
371 'make' flags for that board, or '' if none
373 self._make_flags['target'] = board.target
374 arg_str = self.ResolveReferences(self._make_flags,
375 self._make_flags.get(board.target, ''))
376 args = arg_str.split(' ')
385 def LocateArchUrl(self, fetch_arch):
386 """Find a toolchain available online
388 Look in standard places for available toolchains. At present the
389 only standard place is at kernel.org.
392 arch: Architecture to look for, or 'list' for all
394 If fetch_arch is 'list', a tuple:
395 Machine architecture (e.g. x86_64)
398 URL containing this toolchain, if avaialble, else None
400 arch = command.OutputOneLine('uname', '-m')
401 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
402 versions = ['4.9.0', '4.6.3', '4.6.2', '4.5.1', '4.2.4']
404 for version in versions:
405 url = '%s/%s/%s/' % (base, arch, version)
406 print 'Checking: %s' % url
407 response = urllib2.urlopen(url)
408 html = response.read()
409 parser = MyHTMLParser(fetch_arch)
411 if fetch_arch == 'list':
412 links += parser.links
413 elif parser.arch_link:
414 return url + parser.arch_link
415 if fetch_arch == 'list':
419 def Download(self, url):
420 """Download a file to a temporary directory
426 Temporary directory name
427 Full path to the downloaded archive file in that directory,
428 or None if there was an error while downloading
430 print 'Downloading: %s' % url
431 leaf = url.split('/')[-1]
432 tmpdir = tempfile.mkdtemp('.buildman')
433 response = urllib2.urlopen(url)
434 fname = os.path.join(tmpdir, leaf)
435 fd = open(fname, 'wb')
436 meta = response.info()
437 size = int(meta.getheaders('Content-Length')[0])
442 # Read the file in chunks and show progress as we go
444 buffer = response.read(block_size)
446 print chr(8) * (len(status) + 1), '\r',
451 status = r'%10d MiB [%3d%%]' % (done / 1024 / 1024,
453 status = status + chr(8) * (len(status) + 1)
458 print 'Error, failed to download'
463 def Unpack(self, fname, dest):
467 fname: Filename to unpack
468 dest: Destination directory
470 Directory name of the first entry in the archive, without the
473 stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)
474 return stdout.splitlines()[0][:-1]
476 def TestSettingsHasPath(self, path):
477 """Check if builmand will find this toolchain
480 True if the path is in settings, False if not
482 paths = self.GetPathList()
486 """List architectures with available toolchains to download"""
487 host_arch, archives = self.LocateArchUrl('list')
488 re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*')
490 for archive in archives:
491 # Remove the host architecture from the start
492 arch = re_arch.match(archive[len(host_arch):])
494 arch_set.add(arch.group(1))
495 return sorted(arch_set)
497 def FetchAndInstall(self, arch):
498 """Fetch and install a new toolchain
501 Architecture to fetch, or 'list' to list
503 # Fist get the URL for this architecture
504 url = self.LocateArchUrl(arch)
506 print ("Cannot find toolchain for arch '%s' - use 'list' to list" %
509 home = os.environ['HOME']
510 dest = os.path.join(home, '.buildman-toolchains')
511 if not os.path.exists(dest):
514 # Download the tar file for this toolchain and unpack it
515 tmpdir, tarfile = self.Download(url)
518 print 'Unpacking to: %s' % dest,
520 path = self.Unpack(tarfile, dest)
525 # Check that the toolchain works
527 dirpath = os.path.join(dest, path)
528 compiler_fname_list = self.ScanPath(dirpath, True)
529 if not compiler_fname_list:
530 print 'Could not locate C compiler - fetch failed.'
532 if len(compiler_fname_list) != 1:
533 print ('Internal error, ambiguous toolchains: %s' %
534 (', '.join(compiler_fname)))
536 toolchain = Toolchain(compiler_fname_list[0], True, True)
538 # Make sure that it will be found by buildman
539 if not self.TestSettingsHasPath(dirpath):
540 print ("Adding 'download' to config file '%s'" %
541 bsettings.config_fname)
542 tools_dir = os.path.dirname(dirpath)
543 bsettings.SetItem('toolchain', 'download', '%s/*' % tools_dir)