1 # Copyright (c) 2012 The Chromium OS Authors.
3 # SPDX-License-Identifier: GPL-2.0+
8 from HTMLParser import HTMLParser
18 (PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
19 PRIORITY_CALC) = range(4)
21 # Simple class to collect links from a page
22 class MyHTMLParser(HTMLParser):
23 def __init__(self, arch):
24 """Create a new parser
26 After the parser runs, self.links will be set to a list of the links
27 to .xz archives found in the page, and self.arch_link will be set to
28 the one for the given architecture (or None if not found).
31 arch: Architecture to search for
33 HTMLParser.__init__(self)
36 self._match = '_%s-' % arch
38 def handle_starttag(self, tag, attrs):
40 for tag, value in attrs:
42 if value and value.endswith('.xz'):
43 self.links.append(value)
44 if self._match in value:
45 self.arch_link = value
52 gcc: Full path to C compiler
53 path: Directory path containing C compiler
54 cross: Cross compile string, e.g. 'arm-linux-'
55 arch: Architecture of toolchain as determined from the first
56 component of the filename. E.g. arm-linux-gcc becomes arm
57 priority: Toolchain priority (0=highest, 20=lowest)
59 def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,
61 """Create a new toolchain object.
64 fname: Filename of the gcc component
65 test: True to run the toolchain to test it
66 verbose: True to print out the information
67 priority: Priority to use for this toolchain, or PRIORITY_CALC to
71 self.path = os.path.dirname(fname)
73 # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
74 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
75 basename = os.path.basename(fname)
76 pos = basename.rfind('-')
77 self.cross = basename[:pos + 1] if pos != -1 else ''
79 # The architecture is the first part of the name
80 pos = self.cross.find('-')
84 self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
86 env = self.MakeEnvironment(False)
88 # As a basic sanity check, run the C compiler with --version
89 cmd = [fname, '--version']
90 if priority == PRIORITY_CALC:
91 self.priority = self.GetPriority(fname)
93 self.priority = priority
95 result = command.RunPipe([cmd], capture=True, env=env,
97 self.ok = result.return_code == 0
99 print 'Tool chain test: ',
101 print "OK, arch='%s', priority %d" % (self.arch,
105 print 'Command: ', cmd
111 def GetPriority(self, fname):
112 """Return the priority of the toolchain.
114 Toolchains are ranked according to their suitability by their
118 fname: Filename of toolchain
120 Priority of toolchain, PRIORITY_CALC=highest, 20=lowest.
122 priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
123 '-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux',
124 '-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi',
125 '-linux-gnueabihf', '-le-linux', '-uclinux']
126 for prio in range(len(priority_list)):
127 if priority_list[prio] in fname:
128 return PRIORITY_CALC + prio
129 return PRIORITY_CALC + prio
131 def GetWrapper(self, show_warning=True):
132 """Get toolchain wrapper from the setting file.
135 for name, value in bsettings.GetItems('toolchain-wrapper'):
137 print "Warning: Wrapper not found"
143 def MakeEnvironment(self, full_path):
144 """Returns an environment for using the toolchain.
146 Thie takes the current environment and adds CROSS_COMPILE so that
147 the tool chain will operate correctly.
150 full_path: Return the full path in CROSS_COMPILE and don't set
153 env = dict(os.environ)
154 wrapper = self.GetWrapper()
157 env['CROSS_COMPILE'] = wrapper + os.path.join(self.path, self.cross)
159 env['CROSS_COMPILE'] = wrapper + self.cross
160 env['PATH'] = self.path + ':' + env['PATH']
166 """Manage a list of toolchains for building U-Boot
168 We select one toolchain for each architecture type
171 toolchains: Dict of Toolchain objects, keyed by architecture name
172 prefixes: Dict of prefixes to check, keyed by architecture. This can
173 be a full path and toolchain prefix, for example
174 {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of
175 something on the search path, for example
176 {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported.
177 paths: List of paths to check for toolchains (may contain wildcards)
184 self._make_flags = dict(bsettings.GetItems('make-flags'))
186 def GetPathList(self, show_warning=True):
187 """Get a list of available toolchain paths
190 show_warning: True to show a warning if there are no tool chains.
193 List of strings, each a path to a toolchain mentioned in the
194 [toolchain] section of the settings file.
196 toolchains = bsettings.GetItems('toolchain')
197 if show_warning and not toolchains:
198 print ("Warning: No tool chains. Please run 'buildman "
199 "--fetch-arch all' to download all available toolchains, or "
200 "add a [toolchain] section to your buildman config file "
201 "%s. See README for details" %
202 bsettings.config_fname)
205 for name, value in toolchains:
207 paths += glob.glob(value)
212 def GetSettings(self, show_warning=True):
213 """Get toolchain settings from the settings file.
216 show_warning: True to show a warning if there are no tool chains.
218 self.prefixes = bsettings.GetItems('toolchain-prefix')
219 self.paths += self.GetPathList(show_warning)
221 def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC,
223 """Add a toolchain to our list
225 We select the given toolchain as our preferred one for its
226 architecture if it is a higher priority than the others.
229 fname: Filename of toolchain's gcc driver
230 test: True to run the toolchain to test it
231 priority: Priority to use for this toolchain
232 arch: Toolchain architecture, or None if not known
234 toolchain = Toolchain(fname, test, verbose, priority, arch)
235 add_it = toolchain.ok
236 if toolchain.arch in self.toolchains:
237 add_it = (toolchain.priority <
238 self.toolchains[toolchain.arch].priority)
240 self.toolchains[toolchain.arch] = toolchain
242 print ("Toolchain '%s' at priority %d will be ignored because "
243 "another toolchain for arch '%s' has priority %d" %
244 (toolchain.gcc, toolchain.priority, toolchain.arch,
245 self.toolchains[toolchain.arch].priority))
247 def ScanPath(self, path, verbose):
248 """Scan a path for a valid toolchain
252 verbose: True to print out progress information
254 Filename of C compiler if found, else None
257 for subdir in ['.', 'bin', 'usr/bin']:
258 dirname = os.path.join(path, subdir)
259 if verbose: print " - looking in '%s'" % dirname
260 for fname in glob.glob(dirname + '/*gcc'):
261 if verbose: print " - found '%s'" % fname
265 def ScanPathEnv(self, fname):
266 """Scan the PATH environment variable for a given filename.
269 fname: Filename to scan for
271 List of matching pathanames, or [] if none
274 for path in os.environ["PATH"].split(os.pathsep):
275 path = path.strip('"')
276 pathname = os.path.join(path, fname)
277 if os.path.exists(pathname):
278 pathname_list.append(pathname)
281 def Scan(self, verbose):
282 """Scan for available toolchains and select the best for each arch.
284 We look for all the toolchains we can file, figure out the
285 architecture for each, and whether it works. Then we select the
286 highest priority toolchain for each arch.
289 verbose: True to print out progress information
291 if verbose: print 'Scanning for tool chains'
292 for name, value in self.prefixes:
293 if verbose: print " - scanning prefix '%s'" % value
294 if os.path.exists(value):
295 self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name)
297 fname = value + 'gcc'
298 if os.path.exists(fname):
299 self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name)
301 fname_list = self.ScanPathEnv(fname)
303 self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
305 raise ValueError, ("No tool chain found for prefix '%s'" %
307 for path in self.paths:
308 if verbose: print " - scanning path '%s'" % path
309 fnames = self.ScanPath(path, verbose)
311 self.Add(fname, True, verbose)
314 """List out the selected toolchains for each architecture"""
315 col = terminal.Color()
316 print col.Color(col.BLUE, 'List of available toolchains (%d):' %
317 len(self.toolchains))
318 if len(self.toolchains):
319 for key, value in sorted(self.toolchains.iteritems()):
320 print '%-10s: %s' % (key, value.gcc)
324 def Select(self, arch):
325 """Returns the toolchain for a given architecture
328 args: Name of architecture (e.g. 'arm', 'ppc_8xx')
331 toolchain object, or None if none found
333 for tag, value in bsettings.GetItems('toolchain-alias'):
335 for alias in value.split():
336 if alias in self.toolchains:
337 return self.toolchains[alias]
339 if not arch in self.toolchains:
340 raise ValueError, ("No tool chain found for arch '%s'" % arch)
341 return self.toolchains[arch]
343 def ResolveReferences(self, var_dict, args):
344 """Resolve variable references in a string
346 This converts ${blah} within the string to the value of blah.
347 This function works recursively.
350 var_dict: Dictionary containing variables and their values
351 args: String containing make arguments
355 >>> bsettings.Setup()
356 >>> tcs = Toolchains()
357 >>> tcs.Add('fred', False)
358 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
360 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
362 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
363 'this=OBLIQUE_setfi2ndrstnd'
365 re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
368 m = re_var.search(args)
371 lookup = m.group(0)[2:-1]
372 value = var_dict.get(lookup, '')
373 args = args[:m.start(0)] + value + args[m.end(0):]
376 def GetMakeArguments(self, board):
377 """Returns 'make' arguments for a given board
379 The flags are in a section called 'make-flags'. Flags are named
380 after the target they represent, for example snapper9260=TESTING=1
381 will pass TESTING=1 to make when building the snapper9260 board.
383 References to other boards can be added in the string also. For
387 at91-boards=ENABLE_AT91_TEST=1
388 snapper9260=${at91-boards} BUILD_TAG=442
389 snapper9g45=${at91-boards} BUILD_TAG=443
391 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
392 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
394 A special 'target' variable is set to the board target.
397 board: Board object for the board to check.
399 'make' flags for that board, or '' if none
401 self._make_flags['target'] = board.target
402 arg_str = self.ResolveReferences(self._make_flags,
403 self._make_flags.get(board.target, ''))
404 args = arg_str.split(' ')
413 def LocateArchUrl(self, fetch_arch):
414 """Find a toolchain available online
416 Look in standard places for available toolchains. At present the
417 only standard place is at kernel.org.
420 arch: Architecture to look for, or 'list' for all
422 If fetch_arch is 'list', a tuple:
423 Machine architecture (e.g. x86_64)
426 URL containing this toolchain, if avaialble, else None
428 arch = command.OutputOneLine('uname', '-m')
429 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
430 versions = ['4.9.0', '4.6.3', '4.6.2', '4.5.1', '4.2.4']
432 for version in versions:
433 url = '%s/%s/%s/' % (base, arch, version)
434 print 'Checking: %s' % url
435 response = urllib2.urlopen(url)
436 html = response.read()
437 parser = MyHTMLParser(fetch_arch)
439 if fetch_arch == 'list':
440 links += parser.links
441 elif parser.arch_link:
442 return url + parser.arch_link
443 if fetch_arch == 'list':
447 def Download(self, url):
448 """Download a file to a temporary directory
454 Temporary directory name
455 Full path to the downloaded archive file in that directory,
456 or None if there was an error while downloading
458 print 'Downloading: %s' % url
459 leaf = url.split('/')[-1]
460 tmpdir = tempfile.mkdtemp('.buildman')
461 response = urllib2.urlopen(url)
462 fname = os.path.join(tmpdir, leaf)
463 fd = open(fname, 'wb')
464 meta = response.info()
465 size = int(meta.getheaders('Content-Length')[0])
470 # Read the file in chunks and show progress as we go
472 buffer = response.read(block_size)
474 print chr(8) * (len(status) + 1), '\r',
479 status = r'%10d MiB [%3d%%]' % (done / 1024 / 1024,
481 status = status + chr(8) * (len(status) + 1)
486 print 'Error, failed to download'
491 def Unpack(self, fname, dest):
495 fname: Filename to unpack
496 dest: Destination directory
498 Directory name of the first entry in the archive, without the
501 stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)
502 return stdout.splitlines()[0][:-1]
504 def TestSettingsHasPath(self, path):
505 """Check if buildman will find this toolchain
508 True if the path is in settings, False if not
510 paths = self.GetPathList(False)
514 """List architectures with available toolchains to download"""
515 host_arch, archives = self.LocateArchUrl('list')
516 re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*')
518 for archive in archives:
519 # Remove the host architecture from the start
520 arch = re_arch.match(archive[len(host_arch):])
522 arch_set.add(arch.group(1))
523 return sorted(arch_set)
525 def FetchAndInstall(self, arch):
526 """Fetch and install a new toolchain
529 Architecture to fetch, or 'list' to list
531 # Fist get the URL for this architecture
532 col = terminal.Color()
533 print col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch)
534 url = self.LocateArchUrl(arch)
536 print ("Cannot find toolchain for arch '%s' - use 'list' to list" %
539 home = os.environ['HOME']
540 dest = os.path.join(home, '.buildman-toolchains')
541 if not os.path.exists(dest):
544 # Download the tar file for this toolchain and unpack it
545 tmpdir, tarfile = self.Download(url)
548 print col.Color(col.GREEN, 'Unpacking to: %s' % dest),
550 path = self.Unpack(tarfile, dest)
555 # Check that the toolchain works
556 print col.Color(col.GREEN, 'Testing')
557 dirpath = os.path.join(dest, path)
558 compiler_fname_list = self.ScanPath(dirpath, True)
559 if not compiler_fname_list:
560 print 'Could not locate C compiler - fetch failed.'
562 if len(compiler_fname_list) != 1:
563 print col.Color(col.RED, 'Warning, ambiguous toolchains: %s' %
564 ', '.join(compiler_fname_list))
565 toolchain = Toolchain(compiler_fname_list[0], True, True)
567 # Make sure that it will be found by buildman
568 if not self.TestSettingsHasPath(dirpath):
569 print ("Adding 'download' to config file '%s'" %
570 bsettings.config_fname)
571 bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest)