+# SPDX-License-Identifier: GPL-2.0+
# Copyright (c) 2012 The Chromium OS Authors.
#
-# SPDX-License-Identifier: GPL-2.0+
-#
import re
import glob
+from HTMLParser import HTMLParser
import os
+import sys
+import tempfile
+import urllib2
import bsettings
import command
+import terminal
+
+(PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
+ PRIORITY_CALC) = range(4)
+
+# Simple class to collect links from a page
+class MyHTMLParser(HTMLParser):
+ def __init__(self, arch):
+ """Create a new parser
+
+ After the parser runs, self.links will be set to a list of the links
+ to .xz archives found in the page, and self.arch_link will be set to
+ the one for the given architecture (or None if not found).
+
+ Args:
+ arch: Architecture to search for
+ """
+ HTMLParser.__init__(self)
+ self.arch_link = None
+ self.links = []
+ self.re_arch = re.compile('[-_]%s-' % arch)
+
+ def handle_starttag(self, tag, attrs):
+ if tag == 'a':
+ for tag, value in attrs:
+ if tag == 'href':
+ if value and value.endswith('.xz'):
+ self.links.append(value)
+ if self.re_arch.search(value):
+ self.arch_link = value
+
class Toolchain:
"""A single toolchain
cross: Cross compile string, e.g. 'arm-linux-'
arch: Architecture of toolchain as determined from the first
component of the filename. E.g. arm-linux-gcc becomes arm
+ priority: Toolchain priority (0=highest, 20=lowest)
"""
-
- def __init__(self, fname, test, verbose=False):
+ def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,
+ arch=None):
"""Create a new toolchain object.
Args:
fname: Filename of the gcc component
test: True to run the toolchain to test it
+ verbose: True to print out the information
+ priority: Priority to use for this toolchain, or PRIORITY_CALC to
+ calculate it
"""
self.gcc = fname
self.path = os.path.dirname(fname)
- self.cross = os.path.basename(fname)[:-3]
+
+ # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
+ # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
+ basename = os.path.basename(fname)
+ pos = basename.rfind('-')
+ self.cross = basename[:pos + 1] if pos != -1 else ''
+
+ # The architecture is the first part of the name
pos = self.cross.find('-')
- self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
+ if arch:
+ self.arch = arch
+ else:
+ self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
- env = self.MakeEnvironment()
+ env = self.MakeEnvironment(False)
# As a basic sanity check, run the C compiler with --version
cmd = [fname, '--version']
+ if priority == PRIORITY_CALC:
+ self.priority = self.GetPriority(fname)
+ else:
+ self.priority = priority
if test:
result = command.RunPipe([cmd], capture=True, env=env,
raise_on_error=False)
if verbose:
print 'Tool chain test: ',
if self.ok:
- print 'OK'
+ print "OK, arch='%s', priority %d" % (self.arch,
+ self.priority)
else:
print 'BAD'
print 'Command: ', cmd
print result.stderr
else:
self.ok = True
- self.priority = self.GetPriority(fname)
def GetPriority(self, fname):
"""Return the priority of the toolchain.
Args:
fname: Filename of toolchain
Returns:
- Priority of toolchain, 0=highest, 20=lowest.
+ Priority of toolchain, PRIORITY_CALC=highest, 20=lowest.
"""
- priority_list = ['-elf', '-unknown-linux-gnu', '-linux', '-elf',
- '-none-linux-gnueabi', '-uclinux', '-none-eabi',
- '-gentoo-linux-gnu', '-linux-gnueabi', '-le-linux', '-uclinux']
+ priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
+ '-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux',
+ '-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi',
+ '-linux-gnueabihf', '-le-linux', '-uclinux']
for prio in range(len(priority_list)):
if priority_list[prio] in fname:
- return prio
- return prio
+ return PRIORITY_CALC + prio
+ return PRIORITY_CALC + prio
+
+ def GetWrapper(self, show_warning=True):
+ """Get toolchain wrapper from the setting file.
+ """
+ value = ''
+ for name, value in bsettings.GetItems('toolchain-wrapper'):
+ if not value:
+ print "Warning: Wrapper not found"
+ if value:
+ value = value + ' '
+
+ return value
- def MakeEnvironment(self):
+ def MakeEnvironment(self, full_path):
"""Returns an environment for using the toolchain.
- Thie takes the current environment, adds CROSS_COMPILE and
- augments PATH so that the toolchain will operate correctly.
+ Thie takes the current environment and adds CROSS_COMPILE so that
+ the tool chain will operate correctly. This also disables localized
+ output and possibly unicode encoded output of all build tools by
+ adding LC_ALL=C.
+
+ Args:
+ full_path: Return the full path in CROSS_COMPILE and don't set
+ PATH
"""
env = dict(os.environ)
- env['CROSS_COMPILE'] = self.cross
- env['PATH'] += (':' + self.path)
+ wrapper = self.GetWrapper()
+
+ if full_path:
+ env['CROSS_COMPILE'] = wrapper + os.path.join(self.path, self.cross)
+ else:
+ env['CROSS_COMPILE'] = wrapper + self.cross
+ env['PATH'] = self.path + ':' + env['PATH']
+
+ env['LC_ALL'] = 'C'
+
return env
Public members:
toolchains: Dict of Toolchain objects, keyed by architecture name
+ prefixes: Dict of prefixes to check, keyed by architecture. This can
+ be a full path and toolchain prefix, for example
+ {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of
+ something on the search path, for example
+ {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported.
paths: List of paths to check for toolchains (may contain wildcards)
"""
def __init__(self):
self.toolchains = {}
+ self.prefixes = {}
self.paths = []
- toolchains = bsettings.GetItems('toolchain')
- if not toolchains:
- print ("Warning: No tool chains - please add a [toolchain] section"
- " to your buildman config file %s. See README for details" %
- config_fname)
+ self._make_flags = dict(bsettings.GetItems('make-flags'))
+
+ def GetPathList(self, show_warning=True):
+ """Get a list of available toolchain paths
+ Args:
+ show_warning: True to show a warning if there are no tool chains.
+
+ Returns:
+ List of strings, each a path to a toolchain mentioned in the
+ [toolchain] section of the settings file.
+ """
+ toolchains = bsettings.GetItems('toolchain')
+ if show_warning and not toolchains:
+ print ("Warning: No tool chains. Please run 'buildman "
+ "--fetch-arch all' to download all available toolchains, or "
+ "add a [toolchain] section to your buildman config file "
+ "%s. See README for details" %
+ bsettings.config_fname)
+
+ paths = []
for name, value in toolchains:
if '*' in value:
- self.paths += glob.glob(value)
+ paths += glob.glob(value)
else:
- self.paths.append(value)
- self._make_flags = dict(bsettings.GetItems('make-flags'))
+ paths.append(value)
+ return paths
+
+ def GetSettings(self, show_warning=True):
+ """Get toolchain settings from the settings file.
+
+ Args:
+ show_warning: True to show a warning if there are no tool chains.
+ """
+ self.prefixes = bsettings.GetItems('toolchain-prefix')
+ self.paths += self.GetPathList(show_warning)
- def Add(self, fname, test=True, verbose=False):
+ def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC,
+ arch=None):
"""Add a toolchain to our list
We select the given toolchain as our preferred one for its
Args:
fname: Filename of toolchain's gcc driver
test: True to run the toolchain to test it
+ priority: Priority to use for this toolchain
+ arch: Toolchain architecture, or None if not known
"""
- toolchain = Toolchain(fname, test, verbose)
+ toolchain = Toolchain(fname, test, verbose, priority, arch)
add_it = toolchain.ok
if toolchain.arch in self.toolchains:
add_it = (toolchain.priority <
self.toolchains[toolchain.arch].priority)
if add_it:
self.toolchains[toolchain.arch] = toolchain
+ elif verbose:
+ print ("Toolchain '%s' at priority %d will be ignored because "
+ "another toolchain for arch '%s' has priority %d" %
+ (toolchain.gcc, toolchain.priority, toolchain.arch,
+ self.toolchains[toolchain.arch].priority))
+
+ def ScanPath(self, path, verbose):
+ """Scan a path for a valid toolchain
+
+ Args:
+ path: Path to scan
+ verbose: True to print out progress information
+ Returns:
+ Filename of C compiler if found, else None
+ """
+ fnames = []
+ for subdir in ['.', 'bin', 'usr/bin']:
+ dirname = os.path.join(path, subdir)
+ if verbose: print " - looking in '%s'" % dirname
+ for fname in glob.glob(dirname + '/*gcc'):
+ if verbose: print " - found '%s'" % fname
+ fnames.append(fname)
+ return fnames
+
+ def ScanPathEnv(self, fname):
+ """Scan the PATH environment variable for a given filename.
+
+ Args:
+ fname: Filename to scan for
+ Returns:
+ List of matching pathanames, or [] if none
+ """
+ pathname_list = []
+ for path in os.environ["PATH"].split(os.pathsep):
+ path = path.strip('"')
+ pathname = os.path.join(path, fname)
+ if os.path.exists(pathname):
+ pathname_list.append(pathname)
+ return pathname_list
def Scan(self, verbose):
"""Scan for available toolchains and select the best for each arch.
verbose: True to print out progress information
"""
if verbose: print 'Scanning for tool chains'
+ for name, value in self.prefixes:
+ if verbose: print " - scanning prefix '%s'" % value
+ if os.path.exists(value):
+ self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name)
+ continue
+ fname = value + 'gcc'
+ if os.path.exists(fname):
+ self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name)
+ continue
+ fname_list = self.ScanPathEnv(fname)
+ for f in fname_list:
+ self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
+ if not fname_list:
+ raise ValueError, ("No tool chain found for prefix '%s'" %
+ value)
for path in self.paths:
if verbose: print " - scanning path '%s'" % path
- for subdir in ['.', 'bin', 'usr/bin']:
- dirname = os.path.join(path, subdir)
- if verbose: print " - looking in '%s'" % dirname
- for fname in glob.glob(dirname + '/*gcc'):
- if verbose: print " - found '%s'" % fname
- self.Add(fname, True, verbose)
+ fnames = self.ScanPath(path, verbose)
+ for fname in fnames:
+ self.Add(fname, True, verbose)
def List(self):
"""List out the selected toolchains for each architecture"""
- print 'List of available toolchains (%d):' % len(self.toolchains)
+ col = terminal.Color()
+ print col.Color(col.BLUE, 'List of available toolchains (%d):' %
+ len(self.toolchains))
if len(self.toolchains):
for key, value in sorted(self.toolchains.iteritems()):
print '%-10s: %s' % (key, value.gcc)
returns:
toolchain object, or None if none found
"""
- for name, value in bsettings.GetItems('toolchain-alias'):
- if arch == name:
- arch = value
+ for tag, value in bsettings.GetItems('toolchain-alias'):
+ if arch == tag:
+ for alias in value.split():
+ if alias in self.toolchains:
+ return self.toolchains[alias]
if not arch in self.toolchains:
raise ValueError, ("No tool chain found for arch '%s'" % arch)
>>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
'this=OBLIQUE_setfi2ndrstnd'
"""
- re_var = re.compile('(\$\{[a-z0-9A-Z]{1,}\})')
+ re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
while True:
m = re_var.search(args)
else:
i += 1
return args
+
+ def LocateArchUrl(self, fetch_arch):
+ """Find a toolchain available online
+
+ Look in standard places for available toolchains. At present the
+ only standard place is at kernel.org.
+
+ Args:
+ arch: Architecture to look for, or 'list' for all
+ Returns:
+ If fetch_arch is 'list', a tuple:
+ Machine architecture (e.g. x86_64)
+ List of toolchains
+ else
+ URL containing this toolchain, if avaialble, else None
+ """
+ arch = command.OutputOneLine('uname', '-m')
+ base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
+ versions = ['7.3.0', '6.4.0', '4.9.4']
+ links = []
+ for version in versions:
+ url = '%s/%s/%s/' % (base, arch, version)
+ print 'Checking: %s' % url
+ response = urllib2.urlopen(url)
+ html = response.read()
+ parser = MyHTMLParser(fetch_arch)
+ parser.feed(html)
+ if fetch_arch == 'list':
+ links += parser.links
+ elif parser.arch_link:
+ return url + parser.arch_link
+ if fetch_arch == 'list':
+ return arch, links
+ return None
+
+ def Download(self, url):
+ """Download a file to a temporary directory
+
+ Args:
+ url: URL to download
+ Returns:
+ Tuple:
+ Temporary directory name
+ Full path to the downloaded archive file in that directory,
+ or None if there was an error while downloading
+ """
+ print 'Downloading: %s' % url
+ leaf = url.split('/')[-1]
+ tmpdir = tempfile.mkdtemp('.buildman')
+ response = urllib2.urlopen(url)
+ fname = os.path.join(tmpdir, leaf)
+ fd = open(fname, 'wb')
+ meta = response.info()
+ size = int(meta.getheaders('Content-Length')[0])
+ done = 0
+ block_size = 1 << 16
+ status = ''
+
+ # Read the file in chunks and show progress as we go
+ while True:
+ buffer = response.read(block_size)
+ if not buffer:
+ print chr(8) * (len(status) + 1), '\r',
+ break
+
+ done += len(buffer)
+ fd.write(buffer)
+ status = r'%10d MiB [%3d%%]' % (done / 1024 / 1024,
+ done * 100 / size)
+ status = status + chr(8) * (len(status) + 1)
+ print status,
+ sys.stdout.flush()
+ fd.close()
+ if done != size:
+ print 'Error, failed to download'
+ os.remove(fname)
+ fname = None
+ return tmpdir, fname
+
+ def Unpack(self, fname, dest):
+ """Unpack a tar file
+
+ Args:
+ fname: Filename to unpack
+ dest: Destination directory
+ Returns:
+ Directory name of the first entry in the archive, without the
+ trailing /
+ """
+ stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)
+ return stdout.splitlines()[0][:-1]
+
+ def TestSettingsHasPath(self, path):
+ """Check if buildman will find this toolchain
+
+ Returns:
+ True if the path is in settings, False if not
+ """
+ paths = self.GetPathList(False)
+ return path in paths
+
+ def ListArchs(self):
+ """List architectures with available toolchains to download"""
+ host_arch, archives = self.LocateArchUrl('list')
+ re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*')
+ arch_set = set()
+ for archive in archives:
+ # Remove the host architecture from the start
+ arch = re_arch.match(archive[len(host_arch):])
+ if arch:
+ arch_set.add(arch.group(1))
+ return sorted(arch_set)
+
+ def FetchAndInstall(self, arch):
+ """Fetch and install a new toolchain
+
+ arch:
+ Architecture to fetch, or 'list' to list
+ """
+ # Fist get the URL for this architecture
+ col = terminal.Color()
+ print col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch)
+ url = self.LocateArchUrl(arch)
+ if not url:
+ print ("Cannot find toolchain for arch '%s' - use 'list' to list" %
+ arch)
+ return 2
+ home = os.environ['HOME']
+ dest = os.path.join(home, '.buildman-toolchains')
+ if not os.path.exists(dest):
+ os.mkdir(dest)
+
+ # Download the tar file for this toolchain and unpack it
+ tmpdir, tarfile = self.Download(url)
+ if not tarfile:
+ return 1
+ print col.Color(col.GREEN, 'Unpacking to: %s' % dest),
+ sys.stdout.flush()
+ path = self.Unpack(tarfile, dest)
+ os.remove(tarfile)
+ os.rmdir(tmpdir)
+ print
+
+ # Check that the toolchain works
+ print col.Color(col.GREEN, 'Testing')
+ dirpath = os.path.join(dest, path)
+ compiler_fname_list = self.ScanPath(dirpath, True)
+ if not compiler_fname_list:
+ print 'Could not locate C compiler - fetch failed.'
+ return 1
+ if len(compiler_fname_list) != 1:
+ print col.Color(col.RED, 'Warning, ambiguous toolchains: %s' %
+ ', '.join(compiler_fname_list))
+ toolchain = Toolchain(compiler_fname_list[0], True, True)
+
+ # Make sure that it will be found by buildman
+ if not self.TestSettingsHasPath(dirpath):
+ print ("Adding 'download' to config file '%s'" %
+ bsettings.config_fname)
+ bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest)
+ return 0
projects / u-boot / blobdiff