1 # Copyright (c) 2012 The Chromium OS Authors.
3 # SPDX-License-Identifier: GPL-2.0+
8 from HTMLParser import HTMLParser
17 # Simple class to collect links from a page
18 class MyHTMLParser(HTMLParser):
19 def __init__(self, arch):
20 """Create a new parser
22 After the parser runs, self.links will be set to a list of the links
23 to .xz archives found in the page, and self.arch_link will be set to
24 the one for the given architecture (or None if not found).
27 arch: Architecture to search for
29 HTMLParser.__init__(self)
32 self._match = '_%s-' % arch
34 def handle_starttag(self, tag, attrs):
36 for tag, value in attrs:
38 if value and value.endswith('.xz'):
39 self.links.append(value)
40 if self._match in value:
41 self.arch_link = value
48 gcc: Full path to C compiler
49 path: Directory path containing C compiler
50 cross: Cross compile string, e.g. 'arm-linux-'
51 arch: Architecture of toolchain as determined from the first
52 component of the filename. E.g. arm-linux-gcc becomes arm
54 def __init__(self, fname, test, verbose=False):
55 """Create a new toolchain object.
58 fname: Filename of the gcc component
59 test: True to run the toolchain to test it
62 self.path = os.path.dirname(fname)
64 # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
65 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
66 basename = os.path.basename(fname)
67 pos = basename.rfind('-')
68 self.cross = basename[:pos + 1] if pos != -1 else ''
70 # The architecture is the first part of the name
71 pos = self.cross.find('-')
72 self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
74 env = self.MakeEnvironment(False)
76 # As a basic sanity check, run the C compiler with --version
77 cmd = [fname, '--version']
79 result = command.RunPipe([cmd], capture=True, env=env,
81 self.ok = result.return_code == 0
83 print 'Tool chain test: ',
88 print 'Command: ', cmd
93 self.priority = self.GetPriority(fname)
95 def GetPriority(self, fname):
96 """Return the priority of the toolchain.
98 Toolchains are ranked according to their suitability by their
102 fname: Filename of toolchain
104 Priority of toolchain, 0=highest, 20=lowest.
106 priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
107 '-none-linux-gnueabi', '-uclinux', '-none-eabi',
108 '-gentoo-linux-gnu', '-linux-gnueabi', '-le-linux', '-uclinux']
109 for prio in range(len(priority_list)):
110 if priority_list[prio] in fname:
114 def MakeEnvironment(self, full_path):
115 """Returns an environment for using the toolchain.
117 Thie takes the current environment and adds CROSS_COMPILE so that
118 the tool chain will operate correctly.
121 full_path: Return the full path in CROSS_COMPILE and don't set
124 env = dict(os.environ)
126 env['CROSS_COMPILE'] = os.path.join(self.path, self.cross)
128 env['CROSS_COMPILE'] = self.cross
129 env['PATH'] = self.path + ':' + env['PATH']
135 """Manage a list of toolchains for building U-Boot
137 We select one toolchain for each architecture type
140 toolchains: Dict of Toolchain objects, keyed by architecture name
141 paths: List of paths to check for toolchains (may contain wildcards)
147 self._make_flags = dict(bsettings.GetItems('make-flags'))
149 def GetPathList(self):
150 """Get a list of available toolchain paths
153 List of strings, each a path to a toolchain mentioned in the
154 [toolchain] section of the settings file.
156 toolchains = bsettings.GetItems('toolchain')
158 print ("Warning: No tool chains - please add a [toolchain] section"
159 " to your buildman config file %s. See README for details" %
160 bsettings.config_fname)
163 for name, value in toolchains:
165 paths += glob.glob(value)
170 def GetSettings(self):
171 self.paths += self.GetPathList()
173 def Add(self, fname, test=True, verbose=False):
174 """Add a toolchain to our list
176 We select the given toolchain as our preferred one for its
177 architecture if it is a higher priority than the others.
180 fname: Filename of toolchain's gcc driver
181 test: True to run the toolchain to test it
183 toolchain = Toolchain(fname, test, verbose)
184 add_it = toolchain.ok
185 if toolchain.arch in self.toolchains:
186 add_it = (toolchain.priority <
187 self.toolchains[toolchain.arch].priority)
189 self.toolchains[toolchain.arch] = toolchain
191 def ScanPath(self, path, verbose):
192 """Scan a path for a valid toolchain
196 verbose: True to print out progress information
198 Filename of C compiler if found, else None
201 for subdir in ['.', 'bin', 'usr/bin']:
202 dirname = os.path.join(path, subdir)
203 if verbose: print " - looking in '%s'" % dirname
204 for fname in glob.glob(dirname + '/*gcc'):
205 if verbose: print " - found '%s'" % fname
210 def Scan(self, verbose):
211 """Scan for available toolchains and select the best for each arch.
213 We look for all the toolchains we can file, figure out the
214 architecture for each, and whether it works. Then we select the
215 highest priority toolchain for each arch.
218 verbose: True to print out progress information
220 if verbose: print 'Scanning for tool chains'
221 for path in self.paths:
222 if verbose: print " - scanning path '%s'" % path
223 fnames = self.ScanPath(path, verbose)
225 self.Add(fname, True, verbose)
228 """List out the selected toolchains for each architecture"""
229 print 'List of available toolchains (%d):' % len(self.toolchains)
230 if len(self.toolchains):
231 for key, value in sorted(self.toolchains.iteritems()):
232 print '%-10s: %s' % (key, value.gcc)
236 def Select(self, arch):
237 """Returns the toolchain for a given architecture
240 args: Name of architecture (e.g. 'arm', 'ppc_8xx')
243 toolchain object, or None if none found
245 for tag, value in bsettings.GetItems('toolchain-alias'):
247 for alias in value.split():
248 if alias in self.toolchains:
249 return self.toolchains[alias]
251 if not arch in self.toolchains:
252 raise ValueError, ("No tool chain found for arch '%s'" % arch)
253 return self.toolchains[arch]
255 def ResolveReferences(self, var_dict, args):
256 """Resolve variable references in a string
258 This converts ${blah} within the string to the value of blah.
259 This function works recursively.
262 var_dict: Dictionary containing variables and their values
263 args: String containing make arguments
267 >>> bsettings.Setup()
268 >>> tcs = Toolchains()
269 >>> tcs.Add('fred', False)
270 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
272 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
274 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
275 'this=OBLIQUE_setfi2ndrstnd'
277 re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
280 m = re_var.search(args)
283 lookup = m.group(0)[2:-1]
284 value = var_dict.get(lookup, '')
285 args = args[:m.start(0)] + value + args[m.end(0):]
288 def GetMakeArguments(self, board):
289 """Returns 'make' arguments for a given board
291 The flags are in a section called 'make-flags'. Flags are named
292 after the target they represent, for example snapper9260=TESTING=1
293 will pass TESTING=1 to make when building the snapper9260 board.
295 References to other boards can be added in the string also. For
299 at91-boards=ENABLE_AT91_TEST=1
300 snapper9260=${at91-boards} BUILD_TAG=442
301 snapper9g45=${at91-boards} BUILD_TAG=443
303 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
304 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
306 A special 'target' variable is set to the board target.
309 board: Board object for the board to check.
311 'make' flags for that board, or '' if none
313 self._make_flags['target'] = board.target
314 arg_str = self.ResolveReferences(self._make_flags,
315 self._make_flags.get(board.target, ''))
316 args = arg_str.split(' ')
325 def LocateArchUrl(self, fetch_arch):
326 """Find a toolchain available online
328 Look in standard places for available toolchains. At present the
329 only standard place is at kernel.org.
332 arch: Architecture to look for, or 'list' for all
334 If fetch_arch is 'list', a tuple:
335 Machine architecture (e.g. x86_64)
338 URL containing this toolchain, if avaialble, else None
340 arch = command.OutputOneLine('uname', '-m')
341 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
342 versions = ['4.6.3', '4.6.2', '4.5.1', '4.2.4']
344 for version in versions:
345 url = '%s/%s/%s/' % (base, arch, version)
346 print 'Checking: %s' % url
347 response = urllib2.urlopen(url)
348 html = response.read()
349 parser = MyHTMLParser(fetch_arch)
351 if fetch_arch == 'list':
352 links += parser.links
353 elif parser.arch_link:
354 return url + parser.arch_link
355 if fetch_arch == 'list':
359 def Download(self, url):
360 """Download a file to a temporary directory
366 Temporary directory name
367 Full path to the downloaded archive file in that directory,
368 or None if there was an error while downloading
370 print "Downloading: %s" % url
371 leaf = url.split('/')[-1]
372 tmpdir = tempfile.mkdtemp('.buildman')
373 response = urllib2.urlopen(url)
374 fname = os.path.join(tmpdir, leaf)
375 fd = open(fname, 'wb')
376 meta = response.info()
377 size = int(meta.getheaders("Content-Length")[0])
382 # Read the file in chunks and show progress as we go
384 buffer = response.read(block_size)
386 print chr(8) * (len(status) + 1), '\r',
391 status = r"%10d MiB [%3d%%]" % (done / 1024 / 1024,
393 status = status + chr(8) * (len(status) + 1)
398 print 'Error, failed to download'
403 def Unpack(self, fname, dest):
407 fname: Filename to unpack
408 dest: Destination directory
410 Directory name of the first entry in the archive, without the
413 stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)
414 return stdout.splitlines()[0][:-1]
416 def TestSettingsHasPath(self, path):
417 """Check if builmand will find this toolchain
420 True if the path is in settings, False if not
422 paths = self.GetPathList()
426 """List architectures with available toolchains to download"""
427 host_arch, archives = self.LocateArchUrl('list')
428 re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*')
430 for archive in archives:
431 # Remove the host architecture from the start
432 arch = re_arch.match(archive[len(host_arch):])
434 arch_set.add(arch.group(1))
435 return sorted(arch_set)
437 def FetchAndInstall(self, arch):
438 """Fetch and install a new toolchain
441 Architecture to fetch, or 'list' to list
443 # Fist get the URL for this architecture
444 url = self.LocateArchUrl(arch)
446 print ("Cannot find toolchain for arch '%s' - use 'list' to list" %
449 home = os.environ['HOME']
450 dest = os.path.join(home, '.buildman-toolchains')
451 if not os.path.exists(dest):
454 # Download the tar file for this toolchain and unpack it
455 tmpdir, tarfile = self.Download(url)
458 print 'Unpacking to: %s' % dest,
460 path = self.Unpack(tarfile, dest)
465 # Check that the toolchain works
467 dirpath = os.path.join(dest, path)
468 compiler_fname_list = self.ScanPath(dirpath, True)
469 if not compiler_fname_list:
470 print 'Could not locate C compiler - fetch failed.'
472 if len(compiler_fname_list) != 1:
473 print ('Internal error, ambiguous toolchains: %s' %
474 (', '.join(compiler_fname)))
476 toolchain = Toolchain(compiler_fname_list[0], True, True)
478 # Make sure that it will be found by buildman
479 if not self.TestSettingsHasPath(dirpath):
480 print ("Adding 'download' to config file '%s'" %
481 bsettings.config_fname)
482 tools_dir = os.path.dirname(dirpath)
483 bsettings.SetItem('toolchain', 'download', '%s/*' % tools_dir)