1 # Copyright (c) 2012 The Chromium OS Authors.
3 # SPDX-License-Identifier: GPL-2.0+
8 from HTMLParser import HTMLParser
19 # Simple class to collect links from a page
20 class MyHTMLParser(HTMLParser):
21 def __init__(self, arch):
22 """Create a new parser
24 After the parser runs, self.links will be set to a list of the links
25 to .xz archives found in the page, and self.arch_link will be set to
26 the one for the given architecture (or None if not found).
29 arch: Architecture to search for
31 HTMLParser.__init__(self)
34 self._match = '_%s-' % arch
36 def handle_starttag(self, tag, attrs):
38 for tag, value in attrs:
40 if value and value.endswith('.xz'):
41 self.links.append(value)
42 if self._match in value:
43 self.arch_link = value
50 gcc: Full path to C compiler
51 path: Directory path containing C compiler
52 cross: Cross compile string, e.g. 'arm-linux-'
53 arch: Architecture of toolchain as determined from the first
54 component of the filename. E.g. arm-linux-gcc becomes arm
55 priority: Toolchain priority (0=highest, 20=lowest)
57 def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,
59 """Create a new toolchain object.
62 fname: Filename of the gcc component
63 test: True to run the toolchain to test it
64 verbose: True to print out the information
65 priority: Priority to use for this toolchain, or PRIORITY_CALC to
69 self.path = os.path.dirname(fname)
71 # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
72 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
73 basename = os.path.basename(fname)
74 pos = basename.rfind('-')
75 self.cross = basename[:pos + 1] if pos != -1 else ''
77 # The architecture is the first part of the name
78 pos = self.cross.find('-')
82 self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
84 env = self.MakeEnvironment(False)
86 # As a basic sanity check, run the C compiler with --version
87 cmd = [fname, '--version']
88 if priority == PRIORITY_CALC:
89 self.priority = self.GetPriority(fname)
91 self.priority = priority
93 result = command.RunPipe([cmd], capture=True, env=env,
95 self.ok = result.return_code == 0
97 print 'Tool chain test: ',
99 print "OK, arch='%s', priority %d" % (self.arch,
103 print 'Command: ', cmd
109 def GetPriority(self, fname):
110 """Return the priority of the toolchain.
112 Toolchains are ranked according to their suitability by their
116 fname: Filename of toolchain
118 Priority of toolchain, PRIORITY_CALC=highest, 20=lowest.
120 priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
121 '-none-linux-gnueabi', '-uclinux', '-none-eabi',
122 '-gentoo-linux-gnu', '-linux-gnueabi', '-le-linux', '-uclinux']
123 for prio in range(len(priority_list)):
124 if priority_list[prio] in fname:
125 return PRIORITY_CALC + prio
126 return PRIORITY_CALC + prio
128 def MakeEnvironment(self, full_path):
129 """Returns an environment for using the toolchain.
131 Thie takes the current environment and adds CROSS_COMPILE so that
132 the tool chain will operate correctly.
135 full_path: Return the full path in CROSS_COMPILE and don't set
138 env = dict(os.environ)
140 env['CROSS_COMPILE'] = os.path.join(self.path, self.cross)
142 env['CROSS_COMPILE'] = self.cross
143 env['PATH'] = self.path + ':' + env['PATH']
149 """Manage a list of toolchains for building U-Boot
151 We select one toolchain for each architecture type
154 toolchains: Dict of Toolchain objects, keyed by architecture name
155 paths: List of paths to check for toolchains (may contain wildcards)
161 self._make_flags = dict(bsettings.GetItems('make-flags'))
163 def GetPathList(self):
164 """Get a list of available toolchain paths
167 List of strings, each a path to a toolchain mentioned in the
168 [toolchain] section of the settings file.
170 toolchains = bsettings.GetItems('toolchain')
172 print ('Warning: No tool chains - please add a [toolchain] section'
173 ' to your buildman config file %s. See README for details' %
174 bsettings.config_fname)
177 for name, value in toolchains:
179 paths += glob.glob(value)
184 def GetSettings(self):
185 self.paths += self.GetPathList()
187 def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC,
189 """Add a toolchain to our list
191 We select the given toolchain as our preferred one for its
192 architecture if it is a higher priority than the others.
195 fname: Filename of toolchain's gcc driver
196 test: True to run the toolchain to test it
197 priority: Priority to use for this toolchain
198 arch: Toolchain architecture, or None if not known
200 toolchain = Toolchain(fname, test, verbose, priority, arch)
201 add_it = toolchain.ok
202 if toolchain.arch in self.toolchains:
203 add_it = (toolchain.priority <
204 self.toolchains[toolchain.arch].priority)
206 self.toolchains[toolchain.arch] = toolchain
208 print ("Toolchain '%s' at priority %d will be ignored because "
209 "another toolchain for arch '%s' has priority %d" %
210 (toolchain.gcc, toolchain.priority, toolchain.arch,
211 self.toolchains[toolchain.arch].priority))
213 def ScanPath(self, path, verbose):
214 """Scan a path for a valid toolchain
218 verbose: True to print out progress information
220 Filename of C compiler if found, else None
223 for subdir in ['.', 'bin', 'usr/bin']:
224 dirname = os.path.join(path, subdir)
225 if verbose: print " - looking in '%s'" % dirname
226 for fname in glob.glob(dirname + '/*gcc'):
227 if verbose: print " - found '%s'" % fname
232 def Scan(self, verbose):
233 """Scan for available toolchains and select the best for each arch.
235 We look for all the toolchains we can file, figure out the
236 architecture for each, and whether it works. Then we select the
237 highest priority toolchain for each arch.
240 verbose: True to print out progress information
242 if verbose: print 'Scanning for tool chains'
243 for path in self.paths:
244 if verbose: print " - scanning path '%s'" % path
245 fnames = self.ScanPath(path, verbose)
247 self.Add(fname, True, verbose)
250 """List out the selected toolchains for each architecture"""
251 print 'List of available toolchains (%d):' % len(self.toolchains)
252 if len(self.toolchains):
253 for key, value in sorted(self.toolchains.iteritems()):
254 print '%-10s: %s' % (key, value.gcc)
258 def Select(self, arch):
259 """Returns the toolchain for a given architecture
262 args: Name of architecture (e.g. 'arm', 'ppc_8xx')
265 toolchain object, or None if none found
267 for tag, value in bsettings.GetItems('toolchain-alias'):
269 for alias in value.split():
270 if alias in self.toolchains:
271 return self.toolchains[alias]
273 if not arch in self.toolchains:
274 raise ValueError, ("No tool chain found for arch '%s'" % arch)
275 return self.toolchains[arch]
277 def ResolveReferences(self, var_dict, args):
278 """Resolve variable references in a string
280 This converts ${blah} within the string to the value of blah.
281 This function works recursively.
284 var_dict: Dictionary containing variables and their values
285 args: String containing make arguments
289 >>> bsettings.Setup()
290 >>> tcs = Toolchains()
291 >>> tcs.Add('fred', False)
292 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
294 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
296 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
297 'this=OBLIQUE_setfi2ndrstnd'
299 re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
302 m = re_var.search(args)
305 lookup = m.group(0)[2:-1]
306 value = var_dict.get(lookup, '')
307 args = args[:m.start(0)] + value + args[m.end(0):]
310 def GetMakeArguments(self, board):
311 """Returns 'make' arguments for a given board
313 The flags are in a section called 'make-flags'. Flags are named
314 after the target they represent, for example snapper9260=TESTING=1
315 will pass TESTING=1 to make when building the snapper9260 board.
317 References to other boards can be added in the string also. For
321 at91-boards=ENABLE_AT91_TEST=1
322 snapper9260=${at91-boards} BUILD_TAG=442
323 snapper9g45=${at91-boards} BUILD_TAG=443
325 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
326 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
328 A special 'target' variable is set to the board target.
331 board: Board object for the board to check.
333 'make' flags for that board, or '' if none
335 self._make_flags['target'] = board.target
336 arg_str = self.ResolveReferences(self._make_flags,
337 self._make_flags.get(board.target, ''))
338 args = arg_str.split(' ')
347 def LocateArchUrl(self, fetch_arch):
348 """Find a toolchain available online
350 Look in standard places for available toolchains. At present the
351 only standard place is at kernel.org.
354 arch: Architecture to look for, or 'list' for all
356 If fetch_arch is 'list', a tuple:
357 Machine architecture (e.g. x86_64)
360 URL containing this toolchain, if avaialble, else None
362 arch = command.OutputOneLine('uname', '-m')
363 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
364 versions = ['4.9.0', '4.6.3', '4.6.2', '4.5.1', '4.2.4']
366 for version in versions:
367 url = '%s/%s/%s/' % (base, arch, version)
368 print 'Checking: %s' % url
369 response = urllib2.urlopen(url)
370 html = response.read()
371 parser = MyHTMLParser(fetch_arch)
373 if fetch_arch == 'list':
374 links += parser.links
375 elif parser.arch_link:
376 return url + parser.arch_link
377 if fetch_arch == 'list':
381 def Download(self, url):
382 """Download a file to a temporary directory
388 Temporary directory name
389 Full path to the downloaded archive file in that directory,
390 or None if there was an error while downloading
392 print 'Downloading: %s' % url
393 leaf = url.split('/')[-1]
394 tmpdir = tempfile.mkdtemp('.buildman')
395 response = urllib2.urlopen(url)
396 fname = os.path.join(tmpdir, leaf)
397 fd = open(fname, 'wb')
398 meta = response.info()
399 size = int(meta.getheaders('Content-Length')[0])
404 # Read the file in chunks and show progress as we go
406 buffer = response.read(block_size)
408 print chr(8) * (len(status) + 1), '\r',
413 status = r'%10d MiB [%3d%%]' % (done / 1024 / 1024,
415 status = status + chr(8) * (len(status) + 1)
420 print 'Error, failed to download'
425 def Unpack(self, fname, dest):
429 fname: Filename to unpack
430 dest: Destination directory
432 Directory name of the first entry in the archive, without the
435 stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)
436 return stdout.splitlines()[0][:-1]
438 def TestSettingsHasPath(self, path):
439 """Check if builmand will find this toolchain
442 True if the path is in settings, False if not
444 paths = self.GetPathList()
448 """List architectures with available toolchains to download"""
449 host_arch, archives = self.LocateArchUrl('list')
450 re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*')
452 for archive in archives:
453 # Remove the host architecture from the start
454 arch = re_arch.match(archive[len(host_arch):])
456 arch_set.add(arch.group(1))
457 return sorted(arch_set)
459 def FetchAndInstall(self, arch):
460 """Fetch and install a new toolchain
463 Architecture to fetch, or 'list' to list
465 # Fist get the URL for this architecture
466 url = self.LocateArchUrl(arch)
468 print ("Cannot find toolchain for arch '%s' - use 'list' to list" %
471 home = os.environ['HOME']
472 dest = os.path.join(home, '.buildman-toolchains')
473 if not os.path.exists(dest):
476 # Download the tar file for this toolchain and unpack it
477 tmpdir, tarfile = self.Download(url)
480 print 'Unpacking to: %s' % dest,
482 path = self.Unpack(tarfile, dest)
487 # Check that the toolchain works
489 dirpath = os.path.join(dest, path)
490 compiler_fname_list = self.ScanPath(dirpath, True)
491 if not compiler_fname_list:
492 print 'Could not locate C compiler - fetch failed.'
494 if len(compiler_fname_list) != 1:
495 print ('Internal error, ambiguous toolchains: %s' %
496 (', '.join(compiler_fname)))
498 toolchain = Toolchain(compiler_fname_list[0], True, True)
500 # Make sure that it will be found by buildman
501 if not self.TestSettingsHasPath(dirpath):
502 print ("Adding 'download' to config file '%s'" %
503 bsettings.config_fname)
504 tools_dir = os.path.dirname(dirpath)
505 bsettings.SetItem('toolchain', 'download', '%s/*' % tools_dir)