1 # Copyright (c) 2012 The Chromium OS Authors.
3 # SPDX-License-Identifier: GPL-2.0+
8 from HTMLParser import HTMLParser
17 # Simple class to collect links from a page
18 class MyHTMLParser(HTMLParser):
19 def __init__(self, arch):
20 """Create a new parser
22 After the parser runs, self.links will be set to a list of the links
23 to .xz archives found in the page, and self.arch_link will be set to
24 the one for the given architecture (or None if not found).
27 arch: Architecture to search for
29 HTMLParser.__init__(self)
32 self._match = '_%s-' % arch
34 def handle_starttag(self, tag, attrs):
36 for tag, value in attrs:
38 if value and value.endswith('.xz'):
39 self.links.append(value)
40 if self._match in value:
41 self.arch_link = value
48 gcc: Full path to C compiler
49 path: Directory path containing C compiler
50 cross: Cross compile string, e.g. 'arm-linux-'
51 arch: Architecture of toolchain as determined from the first
52 component of the filename. E.g. arm-linux-gcc becomes arm
54 def __init__(self, fname, test, verbose=False):
55 """Create a new toolchain object.
58 fname: Filename of the gcc component
59 test: True to run the toolchain to test it
60 verbose: True to print out the information
63 self.path = os.path.dirname(fname)
65 # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
66 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
67 basename = os.path.basename(fname)
68 pos = basename.rfind('-')
69 self.cross = basename[:pos + 1] if pos != -1 else ''
71 # The architecture is the first part of the name
72 pos = self.cross.find('-')
73 self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
75 env = self.MakeEnvironment(False)
77 # As a basic sanity check, run the C compiler with --version
78 cmd = [fname, '--version']
80 result = command.RunPipe([cmd], capture=True, env=env,
82 self.ok = result.return_code == 0
84 print 'Tool chain test: ',
89 print 'Command: ', cmd
94 self.priority = self.GetPriority(fname)
96 def GetPriority(self, fname):
97 """Return the priority of the toolchain.
99 Toolchains are ranked according to their suitability by their
103 fname: Filename of toolchain
105 Priority of toolchain, 0=highest, 20=lowest.
107 priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
108 '-none-linux-gnueabi', '-uclinux', '-none-eabi',
109 '-gentoo-linux-gnu', '-linux-gnueabi', '-le-linux', '-uclinux']
110 for prio in range(len(priority_list)):
111 if priority_list[prio] in fname:
115 def MakeEnvironment(self, full_path):
116 """Returns an environment for using the toolchain.
118 Thie takes the current environment and adds CROSS_COMPILE so that
119 the tool chain will operate correctly.
122 full_path: Return the full path in CROSS_COMPILE and don't set
125 env = dict(os.environ)
127 env['CROSS_COMPILE'] = os.path.join(self.path, self.cross)
129 env['CROSS_COMPILE'] = self.cross
130 env['PATH'] = self.path + ':' + env['PATH']
136 """Manage a list of toolchains for building U-Boot
138 We select one toolchain for each architecture type
141 toolchains: Dict of Toolchain objects, keyed by architecture name
142 paths: List of paths to check for toolchains (may contain wildcards)
148 self._make_flags = dict(bsettings.GetItems('make-flags'))
150 def GetPathList(self):
151 """Get a list of available toolchain paths
154 List of strings, each a path to a toolchain mentioned in the
155 [toolchain] section of the settings file.
157 toolchains = bsettings.GetItems('toolchain')
159 print ('Warning: No tool chains - please add a [toolchain] section'
160 ' to your buildman config file %s. See README for details' %
161 bsettings.config_fname)
164 for name, value in toolchains:
166 paths += glob.glob(value)
171 def GetSettings(self):
172 self.paths += self.GetPathList()
174 def Add(self, fname, test=True, verbose=False):
175 """Add a toolchain to our list
177 We select the given toolchain as our preferred one for its
178 architecture if it is a higher priority than the others.
181 fname: Filename of toolchain's gcc driver
182 test: True to run the toolchain to test it
184 toolchain = Toolchain(fname, test, verbose)
185 add_it = toolchain.ok
186 if toolchain.arch in self.toolchains:
187 add_it = (toolchain.priority <
188 self.toolchains[toolchain.arch].priority)
190 self.toolchains[toolchain.arch] = toolchain
192 def ScanPath(self, path, verbose):
193 """Scan a path for a valid toolchain
197 verbose: True to print out progress information
199 Filename of C compiler if found, else None
202 for subdir in ['.', 'bin', 'usr/bin']:
203 dirname = os.path.join(path, subdir)
204 if verbose: print " - looking in '%s'" % dirname
205 for fname in glob.glob(dirname + '/*gcc'):
206 if verbose: print " - found '%s'" % fname
211 def Scan(self, verbose):
212 """Scan for available toolchains and select the best for each arch.
214 We look for all the toolchains we can file, figure out the
215 architecture for each, and whether it works. Then we select the
216 highest priority toolchain for each arch.
219 verbose: True to print out progress information
221 if verbose: print 'Scanning for tool chains'
222 for path in self.paths:
223 if verbose: print " - scanning path '%s'" % path
224 fnames = self.ScanPath(path, verbose)
226 self.Add(fname, True, verbose)
229 """List out the selected toolchains for each architecture"""
230 print 'List of available toolchains (%d):' % len(self.toolchains)
231 if len(self.toolchains):
232 for key, value in sorted(self.toolchains.iteritems()):
233 print '%-10s: %s' % (key, value.gcc)
237 def Select(self, arch):
238 """Returns the toolchain for a given architecture
241 args: Name of architecture (e.g. 'arm', 'ppc_8xx')
244 toolchain object, or None if none found
246 for tag, value in bsettings.GetItems('toolchain-alias'):
248 for alias in value.split():
249 if alias in self.toolchains:
250 return self.toolchains[alias]
252 if not arch in self.toolchains:
253 raise ValueError, ("No tool chain found for arch '%s'" % arch)
254 return self.toolchains[arch]
256 def ResolveReferences(self, var_dict, args):
257 """Resolve variable references in a string
259 This converts ${blah} within the string to the value of blah.
260 This function works recursively.
263 var_dict: Dictionary containing variables and their values
264 args: String containing make arguments
268 >>> bsettings.Setup()
269 >>> tcs = Toolchains()
270 >>> tcs.Add('fred', False)
271 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
273 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
275 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
276 'this=OBLIQUE_setfi2ndrstnd'
278 re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
281 m = re_var.search(args)
284 lookup = m.group(0)[2:-1]
285 value = var_dict.get(lookup, '')
286 args = args[:m.start(0)] + value + args[m.end(0):]
289 def GetMakeArguments(self, board):
290 """Returns 'make' arguments for a given board
292 The flags are in a section called 'make-flags'. Flags are named
293 after the target they represent, for example snapper9260=TESTING=1
294 will pass TESTING=1 to make when building the snapper9260 board.
296 References to other boards can be added in the string also. For
300 at91-boards=ENABLE_AT91_TEST=1
301 snapper9260=${at91-boards} BUILD_TAG=442
302 snapper9g45=${at91-boards} BUILD_TAG=443
304 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
305 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
307 A special 'target' variable is set to the board target.
310 board: Board object for the board to check.
312 'make' flags for that board, or '' if none
314 self._make_flags['target'] = board.target
315 arg_str = self.ResolveReferences(self._make_flags,
316 self._make_flags.get(board.target, ''))
317 args = arg_str.split(' ')
326 def LocateArchUrl(self, fetch_arch):
327 """Find a toolchain available online
329 Look in standard places for available toolchains. At present the
330 only standard place is at kernel.org.
333 arch: Architecture to look for, or 'list' for all
335 If fetch_arch is 'list', a tuple:
336 Machine architecture (e.g. x86_64)
339 URL containing this toolchain, if avaialble, else None
341 arch = command.OutputOneLine('uname', '-m')
342 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
343 versions = ['4.9.0', '4.6.3', '4.6.2', '4.5.1', '4.2.4']
345 for version in versions:
346 url = '%s/%s/%s/' % (base, arch, version)
347 print 'Checking: %s' % url
348 response = urllib2.urlopen(url)
349 html = response.read()
350 parser = MyHTMLParser(fetch_arch)
352 if fetch_arch == 'list':
353 links += parser.links
354 elif parser.arch_link:
355 return url + parser.arch_link
356 if fetch_arch == 'list':
360 def Download(self, url):
361 """Download a file to a temporary directory
367 Temporary directory name
368 Full path to the downloaded archive file in that directory,
369 or None if there was an error while downloading
371 print 'Downloading: %s' % url
372 leaf = url.split('/')[-1]
373 tmpdir = tempfile.mkdtemp('.buildman')
374 response = urllib2.urlopen(url)
375 fname = os.path.join(tmpdir, leaf)
376 fd = open(fname, 'wb')
377 meta = response.info()
378 size = int(meta.getheaders('Content-Length')[0])
383 # Read the file in chunks and show progress as we go
385 buffer = response.read(block_size)
387 print chr(8) * (len(status) + 1), '\r',
392 status = r'%10d MiB [%3d%%]' % (done / 1024 / 1024,
394 status = status + chr(8) * (len(status) + 1)
399 print 'Error, failed to download'
404 def Unpack(self, fname, dest):
408 fname: Filename to unpack
409 dest: Destination directory
411 Directory name of the first entry in the archive, without the
414 stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)
415 return stdout.splitlines()[0][:-1]
417 def TestSettingsHasPath(self, path):
418 """Check if builmand will find this toolchain
421 True if the path is in settings, False if not
423 paths = self.GetPathList()
427 """List architectures with available toolchains to download"""
428 host_arch, archives = self.LocateArchUrl('list')
429 re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*')
431 for archive in archives:
432 # Remove the host architecture from the start
433 arch = re_arch.match(archive[len(host_arch):])
435 arch_set.add(arch.group(1))
436 return sorted(arch_set)
438 def FetchAndInstall(self, arch):
439 """Fetch and install a new toolchain
442 Architecture to fetch, or 'list' to list
444 # Fist get the URL for this architecture
445 url = self.LocateArchUrl(arch)
447 print ("Cannot find toolchain for arch '%s' - use 'list' to list" %
450 home = os.environ['HOME']
451 dest = os.path.join(home, '.buildman-toolchains')
452 if not os.path.exists(dest):
455 # Download the tar file for this toolchain and unpack it
456 tmpdir, tarfile = self.Download(url)
459 print 'Unpacking to: %s' % dest,
461 path = self.Unpack(tarfile, dest)
466 # Check that the toolchain works
468 dirpath = os.path.join(dest, path)
469 compiler_fname_list = self.ScanPath(dirpath, True)
470 if not compiler_fname_list:
471 print 'Could not locate C compiler - fetch failed.'
473 if len(compiler_fname_list) != 1:
474 print ('Internal error, ambiguous toolchains: %s' %
475 (', '.join(compiler_fname)))
477 toolchain = Toolchain(compiler_fname_list[0], True, True)
479 # Make sure that it will be found by buildman
480 if not self.TestSettingsHasPath(dirpath):
481 print ("Adding 'download' to config file '%s'" %
482 bsettings.config_fname)
483 tools_dir = os.path.dirname(dirpath)
484 bsettings.SetItem('toolchain', 'download', '%s/*' % tools_dir)