]> git.sur5r.net Git - u-boot/blobdiff - tools/buildman/builderthread.py
buildman: Extract environment as part of each build
[u-boot] / tools / buildman / builderthread.py
index 7384a72be6da497e4e37c0a87c929fc1a5d6e38b..c84ba6acf11a64a611ae2e7a8d209391da38ccd3 100644 (file)
@@ -1,12 +1,12 @@
+# SPDX-License-Identifier: GPL-2.0+
 # Copyright (c) 2014 Google, Inc
 #
-# SPDX-License-Identifier:      GPL-2.0+
-#
 
 import errno
 import glob
 import os
 import shutil
+import sys
 import threading
 
 import command
@@ -27,6 +27,9 @@ def Mkdir(dirname, parents = False):
             os.mkdir(dirname)
     except OSError as err:
         if err.errno == errno.EEXIST:
+            if os.path.realpath('.') == os.path.realpath(dirname):
+                print "Cannot create the current working directory '%s'!" % dirname
+                sys.exit(1)
             pass
         else:
             raise
@@ -80,11 +83,13 @@ class BuilderThread(threading.Thread):
         thread_num: Our thread number (0-n-1), used to decide on a
                 temporary directory
     """
-    def __init__(self, builder, thread_num):
+    def __init__(self, builder, thread_num, incremental, per_board_out_dir):
         """Set up a new builder thread"""
         threading.Thread.__init__(self)
         self.builder = builder
         self.thread_num = thread_num
+        self.incremental = incremental
+        self.per_board_out_dir = per_board_out_dir
 
     def Make(self, commit, brd, stage, cwd, *args, **kwargs):
         """Run 'make' on a particular commit and board.
@@ -108,8 +113,8 @@ class BuilderThread(threading.Thread):
         return self.builder.do_make(commit, brd, stage, cwd, *args,
                 **kwargs)
 
-    def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build,
-                  force_build_failures):
+    def RunCommit(self, commit_upto, brd, work_dir, do_config, config_only,
+                  force_build, force_build_failures):
         """Build a particular commit.
 
         If the build is already done, and we are not forcing a build, we skip
@@ -120,6 +125,7 @@ class BuilderThread(threading.Thread):
             brd: Board object to build
             work_dir: Directory to which the source will be checked out
             do_config: True to run a make <board>_defconfig on the source
+            config_only: Only configure the source, do not build it
             force_build: Force a build even if one was previously done
             force_build_failures: Force a bulid if the previous result showed
                 failure
@@ -136,7 +142,11 @@ class BuilderThread(threading.Thread):
         if self.builder.in_tree:
             out_dir = work_dir
         else:
-            out_dir = os.path.join(work_dir, 'build')
+            if self.per_board_out_dir:
+                out_rel_dir = os.path.join('..', brd.target)
+            else:
+                out_rel_dir = 'build'
+            out_dir = os.path.join(work_dir, out_rel_dir)
 
         # Check if the job was already completed last time
         done_file = self.builder.GetDoneFile(commit_upto, brd.target)
@@ -197,30 +207,38 @@ class BuilderThread(threading.Thread):
                         #
                         # Symlinks can confuse U-Boot's Makefile since
                         # we may use '..' in our path, so remove them.
-                        work_dir = os.path.realpath(work_dir)
-                        args.append('O=%s/build' % work_dir)
+                        out_dir = os.path.realpath(out_dir)
+                        args.append('O=%s' % out_dir)
                         cwd = None
                         src_dir = os.getcwd()
                     else:
-                        args.append('O=build')
-                if not self.builder.verbose_build:
+                        args.append('O=%s' % out_rel_dir)
+                if self.builder.verbose_build:
+                    args.append('V=1')
+                else:
                     args.append('-s')
                 if self.builder.num_jobs is not None:
                     args.extend(['-j', str(self.builder.num_jobs)])
+                if self.builder.warnings_as_errors:
+                    args.append('KCFLAGS=-Werror')
                 config_args = ['%s_defconfig' % brd.target]
                 config_out = ''
                 args.extend(self.builder.toolchains.GetMakeArguments(brd))
 
                 # If we need to reconfigure, do that now
                 if do_config:
-                    result = self.Make(commit, brd, 'mrproper', cwd,
-                            'mrproper', *args, env=env)
-                    config_out = result.combined
+                    config_out = ''
+                    if not self.incremental:
+                        result = self.Make(commit, brd, 'mrproper', cwd,
+                                'mrproper', *args, env=env)
+                        config_out += result.combined
                     result = self.Make(commit, brd, 'config', cwd,
                             *(args + config_args), env=env)
                     config_out += result.combined
                     do_config = False   # No need to configure next time
                 if result.return_code == 0:
+                    if config_only:
+                        args.append('cfg')
                     result = self.Make(commit, brd, 'build', cwd, *args,
                             env=env)
                 result.stderr = result.stderr.replace(src_dir + '/', '')
@@ -267,13 +285,15 @@ class BuilderThread(threading.Thread):
         outfile = os.path.join(build_dir, 'log')
         with open(outfile, 'w') as fd:
             if result.stdout:
-                fd.write(result.stdout)
+                # We don't want unicode characters in log files
+                fd.write(result.stdout.decode('UTF-8').encode('ASCII', 'replace'))
 
         errfile = self.builder.GetErrFile(result.commit_upto,
                 result.brd.target)
         if result.stderr:
             with open(errfile, 'w') as fd:
-                fd.write(result.stderr)
+                # We don't want unicode characters in log files
+                fd.write(result.stderr.decode('UTF-8').encode('ASCII', 'replace'))
         elif os.path.exists(errfile):
             os.remove(errfile)
 
@@ -294,10 +314,6 @@ class BuilderThread(threading.Thread):
                 print >>fd, 'arch', result.toolchain.arch
                 fd.write('%s' % result.return_code)
 
-            with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
-                print >>fd, 'gcc', result.toolchain.gcc
-                print >>fd, 'path', result.toolchain.path
-
             # Write out the image and function size information and an objdump
             env = result.toolchain.MakeEnvironment(self.builder.full_path)
             lines = []
@@ -335,6 +351,16 @@ class BuilderThread(threading.Thread):
                     lines.append(size_result.stdout.splitlines()[1] + ' ' +
                                  rodata_size)
 
+            # Extract the environment from U-Boot and dump it out
+            cmd = ['%sobjcopy' % self.toolchain.cross, '-O', 'binary',
+                   '-j', '.rodata.default_environment',
+                   'env/built-in.o', 'uboot.env']
+            command.RunPipe([cmd], capture=True,
+                            capture_stderr=True, cwd=result.out_dir,
+                            raise_on_error=False, env=env)
+            ubootenv = os.path.join(result.out_dir, 'uboot.env')
+            self.CopyFiles(result.out_dir, build_dir, '', ['uboot.env'])
+
             # Write out the image sizes file. This is similar to the output
             # of binutil's 'size' utility, but it omits the header line and
             # adds an additional hex value at the end of each line for the
@@ -353,10 +379,9 @@ class BuilderThread(threading.Thread):
 
         # Now write the actual build output
         if keep_outputs:
-            self.CopyFiles(result.out_dir, build_dir, '', ['u-boot', '*.bin',
-                    'u-boot.dtb', '*.map', '*.img',
-                    'spl/u-boot-spl', 'spl/u-boot-spl.bin',
-                    'tpl/u-boot-tpl', 'tpl/u-boot-tpl.bin'])
+            self.CopyFiles(result.out_dir, build_dir, '', ['u-boot*', '*.bin',
+                '*.map', '*.img', 'MLO', 'SPL', 'include/autoconf.mk',
+                'spl/u-boot-spl*'])
 
     def CopyFiles(self, out_dir, build_dir, dirname, patterns):
         """Copy files from the build directory to the output.
@@ -396,7 +421,7 @@ class BuilderThread(threading.Thread):
             force_build = False
             for commit_upto in range(0, len(job.commits), job.step):
                 result, request_config = self.RunCommit(commit_upto, brd,
-                        work_dir, do_config,
+                        work_dir, do_config, self.builder.config_only,
                         force_build or self.builder.force_build,
                         self.builder.force_build_failures)
                 failed = result.return_code or result.stderr
@@ -406,7 +431,7 @@ class BuilderThread(threading.Thread):
                     # with a reconfig.
                     if self.builder.force_config_on_failure:
                         result, request_config = self.RunCommit(commit_upto,
-                            brd, work_dir, True, True, False)
+                            brd, work_dir, True, False, True, False)
                         did_config = True
                 if not self.builder.force_reconfig:
                     do_config = request_config
@@ -450,7 +475,8 @@ class BuilderThread(threading.Thread):
         else:
             # Just build the currently checked-out build
             result, request_config = self.RunCommit(None, brd, work_dir, True,
-                        True, self.builder.force_build_failures)
+                        self.builder.config_only, True,
+                        self.builder.force_build_failures)
             result.commit_upto = 0
             self._WriteResult(result, job.keep_outputs)
             self.builder.out_queue.put(result)
@@ -461,17 +487,7 @@ class BuilderThread(threading.Thread):
         This thread picks a job from the queue, runs it, and then goes to the
         next job.
         """
-        alive = True
         while True:
             job = self.builder.queue.get()
-            if self.builder.active and alive:
-                self.RunJob(job)
-            '''
-            try:
-                if self.builder.active and alive:
-                    self.RunJob(job)
-            except Exception as err:
-                alive = False
-                print err
-            '''
+            self.RunJob(job)
             self.builder.queue.task_done()