3 # Author: Masahiro Yamada <yamada.masahiro@socionext.com>
5 # SPDX-License-Identifier: GPL-2.0+
9 Move config options from headers to defconfig files.
11 Since Kconfig was introduced to U-Boot, we have worked on moving
12 config options from headers to Kconfig (defconfig).
14 This tool intends to help this tremendous work.
20 First, you must edit the Kconfig to add the menu entries for the configs
23 And then run this tool giving CONFIG names you want to move.
24 For example, if you want to move CONFIG_CMD_USB and CONFIG_SYS_TEXT_BASE,
25 simply type as follows:
27 $ tools/moveconfig.py CONFIG_CMD_USB CONFIG_SYS_TEXT_BASE
29 The tool walks through all the defconfig files and move the given CONFIGs.
31 The log is also displayed on the terminal.
33 The log is printed for each defconfig as follows:
41 <defconfig_name> is the name of the defconfig.
43 <action*> shows what the tool did for that defconfig.
44 It looks like one of the followings:
47 This config option was moved to the defconfig
49 - CONFIG_... is not defined in Kconfig. Do nothing.
50 The entry for this CONFIG was not found in Kconfig.
51 There are two common cases:
52 - You forgot to create an entry for the CONFIG before running
53 this tool, or made a typo in a CONFIG passed to this tool.
54 - The entry was hidden due to unmet 'depends on'.
55 This is correct behavior.
57 - 'CONFIG_...' is the same as the define in Kconfig. Do nothing.
58 The define in the config header matched the one in Kconfig.
59 We do not need to touch it.
61 - Undefined. Do nothing.
62 This config option was not found in the config header.
65 - Compiler is missing. Do nothing.
66 The compiler specified for this architecture was not found
67 in your PATH environment.
68 (If -e option is passed, the tool exits immediately.)
71 An error occurred during processing this defconfig. Skipped.
72 (If -e option is passed, the tool exits immediately on error.)
74 Finally, you will be asked, Clean up headers? [y/n]:
76 If you say 'y' here, the unnecessary config defines are removed
77 from the config headers (include/configs/*.h).
78 It just uses the regex method, so you should not rely on it.
79 Just in case, please do 'git diff' to see what happened.
85 This tool runs configuration and builds include/autoconf.mk for every
86 defconfig. The config options defined in Kconfig appear in the .config
87 file (unless they are hidden because of unmet dependency.)
88 On the other hand, the config options defined by board headers are seen
89 in include/autoconf.mk. The tool looks for the specified options in both
90 of them to decide the appropriate action for the options. If the given
91 config option is found in the .config, but its value does not match the
92 one from the board header, the config option in the .config is replaced
93 with the define in the board header. Then, the .config is synced by
94 "make savedefconfig" and the defconfig is updated with it.
96 For faster processing, this tool handles multi-threading. It creates
97 separate build directories where the out-of-tree build is run. The
98 temporary build directories are automatically created and deleted as
99 needed. The number of threads are chosen based on the number of the CPU
100 cores of your system although you can change it via -j (--jobs) option.
106 Appropriate toolchain are necessary to generate include/autoconf.mk
107 for all the architectures supported by U-Boot. Most of them are available
108 at the kernel.org site, some are not provided by kernel.org.
110 The default per-arch CROSS_COMPILE used by this tool is specified by
111 the list below, CROSS_COMPILE. You may wish to update the list to
112 use your own. Instead of modifying the list directly, you can give
113 them via environments.
120 Surround each portion of the log with escape sequences to display it
121 in color on the terminal.
124 Specify a file containing a list of defconfigs to move
127 Perform a trial run that does not make any changes. It is useful to
128 see what is going to happen before one actually runs it.
131 Exit immediately if Make exits with a non-zero status while processing
135 Only cleanup the headers; skip the defconfig processing
138 Specify the number of threads to run simultaneously. If not specified,
139 the number of threads is the same as the number of CPU cores.
142 Show any build errors as boards are built
144 To see the complete list of supported options, run
146 $ tools/moveconfig.py -h
152 import multiprocessing
162 SHOW_GNU_MAKE = 'scripts/show-gnu-make'
165 # Here is the list of cross-tools I use.
166 # Most of them are available at kernel.org
167 # (https://www.kernel.org/pub/tools/crosstool/files/bin/), except the followings:
168 # arc: https://github.com/foss-for-synopsys-dwc-arc-processors/toolchain/releases
169 # blackfin: http://sourceforge.net/projects/adi-toolchain/files/
170 # nds32: http://osdk.andestech.com/packages/nds32le-linux-glibc-v1.tgz
171 # nios2: https://sourcery.mentor.com/GNUToolchain/subscription42545
172 # sh: http://sourcery.mentor.com/public/gnu_toolchain/sh-linux-gnu
174 # openrisc kernel.org toolchain is out of date, download latest one from
175 # http://opencores.org/or1k/OpenRISC_GNU_tool_chain#Prebuilt_versions
178 'aarch64': 'aarch64-linux-',
179 'arm': 'arm-unknown-linux-gnueabi-',
180 'avr32': 'avr32-linux-',
181 'blackfin': 'bfin-elf-',
182 'm68k': 'm68k-linux-',
183 'microblaze': 'microblaze-linux-',
184 'mips': 'mips-linux-',
185 'nds32': 'nds32le-linux-',
186 'nios2': 'nios2-linux-gnu-',
187 'openrisc': 'or1k-elf-',
188 'powerpc': 'powerpc-linux-',
189 'sh': 'sh-linux-gnu-',
190 'sparc': 'sparc-linux-',
197 STATE_SAVEDEFCONFIG = 3
208 COLOR_PURPLE = '0;35'
210 COLOR_LIGHT_GRAY = '0;37'
211 COLOR_DARK_GRAY = '1;30'
212 COLOR_LIGHT_RED = '1;31'
213 COLOR_LIGHT_GREEN = '1;32'
214 COLOR_YELLOW = '1;33'
215 COLOR_LIGHT_BLUE = '1;34'
216 COLOR_LIGHT_PURPLE = '1;35'
217 COLOR_LIGHT_CYAN = '1;36'
220 ### helper functions ###
222 """Get the file object of '/dev/null' device."""
224 devnull = subprocess.DEVNULL # py3k
225 except AttributeError:
226 devnull = open(os.devnull, 'wb')
229 def check_top_directory():
230 """Exit if we are not at the top of source directory."""
231 for f in ('README', 'Licenses'):
232 if not os.path.exists(f):
233 sys.exit('Please run at the top of source directory.')
235 def check_clean_directory():
236 """Exit if the source tree is not clean."""
237 for f in ('.config', 'include/config'):
238 if os.path.exists(f):
239 sys.exit("source tree is not clean, please run 'make mrproper'")
242 """Get the command name of GNU Make.
244 U-Boot needs GNU Make for building, but the command name is not
245 necessarily "make". (for example, "gmake" on FreeBSD).
246 Returns the most appropriate command name on your system.
248 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE)
249 ret = process.communicate()
250 if process.returncode:
251 sys.exit('GNU Make not found')
252 return ret[0].rstrip()
254 def color_text(color_enabled, color, string):
255 """Return colored string."""
257 # LF should not be surrounded by the escape sequence.
258 # Otherwise, additional whitespace or line-feed might be printed.
259 return '\n'.join([ '\033[' + color + 'm' + s + '\033[0m' if s else ''
260 for s in string.split('\n') ])
264 def update_cross_compile(color_enabled):
265 """Update per-arch CROSS_COMPILE via environment variables
267 The default CROSS_COMPILE values are available
268 in the CROSS_COMPILE list above.
270 You can override them via environment variables
271 CROSS_COMPILE_{ARCH}.
273 For example, if you want to override toolchain prefixes
274 for ARM and PowerPC, you can do as follows in your shell:
276 export CROSS_COMPILE_ARM=...
277 export CROSS_COMPILE_POWERPC=...
279 Then, this function checks if specified compilers really exist in your
284 for arch in os.listdir('arch'):
285 if os.path.exists(os.path.join('arch', arch, 'Makefile')):
288 # arm64 is a special case
289 archs.append('aarch64')
292 env = 'CROSS_COMPILE_' + arch.upper()
293 cross_compile = os.environ.get(env)
294 if not cross_compile:
295 cross_compile = CROSS_COMPILE.get(arch, '')
297 for path in os.environ["PATH"].split(os.pathsep):
298 gcc_path = os.path.join(path, cross_compile + 'gcc')
299 if os.path.isfile(gcc_path) and os.access(gcc_path, os.X_OK):
302 print >> sys.stderr, color_text(color_enabled, COLOR_YELLOW,
303 'warning: %sgcc: not found in PATH. %s architecture boards will be skipped'
304 % (cross_compile, arch))
307 CROSS_COMPILE[arch] = cross_compile
309 def cleanup_one_header(header_path, patterns, dry_run):
310 """Clean regex-matched lines away from a file.
313 header_path: path to the cleaned file.
314 patterns: list of regex patterns. Any lines matching to these
315 patterns are deleted.
316 dry_run: make no changes, but still display log.
318 with open(header_path) as f:
319 lines = f.readlines()
322 for i, line in enumerate(lines):
323 for pattern in patterns:
324 m = pattern.search(line)
326 print '%s: %s: %s' % (header_path, i + 1, line),
330 if dry_run or not matched:
333 with open(header_path, 'w') as f:
334 for i, line in enumerate(lines):
338 def cleanup_headers(configs, dry_run):
339 """Delete config defines from board headers.
342 configs: A list of CONFIGs to remove.
343 dry_run: make no changes, but still display log.
346 choice = raw_input('Clean up headers? [y/n]: ').lower()
348 if choice == 'y' or choice == 'n':
355 for config in configs:
356 patterns.append(re.compile(r'#\s*define\s+%s\W' % config))
357 patterns.append(re.compile(r'#\s*undef\s+%s\W' % config))
359 for dir in 'include', 'arch', 'board':
360 for (dirpath, dirnames, filenames) in os.walk(dir):
361 for filename in filenames:
362 if not fnmatch.fnmatch(filename, '*~'):
363 cleanup_one_header(os.path.join(dirpath, filename),
369 """Progress Indicator"""
371 def __init__(self, total):
372 """Create a new progress indicator.
375 total: A number of defconfig files to process.
381 """Increment the number of processed defconfig files."""
386 """Display the progress."""
387 print ' %d defconfigs out of %d\r' % (self.current, self.total),
392 """A parser of .config and include/autoconf.mk."""
394 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"')
395 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"')
397 def __init__(self, configs, options, build_dir):
398 """Create a new parser.
401 configs: A list of CONFIGs to move.
402 options: option flags.
403 build_dir: Build directory.
405 self.configs = configs
406 self.options = options
407 self.dotconfig = os.path.join(build_dir, '.config')
408 self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk')
409 self.config_autoconf = os.path.join(build_dir, 'include', 'config',
411 self.defconfig = os.path.join(build_dir, 'defconfig')
413 def get_cross_compile(self):
414 """Parse .config file and return CROSS_COMPILE.
417 A string storing the compiler prefix for the architecture.
418 Return a NULL string for architectures that do not require
419 compiler prefix (Sandbox and native build is the case).
420 Return None if the specified compiler is missing in your PATH.
421 Caller should distinguish '' and None.
425 for line in open(self.dotconfig):
426 m = self.re_arch.match(line)
430 m = self.re_cpu.match(line)
438 if arch == 'arm' and cpu == 'armv8':
441 return CROSS_COMPILE.get(arch, None)
443 def parse_one_config(self, config, dotconfig_lines, autoconf_lines):
444 """Parse .config, defconfig, include/autoconf.mk for one config.
446 This function looks for the config options in the lines from
447 defconfig, .config, and include/autoconf.mk in order to decide
448 which action should be taken for this defconfig.
451 config: CONFIG name to parse.
452 dotconfig_lines: lines from the .config file.
453 autoconf_lines: lines from the include/autoconf.mk file.
456 A tupple of the action for this defconfig and the line
457 matched for the config.
459 not_set = '# %s is not set' % config
461 for line in dotconfig_lines:
463 if line.startswith(config + '=') or line == not_set:
467 return (ACTION_NO_ENTRY, config)
469 for line in autoconf_lines:
471 if line.startswith(config + '='):
477 if old_val == new_val:
478 return (ACTION_NO_CHANGE, new_val)
480 # If this CONFIG is neither bool nor trisate
481 if old_val[-2:] != '=y' and old_val[-2:] != '=m' and old_val != not_set:
482 # tools/scripts/define2mk.sed changes '1' to 'y'.
483 # This is a problem if the CONFIG is int type.
484 # Check the type in Kconfig and handle it correctly.
485 if new_val[-2:] == '=y':
486 new_val = new_val[:-1] + '1'
488 return (ACTION_MOVE, new_val)
490 def update_dotconfig(self):
491 """Parse files for the config options and update the .config.
493 This function parses the generated .config and include/autoconf.mk
494 searching the target options.
495 Move the config option(s) to the .config as needed.
498 defconfig: defconfig name.
501 Return a tuple of (updated flag, log string).
502 The "updated flag" is True if the .config was updated, False
503 otherwise. The "log string" shows what happend to the .config.
509 with open(self.dotconfig) as f:
510 dotconfig_lines = f.readlines()
512 with open(self.autoconf) as f:
513 autoconf_lines = f.readlines()
515 for config in self.configs:
516 result = self.parse_one_config(config, dotconfig_lines,
518 results.append(result)
522 for (action, value) in results:
523 if action == ACTION_MOVE:
524 actlog = "Move '%s'" % value
525 log_color = COLOR_LIGHT_GREEN
526 elif action == ACTION_NO_ENTRY:
527 actlog = "%s is not defined in Kconfig. Do nothing." % value
528 log_color = COLOR_LIGHT_BLUE
529 elif action == ACTION_NO_CHANGE:
530 actlog = "'%s' is the same as the define in Kconfig. Do nothing." \
532 log_color = COLOR_LIGHT_PURPLE
534 sys.exit("Internal Error. This should not happen.")
536 log += color_text(self.options.color, log_color, actlog) + '\n'
538 with open(self.dotconfig, 'a') as f:
539 for (action, value) in results:
540 if action == ACTION_MOVE:
541 f.write(value + '\n')
544 self.results = results
545 os.remove(self.config_autoconf)
546 os.remove(self.autoconf)
548 return (updated, log)
550 def check_defconfig(self):
551 """Check the defconfig after savedefconfig
554 Return additional log if moved CONFIGs were removed again by
555 'make savedefconfig'.
560 with open(self.defconfig) as f:
561 defconfig_lines = f.readlines()
563 for (action, value) in self.results:
564 if action != ACTION_MOVE:
566 if not value + '\n' in defconfig_lines:
567 log += color_text(self.options.color, COLOR_YELLOW,
568 "'%s' was removed by savedefconfig.\n" %
575 """A slot to store a subprocess.
577 Each instance of this class handles one subprocess.
578 This class is useful to control multiple threads
579 for faster processing.
582 def __init__(self, configs, options, progress, devnull, make_cmd):
583 """Create a new process slot.
586 configs: A list of CONFIGs to move.
587 options: option flags.
588 progress: A progress indicator.
589 devnull: A file object of '/dev/null'.
590 make_cmd: command name of GNU Make.
592 self.options = options
593 self.progress = progress
594 self.build_dir = tempfile.mkdtemp()
595 self.devnull = devnull
596 self.make_cmd = (make_cmd, 'O=' + self.build_dir)
597 self.parser = KconfigParser(configs, options, self.build_dir)
598 self.state = STATE_IDLE
599 self.failed_boards = []
602 """Delete the working directory
604 This function makes sure the temporary directory is cleaned away
605 even if Python suddenly dies due to error. It should be done in here
606 because it is guranteed the destructor is always invoked when the
607 instance of the class gets unreferenced.
609 If the subprocess is still running, wait until it finishes.
611 if self.state != STATE_IDLE:
612 while self.ps.poll() == None:
614 shutil.rmtree(self.build_dir)
616 def add(self, defconfig):
617 """Assign a new subprocess for defconfig and add it to the slot.
619 If the slot is vacant, create a new subprocess for processing the
620 given defconfig and add it to the slot. Just returns False if
621 the slot is occupied (i.e. the current subprocess is still running).
624 defconfig: defconfig name.
627 Return True on success or False on failure
629 if self.state != STATE_IDLE:
631 cmd = list(self.make_cmd)
632 cmd.append(defconfig)
633 self.ps = subprocess.Popen(cmd, stdout=self.devnull,
634 stderr=subprocess.PIPE)
635 self.defconfig = defconfig
636 self.state = STATE_DEFCONFIG
641 """Check the status of the subprocess and handle it as needed.
643 Returns True if the slot is vacant (i.e. in idle state).
644 If the configuration is successfully finished, assign a new
645 subprocess to build include/autoconf.mk.
646 If include/autoconf.mk is generated, invoke the parser to
647 parse the .config and the include/autoconf.mk, moving
648 config options to the .config as needed.
649 If the .config was updated, run "make savedefconfig" to sync
650 it, update the original defconfig, and then set the slot back
654 Return True if the subprocess is terminated, False otherwise
656 if self.state == STATE_IDLE:
659 if self.ps.poll() == None:
662 if self.ps.poll() != 0:
663 self.log += color_text(self.options.color, COLOR_LIGHT_RED,
664 "Failed to process.\n")
665 if self.options.verbose:
666 self.log += color_text(self.options.color, COLOR_LIGHT_CYAN,
667 self.ps.stderr.read())
671 if self.state == STATE_AUTOCONF:
672 (updated, log) = self.parser.update_dotconfig()
678 self.log += color_text(self.options.color, COLOR_LIGHT_GREEN,
679 "Syncing by savedefconfig...\n")
680 cmd = list(self.make_cmd)
681 cmd.append('savedefconfig')
682 self.ps = subprocess.Popen(cmd, stdout=self.devnull,
683 stderr=subprocess.PIPE)
684 self.state = STATE_SAVEDEFCONFIG
687 if self.state == STATE_SAVEDEFCONFIG:
688 self.log += self.parser.check_defconfig()
689 orig_defconfig = os.path.join('configs', self.defconfig)
690 new_defconfig = os.path.join(self.build_dir, 'defconfig')
691 updated = not filecmp.cmp(orig_defconfig, new_defconfig)
694 self.log += color_text(self.options.color, COLOR_LIGHT_GREEN,
695 "defconfig was updated.\n")
697 if not self.options.dry_run and updated:
698 shutil.move(new_defconfig, orig_defconfig)
702 self.cross_compile = self.parser.get_cross_compile()
703 if self.cross_compile is None:
704 self.log += color_text(self.options.color, COLOR_YELLOW,
705 "Compiler is missing. Do nothing.\n")
709 cmd = list(self.make_cmd)
710 if self.cross_compile:
711 cmd.append('CROSS_COMPILE=%s' % self.cross_compile)
712 cmd.append('KCONFIG_IGNORE_DUPLICATES=1')
713 cmd.append('include/config/auto.conf')
714 self.ps = subprocess.Popen(cmd, stdout=self.devnull,
715 stderr=subprocess.PIPE)
716 self.state = STATE_AUTOCONF
719 def finish(self, success):
720 """Display log along with progress and go to the idle state.
723 success: Should be True when the defconfig was processed
724 successfully, or False when it fails.
726 # output at least 30 characters to hide the "* defconfigs out of *".
727 log = self.defconfig.ljust(30) + '\n'
729 log += '\n'.join([ ' ' + s for s in self.log.split('\n') ])
730 # Some threads are running in parallel.
731 # Print log atomically to not mix up logs from different threads.
732 print >> (sys.stdout if success else sys.stderr), log
735 if self.options.exit_on_error:
736 sys.exit("Exit on error.")
737 # If --exit-on-error flag is not set, skip this board and continue.
738 # Record the failed board.
739 self.failed_boards.append(self.defconfig)
743 self.state = STATE_IDLE
745 def get_failed_boards(self):
746 """Returns a list of failed boards (defconfigs) in this slot.
748 return self.failed_boards
752 """Controller of the array of subprocess slots."""
754 def __init__(self, configs, options, progress):
755 """Create a new slots controller.
758 configs: A list of CONFIGs to move.
759 options: option flags.
760 progress: A progress indicator.
762 self.options = options
764 devnull = get_devnull()
765 make_cmd = get_make_cmd()
766 for i in range(options.jobs):
767 self.slots.append(Slot(configs, options, progress, devnull,
770 def add(self, defconfig):
771 """Add a new subprocess if a vacant slot is found.
774 defconfig: defconfig name to be put into.
777 Return True on success or False on failure
779 for slot in self.slots:
780 if slot.add(defconfig):
785 """Check if there is a vacant slot.
788 Return True if at lease one vacant slot is found, False otherwise.
790 for slot in self.slots:
796 """Check if all slots are vacant.
799 Return True if all the slots are vacant, False otherwise.
802 for slot in self.slots:
807 def show_failed_boards(self):
808 """Display all of the failed boards (defconfigs)."""
811 for slot in self.slots:
812 failed_boards += slot.get_failed_boards()
814 if len(failed_boards) > 0:
815 msg = [ "The following boards were not processed due to error:" ]
818 print >> sys.stderr, color_text(self.options.color,
819 COLOR_LIGHT_RED, line)
821 with open('moveconfig.failed', 'w') as f:
822 for board in failed_boards:
823 f.write(board + '\n')
825 def move_config(configs, options):
826 """Move config options to defconfig files.
829 configs: A list of CONFIGs to move.
830 options: option flags
832 if len(configs) == 0:
833 print 'Nothing to do. exit.'
836 print 'Move %s (jobs: %d)' % (', '.join(configs), options.jobs)
838 if options.defconfigs:
839 defconfigs = [line.strip() for line in open(options.defconfigs)]
840 for i, defconfig in enumerate(defconfigs):
841 if not defconfig.endswith('_defconfig'):
842 defconfigs[i] = defconfig + '_defconfig'
843 if not os.path.exists(os.path.join('configs', defconfigs[i])):
844 sys.exit('%s - defconfig does not exist. Stopping.' %
847 # All the defconfig files to be processed
849 for (dirpath, dirnames, filenames) in os.walk('configs'):
850 dirpath = dirpath[len('configs') + 1:]
851 for filename in fnmatch.filter(filenames, '*_defconfig'):
852 defconfigs.append(os.path.join(dirpath, filename))
854 progress = Progress(len(defconfigs))
855 slots = Slots(configs, options, progress)
857 # Main loop to process defconfig files:
858 # Add a new subprocess into a vacant slot.
859 # Sleep if there is no available slot.
860 for defconfig in defconfigs:
861 while not slots.add(defconfig):
862 while not slots.available():
863 # No available slot: sleep for a while
864 time.sleep(SLEEP_TIME)
866 # wait until all the subprocesses finish
867 while not slots.empty():
868 time.sleep(SLEEP_TIME)
871 slots.show_failed_boards()
875 cpu_count = multiprocessing.cpu_count()
876 except NotImplementedError:
879 parser = optparse.OptionParser()
881 parser.add_option('-c', '--color', action='store_true', default=False,
882 help='display the log in color')
883 parser.add_option('-d', '--defconfigs', type='string',
884 help='a file containing a list of defconfigs to move')
885 parser.add_option('-n', '--dry-run', action='store_true', default=False,
886 help='perform a trial run (show log with no changes)')
887 parser.add_option('-e', '--exit-on-error', action='store_true',
889 help='exit immediately on any error')
890 parser.add_option('-H', '--headers-only', dest='cleanup_headers_only',
891 action='store_true', default=False,
892 help='only cleanup the headers')
893 parser.add_option('-j', '--jobs', type='int', default=cpu_count,
894 help='the number of jobs to run simultaneously')
895 parser.add_option('-v', '--verbose', action='store_true', default=False,
896 help='show any build errors as boards are built')
897 parser.usage += ' CONFIG ...'
899 (options, configs) = parser.parse_args()
901 if len(configs) == 0:
905 # prefix the option name with CONFIG_ if missing
906 configs = [ config if config.startswith('CONFIG_') else 'CONFIG_' + config
907 for config in configs ]
909 check_top_directory()
911 check_clean_directory()
913 update_cross_compile(options.color)
915 if not options.cleanup_headers_only:
916 move_config(configs, options)
918 cleanup_headers(configs, options.dry_run)
920 if __name__ == '__main__':