3 # Author: Masahiro Yamada <yamada.masahiro@socionext.com>
5 # SPDX-License-Identifier: GPL-2.0+
9 Move config options from headers to defconfig files.
11 Since Kconfig was introduced to U-Boot, we have worked on moving
12 config options from headers to Kconfig (defconfig).
14 This tool intends to help this tremendous work.
20 This tool takes one input file. (let's say 'recipe' file here.)
21 The recipe describes the list of config options you want to move.
22 Each line takes the form:
23 <config_name> <type> <default>
24 (the fields must be separated with whitespaces.)
26 <config_name> is the name of config option.
28 <type> is the type of the option. It must be one of bool, tristate,
31 <default> is the default value of the option. It must be appropriate
32 value corresponding to the option type. It must be either y or n for
33 the bool type. Tristate options can also take m (although U-Boot has
34 not supported the module feature).
36 You can add two or more lines in the recipe file, so you can move
37 multiple options at once.
39 Let's say, for example, you want to move CONFIG_CMD_USB and
42 The type should be bool, hex, respectively. So, the recipe file
43 should look like this:
47 CONFIG_SYS_TEXT_BASE hex 0x00000000
49 Next you must edit the Kconfig to add the menu entries for the configs
52 And then run this tool giving the file name of the recipe
54 $ tools/moveconfig.py recipe
56 The tool walks through all the defconfig files to move the config
57 options specified by the recipe file.
59 The log is also displayed on the terminal.
61 Each line is printed in the format
62 <defconfig_name> : <action>
64 <defconfig_name> is the name of the defconfig
65 (without the suffix _defconfig).
67 <action> shows what the tool did for that defconfig.
68 It looks like one of the followings:
71 This config option was moved to the defconfig
73 - Default value 'CONFIG_...'. Do nothing.
74 The value of this option is the same as default.
75 We do not have to add it to the defconfig.
77 - 'CONFIG_...' already exists in Kconfig. Do nothing.
78 This config option is already defined in Kconfig.
79 We do not need/want to touch it.
81 - Undefined. Do nothing.
82 This config option was not found in the config header.
85 - Failed to process. Skip.
86 An error occurred during processing this defconfig. Skipped.
87 (If -e option is passed, the tool exits immediately on error.)
89 Finally, you will be asked, Clean up headers? [y/n]:
91 If you say 'y' here, the unnecessary config defines are removed
92 from the config headers (include/configs/*.h).
93 It just uses the regex method, so you should not rely on it.
94 Just in case, please do 'git diff' to see what happened.
100 This tool runs configuration and builds include/autoconf.mk for every
101 defconfig. The config options defined in Kconfig appear in the .config
102 file (unless they are hidden because of unmet dependency.)
103 On the other hand, the config options defined by board headers are seen
104 in include/autoconf.mk. The tool looks for the specified options in both
105 of them to decide the appropriate action for the options. If the option
106 is found in the .config or the value is the same as the specified default,
107 the option does not need to be touched. If the option is found in
108 include/autoconf.mk, but not in the .config, and the value is different
109 from the default, the tools adds the option to the defconfig.
111 For faster processing, this tool handles multi-threading. It creates
112 separate build directories where the out-of-tree build is run. The
113 temporary build directories are automatically created and deleted as
114 needed. The number of threads are chosen based on the number of the CPU
115 cores of your system although you can change it via -j (--jobs) option.
121 Appropriate toolchain are necessary to generate include/autoconf.mk
122 for all the architectures supported by U-Boot. Most of them are available
123 at the kernel.org site, some are not provided by kernel.org.
125 The default per-arch CROSS_COMPILE used by this tool is specified by
126 the list below, CROSS_COMPILE. You may wish to update the list to
127 use your own. Instead of modifying the list directly, you can give
128 them via environments.
135 Surround each portion of the log with escape sequences to display it
136 in color on the terminal.
139 Peform a trial run that does not make any changes. It is useful to
140 see what is going to happen before one actually runs it.
143 Exit immediately if Make exits with a non-zero status while processing
147 Specify the number of threads to run simultaneously. If not specified,
148 the number of threads is the same as the number of CPU cores.
150 To see the complete list of supported options, run
152 $ tools/moveconfig.py -h
157 import multiprocessing
167 SHOW_GNU_MAKE = 'scripts/show-gnu-make'
170 # Here is the list of cross-tools I use.
171 # Most of them are available at kernel.org
172 # (https://www.kernel.org/pub/tools/crosstool/files/bin/), except the followings:
173 # arc: https://github.com/foss-for-synopsys-dwc-arc-processors/toolchain/releases
174 # blackfin: http://sourceforge.net/projects/adi-toolchain/files/
175 # nds32: http://osdk.andestech.com/packages/
176 # nios2: https://sourcery.mentor.com/GNUToolchain/subscription42545
177 # sh: http://sourcery.mentor.com/public/gnu_toolchain/sh-linux-gnu
180 'aarch64': 'aarch64-linux-',
181 'arm': 'arm-unknown-linux-gnueabi-',
182 'avr32': 'avr32-linux-',
183 'blackfin': 'bfin-elf-',
184 'm68k': 'm68k-linux-',
185 'microblaze': 'microblaze-linux-',
186 'mips': 'mips-linux-',
187 'nds32': 'nds32le-linux-',
188 'nios2': 'nios2-linux-gnu-',
189 'openrisc': 'or32-linux-',
190 'powerpc': 'powerpc-linux-',
191 'sh': 'sh-linux-gnu-',
192 'sparc': 'sparc-linux-',
199 STATE_SAVEDEFCONFIG = 3
202 ACTION_DEFAULT_VALUE = 1
203 ACTION_ALREADY_EXIST = 2
211 COLOR_PURPLE = '0;35'
213 COLOR_LIGHT_GRAY = '0;37'
214 COLOR_DARK_GRAY = '1;30'
215 COLOR_LIGHT_RED = '1;31'
216 COLOR_LIGHT_GREEN = '1;32'
217 COLOR_YELLOW = '1;33'
218 COLOR_LIGHT_BLUE = '1;34'
219 COLOR_LIGHT_PURPLE = '1;35'
220 COLOR_LIGHT_CYAN = '1;36'
223 ### helper functions ###
225 """Get the file object of '/dev/null' device."""
227 devnull = subprocess.DEVNULL # py3k
228 except AttributeError:
229 devnull = open(os.devnull, 'wb')
232 def check_top_directory():
233 """Exit if we are not at the top of source directory."""
234 for f in ('README', 'Licenses'):
235 if not os.path.exists(f):
236 sys.exit('Please run at the top of source directory.')
239 """Get the command name of GNU Make.
241 U-Boot needs GNU Make for building, but the command name is not
242 necessarily "make". (for example, "gmake" on FreeBSD).
243 Returns the most appropriate command name on your system.
245 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE)
246 ret = process.communicate()
247 if process.returncode:
248 sys.exit('GNU Make not found')
249 return ret[0].rstrip()
251 def color_text(color_enabled, color, string):
252 """Return colored string."""
254 return '\033[' + color + 'm' + string + '\033[0m'
258 def log_msg(color_enabled, color, defconfig, msg):
259 """Return the formated line for the log."""
260 return defconfig[:-len('_defconfig')].ljust(37) + ': ' + \
261 color_text(color_enabled, color, msg) + '\n'
263 def update_cross_compile():
264 """Update per-arch CROSS_COMPILE via enviroment variables
266 The default CROSS_COMPILE values are available
267 in the CROSS_COMPILE list above.
269 You can override them via enviroment variables
270 CROSS_COMPILE_{ARCH}.
272 For example, if you want to override toolchain prefixes
273 for ARM and PowerPC, you can do as follows in your shell:
275 export CROSS_COMPILE_ARM=...
276 export CROSS_COMPILE_POWERPC=...
280 for arch in os.listdir('arch'):
281 if os.path.exists(os.path.join('arch', arch, 'Makefile')):
284 # arm64 is a special case
285 archs.append('aarch64')
288 env = 'CROSS_COMPILE_' + arch.upper()
289 cross_compile = os.environ.get(env)
291 CROSS_COMPILE[arch] = cross_compile
293 def cleanup_one_header(header_path, patterns, dry_run):
294 """Clean regex-matched lines away from a file.
297 header_path: path to the cleaned file.
298 patterns: list of regex patterns. Any lines matching to these
299 patterns are deleted.
300 dry_run: make no changes, but still display log.
302 with open(header_path) as f:
303 lines = f.readlines()
306 for i, line in enumerate(lines):
307 for pattern in patterns:
308 m = pattern.search(line)
310 print '%s: %s: %s' % (header_path, i + 1, line),
314 if dry_run or not matched:
317 with open(header_path, 'w') as f:
318 for i, line in enumerate(lines):
322 def cleanup_headers(config_attrs, dry_run):
323 """Delete config defines from board headers.
326 config_attrs: A list of dictionaris, each of them includes the name,
327 the type, and the default value of the target config.
328 dry_run: make no changes, but still display log.
331 choice = raw_input('Clean up headers? [y/n]: ').lower()
333 if choice == 'y' or choice == 'n':
340 for config_attr in config_attrs:
341 config = config_attr['config']
342 patterns.append(re.compile(r'#\s*define\s+%s\W' % config))
343 patterns.append(re.compile(r'#\s*undef\s+%s\W' % config))
345 for (dirpath, dirnames, filenames) in os.walk('include'):
346 for filename in filenames:
347 if not fnmatch.fnmatch(filename, '*~'):
348 cleanup_one_header(os.path.join(dirpath, filename), patterns,
354 """A parser of .config and include/autoconf.mk."""
356 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"')
357 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"')
359 def __init__(self, config_attrs, options, build_dir):
360 """Create a new parser.
363 config_attrs: A list of dictionaris, each of them includes the name,
364 the type, and the default value of the target config.
365 options: option flags.
366 build_dir: Build directory.
368 self.config_attrs = config_attrs
369 self.options = options
370 self.build_dir = build_dir
372 def get_cross_compile(self):
373 """Parse .config file and return CROSS_COMPILE.
376 A string storing the compiler prefix for the architecture.
380 dotconfig = os.path.join(self.build_dir, '.config')
381 for line in open(dotconfig):
382 m = self.re_arch.match(line)
386 m = self.re_cpu.match(line)
390 assert arch, 'Error: arch is not defined in %s' % defconfig
393 if arch == 'arm' and cpu == 'armv8':
396 return CROSS_COMPILE.get(arch, '')
398 def parse_one_config(self, config_attr, defconfig_lines, autoconf_lines):
399 """Parse .config, defconfig, include/autoconf.mk for one config.
401 This function looks for the config options in the lines from
402 defconfig, .config, and include/autoconf.mk in order to decide
403 which action should be taken for this defconfig.
406 config_attr: A dictionary including the name, the type,
407 and the default value of the target config.
408 defconfig_lines: lines from the original defconfig file.
409 autoconf_lines: lines from the include/autoconf.mk file.
412 A tupple of the action for this defconfig and the line
413 matched for the config.
415 config = config_attr['config']
416 not_set = '# %s is not set' % config
418 if config_attr['type'] in ('bool', 'tristate') and \
419 config_attr['default'] == 'n':
422 default = config + '=' + config_attr['default']
424 for line in defconfig_lines:
426 if line.startswith(config + '=') or line == not_set:
427 return (ACTION_ALREADY_EXIST, line)
429 if config_attr['type'] in ('bool', 'tristate'):
432 value = '(undefined)'
434 for line in autoconf_lines:
436 if line.startswith(config + '='):
441 action = ACTION_DEFAULT_VALUE
442 elif value == '(undefined)':
443 action = ACTION_UNDEFINED
447 return (action, value)
449 def update_defconfig(self, defconfig):
450 """Parse files for the config options and update the defconfig.
452 This function parses the given defconfig, the generated .config
453 and include/autoconf.mk searching the target options.
454 Move the config option(s) to the defconfig or do nothing if unneeded.
455 Also, display the log to show what happened to this defconfig.
458 defconfig: defconfig name.
461 defconfig_path = os.path.join('configs', defconfig)
462 dotconfig_path = os.path.join(self.build_dir, '.config')
463 autoconf_path = os.path.join(self.build_dir, 'include', 'autoconf.mk')
466 with open(defconfig_path) as f:
467 defconfig_lines = f.readlines()
469 with open(autoconf_path) as f:
470 autoconf_lines = f.readlines()
472 for config_attr in self.config_attrs:
473 result = self.parse_one_config(config_attr, defconfig_lines,
475 results.append(result)
479 for (action, value) in results:
480 if action == ACTION_MOVE:
481 actlog = "Move '%s'" % value
482 log_color = COLOR_LIGHT_GREEN
483 elif action == ACTION_DEFAULT_VALUE:
484 actlog = "Default value '%s'. Do nothing." % value
485 log_color = COLOR_LIGHT_BLUE
486 elif action == ACTION_ALREADY_EXIST:
487 actlog = "'%s' already defined in Kconfig. Do nothing." % value
488 log_color = COLOR_LIGHT_PURPLE
489 elif action == ACTION_UNDEFINED:
490 actlog = "Undefined. Do nothing."
491 log_color = COLOR_DARK_GRAY
493 sys.exit("Internal Error. This should not happen.")
495 log += log_msg(self.options.color, log_color, defconfig, actlog)
497 # Some threads are running in parallel.
498 # Print log in one shot to not mix up logs from different threads.
501 if not self.options.dry_run:
502 with open(dotconfig_path, 'a') as f:
503 for (action, value) in results:
504 if action == ACTION_MOVE:
505 f.write(value + '\n')
507 os.remove(os.path.join(self.build_dir, 'include', 'config', 'auto.conf'))
508 os.remove(autoconf_path)
512 """A slot to store a subprocess.
514 Each instance of this class handles one subprocess.
515 This class is useful to control multiple threads
516 for faster processing.
519 def __init__(self, config_attrs, options, devnull, make_cmd):
520 """Create a new process slot.
523 config_attrs: A list of dictionaris, each of them includes the name,
524 the type, and the default value of the target config.
525 options: option flags.
526 devnull: A file object of '/dev/null'.
527 make_cmd: command name of GNU Make.
529 self.options = options
530 self.build_dir = tempfile.mkdtemp()
531 self.devnull = devnull
532 self.make_cmd = (make_cmd, 'O=' + self.build_dir)
533 self.parser = KconfigParser(config_attrs, options, self.build_dir)
534 self.state = STATE_IDLE
535 self.failed_boards = []
538 """Delete the working directory
540 This function makes sure the temporary directory is cleaned away
541 even if Python suddenly dies due to error. It should be done in here
542 because it is guranteed the destructor is always invoked when the
543 instance of the class gets unreferenced.
545 If the subprocess is still running, wait until it finishes.
547 if self.state != STATE_IDLE:
548 while self.ps.poll() == None:
550 shutil.rmtree(self.build_dir)
552 def add(self, defconfig):
553 """Assign a new subprocess for defconfig and add it to the slot.
555 If the slot is vacant, create a new subprocess for processing the
556 given defconfig and add it to the slot. Just returns False if
557 the slot is occupied (i.e. the current subprocess is still running).
560 defconfig: defconfig name.
563 Return True on success or False on failure
565 if self.state != STATE_IDLE:
567 cmd = list(self.make_cmd)
568 cmd.append(defconfig)
569 self.ps = subprocess.Popen(cmd, stdout=self.devnull)
570 self.defconfig = defconfig
571 self.state = STATE_DEFCONFIG
575 """Check the status of the subprocess and handle it as needed.
577 Returns True if the slot is vacant (i.e. in idle state).
578 If the configuration is successfully finished, assign a new
579 subprocess to build include/autoconf.mk.
580 If include/autoconf.mk is generated, invoke the parser to
581 parse the .config and the include/autoconf.mk, and then set the
582 slot back to the idle state.
585 Return True if the subprocess is terminated, False otherwise
587 if self.state == STATE_IDLE:
590 if self.ps.poll() == None:
593 if self.ps.poll() != 0:
595 print >> sys.stderr, log_msg(self.options.color,
598 "failed to process.")
599 if self.options.exit_on_error:
600 sys.exit("Exit on error.")
602 # If --exit-on-error flag is not set,
603 # skip this board and continue.
604 # Record the failed board.
605 self.failed_boards.append(self.defconfig)
606 self.state = STATE_IDLE
609 if self.state == STATE_AUTOCONF:
610 self.parser.update_defconfig(self.defconfig)
612 """Save off the defconfig in a consistent way"""
613 cmd = list(self.make_cmd)
614 cmd.append('savedefconfig')
615 self.ps = subprocess.Popen(cmd, stdout=self.devnull,
617 self.state = STATE_SAVEDEFCONFIG
620 if self.state == STATE_SAVEDEFCONFIG:
621 defconfig_path = os.path.join(self.build_dir, 'defconfig')
622 shutil.move(defconfig_path,
623 os.path.join('configs', self.defconfig))
624 self.state = STATE_IDLE
627 cross_compile = self.parser.get_cross_compile()
628 cmd = list(self.make_cmd)
630 cmd.append('CROSS_COMPILE=%s' % cross_compile)
631 cmd.append('include/config/auto.conf')
632 self.ps = subprocess.Popen(cmd, stdout=self.devnull)
633 self.state = STATE_AUTOCONF
636 def get_failed_boards(self):
637 """Returns a list of failed boards (defconfigs) in this slot.
639 return self.failed_boards
643 """Controller of the array of subprocess slots."""
645 def __init__(self, config_attrs, options):
646 """Create a new slots controller.
649 config_attrs: A list of dictionaris containing the name, the type,
650 and the default value of the target CONFIG.
651 options: option flags.
653 self.options = options
655 devnull = get_devnull()
656 make_cmd = get_make_cmd()
657 for i in range(options.jobs):
658 self.slots.append(Slot(config_attrs, options, devnull, make_cmd))
660 def add(self, defconfig):
661 """Add a new subprocess if a vacant slot is found.
664 defconfig: defconfig name to be put into.
667 Return True on success or False on failure
669 for slot in self.slots:
670 if slot.add(defconfig):
675 """Check if there is a vacant slot.
678 Return True if at lease one vacant slot is found, False otherwise.
680 for slot in self.slots:
686 """Check if all slots are vacant.
689 Return True if all the slots are vacant, False otherwise.
692 for slot in self.slots:
697 def show_failed_boards(self):
698 """Display all of the failed boards (defconfigs)."""
701 for slot in self.slots:
702 failed_boards += slot.get_failed_boards()
704 if len(failed_boards) > 0:
705 msg = [ "The following boards were not processed due to error:" ]
708 print >> sys.stderr, color_text(self.options.color,
709 COLOR_LIGHT_RED, line)
711 def move_config(config_attrs, options):
712 """Move config options to defconfig files.
715 config_attrs: A list of dictionaris, each of them includes the name,
716 the type, and the default value of the target config.
717 options: option flags
719 check_top_directory()
721 if len(config_attrs) == 0:
722 print 'Nothing to do. exit.'
725 print 'Move the following CONFIG options (jobs: %d)' % options.jobs
726 for config_attr in config_attrs:
727 print ' %s (type: %s, default: %s)' % (config_attr['config'],
729 config_attr['default'])
731 # All the defconfig files to be processed
733 for (dirpath, dirnames, filenames) in os.walk('configs'):
734 dirpath = dirpath[len('configs') + 1:]
735 for filename in fnmatch.filter(filenames, '*_defconfig'):
736 defconfigs.append(os.path.join(dirpath, filename))
738 slots = Slots(config_attrs, options)
740 # Main loop to process defconfig files:
741 # Add a new subprocess into a vacant slot.
742 # Sleep if there is no available slot.
743 for defconfig in defconfigs:
744 while not slots.add(defconfig):
745 while not slots.available():
746 # No available slot: sleep for a while
747 time.sleep(SLEEP_TIME)
749 # wait until all the subprocesses finish
750 while not slots.empty():
751 time.sleep(SLEEP_TIME)
753 slots.show_failed_boards()
755 cleanup_headers(config_attrs, options.dry_run)
757 def bad_recipe(filename, linenum, msg):
758 """Print error message with the file name and the line number and exit."""
759 sys.exit("%s: line %d: error : " % (filename, linenum) + msg)
761 def parse_recipe(filename):
762 """Parse the recipe file and retrieve the config attributes.
764 This function parses the given recipe file and gets the name,
765 the type, and the default value of the target config options.
768 filename: path to file to be parsed.
770 A list of dictionaris, each of them includes the name,
771 the type, and the default value of the target config.
776 for line in open(filename):
777 tokens = line.split()
779 bad_recipe(filename, linenum,
780 "%d fields in this line. Each line must contain 3 fields"
783 (config, type, default) = tokens
785 # prefix the option name with CONFIG_ if missing
786 if not config.startswith('CONFIG_'):
787 config = 'CONFIG_' + config
789 # sanity check of default values
791 if not default in ('y', 'n'):
792 bad_recipe(filename, linenum,
793 "default for bool type must be either y or n")
794 elif type == 'tristate':
795 if not default in ('y', 'm', 'n'):
796 bad_recipe(filename, linenum,
797 "default for tristate type must be y, m, or n")
798 elif type == 'string':
799 if default[0] != '"' or default[-1] != '"':
800 bad_recipe(filename, linenum,
801 "default for string type must be surrounded by double-quotations")
806 bad_recipe(filename, linenum,
807 "type is int, but default value is not decimal")
809 if len(default) < 2 or default[:2] != '0x':
810 bad_recipe(filename, linenum,
811 "default for hex type must be prefixed with 0x")
815 bad_recipe(filename, linenum,
816 "type is hex, but default value is not hexadecimal")
818 bad_recipe(filename, linenum,
819 "unsupported type '%s'. type must be one of bool, tristate, string, int, hex"
822 config_attrs.append({'config': config, 'type': type, 'default': default})
829 cpu_count = multiprocessing.cpu_count()
830 except NotImplementedError:
833 parser = optparse.OptionParser()
835 parser.add_option('-c', '--color', action='store_true', default=False,
836 help='display the log in color')
837 parser.add_option('-n', '--dry-run', action='store_true', default=False,
838 help='perform a trial run (show log with no changes)')
839 parser.add_option('-e', '--exit-on-error', action='store_true',
841 help='exit immediately on any error')
842 parser.add_option('-j', '--jobs', type='int', default=cpu_count,
843 help='the number of jobs to run simultaneously')
844 parser.usage += ' recipe_file\n\n' + \
845 'The recipe_file should describe config options you want to move.\n' + \
846 'Each line should contain config_name, type, default_value\n\n' + \
848 'CONFIG_FOO bool n\n' + \
849 'CONFIG_BAR int 100\n' + \
850 'CONFIG_BAZ string "hello"\n'
852 (options, args) = parser.parse_args()
858 config_attrs = parse_recipe(args[0])
860 update_cross_compile()
862 move_config(config_attrs, options)
864 if __name__ == '__main__':