3 # Author: Masahiro Yamada <yamada.masahiro@socionext.com>
5 # SPDX-License-Identifier: GPL-2.0+
9 Move config options from headers to defconfig files.
11 Since Kconfig was introduced to U-Boot, we have worked on moving
12 config options from headers to Kconfig (defconfig).
14 This tool intends to help this tremendous work.
20 This tool takes one input file. (let's say 'recipe' file here.)
21 The recipe describes the list of config options you want to move.
22 Each line takes the form:
23 <config_name> <type> <default>
24 (the fields must be separated with whitespaces.)
26 <config_name> is the name of config option.
28 <type> is the type of the option. It must be one of bool, tristate,
31 <default> is the default value of the option. It must be appropriate
32 value corresponding to the option type. It must be either y or n for
33 the bool type. Tristate options can also take m (although U-Boot has
34 not supported the module feature).
36 You can add two or more lines in the recipe file, so you can move
37 multiple options at once.
39 Let's say, for example, you want to move CONFIG_CMD_USB and
42 The type should be bool, hex, respectively. So, the recipe file
43 should look like this:
47 CONFIG_SYS_TEXT_BASE hex 0x00000000
49 Next you must edit the Kconfig to add the menu entries for the configs
52 And then run this tool giving the file name of the recipe
54 $ tools/moveconfig.py recipe
56 The tool walks through all the defconfig files to move the config
57 options specified by the recipe file.
59 The log is also displayed on the terminal.
61 Each line is printed in the format
62 <defconfig_name> : <action>
64 <defconfig_name> is the name of the defconfig
65 (without the suffix _defconfig).
67 <action> shows what the tool did for that defconfig.
68 It looks like one of the followings:
71 This config option was moved to the defconfig
73 - Default value 'CONFIG_...'. Do nothing.
74 The value of this option is the same as default.
75 We do not have to add it to the defconfig.
77 - 'CONFIG_...' already exists in Kconfig. Do nothing.
78 This config option is already defined in Kconfig.
79 We do not need/want to touch it.
81 - Undefined. Do nothing.
82 This config option was not found in the config header.
85 - Failed to process. Skip.
86 An error occurred during processing this defconfig. Skipped.
87 (If -e option is passed, the tool exits immediately on error.)
89 Finally, you will be asked, Clean up headers? [y/n]:
91 If you say 'y' here, the unnecessary config defines are removed
92 from the config headers (include/configs/*.h).
93 It just uses the regex method, so you should not rely on it.
94 Just in case, please do 'git diff' to see what happened.
100 This tool runs configuration and builds include/autoconf.mk for every
101 defconfig. The config options defined in Kconfig appear in the .config
102 file (unless they are hidden because of unmet dependency.)
103 On the other hand, the config options defined by board headers are seen
104 in include/autoconf.mk. The tool looks for the specified options in both
105 of them to decide the appropriate action for the options. If the option
106 is found in the .config or the value is the same as the specified default,
107 the option does not need to be touched. If the option is found in
108 include/autoconf.mk, but not in the .config, and the value is different
109 from the default, the tools adds the option to the defconfig.
111 For faster processing, this tool handles multi-threading. It creates
112 separate build directories where the out-of-tree build is run. The
113 temporary build directories are automatically created and deleted as
114 needed. The number of threads are chosen based on the number of the CPU
115 cores of your system although you can change it via -j (--jobs) option.
121 Appropriate toolchain are necessary to generate include/autoconf.mk
122 for all the architectures supported by U-Boot. Most of them are available
123 at the kernel.org site, some are not provided by kernel.org.
125 The default per-arch CROSS_COMPILE used by this tool is specified by
126 the list below, CROSS_COMPILE. You may wish to update the list to
127 use your own. Instead of modifying the list directly, you can give
128 them via environments.
135 Surround each portion of the log with escape sequences to display it
136 in color on the terminal.
139 Peform a trial run that does not make any changes. It is useful to
140 see what is going to happen before one actually runs it.
143 Exit immediately if Make exits with a non-zero status while processing
147 Specify the number of threads to run simultaneously. If not specified,
148 the number of threads is the same as the number of CPU cores.
150 To see the complete list of supported options, run
152 $ tools/moveconfig.py -h
157 import multiprocessing
167 SHOW_GNU_MAKE = 'scripts/show-gnu-make'
170 # Here is the list of cross-tools I use.
171 # Most of them are available at kernel.org
172 # (https://www.kernel.org/pub/tools/crosstool/files/bin/), except the followings:
173 # arc: https://github.com/foss-for-synopsys-dwc-arc-processors/toolchain/releases
174 # blackfin: http://sourceforge.net/projects/adi-toolchain/files/
175 # nds32: http://osdk.andestech.com/packages/
176 # nios2: https://sourcery.mentor.com/GNUToolchain/subscription42545
177 # sh: http://sourcery.mentor.com/public/gnu_toolchain/sh-linux-gnu
180 'aarch64': 'aarch64-linux-',
181 'arm': 'arm-unknown-linux-gnueabi-',
182 'avr32': 'avr32-linux-',
183 'blackfin': 'bfin-elf-',
184 'm68k': 'm68k-linux-',
185 'microblaze': 'microblaze-linux-',
186 'mips': 'mips-linux-',
187 'nds32': 'nds32le-linux-',
188 'nios2': 'nios2-linux-gnu-',
189 'openrisc': 'or32-linux-',
190 'powerpc': 'powerpc-linux-',
191 'sh': 'sh-linux-gnu-',
192 'sparc': 'sparc-linux-',
199 STATE_SAVEDEFCONFIG = 3
202 ACTION_DEFAULT_VALUE = 1
203 ACTION_ALREADY_EXIST = 2
211 COLOR_PURPLE = '0;35'
213 COLOR_LIGHT_GRAY = '0;37'
214 COLOR_DARK_GRAY = '1;30'
215 COLOR_LIGHT_RED = '1;31'
216 COLOR_LIGHT_GREEN = '1;32'
217 COLOR_YELLOW = '1;33'
218 COLOR_LIGHT_BLUE = '1;34'
219 COLOR_LIGHT_PURPLE = '1;35'
220 COLOR_LIGHT_CYAN = '1;36'
223 ### helper functions ###
225 """Get the file object of '/dev/null' device."""
227 devnull = subprocess.DEVNULL # py3k
228 except AttributeError:
229 devnull = open(os.devnull, 'wb')
232 def check_top_directory():
233 """Exit if we are not at the top of source directory."""
234 for f in ('README', 'Licenses'):
235 if not os.path.exists(f):
236 sys.exit('Please run at the top of source directory.')
239 """Get the command name of GNU Make.
241 U-Boot needs GNU Make for building, but the command name is not
242 necessarily "make". (for example, "gmake" on FreeBSD).
243 Returns the most appropriate command name on your system.
245 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE)
246 ret = process.communicate()
247 if process.returncode:
248 sys.exit('GNU Make not found')
249 return ret[0].rstrip()
251 def color_text(color_enabled, color, string):
252 """Return colored string."""
254 return '\033[' + color + 'm' + string + '\033[0m'
258 def log_msg(color_enabled, color, defconfig, msg):
259 """Return the formated line for the log."""
260 return defconfig[:-len('_defconfig')].ljust(37) + ': ' + \
261 color_text(color_enabled, color, msg) + '\n'
263 def update_cross_compile():
264 """Update per-arch CROSS_COMPILE via enviroment variables
266 The default CROSS_COMPILE values are available
267 in the CROSS_COMPILE list above.
269 You can override them via enviroment variables
270 CROSS_COMPILE_{ARCH}.
272 For example, if you want to override toolchain prefixes
273 for ARM and PowerPC, you can do as follows in your shell:
275 export CROSS_COMPILE_ARM=...
276 export CROSS_COMPILE_POWERPC=...
280 for arch in os.listdir('arch'):
281 if os.path.exists(os.path.join('arch', arch, 'Makefile')):
284 # arm64 is a special case
285 archs.append('aarch64')
288 env = 'CROSS_COMPILE_' + arch.upper()
289 cross_compile = os.environ.get(env)
291 CROSS_COMPILE[arch] = cross_compile
293 def cleanup_one_header(header_path, patterns, dry_run):
294 """Clean regex-matched lines away from a file.
297 header_path: path to the cleaned file.
298 patterns: list of regex patterns. Any lines matching to these
299 patterns are deleted.
300 dry_run: make no changes, but still display log.
302 with open(header_path) as f:
303 lines = f.readlines()
306 for i, line in enumerate(lines):
307 for pattern in patterns:
308 m = pattern.search(line)
310 print '%s: %s: %s' % (header_path, i + 1, line),
314 if dry_run or not matched:
317 with open(header_path, 'w') as f:
318 for i, line in enumerate(lines):
322 def cleanup_headers(config_attrs, dry_run):
323 """Delete config defines from board headers.
326 config_attrs: A list of dictionaris, each of them includes the name,
327 the type, and the default value of the target config.
328 dry_run: make no changes, but still display log.
331 choice = raw_input('Clean up headers? [y/n]: ').lower()
333 if choice == 'y' or choice == 'n':
340 for config_attr in config_attrs:
341 config = config_attr['config']
342 patterns.append(re.compile(r'#\s*define\s+%s\W' % config))
343 patterns.append(re.compile(r'#\s*undef\s+%s\W' % config))
345 for (dirpath, dirnames, filenames) in os.walk('include'):
346 for filename in filenames:
347 if not fnmatch.fnmatch(filename, '*~'):
348 cleanup_one_header(os.path.join(dirpath, filename), patterns,
354 """A parser of .config and include/autoconf.mk."""
356 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"')
357 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"')
359 def __init__(self, config_attrs, options, build_dir):
360 """Create a new parser.
363 config_attrs: A list of dictionaris, each of them includes the name,
364 the type, and the default value of the target config.
365 options: option flags.
366 build_dir: Build directory.
368 self.config_attrs = config_attrs
369 self.options = options
370 self.build_dir = build_dir
372 def get_cross_compile(self):
373 """Parse .config file and return CROSS_COMPILE.
376 A string storing the compiler prefix for the architecture.
380 dotconfig = os.path.join(self.build_dir, '.config')
381 for line in open(dotconfig):
382 m = self.re_arch.match(line)
386 m = self.re_cpu.match(line)
390 assert arch, 'Error: arch is not defined in %s' % defconfig
393 if arch == 'arm' and cpu == 'armv8':
396 return CROSS_COMPILE.get(arch, '')
398 def parse_one_config(self, config_attr, defconfig_lines, autoconf_lines):
399 """Parse .config, defconfig, include/autoconf.mk for one config.
401 This function looks for the config options in the lines from
402 defconfig, .config, and include/autoconf.mk in order to decide
403 which action should be taken for this defconfig.
406 config_attr: A dictionary including the name, the type,
407 and the default value of the target config.
408 defconfig_lines: lines from the original defconfig file.
409 autoconf_lines: lines from the include/autoconf.mk file.
412 A tupple of the action for this defconfig and the line
413 matched for the config.
415 config = config_attr['config']
416 not_set = '# %s is not set' % config
418 if config_attr['type'] in ('bool', 'tristate') and \
419 config_attr['default'] == 'n':
422 default = config + '=' + config_attr['default']
424 for line in defconfig_lines:
426 if line.startswith(config + '=') or line == not_set:
427 return (ACTION_ALREADY_EXIST, line)
429 if config_attr['type'] in ('bool', 'tristate'):
432 value = '(undefined)'
434 for line in autoconf_lines:
436 if line.startswith(config + '='):
441 action = ACTION_DEFAULT_VALUE
442 elif value == '(undefined)':
443 action = ACTION_UNDEFINED
447 return (action, value)
449 def update_defconfig(self, defconfig):
450 """Parse files for the config options and update the defconfig.
452 This function parses the given defconfig, the generated .config
453 and include/autoconf.mk searching the target options.
454 Move the config option(s) to the defconfig or do nothing if unneeded.
455 Also, display the log to show what happened to this defconfig.
458 defconfig: defconfig name.
461 defconfig_path = os.path.join('configs', defconfig)
462 dotconfig_path = os.path.join(self.build_dir, '.config')
463 autoconf_path = os.path.join(self.build_dir, 'include', 'autoconf.mk')
466 with open(defconfig_path) as f:
467 defconfig_lines = f.readlines()
469 with open(autoconf_path) as f:
470 autoconf_lines = f.readlines()
472 for config_attr in self.config_attrs:
473 result = self.parse_one_config(config_attr, defconfig_lines,
475 results.append(result)
479 for (action, value) in results:
480 if action == ACTION_MOVE:
481 actlog = "Move '%s'" % value
482 log_color = COLOR_LIGHT_GREEN
483 elif action == ACTION_DEFAULT_VALUE:
484 actlog = "Default value '%s'. Do nothing." % value
485 log_color = COLOR_LIGHT_BLUE
486 elif action == ACTION_ALREADY_EXIST:
487 actlog = "'%s' already defined in Kconfig. Do nothing." % value
488 log_color = COLOR_LIGHT_PURPLE
489 elif action == ACTION_UNDEFINED:
490 actlog = "Undefined. Do nothing."
491 log_color = COLOR_DARK_GRAY
493 sys.exit("Internal Error. This should not happen.")
495 log += log_msg(self.options.color, log_color, defconfig, actlog)
497 # Some threads are running in parallel.
498 # Print log in one shot to not mix up logs from different threads.
501 if not self.options.dry_run:
502 with open(dotconfig_path, 'a') as f:
503 for (action, value) in results:
504 if action == ACTION_MOVE:
505 f.write(value + '\n')
507 os.remove(os.path.join(self.build_dir, 'include', 'config', 'auto.conf'))
508 os.remove(autoconf_path)
512 """A slot to store a subprocess.
514 Each instance of this class handles one subprocess.
515 This class is useful to control multiple threads
516 for faster processing.
519 def __init__(self, config_attrs, options, devnull, make_cmd):
520 """Create a new process slot.
523 config_attrs: A list of dictionaris, each of them includes the name,
524 the type, and the default value of the target config.
525 options: option flags.
526 devnull: A file object of '/dev/null'.
527 make_cmd: command name of GNU Make.
529 self.options = options
530 self.build_dir = tempfile.mkdtemp()
531 self.devnull = devnull
532 self.make_cmd = (make_cmd, 'O=' + self.build_dir)
533 self.parser = KconfigParser(config_attrs, options, self.build_dir)
534 self.state = STATE_IDLE
535 self.failed_boards = []
538 """Delete the working directory
540 This function makes sure the temporary directory is cleaned away
541 even if Python suddenly dies due to error. It should be done in here
542 because it is guranteed the destructor is always invoked when the
543 instance of the class gets unreferenced.
545 If the subprocess is still running, wait until it finishes.
547 if self.state != STATE_IDLE:
548 while self.ps.poll() == None:
550 shutil.rmtree(self.build_dir)
552 def add(self, defconfig):
553 """Assign a new subprocess for defconfig and add it to the slot.
555 If the slot is vacant, create a new subprocess for processing the
556 given defconfig and add it to the slot. Just returns False if
557 the slot is occupied (i.e. the current subprocess is still running).
560 defconfig: defconfig name.
563 Return True on success or False on failure
565 if self.state != STATE_IDLE:
567 cmd = list(self.make_cmd)
568 cmd.append(defconfig)
569 self.ps = subprocess.Popen(cmd, stdout=self.devnull)
570 self.defconfig = defconfig
571 self.state = STATE_DEFCONFIG
575 """Check the status of the subprocess and handle it as needed.
577 Returns True if the slot is vacant (i.e. in idle state).
578 If the configuration is successfully finished, assign a new
579 subprocess to build include/autoconf.mk.
580 If include/autoconf.mk is generated, invoke the parser to
581 parse the .config and the include/autoconf.mk, and then set the
582 slot back to the idle state.
585 Return True if the subprocess is terminated, False otherwise
587 if self.state == STATE_IDLE:
590 if self.ps.poll() == None:
593 if self.ps.poll() != 0:
595 print >> sys.stderr, log_msg(self.options.color,
598 "failed to process.")
599 if self.options.exit_on_error:
600 sys.exit("Exit on error.")
602 # If --exit-on-error flag is not set,
603 # skip this board and continue.
604 # Record the failed board.
605 self.failed_boards.append(self.defconfig)
606 self.state = STATE_IDLE
609 if self.state == STATE_AUTOCONF:
610 self.parser.update_defconfig(self.defconfig)
612 """Save off the defconfig in a consistent way"""
613 cmd = list(self.make_cmd)
614 cmd.append('savedefconfig')
615 self.ps = subprocess.Popen(cmd, stdout=self.devnull,
617 self.state = STATE_SAVEDEFCONFIG
620 if self.state == STATE_SAVEDEFCONFIG:
621 defconfig_path = os.path.join(self.build_dir, 'defconfig')
622 shutil.move(defconfig_path,
623 os.path.join('configs', self.defconfig))
624 self.state = STATE_IDLE
627 cross_compile = self.parser.get_cross_compile()
628 cmd = list(self.make_cmd)
630 cmd.append('CROSS_COMPILE=%s' % cross_compile)
631 cmd.append('KCONFIG_IGNORE_DUPLICATES=1')
632 cmd.append('include/config/auto.conf')
633 self.ps = subprocess.Popen(cmd, stdout=self.devnull)
634 self.state = STATE_AUTOCONF
637 def get_failed_boards(self):
638 """Returns a list of failed boards (defconfigs) in this slot.
640 return self.failed_boards
644 """Controller of the array of subprocess slots."""
646 def __init__(self, config_attrs, options):
647 """Create a new slots controller.
650 config_attrs: A list of dictionaris containing the name, the type,
651 and the default value of the target CONFIG.
652 options: option flags.
654 self.options = options
656 devnull = get_devnull()
657 make_cmd = get_make_cmd()
658 for i in range(options.jobs):
659 self.slots.append(Slot(config_attrs, options, devnull, make_cmd))
661 def add(self, defconfig):
662 """Add a new subprocess if a vacant slot is found.
665 defconfig: defconfig name to be put into.
668 Return True on success or False on failure
670 for slot in self.slots:
671 if slot.add(defconfig):
676 """Check if there is a vacant slot.
679 Return True if at lease one vacant slot is found, False otherwise.
681 for slot in self.slots:
687 """Check if all slots are vacant.
690 Return True if all the slots are vacant, False otherwise.
693 for slot in self.slots:
698 def show_failed_boards(self):
699 """Display all of the failed boards (defconfigs)."""
702 for slot in self.slots:
703 failed_boards += slot.get_failed_boards()
705 if len(failed_boards) > 0:
706 msg = [ "The following boards were not processed due to error:" ]
709 print >> sys.stderr, color_text(self.options.color,
710 COLOR_LIGHT_RED, line)
712 def move_config(config_attrs, options):
713 """Move config options to defconfig files.
716 config_attrs: A list of dictionaris, each of them includes the name,
717 the type, and the default value of the target config.
718 options: option flags
720 check_top_directory()
722 if len(config_attrs) == 0:
723 print 'Nothing to do. exit.'
726 print 'Move the following CONFIG options (jobs: %d)' % options.jobs
727 for config_attr in config_attrs:
728 print ' %s (type: %s, default: %s)' % (config_attr['config'],
730 config_attr['default'])
732 # All the defconfig files to be processed
734 for (dirpath, dirnames, filenames) in os.walk('configs'):
735 dirpath = dirpath[len('configs') + 1:]
736 for filename in fnmatch.filter(filenames, '*_defconfig'):
737 defconfigs.append(os.path.join(dirpath, filename))
739 slots = Slots(config_attrs, options)
741 # Main loop to process defconfig files:
742 # Add a new subprocess into a vacant slot.
743 # Sleep if there is no available slot.
744 for defconfig in defconfigs:
745 while not slots.add(defconfig):
746 while not slots.available():
747 # No available slot: sleep for a while
748 time.sleep(SLEEP_TIME)
750 # wait until all the subprocesses finish
751 while not slots.empty():
752 time.sleep(SLEEP_TIME)
754 slots.show_failed_boards()
756 cleanup_headers(config_attrs, options.dry_run)
758 def bad_recipe(filename, linenum, msg):
759 """Print error message with the file name and the line number and exit."""
760 sys.exit("%s: line %d: error : " % (filename, linenum) + msg)
762 def parse_recipe(filename):
763 """Parse the recipe file and retrieve the config attributes.
765 This function parses the given recipe file and gets the name,
766 the type, and the default value of the target config options.
769 filename: path to file to be parsed.
771 A list of dictionaris, each of them includes the name,
772 the type, and the default value of the target config.
777 for line in open(filename):
778 tokens = line.split()
780 bad_recipe(filename, linenum,
781 "%d fields in this line. Each line must contain 3 fields"
784 (config, type, default) = tokens
786 # prefix the option name with CONFIG_ if missing
787 if not config.startswith('CONFIG_'):
788 config = 'CONFIG_' + config
790 # sanity check of default values
792 if not default in ('y', 'n'):
793 bad_recipe(filename, linenum,
794 "default for bool type must be either y or n")
795 elif type == 'tristate':
796 if not default in ('y', 'm', 'n'):
797 bad_recipe(filename, linenum,
798 "default for tristate type must be y, m, or n")
799 elif type == 'string':
800 if default[0] != '"' or default[-1] != '"':
801 bad_recipe(filename, linenum,
802 "default for string type must be surrounded by double-quotations")
807 bad_recipe(filename, linenum,
808 "type is int, but default value is not decimal")
810 if len(default) < 2 or default[:2] != '0x':
811 bad_recipe(filename, linenum,
812 "default for hex type must be prefixed with 0x")
816 bad_recipe(filename, linenum,
817 "type is hex, but default value is not hexadecimal")
819 bad_recipe(filename, linenum,
820 "unsupported type '%s'. type must be one of bool, tristate, string, int, hex"
823 config_attrs.append({'config': config, 'type': type, 'default': default})
830 cpu_count = multiprocessing.cpu_count()
831 except NotImplementedError:
834 parser = optparse.OptionParser()
836 parser.add_option('-c', '--color', action='store_true', default=False,
837 help='display the log in color')
838 parser.add_option('-n', '--dry-run', action='store_true', default=False,
839 help='perform a trial run (show log with no changes)')
840 parser.add_option('-e', '--exit-on-error', action='store_true',
842 help='exit immediately on any error')
843 parser.add_option('-j', '--jobs', type='int', default=cpu_count,
844 help='the number of jobs to run simultaneously')
845 parser.usage += ' recipe_file\n\n' + \
846 'The recipe_file should describe config options you want to move.\n' + \
847 'Each line should contain config_name, type, default_value\n\n' + \
849 'CONFIG_FOO bool n\n' + \
850 'CONFIG_BAR int 100\n' + \
851 'CONFIG_BAZ string "hello"\n'
853 (options, args) = parser.parse_args()
859 config_attrs = parse_recipe(args[0])
861 update_cross_compile()
863 move_config(config_attrs, options)
865 if __name__ == '__main__':