<defconfig_name> is the name of the defconfig.
<action*> shows what the tool did for that defconfig.
-It looks like one of the followings:
+It looks like one of the following:
- Move 'CONFIG_... '
This config option was moved to the defconfig
The define in the config header matched the one in Kconfig.
We do not need to touch it.
- - Undefined. Do nothing.
- This config option was not found in the config header.
- Nothing to do.
-
- Compiler is missing. Do nothing.
The compiler specified for this architecture was not found
in your PATH environment.
"""
+import copy
+import difflib
import filecmp
import fnmatch
import multiprocessing
# Here is the list of cross-tools I use.
# Most of them are available at kernel.org
-# (https://www.kernel.org/pub/tools/crosstool/files/bin/), except the followings:
+# (https://www.kernel.org/pub/tools/crosstool/files/bin/), except the following:
# arc: https://github.com/foss-for-synopsys-dwc-arc-processors/toolchain/releases
# blackfin: http://sourceforge.net/projects/adi-toolchain/files/
# nds32: http://osdk.andestech.com/packages/nds32le-linux-glibc-v1.tgz
'powerpc': 'powerpc-linux-',
'sh': 'sh-linux-gnu-',
'sparc': 'sparc-linux-',
- 'x86': 'i386-linux-'
+ 'x86': 'i386-linux-',
+ 'xtensa': 'xtensa-linux-'
}
STATE_IDLE = 0
sys.exit('GNU Make not found')
return ret[0].rstrip()
+def get_all_defconfigs():
+ """Get all the defconfig files under the configs/ directory."""
+ defconfigs = []
+ for (dirpath, dirnames, filenames) in os.walk('configs'):
+ dirpath = dirpath[len('configs') + 1:]
+ for filename in fnmatch.filter(filenames, '*_defconfig'):
+ defconfigs.append(os.path.join(dirpath, filename))
+
+ return defconfigs
+
def color_text(color_enabled, color, string):
"""Return colored string."""
if color_enabled:
else:
return string
+def show_diff(a, b, file_path, color_enabled):
+ """Show unidified diff.
+
+ Arguments:
+ a: A list of lines (before)
+ b: A list of lines (after)
+ file_path: Path to the file
+ color_enabled: Display the diff in color
+ """
+
+ diff = difflib.unified_diff(a, b,
+ fromfile=os.path.join('a', file_path),
+ tofile=os.path.join('b', file_path))
+
+ for line in diff:
+ if line[0] == '-' and line[1] != '-':
+ print color_text(color_enabled, COLOR_RED, line),
+ elif line[0] == '+' and line[1] != '+':
+ print color_text(color_enabled, COLOR_GREEN, line),
+ else:
+ print line,
+
def update_cross_compile(color_enabled):
"""Update per-arch CROSS_COMPILE via environment variables
CROSS_COMPILE[arch] = cross_compile
-def cleanup_one_header(header_path, patterns, dry_run):
+def extend_matched_lines(lines, matched, pre_patterns, post_patterns, extend_pre,
+ extend_post):
+ """Extend matched lines if desired patterns are found before/after already
+ matched lines.
+
+ Arguments:
+ lines: A list of lines handled.
+ matched: A list of line numbers that have been already matched.
+ (will be updated by this function)
+ pre_patterns: A list of regular expression that should be matched as
+ preamble.
+ post_patterns: A list of regular expression that should be matched as
+ postamble.
+ extend_pre: Add the line number of matched preamble to the matched list.
+ extend_post: Add the line number of matched postamble to the matched list.
+ """
+ extended_matched = []
+
+ j = matched[0]
+
+ for i in matched:
+ if i == 0 or i < j:
+ continue
+ j = i
+ while j in matched:
+ j += 1
+ if j >= len(lines):
+ break
+
+ for p in pre_patterns:
+ if p.search(lines[i - 1]):
+ break
+ else:
+ # not matched
+ continue
+
+ for p in post_patterns:
+ if p.search(lines[j]):
+ break
+ else:
+ # not matched
+ continue
+
+ if extend_pre:
+ extended_matched.append(i - 1)
+ if extend_post:
+ extended_matched.append(j)
+
+ matched += extended_matched
+ matched.sort()
+
+def cleanup_one_header(header_path, patterns, options):
"""Clean regex-matched lines away from a file.
Arguments:
header_path: path to the cleaned file.
patterns: list of regex patterns. Any lines matching to these
patterns are deleted.
- dry_run: make no changes, but still display log.
+ options: option flags.
"""
with open(header_path) as f:
lines = f.readlines()
matched = []
for i, line in enumerate(lines):
+ if i - 1 in matched and lines[i - 1][-2:] == '\\\n':
+ matched.append(i)
+ continue
for pattern in patterns:
- m = pattern.search(line)
- if m:
- print '%s: %s: %s' % (header_path, i + 1, line),
+ if pattern.search(line):
matched.append(i)
break
- if dry_run or not matched:
+ if not matched:
+ return
+
+ # remove empty #ifdef ... #endif, successive blank lines
+ pattern_if = re.compile(r'#\s*if(def|ndef)?\W') # #if, #ifdef, #ifndef
+ pattern_elif = re.compile(r'#\s*el(if|se)\W') # #elif, #else
+ pattern_endif = re.compile(r'#\s*endif\W') # #endif
+ pattern_blank = re.compile(r'^\s*$') # empty line
+
+ while True:
+ old_matched = copy.copy(matched)
+ extend_matched_lines(lines, matched, [pattern_if],
+ [pattern_endif], True, True)
+ extend_matched_lines(lines, matched, [pattern_elif],
+ [pattern_elif, pattern_endif], True, False)
+ extend_matched_lines(lines, matched, [pattern_if, pattern_elif],
+ [pattern_blank], False, True)
+ extend_matched_lines(lines, matched, [pattern_blank],
+ [pattern_elif, pattern_endif], True, False)
+ extend_matched_lines(lines, matched, [pattern_blank],
+ [pattern_blank], True, False)
+ if matched == old_matched:
+ break
+
+ tolines = copy.copy(lines)
+
+ for i in reversed(matched):
+ tolines.pop(i)
+
+ show_diff(lines, tolines, header_path, options.color)
+
+ if options.dry_run:
return
with open(header_path, 'w') as f:
- for i, line in enumerate(lines):
- if not i in matched:
- f.write(line)
+ for line in tolines:
+ f.write(line)
-def cleanup_headers(configs, dry_run):
+def cleanup_headers(configs, options):
"""Delete config defines from board headers.
Arguments:
configs: A list of CONFIGs to remove.
- dry_run: make no changes, but still display log.
+ options: option flags.
"""
while True:
choice = raw_input('Clean up headers? [y/n]: ').lower()
for dir in 'include', 'arch', 'board':
for (dirpath, dirnames, filenames) in os.walk(dir):
+ if dirpath == os.path.join('include', 'generated'):
+ continue
for filename in filenames:
if not fnmatch.fnmatch(filename, '*~'):
cleanup_one_header(os.path.join(dirpath, filename),
- patterns, dry_run)
+ patterns, options)
+
+def cleanup_one_extra_option(defconfig_path, configs, options):
+ """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in one defconfig file.
+
+ Arguments:
+ defconfig_path: path to the cleaned defconfig file.
+ configs: A list of CONFIGs to remove.
+ options: option flags.
+ """
+
+ start = 'CONFIG_SYS_EXTRA_OPTIONS="'
+ end = '"\n'
+
+ with open(defconfig_path) as f:
+ lines = f.readlines()
+
+ for i, line in enumerate(lines):
+ if line.startswith(start) and line.endswith(end):
+ break
+ else:
+ # CONFIG_SYS_EXTRA_OPTIONS was not found in this defconfig
+ return
+
+ old_tokens = line[len(start):-len(end)].split(',')
+ new_tokens = []
+
+ for token in old_tokens:
+ pos = token.find('=')
+ if not (token[:pos] if pos >= 0 else token) in configs:
+ new_tokens.append(token)
+
+ if new_tokens == old_tokens:
+ return
+
+ tolines = copy.copy(lines)
+
+ if new_tokens:
+ tolines[i] = start + ','.join(new_tokens) + end
+ else:
+ tolines.pop(i)
+
+ show_diff(lines, tolines, defconfig_path, options.color)
+
+ if options.dry_run:
+ return
+
+ with open(defconfig_path, 'w') as f:
+ for line in tolines:
+ f.write(line)
+
+def cleanup_extra_options(configs, options):
+ """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in defconfig files.
+
+ Arguments:
+ configs: A list of CONFIGs to remove.
+ options: option flags.
+ """
+ while True:
+ choice = raw_input('Clean up CONFIG_SYS_EXTRA_OPTIONS? [y/n]: ').lower()
+ print choice
+ if choice == 'y' or choice == 'n':
+ break
+
+ if choice == 'n':
+ return
+
+ configs = [ config[len('CONFIG_'):] for config in configs ]
+
+ defconfigs = get_all_defconfigs()
+
+ for defconfig in defconfigs:
+ cleanup_one_extra_option(os.path.join('configs', defconfig), configs,
+ options)
### classes ###
class Progress:
else:
new_val = not_set
- if old_val == new_val:
- return (ACTION_NO_CHANGE, new_val)
-
# If this CONFIG is neither bool nor trisate
if old_val[-2:] != '=y' and old_val[-2:] != '=m' and old_val != not_set:
# tools/scripts/define2mk.sed changes '1' to 'y'.
if new_val[-2:] == '=y':
new_val = new_val[:-1] + '1'
- return (ACTION_MOVE, new_val)
+ return (ACTION_NO_CHANGE if old_val == new_val else ACTION_MOVE,
+ new_val)
def update_dotconfig(self):
"""Parse files for the config options and update the .config.
self.parser = KconfigParser(configs, options, self.build_dir)
self.state = STATE_IDLE
self.failed_boards = []
+ self.suspicious_boards = []
def __del__(self):
"""Delete the working directory
self.defconfig = defconfig
self.log = ''
- self.use_git_ref = True if self.options.git_ref else False
+ self.current_src_dir = self.reference_src_dir
self.do_defconfig()
return True
if self.ps.poll() != 0:
self.handle_error()
elif self.state == STATE_DEFCONFIG:
- if self.options.git_ref and not self.use_git_ref:
+ if self.reference_src_dir and not self.current_src_dir:
self.do_savedefconfig()
else:
self.do_autoconf()
elif self.state == STATE_AUTOCONF:
- if self.use_git_ref:
- self.use_git_ref = False
+ if self.current_src_dir:
+ self.current_src_dir = None
self.do_defconfig()
else:
self.do_savedefconfig()
cmd = list(self.make_cmd)
cmd.append(self.defconfig)
- if self.use_git_ref:
- cmd.append('-C')
- cmd.append(self.reference_src_dir)
self.ps = subprocess.Popen(cmd, stdout=self.devnull,
- stderr=subprocess.PIPE)
+ stderr=subprocess.PIPE,
+ cwd=self.current_src_dir)
self.state = STATE_DEFCONFIG
def do_autoconf(self):
cmd.append('CROSS_COMPILE=%s' % self.cross_compile)
cmd.append('KCONFIG_IGNORE_DUPLICATES=1')
cmd.append('include/config/auto.conf')
- if self.use_git_ref:
- cmd.append('-C')
- cmd.append(self.reference_src_dir)
self.ps = subprocess.Popen(cmd, stdout=self.devnull,
- stderr=subprocess.PIPE)
+ stderr=subprocess.PIPE,
+ cwd=self.current_src_dir)
self.state = STATE_AUTOCONF
def do_savedefconfig(self):
def update_defconfig(self):
"""Update the input defconfig and go back to the idle state."""
- self.log += self.parser.check_defconfig()
+ log = self.parser.check_defconfig()
+ if log:
+ self.suspicious_boards.append(self.defconfig)
+ self.log += log
orig_defconfig = os.path.join('configs', self.defconfig)
new_defconfig = os.path.join(self.build_dir, 'defconfig')
updated = not filecmp.cmp(orig_defconfig, new_defconfig)
"""
return self.failed_boards
+ def get_suspicious_boards(self):
+ """Returns a list of boards (defconfigs) with possible misconversion.
+ """
+ return self.suspicious_boards
+
class Slots:
"""Controller of the array of subprocess slots."""
def show_failed_boards(self):
"""Display all of the failed boards (defconfigs)."""
- failed_boards = []
+ boards = []
+ output_file = 'moveconfig.failed'
for slot in self.slots:
- failed_boards += slot.get_failed_boards()
+ boards += slot.get_failed_boards()
+
+ if boards:
+ boards = '\n'.join(boards) + '\n'
+ msg = "The following boards were not processed due to error:\n"
+ msg += boards
+ msg += "(the list has been saved in %s)\n" % output_file
+ print >> sys.stderr, color_text(self.options.color, COLOR_LIGHT_RED,
+ msg)
- if len(failed_boards) > 0:
- msg = [ "The following boards were not processed due to error:" ]
- msg += failed_boards
- for line in msg:
- print >> sys.stderr, color_text(self.options.color,
- COLOR_LIGHT_RED, line)
+ with open(output_file, 'w') as f:
+ f.write(boards)
+
+ def show_suspicious_boards(self):
+ """Display all boards (defconfigs) with possible misconversion."""
+ boards = []
+ output_file = 'moveconfig.suspicious'
+
+ for slot in self.slots:
+ boards += slot.get_suspicious_boards()
- with open('moveconfig.failed', 'w') as f:
- for board in failed_boards:
- f.write(board + '\n')
+ if boards:
+ boards = '\n'.join(boards) + '\n'
+ msg = "The following boards might have been converted incorrectly.\n"
+ msg += "It is highly recommended to check them manually:\n"
+ msg += boards
+ msg += "(the list has been saved in %s)\n" % output_file
+ print >> sys.stderr, color_text(self.options.color, COLOR_YELLOW,
+ msg)
-class WorkDir:
- def __init__(self):
- """Create a new working directory."""
- self.work_dir = tempfile.mkdtemp()
+ with open(output_file, 'w') as f:
+ f.write(boards)
+
+class ReferenceSource:
+
+ """Reference source against which original configs should be parsed."""
+
+ def __init__(self, commit):
+ """Create a reference source directory based on a specified commit.
+
+ Arguments:
+ commit: commit to git-clone
+ """
+ self.src_dir = tempfile.mkdtemp()
+ print "Cloning git repo to a separate work directory..."
+ subprocess.check_output(['git', 'clone', os.getcwd(), '.'],
+ cwd=self.src_dir)
+ print "Checkout '%s' to build the original autoconf.mk." % \
+ subprocess.check_output(['git', 'rev-parse', '--short', commit]).strip()
+ subprocess.check_output(['git', 'checkout', commit],
+ stderr=subprocess.STDOUT, cwd=self.src_dir)
def __del__(self):
- """Delete the working directory
+ """Delete the reference source directory
This function makes sure the temporary directory is cleaned away
even if Python suddenly dies due to error. It should be done in here
because it is guaranteed the destructor is always invoked when the
instance of the class gets unreferenced.
"""
- shutil.rmtree(self.work_dir)
+ shutil.rmtree(self.src_dir)
+
+ def get_dir(self):
+ """Return the absolute path to the reference source directory."""
- def get(self):
- return self.work_dir
+ return self.src_dir
def move_config(configs, options):
"""Move config options to defconfig files.
print 'Move ' + ', '.join(configs),
print '(jobs: %d)\n' % options.jobs
- reference_src_dir = ''
-
if options.git_ref:
- work_dir = WorkDir()
- reference_src_dir = work_dir.get()
- print "Cloning git repo to a separate work directory..."
- subprocess.check_output(['git', 'clone', os.getcwd(), '.'],
- cwd=reference_src_dir)
- print "Checkout '%s' to build the original autoconf.mk." % \
- subprocess.check_output(['git', 'rev-parse', '--short',
- options.git_ref]).strip()
- subprocess.check_output(['git', 'checkout', options.git_ref],
- stderr=subprocess.STDOUT,
- cwd=reference_src_dir)
+ reference_src = ReferenceSource(options.git_ref)
+ reference_src_dir = reference_src.get_dir()
+ else:
+ reference_src_dir = None
if options.defconfigs:
defconfigs = [line.strip() for line in open(options.defconfigs)]
sys.exit('%s - defconfig does not exist. Stopping.' %
defconfigs[i])
else:
- # All the defconfig files to be processed
- defconfigs = []
- for (dirpath, dirnames, filenames) in os.walk('configs'):
- dirpath = dirpath[len('configs') + 1:]
- for filename in fnmatch.filter(filenames, '*_defconfig'):
- defconfigs.append(os.path.join(dirpath, filename))
+ defconfigs = get_all_defconfigs()
progress = Progress(len(defconfigs))
slots = Slots(configs, options, progress, reference_src_dir)
print ''
slots.show_failed_boards()
+ slots.show_suspicious_boards()
def main():
try:
check_top_directory()
- check_clean_directory()
-
- update_cross_compile(options.color)
-
if not options.cleanup_headers_only:
+ check_clean_directory()
+ update_cross_compile(options.color)
move_config(configs, options)
if configs:
- cleanup_headers(configs, options.dry_run)
+ cleanup_headers(configs, options)
+ cleanup_extra_options(configs, options)
if __name__ == '__main__':
main()
projects / u-boot / blobdiff