3 # Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
5 # SPDX-License-Identifier: GPL-2.0+
9 Converter from Kconfig and MAINTAINERS to boards.cfg
11 Run 'tools/genboardscfg.py' to create boards.cfg file.
13 Run 'tools/genboardscfg.py -h' for available options.
28 BOARD_FILE = 'boards.cfg'
29 CONFIG_DIR = 'configs'
30 REFORMAT_CMD = [os.path.join('tools', 'reformat.py'),
31 '-i', '-d', '-', '-s', '8']
32 SHOW_GNU_MAKE = 'scripts/show-gnu-make'
37 # Automatically generated by %s: don't edit
39 # Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
43 ### helper functions ###
44 def get_terminal_columns():
45 """Get the width of the terminal.
48 The width of the terminal, or zero if the stdout is not
52 return shutil.get_terminal_size().columns # Python 3.3~
53 except AttributeError:
57 arg = struct.pack('hhhh', 0, 0, 0, 0)
59 ret = fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ, arg)
60 except IOError as exception:
61 # If 'Inappropriate ioctl for device' error occurs,
62 # stdout is probably redirected. Return 0.
64 return struct.unpack('hhhh', ret)[1]
67 """Get the file object of '/dev/null' device."""
69 devnull = subprocess.DEVNULL # py3k
70 except AttributeError:
71 devnull = open(os.devnull, 'wb')
74 def check_top_directory():
75 """Exit if we are not at the top of source directory."""
76 for f in ('README', 'Licenses'):
77 if not os.path.exists(f):
78 sys.exit('Please run at the top of source directory.')
81 """Get the command name of GNU Make."""
82 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE)
83 ret = process.communicate()
84 if process.returncode:
85 sys.exit('GNU Make not found')
86 return ret[0].rstrip()
89 class MaintainersDatabase:
91 """The database of board status and maintainers."""
94 """Create an empty database."""
97 def get_status(self, target):
98 """Return the status of the given board.
101 Either 'Active' or 'Orphan'
103 if not target in self.database:
104 print >> sys.stderr, "WARNING: no status info for '%s'" % target
107 tmp = self.database[target][0]
108 if tmp.startswith('Maintained'):
110 elif tmp.startswith('Orphan'):
113 print >> sys.stderr, ("WARNING: %s: unknown status for '%s'" %
117 def get_maintainers(self, target):
118 """Return the maintainers of the given board.
120 If the board has two or more maintainers, they are separated
123 if not target in self.database:
124 print >> sys.stderr, "WARNING: no maintainers for '%s'" % target
127 return ':'.join(self.database[target][1])
129 def parse_file(self, file):
130 """Parse the given MAINTAINERS file.
132 This method parses MAINTAINERS and add board status and
133 maintainers information to the database.
136 file: MAINTAINERS file to be parsed
141 for line in open(file):
142 tag, rest = line[:2], line[2:].strip()
144 maintainers.append(rest)
146 # expand wildcard and filter by 'configs/*_defconfig'
147 for f in glob.glob(rest):
148 front, match, rear = f.partition('configs/')
149 if not front and match:
150 front, match, rear = rear.rpartition('_defconfig')
151 if match and not rear:
152 targets.append(front)
156 for target in targets:
157 self.database[target] = (status, maintainers)
162 for target in targets:
163 self.database[target] = (status, maintainers)
165 class DotConfigParser:
167 """A parser of .config file.
169 Each line of the output should have the form of:
170 Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
171 Most of them are extracted from .config file.
172 MAINTAINERS files are also consulted for Status and Maintainers fields.
175 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"')
176 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"')
177 re_soc = re.compile(r'CONFIG_SYS_SOC="(.*)"')
178 re_vendor = re.compile(r'CONFIG_SYS_VENDOR="(.*)"')
179 re_board = re.compile(r'CONFIG_SYS_BOARD="(.*)"')
180 re_config = re.compile(r'CONFIG_SYS_CONFIG_NAME="(.*)"')
181 re_options = re.compile(r'CONFIG_SYS_EXTRA_OPTIONS="(.*)"')
182 re_list = (('arch', re_arch), ('cpu', re_cpu), ('soc', re_soc),
183 ('vendor', re_vendor), ('board', re_board),
184 ('config', re_config), ('options', re_options))
185 must_fields = ('arch', 'config')
187 def __init__(self, build_dir, output, maintainers_database):
188 """Create a new .config perser.
191 build_dir: Build directory where .config is located
192 output: File object which the result is written to
193 maintainers_database: An instance of class MaintainersDatabase
195 self.dotconfig = os.path.join(build_dir, '.config')
197 self.database = maintainers_database
199 def parse(self, defconfig):
200 """Parse .config file and output one-line database for the given board.
203 defconfig: Board (defconfig) name
206 for line in open(self.dotconfig):
207 if not line.startswith('CONFIG_SYS_'):
209 for (key, pattern) in self.re_list:
210 m = pattern.match(line)
212 fields[key] = m.group(1)
215 # sanity check of '.config' file
216 for field in self.must_fields:
217 if not field in fields:
218 sys.exit('Error: %s is not defined in %s' % (field, defconfig))
221 if fields['arch'] == 'arm' and 'cpu' in fields:
222 if fields['cpu'] == 'armv8':
223 fields['arch'] = 'aarch64'
225 target, match, rear = defconfig.partition('_defconfig')
226 assert match and not rear, \
227 '%s : invalid defconfig file name' % defconfig
229 fields['status'] = self.database.get_status(target)
230 fields['maintainers'] = self.database.get_maintainers(target)
232 if 'options' in fields:
233 options = fields['config'] + ':' + \
234 fields['options'].replace(r'\"', '"')
235 elif fields['config'] != target:
236 options = fields['config']
240 self.output.write((' '.join(['%s'] * 9) + '\n') %
243 fields.get('cpu', '-'),
244 fields.get('soc', '-'),
245 fields.get('vendor', '-'),
246 fields.get('board', '-'),
249 fields['maintainers']))
253 """A slot to store a subprocess.
255 Each instance of this class handles one subprocess.
256 This class is useful to control multiple processes
257 for faster processing.
260 def __init__(self, output, maintainers_database, devnull, make_cmd):
261 """Create a new slot.
264 output: File object which the result is written to
265 maintainers_database: An instance of class MaintainersDatabase
267 self.occupied = False
268 self.build_dir = tempfile.mkdtemp()
269 self.devnull = devnull
270 self.make_cmd = make_cmd
271 self.parser = DotConfigParser(self.build_dir, output,
272 maintainers_database)
275 """Delete the working directory"""
276 shutil.rmtree(self.build_dir)
278 def add(self, defconfig):
279 """Add a new subprocess to the slot.
281 Fails if the slot is occupied, that is, the current subprocess
285 defconfig: Board (defconfig) name
288 Return True on success or False on fail
292 o = 'O=' + self.build_dir
293 self.ps = subprocess.Popen([self.make_cmd, o, defconfig],
295 self.defconfig = defconfig
300 """Check if the subprocess is running and invoke the .config
301 parser if the subprocess is terminated.
304 Return True if the subprocess is terminated, False otherwise
306 if not self.occupied:
308 if self.ps.poll() == None:
310 self.parser.parse(self.defconfig)
311 self.occupied = False
316 """Controller of the array of subprocess slots."""
318 def __init__(self, jobs, output, maintainers_database):
319 """Create a new slots controller.
322 jobs: A number of slots to instantiate
323 output: File object which the result is written to
324 maintainers_database: An instance of class MaintainersDatabase
327 devnull = get_devnull()
328 make_cmd = get_make_cmd()
329 for i in range(jobs):
330 self.slots.append(Slot(output, maintainers_database,
333 def add(self, defconfig):
334 """Add a new subprocess if a vacant slot is available.
337 defconfig: Board (defconfig) name
340 Return True on success or False on fail
342 for slot in self.slots:
343 if slot.add(defconfig):
348 """Check if there is a vacant slot.
351 Return True if a vacant slot is found, False if all slots are full
353 for slot in self.slots:
359 """Check if all slots are vacant.
362 Return True if all slots are vacant, False if at least one slot
366 for slot in self.slots:
373 """A class to control the progress indicator."""
378 def __init__(self, total):
379 """Create an instance.
382 total: A number of boards
386 width = get_terminal_columns()
387 width = min(width, self.MAX_WIDTH)
388 width -= self.MIN_WIDTH
396 """Increment the counter and show the progress bar."""
400 arrow_len = self.width * self.cur // self.total
401 msg = '%4d/%d [' % (self.cur, self.total)
402 msg += '=' * arrow_len + '>' + ' ' * (self.width - arrow_len) + ']'
403 sys.stdout.write('\r' + msg)
406 def __gen_boards_cfg(jobs):
407 """Generate boards.cfg file.
410 jobs: The number of jobs to run simultaneously
413 The incomplete boards.cfg is left over when an error (including
414 the termination by the keyboard interrupt) occurs on the halfway.
416 check_top_directory()
417 print 'Generating %s ... (jobs: %d)' % (BOARD_FILE, jobs)
419 # All the defconfig files to be processed
421 for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
422 dirpath = dirpath[len(CONFIG_DIR) + 1:]
423 for filename in fnmatch.filter(filenames, '*_defconfig'):
424 if fnmatch.fnmatch(filename, '.*'):
426 defconfigs.append(os.path.join(dirpath, filename))
428 # Parse all the MAINTAINERS files
429 maintainers_database = MaintainersDatabase()
430 for (dirpath, dirnames, filenames) in os.walk('.'):
431 if 'MAINTAINERS' in filenames:
432 maintainers_database.parse_file(os.path.join(dirpath,
435 # Output lines should be piped into the reformat tool
436 reformat_process = subprocess.Popen(REFORMAT_CMD, stdin=subprocess.PIPE,
437 stdout=open(BOARD_FILE, 'w'))
438 pipe = reformat_process.stdin
439 pipe.write(COMMENT_BLOCK)
441 indicator = Indicator(len(defconfigs))
442 slots = Slots(jobs, pipe, maintainers_database)
444 # Main loop to process defconfig files:
445 # Add a new subprocess into a vacant slot.
446 # Sleep if there is no available slot.
447 for defconfig in defconfigs:
448 while not slots.add(defconfig):
449 while not slots.available():
450 # No available slot: sleep for a while
451 time.sleep(SLEEP_TIME)
454 # wait until all the subprocesses finish
455 while not slots.empty():
456 time.sleep(SLEEP_TIME)
459 # wait until the reformat tool finishes
460 reformat_process.communicate()
461 if reformat_process.returncode != 0:
462 sys.exit('"%s" failed' % REFORMAT_CMD[0])
464 def gen_boards_cfg(jobs):
465 """Generate boards.cfg file.
467 The incomplete boards.cfg is deleted if an error (including
468 the termination by the keyboard interrupt) occurs on the halfway.
471 jobs: The number of jobs to run simultaneously
474 __gen_boards_cfg(jobs)
476 # We should remove incomplete boards.cfg
478 os.remove(BOARD_FILE)
479 except OSError as exception:
480 # Ignore 'No such file or directory' error
481 if exception.errno != errno.ENOENT:
486 parser = optparse.OptionParser()
488 parser.add_option('-j', '--jobs',
489 help='the number of jobs to run simultaneously')
490 (options, args) = parser.parse_args()
493 jobs = int(options.jobs)
495 sys.exit('Option -j (--jobs) takes a number')
498 jobs = int(subprocess.Popen(['getconf', '_NPROCESSORS_ONLN'],
499 stdout=subprocess.PIPE).communicate()[0])
500 except (OSError, ValueError):
501 print 'info: failed to get the number of CPUs. Set jobs to 1'
505 if __name__ == '__main__':