1 # Copyright (c) 2011 The Chromium OS Authors.
3 # SPDX-License-Identifier: GPL-2.0+
14 from series import Series
16 # Tags that we detect and remove
17 re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Change-Id:|^Review URL:'
18 '|Reviewed-on:|Commit-\w*:')
20 # Lines which are allowed after a TEST= line
21 re_allowed_after_test = re.compile('^Signed-off-by:')
24 re_signoff = re.compile('^Signed-off-by: *(.*)')
26 # The start of the cover letter
27 re_cover = re.compile('^Cover-letter:')
30 re_cover_cc = re.compile('^Cover-letter-cc: *(.*)')
33 re_series_tag = re.compile('^Series-([a-z-]*): *(.*)')
36 re_commit_tag = re.compile('^Commit-([a-z-]*): *(.*)')
38 # Commit tags that we want to collect and keep
39 re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc): (.*)')
41 # The start of a new commit in the git log
42 re_commit = re.compile('^commit ([0-9a-f]*)$')
44 # We detect these since checkpatch doesn't always do it
45 re_space_before_tab = re.compile('^[+].* \t')
47 # States we can be in - can we use range() and still have comments?
48 STATE_MSG_HEADER = 0 # Still in the message header
49 STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit)
50 STATE_PATCH_HEADER = 2 # In patch header (after the subject)
51 STATE_DIFFS = 3 # In the diff part (past --- line)
54 """Class for detecting/injecting tags in a patch or series of patches
56 We support processing the output of 'git log' to read out the tags we
57 are interested in. We can also process a patch file in order to remove
58 unwanted tags or inject additional ones. These correspond to the two
61 def __init__(self, series, name=None, is_log=False):
62 self.skip_blank = False # True to skip a single blank line
63 self.found_test = False # Found a TEST= line
64 self.lines_after_test = 0 # MNumber of lines found after TEST=
65 self.warn = [] # List of warnings we have collected
66 self.linenum = 1 # Output line number we are up to
67 self.in_section = None # Name of start...END section we are in
68 self.notes = [] # Series notes
69 self.section = [] # The current section...END section
70 self.series = series # Info about the patch series
71 self.is_log = is_log # True if indent like git log
72 self.in_change = 0 # Non-zero if we are in a change list
73 self.blank_count = 0 # Number of blank lines stored up
74 self.state = STATE_MSG_HEADER # What state are we in?
75 self.tags = [] # Tags collected, like Tested-by...
76 self.signoff = [] # Contents of signoff line
77 self.commit = None # Current commit
79 def AddToSeries(self, line, name, value):
80 """Add a new Series-xxx tag.
82 When a Series-xxx tag is detected, we come here to record it, if we
83 are scanning a 'git log'.
86 line: Source line containing tag (useful for debug/error messages)
87 name: Tag name (part after 'Series-')
88 value: Tag value (part after 'Series-xxx: ')
91 self.in_section = name
92 self.skip_blank = False
94 self.series.AddTag(self.commit, line, name, value)
96 def AddToCommit(self, line, name, value):
97 """Add a new Commit-xxx tag.
99 When a Commit-xxx tag is detected, we come here to record it.
102 line: Source line containing tag (useful for debug/error messages)
103 name: Tag name (part after 'Commit-')
104 value: Tag value (part after 'Commit-xxx: ')
107 self.in_section = 'commit-' + name
108 self.skip_blank = False
110 def CloseCommit(self):
111 """Save the current commit into our commit list, and reset our state"""
112 if self.commit and self.is_log:
113 self.series.AddCommit(self.commit)
116 def FormatTags(self, tags):
118 for tag in sorted(tags):
119 if tag.startswith('Cc:'):
120 tag_list = tag[4:].split(',')
121 out_list += gitutil.BuildEmailList(tag_list, 'Cc:')
126 def ProcessLine(self, line):
127 """Process a single line of a patch file or commit log
129 This process a line and returns a list of lines to output. The list
130 may be empty or may contain multiple output lines.
132 This is where all the complicated logic is located. The class's
133 state is used to move between different states and detect things
136 We can be in one of two modes:
137 self.is_log == True: This is 'git log' mode, where most output is
138 indented by 4 characters and we are scanning for tags
140 self.is_log == False: This is 'patch' mode, where we already have
141 all the tags, and are processing patches to remove junk we
142 don't want, and add things we think are required.
145 line: text line to process
148 list of output lines, or [] if nothing should be output
150 # Initially we have no output. Prepare the input line string
152 line = line.rstrip('\n')
157 # Handle state transition and skipping blank lines
158 series_tag_match = re_series_tag.match(line)
159 commit_tag_match = re_commit_tag.match(line)
160 commit_match = re_commit.match(line) if self.is_log else None
161 cover_cc_match = re_cover_cc.match(line)
162 signoff_match = re_signoff.match(line)
164 if self.state == STATE_PATCH_HEADER:
165 tag_match = re_tag.match(line)
166 is_blank = not line.strip()
168 if (self.state == STATE_MSG_HEADER
169 or self.state == STATE_PATCH_SUBJECT):
172 # We don't have a subject in the text stream of patch files
173 # It has its own line with a Subject: tag
174 if not self.is_log and self.state == STATE_PATCH_SUBJECT:
177 self.state = STATE_MSG_HEADER
179 # If we are in a section, keep collecting lines until we see END
182 if self.in_section == 'cover':
183 self.series.cover = self.section
184 elif self.in_section == 'notes':
186 self.series.notes += self.section
187 elif self.in_section == 'commit-notes':
189 self.commit.notes += self.section
191 self.warn.append("Unknown section '%s'" % self.in_section)
192 self.in_section = None
193 self.skip_blank = True
196 self.section.append(line)
198 # Detect the commit subject
199 elif not is_blank and self.state == STATE_PATCH_SUBJECT:
200 self.commit.subject = line
202 # Detect the tags we want to remove, and skip blank lines
203 elif re_remove.match(line) and not commit_tag_match:
204 self.skip_blank = True
206 # TEST= should be the last thing in the commit, so remove
207 # everything after it
208 if line.startswith('TEST='):
209 self.found_test = True
210 elif self.skip_blank and is_blank:
211 self.skip_blank = False
213 # Detect the start of a cover letter section
214 elif re_cover.match(line):
215 self.in_section = 'cover'
216 self.skip_blank = False
219 value = cover_cc_match.group(1)
220 self.AddToSeries(line, 'cover-cc', value)
222 # If we are in a change list, key collected lines until a blank one
225 # Blank line ends this change list
229 out = self.ProcessLine(line)
232 self.series.AddChange(self.in_change, self.commit, line)
233 self.skip_blank = False
235 # Detect Series-xxx tags
236 elif series_tag_match:
237 name = series_tag_match.group(1)
238 value = series_tag_match.group(2)
239 if name == 'changes':
240 # value is the version number: e.g. 1, or 2
243 except ValueError as str:
244 raise ValueError("%s: Cannot decode version info '%s'" %
245 (self.commit.hash, line))
246 self.in_change = int(value)
248 self.AddToSeries(line, name, value)
249 self.skip_blank = True
251 # Detect Commit-xxx tags
252 elif commit_tag_match:
253 name = commit_tag_match.group(1)
254 value = commit_tag_match.group(2)
256 self.AddToCommit(line, name, value)
257 self.skip_blank = True
259 # Detect the start of a new commit
262 # TODO: We should store the whole hash, and just display a subset
263 self.commit = commit.Commit(commit_match.group(1)[:8])
265 # Detect tags in the commit message
267 # Remove Tested-by self, since few will take much notice
268 if (tag_match.group(1) == 'Tested-by' and
269 tag_match.group(2).find(os.getenv('USER') + '@') != -1):
270 self.warn.append("Ignoring %s" % line)
271 elif tag_match.group(1) == 'Patch-cc':
272 self.commit.AddCc(tag_match.group(2).split(','))
274 self.tags.append(line);
276 # Suppress duplicate signoffs
278 if self.commit.CheckDuplicateSignoff(signoff_match.group(1)):
281 # Well that means this is an ordinary line
284 # Look for ugly ASCII characters
286 # TODO: Would be nicer to report source filename and line
288 self.warn.append("Line %d/%d ('%s') has funny ascii char" %
289 (self.linenum, pos, line))
292 # Look for space before tab
293 m = re_space_before_tab.match(line)
295 self.warn.append('Line %d/%d has space before tab' %
296 (self.linenum, m.start()))
298 # OK, we have a valid non-blank line
301 self.skip_blank = False
302 if self.state == STATE_DIFFS:
305 # If this is the start of the diffs section, emit our tags and
308 self.state = STATE_DIFFS
310 # Output the tags (signeoff first), then change list
312 log = self.series.MakeChangeLog(self.commit)
313 out += self.FormatTags(self.tags)
314 out += [line] + self.commit.notes + [''] + log
315 elif self.found_test:
316 if not re_allowed_after_test.match(line):
317 self.lines_after_test += 1
322 """Close out processing of this patch stream"""
324 if self.lines_after_test:
325 self.warn.append('Found %d lines after TEST=' %
326 self.lines_after_test)
328 def ProcessStream(self, infd, outfd):
329 """Copy a stream from infd to outfd, filtering out unwanting things.
331 This is used to process patch files one at a time.
334 infd: Input stream file object
335 outfd: Output stream file object
337 # Extract the filename from each diff, for nice warnings
340 re_fname = re.compile('diff --git a/(.*) b/.*')
342 line = infd.readline()
345 out = self.ProcessLine(line)
347 # Try to detect blank lines at EOF
349 match = re_fname.match(line)
352 fname = match.group(1)
354 self.blank_count += 1
356 if self.blank_count and (line == '-- ' or match):
357 self.warn.append("Found possible blank line(s) at "
358 "end of file '%s'" % last_fname)
359 outfd.write('+\n' * self.blank_count)
360 outfd.write(line + '\n')
365 def GetMetaDataForList(commit_range, git_dir=None, count=None,
367 """Reads out patch series metadata from the commits
369 This does a 'git log' on the relevant commits and pulls out the tags we
373 commit_range: Range of commits to count (e.g. 'HEAD..base')
374 git_dir: Path to git repositiory (None to use default)
375 count: Number of commits to list, or None for no limit
376 series: Series object to add information into. By default a new series
379 A Series object containing information about the commits.
381 params = ['git', 'log', '--no-color', '--reverse', '--no-decorate',
383 if count is not None:
384 params[2:2] = ['-n%d' % count]
386 params[1:1] = ['--git-dir', git_dir]
388 stdout = command.RunPipe(pipe, capture=True).stdout
389 ps = PatchStream(series, is_log=True)
390 for line in stdout.splitlines():
395 def GetMetaData(start, count):
396 """Reads out patch series metadata from the commits
398 This does a 'git log' on the relevant commits and pulls out the tags we
402 start: Commit to start from: 0=HEAD, 1=next one, etc.
403 count: Number of commits to list
405 return GetMetaDataForList('HEAD~%d' % start, None, count)
407 def FixPatch(backup_dir, fname, series, commit):
408 """Fix up a patch file, by adding/removing as required.
410 We remove our tags from the patch file, insert changes lists, etc.
411 The patch file is processed in place, and overwritten.
413 A backup file is put into backup_dir (if not None).
416 fname: Filename to patch file to process
417 series: Series information about this patch set
418 commit: Commit object for this patch file
420 A list of errors, or [] if all ok.
422 handle, tmpname = tempfile.mkstemp()
423 outfd = os.fdopen(handle, 'w')
424 infd = open(fname, 'r')
425 ps = PatchStream(series)
427 ps.ProcessStream(infd, outfd)
431 # Create a backup file if required
433 shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname)))
434 shutil.move(tmpname, fname)
437 def FixPatches(series, fnames):
438 """Fix up a list of patches identified by filenames
440 The patch files are processed in place, and overwritten.
443 series: The series object
444 fnames: List of patch files to process
446 # Current workflow creates patches, so we shouldn't need a backup
447 backup_dir = None #tempfile.mkdtemp('clean-patch')
450 commit = series.commits[count]
452 result = FixPatch(backup_dir, fname, series, commit)
454 print '%d warnings for %s:' % (len(result), fname)
459 print 'Cleaned %d patches' % count
462 def InsertCoverLetter(fname, series, count):
463 """Inserts a cover letter with the required info into patch 0
466 fname: Input / output filename of the cover letter file
467 series: Series object
468 count: Number of patches in the series
470 fd = open(fname, 'r')
471 lines = fd.readlines()
474 fd = open(fname, 'w')
476 prefix = series.GetPatchPrefix()
478 if line.startswith('Subject:'):
479 # TODO: if more than 10 patches this should save 00/xx, not 0/xx
480 line = 'Subject: [%s 0/%d] %s\n' % (prefix, count, text[0])
482 # Insert our cover letter
483 elif line.startswith('*** BLURB HERE ***'):
484 # First the blurb test
485 line = '\n'.join(text[1:]) + '\n'
486 if series.get('notes'):
487 line += '\n'.join(series.notes) + '\n'
489 # Now the change list
490 out = series.MakeChangeLog(None)
491 line += '\n' + '\n'.join(out)