Commit 0d24de9d authored by Simon Glass's avatar Simon Glass Committed by Wolfgang Denk

Add 'patman' patch generation, checking and submission script

What is this?

=============

This tool is a Python script which:
- Creates patch directly from your branch
- Cleans them up by removing unwanted tags
- Inserts a cover letter with change lists
- Runs the patches through checkpatch.pl and its own checks
- Optionally emails them out to selected people

It is intended to automate patch creation and make it a less
error-prone process. It is useful for U-Boot and Linux work so far,
since it uses the checkpatch.pl script.

It is configured almost entirely by tags it finds in your commits.
This means that you can work on a number of different branches at
once, and keep the settings with each branch rather than having to
git format-patch, git send-email, etc. with the correct parameters
each time. So for example if you put:

in one of your commits, the series will be sent there.

See the README file for full details.
END
Signed-off-by: default avatarSimon Glass <sjg@chromium.org>
parent 0d62032e
This diff is collapsed.
# Copyright (c) 2011 The Chromium OS Authors.
#
# See file CREDITS for list of people who contributed to this
# project.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of
# the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307 USA
#
import command
import gitutil
import os
import re
import terminal
def FindCheckPatch():
try_list = [
os.getcwd(),
os.path.join(os.getcwd(), '..', '..'),
os.path.join(gitutil.GetTopLevel(), 'tools'),
'%s/bin' % os.getenv('HOME'),
]
# Look in current dir
for path in try_list:
fname = os.path.join(path, 'checkpatch.pl')
if os.path.isfile(fname):
return fname
# Look upwwards for a Chrome OS tree
while not os.path.ismount(path):
fname = os.path.join(path, 'src', 'third_party', 'kernel', 'files',
'scripts', 'checkpatch.pl')
if os.path.isfile(fname):
return fname
path = os.path.dirname(path)
print 'Could not find checkpatch.pl'
return None
def CheckPatch(fname, verbose=False):
"""Run checkpatch.pl on a file.
Returns:
4-tuple containing:
result: False=failure, True=ok
problems: List of problems, each a dict:
'type'; error or warning
'msg': text message
'file' : filename
'line': line number
lines: Number of lines
"""
result = False
error_count, warning_count, lines = 0, 0, 0
problems = []
chk = FindCheckPatch()
if not chk:
raise OSError, ('Cannot find checkpatch.pl - please put it in your ' +
'~/bin directory')
item = {}
stdout = command.Output(chk, '--no-tree', fname)
#pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE)
#stdout, stderr = pipe.communicate()
# total: 0 errors, 0 warnings, 159 lines checked
re_stats = re.compile('total: (\\d+) errors, (\d+) warnings, (\d+)')
re_ok = re.compile('.*has no obvious style problems')
re_bad = re.compile('.*has style problems, please review')
re_error = re.compile('ERROR: (.*)')
re_warning = re.compile('WARNING: (.*)')
re_file = re.compile('#\d+: FILE: ([^:]*):(\d+):')
for line in stdout.splitlines():
if verbose:
print line
# A blank line indicates the end of a message
if not line and item:
problems.append(item)
item = {}
match = re_stats.match(line)
if match:
error_count = int(match.group(1))
warning_count = int(match.group(2))
lines = int(match.group(3))
elif re_ok.match(line):
result = True
elif re_bad.match(line):
result = False
match = re_error.match(line)
if match:
item['msg'] = match.group(1)
item['type'] = 'error'
match = re_warning.match(line)
if match:
item['msg'] = match.group(1)
item['type'] = 'warning'
match = re_file.match(line)
if match:
item['file'] = match.group(1)
item['line'] = int(match.group(2))
return result, problems, error_count, warning_count, lines, stdout
def GetWarningMsg(col, msg_type, fname, line, msg):
'''Create a message for a given file/line
Args:
msg_type: Message type ('error' or 'warning')
fname: Filename which reports the problem
line: Line number where it was noticed
msg: Message to report
'''
if msg_type == 'warning':
msg_type = col.Color(col.YELLOW, msg_type)
elif msg_type == 'error':
msg_type = col.Color(col.RED, msg_type)
return '%s: %s,%d: %s' % (msg_type, fname, line, msg)
def CheckPatches(verbose, args):
'''Run the checkpatch.pl script on each patch'''
error_count = 0
warning_count = 0
col = terminal.Color()
for fname in args:
ok, problems, errors, warnings, lines, stdout = CheckPatch(fname,
verbose)
if not ok:
error_count += errors
warning_count += warnings
print '%d errors, %d warnings for %s:' % (errors,
warnings, fname)
if len(problems) != error_count + warning_count:
print "Internal error: some problems lost"
for item in problems:
print GetWarningMsg(col, item['type'], item['file'],
item['line'], item['msg'])
#print stdout
if error_count != 0 or warning_count != 0:
str = 'checkpatch.pl found %d error(s), %d warning(s)' % (
error_count, warning_count)
color = col.GREEN
if warning_count:
color = col.YELLOW
if error_count:
color = col.RED
print col.Color(color, str)
return False
return True
# Copyright (c) 2011 The Chromium OS Authors.
#
# See file CREDITS for list of people who contributed to this
# project.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of
# the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307 USA
#
import os
import subprocess
"""Shell command ease-ups for Python."""
def RunPipe(pipeline, infile=None, outfile=None,
capture=False, oneline=False, hide_stderr=False):
"""
Perform a command pipeline, with optional input/output filenames.
hide_stderr Don't allow output of stderr (default False)
"""
last_pipe = None
while pipeline:
cmd = pipeline.pop(0)
kwargs = {}
if last_pipe is not None:
kwargs['stdin'] = last_pipe.stdout
elif infile:
kwargs['stdin'] = open(infile, 'rb')
if pipeline or capture:
kwargs['stdout'] = subprocess.PIPE
elif outfile:
kwargs['stdout'] = open(outfile, 'wb')
if hide_stderr:
kwargs['stderr'] = open('/dev/null', 'wb')
last_pipe = subprocess.Popen(cmd, **kwargs)
if capture:
ret = last_pipe.communicate()[0]
if not ret:
return None
elif oneline:
return ret.rstrip('\r\n')
else:
return ret
else:
return os.waitpid(last_pipe.pid, 0)[1] == 0
def Output(*cmd):
return RunPipe([cmd], capture=True)
def OutputOneLine(*cmd):
return RunPipe([cmd], capture=True, oneline=True)
def Run(*cmd, **kwargs):
return RunPipe([cmd], **kwargs)
def RunList(cmd):
return RunPipe([cmd], capture=True)
# Copyright (c) 2011 The Chromium OS Authors.
#
# See file CREDITS for list of people who contributed to this
# project.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of
# the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307 USA
#
import re
# Separates a tag: at the beginning of the subject from the rest of it
re_subject_tag = re.compile('([^:]*):\s*(.*)')
class Commit:
"""Holds information about a single commit/patch in the series.
Args:
hash: Commit hash (as a string)
Variables:
hash: Commit hash
subject: Subject line
tags: List of maintainer tag strings
changes: Dict containing a list of changes (single line strings).
The dict is indexed by change version (an integer)
cc_list: List of people to aliases/emails to cc on this commit
"""
def __init__(self, hash):
self.hash = hash
self.subject = None
self.tags = []
self.changes = {}
self.cc_list = []
def AddChange(self, version, info):
"""Add a new change line to the change list for a version.
Args:
version: Patch set version (integer: 1, 2, 3)
info: Description of change in this version
"""
if not self.changes.get(version):
self.changes[version] = []
self.changes[version].append(info)
def CheckTags(self):
"""Create a list of subject tags in the commit
Subject tags look like this:
propounder: Change the widget to propound correctly
Multiple tags are supported. The list is updated in self.tag
Returns:
None if ok, else the name of a tag with no email alias
"""
str = self.subject
m = True
while m:
m = re_subject_tag.match(str)
if m:
tag = m.group(1)
self.tags.append(tag)
str = m.group(2)
return None
def AddCc(self, cc_list):
"""Add a list of people to Cc when we send this patch.
Args:
cc_list: List of aliases or email addresses
"""
self.cc_list += cc_list
This diff is collapsed.
This diff is collapsed.
patman.py
\ No newline at end of file
#!/usr/bin/python
#
# Copyright (c) 2011 The Chromium OS Authors.
#
# See file CREDITS for list of people who contributed to this
# project.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of
# the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307 USA
#
"""See README for more information"""
from optparse import OptionParser
import os
import re
import sys
import unittest
# Our modules
import checkpatch
import command
import gitutil
import patchstream
import terminal
import test
parser = OptionParser()
parser.add_option('-H', '--full-help', action='store_true', dest='full_help',
default=False, help='Display the README file')
parser.add_option('-c', '--count', dest='count', type='int',
default=-1, help='Automatically create patches from top n commits')
parser.add_option('-i', '--ignore-errors', action='store_true',
dest='ignore_errors', default=False,
help='Send patches email even if patch errors are found')
parser.add_option('-n', '--dry-run', action='store_true', dest='dry_run',
default=False, help="Do a try run (create but don't email patches)")
parser.add_option('-s', '--start', dest='start', type='int',
default=0, help='Commit to start creating patches from (0 = HEAD)')
parser.add_option('-t', '--test', action='store_true', dest='test',
default=False, help='run tests')
parser.add_option('-v', '--verbose', action='store_true', dest='verbose',
default=False, help='Verbose output of errors and warnings')
parser.add_option('--cc-cmd', dest='cc_cmd', type='string', action='store',
default=None, help='Output cc list for patch file (used by git)')
parser.add_option('--no-tags', action='store_false', dest='process_tags',
default=True, help="Don't process subject tags as aliaes")
parser.usage = """patman [options]
Create patches from commits in a branch, check them and email them as
specified by tags you place in the commits. Use -n to """
(options, args) = parser.parse_args()
# Run our meagre tests
if options.test:
import doctest
sys.argv = [sys.argv[0]]
suite = unittest.TestLoader().loadTestsFromTestCase(test.TestPatch)
result = unittest.TestResult()
suite.run(result)
suite = doctest.DocTestSuite('gitutil')
suite.run(result)
# TODO: Surely we can just 'print' result?
print result
for test, err in result.errors:
print err
for test, err in result.failures:
print err
# Called from git with a patch filename as argument
# Printout a list of additional CC recipients for this patch
elif options.cc_cmd:
fd = open(options.cc_cmd, 'r')
re_line = re.compile('(\S*) (.*)')
for line in fd.readlines():
match = re_line.match(line)
if match and match.group(1) == args[0]:
for cc in match.group(2).split(', '):
cc = cc.strip()
if cc:
print cc
fd.close()
elif options.full_help:
pager = os.getenv('PAGER')
if not pager:
pager = 'more'
fname = os.path.join(os.path.dirname(sys.argv[0]), 'README')
command.Run(pager, fname)
# Process commits, produce patches files, check them, email them
else:
gitutil.Setup()
if options.count == -1:
# Work out how many patches to send if we can
options.count = gitutil.CountCommitsToBranch() - options.start
col = terminal.Color()
if not options.count:
str = 'No commits found to process - please use -c flag'
print col.Color(col.RED, str)
sys.exit(1)
# Read the metadata from the commits
if options.count:
series = patchstream.GetMetaData(options.start, options.count)
cover_fname, args = gitutil.CreatePatches(options.start, options.count,
series)
# Fix up the patch files to our liking, and insert the cover letter
series = patchstream.FixPatches(series, args)
if series and cover_fname and series.get('cover'):
patchstream.InsertCoverLetter(cover_fname, series, options.count)
# Do a few checks on the series
series.DoChecks()
# Check the patches, and run them through 'git am' just to be sure
ok = checkpatch.CheckPatches(options.verbose, args)
if not gitutil.ApplyPatches(options.verbose, args,
options.count + options.start):
ok = False
# Email the patches out (giving the user time to check / cancel)
cmd = ''
if ok or options.ignore_errors:
cc_file = series.MakeCcFile(options.process_tags)
cmd = gitutil.EmailPatches(series, cover_fname, args,
options.dry_run, cc_file)
os.remove(cc_file)
# For a dry run, just show our actions as a sanity check
if options.dry_run:
series.ShowActions(args, cmd, options.process_tags)
# Copyright (c) 2011 The Chromium OS Authors.
#
# See file CREDITS for list of people who contributed to this
# project.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of
# the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307 USA
#
import os
import gitutil
import terminal
# Series-xxx tags that we understand
valid_series = ['to', 'cc', 'version', 'changes', 'prefix', 'notes'];
class Series(dict):
"""Holds information about a patch series, including all tags.
Vars:
cc: List of aliases/emails to Cc all patches to
commits: List of Commit objects, one for each patch
cover: List of lines in the cover letter
notes: List of lines in the notes
changes: (dict) List of changes for each version, The key is
the integer version number
"""
def __init__(self):
self.cc = []
self.to = []
self.commits = []
self.cover = None
self.notes = []
self.changes = {}
# These make us more like a dictionary
def __setattr__(self, name, value):
self[name] = value
def __getattr__(self, name):
return self[name]
def AddTag(self, commit, line, name, value):
"""Add a new Series-xxx tag along with its value.
Args:
line: Source line containing tag (useful for debug/error messages)
name: Tag name (part after 'Series-')
value: Tag value (part after 'Series-xxx: ')
"""
# If we already have it, then add to our list
if name in self:
values = value.split(',')
values = [str.strip() for str in values]
if type(self[name]) != type([]):
raise ValueError("In %s: line '%s': Cannot add another value "
"'%s' to series '%s'" %
(commit.hash, line, values, self[name]))
self[name] += values
# Otherwise just set the value
elif name in valid_series:
self[name] = value
else:
raise ValueError("In %s: line '%s': Unknown 'Series-%s': valid "
"options are %s" % (self.commit.hash, line, name,
', '.join(valid_series)))
def AddCommit(self, commit):
"""Add a commit into our list of commits
We create a list of tags in the commit subject also.
Args:
commit: Commit object to add
"""
commit.CheckTags()
self.commits.append(commit)
def ShowActions(self, args, cmd, process_tags):
"""Show what actions we will/would perform
Args:
args: List of patch files we created
cmd: The git command we would have run
process_tags: Process tags as if they were aliases
"""
col = terminal.Color()
print 'Dry run, so not doing much. But I would do this:'
print
print 'Send a total of %d patch%s with %scover letter.' % (
len(args), '' if len(args) == 1 else 'es',
self.get('cover') and 'a ' or 'no ')
# TODO: Colour the patches according to whether they passed checks
for upto in range(len(args)):
commit = self.commits[upto]
print col.Color(col.GREEN, ' %s' % args[upto])
cc_list = []
if process_tags:
cc_list += gitutil.BuildEmailList(commit.tags)
cc_list += gitutil.BuildEmailList(commit.cc_list)
for email in cc_list:
if email == None:
email = col.Color(col.YELLOW, "<alias '%s' not found>"
% tag)
if email:
print ' Cc: ',email
print
for item in gitutil.BuildEmailList(self.get('to', '<none>')):
print 'To:\t ', item
for item in gitutil.BuildEmailList(self.cc):
print 'Cc:\t ', item
print 'Version: ', self.get('version')
print 'Prefix:\t ', self.get('prefix')
if self.cover:
print 'Cover: %d lines' % len(self.cover)
if cmd:
print 'Git command: %s' % cmd
def MakeChangeLog(self, commit):
"""Create a list of changes for each version.
Return:
The change log as a list of strings, one per line
Changes in v1:
- Fix the widget
- Jog the dial
Changes in v2:
- Jog the dial back closer to the widget
etc.
"""
final = []
need_blank = False
for change in sorted(self.changes):
out = []
for this_commit, text in self.changes[change]:
if commit and this_commit != commit:
continue
if text not in out:
out.append(text)
if out:
out = ['Changes in v%d:' % change] + sorted(out)
if need_blank:
out = [''] + out
final += out
need_blank = True
if self.changes:
final.append('')
return final
def DoChecks(self):
"""Check that each version has a change log
Print an error if something is wrong.
"""
col = terminal.Color()
if self.get('version'):
changes_copy = dict(self.changes)
for version in range(2, int(self.version) + 1):
if self.changes.get(version):
del changes_copy[version]
else:
str = 'Change log missing for v%d' % version
print col.Color(col.RED, str)
for version in changes_copy:
str = 'Change log for unknown version v%d' % version
print col.Color(col.RED, str)
elif self.changes:
str = 'Change log exists, but no version is set'
print col.Color(col.RED, str)
def MakeCcFile(self, process_tags):
"""Make a cc file for us to use for per-commit Cc automation
Args:
process_tags: Process tags as if they were aliases
Return:
Filename of temp file created
"""
# Look for commit tags (of the form 'xxx:' at the start of the subject)
fname = '/tmp/patman.%d' % os.getpid()
fd = open(fname, 'w')
for commit in self.commits:
list = []
if process_tags:
list += gitutil.BuildEmailList(commit.tags)
list += gitutil.BuildEmailList(commit.cc_list)
print >>fd, commit.patch, ', '.join(list)
fd.close()
return fname
def AddChange(self, version, commit, info):
"""Add a new change line to a version.
This will later appear in the change log.
Args:
version: version number to add change list to
info: change line for this version
"""
if not self.changes.get(version):
self.changes[version] = []
self.changes[version].append([commit, info])
def GetPatchPrefix(self):
"""Get the patch version string
Return:
Patch string, like 'RFC PATCH v5' or just 'PATCH'
"""
version = ''
if self.get('version'):
version = ' v%s' % self['version']
# Get patch name prefix
prefix = ''
if self.get('prefix'):
prefix = '%s ' % self['prefix']
return '%sPATCH%s' % (prefix, version)
# Copyright (c) 2011 The Chromium OS Authors.
#
# See file CREDITS for list of people who contributed to this
# project.
#
# This program is free software; you can redistribute it and/or