[Asterisk-code-review] release summary: Refactor module to support Git (repotools[master])
Matt Jordan
asteriskteam at digium.com
Tue Apr 21 08:13:49 CDT 2015
Matt Jordan has uploaded a new change for review.
https://gerrit.asterisk.org/179
Change subject: release_summary: Refactor module to support Git
......................................................................
release_summary: Refactor module to support Git
This patch refactors the release-summary script to support Git. Along
the way, the module was also refactored to make it easier to use the
release-summary functionality within another Python module, by placing
the logic for generating a release summary within a Python class and
by renaming the module to 'release_summary'.
The release summary that is generated will now also show:
* A break down of New Features, Improvements, and Bug Fixes in the
release summary. Since point releases can now include New Features
and Improvements, the release summary makes a point of
differentiating between these issues and Bugs.
* All JIRA issues that are still open but that had commits made against
them in this release. This is separate from the miscellaneous
commits, which have no JIRA issue related to them.
* All Git commits associated with a JIRA issue. This is particularly
helpful in mapping a solution to a particular problem, especially
when multiple commits were needed to resolve an issue.
REP-13 #close
Reported by: Matt Jordan
Change-Id: I97275322fe5b8067e2c7317b6fbbfab532e6cd48
---
D release-summary.py
A release_summary.py
2 files changed, 704 insertions(+), 621 deletions(-)
git pull ssh://gerrit.asterisk.org:29418/repotools refs/changes/79/179/3
diff --git a/release-summary.py b/release-summary.py
deleted file mode 100755
index 9ab32f1..0000000
--- a/release-summary.py
+++ /dev/null
@@ -1,621 +0,0 @@
-#!/usr/bin/env python
-'''Generate a release summary document based on data from commits.
-
-Copyright (C) 2009 - 2011, Digium, Inc.
-David Vossel (Batman) <dvossel at digium.com>
-Russell Bryant <russell at digium.com>
-'''
-
-import sys
-import string
-import os
-import operator
-import re
-import string
-import datetime
-import tempfile
-import subprocess
-import pysvn
-import getpass
-from optparse import OptionParser
-from progressbar import ProgressBar
-from digium_commits import DigiumCommitMessageParser
-from digium_commits import convert_name
-from jira.client import JIRA
-
-
-RELEASE_TYPE_FEATURE = 1
-RELEASE_TYPE_BUGFIX = 2
-RELEASE_TYPE_SECURITY = 3
-
-release_type_dic = {
- "feature": RELEASE_TYPE_FEATURE,
- "bugfix": RELEASE_TYPE_BUGFIX,
- "security": RELEASE_TYPE_SECURITY
-}
-
-'''Maximum column width for diffstat and plain text output'''
-MAX_WIDTH = 80
-
-'''URL prefix for JIRA issues'''
-JIRA_URL = "https://issues.asterisk.org/jira/browse/"
-
-'''URL prefix for security advisories'''
-ADVISORY_URL = "http://downloads.asterisk.org/pub/security/"
-
-
-class Issue:
- def __init__(self, issue, rev, parser, jira):
- self.rev = rev
- self.issue_id = issue["id"]
- if issue["type"] != "JIRA":
- self.jira_issue = None
- return
- self.jira_issue = jira.issue(self.issue_id)
- self.reporter_name = self.jira_issue.fields.reporter.name
- self.testers = parser.get_testers()
- self.coders = parser.get_coders()
-
-
-class MiscCommit:
- pass
-
-
-class Contributors:
- def __init__(self):
- self.reporter_dic = {}
- self.coder_dic = {}
- self.tester_dic = {}
- self.reporter_sorted = []
- self.coder_sorted = []
- self.tester_sorted = []
-
- def sort_lists(self):
- for key in self.reporter_dic.iterkeys():
- self.reporter_sorted.append((self.reporter_dic[key], key))
- self.reporter_sorted.sort(key=lambda x:
- (sys.maxint - x[0], x[1].lower()))
-
- for key in self.tester_dic.iterkeys():
- self.tester_sorted.append((self.tester_dic[key], key))
- self.tester_sorted.sort(key=lambda x:
- (sys.maxint - x[0], x[1].lower()))
-
- for key in self.coder_dic.iterkeys():
- self.coder_sorted.append((self.coder_dic[key], key))
- self.coder_sorted.sort(key=lambda x:
- (sys.maxint - x[0], x[1].lower()))
-
- def add_coder(self, coder):
- if coder in self.coder_dic:
- self.coder_dic[coder] += 1
- else:
- self.coder_dic[coder] = 1
-
- def add_count(self, jira_issue):
- reporter = jira_issue.reporter_name
- if reporter in self.reporter_dic:
- self.reporter_dic[reporter] += 1
- elif reporter != "":
- self.reporter_dic[reporter] = 1
-
- for tester in jira_issue.testers:
- if tester in self.tester_dic:
- self.tester_dic[tester] += 1
- else:
- self.tester_dic[tester] = 1
-
- for coder in jira_issue.coders:
- self.add_coder(coder)
-
-
-def main(argv=None):
- if argv is None:
- argv = sys.argv
-
- parser = OptionParser()
-
- parser.add_option("-u", "--svn-url", action="store", type="string",
- dest="svn_path", default="", help="SVN URL to use")
- parser.add_option("-s", "--start-rev", action="store", type="string",
- dest="start_rev", default="", help="Starting SVN Revision")
- parser.add_option("-e", "--end-rev", action="store", type="string",
- dest="end_rev", default="", help="Ending SVN Revision")
- parser.add_option("-p", "--project", action="store", type="string",
- dest="project", default="",
- help="Name of the project (such as Asterisk)")
- parser.add_option("-v", "--version", action="store", type="string",
- dest="version", default="",
- help="Version ID (the SVN tag)")
- parser.add_option("-P", "--prev-version", action="store", type="string",
- dest="prev_version", default="",
- help="Previous version ID (the SVN tag)")
- parser.add_option("-t", "--release-type", action="store",
- type="string", dest="release_type_str", default="bugfix",
- help="Type of release: \"feature\", \"bugfix\", or \"security\"")
- parser.add_option("-a", "--advisories", action="store", type="string",
- dest="advisories", default="",
- help="A comma separated list of security advisories addressed " \
- "in this release")
- parser.add_option("-A", action="store", type="string",
- dest="additional_tags", default=None,
- help="A comma separated list of additional tags to include changes" \
- "from.")
- parser.add_option("-o", "--at-only", action="store_true",
- dest="at_only", default=False,
- help="Only process additional tags")
-
- (options, args) = parser.parse_args(argv)
-
- if len(options.svn_path) == 0:
- parser.error("An SVN path is required.")
-
- if len(options.start_rev) == 0:
- parser.error("A beginning SVN revision is required.")
-
- if len(options.end_rev) == 0:
- parser.error("An ending SVN revision is required.")
-
- if len(options.project) == 0:
- parser.error("A project name is required.")
-
- if len(options.version) == 0:
- parser.error("A version ID is required.")
-
- if len(options.prev_version) == 0:
- parser.error("A previous version ID is required.")
-
- if options.release_type_str in release_type_dic:
- options.release_type = release_type_dic[options.release_type_str]
- else:
- parser.error("The release type provided is not valid.")
-
- if options.release_type == "security" and len(options.advisories) == 0:
- parser.error("You must provide at least one security advisory for " \
- "a security release.")
-
- collect_data(options)
-
-
-def parse_additional_tags(additional_tags):
- tags = []
-
- if not additional_tags:
- return tags
-
- for t in additional_tags.split(","):
- t = t.strip()
- tags.append(t)
-
- return tags
-
-
-def get_tag_start(tag):
- tmpf = tempfile.NamedTemporaryFile()
- os.system("svn log --verbose --stop-on-copy --xml %s | grep "
- "copyfrom-rev | cut --delimiter== --fields=2 | sed -e "
- "s/\\\"//g > %s" % (tag, tmpf.name))
- tag_start = tmpf.read().strip()
- tmpf.close()
- return tag_start
-
-
-def build_tag_path(options, tag):
- return "%s/tags/%s" % (options.svn_path.rsplit("/", 2)[0], tag)
-
-
-def __get_jira_auth():
- '''Private method that looks up JIRA login credentials.
- '''
- try:
- f = open(os.path.expanduser('~') + "/.jira_login", "r")
- jira_user = f.readline().strip()
- jira_pw = f.readline().strip()
- f.close()
- return (jira_user, jira_pw)
- except:
- pass
-
- # Didn't get auth deatils from file, try interactive instead.
- print "Please enter your username and pw for JIRA."
- jira_user = raw_input("Username: ")
- jira_pw = getpass.getpass("Password: ")
-
- return (jira_user, jira_pw)
-
-def collect_data(options):
- print "Generating release summary for %s-%s ..." % \
- (options.project, options.version)
-
- contributors = Contributors()
-
- (jira_user, jira_password) = __get_jira_auth()
-
- jira_options = {
- 'server': 'https://issues.asterisk.org/jira/'
- }
-
- jira = JIRA(options = jira_options, basic_auth = (jira_user, jira_password))
-
- client = pysvn.Client()
- client.update(options.svn_path)
-
- start_rev = pysvn.Revision(pysvn.opt_revision_kind.number,
- options.start_rev)
- if options.end_rev == "HEAD":
- end_rev = pysvn.Revision(pysvn.opt_revision_kind.head)
- else:
- end_rev = pysvn.Revision(pysvn.opt_revision_kind.number,
- options.end_rev)
-
- # XXX BEGIN HACK THAT SHOULD BE REMOVED
- #hack_start = pysvn.Revision(pysvn.opt_revision_kind.number, 182359) # when 1.6.2 was branched
- #hack_end = pysvn.Revision(pysvn.opt_revision_kind.number, 279056) # when 1.8 was branched
- #log_messages = client.log("https://origsvn.digium.com/svn/asterisk/trunk",
- # hack_start, hack_end)
- #log_messages.extend(client.log(options.svn_path, start_rev, end_rev))
- # XXX END HACK
-
- # Uncomment after hack removed
- log_messages = client.log(options.svn_path, start_rev, end_rev)
-
- additional_tags = parse_additional_tags(options.additional_tags)
- extra_logs = []
- for t in additional_tags:
- tag_path = build_tag_path(options, t)
- tag_start = get_tag_start(tag_path)
- extra_logs.extend(client.log(tag_path,
- pysvn.Revision(pysvn.opt_revision_kind.number, tag_start),
- pysvn.Revision(pysvn.opt_revision_kind.head)))
-
- if options.at_only:
- log_messages = extra_logs
- else:
- log_messages.extend(extra_logs)
-
- jira_issues = {}
- misc_commits = []
-
- num_log_msgs = len(log_messages)
-
- if options.at_only:
- print "Processing logs for commits on tags: %s ..." % options.additional_tags
- elif options.additional_tags:
- print "Processing revisions %s through %s of %s, and commits on tags: %s ..." % \
- (options.start_rev, options.end_rev, options.svn_path,
- options.additional_tags)
- else:
- print "Processing revisions %s through %s of %s ..." % \
- (options.start_rev, options.end_rev, options.svn_path)
-
- pbar = ProgressBar()
- pbar.maxval = num_log_msgs
- pbar.start()
-
- for i in range(0, num_log_msgs):
- log_message = log_messages[i]
- try:
- parser = DigiumCommitMessageParser(log_message.message,
- log_message.author)
- except:
- print "Failed to create message parser for %s" % str(log_message)
- pbar.update(i + 1)
- continue
-
- if parser.block:
- # Don't process the log message if it was actually a block
- pbar.update(i + 1)
- continue
-
- issues = parser.get_issues(closed=True)
-
- if len(issues) == 0:
- commit = MiscCommit()
- commit.rev = log_message.revision.number
- commit.author = log_message.author
- commit.summary = parser.get_commit_summary()
- try:
- commit.issues = [
- Issue(issue_id, log_message.revision.number, parser, jira)
- for issue_id in parser.get_issues(closed=False)
- ]
- except:
- print "Failed to create issue %s; bypassing" % issue_id
- continue
- misc_commits.append(commit)
- contributors.add_coder(commit.author)
- pbar.update(i + 1)
- continue
-
- for issue_id in issues:
- try:
- issue = Issue(issue_id, log_message.revision.number, parser, jira)
- except:
- print "Failed to create issue %s; bypassing" % issue_id
- continue
- if issue.jira_issue is None:
- print "Could not get issue %s" % issue.issue_id
- continue
-
- for c in issue.jira_issue.fields.components:
- if c.name not in jira_issues:
- jira_issues[c.name] = []
- jira_issues[c.name].append(issue)
- contributors.add_count(issue)
-
- pbar.update(i + 1)
-
- pbar.finish()
-
- print "Generating diffstat ..."
- diff_list = get_diff_stat(options)
-
- contributors.sort_lists()
-
- jira_category_list = []
-
- for category, issues in jira_issues.items():
- issues.sort(key=lambda x:x.issue_id)
- jira_category_list.append((category, issues))
- jira_category_list.sort(key=lambda x:x[0])
-
- print "Generating output files ..."
- html_print_out(options, contributors, jira_category_list, misc_commits,
- diff_list)
-
- print "Done!"
-
-
-def get_diff_stat(options):
- tmpf = tempfile.NamedTemporaryFile()
- diff_tmpf = tempfile.NamedTemporaryFile()
-
- additional_tags = parse_additional_tags(options.additional_tags)
- for t in additional_tags:
- tag_path = build_tag_path(options, t)
- os.system("svn diff -r %s:HEAD %s > %s" % (get_tag_start(tag_path),
- tag_path, diff_tmpf.name))
-
- if not options.at_only:
- if options.start_rev == options.end_rev:
- os.system("svn diff -c %s %s >> %s" %
- (options.start_rev, options.svn_path, diff_tmpf.name))
- else:
- os.system("svn diff -r %s:%s %s >> %s" %
- (options.start_rev, options.end_rev, options.svn_path,
- diff_tmpf.name))
-
- os.system("diffstat -w %d %s > %s" % (MAX_WIDTH, diff_tmpf.name, tmpf.name))
-
- diff_list = []
- for line in tmpf:
- diff_list.append(line.strip())
-
- tmpf.close()
- diff_tmpf.close()
-
- return diff_list
-
-
-def html_print_out(options, contributors, jira_category_list, misc_commits,
- diff_list):
- view_path = options.svn_path.replace("/svn/", "/view/")
- view_path = view_path.replace("https", "http")
- view_path = view_path.replace("origsvn", "svn")
-
- feature_release = "This release includes new features. For a list of " \
- "new features that have been included with this release, please " \
- "see the CHANGES file inside the source package. Since this is " \
- "new major release, users are encouraged to do extended testing " \
- "before upgrading to this version in a production environment."
-
- bugfix_release = "This release includes only bug fixes. The changes " \
- "included were made only to address problems that have been " \
- "identified in this release series. Users should be able to " \
- "safely upgrade to this version if this release series is already " \
- "in use. Users considering upgrading from a previous release series " \
- "are strongly encouraged to review the UPGRADE.txt document " \
- "as well as the CHANGES document for information about upgrading " \
- "to this release series."
-
- security_release = "This release has been made to address one or more " \
- "security vulnerabilities that have been identified. A security " \
- "advisory document has been published for each vulnerability " \
- "that includes additional information. Users of versions of " \
- "Asterisk that are affected are strongly encouraged to review " \
- "the advisories and determine what action they should take to " \
- "protect their systems from these issues."
-
- fn_base = "%s-%s-summary" % (options.project, options.version)
- f = open("%s.html" % fn_base, "w")
-
- f.write("<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\""
- " http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\n")
- f.write("<html xmlns=\"http://www.w3.org/1999/xhtml\">\n")
- f.write("<head><meta http-equiv=\"Content-Type\" content=\"text/html; "
- "charset=iso-8859-1\" />"
- "<title>Release Summary - %s-%s</title>"
- "</head>\n" % (options.project, options.version))
- f.write("<body>\n")
-
- f.write("<h1 align=\"center\"><a name=\"top\">Release Summary</a></h1>\n")
- back_to_top = "<center><a href=\"#top\">[Back to Top]</a></center><br/>"
-
- f.write("<h3 align=\"center\">%s-%s</h3>\n" %
- (options.project, options.version))
- f.write("<h3 align=\"center\">Date: %s</h3>\n" %
- str(datetime.date.today()))
- f.write("<h3 align=\"center\"><asteriskteam at digium.com></h3>\n")
-
- f.write("<hr/>\n")
-
- f.write("<h2 align=\"center\">Table of Contents</h2>\n")
-
- f.write("<ol>\n"
- " <li><a href=\"#summary\">Summary</a></li>\n"
- " <li><a href=\"#contributors\">Contributors</a></li>\n");
- if len(jira_category_list) > 0:
- f.write(" <li><a href=\"#issues\">Closed Issues</a></li>\n")
- if len(misc_commits) > 0:
- f.write(" <li><a href=\"#commits\">Other Changes</a></li>\n")
- f.write(" <li><a href=\"#diffstat\">Diffstat</a></li>\n"
- "</ol>\n")
-
- f.write("<hr/>\n")
-
- f.write("<a name=\"summary\"><h2 align=\"center\">Summary</h2></a>\n")
- f.write(back_to_top)
- if options.release_type == RELEASE_TYPE_FEATURE:
- f.write("<p>%s</p>\n" % feature_release)
- elif options.release_type == RELEASE_TYPE_BUGFIX:
- f.write("<p>%s</p>\n" % bugfix_release)
- elif options.release_type == RELEASE_TYPE_SECURITY:
- f.write("<p>%s</p>\n" % security_release)
- f.write("<p>Security Advisories: ")
- advs = options.advisories.split(",")
- for i in range(0, len(advs)):
- adv = advs[i].strip()
- f.write("<a href=\"%s%s.html\">%s</a>" % (ADVISORY_URL, adv, adv))
- if i < len(advs) - 1:
- f.write(", ")
- f.write("</p>\n")
-
- f.write("<p>The data in this summary reflects changes that have been made "
- "since the previous release, %s-%s.</p>\n" % (options.project,
- options.prev_version))
-
- f.write("<hr/>\n")
-
- #contrib,tester,reporters table
- f.write("<a name=\"contributors\">"
- "<h2 align=\"center\">Contributors</h2></a>\n")
- f.write(back_to_top)
- f.write("<p>%s</p>\n" %
- "This table lists the people who have submitted code, "
- "those that have tested patches, as well as those that reported issues "
- "on the issue tracker that were resolved in this release. "
- "For coders, the number is how many of their patches (of any size) "
- "were committed into this release. For testers, the number is the number "
- "of times their name was listed as assisting with testing a patch. Finally, "
- "for reporters, the number is the number of issues that they reported that "
- "were closed by commits that went into this release.")
- f.write("<table width=\"100%\" border=\"0\">\n")
- f.write("<tr>\n")
- f.write("<td width=\"33%\"><h3>Coders</h3></td>\n")
- f.write("<td width=\"33%\"><h3>Testers</h3></td>\n")
- f.write("<td width=\"33%\"><h3>Reporters</h3></td>\n")
- f.write("</tr>\n")
- f.write("<tr valign=\"top\">\n")
- f.write("<td>\n")
- for item in contributors.coder_sorted:
- f.write(str(item[0]) + " " + unicode(item[1]).encode('utf-8') + "<br/>\n")
- f.write("</td>\n")
- f.write("<td>\n")
- for item in contributors.tester_sorted:
- f.write(str(item[0]) + " " + unicode(item[1]).encode('utf-8') + "<br/>\n")
- f.write("</td>\n")
- f.write("<td>\n")
- for item in contributors.reporter_sorted:
- f.write(str(item[0]) + " " + unicode(item[1]).encode('utf-8') + "<br/>\n")
- f.write("</td>\n")
- f.write("</tr>\n")
- f.write("</table>\n")
-
- f.write("<hr/>\n")
-
- #issues table
- if len(jira_category_list) > 0:
- f.write("<a name=\"issues\">"
- "<h2 align=\"center\">Closed Issues</h2></a>\n")
- f.write(back_to_top)
-
- f.write("<p>%s</p>\n" %
- "This is a list of all issues from the issue tracker that were closed "
- "by changes that went into this release.")
- for category, issues in jira_category_list:
- f.write("<h3>Category: %s</h3><br/>\n" % category)
- for issue in issues:
- try:
- f.write("<a href=\"%s%s\">%s</a>: %s<br/>\n" %
- (JIRA_URL, issue.issue_id, issue.issue_id,
- issue.jira_issue.fields.summary))
- except:
- print "Error in issue %s" % issue.issue_id
- continue
- f.write("Revision: <a href=\"%s?view=revision&revision=%d\">"
- "%d</a><br/>\n" %
- (view_path, issue.rev, issue.rev))
- f.write("Reporter: %s<br/>\n" % unicode(issue.reporter_name).encode('utf-8'))
- if len(issue.testers) > 0:
- f.write("Testers: %s<br/>\n" % unicode(string.join(issue.testers,
- ", ")).encode('utf-8'))
- f.write("Coders: %s<br/>\n" % unicode(string.join(issue.coders, ", ")).encode('utf-8'))
- f.write("<br/>\n")
-
- f.write("<hr/>\n")
-
- # Misc Commit List
- if len(misc_commits) > 0:
- f.write("<a name=\"commits\">"
- "<h2 align=\"center\">Commits Not Associated with an Issue"
- "</h2></a>\n")
- f.write(back_to_top)
- f.write("<p>%s</p>\n" %
- "This is a list of all changes that went into this release that did not "
- "directly close an issue from the issue tracker. The commits may have "
- "been marked as being related to an issue. If that is the case, the "
- "issue numbers are listed here, as well.")
- f.write("<table width=\"100%\" border=\"1\">\n")
- f.write("<tr>"
- "<td><b>Revision</b></td>"
- "<td><b>Author</b></td>"
- "<td><b>Summary</b></td>"
- "<td><b>Issues Referenced</b></td>"
- "</tr>")
- for commit in misc_commits:
- try:
- f.write("<tr><td><a href=\"%s?view=revision&revision=%d\">%d</a></td>"
- "<td>%s</td><td>%s</td>\n" %
- (view_path, commit.rev, commit.rev, commit.author,
- commit.summary))
- except:
- print "Error in rev %d" % commit.rev
- continue
- f.write("<td>")
- for i in range(0, len(commit.issues)):
- f.write("<a href=\"%s%s\">%s</a>" %
- (JIRA_URL, commit.issues[i].issue_id,
- commit.issues[i].issue_id))
- if i < len(commit.issues) - 1:
- f.write(", ")
- f.write("</td></tr>")
- f.write("</table>\n")
-
- f.write("<hr/>\n")
-
- #diffstat table
- f.write("<a name=\"diffstat\">"
- "<h2 align=\"center\">Diffstat Results</h2></a>\n")
- f.write(back_to_top)
- f.write("<p>%s</p>\n" % \
- "This is a summary of the changes to the source code that went into "
- "this release that was generated using the diffstat utility.")
- f.write("<pre>\n")
- for item in diff_list:
- f.write(item + "\n")
- f.write("</pre><br/>\n")
-
- f.write("<hr/>\n")
-
- f.write("</body>\n")
- f.write("</html>\n")
-
- f.close()
-
- os.system("links2 -dump -width %d %s.html > %s.txt" %
- (MAX_WIDTH, fn_base, fn_base))
-
-
-if __name__ == "__main__":
- main(sys.argv)
-
diff --git a/release_summary.py b/release_summary.py
new file mode 100755
index 0000000..5048bfd
--- /dev/null
+++ b/release_summary.py
@@ -0,0 +1,704 @@
+#!/usr/bin/env python
+"""Generate a release summary document based on data from commits.
+
+Copyright (C) 2009 - 2015, Digium, Inc.
+Matt Jordan <mjordan at digium.com>
+David Vossel (Batman) <dvossel at digium.com>
+Russell Bryant <russell at digium.com>
+"""
+
+import sys
+import os
+import datetime
+import tempfile
+from optparse import OptionParser
+from progressbar import ProgressBar
+
+from digium_jira import get_jira_client
+from digium_jira_user import AsteriskUser
+from digium_git import DigiumGitRepo
+
+# Maximum column width for diffstat and plain text output
+MAX_WIDTH = 80
+
+# URL prefix for JIRA issues
+JIRA_URL = "https://issues.asterisk.org/jira/browse/"
+
+# URL prefix for Fisheye log/code views
+FISHEYE_URL = "https://code.asterisk.org/code/changelog/"
+
+# URL prefix for security advisories
+ADVISORY_URL = "http://downloads.asterisk.org/pub/security/"
+
+# URL for Gerrit/Git repos
+GERRIT = "https://gerrit.asterisk.org"
+
+# Default repo location
+DEFAULT_REPO_ROOT = "/tmp"
+
+class Contributors(object):
+ """Tracking object for contributors"""
+
+ def __init__(self):
+ """Constructor"""
+ self.reporter_dic = {}
+ self.coder_dic = {}
+ self.tester_dic = {}
+ self.reporter_sorted = []
+ self.coder_sorted = []
+ self.tester_sorted = []
+
+ def sort_lists(self):
+ """Populate and sort the various contributor lists"""
+
+ for key in self.reporter_dic.iterkeys():
+ self.reporter_sorted.append(self.reporter_dic[key])
+ self.reporter_sorted.sort(key=lambda x:
+ (sys.maxint - x[0], x[1]))
+
+ for key in self.tester_dic.iterkeys():
+ self.tester_sorted.append(self.tester_dic[key])
+ self.tester_sorted.sort(key=lambda x:
+ (sys.maxint - x[0], x[1]))
+
+ for key in self.coder_dic.iterkeys():
+ self.coder_sorted.append(self.coder_dic[key])
+ self.coder_sorted.sort(key=lambda x:
+ (sys.maxint - x[0], x[1]))
+
+ def _add_user(self, user_dic, user):
+ """Add a user to a specific dictionary
+
+ Keyword Arguments:
+ user_dic - the dictionary of users to add to
+ user - the AsteriskUser object to add
+ """
+ if user.username in user_dic:
+ count, user = user_dic[user.username]
+ user_dic[user.username] = (count + 1, user)
+ else:
+ user_dic[user.username] = (1, user)
+
+ def add_coder(self, coder):
+ """Add a coder
+
+ Keyword Arguments:
+ coder - the AsteriskUser object to add
+ """
+ self._add_user(self.coder_dic, coder)
+
+ def add_tester(self, tester):
+ """Add a tester
+
+ Keyword Arguments:
+ tester - the AsteriskUser to add
+ """
+ self._add_user(self.tester_dic, tester)
+
+ def add_reporter(self, reporter):
+ """Add a reporter
+
+ Keyword Arguments:
+ reporter - the AsteriskUser to add
+ """
+ self._add_user(self.reporter_dic, reporter)
+
+
+class ReleaseSummaryOptions(object):
+ """Configuration object for ReleaseSummary instances"""
+
+ RELEASE_TYPE_FEATURE = 1
+ RELEASE_TYPE_BUGFIX = 2
+ RELEASE_TYPE_SECURITY = 3
+
+ @classmethod
+ def get_release_type(cls, release_type):
+ """Get the release type from a string representation
+
+ Keyword Arguments:
+ release_type - The type of release
+
+ Returns:
+ An integer value that represents the type of release
+ """
+ if release_type == "bugfix":
+ return ReleaseSummaryOptions.RELEASE_TYPE_BUGFIX
+ elif release_type == "feature":
+ return ReleaseSummaryOptions.RELEASE_TYPE_FEATURE
+ elif release_type == "security":
+ return ReleaseSummaryOptions.RELEASE_TYPE_SECURITY
+ else:
+ raise Exception("Unknown release type: {0}".format(
+ release_type))
+
+ def __init__(self):
+ """Constructor
+
+ This sets most initial values to None. Users of the object
+ should populate it as needed.
+ """
+ self.project = None
+ self.version = None
+ self.previous_version = None
+ self.start_tag = None
+ self.end_tag = None
+ self.release_type = ReleaseSummaryOptions.RELEASE_TYPE_BUGFIX
+ self.advisories = []
+ self.local_root = DEFAULT_REPO_ROOT
+
+
+class ReleaseSummary(object):
+ """A release summary document"""
+
+ FEATURE_RELEASE = "This is the first release of a major new version of " \
+ "Asterisk. For a list of new features that have been included with " \
+ "this release, please see the CHANGES file inside the source " \
+ "package. Since this is new major release, users are encouraged to " \
+ "do extended testing before upgrading to this version in a " \
+ "production environment."
+
+ BUGFIX_RELEASE = "This release is a point release of an existing major " \
+ "version. The changes included were made to address problems that " \
+ "have been identified in this release series, or are minor, " \
+ "backwards compatible new features or improvements. Users should be " \
+ "able to safely upgrade to this version if this release series is " \
+ "already in use. Users considering upgrading from a previous " \
+ "version are strongly encouraged to review the UPGRADE.txt " \
+ "document as well as the CHANGES document for information about " \
+ "upgrading to this release series."
+
+ SECURITY_RELEASE = "This release has been made to address one or more " \
+ "security vulnerabilities that have been identified. A security " \
+ "advisory document has been published for each vulnerability " \
+ "that includes additional information. Users of versions of " \
+ "Asterisk that are affected are strongly encouraged to review " \
+ "the advisories and determine what action they should take to " \
+ "protect their systems from these issues."
+
+
+ def __init__(self, options, jira=None, debug=False):
+ """Constructor
+
+ Keyword Arguments:
+ options - An instance of ReleaseSummaryOptions.
+ jira - The JIRA client to use.
+ debug - If true, display progress as the summary is built
+ """
+
+ self.jira = jira
+ self.debug = debug
+ self.options = options
+
+ if not self.jira:
+ self.jira = get_jira_client()
+
+ self.open_issues = {} # All open JIRA issues in this release
+ self.closed_issues = {} # All closed JIRA issues in this release
+ self.raw_issues = {} # All JIRA issues in this release
+ self.jira_commits = {} # Commits associated with each JIRA issue
+ self.misc_commits = [] # Commits not associated with an issue
+ self.contributors = Contributors()
+
+ path = os.path.join(self.options.local_root, self.options.project)
+ gerrit_repo = '{0}/{1}'.format(GERRIT, self.options.project)
+ self.repo = DigiumGitRepo(path, gerrit_repo, show_progress=self.debug)
+
+ # Infer the branch from the start tag
+ branch = self.options.start_tag[:self.options.start_tag.rindex('.')]
+ repo.checkout_remote_branch(branch)
+
+ self.raw_log_messages = self.repo.get_commits_by_tags(
+ options.start_tag,
+ options.end_tag)
+
+ self._process_log_messages()
+
+ def get_jira_user(self, user_obj):
+ """Build an AsteriskUser from JIRA
+
+ Parsed commit logs will generate AsteriskUser objects from the text in
+ the commit message. In order to standardize that information as much
+ as possible, we ask JIRA for the user and try to build a standard set
+ of user names.
+
+ Keyword Arguments:
+ user_obj - The Commit mesage AsteriskUser object
+
+ Returns:
+ An AsteriskUser object. If the user could not be found in JIRA, this
+ will be the original object.
+ """
+ jira_users = self.jira.search_users(user_obj.username)
+ if len(jira_users) == 0 and user_obj.email:
+ jira_users = self.jira.search_users(user_obj.email)
+ if len(jira_users) == 0:
+ # JIRA couldn't give us anything better, return what we had
+ return user_obj
+
+ user_obj = AsteriskUser(jira_users[0].raw.get('name'))
+ user_obj.full_name = jira_users[0].raw.get('displayName')
+ user_obj.email = jira_users[0].raw.get('emailAddress')
+ return user_obj
+
+ def _process_log_messages(self):
+ """Process the raw log messages into JIRA issues and commits
+
+ This will populate the various dictionaries with the JIRA issues fixed
+ by this release, as well as types of contributors, how many commits
+ each contributor made, etc.
+
+ All of this forms the raw data for the release summary.
+ """
+
+ if self.debug:
+ print "Process log messages..."
+ pbar = ProgressBar()
+ pbar.maxval = len(self.raw_log_messages)
+ pbar.start()
+
+ for i, log_message in enumerate(self.raw_log_messages):
+
+ if log_message.get_commit_summary().startswith("Merge"):
+ # Ignore merge commits
+ continue
+
+ # Reporters is the aggregate of what is reportee, and
+ # what JIRA reports back to us
+ reporters = []
+
+ # Calculate participants
+ for coder in log_message.get_coders():
+ self.contributors.add_coder(
+ self.get_jira_user(coder))
+ for tester in log_message.get_testers():
+ self.contributors.add_tester(
+ self.get_jira_user(tester))
+ reporter = log_message.get_reporter()
+ if reporter:
+ reporters.append(self.get_jira_user(reporter))
+
+ # Gather JIRA issues
+ issues = log_message.get_issues()
+ if len(issues) == 0:
+ self.misc_commits.append(log_message)
+ continue
+
+ # For each issue, organize and categorize based on
+ # issue status, followed by issue type
+ for raw_issue in issues:
+ issue_id = raw_issue['id']
+ issue = self.raw_issues.get(issue_id)
+ if not issue:
+ try:
+ issue = self.jira.issue(issue_id)
+ except Exception:
+ print "Could not get issue {0}, treating as misc " \
+ "commit".format(issue_id)
+ self.misc_commits.append(log_message)
+ continue
+ if not issue:
+ continue
+
+ self.jira_commits[issue_id] = []
+ self.raw_issues[issue_id] = issue
+
+ issue_dict = None
+ status = str(issue.fields.status).lower()
+ if status == 'closed' or status == 'complete':
+ issue_dict = self.closed_issues
+ else:
+ issue_dict = self.open_issues
+
+ issuetype = str(issue.fields.issuetype)
+ if issuetype not in issue_dict:
+ issue_dict[issuetype] = []
+ issue_dict[issuetype].append(issue)
+
+ # The issue reporter may not be in the commit message.
+ # If not, use what is in the JIRA issue and append
+ # to our reporters.
+ issue_reporter = AsteriskUser(issue.fields.reporter.name)
+ issue_reporter.full_name = issue.fields.reporter.displayName
+ issue_reporter.email = issue.fields.reporter.emailAddress
+ if issue_reporter not in reporters:
+ reporters.append(issue_reporter)
+ self.jira_commits[issue_id].append(log_message)
+
+ for reporter in reporters:
+ self.contributors.add_reporter(reporter)
+
+ if self.debug:
+ pbar.update(i + 1)
+
+ self.contributors.sort_lists()
+ if self.debug:
+ pbar.finish()
+ return
+
+ def get_diff_stat(self):
+ """Build a diff stat from the start/end tags
+
+ Returns:
+ A list of diff statistics
+ """
+
+ diff_list = []
+ tmpf = tempfile.NamedTemporaryFile()
+ diff_tmpf = tempfile.NamedTemporaryFile()
+
+ git = self.repo.repo.git
+ diff = git.diff('{0}..{1}'.format(
+ self.options.start_tag,
+ self.options.end_tag))
+
+ diff_tmpf.write(unicode(diff).encode('utf-8'))
+
+ os.system("diffstat -w {0} {1} > {2}".format(
+ MAX_WIDTH, diff_tmpf.name, tmpf.name))
+
+ for line in tmpf:
+ diff_list.append(line.strip())
+
+ tmpf.close()
+ diff_tmpf.close()
+
+ return diff_list
+
+ def write_jira_issue_to_html(self, f, issue):
+ """Write a single JIRA issue out, along with its commits
+
+ This will write out a single JIRA issue, then look up that issue's
+ commits and write them out as well.
+
+ Keyword Arguments:
+ f - The file object to write to
+ issue - The JIRA issue to write out
+ """
+ fisheye_url = '{0}{1}?cs='.format(FISHEYE_URL, self.options.project)
+
+ f.write("<a href=\"{0}{1}\">{2}</a>: {3}<br/>\n".format(
+ JIRA_URL, issue.key, issue.key,
+ issue.fields.summary))
+ f.write("Reported by: {0}".format(
+ unicode(issue.fields.reporter.displayName).encode('utf-8')))
+ f.write("<ul>\n")
+ for log_message in self.jira_commits[issue.key]:
+ f.write("<li>\n")
+ if log_message.raw:
+ f.write("<a href=\"{0}{1}\">[{2}]</a> ".format(
+ fisheye_url,
+ log_message.raw.hexsha,
+ log_message.raw.hexsha[:10]))
+ coders = ','.join([coder.full_name or coder.username for coder in
+ log_message.get_coders()])
+ f.write("{0} -- ".format(coders))
+ f.write("{0}\n".format(log_message.get_commit_summary()))
+ f.write("</li>\n")
+ f.write("</ul>\n")
+
+ def write_jira_issues_to_html(self, f, issue_type, issues):
+ """Write a series of JIRA issues out
+
+ This will organize the issues by component, then write out all of the
+ issues for those components.
+
+ Keyword Arguments:
+ f - The file object to write to
+ issue_type - The type of issue
+ issues - A list of issues
+ """
+ components = {}
+ # Build up a list of components
+ for issue in issues:
+ for component in issue.fields.components:
+ if component.name not in components:
+ components[component.name] = []
+ components[component.name].append(issue)
+
+ f.write("<h3>{0}</h3>\n".format(issue_type))
+ for key in sorted(components):
+ f.write("<h4>Category: {0}</h4>\n".format(key))
+ for issue in components[key]:
+ self.write_jira_issue_to_html(f, issue)
+ f.write("<br />\n")
+
+ def to_html(self, out_file=None):
+ """Write out the HTML release summary
+
+ Keyword Arguments:
+ out_file - optional, the file to write out. If not specified,
+ defaults to [project]-[version].html in the current
+ directory
+ """
+ if out_file:
+ fn_base = out_file
+ else:
+ fn_base = "{0}-{1}-summary".format(self.options.project,
+ self.options.version)
+
+ with open("{0}.html".format(fn_base), "w") as f:
+
+ f.write("<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\""
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\n")
+ f.write("<html xmlns=\"http://www.w3.org/1999/xhtml\">\n")
+ f.write("<head><meta http-equiv=\"Content-Type\" content=\"text/html; "
+ "charset=iso-8859-1\" />"
+ "<title>Release Summary - {0}-{1}</title></head>\n".format(
+ self.options.project, self.options.version))
+ f.write("<body>\n")
+ f.write("<h1 align=\"center\"><a name=\"top\">Release Summary</a></h1>\n")
+
+ back_to_top = "<center><a href=\"#top\">[Back to Top]</a></center><br/>"
+
+ f.write("<h3 align=\"center\">{0}-{1}</h3>\n".format(
+ self.options.project, self.options.version))
+ f.write("<h3 align=\"center\">Date: {0}</h3>\n".format(
+ str(datetime.date.today())))
+ f.write("<h3 align=\"center\"><asteriskteam at digium.com></h3>\n")
+
+ f.write("<hr/>\n")
+
+ f.write("<h2 align=\"center\">Table of Contents</h2>\n")
+ f.write("<ol>\n"
+ " <li><a href=\"#summary\">Summary</a></li>\n"
+ " <li><a href=\"#contributors\">Contributors</a></li>\n")
+ if len(self.closed_issues) > 0:
+ f.write(" <li><a href=\"#closed_issues\">Closed Issues</a></li>\n")
+ if len(self.open_issues) > 0:
+ f.write(" <li><a href=\"#open_issues\">Referenced Issues</a></li>\n")
+ if len(self.misc_commits) > 0:
+ f.write(" <li><a href=\"#commits\">Other Changes</a></li>\n")
+ f.write(" <li><a href=\"#diffstat\">Diffstat</a></li>\n"
+ "</ol>\n")
+
+ f.write("<hr/>\n")
+
+ f.write("<a name=\"summary\"><h2 align=\"center\">Summary</h2></a>\n")
+ f.write(back_to_top)
+ if self.options.release_type == ReleaseSummaryOptions.RELEASE_TYPE_FEATURE:
+ f.write("<p>%s</p>\n" % ReleaseSummary.FEATURE_RELEASE)
+ elif self.options.release_type == ReleaseSummaryOptions.RELEASE_TYPE_BUGFIX:
+ f.write("<p>%s</p>\n" % ReleaseSummary.BUGFIX_RELEASE)
+ elif self.options.release_type == ReleaseSummaryOptions.RELEASE_TYPE_SECURITY:
+ f.write("<p>%s</p>\n" % ReleaseSummary.SECURITY_RELEASE)
+ if len(self.options.advisories):
+ f.write("<p>Security Advisories: ")
+ for i, adv in enumerate(self.options.advisories):
+ adv = adv.strip()
+ f.write("<a href=\"%s%s.html\">%s</a>" % (ADVISORY_URL, adv, adv))
+ if i < len(self.options.advisories) - 1:
+ f.write(", ")
+ f.write("</p>\n")
+
+ f.write("<p>The data in this summary reflects changes that have been made "
+ "since the previous release, {0}-{1}.</p>\n".format(
+ self.options.project, self.options.previous_version))
+
+ f.write("<hr/>\n")
+
+ f.write("<a name=\"contributors\">"
+ "<h2 align=\"center\">Contributors</h2></a>\n")
+ f.write(back_to_top)
+ f.write("<p>%s</p>\n" %
+ "This table lists the people who have submitted code, "
+ "those that have tested patches, as well as those that "
+ "reported issues on the issue tracker that were resolved "
+ "in this release. For coders, the number is how many of "
+ "their patches (of any size) were committed into this "
+ "release. For testers, the number is the number of times "
+ "their name was listed as assisting with testing a patch. "
+ "Finally, for reporters, the number is the number of issues "
+ "that they reported that were affected by commits that went "
+ "into this release.")
+ f.write("<table width=\"100%\" border=\"0\">\n")
+ f.write("<tr>\n")
+ f.write("<td width=\"33%\"><h3>Coders</h3></td>\n")
+ f.write("<td width=\"33%\"><h3>Testers</h3></td>\n")
+ f.write("<td width=\"33%\"><h3>Reporters</h3></td>\n")
+ f.write("</tr>\n")
+ f.write("<tr valign=\"top\">\n")
+ f.write("<td>\n")
+ for commits, user in self.contributors.coder_sorted:
+ f.write("{0} {1}<br/>\n".format(commits, str(user)))
+ f.write("</td>\n")
+ f.write("<td>\n")
+ for commits, user in self.contributors.tester_sorted:
+ f.write("{0} {1}<br/>\n".format(commits, str(user)))
+ f.write("</td>\n")
+ f.write("<td>\n")
+ for commits, user in self.contributors.reporter_sorted:
+ f.write("{0} {1}<br/>\n".format(commits, str(user)))
+ f.write("</td>\n")
+ f.write("</tr>\n")
+ f.write("</table>\n")
+
+ f.write("<hr/>\n")
+
+ if len(self.closed_issues) > 0:
+ f.write("<a name=\"closed_issues\">"
+ "<h2 align=\"center\">Closed Issues</h2></a>\n")
+ f.write(back_to_top)
+
+ f.write("<p>{0}</p>\n".format(
+ "This is a list of all issues from the issue tracker that "
+ "were closed by changes that went into this release."))
+
+ for category, issues in self.closed_issues.items():
+ self.write_jira_issues_to_html(f, category, issues)
+
+ f.write("<hr />\n")
+
+ if len(self.open_issues) > 0:
+ f.write("<a name=\"open_issues\">"
+ "<h2 align=\"center\">Open Issues</h2></a>\n")
+ f.write(back_to_top)
+ f.write("<p>{0}</p>\n".format(
+ "This is a list of all open issues from the issue tracker "
+ "that were referenced by changes that went into this "
+ "release."))
+
+ for category, issues in self.open_issues.items():
+ self.write_jira_issues_to_html(f, category, issues)
+
+ f.write("<hr />\n")
+
+ if len(self.misc_commits) > 0:
+ f.write("<a name=\"commits\">"
+ "<h2 align=\"center\">Commits Not Associated with an Issue</h2></a>\n")
+ f.write(back_to_top)
+ f.write("<p>%s</p>\n" %
+ "This is a list of all changes that went into this "
+ "release that did not directly close an issue from "
+ "the issue tracker.")
+ f.write("<table width=\"100%\" border=\"1\">\n")
+ f.write("<tr>"
+ "<th>Revision</td>"
+ "<th>Author</th>"
+ "<th>Summary</th>"
+ "</tr>")
+
+ fisheye_url = '{0}{1}?cs='.format(FISHEYE_URL,
+ self.options.project)
+ for log_message in self.misc_commits:
+ coders = [coder.full_name or coder.username for coder in
+ log_message.get_coders()]
+ coders = ','.join(coders)
+ f.write("<tr>")
+ if log_message.raw:
+ f.write("<td><a href=\"{0}{1}\">{2}</a></td>"
+ "<td>{3}</td><td>{4}</td>".format(
+ fisheye_url,
+ log_message.raw.hexsha,
+ log_message.raw.hexsha[:10],
+ coders,
+ log_message.get_commit_summary()))
+ else:
+ f.write("<td/><td>{0}</td><td>{1}</td>".format(
+ coders, log_message.get_commit_summary()))
+ f.write("</tr>\n")
+ f.write("</table>\n")
+
+ f.write("<hr/>\n")
+
+ f.write("<a name=\"diffstat\">"
+ "<h2 align=\"center\">Diffstat Results</h2></a>\n")
+ f.write(back_to_top)
+ f.write("<p>%s</p>\n" % \
+ "This is a summary of the changes to the source code that went into "
+ "this release that was generated using the diffstat utility.")
+ f.write("<pre>\n")
+ diff_list = self.get_diff_stat()
+ for item in diff_list:
+ f.write(item + "\n")
+ f.write("</pre><br/>\n")
+
+ f.write("<hr/>\n")
+
+ f.write("</body>\n")
+ f.write("</html>\n")
+
+ # And... done! Write out the txt file too.
+ os.system("links2 -dump -width %d %s.html > %s.txt" %
+ (MAX_WIDTH, fn_base, fn_base))
+ return
+
+
+def main(argv=None):
+ """Main entry point
+
+ Keyword Arguments:
+ argv - Command line parameters
+ """
+ if argv is None:
+ argv = sys.argv
+
+ parser = OptionParser()
+
+ parser.add_option("-l", "--local-root", action="store", type="string",
+ dest="local_root", default="/tmp",
+ help="Root location on disk to store Git repositories")
+ parser.add_option("-s", "--start-tag", action="store", type="string",
+ dest="start_tag", default="", help="Tag to start from")
+ parser.add_option("-e", "--end-tag", action="store", type="string",
+ dest="end_tag", default="", help="Tag to end on")
+ parser.add_option("-p", "--project", action="store", type="string",
+ dest="project", default="asterisk",
+ help="Name of the project (such as Asterisk)")
+ parser.add_option("-v", "--version", action="store", type="string",
+ dest="version", default="",
+ help="Version ID.")
+ parser.add_option("-P", "--prev-version", action="store", type="string",
+ dest="prev_version", default="",
+ help="Previous version ID.")
+ parser.add_option("-t", "--release-type", action="store",
+ type="string", dest="release_type_str", default="bugfix",
+ help="Type of release: \"feature\", \"bugfix\", or \"security\"")
+ parser.add_option("-a", "--advisories", action="store", type="string",
+ dest="advisories", default="",
+ help="A comma separated list of security advisories addressed " \
+ "in this release")
+
+ (options, args) = parser.parse_args(argv)
+
+ if len(options.start_tag) == 0:
+ parser.error("A start tag is required.")
+
+ if len(options.end_tag) == 0:
+ parser.error("An end tag is required.")
+
+ if len(options.version) == 0:
+ parser.error("A version ID is required.")
+
+ if len(options.prev_version) == 0:
+ parser.error("A previous version ID is required.")
+
+ if options.release_type_str == "security" and len(options.advisories) == 0:
+ parser.error("You must provide at least one security advisory for "
+ "a security release.")
+
+ print "Generating release summary for {0}-{1} ...".format(options.project,
+ options.version)
+
+ jira = get_jira_client()
+
+ release_options = ReleaseSummaryOptions()
+ release_options.release_type = ReleaseSummaryOptions.get_release_type(
+ options.release_type_str)
+ release_options.project = options.project
+ release_options.version = options.version
+ release_options.previous_version = options.prev_version
+ release_options.advisories = options.advisories.split(',')
+ release_options.local_root = options.local_root
+ release_options.start_tag = options.start_tag
+ release_options.end_tag = options.end_tag
+
+ summary = ReleaseSummary(release_options, jira=jira, debug=True)
+ summary.to_html()
+
+ print "Done!"
+ return
+
+
+if __name__ == "__main__":
+ main(sys.argv)
+
--
To view, visit https://gerrit.asterisk.org/179
To unsubscribe, visit https://gerrit.asterisk.org/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I97275322fe5b8067e2c7317b6fbbfab532e6cd48
Gerrit-PatchSet: 3
Gerrit-Project: repotools
Gerrit-Branch: master
Gerrit-Owner: Matt Jordan <mjordan at digium.com>
More information about the asterisk-code-review
mailing list