[asterisk-dev] Change in testsuite[master]: Memory Debugging Improvements
Matt Jordan (Code Review)
asteriskteam at digium.com
Thu Apr 2 20:33:22 CDT 2015
Matt Jordan has submitted this change and it was merged.
Change subject: Memory Debugging Improvements
......................................................................
Memory Debugging Improvements
* Enable XML output from valgrind.
* Display and save a summary of valgrind errors and leaks.
* Enable use of contrib/valgrind/suppressions.txt if it exists
to suppress expected leaks or system library errors. Added
entry to .gitignore for this file.
* Supply a sample suppressions file that ignores some reachable
memory.
* Create stdout_print for printing messages to the terminal and
including in the failure message.
* Switch some failure notifications to use stdout_print() so the
messages are included in asterisk-test-suite-report.xml.
* Create function for archiving a list of files from source folder
to destination folder. Switch all archive functions to use this.
Valgrind Summaries require the lxml module. If this module is not
found summaries will not be produced, but valgrind XML output will
still be available in the Asterisk logs directory.
Change-Id: I21634673508a01eea1f489c751d3cf75ea55cf06
---
M .gitignore
M README.txt
A contrib/valgrind/summary-lines.xsl
A contrib/valgrind/suppressions-sample.txt
M lib/python/asterisk/asterisk.py
M runtests.py
6 files changed, 192 insertions(+), 47 deletions(-)
Approvals:
Matt Jordan: Looks good to me, approved; Verified
Ashley Sanders: Looks good to me, but someone else must approve
diff --git a/.gitignore b/.gitignore
index cf96d9e..b7f92b6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
*.pyc
asterisk-test-suite-report.xml
/astroot
+/contrib/valgrind/suppressions.txt
/logs
diff --git a/README.txt b/README.txt
index 30027f2..f6c5b80 100644
--- a/README.txt
+++ b/README.txt
@@ -127,6 +127,7 @@
$ make update
$ make install
- python-twisted
+ - python-lxml
- pjsua
- Download and build pjproject 1.x from source
- http://www.pjsip.org/download.htm
diff --git a/contrib/valgrind/summary-lines.xsl b/contrib/valgrind/summary-lines.xsl
new file mode 100644
index 0000000..c06faaf
--- /dev/null
+++ b/contrib/valgrind/summary-lines.xsl
@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="utf-8"?>
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+<!--
+ The XML output of this stylesheet is used internally for producing
+ text based summary. The output is processed by runtests.py,
+ TestRun._process_valgrind.
+-->
+
+<xsl:template match="/valgrindoutput">
+ <xml>
+ <xsl:if test="error | suppcounts/pair">
+ <line>Valgrind <xsl:value-of select="tool" /> results<xsl:if test="usercomment"> - <xsl:value-of select="usercomment" /></xsl:if></line>
+ <line />
+ </xsl:if>
+ <xsl:if test="error/what">
+ <line>================= Errors ==================</line>
+ <xsl:for-each select="error[what]">
+ <cols>
+ <xsl:attribute name="col1"><xsl:value-of select="kind"/></xsl:attribute>
+ <xsl:attribute name="col2"><xsl:value-of select="what"/></xsl:attribute>
+ </cols>
+ </xsl:for-each>
+ <line />
+ </xsl:if>
+ <xsl:if test="error/xwhat/leakedbytes">
+ <line>============== Leaked Memory ==============</line>
+ <xsl:if test="error[kind='Leak_DefinitelyLost']/xwhat/leakedbytes">
+ <cols col1="Definitely Lost">
+ <xsl:attribute name="col2">
+ <xsl:value-of select="sum(error[kind='Leak_DefinitelyLost']/xwhat/leakedbytes | error[kind='Leak_IndirectlyLost']/xwhat/leakedbytes)" /> bytes
+ </xsl:attribute>
+ </cols>
+ </xsl:if>
+ <xsl:if test="error[kind='Leak_PossiblyLost']/xwhat/leakedbytes">
+ <cols col1="Possible Lost">
+ <xsl:attribute name="col2">
+ <xsl:value-of select="sum(error[kind='Leak_PossiblyLost']/xwhat/leakedbytes)" /> bytes
+ </xsl:attribute>
+ </cols>
+ </xsl:if>
+ <xsl:if test="error[kind='Leak_StillReachable']/xwhat/leakedbytes">
+ <cols col1="Reachable Memory">
+ <xsl:attribute name="col2">
+ <xsl:value-of select="sum(error[kind='Leak_StillReachable']/xwhat/leakedbytes)" /> bytes
+ </xsl:attribute>
+ </cols>
+ </xsl:if>
+ <line />
+ </xsl:if>
+ <xsl:if test="suppcounts/pair">
+ <line>============== Suppressions ===============</line>
+ <xsl:for-each select="suppcounts/pair">
+ <cols>
+ <xsl:attribute name="col1"><xsl:value-of select="count" /></xsl:attribute>
+ <xsl:attribute name="col2"><xsl:value-of select="name" /></xsl:attribute>
+ </cols>
+ </xsl:for-each>
+ <line/>
+ </xsl:if>
+ </xml>
+</xsl:template>
+
+</xsl:stylesheet>
diff --git a/contrib/valgrind/suppressions-sample.txt b/contrib/valgrind/suppressions-sample.txt
new file mode 100644
index 0000000..1ee57a7
--- /dev/null
+++ b/contrib/valgrind/suppressions-sample.txt
@@ -0,0 +1,38 @@
+#
+# This is a sample valgrind suppressions file for Asterisk.
+#
+# To use this suppression file with the testsuite, copy it to:
+# contrib/valgrind/suppressions.txt
+#
+# This is meant for use when valgrind parameters include:
+# --show-leak-kinds=all
+#
+# Extra valgrind options are read from:
+# ~/.valgrindrc, $VALGRIND_OPTS, ./.valgrindrc
+#
+{
+ <ast_threadstorage_get>
+ # The threadstorage for the main thread is not cleaned by exit().
+ Memcheck:Leak
+ match-leak-kinds: reachable
+ fun:calloc
+ ...
+ fun:ast_threadstorage_get
+}
+{
+ <ast_ssl_init>
+ # libasteriskssl doesn't cleanup the init items.
+ Memcheck:Leak
+ match-leak-kinds: reachable
+ ...
+ fun:ast_ssl_init
+}
+{
+ <load_dynamic_module>
+ # dlclose is not run on modules at shutdown.
+ # This does not suppress allocations from module_load.
+ Memcheck:Leak
+ match-leak-kinds: reachable
+ ...
+ fun:load_dynamic_module
+}
diff --git a/lib/python/asterisk/asterisk.py b/lib/python/asterisk/asterisk.py
index 5dcb784..53342ce 100755
--- a/lib/python/asterisk/asterisk.py
+++ b/lib/python/asterisk/asterisk.py
@@ -349,19 +349,29 @@
self.install_configs(os.getcwd() + "/configs", deps)
self._setup_configs()
- cmd = [
- self.ast_binary,
- "-f", "-g", "-q", "-m", "-n",
- "-C", "%s" % os.path.join(self.astetcdir, "asterisk.conf")
- ]
+ cmd_prefix = []
if os.getenv("VALGRIND_ENABLE") == "true":
valgrind_path = test_suite_utils.which('valgrind')
if valgrind_path:
- cmd = [valgrind_path] + cmd
+ cmd_prefix = [
+ valgrind_path,
+ '--xml=yes',
+ '--xml-file=%s' % self.get_path("astlogdir", 'valgrind.xml'),
+ '--xml-user-comment=%s (%s)' % (
+ os.environ['TESTSUITE_ACTIVE_TEST'], self.host)]
+ suppression_file = 'contrib/valgrind/suppressions.txt'
+ if os.path.exists(suppression_file):
+ cmd_prefix.append('--suppressions=%s' % suppression_file)
else:
LOGGER.error('Valgrind not found')
+ cmd = cmd_prefix + [
+ self.ast_binary,
+ "-f", "-g", "-q", "-m", "-n",
+ "-C", "%s" % os.path.join(self.astetcdir, "asterisk.conf")
+ ]
+
# Make the start/stop deferreds - this method will return
# the start deferred, and pass the stop deferred to the AsteriskProtocol
# object. The stop deferred will be raised when the Asterisk process
diff --git a/runtests.py b/runtests.py
index f0fac91..f745266 100755
--- a/runtests.py
+++ b/runtests.py
@@ -20,6 +20,12 @@
import random
import select
+try:
+ import lxml.etree as ET
+except:
+ # Ensure ET is defined
+ ET = None
+
# Re-open stdout so it's line buffered.
# This allows timely processing of piped output.
newfno = os.dup(sys.stdout.fileno())
@@ -59,10 +65,15 @@
assert self.test_name.startswith('tests/')
self.test_relpath = self.test_name[6:]
+ def stdout_print(self, msg):
+ self.stdout += msg + "\n"
+ print msg
+
def run(self):
self.passed = False
self.did_run = True
start_time = time.time()
+ os.environ['TESTSUITE_ACTIVE_TEST'] = self.test_name
cmd = [
"%s/run-test" % self.test_name,
]
@@ -71,9 +82,7 @@
cmd = ["./lib/python/asterisk/test_runner.py",
"%s" % self.test_name]
if os.path.exists(cmd[0]) and os.access(cmd[0], os.X_OK):
- msg = "Running %s ..." % cmd
- print msg
- self.stdout += msg + "\n"
+ self.stdout_print("Running %s ..." % cmd)
cmd.append(str(self.ast_version).rstrip())
p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
@@ -91,8 +100,7 @@
l = p.stdout.readline()
if not l:
break
- print l
- self.stdout += l
+ self.stdout_print(l)
except IOError:
pass
p.wait()
@@ -100,22 +108,19 @@
# Sanitize p.returncode so it's always a boolean.
did_pass = (p.returncode == 0)
if did_pass and not self.test_config.expect_pass:
- msg = "Test passed but was expected to fail."
- print msg
- self.stdout += msg + "\n"
+ self.stdout_print("Test passed but was expected to fail.")
if not did_pass and not self.test_config.expect_pass:
print "Test failed as expected."
-
- self.__parse_run_output(self.stdout)
self.passed = (did_pass == self.test_config.expect_pass)
core_dumps = self._check_for_core()
if (len(core_dumps)):
- print "Core dumps detected; failing test"
+ self.stdout_print("Core dumps detected; failing test")
self.passed = False
self._archive_core_dumps(core_dumps)
+ self._process_valgrind()
self._process_ref_debug()
if not self.passed:
@@ -130,6 +135,7 @@
except:
print "Unable to clean up directory for test %s (non-fatal)" % self.test_name
+ self.__parse_run_output(self.stdout)
print 'Test %s %s\n' % (cmd, 'timedout' if timedout else 'passed' if self.passed else 'failed')
else:
@@ -189,6 +195,42 @@
'run_%d' % run_num)
return (run_num, run_dir, archive_dir)
+ def _process_valgrind(self):
+ (run_num, run_dir, archive_dir) = self._find_run_dirs()
+ if (run_num == 0):
+ return
+ if not ET:
+ return
+
+ i = 1
+ while os.path.isdir(os.path.join(run_dir, 'ast%d/var/log/asterisk' % i)):
+ ast_dir = "%s/ast%d/var/log/asterisk" % (run_dir, i)
+ valgrind_xml = os.path.join(ast_dir, 'valgrind.xml')
+ valgrind_txt = os.path.join(ast_dir, 'valgrind-summary.txt')
+
+ # All instances either use valgrind or not.
+ if not os.path.exists(valgrind_xml):
+ return
+
+ dom = ET.parse(valgrind_xml)
+ xslt = ET.parse('contrib/valgrind/summary-lines.xsl')
+ transform = ET.XSLT(xslt)
+ newdom = transform(dom)
+ lines = []
+ for node in newdom.getroot():
+ if node.tag == 'line':
+ lines.append((node.text or '').strip())
+ elif node.tag == 'cols':
+ lines.append("%s: %s" % (
+ node.attrib['col1'].strip().rjust(20),
+ node.attrib['col2'].strip()))
+
+ self.stdout_print("\n".join(lines))
+ with open(valgrind_txt, 'a') as txtfile:
+ txtfile.write("\n".join(lines))
+ txtfile.close()
+ i += 1
+
def _process_ref_debug(self):
(run_num, run_dir, archive_dir) = self._find_run_dirs()
if (run_num == 0):
@@ -216,7 +258,7 @@
stdout=dest_file,
stderr=subprocess.STDOUT)
except Exception, e:
- print "Exception occurred while processing REF_DEBUG"
+ self.stdout_print("Exception occurred while processing REF_DEBUG")
finally:
dest_file.close()
if res != 0:
@@ -228,20 +270,26 @@
os.path.join(dest_dir, "refs.txt"))
hardlink_or_copy(refs_in,
os.path.join(dest_dir, "refs"))
- print "REF_DEBUG identified leaks, mark test as failure"
+ self.stdout_print("REF_DEBUG identified leaks, mark test as failure")
self.passed = False
i += 1
+
+ def _archive_files(self, src_dir, dest_dir, *filenames):
+ for filename in filenames:
+ try:
+ srcfile = os.path.join(src_dir, filename)
+ if os.path.exists(srcfile):
+ hardlink_or_copy(srcfile, os.path.join(dest_dir, filename))
+ except Exception, e:
+ print "Exception occurred while archiving file '%s' to %s: %s" % (
+ srcfile, dest_dir, e
+ )
def _archive_logs(self):
(run_num, run_dir, archive_dir) = self._find_run_dirs()
self._archive_ast_logs(run_num, run_dir, archive_dir)
self._archive_pcap_dump(run_dir, archive_dir)
- if os.path.exists(os.path.join(run_dir, 'messages.txt')):
- hardlink_or_copy(os.path.join(run_dir, 'messages.txt'),
- os.path.join(archive_dir, 'messages.txt'))
- if os.path.exists(os.path.join(run_dir, 'full.txt')):
- hardlink_or_copy(os.path.join(run_dir, 'full.txt'),
- os.path.join(archive_dir, 'full.txt'))
+ self._archive_files(run_dir, archive_dir, 'messages.txt', 'full.txt')
def _archive_ast_logs(self, run_num, run_dir, archive_dir):
"""Archive the Asterisk logs"""
@@ -250,31 +298,13 @@
ast_dir = "%s/ast%d/var/log/asterisk" % (run_dir, i)
dest_dir = os.path.join(archive_dir,
'ast%d/var/log/asterisk' % i)
- try:
- hardlink_or_copy(ast_dir + "/messages.txt",
- dest_dir + "/messages.txt")
- hardlink_or_copy(ast_dir + "/full.txt",
- dest_dir + "/full.txt")
- if os.path.exists(ast_dir + "/mmlog"):
- hardlink_or_copy(ast_dir + "/mmlog",
- dest_dir + "/mmlog")
- except Exception, e:
- print "Exception occurred while archiving logs from %s to %s: %s" % (
- ast_dir, dest_dir, e
- )
+ self._archive_files(ast_dir, dest_dir,
+ 'messages.txt', 'full.txt', 'mmlog',
+ 'valgrind.xml', 'valgrind-summary.txt')
i += 1
def _archive_pcap_dump(self, run_dir, archive_dir):
- filename = "dumpfile.pcap"
- src = os.path.join(run_dir, filename)
- dst = os.path.join(archive_dir, filename)
- if os.path.exists(src):
- try:
- hardlink_or_copy(src, dst)
- except Exception, e:
- print "Exeception occured while archiving pcap file from %s to %s: %s" % (
- src, dst, e
- )
+ self._archive_files(run_dir, archive_dir, 'dumpfile.pcap')
def __check_can_run(self, ast_version):
"""Check tags and dependencies in the test config."""
@@ -577,6 +607,8 @@
return 0
if options.valgrind:
+ if not ET:
+ print "python lxml module not loaded, text summaries from valgrind will not be produced.\n"
os.environ["VALGRIND_ENABLE"] = "true"
print "Running tests for Asterisk %s ...\n" % str(ast_version)
--
To view, visit https://gerrit.asterisk.org/15
To unsubscribe, visit https://gerrit.asterisk.org/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I21634673508a01eea1f489c751d3cf75ea55cf06
Gerrit-PatchSet: 4
Gerrit-Project: testsuite
Gerrit-Branch: master
Gerrit-Owner: Corey Farrell <git at cfware.com>
Gerrit-Reviewer: Ashley Sanders <asanders at digium.com>
Gerrit-Reviewer: Corey Farrell <git at cfware.com>
Gerrit-Reviewer: Matt Jordan <mjordan at digium.com>
More information about the asterisk-dev
mailing list