<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0" />
<base href="https://wiki.asterisk.org/wiki" />
<title>Message Title</title>
<style type="text/css">@media only screen and (max-device-width: 480px) {.mobile-only {
width: auto !important;
height: auto !important;
overflow: visible !important;
line-height: normal !important;
font-size: inherit !important;
mso-hide: all;
}
.desktop-only {
display: none !important;
}
/* iPhone 3GS fix for unwanted 20px right margin */
body { min-width: 100% !important; padding: 0; margin: 0; }
#center-content-table { max-width: none; !important; }
#header-pattern-container { padding: 10px 10px 10px 10px !important; line-height: 20px !important; }
#header-avatar-image-container { padding-right: 8px !important; }
#email-content-container { padding: 0 !important; }
.mobile-expand { border-radius: 0 !important; border-left: 0 !important; border-right: 0 !important; padding-left: 26px !important;}
.mobile-resize-text { font-size: 16px !important; line-height: 22px !important; }
#page-title-pattern-header { font-size: 20px !important; line-height: 28px !important; }
#page-title-pattern-icon-image-container-cell { padding-top: 7px !important; }
#inline-user-pattern { display: block !important; }
#inline-user-pattern-avatar { padding-top: 3px !important; }
.contextual-area-pattern { border-bottom: 1px solid #ccc !important; padding: 15px 10px 0 10px !important;}
.users-involved-pattern-column-table { width: 100% !important; }
.users-involved-pattern-avatar-table-cell { padding: 3px 5px 5px 0 !important; }
.users-involved-pattern-column-container { padding-right: 0 !important; }
.contextual-excerpt-pattern, #users-involved-pattern { border: 0 !important; }
/** Aui Typography upsized for mobile **/
#content-excerpt-pattern-container, #contextual-excerpt-pattern-text-container { font-size: 16px !important; line-height: 22px !important; }
#content-excerpt-pattern-container h1, #contextual-excerpt-pattern-text-container h1 { font-size: 24px !important; line-height: 28px !important; }
#content-excerpt-pattern-container h2, #contextual-excerpt-pattern-text-container h2 { font-size: 20px !important; line-height: 28px !important; }
#content-excerpt-pattern-container h3, #contextual-excerpt-pattern-text-container h3 { font-size: 18px !important; line-height: 24px !important; }
#content-excerpt-pattern-container h4, #contextual-excerpt-pattern-text-container h4 { font-size: 16px !important; line-height: 22px !important; }
#content-excerpt-pattern-container h5, #contextual-excerpt-pattern-text-container h5 { font-size: 14px !important; line-height: 20px !important; }
#content-excerpt-pattern-container h6, #contextual-excerpt-pattern-text-container h6 { font-size: 14px !important; line-height: 20px !important; }
.user-mention { line-height: 18px !important; }
/** Aui Typography end **/
/* Show appropriate footer logo on mobile, display links vertically */
#footer-pattern { padding: 15px 10px !important; }
#footer-pattern-logo-desktop-container { padding: 0 !important; }
#footer-pattern-logo-desktop { width: 0 !important; height: 0 !important; }
#footer-pattern-logo-mobile {
padding-top: 10px !important;
width: 30px !important;
height: 27px !important;
display: inline !important;
}
#footer-pattern-text {
display: block !important;
}
#footer-pattern-links-container { line-height: 0 !important;}
.footer-pattern-links.mobile-resize-text,
.footer-pattern-links.mobile-resize-text,
#footer-pattern-text.mobile-resize-text,
#footer-pattern-links-container.no-footer-links {
font-size: 14px !important;
line-height: 20px !important;
}
.footer-link { display: block !important; }
#footer-pattern-links-container table { display: inline-block !important; float: none !important; }
#footer-pattern-links-container, #footer-pattern-text { text-align: center !important; }
#footer-pattern-links { padding-bottom: 5px !important; }
/** Team Calendar overrides, these should be removed when notifications are updated in Team Calendars. For now CSS
overrides are being used because the structure of the content can't change without rereleasing the plugin */
.mail-calendar-container .day-header + table tr td:first-child {
vertical-align: top !important;
padding-top: 5px !important;
}}
@media (min-width: 900px) {#center-content-table { width: 900px; }}
@media all {#outlook a {padding:0;} /* Force Outlook to provide a "view in browser" menu link. */
/* Prevent Webkit and Windows Mobile platforms from changing default font sizes.*/
body{-webkit-text-size-adjust:100%; -ms-text-size-adjust:100%;}
.ExternalClass {width:100%;} /* Force Hotmail to display emails at full width */
#background-table {margin:0; padding:0; width:100% !important; }
/* Needed to override highlighting on date and time links in iOS */
.grey a {color: #707070; text-decoration: none; }/* These styles are appended to the head element of a notification in order to prevent Apple Mail and similar
clients from underlining the due dates with a blue hyperlink */
/* a lozenge outside an inline task should always be #333, lozenges inside an inline task should be
colored according to their upcoming due dates, a completed task date lozenge or deleted task date
lozenge should always be #707070 */
.date-time-lozenge a {color: #333333; text-decoration: none; }
.inline-task-text-container .date-time-lozenge.date-upcoming a {color: #DF6F00; text-decoration: none; }
.inline-task-text-container .date-time-lozenge.date-past a {color: #D04437; text-decoration: none; }
.inline-task-text-container.content-deleted-color .date-time-lozenge a,
.inline-task-text-container.checked .date-time-lozenge a {
color: #707070; text-decoration: none;
}}
</style>
</head>
<body>
<table id="background-table" cellpadding="0" cellspacing="0" width="100%" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; color: #333; background-color: #f5f5f5">
<tbody>
<tr>
<td id="header-pattern-container" style="padding: 0px; border-collapse: collapse; padding: 10px 20px">
<table id="header-pattern" cellspacing="0" cellpadding="0" border="0" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; color: #333">
<tbody>
<tr>
<td id="header-avatar-image-container" valign="top" style="padding: 0px; border-collapse: collapse; vertical-align: top; width: 32px; padding-right: 9px"><a href="https://wiki.asterisk.org/wiki/display/~gjoseph?src=email" style="color: #3b73af; text-decoration: none"><img id="header-avatar-image" class="image_fix" src="cid:avatar_56b2a47747d2721d6dcf26979b4b605a" height="32" width="32" border="0" style="border-radius: 3px; vertical-align: top" /></a></td>
<td id="header-text-container" valign="middle" style="padding: 0px; border-collapse: collapse; vertical-align: middle; font-family: Arial, sans-serif; font-size: 14px; line-height: 20px; mso-line-height-rule: exactly; mso-text-raise: 1px">George Joseph <strong>edited</strong> a page</td>
</tr>
</tbody>
</table> </td>
</tr>
<!-- End Header pattern -->
<tr>
<td id="email-content-container" style="padding: 0px; border-collapse: collapse; padding: 0 20px">
<table id="email-content-table" cellspacing="0" cellpadding="0" border="0" width="100%" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; color: #333; border-spacing: 0; border-collapse: separate">
<tbody>
<tr>
<td class="email-content-rounded-top mobile-expand" style="padding: 0px; border-collapse: collapse; color: #fff; padding: 0 15px 0 16px; height: 15px; background-color: #fff; border-left: 1px solid #ccc; border-top: 1px solid #ccc; border-right: 1px solid #ccc; border-bottom: 0; border-top-right-radius: 5px; border-top-left-radius: 5px"> </td>
</tr>
<tr>
<td class="email-content-main mobile-expand" style="padding: 0px; border-collapse: collapse; border-left: 1px solid #ccc; border-right: 1px solid #ccc; border-top: 0; border-bottom: 0; padding: 0 15px 15px 16px; background-color: #fff">
<table id="page-title-pattern" cellspacing="0" cellpadding="0" border="0" width="100%" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; color: #333">
<tbody>
<tr>
<td id="page-title-pattern-icon-image-container" valign="top" style="padding: 0px; border-collapse: collapse; width: 16px; vertical-align: top">
<table cellspacing="0" cellpadding="0" border="0" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; color: #333">
<tbody>
<tr>
<td id="page-title-pattern-icon-image-container-cell" style="padding: 0px; border-collapse: collapse; width: 16px; padding: 9px 8px 0px 0px; mso-text-raise: 5px; mso-line-height-rule: exactly"><a href="https://wiki.asterisk.org/wiki/display/AST/Getting+a+Backtrace?src=email" title="page icon" style="vertical-align: top;; color: #3b73af; text-decoration: none"><img style="vertical-align: top; display: block;" src="cid:page-icon" alt="page icon" title="page icon" height="16" width="16" border="0" /></a></td>
</tr>
</tbody>
</table> </td>
<td style="vertical-align: top;; padding: 0px; border-collapse: collapse; padding-right: 5px; font-size: 20px; line-height: 30px; mso-line-height-rule: exactly" id="page-title-pattern-header-container"><span id="page-title-pattern-header" style="font-family: Arial, sans-serif; padding: 0; font-size: 20px; line-height: 30px; mso-text-raise: 2px; mso-line-height-rule: exactly; vertical-align: middle"><a href="https://wiki.asterisk.org/wiki/display/AST/Getting+a+Backtrace?src=email" title="Getting a Backtrace" style="color: #3b73af; text-decoration: none">Getting a Backtrace</a></span></td>
</tr>
</tbody>
</table> </td>
</tr>
<tr>
<td class="email-content-main mobile-expand" style="padding: 0px; border-collapse: collapse; border-left: 1px solid #ccc; border-right: 1px solid #ccc; border-top: 0; border-bottom: 0; padding: 0 15px 15px 16px; background-color: #fff">
<table class="content-excerpt-pattern" cellspacing="0" cellpadding="0" border="0" width="100%" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; color: #333; font-family: Arial, sans-serif; font-size: 14px; line-height: 20px; mso-line-height-rule: exactly; mso-text-raise: 1px">
<tbody>
<tr>
<td class="content-excerpt-pattern-container mobile-resize-text " style="padding: 0px; border-collapse: collapse; padding: 0 0 0 24px"> <p class="diff-context-placeholder" style="margin: 10px 0 0 0; margin-top: 0">...</p> <p class="diff-block-target diff-block-context" style="margin: 10px 0 0 0">Second, your copy of Asterisk must have been built without optimization or the backtrace will be (nearly) unusable. This can be done by selecting the 'DONT_OPTIMIZE' option in the Compiler Flags submenu in the 'make menuselect' tree before building Asterisk.</p>
<table class="diff-macro diff-html-removed diff-block-target diff-block-context" style="background-color: #f0f0f0;border: 1px solid #dddddd;margin: 10px 1px;padding: 0 2px 2px;width: 100%;background-color: #ffe7e7;border-color: #df9898;; border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; color: #333">
<thead>
<tr>
<th class="diff-macro-title" style="background-color: transparent; text-align: left; font-weight: normal;padding: 5px;"><span class="diff-html-removed" style="font-size: 100%; background-color: #ffe7e7; text-decoration: line-through;"><span class="icon macro-placeholder-icon" style="background-color: ;line-height: 20px;"><img src="https://wiki.asterisk.org/wiki/s/en_GB/5639/a252d7f5e75d7a8bf7047b4b2c92f71a56a8f048.48/_/images/icons/macrobrowser/dropdown/note.png" style="padding-right: 5px; vertical-align: text-bottom;" /> </span>Note</span></th>
</tr>
</thead>
<tbody>
<tr>
<td class="diff-macro-body" style="background-color: #fff;border: 1px solid #dddddd;padding: 10px;; padding: 0px; border-collapse: collapse">
<div class="table-wrap" style="margin: 10px 0 0 0; overflow-x: auto; margin-top: 0">
<table class="confluenceTable" style="border-collapse: collapse; border: 1px solid #ddd;; border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; color: #333; border-collapse: collapse; margin: 10px 0 0 0; overflow-x: auto; margin-top: 0">
<tbody>
<tr>
<th class="confluenceTh" style="background-color: transparent; text-align: left; font-weight: normal;; border: 1px solid #ddd; padding: 7px 10px; vertical-align: top; text-align: left; background-color: #f0f0f0; font-weight: bold"> <p style="margin: 10px 0 0 0; margin-top: 0; background-color: #f0f0f0; font-weight: bold"> <span class="diff-html-removed" style="font-size: 100%; background-color: #ffe7e7; text-decoration: line-through;">Using BETTER_BACKTRACES</span> </p> </th>
</tr>
<tr>
<td class="confluenceTd" style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align: top;; padding: 0px; border-collapse: collapse; border: 1px solid #ddd; padding: 7px 10px; vertical-align: top; text-align: left"> <p style="margin: 10px 0 0 0; margin-top: 0"> <span class="diff-html-removed" style="font-size: 100%; background-color: #ffe7e7; text-decoration: line-through;">Starting with Asterisk versions 1.4.40, 1.6.2.17, 1.8.3, and later release branches, the option BETTER_BACKTRACES (which uses libbfd) will provide better symbol information within the Asterisk binary and loaded modules when creating inline backtraces to track down problems. </span><strong><span class="diff-html-removed" style="font-size: 100%; background-color: #ffe7e7; text-decoration: line-through;">It is recommended that you enable both DONT_OPTIMIZE and BETTER_BACKTRACES. You do NOT need COMPILE_DOUBLE.</span></strong> </p> </td>
</tr>
</tbody>
</table>
</div> </td>
</tr>
</tbody>
</table>
<table class="diff-macro diff-html-removed diff-block-target diff-block-context" style="background-color: #f0f0f0;border: 1px solid #dddddd;margin: 10px 1px;padding: 0 2px 2px;width: 100%;background-color: #ffe7e7;border-color: #df9898;; border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; color: #333">
<thead>
<tr>
<th class="diff-macro-title" style="background-color: transparent; text-align: left; font-weight: normal;padding: 5px;"><span class="diff-html-removed" style="font-size: 100%; background-color: #ffe7e7; text-decoration: line-through;"><span class="icon macro-placeholder-icon" style="background-color: ;line-height: 20px;"><img src="https://wiki.asterisk.org/wiki/s/en_GB/5639/a252d7f5e75d7a8bf7047b4b2c92f71a56a8f048.48/_/images/icons/macrobrowser/dropdown/tip.png" style="padding-right: 5px; vertical-align: text-bottom;" /> </span>Tip</span></th>
</tr>
</thead>
<tbody>
<tr>
<td class="diff-macro-body" style="background-color: #fff;border: 1px solid #dddddd;padding: 10px;; padding: 0px; border-collapse: collapse"> <p style="margin: 10px 0 0 0; margin-top: 0"> <span class="diff-html-removed" style="font-size: 100%; background-color: #ffe7e7; text-decoration: line-through;">libbfd is included in the binutils-devel package on CentOS / RHEL, or the binutils-dev package on Debian / Ubuntu.</span> </p> </td>
</tr>
</tbody>
</table> <p class="diff-block-target diff-block-context" style="margin: 10px 0 0 0"> <span class="diff-html-added" id="added-diff-0" style="font-size: 100%; background-color: #ddfade;"> </span>Running a production server with DONT_OPTIMIZE is generally safe. You'll notice the binary files may be a bit larger, but in terms of Asterisk performance, impact should be negligible.</p> <p class="diff-block-target" style="margin: 10px 0 0 0"> <span class="diff-html-added" id="added-diff-1" style="font-size: 100%; background-color: #ddfade;">Third, your copy of Asterisk must have the debug symbols still included in the binaries. This is the default unless you, or the package maintainer if installing from packages, strips the binaries. Normally a package maintainer will strip the binaries and then provide separate "debug" packages that contain just the symbols. If so, make sure those debug packages are installed.</span> </p> <p class="diff-block-target diff-block-context" style="margin: 10px 0 0 0">After Asterisk crashes, a file named "core" will be dumped in the present working directory of the Linux shell from which Asterisk was started or in the location specified by the <code style="font-family: monospace">kernel.core_pattern</code> sysctl setting. In the event that there are multiple core files present, it is important to look at the file timestamps in order to determine which one you really intend to look at.</p> <p class="diff-context-placeholder" style="margin: 10px 0 0 0">...</p>
<div class="diff-block-target">
<h2 id="GettingaBacktrace-GettingInformationAfterACrash" style="margin: 10px 0 0 0; margin-top: 0; font-size: 20px; font-weight: normal; line-height: 30px; margin: 40px 0 0 0; margin-top: 0">Getting Information After A Crash</h2>
<p style="margin: 10px 0 0 0">Before you go further, you must have the GNU Debugger (gdb) installed on the machine that experienced the crash. Use your package manager to install it if it isn't already.</p>
<table class="diff-macro" style="background-color: #f0f0f0;border: 1px solid #dddddd;margin: 10px 1px;padding: 0 2px 2px;width: 100%;; border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; color: #333">
<thead>
<tr>
<th class="diff-macro-title" style="background-color: transparent; text-align: left; font-weight: normal;padding: 5px;"><span class="icon macro-placeholder-icon" style="background-color: ;line-height: 20px;"><img src="https://wiki.asterisk.org/wiki/s/en_GB/5639/a252d7f5e75d7a8bf7047b4b2c92f71a56a8f048.48/_/images/icons/macrobrowser/dropdown/note.png" style="padding-right: 5px; vertical-align: text-bottom;" /> </span>Note</th>
</tr>
</thead>
<tbody>
<tr>
<td class="diff-macro-body" style="background-color: #fff;border: 1px solid #dddddd;padding: 10px;; padding: 0px; border-collapse: collapse"> <p style="margin: 10px 0 0 0; margin-top: 0">Don't attach core files on the bug tracker as they are only useful on the machine they were generated on. We only need the output of ast_coredumper.</p> </td>
</tr>
</tbody>
</table>
<h3 id="GettingaBacktrace-ast_coredumper" style="margin: 10px 0 0 0; font-size: 16px; line-height: 25px; margin: 30px 0 0 0">ast_coredumper</h3>
<p style="margin: 10px 0 0 0">Asterisk versions 13.14.0, 14.3.0, and later release branches added a few tools to make debugging easier. One of these is <code style="font-family: monospace">ast_coredumper</code>. By default, it's installed in <code style="font-family: monospace">/var/lib/asterisk/scripts</code> and it takes in a core file and produces backtraces and lock dumps in a format for uploading to Jira. </p>
<table class="diff-macro" style="background-color: #f0f0f0;border: 1px solid #dddddd;margin: 10px 1px;padding: 0 2px 2px;width: 100%;; border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; color: #333">
<thead>
<tr>
<th class="diff-macro-title" style="background-color: transparent; text-align: left; font-weight: normal;padding: 5px;"><span class="icon macro-placeholder-icon" style="background-color: ;line-height: 20px;"><img src="https://wiki.asterisk.org/wiki/s/en_GB/5639/a252d7f5e75d7a8bf7047b4b2c92f71a56a8f048.48/_/plugins/servlet/confluence/placeholder/macro-icon?name=code" style="padding-right: 5px; vertical-align: text-bottom;" /> </span>Code Block</th>
</tr>
</thead>
<tbody>
<tr>
<td class="diff-macro-properties" style="background-color: #fafafa; padding: 0 0 0 5px; font-size: 12px; text-align: left;padding: 0; border: 1px solid #dddddd;; padding: 0px; border-collapse: collapse">
<table style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; color: #333">
<tbody>
<tr>
<td style="background-color: #fafafa; padding: 0 0 0 5px; font-size: 12px; text-align: left;; padding: 0px; border-collapse: collapse">title</td>
<td style="background-color: #fafafa; padding: 0 0 0 5px; font-size: 12px; text-align: left;; padding: 0px; border-collapse: collapse">/var/lib/asterisk/scripts/ast_coredumper --help</td>
</tr>
<tr>
<td style="background-color: #fafafa; padding: 0 0 0 5px; font-size: 12px; text-align: left;; padding: 0px; border-collapse: collapse">collapse</td>
<td style="background-color: #fafafa; padding: 0 0 0 5px; font-size: 12px; text-align: left;; padding: 0px; border-collapse: collapse">true</td>
</tr>
</tbody>
</table> </td>
</tr>
</tbody>
<tbody>
<tr>
<td class="diff-macro-body" style="background-color: #fff;border: 1px solid #dddddd;padding: 10px;; padding: 0px; border-collapse: collapse"> <pre style="margin: 10px 0 0 0; margin-top: 0">NAME
ast_coredumper - Dump and/or format asterisk coredump files
SYNOPSIS
ast_coredumper [ --help ] [ --running | --RUNNING ] [ --latest ]
[ --tarball-coredumps ] [ --delete-coredumps-after ]
[ --tarball-results ] [ --delete-results-after ]
[ --tarball-uniqueid="<uniqueid>" ]
[ --no-default-search ] [ --append-coredumps ]
[ <coredump> | <pattern> ... ]
DESCRIPTION
Extracts backtraces and lock tables from Asterisk coredump files.
For each coredump found, 4 new result files are created:
- <coredump>.brief.txt: The output of "thread apply all bt".
- <coredump>.thread1.txt: The output of "thread apply 1 bt full".
- <coredump>.full.txt: The output of "thread apply all bt full".
- <coredump>.locks.txt: If asterisk was compiled with
"DEBUG_THREADS", this file will contain a dump of the locks
table similar to doing a "core show locks" from the asterisk
CLI.
Optional features:
- The running asterisk process can be suspended and dumped.
- The coredumps can be merged into a tarball.
- The coredumps can be deleted after processing.
- The results files can be merged into a tarball.
- The results files can be deleted after processing.
Options:
--help
Print this help.
--running
Create a coredump from the running asterisk instance and
process it along with any other coredumps found (if any).
WARNING: This WILL interrupt call processing. You will be
asked to confirm.
--RUNNING
Same as --running but without the confirmation prompt.
DANGEROUS!!
--latest
Process only the latest coredump from those specified (based
on last-modified time). If a dump of the running process was
requested, it is always included in addition to the latest
from the existing coredumps.
--tarball-coredumps
Creates a gzipped tarball of all coredumps processed.
The tarball name will be:
/tmp/asterisk.<timestamp>.coredumps.tar.gz
--delete-coredumps-after
Deletes all processed coredumps regardless of whether
a tarball was created.
--tarball-results
Creates a gzipped tarball of all result files produced.
The tarball name will be:
/tmp/asterisk.<timestamp>.results.tar.gz
--delete-results-after
Deletes all processed results regardless of whether
a tarball was created. It probably doesn't make sense
to use this option unless you have also specified
--tarball-results.
--tarball-uniqueid="<uniqueid>"
Normally DATEFORMAT is used to make the tarballs unique
but you can use your own unique id in the tarball names
such as the Jira issue id.
--no-default-search
Ignore COREDUMPS from the config files and process only
coredumps listed on the command line (if any) and/or
the running asterisk instance (if requested).
--append-coredumps
Append any coredumps specified on the command line to the
config file specified ones instead of overriding them.
<coredump> | <pattern>
A list of coredumps or coredump search patterns. Unless
--append-coredumps was specified, these entries will override
those specified in the config files.
Any resulting file that isn't actually a coredump is silently
ignored. If your patterns contains spaces be sure to only
quote the portion of the pattern that DOESN'T contain wildcard
expressions. If you quote the whole pattern, it won't be
expanded.
If --no-default-search is specified and no files are specified
on the command line, then the only the running asterisk process
will be dumped (if requested). Otherwise if no files are
specified on the command line the value of COREDUMPS from
ast_debug_tools.conf will be used. Failing that, the following
patterns will be used:
/tmp/core[-._]asterisk!(*.txt)
/tmp/core[-._]$(hostname)!(*.txt)
NOTES
You must be root to use ast_coredumper.
The script relies on not only bash, but also recent GNU date and
gdb with python support. *BSD operating systems may require
installation of the 'coreutils' and 'devel/gdb' packagess and minor
tweaking of the ast_debug_tools.conf file.
Any files output will have ':' characters changed to '-'. This is
to facilitate uploading those files to Jira which doesn't like the
colons.
FILES
/etc/asterisk/ast_debug_tools.conf
~/ast_debug_tools.conf
./ast_debug_tools.conf
#
# This file is used by the Asterisk debug tools.
# Unlike other Asterisk config files, this one is
# "sourced" by bash and must adhere to bash semantics.
#
# A list of coredumps and/or coredump search patterns.
# Bash extended globs are enabled and any resulting files
# that aren't actually coredumps are silently ignored
# so you can be liberal with the globs.
#
# If your patterns contains spaces be sure to only quote
# the portion of the pattern that DOESN'T contain wildcard
# expressions. If you quote the whole pattern, it won't
# be expanded and the glob characters will be treated as
# literals.
#
# The exclusion of files ending ".txt" is just for
# demonstration purposes as non-coredumps will be ignored
# anyway.
COREDUMPS=(/tmp/core[-._]asterisk!(*.txt) /tmp/core[-._]$(hostname)!(*.txt))
# Date command for the "running" coredump and tarballs.
# DATEFORMAT will be executed to get the timestamp.
# Don't put quotes around the format string or they'll be
# treated as literal characters. Also be aware of colons
# in the output as you can't upload files with colons in
# the name to Jira.
#
# Unix timestamp
#DATEFORMAT='date +%s.%N'
#
# *BSD/MacOS doesn't support %N but after installing GNU
# coreutils...
#DATEFORMAT='gdate +%s.%N'
#
# Readable GMT
#DATEFORMAT='date -u +%FT%H-%M-%S%z'
#
# Readable Local time
DATEFORMAT='date +%FT%H-%M-%S%z'
</pre> </td>
</tr>
</tbody>
</table>
<p style="margin: 10px 0 0 0">As you can see, there are lots of options but if the core file is simply named <code style="font-family: monospace">core</code> in your current directory, running <code style="font-family: monospace">/var/lib/asterisk/scripts/ast_coredumper core</code> will usually be sufficient.</p>
<table class="diff-macro" style="background-color: #f0f0f0;border: 1px solid #dddddd;margin: 10px 1px;padding: 0 2px 2px;width: 100%;; border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; color: #333">
<thead>
<tr>
<th class="diff-macro-title" style="background-color: transparent; text-align: left; font-weight: normal;padding: 5px;"><span class="icon macro-placeholder-icon" style="background-color: ;line-height: 20px;"><img src="https://wiki.asterisk.org/wiki/s/en_GB/5639/a252d7f5e75d7a8bf7047b4b2c92f71a56a8f048.48/_/plugins/servlet/confluence/placeholder/macro-icon?name=code" style="padding-right: 5px; vertical-align: text-bottom;" /> </span>Code Block</th>
</tr>
</thead>
<tbody>
<tr>
<td class="diff-macro-body" style="background-color: #fff;border: 1px solid #dddddd;padding: 10px;; padding: 0px; border-collapse: collapse"> <pre style="margin: 10px 0 0 0; margin-top: 0">$ sudo /var/lib/asterisk/scripts/ast_coredumper core
Processing core
Creating core-thread1.txt
Creating core-brief.txt
Creating core-full.txt
Creating core-locks.txt
$ </pre> </td>
</tr>
</tbody>
</table>
<p style="margin: 10px 0 0 0">Unless you've compiled Asterisk with the <code style="font-family: monospace">DEBUG_THREADS</code> compiler flag (see below), the locks.txt file will be empty.</p>
<p style="margin: 10px 0 0 0">Many system administrators use the sysctl <code style="font-family: monospace">kernel.core_pattern</code> parameter to control where core files are dumped and what they are named.</p>
<table class="diff-macro" style="background-color: #f0f0f0;border: 1px solid #dddddd;margin: 10px 1px;padding: 0 2px 2px;width: 100%;; border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; color: #333">
<thead>
<tr>
<th class="diff-macro-title" style="background-color: transparent; text-align: left; font-weight: normal;padding: 5px;"><span class="icon macro-placeholder-icon" style="background-color: ;line-height: 20px;"><img src="https://wiki.asterisk.org/wiki/s/en_GB/5639/a252d7f5e75d7a8bf7047b4b2c92f71a56a8f048.48/_/plugins/servlet/confluence/placeholder/macro-icon?name=code" style="padding-right: 5px; vertical-align: text-bottom;" /> </span>Code Block</th>
</tr>
</thead>
<tbody>
<tr>
<td class="diff-macro-body" style="background-color: #fff;border: 1px solid #dddddd;padding: 10px;; padding: 0px; border-collapse: collapse"> <pre style="margin: 10px 0 0 0; margin-top: 0">$ sysctl -n kernel.core_pattern
/tmp/core-%e-%t
$ ls -al /tmp/core-asterisk*
-rw-------. 1 root root 173936640 Jun 16 07:44 /tmp/core-asterisk-1497620664.32259
$ sudo /var/lib/asterisk/scripts/ast_coredumper /tmp/core-asterisk-1497620664.32259
Processing /tmp/core-asterisk-1497620664.32259
Creating /tmp/core-asterisk-1497620664.32259-thread1.txt
Creating /tmp/core-asterisk-1497620664.32259-brief.txt
Creating /tmp/core-asterisk-1497620664.32259-full.txt
Creating /tmp/core-asterisk-1497620664.32259-locks.txt
</pre> </td>
</tr>
</tbody>
</table>
<p style="margin: 10px 0 0 0">You'll notice that the output file names include the full core file name.</p>
<p style="margin: 10px 0 0 0">If the <code style="font-family: monospace">/etc/asterisk/ast_debug_tools.conf</code> file contained a <code style="font-family: monospace">COREDUMPS</code> entry that would have matched the core file name in /tmp, then you don't even have to supply a path.</p>
<table class="diff-macro" style="background-color: #f0f0f0;border: 1px solid #dddddd;margin: 10px 1px;padding: 0 2px 2px;width: 100%;; border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; color: #333">
<thead>
<tr>
<th class="diff-macro-title" style="background-color: transparent; text-align: left; font-weight: normal;padding: 5px;"><span class="icon macro-placeholder-icon" style="background-color: ;line-height: 20px;"><img src="https://wiki.asterisk.org/wiki/s/en_GB/5639/a252d7f5e75d7a8bf7047b4b2c92f71a56a8f048.48/_/plugins/servlet/confluence/placeholder/macro-icon?name=code" style="padding-right: 5px; vertical-align: text-bottom;" /> </span>Code Block</th>
</tr>
</thead>
<tbody>
<tr>
<td class="diff-macro-body" style="background-color: #fff;border: 1px solid #dddddd;padding: 10px;; padding: 0px; border-collapse: collapse"> <pre style="margin: 10px 0 0 0; margin-top: 0">$ sudo /var/lib/asterisk/scripts/ast_coredumper
Processing /tmp/core-asterisk-1497620664.32259
Creating /tmp/core-asterisk-1497620664.32259-thread1.txt
Creating /tmp/core-asterisk-1497620664.32259-brief.txt
Creating /tmp/core-asterisk-1497620664.32259-full.txt
Creating /tmp/core-asterisk-1497620664.32259-locks.txt</pre> </td>
</tr>
</tbody>
</table>
<p style="margin: 10px 0 0 0">Finally, you might want to create a tarball of the text files and include the Jira issue id in the tarball name.</p>
<table class="diff-macro" style="background-color: #f0f0f0;border: 1px solid #dddddd;margin: 10px 1px;padding: 0 2px 2px;width: 100%;; border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; color: #333">
<thead>
<tr>
<th class="diff-macro-title" style="background-color: transparent; text-align: left; font-weight: normal;padding: 5px;"><span class="icon macro-placeholder-icon" style="background-color: ;line-height: 20px;"><img src="https://wiki.asterisk.org/wiki/s/en_GB/5639/a252d7f5e75d7a8bf7047b4b2c92f71a56a8f048.48/_/plugins/servlet/confluence/placeholder/macro-icon?name=code" style="padding-right: 5px; vertical-align: text-bottom;" /> </span>Code Block</th>
</tr>
</thead>
<tbody>
<tr>
<td class="diff-macro-body" style="background-color: #fff;border: 1px solid #dddddd;padding: 10px;; padding: 0px; border-collapse: collapse"> <pre style="margin: 10px 0 0 0; margin-top: 0">$ sudo /var/lib/asterisk/scripts/ast_coredumper --tarball-uniqueid=ASTERISK-99999 --tarball-results
Processing /tmp/core-asterisk-1497620664.32259
Creating /tmp/core-asterisk-1497620664.32259-thread1.txt
Creating /tmp/core-asterisk-1497620664.32259-brief.txt
Creating /tmp/core-asterisk-1497620664.32259-full.txt
Creating /tmp/core-asterisk-1497620664.32259-locks.txt
Creating /tmp/asterisk-ASTERISK-99999-results.tar.gz
/tmp/core-asterisk-1497620664.32259-brief.txt
/tmp/core-asterisk-1497620664.32259-full.txt
/tmp/core-asterisk-1497620664.32259-thread1.txt
/tmp/core-asterisk-1497620664.32259-locks.txt
</pre> </td>
</tr>
</tbody>
</table>
<p style="margin: 10px 0 0 0"> <span style="color: rgb(133,120,102);"> <br /> </span> </p>
<h2 id="GettingaBacktrace-GettingInformationForADeadlock" style="margin: 10px 0 0 0; font-size: 20px; font-weight: normal; line-height: 30px; margin: 40px 0 0 0"> <span style="color: rgb(133,120,102);">Getting Information For A Deadlock</span> </h2>
<p style="margin: 10px 0 0 0">Whenever collecting information about a deadlock it is useful to have additional information about the threads involved. We can generate this information by attaching to a running Asterisk process and gathering that information. Follow the steps below to collect debug that will be useful to Asterisk developers.</p>
<span class="diff-html-removed" style="font-size: 100%; background-color: #ffe7e7; text-decoration: line-through;">Enabling </span>
<p style="margin: 10px 0 0 0"> <span><span class="diff-html-removed" id="removed-diff-2" style="font-size: 100%; background-color: #ffe7e7; text-decoration: line-through;">In the </span><span class="diff-html-added" id="added-diff-2" style="font-size: 100%; background-color: #ddfade;">If you can easily reproduce the deadlock, in the </span>Compiler Flags menu of menuselect you should enable </span><strong>DEBUG_THREADS</strong><span>, <strong>MALLOC_DEBUG</strong>, </span><strong>DONT_OPTIMIZE</strong><span><span class="diff-html-removed" style="font-size: 100%; background-color: #ffe7e7; text-decoration: line-through;"> </span><span class="diff-html-removed" id="removed-diff-3" style="font-size: 100%; background-color: #ffe7e7; text-decoration: line-through;">and </span></span><strong><span class="diff-html-removed" style="font-size: 100%; background-color: #ffe7e7; text-decoration: line-through;">BETTER_BACKTRACES</span></strong><span>. Then, you need to recompile, re-install, and restart Asterisk before following the steps below.</span> </p>
<span class="diff-html-removed" style="font-size: 100%; background-color: #ffe7e7; text-decoration: line-through;"><span class="icon macro-placeholder-icon" style="background-color: ;line-height: 20px;"><img src="https://wiki.asterisk.org/wiki/s/en_GB/5639/a252d7f5e75d7a8bf7047b4b2c92f71a56a8f048.48/_/images/icons/macrobrowser/dropdown/warning.png" style="padding-right: 5px; vertical-align: text-bottom;" /> </span>Warning</span>
<p style="margin: 10px 0 0 0"> <span><span class="diff-html-added" id="added-diff-3" style="font-size: 100%; background-color: #ddfade;"> </span><strong><span class="diff-html-changed" id="changed-diff-0" style="background-color: #d6f0ff;">DEBUG_THREADS</span></strong><span class="diff-html-changed" style="background-color: #d6f0ff;"> </span><span class="diff-html-added" id="added-diff-4" style="font-size: 100%; background-color: #ddfade;">is very resource intensive and </span><span class="diff-html-changed" id="changed-diff-1" style="background-color: #d6f0ff;">can seriously impact performance </span></span> </p>
<span class="diff-html-removed" style="font-size: 100%; background-color: #ffe7e7; text-decoration: line-through;"> </span>
<span class="diff-html-removed" id="removed-diff-5" style="font-size: 100%; background-color: #ffe7e7; text-decoration: line-through;">and </span>
<p style="margin: 10px 0 0 0"> <span><span class="diff-html-added" id="added-diff-5" style="font-size: 100%; background-color: #ddfade;">so you </span><span class="diff-html-changed" id="changed-diff-2" style="background-color: #d6f0ff;">should only </span></span> </p>
<span class="diff-html-removed" style="font-size: 100%; background-color: #ffe7e7; text-decoration: line-through;"> </span>
<span class="diff-html-removed" id="removed-diff-6" style="font-size: 100%; background-color: #ffe7e7; text-decoration: line-through;">be enabled if you suspect a deadlock may be occurring.</span>
<strong><span class="diff-html-removed" style="font-size: 100%; background-color: #ffe7e7; text-decoration: line-through;">MALLOC_DEBUG can also impact performance but the impact is not very noticeable.</span></strong>
<p style="margin: 10px 0 0 0"> <span><span class="diff-html-added" id="added-diff-6" style="font-size: 100%; background-color: #ddfade;">use it if you can reproduce the issue quickly. Otherwise, </span><strong><span class="diff-html-added" style="font-size: 100%; background-color: #ddfade;">DONT_OPTIMIZE</span></strong><span class="diff-html-added" style="font-size: 100%; background-color: #ddfade;"> is the only option you really need. </span></span> </p>
<p style="margin: 10px 0 0 0">When you suspect asterisk is deadlocked, you can use ast_coredumper again to dump the currently running asterisk instance.</p>
<table class="diff-macro" style="background-color: #f0f0f0;border: 1px solid #dddddd;margin: 10px 1px;padding: 0 2px 2px;width: 100%;; border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; color: #333">
<thead>
<tr>
<th class="diff-macro-title" style="background-color: transparent; text-align: left; font-weight: normal;padding: 5px;"><span class="icon macro-placeholder-icon" style="background-color: ;line-height: 20px;"><img src="https://wiki.asterisk.org/wiki/s/en_GB/5639/a252d7f5e75d7a8bf7047b4b2c92f71a56a8f048.48/_/plugins/servlet/confluence/placeholder/macro-icon?name=code" style="padding-right: 5px; vertical-align: text-bottom;" /> </span>Code Block</th>
</tr>
</thead>
<tbody>
<tr>
<td class="diff-macro-body" style="background-color: #fff;border: 1px solid #dddddd;padding: 10px;; padding: 0px; border-collapse: collapse"> <pre style="margin: 10px 0 0 0; margin-top: 0">$ sudo /var/lib/asterisk/scripts/ast_coredumper --running
WARNING: Taking a core dump of the running asterisk instance will suspend call processing while the dump is saved. Do you wish to continue? (y/N) y
Dumping running asterisk process to /tmp/core-asterisk-running-2017-06-16T09-56-53-0600
Processing /tmp/core-asterisk-1497620664.32259
Creating /tmp/core-asterisk-1497620664.32259-thread1.txt
Creating /tmp/core-asterisk-1497620664.32259-brief.txt
Creating /tmp/core-asterisk-1497620664.32259-full.txt
Creating /tmp/core-asterisk-1497620664.32259-locks.txt
Processing /tmp/core-asterisk-running-2017-06-16T09-56-53-0600
Creating /tmp/core-asterisk-running-2017-06-16T09-56-53-0600-thread1.txt
Creating /tmp/core-asterisk-running-2017-06-16T09-56-53-0600-brief.txt
Creating /tmp/core-asterisk-running-2017-06-16T09-56-53-0600-full.txt
Creating /tmp/core-asterisk-running-2017-06-16T09-56-53-0600-locks.txt
</pre> </td>
</tr>
</tbody>
</table>
<p style="margin: 10px 0 0 0">By default, ast_coredumper also processes existing core files it detects. You can suppress that using the <code style="font-family: monospace">--no-default-search</code> option. You can also suppress the warning and prompt using the <code style="font-family: monospace">--RUNNING</code> option.</p>
<table class="diff-macro" style="background-color: #f0f0f0;border: 1px solid #dddddd;margin: 10px 1px;padding: 0 2px 2px;width: 100%;; border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; color: #333">
<thead>
<tr>
<th class="diff-macro-title" style="background-color: transparent; text-align: left; font-weight: normal;padding: 5px;"><span class="icon macro-placeholder-icon" style="background-color: ;line-height: 20px;"><img src="https://wiki.asterisk.org/wiki/s/en_GB/5639/a252d7f5e75d7a8bf7047b4b2c92f71a56a8f048.48/_/plugins/servlet/confluence/placeholder/macro-icon?name=code" style="padding-right: 5px; vertical-align: text-bottom;" /> </span>Code Block</th>
</tr>
</thead>
<tbody>
<tr>
<td class="diff-macro-body" style="background-color: #fff;border: 1px solid #dddddd;padding: 10px;; padding: 0px; border-collapse: collapse"> <pre style="margin: 10px 0 0 0; margin-top: 0">$ sudo /var/lib/asterisk/scripts/ast_coredumper --RUNNING --no-default-search
Dumping running asterisk process to /tmp/core-asterisk-running-2017-06-16T10-00-03-0600
Processing /tmp/core-asterisk-running-2017-06-16T10-00-03-0600
Creating /tmp/core-asterisk-running-2017-06-16T10-00-03-0600-thread1.txt
Creating /tmp/core-asterisk-running-2017-06-16T10-00-03-0600-brief.txt
Creating /tmp/core-asterisk-running-2017-06-16T10-00-03-0600-full.txt
Creating /tmp/core-asterisk-running-2017-06-16T10-00-03-0600-locks.txt
</pre> </td>
</tr>
</tbody>
</table>
<p style="margin: 10px 0 0 0">As above, you can also use the tarball options should you desire.</p>
<p style="margin: 10px 0 0 0">If Asterisk is truly deadlocked and you compiled with <code style="font-family: monospace">DEBUG_THREADS</code>, the locks.txt file should now contain a table of locks including who's waiting and who's holding.</p>
<p style="margin: 10px 0 0 0"> </p>
<p style="margin: 10px 0 0 0"> </p>
</div> </td>
</tr>
</tbody>
</table> </td>
</tr>
<tr>
<td class="email-content-main mobile-expand action-padding last-row-padding" style="padding: 0px; border-collapse: collapse; border-left: 1px solid #ccc; border-right: 1px solid #ccc; border-top: 0; border-bottom: 0; padding: 0 15px 15px 16px; background-color: #fff; padding-bottom: 10px; padding-bottom: 10px">
<table id="actions-pattern" cellspacing="0" cellpadding="0" border="0" width="100%" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; color: #333; font-family: Arial, sans-serif; font-size: 14px; line-height: 20px; mso-line-height-rule: exactly; mso-text-raise: 1px">
<tbody>
<tr>
<td id="actions-pattern-container" valign="middle" style="padding: 0px; border-collapse: collapse; padding: 15px 0 0 24px; vertical-align: middle">
<table align="left" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; color: #333">
<tbody>
<tr>
<td class="actions-pattern-action-icon-container" style="padding: 0px; border-collapse: collapse; font-family: Arial, sans-serif; font-size: 14px; line-height: 20px; mso-line-height-rule: exactly; mso-text-raise: 0px; vertical-align: middle"><a href="https://wiki.asterisk.org/wiki/display/AST/Getting+a+Backtrace?src=email" title="View page Icon" style="color: #3b73af; text-decoration: none"><img class="actions-pattern-action-icon-image" height="16" width="16" border="0" title="View page Icon" src="cid:com.atlassian.confluence.plugins.confluence-email-resources%3Aview-page-email-adg-footer-item%3Aicon" alt="View page Icon" style="vertical-align: middle" /></a></td>
<td class="actions-pattern-action-text-container" style="padding: 0px; border-collapse: collapse; font-family: Arial, sans-serif; font-size: 14px; line-height: 20px; mso-line-height-rule: exactly; mso-text-raise: 4px; padding-left: 5px; white-space: nowrap"><a href="https://wiki.asterisk.org/wiki/display/AST/Getting+a+Backtrace?src=email" title="View page" style="color: #3b73af; text-decoration: none">View page</a></td>
<td class="actions-pattern-action-bull" style="padding: 0px; border-collapse: collapse; font-family: Arial, sans-serif; font-size: 14px; line-height: 20px; mso-line-height-rule: exactly; mso-text-raise: 4px; color: #999; padding: 0 5px">•</td>
</tr>
</tbody>
</table>
<table align="left" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; color: #333">
<tbody>
<tr>
<td class="actions-pattern-action-icon-container" style="padding: 0px; border-collapse: collapse; font-family: Arial, sans-serif; font-size: 14px; line-height: 20px; mso-line-height-rule: exactly; mso-text-raise: 0px; vertical-align: middle"><a href="https://wiki.asterisk.org/wiki/display/AST/Getting+a+Backtrace?showComments=true&showCommentArea=true&src=email#addcomment" title="Add comment Icon" style="color: #3b73af; text-decoration: none"><img class="actions-pattern-action-icon-image" height="16" width="16" border="0" title="Add comment Icon" src="cid:com.atlassian.confluence.plugins.confluence-email-resources%3Aadd-comment-to-content-email-adg-footer-item%3Aicon" alt="Add comment Icon" style="vertical-align: middle" /></a></td>
<td class="actions-pattern-action-text-container" style="padding: 0px; border-collapse: collapse; font-family: Arial, sans-serif; font-size: 14px; line-height: 20px; mso-line-height-rule: exactly; mso-text-raise: 4px; padding-left: 5px; white-space: nowrap"><a href="https://wiki.asterisk.org/wiki/display/AST/Getting+a+Backtrace?showComments=true&showCommentArea=true&src=email#addcomment" title="Add comment" style="color: #3b73af; text-decoration: none">Add comment</a></td>
<td class="actions-pattern-action-bull" style="padding: 0px; border-collapse: collapse; font-family: Arial, sans-serif; font-size: 14px; line-height: 20px; mso-line-height-rule: exactly; mso-text-raise: 4px; color: #999; padding: 0 5px">•</td>
</tr>
</tbody>
</table>
<table style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; color: #333">
<tbody>
<tr>
<td class="actions-pattern-action-icon-container" style="padding: 0px; border-collapse: collapse; font-family: Arial, sans-serif; font-size: 14px; line-height: 20px; mso-line-height-rule: exactly; mso-text-raise: 0px; vertical-align: middle"><a href="https://wiki.asterisk.org/wiki/plugins/likes/like.action?contentId=36803756&src=email" title="Like Icon" style="color: #3b73af; text-decoration: none"><img class="actions-pattern-action-icon-image" height="16" width="16" border="0" title="Like Icon" src="cid:com.atlassian.confluence.plugins.confluence-like%3Aview-email-adg-content-item%3Aicon" alt="Like Icon" style="vertical-align: middle" /></a></td>
<td class="actions-pattern-action-text-container" style="padding: 0px; border-collapse: collapse; font-family: Arial, sans-serif; font-size: 14px; line-height: 20px; mso-line-height-rule: exactly; mso-text-raise: 4px; padding-left: 5px; white-space: nowrap"><a href="https://wiki.asterisk.org/wiki/plugins/likes/like.action?contentId=36803756&src=email" title="Like" style="color: #3b73af; text-decoration: none">Like</a></td>
</tr>
</tbody>
</table> </td>
</tr>
</tbody>
</table> </td>
</tr>
<tr>
<td class="email-content-rounded-bottom mobile-expand" style="padding: 0px; border-collapse: collapse; color: #fff; height: 5px; line-height: 5px; padding: 0 15px 0 16px; background-color: #fff; border-bottom-right-radius: 5px; border-bottom-left-radius: 5px; border-top: 0; border-left: 1px solid #ccc; border-bottom: 1px solid #ccc; border-right: 1px solid #ccc; mso-line-height-rule: exactly"> </td>
</tr>
</tbody>
</table> </td>
</tr>
<tr>
<td id="footer-pattern" style="padding: 0px; border-collapse: collapse; padding: 12px 20px">
<table id="footer-pattern-container" cellspacing="0" cellpadding="0" border="0" width="100%" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; color: #333">
<tbody>
<tr>
<td id="footer-pattern-links-container" width="100%" style="padding: 0px; border-collapse: collapse; color: #999; font-size: 12px; line-height: 18px; font-family: Arial, sans-serif; mso-line-height-rule: exactly; mso-text-raise: 2px">
<table align="left" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; color: #333; font-size: 12px; line-height: 18px; font-family: Arial, sans-serif; mso-line-height-rule: exactly; mso-text-raise: 2px">
<tbody>
<tr>
<td class="footer-pattern-links mobile-resize-text" style="padding: 0px; border-collapse: collapse"><a href="https://wiki.asterisk.org/wiki/users/removespacenotification.action?spaceKey=AST&src=email" title="" style="color: #3b73af; text-decoration: none">Stop watching space</a></td>
<td class="footer-pattern-links-bull" style="padding: 0px; border-collapse: collapse; padding: 0 5px; color: #999">•</td>
</tr>
</tbody>
</table>
<table style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; color: #333; font-size: 12px; line-height: 18px; font-family: Arial, sans-serif; mso-line-height-rule: exactly; mso-text-raise: 2px">
<tbody>
<tr>
<td class="footer-pattern-links mobile-resize-text" style="padding: 0px; border-collapse: collapse"><a href="https://wiki.asterisk.org/wiki/users/editmyemailsettings.action?src=email" title="" style="color: #3b73af; text-decoration: none">Manage notifications</a></td>
</tr>
</tbody>
</table> </td>
</tr>
<tr>
<td id="footer-pattern-text" class="mobile-resize-text" width="100%" style="padding: 0px; border-collapse: collapse; color: #999; font-size: 12px; line-height: 18px; font-family: Arial, sans-serif; mso-line-height-rule: exactly; mso-text-raise: 2px; display: none">This message was sent by Atlassian Confluence 5.6.6</td>
</tr>
</tbody>
</table> </td>
</tr>
</tbody>
</table>
<table id="sealed-section" border="0" cellpadding="0" cellspacing="0" width="0" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; color: #333; display: none">
<tbody>
<tr>
<td style="padding: 0px; border-collapse: collapse; border: 0; font-size: 0px; line-height: 0; mso-line-height-rule: exactly"></td>
</tr>
</tbody>
</table>
</body>
</html>