<p>Michael Bradeen has uploaded this change for <strong>review</strong>.</p><p><a href="https://gerrit.asterisk.org/c/asterisk/+/17644">View Change</a></p><pre style="font-family: monospace,monospace; white-space: pre-wrap;">various: fix and test a double deref on a scheduled delete of an<br>executing call back.<br><br>sched: Avoid a double deref when AST_SCHED_DEL_UNREF is called on an<br>executing call-back. This is done by adding a new variable 'rescheduled'<br>to the struct sched which is set in ast_sched_runq and checked in<br>ast_sched_del. ast_sched_del will now return a new possible value -2 if<br>called on an executing call-back with rescheduled set.<br>AST_SCHED_DEL_UNREF is also updated to look for the -2 in which case it<br>will not throw a warning or invoke refcall.<br>test_sched: Add a new unit test sched_test_freebird that will check the<br>reference count in the resolved scenario.<br><br>ASTERISK-29698<br><br>Change-Id: Icfb16b3acbc29cf5b4cef74183f7531caaefe21d<br>---<br>M include/asterisk/sched.h<br>M main/sched.c<br>M tests/test_sched.c<br>3 files changed, 154 insertions(+), 5 deletions(-)<br><br></pre><pre style="font-family: monospace,monospace; white-space: pre-wrap;">git pull ssh://gerrit.asterisk.org:29418/asterisk refs/changes/44/17644/1</pre><pre style="font-family: monospace,monospace; white-space: pre-wrap;"><span>diff --git a/include/asterisk/sched.h b/include/asterisk/sched.h</span><br><span>index bbef914..cd6d2ad 100644</span><br><span>--- a/include/asterisk/sched.h</span><br><span>+++ b/include/asterisk/sched.h</span><br><span>@@ -79,13 +79,13 @@</span><br><span> */</span><br><span> #define AST_SCHED_DEL_UNREF(sched, id, refcall) \</span><br><span> do { \</span><br><span style="color: hsl(0, 100%, 40%);">- int _count = 0, _id; \</span><br><span style="color: hsl(0, 100%, 40%);">- while ((_id = id) > -1 && ast_sched_del(sched, _id) && ++_count < 10) { \</span><br><span style="color: hsl(120, 100%, 40%);">+ int _count = 0, _id, _ret = 0; \</span><br><span style="color: hsl(120, 100%, 40%);">+ while ((_id = id) > -1 && (( _ret = ast_sched_del(sched, _id)) == -1) && ++_count < 10) { \</span><br><span> usleep(1); \</span><br><span> } \</span><br><span> if (_count == 10) { \</span><br><span> ast_log(LOG_WARNING, "Unable to cancel schedule ID %d. This is probably a bug (%s: %s, line %d).\n", _id, __FILE__, __PRETTY_FUNCTION__, __LINE__); \</span><br><span style="color: hsl(0, 100%, 40%);">- } else if (_id > -1) { \</span><br><span style="color: hsl(120, 100%, 40%);">+ } else if (_id > -1 && _ret >-2) { \</span><br><span> refcall; \</span><br><span> id = -1; \</span><br><span> } \</span><br><span>@@ -293,6 +293,7 @@</span><br><span> * \param id ID of the scheduled item to delete</span><br><span> *</span><br><span> * \retval -1 on failure</span><br><span style="color: hsl(120, 100%, 40%);">+ * \retval -2 event was running and not rescheduled</span><br><span> * \retval 0 on success</span><br><span> */</span><br><span> int ast_sched_del(struct ast_sched_context *con, int id) attribute_warn_unused_result;</span><br><span>diff --git a/main/sched.c b/main/sched.c</span><br><span>index e3a7d30..b759390 100644</span><br><span>--- a/main/sched.c</span><br><span>+++ b/main/sched.c</span><br><span>@@ -98,6 +98,8 @@</span><br><span> ast_cond_t cond;</span><br><span> /*! Indication that a running task was deleted. */</span><br><span> unsigned int deleted:1;</span><br><span style="color: hsl(120, 100%, 40%);">+ /*! Indication that a running task was rescheduled. */</span><br><span style="color: hsl(120, 100%, 40%);">+ unsigned int rescheduled:1;</span><br><span> };</span><br><span> </span><br><span> struct sched_thread {</span><br><span>@@ -611,6 +613,7 @@</span><br><span> {</span><br><span> struct sched *s = NULL;</span><br><span> int *last_id = ast_threadstorage_get(&last_del_id, sizeof(int));</span><br><span style="color: hsl(120, 100%, 40%);">+ int res = 0;</span><br><span> </span><br><span> DEBUG(ast_debug(1, "ast_sched_del(%d)\n", id));</span><br><span> </span><br><span>@@ -646,6 +649,13 @@</span><br><span> ast_cond_wait(&s->cond, &con->lock);</span><br><span> }</span><br><span> /* Do not sched_release() here because ast_sched_runq() will do it */</span><br><span style="color: hsl(120, 100%, 40%);">+ /* This was not rescheduled so the caller of ast_sched_del can not remove any</span><br><span style="color: hsl(120, 100%, 40%);">+ * references as they already were.</span><br><span style="color: hsl(120, 100%, 40%);">+ */</span><br><span style="color: hsl(120, 100%, 40%);">+ if (!s->rescheduled) {</span><br><span style="color: hsl(120, 100%, 40%);">+ res = -2;</span><br><span style="color: hsl(120, 100%, 40%);">+ }</span><br><span style="color: hsl(120, 100%, 40%);">+ sched_release(con, s);</span><br><span> }</span><br><span> }</span><br><span> </span><br><span>@@ -668,7 +678,7 @@</span><br><span> return -1;</span><br><span> }</span><br><span> </span><br><span style="color: hsl(0, 100%, 40%);">- return 0;</span><br><span style="color: hsl(120, 100%, 40%);">+ return res;</span><br><span> }</span><br><span> </span><br><span> void ast_sched_report(struct ast_sched_context *con, struct ast_str **buf, struct ast_cb_names *cbnames)</span><br><span>@@ -793,7 +803,9 @@</span><br><span> con->currently_executing = NULL;</span><br><span> ast_cond_signal(¤t->cond);</span><br><span> </span><br><span style="color: hsl(0, 100%, 40%);">- if (res && !current->deleted) {</span><br><span style="color: hsl(120, 100%, 40%);">+ if (current->deleted) {</span><br><span style="color: hsl(120, 100%, 40%);">+ current->rescheduled = res ? 1 : 0;</span><br><span style="color: hsl(120, 100%, 40%);">+ } else if (res) {</span><br><span> /*</span><br><span> * If they return non-zero, we should schedule them to be</span><br><span> * run again.</span><br><span>diff --git a/tests/test_sched.c b/tests/test_sched.c</span><br><span>index e995c2c..90c461f 100644</span><br><span>--- a/tests/test_sched.c</span><br><span>+++ b/tests/test_sched.c</span><br><span>@@ -37,6 +37,7 @@</span><br><span> #include "asterisk/sched.h"</span><br><span> #include "asterisk/test.h"</span><br><span> #include "asterisk/cli.h"</span><br><span style="color: hsl(120, 100%, 40%);">+#include "asterisk/astobj2.h"</span><br><span> </span><br><span> static int sched_cb(const void *data)</span><br><span> {</span><br><span>@@ -336,6 +337,139 @@</span><br><span> return CLI_SUCCESS;</span><br><span> }</span><br><span> </span><br><span style="color: hsl(120, 100%, 40%);">+struct test_obj {</span><br><span style="color: hsl(120, 100%, 40%);">+ ast_mutex_t lock;</span><br><span style="color: hsl(120, 100%, 40%);">+ ast_cond_t cond;</span><br><span style="color: hsl(120, 100%, 40%);">+ int scheduledCBstarted;</span><br><span style="color: hsl(120, 100%, 40%);">+ int id;</span><br><span style="color: hsl(120, 100%, 40%);">+};</span><br><span style="color: hsl(120, 100%, 40%);">+</span><br><span style="color: hsl(120, 100%, 40%);">+static void test_obj_cleanup(void *data)</span><br><span style="color: hsl(120, 100%, 40%);">+{</span><br><span style="color: hsl(120, 100%, 40%);">+ struct test_obj *obj = data;</span><br><span style="color: hsl(120, 100%, 40%);">+ ast_mutex_destroy(&obj->lock);</span><br><span style="color: hsl(120, 100%, 40%);">+ ast_cond_destroy(&obj->cond);</span><br><span style="color: hsl(120, 100%, 40%);">+}</span><br><span style="color: hsl(120, 100%, 40%);">+</span><br><span style="color: hsl(120, 100%, 40%);">+//static void context_cleanup(void *data)</span><br><span style="color: hsl(120, 100%, 40%);">+//{</span><br><span style="color: hsl(120, 100%, 40%);">+// struct ast_sched_context *c = data;</span><br><span style="color: hsl(120, 100%, 40%);">+// ast_sched_context_destroy(c);</span><br><span style="color: hsl(120, 100%, 40%);">+//}</span><br><span style="color: hsl(120, 100%, 40%);">+</span><br><span style="color: hsl(120, 100%, 40%);">+static int lockingcb(const void *data)</span><br><span style="color: hsl(120, 100%, 40%);">+{</span><br><span style="color: hsl(120, 100%, 40%);">+ struct test_obj *obj = (struct test_obj *)data;</span><br><span style="color: hsl(120, 100%, 40%);">+</span><br><span style="color: hsl(120, 100%, 40%);">+ ast_mutex_lock(&obj->lock);</span><br><span style="color: hsl(120, 100%, 40%);">+</span><br><span style="color: hsl(120, 100%, 40%);">+ obj->scheduledCBstarted = 1;</span><br><span style="color: hsl(120, 100%, 40%);">+ ast_cond_signal(&obj->cond);</span><br><span style="color: hsl(120, 100%, 40%);">+</span><br><span style="color: hsl(120, 100%, 40%);">+ ast_mutex_unlock(&obj->lock);</span><br><span style="color: hsl(120, 100%, 40%);">+</span><br><span style="color: hsl(120, 100%, 40%);">+ ao2_ref(obj, -1);</span><br><span style="color: hsl(120, 100%, 40%);">+ usleep(3000);</span><br><span style="color: hsl(120, 100%, 40%);">+</span><br><span style="color: hsl(120, 100%, 40%);">+ return 0;</span><br><span style="color: hsl(120, 100%, 40%);">+}</span><br><span style="color: hsl(120, 100%, 40%);">+</span><br><span style="color: hsl(120, 100%, 40%);">+AST_TEST_DEFINE(sched_test_freebird)</span><br><span style="color: hsl(120, 100%, 40%);">+{</span><br><span style="color: hsl(120, 100%, 40%);">+</span><br><span style="color: hsl(120, 100%, 40%);">+ //RAII_VAR(struct test_obj *, obj, ao2_alloc(sizeof(*obj), test_obj_cleanup), ao2_cleanup);</span><br><span style="color: hsl(120, 100%, 40%);">+ //RAII_VAR(struct ast_sched_context *, con, ast_sched_context_create(), context_cleanup), ao2_cleanup);</span><br><span style="color: hsl(120, 100%, 40%);">+</span><br><span style="color: hsl(120, 100%, 40%);">+ struct ast_sched_context *con;</span><br><span style="color: hsl(120, 100%, 40%);">+ enum ast_test_result_state res = AST_TEST_FAIL;</span><br><span style="color: hsl(120, 100%, 40%);">+ struct test_obj *obj;</span><br><span style="color: hsl(120, 100%, 40%);">+ int refs;</span><br><span style="color: hsl(120, 100%, 40%);">+</span><br><span style="color: hsl(120, 100%, 40%);">+ switch (cmd) {</span><br><span style="color: hsl(120, 100%, 40%);">+ case TEST_INIT:</span><br><span style="color: hsl(120, 100%, 40%);">+ info->name = "sched_test_freebird";</span><br><span style="color: hsl(120, 100%, 40%);">+ info->category = "/main/sched/";</span><br><span style="color: hsl(120, 100%, 40%);">+ info->summary = "Test deadlock avoidance and double-unref";</span><br><span style="color: hsl(120, 100%, 40%);">+ info->description =</span><br><span style="color: hsl(120, 100%, 40%);">+ "This test forces a thread conflict issue on res mem sorcery.";</span><br><span style="color: hsl(120, 100%, 40%);">+ return AST_TEST_NOT_RUN;</span><br><span style="color: hsl(120, 100%, 40%);">+ case TEST_EXECUTE:</span><br><span style="color: hsl(120, 100%, 40%);">+ res = AST_TEST_PASS;</span><br><span style="color: hsl(120, 100%, 40%);">+ break;</span><br><span style="color: hsl(120, 100%, 40%);">+ }</span><br><span style="color: hsl(120, 100%, 40%);">+</span><br><span style="color: hsl(120, 100%, 40%);">+ if (!(obj = ao2_alloc(sizeof(struct test_obj), test_obj_cleanup))) {</span><br><span style="color: hsl(120, 100%, 40%);">+ ast_test_status_update(test,</span><br><span style="color: hsl(120, 100%, 40%);">+ "ao2_alloc() did not return an object\n");</span><br><span style="color: hsl(120, 100%, 40%);">+ return AST_TEST_FAIL;</span><br><span style="color: hsl(120, 100%, 40%);">+ }</span><br><span style="color: hsl(120, 100%, 40%);">+</span><br><span style="color: hsl(120, 100%, 40%);">+ /* This additional reference is to ensure that the object isn't destroyed prematurely</span><br><span style="color: hsl(120, 100%, 40%);">+ * in a case where it is unreffed an additional time.</span><br><span style="color: hsl(120, 100%, 40%);">+ */</span><br><span style="color: hsl(120, 100%, 40%);">+ ao2_ref(obj, +1);</span><br><span style="color: hsl(120, 100%, 40%);">+ obj->scheduledCBstarted = 0;</span><br><span style="color: hsl(120, 100%, 40%);">+</span><br><span style="color: hsl(120, 100%, 40%);">+ if (!(con = ast_sched_context_create())) {</span><br><span style="color: hsl(120, 100%, 40%);">+ ast_test_status_update(test,</span><br><span style="color: hsl(120, 100%, 40%);">+ "ast_sched_context_create() did not return a context\n");</span><br><span style="color: hsl(120, 100%, 40%);">+ return AST_TEST_FAIL;</span><br><span style="color: hsl(120, 100%, 40%);">+ }</span><br><span style="color: hsl(120, 100%, 40%);">+</span><br><span style="color: hsl(120, 100%, 40%);">+ if(ast_sched_start_thread(con)) {</span><br><span style="color: hsl(120, 100%, 40%);">+ ast_test_status_update(test, "Failed to start test thread\n");</span><br><span style="color: hsl(120, 100%, 40%);">+ ast_sched_context_destroy(con);</span><br><span style="color: hsl(120, 100%, 40%);">+ return AST_TEST_FAIL;</span><br><span style="color: hsl(120, 100%, 40%);">+ }</span><br><span style="color: hsl(120, 100%, 40%);">+</span><br><span style="color: hsl(120, 100%, 40%);">+ ao2_bump(obj);</span><br><span style="color: hsl(120, 100%, 40%);">+ if ((obj->id = ast_sched_add(con, 0, lockingcb, obj)) == -1) {</span><br><span style="color: hsl(120, 100%, 40%);">+ ast_test_status_update(test, "Failed to add scheduler entry\n");</span><br><span style="color: hsl(120, 100%, 40%);">+ ast_sched_context_destroy(con);</span><br><span style="color: hsl(120, 100%, 40%);">+ return AST_TEST_FAIL;</span><br><span style="color: hsl(120, 100%, 40%);">+ }</span><br><span style="color: hsl(120, 100%, 40%);">+</span><br><span style="color: hsl(120, 100%, 40%);">+ ast_mutex_lock(&obj->lock);</span><br><span style="color: hsl(120, 100%, 40%);">+ while(obj->scheduledCBstarted == 0) {</span><br><span style="color: hsl(120, 100%, 40%);">+ ast_cond_wait(&obj->cond,&obj->lock);</span><br><span style="color: hsl(120, 100%, 40%);">+ }</span><br><span style="color: hsl(120, 100%, 40%);">+ ast_mutex_unlock(&obj->lock);</span><br><span style="color: hsl(120, 100%, 40%);">+</span><br><span style="color: hsl(120, 100%, 40%);">+ ast_test_status_update(test, "Received signal, calling Scedule and UNREF\n");</span><br><span style="color: hsl(120, 100%, 40%);">+ ast_test_status_update(test, "ID: %d\n", obj->id);</span><br><span style="color: hsl(120, 100%, 40%);">+ AST_SCHED_DEL_UNREF(con, obj->id, ao2_ref(obj, -1));</span><br><span style="color: hsl(120, 100%, 40%);">+</span><br><span style="color: hsl(120, 100%, 40%);">+/* do{</span><br><span style="color: hsl(120, 100%, 40%);">+ int _count = 0, _id, ret;</span><br><span style="color: hsl(120, 100%, 40%);">+ while ((_id = obj->id) > -1 && ((ret = ast_sched_del(con, _id)) ==-1) && ++_count < 10) {</span><br><span style="color: hsl(120, 100%, 40%);">+ usleep(1);</span><br><span style="color: hsl(120, 100%, 40%);">+ }</span><br><span style="color: hsl(120, 100%, 40%);">+ if (_count == 10) {</span><br><span style="color: hsl(120, 100%, 40%);">+ ast_log(LOG_WARNING, "Unable to cancel schedule ID %d. This is probably a bug (%s: %s, line %d).\n", _id, __FILE__, __PRETTY_FUNCTION__, __LINE__);</span><br><span style="color: hsl(120, 100%, 40%);">+ } else if (_id > -1 && ret >-2) {</span><br><span style="color: hsl(120, 100%, 40%);">+ //refcall;</span><br><span style="color: hsl(120, 100%, 40%);">+ ao2_ref(obj, -1);</span><br><span style="color: hsl(120, 100%, 40%);">+ obj->id = -1;</span><br><span style="color: hsl(120, 100%, 40%);">+ }</span><br><span style="color: hsl(120, 100%, 40%);">+ } while(0); */</span><br><span style="color: hsl(120, 100%, 40%);">+</span><br><span style="color: hsl(120, 100%, 40%);">+ refs = ao2_ref(obj, 0);</span><br><span style="color: hsl(120, 100%, 40%);">+</span><br><span style="color: hsl(120, 100%, 40%);">+ switch(refs){</span><br><span style="color: hsl(120, 100%, 40%);">+ case 2:</span><br><span style="color: hsl(120, 100%, 40%);">+ ast_test_status_update(test, "Correct number of references '2'\n");</span><br><span style="color: hsl(120, 100%, 40%);">+ break;</span><br><span style="color: hsl(120, 100%, 40%);">+ default:</span><br><span style="color: hsl(120, 100%, 40%);">+ ast_test_status_update(test, "Inorrect number of references '%d'\n",</span><br><span style="color: hsl(120, 100%, 40%);">+ refs);</span><br><span style="color: hsl(120, 100%, 40%);">+ res = AST_TEST_FAIL;</span><br><span style="color: hsl(120, 100%, 40%);">+ break;</span><br><span style="color: hsl(120, 100%, 40%);">+ }</span><br><span style="color: hsl(120, 100%, 40%);">+</span><br><span style="color: hsl(120, 100%, 40%);">+ ast_sched_context_destroy(con);</span><br><span style="color: hsl(120, 100%, 40%);">+ return res;</span><br><span style="color: hsl(120, 100%, 40%);">+}</span><br><span style="color: hsl(120, 100%, 40%);">+</span><br><span> static struct ast_cli_entry cli_sched[] = {</span><br><span> AST_CLI_DEFINE(handle_cli_sched_bench, "Benchmark ast_sched add/del performance"),</span><br><span> };</span><br><span>@@ -343,6 +477,7 @@</span><br><span> static int unload_module(void)</span><br><span> {</span><br><span> AST_TEST_UNREGISTER(sched_test_order);</span><br><span style="color: hsl(120, 100%, 40%);">+ AST_TEST_UNREGISTER(sched_test_freebird);</span><br><span> ast_cli_unregister_multiple(cli_sched, ARRAY_LEN(cli_sched));</span><br><span> return 0;</span><br><span> }</span><br><span>@@ -350,6 +485,7 @@</span><br><span> static int load_module(void)</span><br><span> {</span><br><span> AST_TEST_REGISTER(sched_test_order);</span><br><span style="color: hsl(120, 100%, 40%);">+ AST_TEST_REGISTER(sched_test_freebird);</span><br><span> ast_cli_register_multiple(cli_sched, ARRAY_LEN(cli_sched));</span><br><span> return AST_MODULE_LOAD_SUCCESS;</span><br><span> }</span><br><span></span><br></pre><p>To view, visit <a href="https://gerrit.asterisk.org/c/asterisk/+/17644">change 17644</a>. To unsubscribe, or for help writing mail filters, visit <a href="https://gerrit.asterisk.org/settings">settings</a>.</p><div itemscope itemtype="http://schema.org/EmailMessage"><div itemscope itemprop="action" itemtype="http://schema.org/ViewAction"><link itemprop="url" href="https://gerrit.asterisk.org/c/asterisk/+/17644"/><meta itemprop="name" content="View Change"/></div></div>
<div style="display:none"> Gerrit-Project: asterisk </div>
<div style="display:none"> Gerrit-Branch: 16 </div>
<div style="display:none"> Gerrit-Change-Id: Icfb16b3acbc29cf5b4cef74183f7531caaefe21d </div>
<div style="display:none"> Gerrit-Change-Number: 17644 </div>
<div style="display:none"> Gerrit-PatchSet: 1 </div>
<div style="display:none"> Gerrit-Owner: Michael Bradeen <mbradeen@sangoma.com> </div>
<div style="display:none"> Gerrit-MessageType: newchange </div>