Skip to content

Commit

Permalink
Nullify dart_handle in dart_test, dart_wait, and dart_wait_local
Browse files Browse the repository at this point in the history
  • Loading branch information
@devreal
devreal committed Aug 16, 2017
1 parent ad7560a commit efd6ac8
Show file tree
Hide file tree
Showing 2 changed files with 76 additions and 68 deletions.
26 changes: 18 additions & 8 deletions dart-if/include/dash/dart/if/dart_communication.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -515,9 +515,11 @@ dart_ret_t dart_put_handle(
*/

dart_ret_t dart_wait(
dart_handle_t handle) DART_NOTHROW;
dart_handle_t * handle) DART_NOTHROW;
/**
* Wait for the local and remote completion of operations.
* Upon success, the handle is invalidated and may not be used in another
* \c dart_wait or \c dart_test operation.
*
* \param handles Array of handles of operations to wait for.
* \param n Number of \c handles to wait for.
Expand All @@ -528,11 +530,13 @@ dart_ret_t dart_wait(
* \ingroup DartCommunication
*/
dart_ret_t dart_waitall(
dart_handle_t * handles,
size_t n) DART_NOTHROW;
dart_handle_t handles[],
size_t n) DART_NOTHROW;

/**
* Wait for the local completion of an operation.
* Upon success, the handle is invalidated and may not be used in another
* \c dart_wait or \c dart_test operation.
*
* \param handle Handle of an operations to wait for.
*
Expand All @@ -542,10 +546,12 @@ dart_ret_t dart_waitall(
* \ingroup DartCommunication
*/
dart_ret_t dart_wait_local(
dart_handle_t handle);
dart_handle_t * handle) DART_NOTHROW;

/**
* Wait for the local completion of operations.
* Upon success, the handles are invalidated and may not be used in another
* \c dart_wait or \c dart_test operation.
*
* \param handles Array of handles of operations to wait for.
* \param n Number of \c handles to wait for.
Expand All @@ -556,11 +562,13 @@ dart_ret_t dart_wait_local(
* \ingroup DartCommunication
*/
dart_ret_t dart_waitall_local(
dart_handle_t * handles,
size_t n) DART_NOTHROW;
dart_handle_t handles[],
size_t n) DART_NOTHROW;

/**
* Test for the local completion of an operation.
* If the transfer completed, the handle is invalidated and may not be used
* in another \c dart_wait or \c dart_test operation.
*
* \param handle The handle of an operation to test for completion.
* \param[out] result \c True if the operation has completed.
Expand All @@ -571,11 +579,13 @@ dart_ret_t dart_waitall_local(
* \ingroup DartCommunication
*/
dart_ret_t dart_test_local(
dart_handle_t handle,
dart_handle_t * handle,
int32_t * result) DART_NOTHROW;

/**
* Test for the local completion of operations.
* If the transfers completed, the handles are invalidated and may not be
* used in another \c dart_wait or \c dart_test operation.
*
* \param handles Array of handles of operations to test for completion.
* \param n Number of \c handles to test for completion.
Expand All @@ -587,7 +597,7 @@ dart_ret_t dart_test_local(
* \ingroup DartCommunication
*/
dart_ret_t dart_testall_local(
dart_handle_t * handles,
dart_handle_t handles[],
size_t n,
int32_t * result) DART_NOTHROW;

Expand Down
118 changes: 58 additions & 60 deletions dart-impl/mpi/src/dart_communication.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1176,11 +1176,12 @@ dart_ret_t dart_flush_local_all(
}

dart_ret_t dart_wait_local(
dart_handle_t handle)
dart_handle_t * handleptr)
{
int mpi_ret;
DART_LOG_DEBUG("dart_wait_local() handle:%p", (void*)(handle));
if (handle != NULL) {
DART_LOG_DEBUG("dart_wait_local() handle:%p", (void*)(handleptr));
if (handleptr != NULL && *handleptr != DART_HANDLE_NULL) {
dart_handle_t handle = *handleptr;
DART_LOG_TRACE("dart_wait_local: handle->dest: %d",
handle->dest);
DART_LOG_TRACE("dart_wait_local: handle->win: %p",
Expand All @@ -1202,20 +1203,19 @@ dart_ret_t dart_wait_local(
} else {
DART_LOG_TRACE("dart_wait_local: handle->req == MPI_REQUEST_NULL");
}
/*
* Do not free handle resource, it could be needed for a following
* dart_wait() or dart_wait_all() to ensure remote completion.
*/
free(handle);
*handleptr = DART_HANDLE_NULL;
}
DART_LOG_DEBUG("dart_wait_local > finished");
return DART_OK;
}

dart_ret_t dart_wait(
dart_handle_t handle)
dart_handle_t * handleptr)
{
DART_LOG_DEBUG("dart_wait() handle:%p", (void*)(handle));
if (handle != DART_HANDLE_NULL) {
DART_LOG_DEBUG("dart_wait() handle:%p", (void*)(handleptr));
if (handleptr != NULL && *handleptr != DART_HANDLE_NULL) {
dart_handle_t handle = *handleptr;
DART_LOG_TRACE("dart_wait_local: handle->dest: %d",
handle->dest);
DART_LOG_TRACE("dart_wait_local: handle->win: %"PRIu64"",
Expand Down Expand Up @@ -1244,15 +1244,15 @@ dart_ret_t dart_wait(
/* Free handle resource */
handle->request = MPI_REQUEST_NULL;
free(handle);
handle = NULL;
*handleptr = DART_HANDLE_NULL;
}
DART_LOG_DEBUG("dart_wait > finished");
return DART_OK;
}

dart_ret_t dart_waitall_local(
dart_handle_t * handle,
size_t num_handles)
dart_handle_t handles[],
size_t num_handles)
{
dart_ret_t ret = DART_OK;

Expand All @@ -1265,16 +1265,16 @@ dart_ret_t dart_waitall_local(
DART_LOG_ERROR("dart_waitall_local ! number of handles > INT_MAX");
return DART_ERR_INVAL;
}
if (handle != NULL) {
if (handles != NULL) {
size_t r_n = 0;
MPI_Request *mpi_req = ALLOC_TMP(num_handles * sizeof(MPI_Request));
for (size_t i = 0; i < num_handles; i++) {
if (handle[i] != NULL && handle[i]->request != MPI_REQUEST_NULL) {
if (handles[i] != NULL && handles[i]->request != MPI_REQUEST_NULL) {
DART_LOG_TRACE("dart_waitall_local: -- handle[%"PRIu64"]: %p)",
i, (void*)handle[i]);
i, (void*)handles[i]);
DART_LOG_TRACE("dart_waitall_local: handle[%"PRIu64"]->dest: %d",
i, handle[i]->dest);
mpi_req[r_n] = handle[i]->request;
i, handles[i]->dest);
mpi_req[r_n] = handles[i]->request;
r_n++;
}
}
Expand Down Expand Up @@ -1304,13 +1304,13 @@ dart_ret_t dart_waitall_local(
*/
DART_LOG_TRACE("dart_waitall_local: releasing DART handles");
for (size_t i = 0; i < num_handles; i++) {
if (handle[i]) {
if (handles[i]) {
DART_LOG_TRACE("dart_waitall_local: free handle[%zu] %p",
i, (void*)(handle[i]));
i, (void*)(handles[i]));
// nullify request and free the handle
handle[i]->request = MPI_REQUEST_NULL;
free(handle[i]);
handle[i] = NULL;
handles[i]->request = MPI_REQUEST_NULL;
free(handles[i]);
handles[i] = DART_HANDLE_NULL;
}
}
FREE_TMP(num_handles * sizeof(MPI_Request), mpi_req);
Expand All @@ -1320,12 +1320,12 @@ dart_ret_t dart_waitall_local(
}

dart_ret_t dart_waitall(
dart_handle_t * handle,
size_t n)
dart_handle_t handles[],
size_t n)
{
DART_LOG_DEBUG("dart_waitall()");
if (n == 0) {
DART_LOG_ERROR("dart_waitall > number of handles = 0");
DART_LOG_DEBUG("dart_waitall > number of handles = 0");
return DART_OK;
}
if (n > INT_MAX) {
Expand All @@ -1335,22 +1335,22 @@ dart_ret_t dart_waitall(

DART_LOG_DEBUG("dart_waitall: number of handles: %zu", n);

if (handle) {
if (handles != NULL) {
MPI_Request *mpi_req = ALLOC_TMP(n * sizeof(MPI_Request));
/*
* copy requests from DART handles to MPI request array:
*/
DART_LOG_TRACE("dart_waitall: copying DART handles to MPI request array");
size_t r_n = 0;
for (size_t i = 0; i < n; i++) {
if (handle[i] != NULL) {
if (handles[i] != DART_HANDLE_NULL) {
DART_LOG_DEBUG("dart_waitall: -- handle[%zu](%p): "
"dest:%d win:%"PRIu64" req:%"PRIu64"",
i, (void*)handle[i],
handle[i]->dest,
(unsigned long)handle[i]->win,
(unsigned long)handle[i]->request);
mpi_req[r_n] = handle[i]->request;
i, (void*)handles[i],
handles[i]->dest,
(unsigned long)handles[i]->win,
(unsigned long)handles[i]->request);
mpi_req[r_n] = handles[i]->request;
r_n++;
}
}
Expand Down Expand Up @@ -1389,20 +1389,16 @@ dart_ret_t dart_waitall(
*/
DART_LOG_DEBUG("dart_waitall: waiting for remote completion");
for (size_t i = 0; i < n; i++) {
if (handle[i] && handle[i]->needs_flush) {
DART_LOG_DEBUG("dart_waitall: -- MPI_Win_flush(handle[%zu]: %p))",
i, (void*)handle[i]);
DART_LOG_TRACE("dart_waitall: handle[%zu]->dest: %d",
i, handle[i]->dest);
if (handles[i] != DART_HANDLE_NULL && handles[i]->needs_flush) {
DART_LOG_DEBUG("dart_waitall: -- MPI_Win_flush(handle[%zu]: %p, dest: %d))",
i, (void*)handles[i], handles[i]->dest);
/*
* MPI_Win_flush to wait for remote completion if required:
*/
if (handle[i]->needs_flush) {
if (MPI_Win_flush(handle[i]->dest, handle[i]->win) != MPI_SUCCESS) {
DART_LOG_ERROR("dart_waitall: MPI_Win_flush failed");
FREE_TMP(n * sizeof(MPI_Request), mpi_req);
return DART_ERR_INVAL;
}
if (MPI_Win_flush(handles[i]->dest, handles[i]->win) != MPI_SUCCESS) {
DART_LOG_ERROR("dart_waitall: MPI_Win_flush failed");
FREE_TMP(n * sizeof(MPI_Request), mpi_req);
return DART_ERR_INVAL;
}
}
}
Expand All @@ -1412,14 +1408,14 @@ dart_ret_t dart_waitall(
*/
DART_LOG_DEBUG("dart_waitall: free handles");
for (size_t i = 0; i < n; i++) {
if (handle[i]) {
if (handles[i]) {
/* Free handle resource */
DART_LOG_TRACE("dart_waitall: -- free handle[%zu]: %p",
i, (void*)(handle[i]));
i, (void*)(handles[i]));
// nullify request and free the handle
handle[i]->request = MPI_REQUEST_NULL;
free(handle[i]);
handle[i] = NULL;
handles[i]->request = MPI_REQUEST_NULL;
free(handles[i]);
handles[i] = DART_HANDLE_NULL;
}
}
DART_LOG_TRACE("dart_waitall: free MPI_Request temporaries");
Expand All @@ -1430,14 +1426,15 @@ dart_ret_t dart_waitall(
}

dart_ret_t dart_test_local(
dart_handle_t handle,
int32_t* is_finished)
dart_handle_t * handleptr,
int32_t * is_finished)
{
DART_LOG_DEBUG("dart_test_local()");
if (!handle) {
if (handleptr == NULL || *handleptr == DART_HANDLE_NULL) {
*is_finished = 1;
return DART_OK;
}
dart_handle_t handle = *handleptr;
if (MPI_Test(&(handle->request),
is_finished, MPI_STATUS_IGNORE) != MPI_SUCCESS) {
DART_LOG_ERROR("dart_test_local: MPI_Test failed!");
Expand All @@ -1446,27 +1443,28 @@ dart_ret_t dart_test_local(
if (is_finished) {
// deallocate handle
free(handle);
*handleptr = DART_HANDLE_NULL;
}
DART_LOG_DEBUG("dart_test_local > finished");
return DART_OK;
}

dart_ret_t dart_testall_local(
dart_handle_t * handle,
dart_handle_t handles[],
size_t n,
int32_t * is_finished)
{
DART_LOG_DEBUG("dart_testall_local()");
if (handle == NULL || n == 0) {
if (handles == NULL || n == 0) {
DART_LOG_DEBUG("dart_testall_local: empty handles");
return DART_OK;
}

MPI_Request *mpi_req = ALLOC_TMP(n * sizeof (MPI_Request));
size_t r_n = 0;
for (size_t i = 0; i < n; ++i) {
if (handle[i] && handle[i]->request != MPI_REQUEST_NULL){
mpi_req[r_n] = handle[i]->request;
if (handles[i] && handles[i]->request != MPI_REQUEST_NULL){
mpi_req[r_n] = handles[i]->request;
++r_n;
}
}
Expand All @@ -1481,11 +1479,11 @@ dart_ret_t dart_testall_local(

if (*is_finished) {
for (size_t i = 0; i < n; i++) {
if (handle[i]) {
if (handles[i] != DART_HANDLE_NULL) {
// nullify request and free the handle
handle[i]->request = MPI_REQUEST_NULL;
free(handle[i]);
handle[i] = NULL;
handles[i]->request = MPI_REQUEST_NULL;
free(handles[i]);
handles[i] = DART_HANDLE_NULL;
}
}
}
Expand Down

0 comments on commit efd6ac8

Please sign in to comment.