From 2efa45f3f1d2533fc3c73226f02ec7dd00b4837a Mon Sep 17 00:00:00 2001 From: Neil Fortner Date: Mon, 21 Feb 2022 08:45:17 -0600 Subject: Fix another outdated H5FD_class_t struct. --- test/null_vfd_plugin.c | 74 ++++++++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/test/null_vfd_plugin.c b/test/null_vfd_plugin.c index f41da00..125b510 100644 --- a/test/null_vfd_plugin.c +++ b/test/null_vfd_plugin.c @@ -35,41 +35,45 @@ static herr_t H5FD_null_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr); static haddr_t H5FD_null_get_eof(const H5FD_t *_file, H5FD_mem_t type); static const H5FD_class_t H5FD_null_g = { - NULL_VFD_VALUE, /* value */ - NULL_VFD_NAME, /* name */ - 1, /* maxaddr */ - H5F_CLOSE_WEAK, /* fc_degree */ - NULL, /* terminate */ - NULL, /* sb_size */ - NULL, /* sb_encode */ - NULL, /* sb_decode */ - 0, /* fapl_size */ - NULL, /* fapl_get */ - NULL, /* fapl_copy */ - NULL, /* fapl_free */ - 0, /* dxpl_size */ - NULL, /* dxpl_copy */ - NULL, /* dxpl_free */ - H5FD_null_open, /* open */ - H5FD_null_close, /* close */ - NULL, /* cmp */ - NULL, /* query */ - NULL, /* get_type_map */ - NULL, /* alloc */ - NULL, /* free */ - H5FD_null_get_eoa, /* get_eoa */ - H5FD_null_set_eoa, /* set_eoa */ - H5FD_null_get_eof, /* get_eof */ - NULL, /* get_handle */ - H5FD_null_read, /* read */ - H5FD_null_write, /* write */ - NULL, /* flush */ - NULL, /* truncate */ - NULL, /* lock */ - NULL, /* unlock */ - NULL, /* del */ - NULL, /* ctl */ - H5FD_FLMAP_DICHOTOMY /* fl_map */ + NULL_VFD_VALUE, /* value */ + NULL_VFD_NAME, /* name */ + 1, /* maxaddr */ + H5F_CLOSE_WEAK, /* fc_degree */ + NULL, /* terminate */ + NULL, /* sb_size */ + NULL, /* sb_encode */ + NULL, /* sb_decode */ + 0, /* fapl_size */ + NULL, /* fapl_get */ + NULL, /* fapl_copy */ + NULL, /* fapl_free */ + 0, /* dxpl_size */ + NULL, /* dxpl_copy */ + NULL, /* dxpl_free */ + H5FD_null_open, /* open */ + H5FD_null_close, /* close */ + NULL, /* cmp */ + NULL, /* query */ + NULL, /* get_type_map */ + NULL, /* alloc */ + NULL, /* free */ + H5FD_null_get_eoa, /* get_eoa */ + H5FD_null_set_eoa, /* set_eoa */ + H5FD_null_get_eof, /* get_eof */ + NULL, /* get_handle */ + H5FD_null_read, /* read */ + H5FD_null_write, /* write */ + NULL, /* read_vector */ + NULL, /* write_vector */ + NULL, /* read_selection */ + NULL, /* write_selection */ + NULL, /* flush */ + NULL, /* truncate */ + NULL, /* lock */ + NULL, /* unlock */ + NULL, /* del */ + NULL, /* ctl */ + H5FD_FLMAP_DICHOTOMY /* fl_map */ }; static H5FD_t * -- cgit v0.12 From d50b11f37e88af15550226b8e4bd3155d7d0e5b6 Mon Sep 17 00:00:00 2001 From: Neil Fortner Date: Mon, 21 Feb 2022 09:19:35 -0600 Subject: Free arrays in H5FD__mpio_read/write_vector() as soon as they're not needed, to cut down on memory usage during I/O. --- src/H5FDmpio.c | 196 +++++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 134 insertions(+), 62 deletions(-) diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c index 4775948..5b9af7e 100644 --- a/src/H5FDmpio.c +++ b/src/H5FDmpio.c @@ -1695,7 +1695,7 @@ H5FD__mpio_read_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t cou void * mpi_bufs_base = NULL; MPI_Aint mpi_bufs_base_Aint; MPI_Aint * mpi_bufs = NULL; - MPI_Aint * mpi_displacments = NULL; + MPI_Aint * mpi_displacements = NULL; MPI_Datatype buf_type = MPI_BYTE; /* MPI description of the selection in memory */ hbool_t buf_type_created = FALSE; MPI_Datatype file_type = MPI_BYTE; /* MPI description of the selection in file */ @@ -1816,7 +1816,7 @@ H5FD__mpio_read_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t cou HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "can't sort vector I/O request") if ((NULL == (mpi_block_lengths = (int *)HDmalloc((size_t)count * sizeof(int)))) || - (NULL == (mpi_displacments = (MPI_Aint *)HDmalloc((size_t)count * sizeof(MPI_Aint)))) || + (NULL == (mpi_displacements = (MPI_Aint *)HDmalloc((size_t)count * sizeof(MPI_Aint)))) || (NULL == (mpi_bufs = (MPI_Aint *)HDmalloc((size_t)count * sizeof(MPI_Aint))))) { HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't alloc mpi block lengths / displacement") @@ -1868,7 +1868,7 @@ H5FD__mpio_read_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t cou /* Add to block lengths and displacements arrays */ mpi_block_lengths[i] = (int)size; - mpi_displacments[i] = (MPI_Aint)s_addrs[i]; + mpi_displacements[i] = (MPI_Aint)s_addrs[i]; /* convert s_bufs[i] to MPI_Aint... */ if (MPI_SUCCESS != (mpi_code = MPI_Get_address(s_bufs[i], &(mpi_bufs[i])))) @@ -1939,12 +1939,12 @@ H5FD__mpio_read_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t cou /* create the file MPI derived type */ if (sub_types) { if (MPI_SUCCESS != - (mpi_code = MPI_Type_create_struct((int)count, mpi_block_lengths, mpi_displacments, + (mpi_code = MPI_Type_create_struct((int)count, mpi_block_lengths, mpi_displacements, sub_types, &file_type))) HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_struct for file_type failed", mpi_code) } else if (MPI_SUCCESS != - (mpi_code = MPI_Type_create_hindexed((int)count, mpi_block_lengths, mpi_displacments, + (mpi_code = MPI_Type_create_hindexed((int)count, mpi_block_lengths, mpi_displacements, MPI_BYTE, &file_type))) HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed for file_type failed", mpi_code) @@ -1954,6 +1954,32 @@ H5FD__mpio_read_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t cou HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit for file_type failed", mpi_code) + /* Free up memory used to build types */ + HDassert(mpi_block_lengths); + HDfree(mpi_block_lengths); + mpi_block_lengths = NULL; + + HDassert(mpi_displacements); + HDfree(mpi_displacements); + mpi_displacements = NULL; + + HDassert(mpi_bufs); + HDfree(mpi_bufs); + mpi_bufs = NULL; + + if(sub_types) { + HDassert(sub_types); + + for (i = 0; i < (int)count; i++) + if (sub_types_created[i]) + MPI_Type_free(&sub_types[i]); + + HDfree(sub_types); + sub_types = NULL; + HDfree(sub_types_created); + sub_types_created = NULL; + } + /* some numeric conversions */ if (H5FD_mpi_haddr_to_MPIOff((haddr_t)0, &mpi_off) < 0) HGOTO_ERROR(H5E_INTERNAL, H5E_BADRANGE, FAIL, "can't set MPI off to 0") @@ -2249,45 +2275,55 @@ done: } } - if (mpi_block_lengths) { + if (buf_type_created) { + MPI_Type_free(&buf_type); + } - HDfree(mpi_block_lengths); - mpi_block_lengths = NULL; + if (file_type_created) { + MPI_Type_free(&file_type); } - if (mpi_displacments) { + /* Clean up on error */ + if (ret_value < 0) { + if (mpi_block_lengths) { - HDfree(mpi_displacments); - mpi_displacments = NULL; - } + HDfree(mpi_block_lengths); + mpi_block_lengths = NULL; + } - if (mpi_bufs) { + if (mpi_displacements) { - HDfree(mpi_bufs); - mpi_bufs = NULL; - } + HDfree(mpi_displacements); + mpi_displacements = NULL; + } - if (buf_type_created) { - MPI_Type_free(&buf_type); - } + if (mpi_bufs) { - if (sub_types) { - HDassert(sub_types_created); + HDfree(mpi_bufs); + mpi_bufs = NULL; + } - for (i = 0; i < (int)count; i++) - if (sub_types_created[i]) - MPI_Type_free(&sub_types[i]); + if (sub_types) { + HDassert(sub_types_created); - HDfree(sub_types); - sub_types = NULL; - HDfree(sub_types_created); - sub_types_created = NULL; - } + for (i = 0; i < (int)count; i++) + if (sub_types_created[i]) + MPI_Type_free(&sub_types[i]); - if (file_type_created) { - MPI_Type_free(&file_type); + HDfree(sub_types); + sub_types = NULL; + HDfree(sub_types_created); + sub_types_created = NULL; + } } + /* Make sure we cleaned up */ + HDassert(!mpi_block_lengths); + HDassert(!mpi_displacements); + HDassert(!mpi_bufs); + HDassert(!sub_types); + HDassert(!sub_types_created); + #ifdef H5FDmpio_DEBUG if (H5FD_mpio_debug_t_flag) HDfprintf(stdout, "%s: Leaving, proc %d: ret_value = %d\n", __func__, file->mpi_rank, ret_value); @@ -2355,7 +2391,7 @@ H5FD__mpio_write_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t co void * mpi_bufs_base = NULL; MPI_Aint mpi_bufs_base_Aint; MPI_Aint * mpi_bufs = NULL; - MPI_Aint * mpi_displacments = NULL; + MPI_Aint * mpi_displacements = NULL; MPI_Datatype buf_type = MPI_BYTE; /* MPI description of the selection in memory */ hbool_t buf_type_created = FALSE; MPI_Datatype file_type = MPI_BYTE; /* MPI description of the selection in file */ @@ -2431,7 +2467,7 @@ H5FD__mpio_write_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t co hsize_t bigio_count; /* Transition point to create derived type */ if ((NULL == (mpi_block_lengths = (int *)HDmalloc((size_t)count * sizeof(int)))) || - (NULL == (mpi_displacments = (MPI_Aint *)HDmalloc((size_t)count * sizeof(MPI_Aint)))) || + (NULL == (mpi_displacements = (MPI_Aint *)HDmalloc((size_t)count * sizeof(MPI_Aint)))) || (NULL == (mpi_bufs = (MPI_Aint *)HDmalloc((size_t)count * sizeof(MPI_Aint))))) { HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't alloc mpi block lengths / displacement") @@ -2486,7 +2522,7 @@ H5FD__mpio_write_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t co /* Add to block lengths and displacements arrays */ mpi_block_lengths[i] = (int)size; - mpi_displacments[i] = (MPI_Aint)s_addrs[i]; + mpi_displacements[i] = (MPI_Aint)s_addrs[i]; /* convert s_bufs[i] to MPI_Aint... */ if (MPI_SUCCESS != (mpi_code = MPI_Get_address(s_bufs[i], &(mpi_bufs[i])))) @@ -2557,12 +2593,12 @@ H5FD__mpio_write_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t co /* create the file MPI derived type */ if (sub_types) { if (MPI_SUCCESS != - (mpi_code = MPI_Type_create_struct((int)count, mpi_block_lengths, mpi_displacments, + (mpi_code = MPI_Type_create_struct((int)count, mpi_block_lengths, mpi_displacements, sub_types, &file_type))) HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_struct for file_type failed", mpi_code) } else if (MPI_SUCCESS != - (mpi_code = MPI_Type_create_hindexed((int)count, mpi_block_lengths, mpi_displacments, + (mpi_code = MPI_Type_create_hindexed((int)count, mpi_block_lengths, mpi_displacements, MPI_BYTE, &file_type))) HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed for file_type failed", mpi_code) @@ -2571,6 +2607,32 @@ H5FD__mpio_write_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t co if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(&file_type))) HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit for file_type failed", mpi_code) + + /* Free up memory used to build types */ + HDassert(mpi_block_lengths); + HDfree(mpi_block_lengths); + mpi_block_lengths = NULL; + + HDassert(mpi_displacements); + HDfree(mpi_displacements); + mpi_displacements = NULL; + + HDassert(mpi_bufs); + HDfree(mpi_bufs); + mpi_bufs = NULL; + + if(sub_types) { + HDassert(sub_types); + + for (i = 0; i < (int)count; i++) + if (sub_types_created[i]) + MPI_Type_free(&sub_types[i]); + + HDfree(sub_types); + sub_types = NULL; + HDfree(sub_types_created); + sub_types_created = NULL; + } } else { @@ -2750,45 +2812,55 @@ done: } } - if (mpi_block_lengths) { + if (buf_type_created) { + MPI_Type_free(&buf_type); + } - HDfree(mpi_block_lengths); - mpi_block_lengths = NULL; + if (file_type_created) { + MPI_Type_free(&file_type); } - if (mpi_displacments) { + /* Clean up on error */ + if (ret_value < 0) { + if (mpi_block_lengths) { - HDfree(mpi_displacments); - mpi_displacments = NULL; - } + HDfree(mpi_block_lengths); + mpi_block_lengths = NULL; + } - if (mpi_bufs) { + if (mpi_displacements) { - HDfree(mpi_bufs); - mpi_bufs = NULL; - } + HDfree(mpi_displacements); + mpi_displacements = NULL; + } - if (buf_type_created) { - MPI_Type_free(&buf_type); - } + if (mpi_bufs) { - if (file_type_created) { - MPI_Type_free(&file_type); - } + HDfree(mpi_bufs); + mpi_bufs = NULL; + } - if (sub_types) { - HDassert(sub_types_created); + if (sub_types) { + HDassert(sub_types_created); - for (i = 0; i < (int)count; i++) - if (sub_types_created[i]) - MPI_Type_free(&sub_types[i]); + for (i = 0; i < (int)count; i++) + if (sub_types_created[i]) + MPI_Type_free(&sub_types[i]); - HDfree(sub_types); - sub_types = NULL; - HDfree(sub_types_created); - sub_types_created = NULL; + HDfree(sub_types); + sub_types = NULL; + HDfree(sub_types_created); + sub_types_created = NULL; + } } + /* Make sure we cleaned up */ + HDassert(!mpi_block_lengths); + HDassert(!mpi_displacements); + HDassert(!mpi_bufs); + HDassert(!sub_types); + HDassert(!sub_types_created); + #ifdef H5FDmpio_DEBUG if (H5FD_mpio_debug_t_flag) HDfprintf(stdout, "%s: Leaving, proc %d: ret_value = %d\n", __func__, file->mpi_rank, ret_value); -- cgit v0.12 From 5a93c181de4187fa934b02ad913ea0a953cba65c Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 21 Feb 2022 15:22:09 +0000 Subject: Committing clang-format changes --- src/H5FDmpio.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c index 5b9af7e..211da9c 100644 --- a/src/H5FDmpio.c +++ b/src/H5FDmpio.c @@ -1967,7 +1967,7 @@ H5FD__mpio_read_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t cou HDfree(mpi_bufs); mpi_bufs = NULL; - if(sub_types) { + if (sub_types) { HDassert(sub_types); for (i = 0; i < (int)count; i++) @@ -2621,7 +2621,7 @@ H5FD__mpio_write_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t co HDfree(mpi_bufs); mpi_bufs = NULL; - if(sub_types) { + if (sub_types) { HDassert(sub_types); for (i = 0; i < (int)count; i++) -- cgit v0.12