From fdeac5309eceeed5159dcbc60de4161f144593ef Mon Sep 17 00:00:00 2001 From: jhendersonHDF Date: Fri, 1 Sep 2023 07:20:05 -0500 Subject: Fix serial to parallel chunked dataset file space allocation bug (#3394) (#3467) --- release_docs/RELEASE.txt | 11 ++ src/H5Dint.c | 35 ++++-- testpar/t_chunk_alloc.c | 277 +++++++++++++++++++++++++++++++++++++++++++++++ testpar/testphdf5.c | 2 + testpar/testphdf5.h | 1 + 5 files changed, 315 insertions(+), 11 deletions(-) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 9b8946d..1059a46 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -109,6 +109,17 @@ Bug Fixes since HDF5-1.14.2 release =================================== Library ------- + - Fixed a file space allocation bug in the parallel library for chunked + datasets + + With the addition of support for incremental file space allocation for + chunked datasets with filters applied to them that are created/accessed + in parallel, a bug was introduced to the library's parallel file space + allocation code. This could cause file space to not be allocated correctly + for datasets without filters applied to them that are created with serial + file access and later opened with parallel file access. In turn, this could + cause parallel writes to those datasets to place incorrect data in the file. + - Fixed an assertion failure in Parallel HDF5 when a file can't be created due to an invalid library version bounds setting diff --git a/src/H5Dint.c b/src/H5Dint.c index e6a709e..a547a72 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -1664,12 +1664,13 @@ done: static herr_t H5D__open_oid(H5D_t *dataset, hid_t dapl_id) { - H5P_genplist_t *plist; /* Property list */ - H5O_fill_t *fill_prop; /* Pointer to dataset's fill value info */ - unsigned alloc_time_state; /* Allocation time state */ - htri_t msg_exists; /* Whether a particular type of message exists */ - hbool_t layout_init = FALSE; /* Flag to indicate that chunk information was initialized */ - herr_t ret_value = SUCCEED; /* Return value */ + H5P_genplist_t *plist; /* Property list */ + H5O_fill_t *fill_prop; /* Pointer to dataset's fill value info */ + unsigned alloc_time_state; /* Allocation time state */ + htri_t msg_exists; /* Whether a particular type of message exists */ + hbool_t layout_init = FALSE; /* Flag to indicate that chunk information was initialized */ + hbool_t must_init_storage = FALSE; + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE_TAG(dataset->oloc.addr) @@ -1812,12 +1813,24 @@ H5D__open_oid(H5D_t *dataset, hid_t dapl_id) * Make sure all storage is properly initialized. * This is important only for parallel I/O where the space must * be fully allocated before I/O can happen. + * + * Storage will be initialized here if either the VFD being used + * has set the H5FD_FEAT_ALLOCATE_EARLY flag to indicate that it + * wishes to force early space allocation OR a parallel VFD is + * being used and the dataset in question doesn't have any filters + * applied to it. If filters are applied to the dataset, collective + * I/O will be required when writing to the dataset, so we don't + * need to initialize storage here, as the collective I/O process + * will coordinate that. */ - if ((H5F_INTENT(dataset->oloc.file) & H5F_ACC_RDWR) && - !(*dataset->shared->layout.ops->is_space_alloc)(&dataset->shared->layout.storage) && - H5F_HAS_FEATURE(dataset->oloc.file, H5FD_FEAT_ALLOCATE_EARLY)) - if (H5D__alloc_storage(dataset, H5D_ALLOC_OPEN, FALSE, NULL) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize file storage"); + must_init_storage = (H5F_INTENT(dataset->oloc.file) & H5F_ACC_RDWR) && + !(*dataset->shared->layout.ops->is_space_alloc)(&dataset->shared->layout.storage); + must_init_storage = must_init_storage && (H5F_HAS_FEATURE(dataset->oloc.file, H5FD_FEAT_ALLOCATE_EARLY) || + (H5F_HAS_FEATURE(dataset->oloc.file, H5FD_FEAT_HAS_MPI) && + dataset->shared->dcpl_cache.pline.nused == 0)); + + if (must_init_storage && (H5D__alloc_storage(dataset, H5D_ALLOC_OPEN, FALSE, NULL) < 0)) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize file storage"); done: if (ret_value < 0) { diff --git a/testpar/t_chunk_alloc.c b/testpar/t_chunk_alloc.c index 6f862c0..0ffe695 100644 --- a/testpar/t_chunk_alloc.c +++ b/testpar/t_chunk_alloc.c @@ -480,3 +480,280 @@ test_chunk_alloc(void) /* reopen dataset in parallel, read and verify the data */ verify_data(filename, CHUNK_FACTOR, all, CLOSE, &file_id, &dataset); } + +/* + * A test to verify the following: + * + * - That the library forces allocation of all space in the file + * for a chunked dataset opened with parallel file access when + * that dataset: + * + * - was created with serial file access + * - was created with the default incremental file space + * allocation time + * - has no filters applied to it + * + * In this case, the library has to ensure that all the file + * space for the dataset is allocated so that the MPI processes + * can write to chunks independently of each other and still have + * a consistent view of the file. + * + * - That the library DOES NOT force allocation of all space in + * the file for a chunked dataset opened with parallel file access + * when that dataset: + * + * - was created with serial file access + * - was created with the default incremental file space + * allocation time + * - has filters applied to it + * + * In this case, writes to the dataset are required to be collective, + * so file space can be allocated incrementally in a coordinated + * fashion. + */ +void +test_chunk_alloc_incr_ser_to_par(void) +{ + H5D_space_status_t space_status; + const char *filename; + hsize_t dset_dims[1]; + hsize_t start[1]; + hsize_t stride[1]; + hsize_t count[1]; + hsize_t block[1]; + hsize_t alloc_size; + size_t nchunks; + herr_t ret; + hid_t fid = H5I_INVALID_HID; + hid_t fapl_id = H5I_INVALID_HID; + hid_t dset_id = H5I_INVALID_HID; + hid_t fspace_id = H5I_INVALID_HID; + hid_t dxpl_id = H5I_INVALID_HID; + int *data = NULL; + int *correct_data = NULL; + int *read_data = NULL; + + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + + filename = (const char *)GetTestParameters(); + if (MAINPROCESS && VERBOSE_MED) + printf("Chunked dataset incremental file space allocation serial to parallel test on file %s\n", + filename); + + nchunks = (size_t)(CHUNK_FACTOR * mpi_size); + dset_dims[0] = (hsize_t)(nchunks * CHUNK_SIZE); + + if (mpi_rank == 0) { + hsize_t chunk_dims[1] = {CHUNK_SIZE}; + hid_t space_id = H5I_INVALID_HID; + hid_t dcpl_id = H5I_INVALID_HID; + + fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + VRFY((fid >= 0), "H5Fcreate"); + + dcpl_id = H5Pcreate(H5P_DATASET_CREATE); + VRFY((dcpl_id >= 0), "H5Pcreate"); + + ret = H5Pset_chunk(dcpl_id, 1, chunk_dims); + VRFY((ret == SUCCEED), "H5Pset_chunk"); + + ret = H5Pset_alloc_time(dcpl_id, H5D_ALLOC_TIME_INCR); + VRFY((ret == SUCCEED), "H5Pset_alloc_time"); + + space_id = H5Screate_simple(1, dset_dims, NULL); + VRFY((space_id >= 0), "H5Screate_simple"); + + /* Create a chunked dataset without a filter applied to it */ + dset_id = + H5Dcreate2(fid, "dset_no_filter", H5T_NATIVE_INT, space_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "H5Dcreate2"); + + ret = H5Dclose(dset_id); + VRFY((ret == SUCCEED), "H5Dclose"); + + /* Create a chunked dataset with a filter applied to it */ + ret = H5Pset_shuffle(dcpl_id); + VRFY((ret == SUCCEED), "H5Pset_shuffle"); + + dset_id = H5Dcreate2(fid, "dset_filter", H5T_NATIVE_INT, space_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "H5Dcreate2"); + + ret = H5Dclose(dset_id); + VRFY((ret == SUCCEED), "H5Dclose"); + ret = H5Pclose(dcpl_id); + VRFY((ret == SUCCEED), "H5Pclose"); + ret = H5Sclose(space_id); + VRFY((ret == SUCCEED), "H5Sclose"); + ret = H5Fclose(fid); + VRFY((ret == SUCCEED), "H5Fclose"); + } + + MPI_Barrier(MPI_COMM_WORLD); + + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + VRFY((fapl_id >= 0), "H5Pcreate"); + + ret = H5Pset_fapl_mpio(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL); + VRFY((ret == SUCCEED), "H5Pset_fapl_mpio"); + + fid = H5Fopen(filename, H5F_ACC_RDWR, fapl_id); + VRFY((fid >= 0), "H5Fopen"); + + data = malloc((dset_dims[0] / (hsize_t)mpi_size) * sizeof(int)); + VRFY(data, "malloc"); + read_data = malloc(dset_dims[0] * sizeof(int)); + VRFY(read_data, "malloc"); + correct_data = malloc(dset_dims[0] * sizeof(int)); + VRFY(correct_data, "malloc"); + + /* + * Check the file space allocation status/size and dataset + * data before and after writing to the dataset without a + * filter + */ + dset_id = H5Dopen2(fid, "dset_no_filter", H5P_DEFAULT); + VRFY((dset_id >= 0), "H5Dopen2"); + + ret = H5Dget_space_status(dset_id, &space_status); + VRFY((ret == SUCCEED), "H5Dread"); + + VRFY((space_status == H5D_SPACE_STATUS_ALLOCATED), "file space allocation status verification succeeded"); + + alloc_size = H5Dget_storage_size(dset_id); + VRFY(((dset_dims[0] * sizeof(int)) == alloc_size), "file space allocation size verification succeeded"); + + memset(read_data, 255, dset_dims[0] * sizeof(int)); + memset(correct_data, 0, dset_dims[0] * sizeof(int)); + + ret = H5Dread(dset_id, H5T_NATIVE_INT, H5S_BLOCK, H5S_ALL, H5P_DEFAULT, read_data); + VRFY((ret == SUCCEED), "H5Dread"); + + MPI_Barrier(MPI_COMM_WORLD); + + VRFY((0 == memcmp(read_data, correct_data, dset_dims[0] * sizeof(int))), "data verification succeeded"); + + fspace_id = H5Dget_space(dset_id); + VRFY((ret == SUCCEED), "H5Dget_space"); + + start[0] = ((hsize_t)mpi_rank * (dset_dims[0] / (hsize_t)mpi_size)); + stride[0] = 1; + count[0] = (dset_dims[0] / (hsize_t)mpi_size); + block[0] = 1; + + ret = H5Sselect_hyperslab(fspace_id, H5S_SELECT_SET, start, stride, count, block); + VRFY((ret == SUCCEED), "H5Sselect_hyperslab"); + + memset(data, 255, (dset_dims[0] / (hsize_t)mpi_size) * sizeof(int)); + + ret = H5Dwrite(dset_id, H5T_NATIVE_INT, H5S_BLOCK, fspace_id, H5P_DEFAULT, data); + VRFY((ret == SUCCEED), "H5Dwrite"); + + MPI_Barrier(MPI_COMM_WORLD); + + ret = H5Dget_space_status(dset_id, &space_status); + VRFY((ret == SUCCEED), "H5Dread"); + + VRFY((space_status == H5D_SPACE_STATUS_ALLOCATED), "file space allocation status verification succeeded"); + + alloc_size = H5Dget_storage_size(dset_id); + VRFY(((dset_dims[0] * sizeof(int)) == alloc_size), "file space allocation size verification succeeded"); + + memset(read_data, 0, dset_dims[0] * sizeof(int)); + memset(correct_data, 255, dset_dims[0] * sizeof(int)); + + ret = H5Dread(dset_id, H5T_NATIVE_INT, H5S_BLOCK, H5S_ALL, H5P_DEFAULT, read_data); + VRFY((ret == SUCCEED), "H5Dread"); + + MPI_Barrier(MPI_COMM_WORLD); + + VRFY((0 == memcmp(read_data, correct_data, dset_dims[0] * sizeof(int))), "data verification succeeded"); + + ret = H5Sclose(fspace_id); + VRFY((ret == SUCCEED), "H5Sclose"); + ret = H5Dclose(dset_id); + VRFY((ret == SUCCEED), "H5Dclose"); + + /* + * Check the file space allocation status/size and dataset + * data before and after writing to the dataset with a + * filter + */ + dset_id = H5Dopen2(fid, "dset_filter", H5P_DEFAULT); + VRFY((dset_id >= 0), "H5Dopen2"); + + ret = H5Dget_space_status(dset_id, &space_status); + VRFY((ret == SUCCEED), "H5Dread"); + + VRFY((space_status == H5D_SPACE_STATUS_NOT_ALLOCATED), + "file space allocation status verification succeeded"); + + alloc_size = H5Dget_storage_size(dset_id); + VRFY((0 == alloc_size), "file space allocation size verification succeeded"); + + memset(read_data, 255, dset_dims[0] * sizeof(int)); + memset(correct_data, 0, dset_dims[0] * sizeof(int)); + + ret = H5Dread(dset_id, H5T_NATIVE_INT, H5S_BLOCK, H5S_ALL, H5P_DEFAULT, read_data); + VRFY((ret == SUCCEED), "H5Dread"); + + MPI_Barrier(MPI_COMM_WORLD); + + VRFY((0 == memcmp(read_data, correct_data, dset_dims[0] * sizeof(int))), "data verification succeeded"); + + fspace_id = H5Dget_space(dset_id); + VRFY((ret == SUCCEED), "H5Dget_space"); + + start[0] = ((hsize_t)mpi_rank * (dset_dims[0] / (hsize_t)mpi_size)); + stride[0] = 1; + count[0] = (dset_dims[0] / (hsize_t)mpi_size); + block[0] = 1; + + ret = H5Sselect_hyperslab(fspace_id, H5S_SELECT_SET, start, stride, count, block); + VRFY((ret == SUCCEED), "H5Sselect_hyperslab"); + + memset(data, 255, (dset_dims[0] / (hsize_t)mpi_size) * sizeof(int)); + + dxpl_id = H5Pcreate(H5P_DATASET_XFER); + VRFY((dxpl_id >= 0), "H5Pcreate"); + + ret = H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE); + VRFY((ret == SUCCEED), "H5Pset_dxpl_mpio"); + + ret = H5Dwrite(dset_id, H5T_NATIVE_INT, H5S_BLOCK, fspace_id, dxpl_id, data); + VRFY((ret == SUCCEED), "H5Dwrite"); + + MPI_Barrier(MPI_COMM_WORLD); + + ret = H5Dget_space_status(dset_id, &space_status); + VRFY((ret == SUCCEED), "H5Dread"); + + VRFY((space_status == H5D_SPACE_STATUS_ALLOCATED), "file space allocation status verification succeeded"); + + alloc_size = H5Dget_storage_size(dset_id); + VRFY(((dset_dims[0] * sizeof(int)) == alloc_size), "file space allocation size verification succeeded"); + + memset(read_data, 0, dset_dims[0] * sizeof(int)); + memset(correct_data, 255, dset_dims[0] * sizeof(int)); + + ret = H5Dread(dset_id, H5T_NATIVE_INT, H5S_BLOCK, H5S_ALL, H5P_DEFAULT, read_data); + VRFY((ret == SUCCEED), "H5Dread"); + + MPI_Barrier(MPI_COMM_WORLD); + + VRFY((0 == memcmp(read_data, correct_data, dset_dims[0] * sizeof(int))), "data verification succeeded"); + + ret = H5Pclose(dxpl_id); + VRFY((ret == SUCCEED), "H5Pclose"); + ret = H5Sclose(fspace_id); + VRFY((ret == SUCCEED), "H5Sclose"); + ret = H5Dclose(dset_id); + VRFY((ret == SUCCEED), "H5Dclose"); + + free(correct_data); + free(read_data); + free(data); + + H5Pclose(fapl_id); + H5Fclose(fid); +} diff --git a/testpar/testphdf5.c b/testpar/testphdf5.c index 1cca1fd..430b8a6 100644 --- a/testpar/testphdf5.c +++ b/testpar/testphdf5.c @@ -376,6 +376,8 @@ main(int argc, char **argv) AddTest("selnone", none_selection_chunk, NULL, "chunked dataset with none-selection", PARATESTFILE); AddTest("calloc", test_chunk_alloc, NULL, "parallel extend Chunked allocation on serial file", PARATESTFILE); + AddTest("chkallocser2par", test_chunk_alloc_incr_ser_to_par, NULL, + "chunk allocation from serial to parallel file access", PARATESTFILE); AddTest("fltread", test_filter_read, NULL, "parallel read of dataset written serially with filters", PARATESTFILE); diff --git a/testpar/testphdf5.h b/testpar/testphdf5.h index 820fdc3..631ad44 100644 --- a/testpar/testphdf5.h +++ b/testpar/testphdf5.h @@ -256,6 +256,7 @@ void none_selection_chunk(void); void actual_io_mode_tests(void); void no_collective_cause_tests(void); void test_chunk_alloc(void); +void test_chunk_alloc_incr_ser_to_par(void); void test_filter_read(void); void compact_dataset(void); void null_dataset(void); -- cgit v0.12