From 5a61dfb242371f233c54c391f6919055a020df5e Mon Sep 17 00:00:00 2001 From: mainzer Date: Tue, 20 Apr 2021 18:40:45 -0500 Subject: Added vector read / write support to the MPIO VFD, with associated test code (see testpar/t_vfd.c). Note that this implementation does NOT support vector entries of size greater than 2 GB. This must be repaired before release, but it should be good enough for correctness testing. As MPIO requires vector I/O requests to be sorted in increasing address order, also added a vector sort utility in H5FDint.c This function is tested in passing by the MPIO vector I/O extension. In passing, repaired a bug in size / type vector extension management in H5FD_read/write_vector() Tested parallel debug and production on charis and Jelly. --- MANIFEST | 1 + src/H5FD.c | 4 +- src/H5FDint.c | 354 ++++- src/H5FDmpio.c | 908 +++++++++++- src/H5FDprivate.h | 4 + testpar/Makefile.am | 2 +- testpar/t_vfd.c | 4057 +++++++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 5282 insertions(+), 48 deletions(-) create mode 100644 testpar/t_vfd.c diff --git a/MANIFEST b/MANIFEST index f386724..4d3c027 100644 --- a/MANIFEST +++ b/MANIFEST @@ -1462,6 +1462,7 @@ ./testpar/t_pshutdown.c ./testpar/t_prestart.c ./testpar/t_span_tree.c +./testpar/t_vfd.c ./testpar/t_init_term.c ./testpar/t_2Gio.c ./testpar/testpar.h diff --git a/src/H5FD.c b/src/H5FD.c index 1e9a812..5a853e3 100644 --- a/src/H5FD.c +++ b/src/H5FD.c @@ -1467,7 +1467,7 @@ H5FDread_vector(H5FD_t *file, hid_t dxpl_id, uint32_t count, H5FD_mem_t types[], herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE7("e", "*xiIu*Mt*a*zx", file, dxpl_id, count, types, addrs, sizes, bufs); + H5TRACE7("e", "*#iIu*Mt*a*zx", file, dxpl_id, count, types, addrs, sizes, bufs); /* Check arguments */ if (!file) @@ -1548,7 +1548,7 @@ H5FDwrite_vector(H5FD_t *file, hid_t dxpl_id, uint32_t count, H5FD_mem_t types[] herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE7("e", "*xiIu*Mt*a*z**x", file, dxpl_id, count, types, addrs, sizes, bufs); + H5TRACE7("e", "*#iIu*Mt*a*z**x", file, dxpl_id, count, types, addrs, sizes, bufs); /* Check arguments */ if (!file) diff --git a/src/H5FDint.c b/src/H5FDint.c index 141e7ab..add69f3 100644 --- a/src/H5FDint.c +++ b/src/H5FDint.c @@ -44,6 +44,33 @@ /* Local Typedefs */ /******************/ +/************************************************************************* + * + * H5FD_vsrt_tmp_t + * + * Structure used to store vector I/O request addresses and the associated + * indexes in the addrs[] array for the purpose of determine the sorted + * order. + * + * This is done by allocating an array of H5FD_vsrt_tmp_t of length + * count, loading it with the contents of the addrs[] array and the + * associated indicies, and then sorting it. + * + * This sorted array of H5FD_vsrt_tmp_t is then used to populate sorted + * versions of the types[], addrs[], sizes[] and bufs[] vectors. + * + * addr: haddr_t containing the value of addrs[i], + * + * index: integer containing the value of i used to obtain the + * value of the addr field from the addrs[] vector. + * + *************************************************************************/ + +typedef struct H5FD_vsrt_tmp_t { + haddr_t addr; + int index; +} H5FD_vsrt_tmp_t; + /********************/ /* Package Typedefs */ /********************/ @@ -345,19 +372,48 @@ H5FD_read_vector(H5FD_t *file, uint32_t count, H5FD_mem_t types[], haddr_t addrs * objects being written within the file by the application performing * SWMR write operations. */ - if (!(file->access_flags & H5F_ACC_SWMR_READ)) { + if ((!(file->access_flags & H5F_ACC_SWMR_READ)) && (count > 0)) { haddr_t eoa; + extend_sizes = FALSE; + extend_types = FALSE; + for (i = 0; i < count; i++) { - if (HADDR_UNDEF == (eoa = (file->cls->get_eoa)(file, types[i]))) + if (!extend_sizes) { + + if (sizes[i] == 0) { + + extend_sizes = TRUE; + size = sizes[i - 1]; + } + else { + + size = sizes[i]; + } + } + + if (!extend_types) { + + if (types[i] == H5FD_MEM_NOLIST) { + + extend_types = TRUE; + type = types[i - 1]; + } + else { + + type = types[i]; + } + } + + if (HADDR_UNDEF == (eoa = (file->cls->get_eoa)(file, type))) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver get_eoa request failed") - if ((addrs[i] + sizes[i]) > eoa) + if ((addrs[i] + size) > eoa) HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "addr overflow, addrs[%d] = %llu, sizes[%d] = %llu, eoa = %llu", (int)i, - (unsigned long long)(addrs[i]), (int)i, (unsigned long long)sizes[i], + (unsigned long long)(addrs[i]), (int)i, (unsigned long long)size, (unsigned long long)eoa) } } @@ -522,17 +578,46 @@ H5FD_write_vector(H5FD_t *file, uint32_t count, H5FD_mem_t types[], haddr_t addr addrs_cooked = TRUE; } + extend_sizes = FALSE; + extend_types = FALSE; + for (i = 0; i < count; i++) { - if (HADDR_UNDEF == (eoa = (file->cls->get_eoa)(file, types[i]))) + if (!extend_sizes) { + + if (sizes[i] == 0) { + + extend_sizes = TRUE; + size = sizes[i - 1]; + } + else { + + size = sizes[i]; + } + } + + if (!extend_types) { + + if (types[i] == H5FD_MEM_NOLIST) { + + extend_types = TRUE; + type = types[i - 1]; + } + else { + + type = types[i]; + } + } + + if (HADDR_UNDEF == (eoa = (file->cls->get_eoa)(file, type))) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver get_eoa request failed") - if ((addrs[i] + sizes[i]) > eoa) + if ((addrs[i] + size) > eoa) HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "addr overflow, addrs[%d] = %llu, sizes[%d] = %llu, \ eoa = %llu", - (int)i, (unsigned long long)(addrs[i]), (int)i, (unsigned long long)sizes[i], + (int)i, (unsigned long long)(addrs[i]), (int)i, (unsigned long long)size, (unsigned long long)eoa) } @@ -739,3 +824,258 @@ H5FD_driver_query(const H5FD_class_t *driver, unsigned long *flags /*out*/) FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_driver_query() */ + +/*------------------------------------------------------------------------- + * Function: H5FD_sort_vector_io_req + * + * Purpose: Determine whether the supplied vector I/O request is + * sorted. + * + * if is is, set *vector_was_sorted to TRUE, set: + * + * *s_types_ptr = types + * *s_addrs_ptr = addrs + * *s_sizes_ptr = sizes + * *s_bufs_ptr = bufs + * + * and return. + * + * If it is not sorted, duplicate the type, addrs, sizes, + * and bufs vectors, storing the base addresses of the new + * vectors in *s_types_ptr, *s_addrs_ptr, *s_sizes_ptr, and + * *s_bufs_ptr respectively. Determine the sorted order + * of the vector I/O request, and load it into the new + * vectors in sorted order. + * + * Note that in this case, it is the callers responsibility + * to free the sorted vectors. + * + * JRM -- 3/15/21 + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ + +static int +H5FD__vsrt_tmp_cmp(const void *element_1, const void *element_2) +{ + haddr_t addr_1 = ((const H5FD_vsrt_tmp_t *)element_1)->addr; + haddr_t addr_2 = ((const H5FD_vsrt_tmp_t *)element_2)->addr; + int ret_value = 0; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Sanity checks */ + HDassert(H5F_addr_defined(addr_1)); + HDassert(H5F_addr_defined(addr_2)); + + if (H5F_addr_gt(addr_1, addr_2)) { + + ret_value = 1; + } + else if (H5F_addr_lt(addr_1, addr_2)) { + + ret_value = -1; + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD__vsrt_tmp_cmp() */ + +herr_t +H5FD_sort_vector_io_req(hbool_t *vector_was_sorted, uint32_t count, H5FD_mem_t types[], haddr_t addrs[], + size_t sizes[], void *bufs[], H5FD_mem_t **s_types_ptr, haddr_t **s_addrs_ptr, + size_t **s_sizes_ptr, void ***s_bufs_ptr) +{ + herr_t ret_value = SUCCEED; /* Return value */ + int i; + struct H5FD_vsrt_tmp_t *srt_tmp = NULL; + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(vector_was_sorted); + + HDassert((types) || (count == 0)); + HDassert((addrs) || (count == 0)); + HDassert((sizes) || (count == 0)); + HDassert((bufs) || (count == 0)); + + /* verify that the first elements of the sizes and types arrays are + * valid. + */ + HDassert((count == 0) || (sizes[0] != 0)); + HDassert((count == 0) || (types[0] != H5FD_MEM_NOLIST)); + + HDassert((count == 0) || ((s_types_ptr) && (NULL == *s_types_ptr))); + HDassert((count == 0) || ((s_addrs_ptr) && (NULL == *s_addrs_ptr))); + HDassert((count == 0) || ((s_sizes_ptr) && (NULL == *s_sizes_ptr))); + HDassert((count == 0) || ((s_bufs_ptr) && (NULL == *s_bufs_ptr))); + + *vector_was_sorted = TRUE; + + /* if count <= 1, vector is sorted by definition */ + if (count > 1) { + + /* scan the addrs array to see if it is sorted */ + i = 1; + + while ((*vector_was_sorted) && (i < (int)(count - 1))) { + + if (H5F_addr_gt(addrs[i - 1], addrs[i])) { + + *vector_was_sorted = FALSE; + } + else if (H5F_addr_eq(addrs[i - 1], addrs[i])) { + + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "duplicate addr in vector") + } + i++; + } + } + + if (*vector_was_sorted) { + + *s_types_ptr = types; + *s_addrs_ptr = addrs; + *s_sizes_ptr = sizes; + *s_bufs_ptr = bufs; + } + else { + + /* must sort the addrs array in increasing addr order, while + * maintaining the association between each addr, and the + * sizes[], types[], and bufs[] values at the same index. + * + * Do this by allocating an array of struct H5FD_vsrt_tmp_t, where + * each instance of H5FD_vsrt_tmp_t has two fields, addr and index. + * Load the array with the contents of the addrs array and + * the index of the associated entry. Sort the array, allocate + * the s_types_ptr, s_addrs_ptr, s_sizes_ptr, and s_bufs_ptr + * arrays and populate them using the mapping provided by + * the sorted array of H5FD_vsrt_tmp_t. + */ + int j; + int fixed_size_index = (int)count; + int fixed_type_index = (int)count; + size_t srt_tmp_size; + + srt_tmp_size = ((size_t)count * sizeof(struct H5FD_vsrt_tmp_t)); + + if (NULL == (srt_tmp = (H5FD_vsrt_tmp_t *)HDmalloc(srt_tmp_size))) + + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't alloc srt_tmp") + + for (i = 0; i < (int)count; i++) { + + srt_tmp[i].addr = addrs[i]; + srt_tmp[i].index = i; + } + + /* sort the srt_tmp array */ + HDqsort(srt_tmp, (size_t)count, sizeof(struct H5FD_vsrt_tmp_t), H5FD__vsrt_tmp_cmp); + + /* verify no duplicate entries */ + i = 1; + + while ((*vector_was_sorted) && (i < (int)(count - 1))) { + + HDassert(H5F_addr_lt(srt_tmp[i - 1].addr, srt_tmp[i].addr)); + + if (H5F_addr_eq(addrs[i - 1], addrs[i])) { + + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "duplicate addr in vector") + } + i++; + } + + if ((NULL == (*s_types_ptr = (H5FD_mem_t *)HDmalloc((size_t)count * sizeof(H5FD_mem_t)))) || + (NULL == (*s_addrs_ptr = (haddr_t *)HDmalloc((size_t)count * sizeof(haddr_t)))) || + (NULL == (*s_sizes_ptr = (size_t *)HDmalloc((size_t)count * sizeof(size_t)))) || + (NULL == (*s_bufs_ptr = (void *)HDmalloc((size_t)count * sizeof(void *))))) { + + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't alloc sorted vector(s)") + } + + HDassert(sizes[0] != 0); + HDassert(types[0] != H5FD_MEM_NOLIST); + + /* scan the sizes and types vectors to determine if the fixed size / type + * optimization is in use, and if so, to determine the index of the last + * valid value on each vector. + */ + i = 0; + while ((i < (int)count) && ((fixed_size_index == (int)count) || (fixed_type_index == (int)count))) { + + if ((fixed_size_index == (int)count) && (sizes[i] == 0)) { + + fixed_size_index = i - 1; + } + + if ((fixed_type_index == (int)count) && (types[i] == H5FD_MEM_NOLIST)) { + + fixed_type_index = i - 1; + } + + i++; + } + + HDassert((fixed_size_index >= 0) && (fixed_size_index <= (int)count)); + HDassert((fixed_type_index >= 0) && (fixed_size_index <= (int)count)); + + /* populate the sorted vectors */ + for (i = 0; i < (int)count; i++) { + + j = srt_tmp[i].index; + + (*s_types_ptr)[j] = types[MIN(i, fixed_type_index)]; + (*s_addrs_ptr)[j] = addrs[i]; + (*s_sizes_ptr)[j] = sizes[MIN(i, fixed_size_index)]; + (*s_bufs_ptr)[j] = bufs[i]; + } + } + +done: + if (srt_tmp) { + + HDfree(srt_tmp); + srt_tmp = NULL; + } + + /* On failure, free the sorted vectors if they were allocated. + * Note that we only allocate these vectors if the original array + * was not sorted -- thus we check both for failure, and for + * the flag indicating that the original vector was not sorted + * in increasing address order. + */ + if ((ret_value != SUCCEED) && (!(*vector_was_sorted))) { + + /* free space allocated for sorted vectors */ + if (*s_types_ptr) { + + HDfree(*s_types_ptr); + *s_types_ptr = NULL; + } + + if (*s_addrs_ptr) { + + HDfree(*s_addrs_ptr); + *s_addrs_ptr = NULL; + } + + if (*s_sizes_ptr) { + + HDfree(*s_sizes_ptr); + *s_sizes_ptr = NULL; + } + + if (*s_bufs_ptr) { + + HDfree(*s_bufs_ptr); + *s_bufs_ptr = NULL; + } + } + + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5FD_sort_vector_io_req() */ diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c index b5c2684..3ec6802 100644 --- a/src/H5FDmpio.c +++ b/src/H5FDmpio.c @@ -84,6 +84,10 @@ static herr_t H5FD__mpio_read(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, h void *buf); static herr_t H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, const void *buf); +static herr_t H5FD__mpio_read_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t count, + H5FD_mem_t types[], haddr_t addrs[], size_t sizes[], void *bufs[]); +static herr_t H5FD__mpio_write_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t count, + H5FD_mem_t types[], haddr_t addrs[], size_t sizes[], void *bufs[]); static herr_t H5FD__mpio_flush(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); static herr_t H5FD__mpio_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); static int H5FD__mpio_mpi_rank(const H5FD_t *_file); @@ -94,44 +98,44 @@ static MPI_Comm H5FD__mpio_communicator(const H5FD_t *_file); static const H5FD_class_mpi_t H5FD_mpio_g = { { /* Start of superclass information */ - "mpio", /*name */ - HADDR_MAX, /*maxaddr */ - H5F_CLOSE_SEMI, /*fc_degree */ - H5FD__mpio_term, /*terminate */ - NULL, /*sb_size */ - NULL, /*sb_encode */ - NULL, /*sb_decode */ - 0, /*fapl_size */ - NULL, /*fapl_get */ - NULL, /*fapl_copy */ - NULL, /*fapl_free */ - 0, /*dxpl_size */ - NULL, /*dxpl_copy */ - NULL, /*dxpl_free */ - H5FD__mpio_open, /*open */ - H5FD__mpio_close, /*close */ - NULL, /*cmp */ - H5FD__mpio_query, /*query */ - NULL, /*get_type_map */ - NULL, /*alloc */ - NULL, /*free */ - H5FD__mpio_get_eoa, /*get_eoa */ - H5FD__mpio_set_eoa, /*set_eoa */ - H5FD__mpio_get_eof, /*get_eof */ - H5FD__mpio_get_handle, /*get_handle */ - H5FD__mpio_read, /*read */ - H5FD__mpio_write, /*write */ - NULL, /*read_vector */ - NULL, /*write_vector */ - H5FD__mpio_flush, /*flush */ - H5FD__mpio_truncate, /*truncate */ - NULL, /*lock */ - NULL, /*unlock */ - H5FD_FLMAP_DICHOTOMY /*fl_map */ - }, /* End of superclass information */ - H5FD__mpio_mpi_rank, /*get_rank */ - H5FD__mpio_mpi_size, /*get_size */ - H5FD__mpio_communicator /*get_comm */ + "mpio", /*name */ + HADDR_MAX, /*maxaddr */ + H5F_CLOSE_SEMI, /*fc_degree */ + H5FD__mpio_term, /*terminate */ + NULL, /*sb_size */ + NULL, /*sb_encode */ + NULL, /*sb_decode */ + 0, /*fapl_size */ + NULL, /*fapl_get */ + NULL, /*fapl_copy */ + NULL, /*fapl_free */ + 0, /*dxpl_size */ + NULL, /*dxpl_copy */ + NULL, /*dxpl_free */ + H5FD__mpio_open, /*open */ + H5FD__mpio_close, /*close */ + NULL, /*cmp */ + H5FD__mpio_query, /*query */ + NULL, /*get_type_map */ + NULL, /*alloc */ + NULL, /*free */ + H5FD__mpio_get_eoa, /*get_eoa */ + H5FD__mpio_set_eoa, /*set_eoa */ + H5FD__mpio_get_eof, /*get_eof */ + H5FD__mpio_get_handle, /*get_handle */ + H5FD__mpio_read, /*read */ + H5FD__mpio_write, /*write */ + H5FD__mpio_read_vector, /*read_vector */ + H5FD__mpio_write_vector, /*write_vector */ + H5FD__mpio_flush, /*flush */ + H5FD__mpio_truncate, /*truncate */ + NULL, /*lock */ + NULL, /*unlock */ + H5FD_FLMAP_DICHOTOMY /*fl_map */ + }, /* End of superclass information */ + H5FD__mpio_mpi_rank, /*get_rank */ + H5FD__mpio_mpi_size, /*get_size */ + H5FD__mpio_communicator /*get_comm */ }; #ifdef H5FDmpio_DEBUG @@ -1521,6 +1525,834 @@ done: } /* end H5FD__mpio_write() */ /*------------------------------------------------------------------------- + * Function: H5FD__mpio_read_vector() + * + * Purpose: The behaviour of this function dependes on the value of + * the io_xfer_mode obtained from the context. + * + * If it is H5FD_MPIO_COLLECTIVE, this is a collective + * operation, which allows us to use MPI_File_set_view, and + * then perform the entire vector read in a single MPI call. + * + * Do this (if count is positive), by constructing memory + * and file derived types from the supplied vector, using + * file type to set the file view, and then reading the + * the memory type from file. Note that this read is + * either independent or collective depending on the + * value of mpio_coll_opt -- again obtained from the context. + * + * If count is zero, participate in the collective read + * (if so configured) with an empty read. + * + * Finally, set the file view back to its default state. + * + * In contrast, if io_xfer_mode is H5FD_MPIO_INDEPENDENT, + * this call is independent, and thus we cannot use + * MPI_File_set_view(). + * + * In this case, simply walk the vector, and issue an + * independent read for each entry. + * + * WARNING: At present, this function makes no provision + * entries of size greater than 2 GB in the vector. This + * will have to be fixed before release. + * + * Return: Success: SUCCEED. + * Failure: FAIL. + * + * Programmer: John Mainzer + * March 15, 2021 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__mpio_read_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t count, H5FD_mem_t types[], + haddr_t addrs[], size_t sizes[], void *bufs[]) +{ + H5FD_mpio_t * file = (H5FD_mpio_t *)_file; + hbool_t vector_was_sorted = TRUE; + hbool_t fixed_size = FALSE; + size_t size; + H5FD_mem_t * s_types = NULL; + haddr_t * s_addrs = NULL; + size_t * s_sizes = NULL; + void ** s_bufs = NULL; + int * mpi_block_lengths = NULL; + char unused = 0; /* Unused, except for non-NULL pointer value */ + void * mpi_bufs_base = NULL; + MPI_Aint mpi_bufs_base_Aint; + MPI_Aint * mpi_bufs = NULL; + MPI_Aint * mpi_displacments = NULL; + MPI_Datatype buf_type = MPI_BYTE; /* MPI description of the selection in memory */ + hbool_t buf_type_created = FALSE; + MPI_Datatype file_type = MPI_BYTE; /* MPI description of the selection in file */ + hbool_t file_type_created = FALSE; + int i; + int j; + int mpi_code; /* MPI return code */ + MPI_Offset mpi_off = 0; + MPI_Status mpi_stat; /* Status from I/O operation */ + H5FD_mpio_xfer_t xfer_mode; /* I/O transfer mode */ + H5FD_mpio_collective_opt_t coll_opt_mode; /* whether we are doing collective or independent I/O */ + int size_i; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + +#ifdef H5FDmpio_DEBUG + if (H5FD_mpio_Debug[(int)'t']) + HDfprintf(stdout, "%s: Entering\n", FUNC); +#endif + + /* Sanity checks */ + HDassert(file); + HDassert(H5FD_MPIO == file->pub.driver_id); + HDassert((types) || (count == 0)); + HDassert((addrs) || (count == 0)); + HDassert((sizes) || (count == 0)); + HDassert((bufs) || (count == 0)); + + /* verify that the first elements of the sizes and types arrays are + * valid. + */ + HDassert((count == 0) || (sizes[0] != 0)); + HDassert((count == 0) || (types[0] != H5FD_MEM_NOLIST)); + + + /* sort the vector I/O request into increasing address order if required + * + * If the vector is already sorted, the base addresses of types, addrs, sizes, + * and bufs will be returned in s_types, s_addrs, s_sizes, and s_bufs respectively. + * + * If the vector was not already sorted, new, sorted versions of types, addrs, sizes, and bufs + * are allocated, populated, and returned in s_types, s_addrs, s_sizes, and s_bufs respectively. + * In this case, this function must free the memory allocated for the sorted vectors. + */ + if (H5FD_sort_vector_io_req(&vector_was_sorted, count, types, addrs, sizes, bufs, &s_types, &s_addrs, + &s_sizes, &s_bufs) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "can't sort vector I/O request") + + + /* Get the transfer mode from the API context + * + * This flag is set to H5FD_MPIO_COLLECTIVE if the API call is + * collective, and to H5FD_MPIO_INDEPENDENT if it is not. + * + * While this doesn't mean that we are actually about to do a collective + * read, it does mean that all ranks are here, so we can use MPI_File_set_view(). + */ + if (H5CX_get_io_xfer_mode(&xfer_mode) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't get MPI-I/O transfer mode") + + if (xfer_mode == H5FD_MPIO_COLLECTIVE) { + + if (count > 0) { /* create MPI derived types describing the vector write */ + + if ((NULL == (mpi_block_lengths = (int *)HDmalloc((size_t)count * sizeof(int)))) || + (NULL == (mpi_displacments = (MPI_Aint *)HDmalloc((size_t)count * sizeof(MPI_Aint)))) || + (NULL == (mpi_bufs = (MPI_Aint *)HDmalloc((size_t)count * sizeof(MPI_Aint))))) { + + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't alloc mpi block lengths / displacement") + } + + /* when we setup mpi_bufs[] below, all addresses are offsets from + * mpi_bufs_base. + * + * Since these offsets must all be positive, we must scan through + * s_bufs[] to find the smallest value, and choose that for + * mpi_bufs_base. + */ + + j = 0; /* guess at the index of the smallest value of s_bufs[] */ + + for (i = 1; i < (int)count; i++) { + + if (s_bufs[i] < s_bufs[j]) { + + j = i; + } + } + + mpi_bufs_base = s_bufs[j]; + + if (MPI_SUCCESS != (mpi_code = MPI_Get_address(mpi_bufs_base, &mpi_bufs_base_Aint))) + + HMPI_GOTO_ERROR(FAIL, "MPI_Get_address for s_bufs[] to mpi_bufs_base failed", mpi_code) + + size_i = 1; + + fixed_size = FALSE; + + /* load the mpi_block_lengths and mpi_displacements arrays */ + for (i = 0; i < (int)count; i++) { + + if (!fixed_size) { + + if (sizes[i] == 0) { + + fixed_size = TRUE; + size = sizes[i - 1]; + } + else { + + size = s_sizes[i]; + } + } + + /* There is an obvious possibility of an overflow here, as size_t + * will typically be 64 bits, where as int will typically be 32 bits. + * This must be fixed, but it should be good enough for initial + * correctness testing. + * JRM -- 3/17/21 + */ + mpi_block_lengths[i] = (int)size; + mpi_displacments[i] = (MPI_Aint)s_addrs[i]; + + /* convert s_bufs[i] to MPI_Aint... */ + if (MPI_SUCCESS != (mpi_code = MPI_Get_address(s_bufs[i], &(mpi_bufs[i])))) + + HMPI_GOTO_ERROR(FAIL, "MPI_Get_address for s_bufs[] - mpi_bufs_base failed", mpi_code) + + /*... and then subtract mpi_bufs_base_Aint from it. */ +#if ((MPI_VERSION > 3) || ((MPI_VERSION == 3) && (MPI_SUBVERSION >= 1))) + mpi_bufs[i] = MPI_Aint_diff(mpi_bufs[i], mpi_bufs_base_Aint); +#else + mpi_bufs[i] = mpi_bufs[i] - mpi_bufs_base_Aint; +#endif + } + + + /* create the memory MPI derived types */ + if (MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int)count, mpi_block_lengths, mpi_bufs, + MPI_BYTE, &buf_type))) + + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed for buf_type failed", mpi_code) + + buf_type_created = TRUE; + + if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(&buf_type))) + + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit for buf_type failed", mpi_code) + + + /* create the file MPI derived type */ + if (MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int)count, mpi_block_lengths, + mpi_displacments, MPI_BYTE, &file_type))) + + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed for file_type failed", mpi_code) + + file_type_created = TRUE; + + if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(&file_type))) + + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit for file_type failed", mpi_code) + } + else { + + /* setup for null participation in the collective operation. */ + + buf_type = MPI_BYTE; + file_type = MPI_BYTE; + + /* Set non-NULL pointer for I/O operation */ + mpi_bufs_base = (void *)(&unused); + + /* MPI count to read */ + size_i = 0; + } + + /* Portably initialize MPI status variable */ + HDmemset(&mpi_stat, 0, sizeof(MPI_Status)); + + /* some numeric conversions */ + if (H5FD_mpi_haddr_to_MPIOff((haddr_t)0, &mpi_off) < 0) + HGOTO_ERROR(H5E_INTERNAL, H5E_BADRANGE, FAIL, "can't set MPI off to 0") + +#ifdef H5FDmpio_DEBUG + if (H5FD_mpio_Debug[(int)'r']) + HDfprintf(stdout, "%s: mpi_off = %ld size_i = %d\n", FUNC, (long)mpi_off, size_i); +#endif + + /* Setup the file view. */ + if (MPI_SUCCESS != (mpi_code = MPI_File_set_view(file->f, mpi_off, MPI_BYTE, file_type, + H5FD_mpi_native_g, file->info))) + HMPI_GOTO_ERROR(FAIL, "MPI_File_set_view failed", mpi_code) + + /* Get the collective_opt property to check whether the application wants to do IO individually. + */ + if (H5CX_get_mpio_coll_opt(&coll_opt_mode) < 0) + + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't get MPI-I/O collective_op property") + + /* Read the data. */ +#ifdef H5FDmpio_DEBUG + if (H5FD_mpio_Debug[(int)'r']) + HDfprintf(stdout, "%s: using MPIO collective mode\n", FUNC); +#endif + + if (coll_opt_mode == H5FD_MPIO_COLLECTIVE_IO) { +#ifdef H5FDmpio_DEBUG + if (H5FD_mpio_Debug[(int)'r']) + HDfprintf(stdout, "%s: doing MPI collective IO\n", FUNC); +#endif + + if (MPI_SUCCESS != (mpi_code = MPI_File_read_at_all(file->f, mpi_off, mpi_bufs_base, size_i, + buf_type, &mpi_stat))) + HMPI_GOTO_ERROR(FAIL, "MPI_File_read_at_all failed", mpi_code) + } /* end if */ + else if (size_i > 0) { +#ifdef H5FDmpio_DEBUG + if (H5FD_mpio_Debug[(int)'r']) + HDfprintf(stdout, "%s: doing MPI independent IO\n", FUNC); +#endif + + if (MPI_SUCCESS != + (mpi_code = MPI_File_read_at(file->f, mpi_off, mpi_bufs_base, size_i, buf_type, &mpi_stat))) + + HMPI_GOTO_ERROR(FAIL, "MPI_File_read_at failed", mpi_code) + + } /* end else */ + + /* Reset the file view */ + if (MPI_SUCCESS != (mpi_code = MPI_File_set_view(file->f, (MPI_Offset)0, MPI_BYTE, MPI_BYTE, + H5FD_mpi_native_g, file->info))) + HMPI_GOTO_ERROR(FAIL, "MPI_File_set_view failed", mpi_code) + } + else if (count > 0) { + + /* The read is part of an independent operation. As a result, + * we can't use MPI_File_set_view() (since it it a collective operation), + * and thus there is no point in setting up an MPI derived type, as + * (to the best of my knowlege) MPI I/O doesn't have support for + * non-contiguous I/O in independent mode. + * + * Thus we have to read in each element of the vector in a separate + * MPI_File_read_at() call. + */ + + fixed_size = FALSE; + +#ifdef H5FDmpio_DEBUG + if (H5FD_mpio_Debug[(int)'r']) + HDfprintf(stdout, "%s: doing MPI independent IO\n", FUNC); +#endif + + for (i = 0; i < (int)count; i++) { + + if (H5FD_mpi_haddr_to_MPIOff(s_addrs[i], &mpi_off) < 0) + + HGOTO_ERROR(H5E_INTERNAL, H5E_BADRANGE, FAIL, "can't convert from haddr to MPI off") + + if (!fixed_size) { + + if (sizes[i] == 0) { + + fixed_size = TRUE; + size = sizes[i - 1]; + } + else { + + size = s_sizes[i]; + } + } + + size_i = (int)size; /* todo: fix potential for overflow */ + + if (MPI_SUCCESS != + (mpi_code = MPI_File_read_at(file->f, mpi_off, s_bufs[i], size_i, MPI_BYTE, &mpi_stat))) + + HMPI_GOTO_ERROR(FAIL, "MPI_File_read_at failed", mpi_code) + } + } + +done: + + if (!vector_was_sorted) { /* free sorted vectors if they exist */ + + if (s_types) { + + HDfree(s_types); + s_types = NULL; + } + + if (s_addrs) { + + HDfree(s_addrs); + s_addrs = NULL; + } + + if (s_sizes) { + + HDfree(s_sizes); + s_sizes = NULL; + } + + if (s_bufs) { + + HDfree(s_bufs); + s_bufs = NULL; + } + } + + if (mpi_block_lengths) { + + HDfree(mpi_block_lengths); + mpi_block_lengths = NULL; + } + + if (mpi_displacments) { + + HDfree(mpi_displacments); + mpi_displacments = NULL; + } + + if (mpi_bufs) { + + HDfree(mpi_bufs); + mpi_bufs = NULL; + } + + if (buf_type_created) { + MPI_Type_free(&buf_type); + } + + if (file_type_created) { + MPI_Type_free(&file_type); + } + +#ifdef H5FDmpio_DEBUG + if (H5FD_mpio_Debug[(int)'t']) + HDfprintf(stdout, "%s: Leaving, proc %d: ret_value = %d\n", FUNC, file->mpi_rank, ret_value); +#endif + + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5FD__mpio_read_vector() */ + +/*------------------------------------------------------------------------- + * Function: H5FD__mpio_write_vector + * + * Purpose: The behaviour of this function dependes on the value of + * the io_xfer_mode obtained from the context. + * + * If it is H5FD_MPIO_COLLECTIVE, this is a collective + * operation, which allows us to use MPI_File_set_view, and + * then perform the entire vector write in a single MPI call. + * + * Do this (if count is positive), by constructing memory + * and file derived types from the supplied vector, using + * file type to set the file view, and then writing the + * the memory type to file. Note that this write is + * either independent or collective depending on the + * value of mpio_coll_opt -- again obtained from the context. + * + * If count is zero, participate in the collective write + * (if so configured) with an empty write. + * + * Finally, set the file view back to its default state. + * + * In contrast, if io_xfer_mode is H5FD_MPIO_INDEPENDENT, + * this call is independent, and thus we cannot use + * MPI_File_set_view(). + * + * In this case, simply walk the vector, and issue an + * independent write for each entry. + * + * WARNING: At present, this function makes no provision + * entries of size greater than 2 GB in the vector. This + * will have to be fixed before release. + * + * Return: Success: SUCCEED. + * Failure: FAIL. + * + * Programmer: John Mainzer + * March 15, 2021 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__mpio_write_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t count, H5FD_mem_t types[], + haddr_t addrs[], size_t sizes[], void *bufs[]) +{ + H5FD_mpio_t * file = (H5FD_mpio_t *)_file; + hbool_t vector_was_sorted = TRUE; + hbool_t fixed_size = FALSE; + size_t size; + H5FD_mem_t * s_types = NULL; + haddr_t * s_addrs = NULL; + size_t * s_sizes = NULL; + void ** s_bufs = NULL; + int * mpi_block_lengths = NULL; + char unused = 0; /* Unused, except for non-NULL pointer value */ + void * mpi_bufs_base = NULL; + MPI_Aint mpi_bufs_base_Aint; + MPI_Aint * mpi_bufs = NULL; + MPI_Aint * mpi_displacments = NULL; + MPI_Datatype buf_type = MPI_BYTE; /* MPI description of the selection in memory */ + hbool_t buf_type_created = FALSE; + MPI_Datatype file_type = MPI_BYTE; /* MPI description of the selection in file */ + hbool_t file_type_created = FALSE; + int i; + int j; + int mpi_code; /* MPI return code */ + MPI_Offset mpi_off = 0; + MPI_Status mpi_stat; /* Status from I/O operation */ + H5FD_mpio_xfer_t xfer_mode; /* I/O transfer mode */ + H5FD_mpio_collective_opt_t coll_opt_mode; /* whether we are doing collective or independent I/O */ + int size_i; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + +#ifdef H5FDmpio_DEBUG + if (H5FD_mpio_Debug[(int)'t']) + HDfprintf(stdout, "%s: Entering\n", FUNC); +#endif + + /* Sanity checks */ + HDassert(file); + HDassert(H5FD_MPIO == file->pub.driver_id); + HDassert((types) || (count == 0)); + HDassert((addrs) || (count == 0)); + HDassert((sizes) || (count == 0)); + HDassert((bufs) || (count == 0)); + + /* verify that the first elements of the sizes and types arrays are + * valid. + */ + HDassert((count == 0) || (sizes[0] != 0)); + HDassert((count == 0) || (types[0] != H5FD_MEM_NOLIST)); + + + /* Verify that no data is written when between MPI_Barrier()s during file flush */ + + HDassert(!H5CX_get_mpi_file_flushing()); + + /* sort the vector I/O request into increasing address order if required + * + * If the vector is already sorted, the base addresses of types, addrs, sizes, + * and bufs will be returned in s_types, s_addrs, s_sizes, and s_bufs respectively. + * + * If the vector was not already sorted, new, sorted versions of types, addrs, sizes, and bufs + * are allocated, populated, and returned in s_types, s_addrs, s_sizes, and s_bufs respectively. + * In this case, this function must free the memory allocated for the sorted vectors. + */ + if (H5FD_sort_vector_io_req(&vector_was_sorted, count, types, addrs, sizes, bufs, &s_types, &s_addrs, + &s_sizes, &s_bufs) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "can't sort vector I/O request") + + + /* Get the transfer mode from the API context + * + * This flag is set to H5FD_MPIO_COLLECTIVE if the API call is + * collective, and to H5FD_MPIO_INDEPENDENT if it is not. + * + * While this doesn't mean that we are actually about to do a collective + * write, it does mean that all ranks are here, so we can use MPI_File_set_view(). + */ + if (H5CX_get_io_xfer_mode(&xfer_mode) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't get MPI-I/O transfer mode") + + if (xfer_mode == H5FD_MPIO_COLLECTIVE) { + + if (count > 0) { /* create MPI derived types describing the vector write */ + + if ((NULL == (mpi_block_lengths = (int *)HDmalloc((size_t)count * sizeof(int)))) || + (NULL == (mpi_displacments = (MPI_Aint *)HDmalloc((size_t)count * sizeof(MPI_Aint)))) || + (NULL == (mpi_bufs = (MPI_Aint *)HDmalloc((size_t)count * sizeof(MPI_Aint))))) { + + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't alloc mpi block lengths / displacement") + } + + /* when we setup mpi_bufs[] below, all addresses are offsets from + * mpi_bufs_base. + * + * Since these offsets must all be positive, we must scan through + * s_bufs[] to find the smallest value, and choose that for + * mpi_bufs_base. + */ + + j = 0; /* guess at the index of the smallest value of s_bufs[] */ + + for (i = 1; i < (int)count; i++) { + + if (s_bufs[i] < s_bufs[j]) { + + j = i; + } + } + + mpi_bufs_base = s_bufs[j]; + + if (MPI_SUCCESS != (mpi_code = MPI_Get_address(mpi_bufs_base, &mpi_bufs_base_Aint))) + + HMPI_GOTO_ERROR(FAIL, "MPI_Get_address for s_bufs[] to mpi_bufs_base failed", mpi_code) + + size_i = 1; + + fixed_size = FALSE; + + /* load the mpi_block_lengths and mpi_displacements arrays */ + for (i = 0; i < (int)count; i++) { + + if (!fixed_size) { + + if (sizes[i] == 0) { + + fixed_size = TRUE; + size = sizes[i - 1]; + } + else { + + size = s_sizes[i]; + } + } + + /* There is an obvious possibility of an overflow here, as size_t + * will typically be 64 bits, where as int will typically be 32 bits. + * This must be fixed, but it should be good enough for initial + * correctness testing. + * JRM -- 3/17/21 + */ + mpi_block_lengths[i] = (int)size; + mpi_displacments[i] = (MPI_Aint)s_addrs[i]; + + /* convert s_bufs[i] to MPI_Aint... */ + if (MPI_SUCCESS != (mpi_code = MPI_Get_address(s_bufs[i], &(mpi_bufs[i])))) + + HMPI_GOTO_ERROR(FAIL, "MPI_Get_address for s_bufs[] - mpi_bufs_base failed", mpi_code) + + /*... and then subtract mpi_bufs_base_Aint from it. */ +#if ((MPI_VERSION > 3) || ((MPI_VERSION == 3) && (MPI_SUBVERSION >= 1))) + mpi_bufs[i] = MPI_Aint_diff(mpi_bufs[i], mpi_bufs_base_Aint); +#else + mpi_bufs[i] = mpi_bufs[i] - mpi_bufs_base_Aint; +#endif + } + + + /* create the memory MPI derived types */ + if (MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int)count, mpi_block_lengths, mpi_bufs, + MPI_BYTE, &buf_type))) + + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed for buf_type failed", mpi_code) + + buf_type_created = TRUE; + + if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(&buf_type))) + + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit for buf_type failed", mpi_code) + + /* create the file MPI derived type */ + if (MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int)count, mpi_block_lengths, + mpi_displacments, MPI_BYTE, &file_type))) + + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed for file_type failed", mpi_code) + + file_type_created = TRUE; + + if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(&file_type))) + + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit for file_type failed", mpi_code) + } + else { + + /* setup for null participation in the collective operation. */ + + buf_type = MPI_BYTE; + file_type = MPI_BYTE; + + /* Set non-NULL pointer for I/O operation */ + mpi_bufs_base = (void *)(&unused); + + /* MPI count to write */ + size_i = 0; + } + + /* Portably initialize MPI status variable */ + HDmemset(&mpi_stat, 0, sizeof(MPI_Status)); + + /* some numeric conversions */ + if (H5FD_mpi_haddr_to_MPIOff((haddr_t)0, &mpi_off) < 0) + HGOTO_ERROR(H5E_INTERNAL, H5E_BADRANGE, FAIL, "can't set MPI off to 0") + +#ifdef H5FDmpio_DEBUG + if (H5FD_mpio_Debug[(int)'w']) + HDfprintf(stdout, "%s: mpi_off = %ld size_i = %d\n", FUNC, (long)mpi_off, size_i); +#endif + + /* Setup the file view. */ + if (MPI_SUCCESS != (mpi_code = MPI_File_set_view(file->f, mpi_off, MPI_BYTE, file_type, + H5FD_mpi_native_g, file->info))) + HMPI_GOTO_ERROR(FAIL, "MPI_File_set_view failed", mpi_code) + + /* Get the collective_opt property to check whether the application wants to do IO individually. + */ + if (H5CX_get_mpio_coll_opt(&coll_opt_mode) < 0) + + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't get MPI-I/O collective_op property") + + /* Write the data. */ +#ifdef H5FDmpio_DEBUG + if (H5FD_mpio_Debug[(int)'w']) + HDfprintf(stdout, "%s: using MPIO collective mode\n", FUNC); +#endif + + if (coll_opt_mode == H5FD_MPIO_COLLECTIVE_IO) { +#ifdef H5FDmpio_DEBUG + if (H5FD_mpio_Debug[(int)'w']) + HDfprintf(stdout, "%s: doing MPI collective IO\n", FUNC); +#endif + + if (MPI_SUCCESS != (mpi_code = MPI_File_write_at_all(file->f, mpi_off, mpi_bufs_base, size_i, + buf_type, &mpi_stat))) + HMPI_GOTO_ERROR(FAIL, "MPI_File_write_at_all failed", mpi_code) + } /* end if */ + else if (size_i > 0) { +#ifdef H5FDmpio_DEBUG + if (H5FD_mpio_Debug[(int)'w']) + HDfprintf(stdout, "%s: doing MPI independent IO\n", FUNC); +#endif + + if (MPI_SUCCESS != + (mpi_code = MPI_File_write_at(file->f, mpi_off, mpi_bufs_base, size_i, buf_type, &mpi_stat))) + HMPI_GOTO_ERROR(FAIL, "MPI_File_write_at failed", mpi_code) + } /* end else */ + + /* Reset the file view */ + if (MPI_SUCCESS != (mpi_code = MPI_File_set_view(file->f, (MPI_Offset)0, MPI_BYTE, MPI_BYTE, + H5FD_mpi_native_g, file->info))) + HMPI_GOTO_ERROR(FAIL, "MPI_File_set_view failed", mpi_code) + } + else if (count > 0) { + + /* The write is part of an independent operation. As a result, + * we can't use MPI_File_set_view() (since it it a collective operation), + * and thus there is no point in setting up an MPI derived type, as + * (to the best of my knowlege) MPI I/O doesn't have support for + * non-contiguous I/O in independent mode. + * + * Thus we have to write out each element of the vector in a separate + * MPI_File_write_at() call. + */ + + fixed_size = FALSE; + +#ifdef H5FDmpio_DEBUG + if (H5FD_mpio_Debug[(int)'w']) + HDfprintf(stdout, "%s: doing MPI independent IO\n", FUNC); +#endif + + for (i = 0; i < (int)count; i++) { + + if (H5FD_mpi_haddr_to_MPIOff(s_addrs[i], &mpi_off) < 0) + + HGOTO_ERROR(H5E_INTERNAL, H5E_BADRANGE, FAIL, "can't convert from haddr to MPI off") + + if (!fixed_size) { + + if (sizes[i] == 0) { + + fixed_size = TRUE; + size = sizes[i - 1]; + } + else { + + size = s_sizes[i]; + } + } + + size_i = (int)size; /* todo: fix potential for overflow */ + + if (MPI_SUCCESS != + (mpi_code = MPI_File_write_at(file->f, mpi_off, s_bufs[i], size_i, MPI_BYTE, &mpi_stat))) + + HMPI_GOTO_ERROR(FAIL, "MPI_File_write_at failed", mpi_code) + } + } + + /* Each process will keep track of its perceived EOF value locally, and + * ultimately we will reduce this value to the maximum amongst all + * processes, but until then keep the actual eof at HADDR_UNDEF just in + * case something bad happens before that point. (rather have a value + * we know is wrong sitting around rather than one that could only + * potentially be wrong.) + */ + file->eof = HADDR_UNDEF; + + /* check to see if the local eof has changed been extended, and update if so. + * Since the vector write request has been sorted in increasing address order, + * we need only look at the address and size of the last element in the vector. + */ + if ((count > 0) && ((s_addrs[count - 1] + (haddr_t)(s_sizes[count - 1])) > file->local_eof)) { + + file->local_eof = (s_addrs[count - 1] + (haddr_t)(s_sizes[count - 1])); + } + +done: + + if (!vector_was_sorted) { /* free sorted vectors if they exist */ + + if (s_types) { + + HDfree(s_types); + s_types = NULL; + } + + if (s_addrs) { + + HDfree(s_addrs); + s_addrs = NULL; + } + + if (s_sizes) { + + HDfree(s_sizes); + s_sizes = NULL; + } + + if (s_bufs) { + + HDfree(s_bufs); + s_bufs = NULL; + } + } + + if (mpi_block_lengths) { + + HDfree(mpi_block_lengths); + mpi_block_lengths = NULL; + } + + if (mpi_displacments) { + + HDfree(mpi_displacments); + mpi_displacments = NULL; + } + + if (mpi_bufs) { + + HDfree(mpi_bufs); + mpi_bufs = NULL; + } + + if (buf_type_created) { + MPI_Type_free(&buf_type); + } + + if (file_type_created) { + MPI_Type_free(&file_type); + } + +#ifdef H5FDmpio_DEBUG + if (H5FD_mpio_Debug[(int)'t']) + HDfprintf(stdout, "%s: Leaving, proc %d: ret_value = %d\n", FUNC, file->mpi_rank, ret_value); +#endif + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__mpio_write_vector() */ + +/*------------------------------------------------------------------------- * Function: H5FD__mpio_flush * * Purpose: Makes sure that all data is on disk. This is collective. diff --git a/src/H5FDprivate.h b/src/H5FDprivate.h index d485ea9..82f51e8 100644 --- a/src/H5FDprivate.h +++ b/src/H5FDprivate.h @@ -145,6 +145,10 @@ H5_DLL herr_t H5FD_set_base_addr(H5FD_t *file, haddr_t base_addr); H5_DLL haddr_t H5FD_get_base_addr(const H5FD_t *file); H5_DLL herr_t H5FD_set_paged_aggr(H5FD_t *file, hbool_t paged); +H5_DLL herr_t H5FD_sort_vector_io_req(hbool_t *vector_was_sorted, uint32_t count, H5FD_mem_t types[], + haddr_t addrs[], size_t sizes[], void *bufs[], H5FD_mem_t **s_types_ptr, + haddr_t **s_addrs_ptr, size_t **s_sizes_ptr, void ***s_bufs_ptr); + /* Function prototypes for MPI based VFDs*/ #ifdef H5_HAVE_PARALLEL /* General routines */ diff --git a/testpar/Makefile.am b/testpar/Makefile.am index 6a8cc2b..cbde0c1 100644 --- a/testpar/Makefile.am +++ b/testpar/Makefile.am @@ -30,7 +30,7 @@ check_SCRIPTS = $(TEST_SCRIPT_PARA) # Test programs. These are our main targets. # -TEST_PROG_PARA=t_mpi t_bigio testphdf5 t_cache t_cache_image t_pread t_pshutdown t_prestart t_init_term t_shapesame t_filters_parallel t_2Gio +TEST_PROG_PARA=t_mpi t_bigio testphdf5 t_cache t_cache_image t_pread t_pshutdown t_prestart t_init_term t_shapesame t_filters_parallel t_2Gio t_vfd # t_pflush1 and t_pflush2 are used by testpflush.sh check_PROGRAMS = $(TEST_PROG_PARA) t_pflush1 t_pflush2 diff --git a/testpar/t_vfd.c b/testpar/t_vfd.c new file mode 100644 index 0000000..880cc96 --- /dev/null +++ b/testpar/t_vfd.c @@ -0,0 +1,4057 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* Programmer: John Mainzer + * 7/13/15 + * + * This file is a catchall for parallel VFD tests. + */ + +#include "testphdf5.h" + +/* Must be a power of 2. Reducing it below 1024 may cause problems */ +#define INTS_PER_RANK 1024 + +/* global variable declarations: */ + +hbool_t pass = TRUE; /* set to FALSE on error */ +const char *failure_mssg = NULL; + +const char *FILENAMES[] = {"mpio_vfd_test_file_0", /*0*/ + "mpio_vfd_test_file_1", /*1*/ + "mpio_vfd_test_file_2", /*2*/ + "mpio_vfd_test_file_3", /*3*/ + "mpio_vfd_test_file_4", /*4*/ + "mpio_vfd_test_file_5", /*5*/ + NULL}; + +/* File Test Images + * + * Pointers to dynamically allocated buffers of size + * INTS_PER_RANK * sizeof(int32_t) * mpi_size(). These + * buffers are used to put the test file in a known + * state, and to test if the test file contains the + * expected data. + */ + +int32_t *increasing_fi_buf = NULL; +int32_t *decreasing_fi_buf = NULL; +int32_t *negative_fi_buf = NULL; +int32_t *zero_fi_buf = NULL; +int32_t *read_fi_buf = NULL; + +/* local utility function declarations */ + +static unsigned alloc_and_init_file_images(int mpi_size); +static void free_file_images(void); +static void setup_vfd_test_file(int file_name_id, char *file_name, int mpi_size, H5FD_mpio_xfer_t xfer_mode, + H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name, haddr_t eoa, + H5FD_t **lf_ptr, hid_t *fapl_id_ptr, hid_t *dxpl_id_ptr); +static void takedown_vfd_test_file(int mpi_rank, char *filename, H5FD_t **lf_ptr, hid_t *fapl_id_ptr, + hid_t *dxpl_id_ptr); + +/* test functions */ +static unsigned vector_read_test_1(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode, + H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name); +static unsigned vector_read_test_2(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode, + H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name); +static unsigned vector_read_test_3(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode, + H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name); +static unsigned vector_read_test_4(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode, + H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name); +static unsigned vector_read_test_5(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode, + H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name); + +static unsigned vector_write_test_1(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode, + H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name); +static unsigned vector_write_test_2(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode, + H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name); +static unsigned vector_write_test_3(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode, + H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name); +static unsigned vector_write_test_4(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode, + H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name); +static unsigned vector_write_test_5(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode, + H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name); +static unsigned vector_write_test_6(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode, + H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name); + +/****************************************************************************/ +/***************************** Utility Functions ****************************/ +/****************************************************************************/ + +/*------------------------------------------------------------------------- + * Function: alloc_and_init_file_images + * + * Purpose: Allocate and initialize the global buffers used to construct, + * load and verify test file contents. + * + * Return: void + * + * Programmer: John Mainzer + * 3/25/26 + * + * Modifications: + * + * None. + * + *------------------------------------------------------------------------- + */ + +static unsigned +alloc_and_init_file_images(int mpi_size) +{ + const char *fcn_name = "alloc_and_init_file_images()"; + int cp = 0; + int buf_len; + size_t buf_size; + int i; + hbool_t show_progress = FALSE; + + pass = TRUE; + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* allocate the file image buffers */ + if (pass) { + + buf_len = INTS_PER_RANK * mpi_size; + buf_size = sizeof(int32_t) * (size_t)INTS_PER_RANK * (size_t)mpi_size; + + increasing_fi_buf = (int32_t *)HDmalloc(buf_size); + decreasing_fi_buf = (int32_t *)HDmalloc(buf_size); + negative_fi_buf = (int32_t *)HDmalloc(buf_size); + zero_fi_buf = (int32_t *)HDmalloc(buf_size); + read_fi_buf = (int32_t *)HDmalloc(buf_size); + + if ((!increasing_fi_buf) || (!decreasing_fi_buf) || (!negative_fi_buf) || (!zero_fi_buf) || + (!read_fi_buf)) { + + pass = FALSE; + failure_mssg = "Can't allocate one or more file image buffers."; + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* initialize the file image buffers */ + if (pass) { + + for (i = 0; i < buf_len; i++) { + + increasing_fi_buf[i] = i; + decreasing_fi_buf[i] = buf_len - i; + negative_fi_buf[i] = -i; + zero_fi_buf[i] = 0; + read_fi_buf[i] = 0; + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* discard file image buffers if there was an error */ + if (!pass) { + + free_file_images(); + } + + return !pass; + +} /* alloc_and_init_file_images() */ + +/*------------------------------------------------------------------------- + * Function: free_file_images + * + * Purpose: Deallocate any glogal file image buffers that exist, and + * set their associated pointers to NULL. + * + * Return: void + * + * Programmer: John Mainzer + * 1/25/17 + * + * Modifications: + * + * None. + * + *------------------------------------------------------------------------- + */ + +static void +free_file_images(void) +{ + if (increasing_fi_buf) { + + HDfree(increasing_fi_buf); + increasing_fi_buf = NULL; + } + + if (decreasing_fi_buf) { + + HDfree(decreasing_fi_buf); + decreasing_fi_buf = NULL; + } + + if (negative_fi_buf) { + + HDfree(negative_fi_buf); + negative_fi_buf = NULL; + } + + if (zero_fi_buf) { + + HDfree(zero_fi_buf); + zero_fi_buf = NULL; + } + + if (read_fi_buf) { + + HDfree(read_fi_buf); + read_fi_buf = NULL; + } + + return; + +} /* free_file_images() */ + +/*------------------------------------------------------------------------- + * Function: setup_vfd_test_file + * + * Purpose: Create / open the specified test file with the specified + * VFD, and set the EOA to the specified value. + * + * Setup the dxpl for subsequent I/O via the target VFD. + * + * Return a pointer to the instance of H5FD_t created on + * file open in *lf_ptr, and the FAPL and DXPL ids in + * *fapl_id_ptr and *dxpl_id_ptr. Similarly, copy the + * "fixed" file name into file_name on exit. + * + * Return: void + * + * Programmer: John Mainzer + * 3/25/26 + * + * Modifications: + * + * None. + * + *------------------------------------------------------------------------- + */ + +static void +setup_vfd_test_file(int file_name_id, char *file_name, int mpi_size, H5FD_mpio_xfer_t xfer_mode, + H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name, haddr_t eoa, + H5FD_t **lf_ptr, hid_t *fapl_id_ptr, hid_t *dxpl_id_ptr) +{ + const char *fcn_name = "setup_vfd_test_file()"; + char filename[512]; + int cp = 0; + hbool_t show_progress = FALSE; + hid_t fapl_id = -1; /* file access property list ID */ + hid_t dxpl_id = -1; /* data access property list ID */ + unsigned flags = 0; /* file open flags */ + H5FD_t * lf = NULL; /* VFD struct ptr */ + + HDassert(vfd_name); + HDassert(lf_ptr); + HDassert(fapl_id_ptr); + HDassert(dxpl_id_ptr); + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* setupf fapl for target VFD */ + if (pass) { + + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) { + + pass = FALSE; + failure_mssg = "Can't create fapl."; + } + } + + if (pass) { + + if (strcmp(vfd_name, "mpio") == 0) { + + if (H5Pset_fapl_mpio(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL) < 0) { + + pass = FALSE; + failure_mssg = "Can't set mpio fapl."; + } + } + else { + + pass = FALSE; + failure_mssg = "un-supported VFD"; + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* setup the file name */ + if (pass) { + + if (h5_fixname(FILENAMES[file_name_id], H5P_DEFAULT, filename, sizeof(filename)) == NULL) { + + pass = FALSE; + failure_mssg = "h5_fixname() failed.\n"; + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* Open the VFD test file with the specified VFD. */ + + if (pass) { + + flags = H5F_ACC_RDWR | H5F_ACC_CREAT | H5F_ACC_TRUNC; + + if (NULL == (lf = H5FDopen(filename, flags, fapl_id, HADDR_UNDEF))) { + + pass = FALSE; + failure_mssg = "H5FDopen() failed.\n"; + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* set eoa as specified */ + + if (pass) { + + eoa = (haddr_t)mpi_size * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t)); + + if (H5FDset_eoa(lf, H5FD_MEM_DEFAULT, eoa) < 0) { + + pass = FALSE; + failure_mssg = "H5FDset_eoa() failed.\n"; + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if (pass) { /* setup dxpl */ + + dxpl_id = H5Pcreate(H5P_DATASET_XFER); + + if (dxpl_id < 0) { + + pass = FALSE; + failure_mssg = "H5Pcreate(H5P_DATASET_XFER) failed."; + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if (pass) { + + if (H5Pset_dxpl_mpio(dxpl_id, xfer_mode) < 0) { + + pass = FALSE; + failure_mssg = "H5Pset_dxpl_mpio() failed."; + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if (pass) { + + if (H5Pset_dxpl_mpio_collective_opt(dxpl_id, coll_opt_mode) < 0) { + + pass = FALSE; + failure_mssg = "H5Pset_dxpl_mpio() failed."; + } + } + + if (pass) { /* setup pointers with return values */ + + strncpy(file_name, filename, 512); + *lf_ptr = lf; + *fapl_id_ptr = fapl_id; + *dxpl_id_ptr = dxpl_id; + } + else { /* tidy up from failure as posible */ + + if (lf) + H5FDclose(lf); + + if (fapl_id != -1) + H5Pclose(fapl_id); + + if (dxpl_id != -1) + H5Pclose(dxpl_id); + } + + return; + +} /* setup_vfd_test_file() */ + +/*------------------------------------------------------------------------- + * Function: takedown_vfd_test_file + * + * Purpose: Close and delete the specified test file. Close the + * FAPL & DXPL. + * + * Return: void + * + * Programmer: John Mainzer + * 3/25/26 + * + * Modifications: + * + * None. + * + *------------------------------------------------------------------------- + */ + +static void +takedown_vfd_test_file(int mpi_rank, char *filename, H5FD_t **lf_ptr, hid_t *fapl_id_ptr, hid_t *dxpl_id_ptr) +{ + const char *fcn_name = "takedown_vfd_test_file()"; + int cp = 0; + hbool_t show_progress = FALSE; + + HDassert(lf_ptr); + HDassert(fapl_id_ptr); + HDassert(dxpl_id_ptr); + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* Close the test file if it is open, regardless of the value of pass. + * This should let the test program shut down more cleanly. + */ + + if (*lf_ptr) { + + if (H5FDclose(*lf_ptr) < 0) { + + pass = FALSE; + failure_mssg = "H5FDclose() failed.\n"; + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 6) On rank 0, delete the test file. + */ + + if (pass) { + + /* wait for everyone to close the file */ + MPI_Barrier(MPI_COMM_WORLD); + + if ((mpi_rank == 0) && (HDremove(filename) < 0)) { + + pass = FALSE; + failure_mssg = "HDremove() failed.\n"; + } + + /* wait for the file delete to complete */ + MPI_Barrier(MPI_COMM_WORLD); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* Close the fapl */ + if (H5Pclose(*fapl_id_ptr) < 0) { + + pass = FALSE; + failure_mssg = "can't close fapl.\n"; + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* Close the dxpl */ + if (H5Pclose(*dxpl_id_ptr) < 0) { + + pass = FALSE; + failure_mssg = "can't close dxpl.\n"; + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + return; + +} /* takedown_vfd_test_file() */ + +/****************************************************************************/ +/******************************* Test Functions *****************************/ +/****************************************************************************/ + +/*------------------------------------------------------------------------- + * Function: vector_read_test_1() + * + * Purpose: Simple vector read test: + * + * 1) Open the test file with the specified VFD, set the eoa, + * and setup the DXPL. + * + * 2) Using rank zero, write the entire increasing_fi_buf to + * the file. + * + * 3) Barrier + * + * 4) On each rank, zero the read buffer, and then read + * INTS_PER_RANK * sizeof(int32) bytes from the file + * starting at offset mpi_rank * INTS_PER_RANK * + * sizeof(int32_t) in both the file and read_fi_buf. + * Do this with a vector read containing a single + * element. + * + * Verify that read_fi_buf contains zeros for all + * indices less than mpi_rank * INTS_PER_RANK, or + * greater than or equal to (mpi_rank + 1) * INTS_PER_RANK. + * For all other indicies, read_fi_buf should equal + * increasing_fi_buf. + * + * 5) Barrier + * + * 6) Close the test file. + * + * 7) On rank 0, delete the test file. + * + * Return: FALSE on success, TRUE if any errors are detected. + * + * Programmer: John Mainzer + * 3/26/21 + * + * Modifications: + * + * None. + * + *------------------------------------------------------------------------- + */ + +static unsigned +vector_read_test_1(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode, + H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name) +{ + const char *fcn_name = "vector_read_test_1()"; + char test_title[120]; + char filename[512]; + haddr_t eoa; + hbool_t show_progress = FALSE; + hid_t fapl_id = -1; /* file access property list ID */ + hid_t dxpl_id = -1; /* data access property list ID */ + H5FD_t * lf = NULL; /* VFD struct ptr */ + int cp = 0; + int i; + uint32_t count; + H5FD_mem_t types[1]; + haddr_t addrs[1]; + size_t sizes[1]; + void * bufs[1]; + + pass = TRUE; + + if (mpi_rank == 0) { + + if (xfer_mode == H5FD_MPIO_INDEPENDENT) { + + sprintf(test_title, "parallel vector read test 1 -- %s / independent", vfd_name); + } + else if (coll_opt_mode == H5FD_MPIO_INDIVIDUAL_IO) { + + sprintf(test_title, "parallel vector read test 1 -- %s / col op / ind I/O", vfd_name); + } + else { + + HDassert(coll_opt_mode == H5FD_MPIO_COLLECTIVE_IO); + + sprintf(test_title, "parallel vector read test 1 -- %s / col op / col I/O", vfd_name); + } + + TESTING(test_title); + } + + show_progress = ((show_progress) && (mpi_rank == 0)); + + if (show_progress) + HDfprintf(stdout, "\n%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 1) Open the test file with the specified VFD, set the eoa, and setup the dxpl */ + if (pass) { + + eoa = (haddr_t)mpi_size * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t)); + + setup_vfd_test_file(file_name_id, filename, mpi_size, xfer_mode, coll_opt_mode, vfd_name, eoa, &lf, + &fapl_id, &dxpl_id); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 2) Using rank zero, write the entire increasing_fi_buf to + * the file. + */ + if (pass) { + + size_t image_size = (size_t)mpi_size * (size_t)INTS_PER_RANK * sizeof(int32_t); + + if (mpi_rank == 0) { + + if (H5FDwrite(lf, H5FD_MEM_DRAW, H5P_DEFAULT, (haddr_t)0, image_size, (void *)increasing_fi_buf) < + 0) { + + pass = FALSE; + failure_mssg = "H5FDwrite() on rank 0 failed.\n"; + } + } + } + + /* 3) Barrier */ + + if (pass) { + + MPI_Barrier(MPI_COMM_WORLD); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 4) On each rank, zero the read buffer, and then read + * INTS_PER_RANK * sizeof(int32) bytes from the file + * starting at offset mpi_rank * INTS_PER_RANK * + * sizeof(int32_t) in both the file and read_fi_buf. + * Do this with a vector read containing a single + * element. + * + * Verify that read_fi_buf contains zeros for all + * indices less than mpi_rank * INTS_PER_RANK, or + * greater than or equal to (mpi_rank + 1) * INTS_PER_RANK. + * For all other indicies, read_fi_buf should equal + * increasing_fi_buf. + */ + if (pass) { + + for (i = 0; i < mpi_size * INTS_PER_RANK; i++) { + + read_fi_buf[i] = 0; + } + + count = 1; + types[0] = H5FD_MEM_DRAW; + addrs[0] = (haddr_t)mpi_rank * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t)); + sizes[0] = (size_t)INTS_PER_RANK * sizeof(int32_t); + bufs[0] = (void *)(&(read_fi_buf[mpi_rank * INTS_PER_RANK])); + + if (H5FDread_vector(lf, dxpl_id, count, types, addrs, sizes, bufs) < 0) { + + pass = FALSE; + failure_mssg = "H5FDread_vector() failed.\n"; + } + + for (i = 0; i < mpi_size * INTS_PER_RANK; i++) { + + if ((i < mpi_rank * INTS_PER_RANK) || (i >= (mpi_rank + 1) * INTS_PER_RANK)) { + + if (read_fi_buf[i] != 0) { + + pass = FALSE; + failure_mssg = "Unexpected value in read_fi_buf (1).\n"; + break; + } + } + else { + + if (read_fi_buf[i] != increasing_fi_buf[i]) { + + pass = FALSE; + failure_mssg = "Unexpected value in read_fi_buf (2).\n"; + break; + } + } + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 5) Barrier */ + + if (pass) { + + MPI_Barrier(MPI_COMM_WORLD); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 6) Close the test file and delete it (on rank 0 only). + * Close FAPL and DXPL. + */ + + if (pass) { + + takedown_vfd_test_file(mpi_rank, filename, &lf, &fapl_id, &dxpl_id); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* report results */ + if (mpi_rank == 0) { + + if (pass) { + + PASSED(); + } + else { + + H5_FAILED(); + + if (show_progress) { + HDfprintf(stdout, "%s: failure_mssg = \"%s\"\n", fcn_name, failure_mssg); + } + } + } + + return (!pass); + +} /* vector_read_test_1() */ + +/*------------------------------------------------------------------------- + * Function: vector_read_test_2() + * + * Purpose: Simple vector read test with only half of ranks + * participating in each vector read. + * + * 1) Open the test file with the specified VFD, set the eoa, + * and setup the DXPL. + * + * 2) Using rank zero, write the entire decreasing_fi_buf to + * the file. + * + * 3) Barrier + * + * 4) On each rank, zero the read buffer. + * + * 5) On even ranks, read INTS_PER_RANK * sizeof(int32) + * bytes from the file starting at offset mpi_rank * + * INTS_PER_RANK * sizeof(int32_t) in both the file and + * read_fi_buf. Do this with a vector read containing + * a single element. + * + * Odd ranks perform an empty read. + * + * 6) Barrier. + * + * 7) On odd ranks, read INTS_PER_RANK * sizeof(int32) + * bytes from the file starting at offset mpi_rank * + * INTS_PER_RANK * sizeof(int32_t) in both the file and + * read_fi_buf. Do this with a vector read containing + * a single element. + * + * Even ranks perform an empty read. + * + * 8) Verify that read_fi_buf contains zeros for all + * indices less than mpi_rank * INTS_PER_RANK, or + * greater than or equal to (mpi_rank + 1) * INTS_PER_RANK. + * For all other indicies, read_fi_buf should equal + * decreasing_fi_buf. + * + * 9) Barrier + * + * 10) Close the test file. + * + * 11) On rank 0, delete the test file. + * + * Return: FALSE on success, TRUE if any errors are detected. + * + * Programmer: John Mainzer + * 3/26/21 + * + * Modifications: + * + * None. + * + *------------------------------------------------------------------------- + */ + +static unsigned +vector_read_test_2(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode, + H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name) +{ + const char *fcn_name = "vector_read_test_2()"; + char test_title[120]; + char filename[512]; + haddr_t eoa; + hbool_t show_progress = FALSE; + hid_t fapl_id = -1; /* file access property list ID */ + hid_t dxpl_id = -1; /* data access property list ID */ + H5FD_t * lf = NULL; /* VFD struct ptr */ + int cp = 0; + int i; + uint32_t count; + H5FD_mem_t types[1]; + haddr_t addrs[1]; + size_t sizes[1]; + void * bufs[1]; + + pass = TRUE; + + if (mpi_rank == 0) { + + if (xfer_mode == H5FD_MPIO_INDEPENDENT) { + + sprintf(test_title, "parallel vector read test 2 -- %s / independent", vfd_name); + } + else if (coll_opt_mode == H5FD_MPIO_INDIVIDUAL_IO) { + + sprintf(test_title, "parallel vector read test 2 -- %s / col op / ind I/O", vfd_name); + } + else { + + HDassert(coll_opt_mode == H5FD_MPIO_COLLECTIVE_IO); + + sprintf(test_title, "parallel vector read test 2 -- %s / col op / col I/O", vfd_name); + } + + TESTING(test_title); + } + + show_progress = ((show_progress) && (mpi_rank == 0)); + + if (show_progress) + HDfprintf(stdout, "\n%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 1) Open the test file with the specified VFD, set the eoa, and setup the dxpl */ + if (pass) { + + eoa = (haddr_t)mpi_size * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t)); + + setup_vfd_test_file(file_name_id, filename, mpi_size, xfer_mode, coll_opt_mode, vfd_name, eoa, &lf, + &fapl_id, &dxpl_id); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 2) Using rank zero, write the entire decreasing_fi_buf to + * the file. + */ + if (pass) { + + size_t image_size = (size_t)mpi_size * (size_t)INTS_PER_RANK * sizeof(int32_t); + + if (mpi_rank == 0) { + + if (H5FDwrite(lf, H5FD_MEM_DRAW, H5P_DEFAULT, (haddr_t)0, image_size, (void *)decreasing_fi_buf) < + 0) { + + pass = FALSE; + failure_mssg = "H5FDwrite() on rank 0 failed.\n"; + } + } + } + + /* 3) Barrier */ + + if (pass) { + + MPI_Barrier(MPI_COMM_WORLD); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 4) On each rank, zero the read buffer. */ + if (pass) { + + for (i = 0; i < mpi_size * INTS_PER_RANK; i++) { + + read_fi_buf[i] = 0; + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 5) On even ranks, read INTS_PER_RANK * sizeof(int32) + * bytes from the file starting at offset mpi_rank * + * INTS_PER_RANK * sizeof(int32_t) in both the file and + * read_fi_buf. Do this with a vector read containing + * a single element. + * + * Odd ranks perform an empty read. + */ + if (pass) { + + if (mpi_rank % 2 == 0) { + + count = 1; + types[0] = H5FD_MEM_DRAW; + addrs[0] = (haddr_t)mpi_rank * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t)); + sizes[0] = (size_t)INTS_PER_RANK * sizeof(int32_t); + bufs[0] = (void *)(&(read_fi_buf[mpi_rank * INTS_PER_RANK])); + } + else { + + count = 0; + } + + if (H5FDread_vector(lf, dxpl_id, count, types, addrs, sizes, bufs) < 0) { + + pass = FALSE; + failure_mssg = "H5FDread_vector() failed.\n"; + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 6) Barrier */ + + if (pass) { + + MPI_Barrier(MPI_COMM_WORLD); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 7) On odd ranks, read INTS_PER_RANK * sizeof(int32) + * bytes from the file starting at offset mpi_rank * + * INTS_PER_RANK * sizeof(int32_t) in both the file and + * read_fi_buf. Do this with a vector read containing + * a single element. + * + * Even ranks perform an empty read. + */ + if (pass) { + + if (mpi_rank % 2 == 1) { + + count = 1; + types[0] = H5FD_MEM_DRAW; + addrs[0] = (haddr_t)mpi_rank * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t)); + sizes[0] = (size_t)INTS_PER_RANK * sizeof(int32_t); + bufs[0] = (void *)(&(read_fi_buf[mpi_rank * INTS_PER_RANK])); + } + else { + + count = 0; + } + + if (H5FDread_vector(lf, dxpl_id, count, types, addrs, sizes, bufs) < 0) { + + pass = FALSE; + failure_mssg = "H5FDread_vector() failed.\n"; + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 8) Verify that read_fi_buf contains zeros for all + * indices less than mpi_rank * INTS_PER_RANK, or + * greater than or equal to (mpi_rank + 1) * INTS_PER_RANK. + * For all other indicies, read_fi_buf should equal + * decreasing_fi_buf. + */ + + if (pass) { + + for (i = 0; i < mpi_size * INTS_PER_RANK; i++) { + + if ((i < mpi_rank * INTS_PER_RANK) || (i >= (mpi_rank + 1) * INTS_PER_RANK)) { + + if (read_fi_buf[i] != 0) { + + pass = FALSE; + failure_mssg = "Unexpected value in read_fi_buf (1).\n"; + break; + } + } + else { + + if (read_fi_buf[i] != decreasing_fi_buf[i]) { + + pass = FALSE; + failure_mssg = "Unexpected value in read_fi_buf (2).\n"; + break; + } + } + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 9) Barrier */ + + if (pass) { + + MPI_Barrier(MPI_COMM_WORLD); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 10) Close the test file and delete it (on rank 0 only). + * Close FAPL and DXPL. + */ + + if (pass) { + + takedown_vfd_test_file(mpi_rank, filename, &lf, &fapl_id, &dxpl_id); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* report results */ + if (mpi_rank == 0) { + + if (pass) { + + PASSED(); + } + else { + + H5_FAILED(); + + if (show_progress) { + HDfprintf(stdout, "%s: failure_mssg = \"%s\"\n", fcn_name, failure_mssg); + } + } + } + + return (!pass); + +} /* vector_read_test_2() */ + +/*------------------------------------------------------------------------- + * Function: vector_read_test_3() + * + * Purpose: Verify that vector read works with multiple entries in + * the vector in each read, and that read buffers need not + * be in increasing (memory) address order. + * + * 1) Open the test file with the specified VFD, set the eoa, + * and setup the DXPL. + * + * 2) Using rank zero, write the entire negative_fi_buf to + * the file. + * + * 3) Barrier + * + * 4) On each rank, zero the four read buffers. + * + * 5) On each rank, do a vector read from the file, with + * each rank's vector having four elements, with each + * element reading INTS_PER_RANK / 4 * sizeof(int32) + * bytes, and the reads starting at address: + * + * (mpi_rank * INTS_PER_RANK) * sizeof(int32_t) + * + * (mpi_rank * INTS_PER_RANK + INTS_PER_RANK / 4) * + * sizeof(int32_t) + * + * (mpi_rank * INTS_PER_RANK + INTS_PER_RANK / 2) * + * sizeof(int32_t) + * + * (mpi_rank * INTS_PER_RANK + 3 * INTS_PER_RANK / 2) * + * sizeof(int32_t) + * + * On even ranks, the targets of the reads should be + * buf_0, buf_1, buf_2, and buf_3 respectively. + * + * On odd ranks, the targets of the reads should be + * buf_3, buf_2, buf_1, and buf_0 respectively. + * + * This has the effect of ensuring that on at least + * some ranks, the read buffers are not in increasing + * address order. + * + * 6) Verify that buf_0, buf_1, buf_2, and buf_3 contain + * the expected data. Note that this will be different + * on even vs. odd ranks. + * + * 7) Barrier. + * + * 8) Close the test file. + * + * 9) On rank 0, delete the test file. + * + * Return: FALSE on success, TRUE if any errors are detected. + * + * Programmer: John Mainzer + * 3/26/21 + * + * Modifications: + * + * None. + * + *------------------------------------------------------------------------- + */ + +static unsigned +vector_read_test_3(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode, + H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name) +{ + const char *fcn_name = "vector_read_test_3()"; + char test_title[120]; + char filename[512]; + int32_t buf_0[(INTS_PER_RANK / 4) + 1]; + int32_t buf_1[(INTS_PER_RANK / 4) + 1]; + int32_t buf_2[(INTS_PER_RANK / 4) + 1]; + int32_t buf_3[(INTS_PER_RANK / 4) + 1]; + haddr_t eoa; + hbool_t show_progress = FALSE; + hid_t fapl_id = -1; /* file access property list ID */ + hid_t dxpl_id = -1; /* data access property list ID */ + H5FD_t * lf = NULL; /* VFD struct ptr */ + int cp = 0; + int i; + uint32_t count; + H5FD_mem_t types[4]; + haddr_t addrs[4]; + size_t sizes[4]; + void * bufs[4]; + + pass = TRUE; + + if (mpi_rank == 0) { + + if (xfer_mode == H5FD_MPIO_INDEPENDENT) { + + sprintf(test_title, "parallel vector read test 3 -- %s / independent", vfd_name); + } + else if (coll_opt_mode == H5FD_MPIO_INDIVIDUAL_IO) { + + sprintf(test_title, "parallel vector read test 3 -- %s / col op / ind I/O", vfd_name); + } + else { + + HDassert(coll_opt_mode == H5FD_MPIO_COLLECTIVE_IO); + + sprintf(test_title, "parallel vector read test 3 -- %s / col op / col I/O", vfd_name); + } + + TESTING(test_title); + } + + show_progress = ((show_progress) && (mpi_rank == 0)); + + if (show_progress) + HDfprintf(stdout, "\n%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 1) Open the test file with the specified VFD, set the eoa, and setup the dxpl */ + if (pass) { + + eoa = (haddr_t)mpi_size * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t)); + + setup_vfd_test_file(file_name_id, filename, mpi_size, xfer_mode, coll_opt_mode, vfd_name, eoa, &lf, + &fapl_id, &dxpl_id); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 2) Using rank zero, write the entire negative_fi_buf to + * the file. + */ + if (pass) { + + size_t image_size = (size_t)mpi_size * (size_t)INTS_PER_RANK * sizeof(int32_t); + + if (mpi_rank == 0) { + + if (H5FDwrite(lf, H5FD_MEM_DRAW, H5P_DEFAULT, (haddr_t)0, image_size, (void *)negative_fi_buf) < + 0) { + + pass = FALSE; + failure_mssg = "H5FDwrite() on rank 0 failed.\n"; + } + } + } + + /* 3) Barrier */ + + if (pass) { + + MPI_Barrier(MPI_COMM_WORLD); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 4) On each rank, zero the four read buffers. */ + if (pass) { + + for (i = 0; i <= INTS_PER_RANK / 4; i++) { + + buf_0[i] = 0; + buf_1[i] = 0; + buf_2[i] = 0; + buf_3[i] = 0; + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 5) On each rank, do a vector read from the file, with + * each rank's vector having four elements, with each + * element reading INTS_PER_RANK / 4 * sizeof(int32) + * bytes, and the reads starting at address: + * + * (mpi_rank * INTS_PER_RANK) * sizeof(int32_t) + * + * (mpi_rank * INTS_PER_RANK + INTS_PER_RANK / 4) * + * sizeof(int32_t) + * + * (mpi_rank * INTS_PER_RANK + INTS_PER_RANK / 2) * + * sizeof(int32_t) + * + * (mpi_rank * INTS_PER_RANK + 3 * INTS_PER_RANK / 2) * + * sizeof(int32_t) + * + * On even ranks, the targets of the reads should be + * buf_0, buf_1, buf_2, and buf_3 respectively. + * + * On odd ranks, the targets of the reads should be + * buf_3, buf_2, buf_1, and buf_0 respectively. + * + * This has the effect of ensuring that on at least + * some ranks, the read buffers are not in increasing + * address order. + */ + if (pass) { + + haddr_t base_addr = (haddr_t)mpi_rank * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t)); + + count = 4; + + types[0] = H5FD_MEM_DRAW; + addrs[0] = base_addr; + sizes[0] = (size_t)(INTS_PER_RANK / 4) * sizeof(int32_t); + + types[1] = H5FD_MEM_DRAW; + addrs[1] = base_addr + ((haddr_t)(INTS_PER_RANK / 4) * (haddr_t)(sizeof(int32_t))); + sizes[1] = (size_t)(INTS_PER_RANK / 4) * sizeof(int32_t); + + types[2] = H5FD_MEM_DRAW; + addrs[2] = base_addr + ((haddr_t)(INTS_PER_RANK / 2) * (haddr_t)(sizeof(int32_t))); + sizes[2] = (size_t)(INTS_PER_RANK / 4) * sizeof(int32_t); + + types[3] = H5FD_MEM_DRAW; + addrs[3] = base_addr + ((haddr_t)(3 * INTS_PER_RANK / 4) * (haddr_t)(sizeof(int32_t))); + sizes[3] = (size_t)INTS_PER_RANK / 4 * sizeof(int32_t); + + if (mpi_rank % 2 == 0) { + + bufs[0] = (void *)(&(buf_0[0])); + bufs[1] = (void *)(buf_1); + bufs[2] = (void *)(buf_2); + bufs[3] = (void *)(buf_3); + } + else { + + bufs[0] = (void *)(&(buf_3[0])); + bufs[1] = (void *)(buf_2); + bufs[2] = (void *)(buf_1); + bufs[3] = (void *)(buf_0); + } + + if (H5FDread_vector(lf, dxpl_id, count, types, addrs, sizes, bufs) < 0) { + + pass = FALSE; + failure_mssg = "H5FDread_vector() failed.\n"; + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 6) Verify that buf_0, buf_1, buf_2, and buf_3 contain + * the expected data. Note that this will be different + * on even vs. odd ranks. + */ + if (pass) { + + int base_index = mpi_rank * INTS_PER_RANK; + + for (i = 0; ((pass) && (i < INTS_PER_RANK / 4)); i++) { + + if (((mpi_rank % 2 == 0) && (buf_0[i] != negative_fi_buf[base_index + i])) || + ((mpi_rank % 2 == 1) && (buf_3[i] != negative_fi_buf[base_index + i]))) { + + pass = FALSE; + failure_mssg = "Unexpected value in buf (1).\n"; + } + } + + base_index += INTS_PER_RANK / 4; + + for (i = 0; ((pass) && (i < INTS_PER_RANK / 4)); i++) { + + if (((mpi_rank % 2 == 0) && (buf_1[i] != negative_fi_buf[base_index + i])) || + ((mpi_rank % 2 == 1) && (buf_2[i] != negative_fi_buf[base_index + i]))) { + + pass = FALSE; + failure_mssg = "Unexpected value in buf (2).\n"; + } + } + + base_index += INTS_PER_RANK / 4; + + for (i = 0; ((pass) && (i < INTS_PER_RANK / 4)); i++) { + + if (((mpi_rank % 2 == 0) && (buf_2[i] != negative_fi_buf[base_index + i])) || + ((mpi_rank % 2 == 1) && (buf_1[i] != negative_fi_buf[base_index + i]))) { + + pass = FALSE; + failure_mssg = "Unexpected value in buf (3).\n"; + } + } + + base_index += INTS_PER_RANK / 4; + + for (i = 0; ((pass) && (i < INTS_PER_RANK / 4)); i++) { + + if (((mpi_rank % 2 == 0) && (buf_3[i] != negative_fi_buf[base_index + i])) || + ((mpi_rank % 2 == 1) && (buf_0[i] != negative_fi_buf[base_index + i]))) { + + pass = FALSE; + failure_mssg = "Unexpected value in buf (4).\n"; + } + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 7) Barrier */ + + if (pass) { + + MPI_Barrier(MPI_COMM_WORLD); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 8) Close the test file and delete it (on rank 0 only). + * Close FAPL and DXPL. + */ + + if (pass) { + + takedown_vfd_test_file(mpi_rank, filename, &lf, &fapl_id, &dxpl_id); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* report results */ + if (mpi_rank == 0) { + + if (pass) { + + PASSED(); + } + else { + + H5_FAILED(); + + if (show_progress) { + HDfprintf(stdout, "%s: failure_mssg = \"%s\"\n", fcn_name, failure_mssg); + } + } + } + + return (!pass); + +} /* vector_read_test_3() */ + +/*------------------------------------------------------------------------- + * Function: vector_read_test_4() + * + * Purpose: Test vector I/O reads with vectors of different lengths + * and entry sizes across the ranks. Vectors are not, in + * general, sorted in increasing address order. Further, + * reads are not, in general, contiguous. + * + * 1) Open the test file with the specified VFD, set the eoa. + * and setup the DXPL. + * + * 2) Using rank zero, write the entire increasing_fi_buf to + * the file. + * + * 3) Barrier + * + * 4) Set all cells of read_fi_buf to zero. + * + * 5) For each rank, define base_index equal to: + * + * mpi_rank * INTS_PER_RANK + * + * and define base_addr equal to + * + * base_index * sizeof(int32_t). + * + * Setup a vector read between base_addr and + * base_addr + INTS_PER_RANK * sizeof(int32_t) - 1 + * as follows: + * + * if ( rank % 4 == 0 ) construct a vector that reads: + * + * INTS_PER_RANK / 4 * sizeof(int32_t) bytes + * starting at base_addr + INTS_PER_RANK / 2 * + * sizeof(int32_t), + * + * INTS_PER_RANK / 8 * sizeof(int32_t) bytes + * starting at base_addr + INTS_PER_RANK / 4 * + * sizeof(int32_t), and + * + * INTS_PER_RANK / 16 * sizeof(int32_t) butes + * starting at base_addr + INTS_PER_RANK / 16 * + * sizeof(int32_t) + * + * to the equivalent locations in read_fi_buf + * + * if ( rank % 4 == 1 ) construct a vector that reads: + * + * ((INTS_PER_RANK / 2) - 2) * sizeof(int32_t) + * bytes starting at base_addr + sizeof(int32_t), and + * + * ((INTS_PER_RANK / 2) - 2) * sizeof(int32_t) bytes + * starting at base_addr + (INTS_PER_RANK / 2 + 1) * + * sizeof(int32_t). + * + * to the equivalent locations in read_fi_buf + * + * if ( rank % 4 == 2 ) construct a vector that reads: + * + * sizeof(int32_t) bytes starting at base_index + + * (INTS_PER_RANK / 2) * sizeof int32_t. + * + * to the equivalent locations in read_fi_buf + * + * if ( rank % 4 == 3 ) construct and read the empty vector + * + * 6) On each rank, verify that read_fi_buf contains the + * the expected values -- that is the matching values from + * increasing_fi_buf where ever there was a read, and zero + * otherwise. + * + * 7) Barrier. + * + * 8) Close the test file. + * + * 9) On rank 0, delete the test file. + * + * Return: FALSE on success, TRUE if any errors are detected. + * + * Programmer: John Mainzer + * 3/26/21 + * + * Modifications: + * + * None. + * + *------------------------------------------------------------------------- + */ + +static unsigned +vector_read_test_4(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode, + H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name) +{ + const char *fcn_name = "vector_read_test_4()"; + char test_title[120]; + char filename[512]; + haddr_t eoa; + haddr_t base_addr; + hbool_t show_progress = FALSE; + hid_t fapl_id = -1; /* file access property list ID */ + hid_t dxpl_id = -1; /* data access property list ID */ + H5FD_t * lf = NULL; /* VFD struct ptr */ + int cp = 0; + int i; + int j; + int k; + int base_index; + uint32_t count = 0; + H5FD_mem_t types[4]; + haddr_t addrs[4]; + size_t sizes[4]; + void * bufs[4]; + + pass = TRUE; + + if (mpi_rank == 0) { + + if (xfer_mode == H5FD_MPIO_INDEPENDENT) { + + sprintf(test_title, "parallel vector read test 4 -- %s / independent", vfd_name); + } + else if (coll_opt_mode == H5FD_MPIO_INDIVIDUAL_IO) { + + sprintf(test_title, "parallel vector read test 4 -- %s / col op / ind I/O", vfd_name); + } + else { + + HDassert(coll_opt_mode == H5FD_MPIO_COLLECTIVE_IO); + + sprintf(test_title, "parallel vector read test 4 -- %s / col op / col I/O", vfd_name); + } + + TESTING(test_title); + } + + show_progress = ((show_progress) && (mpi_rank == 0)); + + if (show_progress) + HDfprintf(stdout, "\n%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 1) Open the test file with the specified VFD, set the eoa, and setup the dxpl */ + if (pass) { + + eoa = (haddr_t)mpi_size * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t)); + + setup_vfd_test_file(file_name_id, filename, mpi_size, xfer_mode, coll_opt_mode, vfd_name, eoa, &lf, + &fapl_id, &dxpl_id); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 2) Using rank zero, write the entire negative_fi_buf to + * the file. + */ + if (pass) { + + size_t image_size = (size_t)mpi_size * (size_t)INTS_PER_RANK * sizeof(int32_t); + + if (mpi_rank == 0) { + + if (H5FDwrite(lf, H5FD_MEM_DRAW, H5P_DEFAULT, (haddr_t)0, image_size, (void *)increasing_fi_buf) < + 0) { + + pass = FALSE; + failure_mssg = "H5FDwrite() on rank 0 failed.\n"; + } + } + } + + /* 3) Barrier */ + + if (pass) { + + MPI_Barrier(MPI_COMM_WORLD); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 4) Set all cells of read_fi_buf to zero. */ + if (pass) { + + for (i = 0; i < mpi_size * INTS_PER_RANK; i++) { + + read_fi_buf[i] = 0; + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 5) For each rank, define base_index equal to: + * + * mpi_rank * INTS_PER_RANK + * + * and define base_addr equal to + * + * base_index * sizeof(int32_t). + * + * Setup a vector read between base_addr and + * base_addr + INTS_PER_RANK * sizeof(int32_t) - 1 + * as follows: + */ + if (pass) { + + base_index = mpi_rank * INTS_PER_RANK; + base_addr = (haddr_t)base_index * (haddr_t)sizeof(int32_t); + + if ((mpi_rank % 4) == 0) { + + /* if ( rank % 4 == 0 ) construct a vector that reads: + * + * INTS_PER_RANK / 4 * sizeof(int32_t) bytes + * starting at base_addr + INTS_PER_RANK / 2 * + * sizeof(int32_t), + * + * INTS_PER_RANK / 8 * sizeof(int32_t) bytes + * starting at base_addr + INTS_PER_RANK / 4 * + * sizeof(int32_t), and + * + * INTS_PER_RANK / 16 * sizeof(int32_t) butes + * starting at base_addr + INTS_PER_RANK / 16 * + * sizeof(int32_t) + * + * to the equivalent locations in read_fi_buf + */ + + count = 3; + + types[0] = H5FD_MEM_DRAW; + addrs[0] = base_addr + (haddr_t)((size_t)(INTS_PER_RANK / 2) * sizeof(int32_t)); + sizes[0] = (size_t)(INTS_PER_RANK / 4) * sizeof(int32_t); + bufs[0] = (void *)(&(read_fi_buf[base_index + (INTS_PER_RANK / 2)])); + + types[1] = H5FD_MEM_DRAW; + addrs[1] = base_addr + (haddr_t)((size_t)(INTS_PER_RANK / 4) * sizeof(int32_t)); + sizes[1] = (size_t)(INTS_PER_RANK / 8) * sizeof(int32_t); + bufs[1] = (void *)(&(read_fi_buf[base_index + (INTS_PER_RANK / 4)])); + + types[2] = H5FD_MEM_DRAW; + addrs[2] = base_addr + (haddr_t)((size_t)(INTS_PER_RANK / 16) * sizeof(int32_t)); + sizes[2] = (size_t)(INTS_PER_RANK / 16) * sizeof(int32_t); + bufs[2] = (void *)(&(read_fi_buf[base_index + (INTS_PER_RANK / 16)])); + } + else if ((mpi_rank % 4) == 1) { + + /* if ( rank % 4 == 1 ) construct a vector that reads: + * + * ((INTS_PER_RANK / 2) - 2) * sizeof(int32_t) + * bytes starting at base_addr + sizeof(int32_t), and + * + * ((INTS_PER_RANK / 2) - 2) * sizeof(int32_t) bytes + * starting at base_addr + (INTS_PER_RANK / 2 + 1) * + * sizeof(int32_t). + * + * to the equivalent locations in read_fi_buf + */ + count = 2; + + types[0] = H5FD_MEM_DRAW; + addrs[0] = base_addr + (haddr_t)(sizeof(int32_t)); + sizes[0] = (size_t)((INTS_PER_RANK / 2) - 2) * sizeof(int32_t); + bufs[0] = (void *)(&(read_fi_buf[base_index + 1])); + + types[1] = H5FD_MEM_DRAW; + addrs[1] = base_addr + (haddr_t)((size_t)((INTS_PER_RANK / 2) + 1) * sizeof(int32_t)); + sizes[1] = (size_t)((INTS_PER_RANK / 2) - 2) * sizeof(int32_t); + bufs[1] = (void *)(&(read_fi_buf[base_index + (INTS_PER_RANK / 2) + 1])); + } + else if ((mpi_rank % 4) == 2) { + + /* if ( rank % 4 == 2 ) construct a vector that reads: + * + * sizeof(int32_t) bytes starting at base_index + + * (INTS_PER_RANK / 2) * sizeof int32_t. + * + * to the equivalent locations in read_fi_buf + */ + count = 1; + + types[0] = H5FD_MEM_DRAW; + addrs[0] = base_addr + (haddr_t)((size_t)(INTS_PER_RANK / 2) * sizeof(int32_t)); + sizes[0] = sizeof(int32_t); + bufs[0] = (void *)(&(read_fi_buf[base_index + (INTS_PER_RANK / 2)])); + } + else if ((mpi_rank % 4) == 3) { + + /* if ( rank % 4 == 3 ) construct and read the empty vector */ + + count = 0; + } + + if (H5FDread_vector(lf, dxpl_id, count, types, addrs, sizes, bufs) < 0) { + + pass = FALSE; + failure_mssg = "H5FDread_vector() failed (1).\n"; + } + } + + /* 6) On each rank, verify that read_fi_buf contains the + * the expected values -- that is the matching values from + * increasing_fi_buf where ever there was a read, and zero + * otherwise. + */ + if (pass) { + + for (i = 0; ((pass) && (i < mpi_size)); i++) { + + base_index = i * INTS_PER_RANK; +#if 1 + for (j = base_index; j < base_index + INTS_PER_RANK; j++) { + + k = j - base_index; +#else + for (k = 0; k < INTS_PER_RANK; k++) { + + j = k + base_index; +#endif + + if (i == mpi_rank) { + + switch (i % 4) { + + case 0: + if (((INTS_PER_RANK / 2) <= k) && (k < (3 * (INTS_PER_RANK / 4)))) { + + if (read_fi_buf[j] != increasing_fi_buf[j]) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (1.1)"; + HDfprintf(stdout, "\nread_fi_buf[%d] = %d, increasing_fi_buf[%d] = %d\n", + j, read_fi_buf[j], j, increasing_fi_buf[j]); + } + } + else if (((INTS_PER_RANK / 4) <= k) && (k < (3 * (INTS_PER_RANK / 8)))) { + + if (read_fi_buf[j] != increasing_fi_buf[j]) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (1.2)"; + } + } + else if (((INTS_PER_RANK / 16) <= k) && (k < (INTS_PER_RANK / 8))) { + + if (read_fi_buf[j] != increasing_fi_buf[j]) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (1.3)"; + } + } + else { + + if (read_fi_buf[j] != 0) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (1.4)"; + } + } + break; + + case 1: + if ((1 <= k) && (k <= ((INTS_PER_RANK / 2) - 2))) { + + if (read_fi_buf[j] != increasing_fi_buf[j]) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (2.1)"; + } + } + else if ((((INTS_PER_RANK / 2) + 1) <= k) && (k <= (INTS_PER_RANK - 2))) { + + if (read_fi_buf[j] != increasing_fi_buf[j]) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (2.2)"; + } + } + else { + + if (read_fi_buf[j] != 0) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (2.3)"; + } + } + break; + + case 2: + if (k == INTS_PER_RANK / 2) { + + if (read_fi_buf[j] != increasing_fi_buf[j]) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (3.1)"; + } + } + else { + + if (read_fi_buf[j] != 0) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (3.2)"; + } + } + break; + + case 3: + if (read_fi_buf[j] != 0) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (4)"; + } + break; + + default: + HDassert(FALSE); /* should be un-reachable */ + break; + } + } + else if (read_fi_buf[j] != 0) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (5)"; + } + } /* end for loop */ + } /* end for loop */ + } /* end if */ + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 7) Barrier */ + + if (pass) { + + MPI_Barrier(MPI_COMM_WORLD); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 8) Close the test file and delete it (on rank 0 only). + * Close FAPL and DXPL. + */ + + if (pass) { + + takedown_vfd_test_file(mpi_rank, filename, &lf, &fapl_id, &dxpl_id); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* report results */ + if (mpi_rank == 0) { + + if (pass) { + + PASSED(); + } + else { + + H5_FAILED(); + + if (show_progress) { + HDfprintf(stdout, "%s: failure_mssg = \"%s\"\n", fcn_name, failure_mssg); + } + } + } + + return (!pass); + +} /* vector_read_test_4() */ + +/*------------------------------------------------------------------------- + * Function: vector_read_test_5() + * + * Purpose: Test correct management of the sizes[] array optimization, + * where, if sizes[i] == 0, we use sizes[i - 1] as the value + * of size[j], for j >= i. + * + * 1) Open the test file with the specified VFD, set the eoa. + * and setup the DXPL. + * + * 2) Using rank zero, write the entire increasing_fi_buf to + * the file. + * + * 3) Barrier + * + * 4) Set all cells of read_fi_buf to zero. + * + * 5) For each rank, define base_index equal to: + * + * mpi_rank * INTS_PER_RANK + * + * and define base_addr equal to + * + * base_index * sizeof(int32_t). + * + * Setup a vector read between base_addr and + * base_addr + INTS_PER_RANK * sizeof(int32_t) - 1 + * that reads every 16th integer located in that + * that range starting at base_addr. Use a sizes[] + * array of length 2, with sizes[0] set to sizeof(int32_t), + * and sizes[1] = 0. + * + * Read the integers into the corresponding locations in + * read_fi_buf. + * + * 6) On each rank, verify that read_fi_buf contains the + * the expected values -- that is the matching values from + * increasing_fi_buf where ever there was a read, and zero + * otherwise. + * + * 7) Barrier. + * + * 8) Close the test file. + * + * 9) On rank 0, delete the test file. + * + * Return: FALSE on success, TRUE if any errors are detected. + * + * Programmer: John Mainzer + * 3/26/21 + * + * Modifications: + * + * None. + * + *------------------------------------------------------------------------- + */ + +static unsigned +vector_read_test_5(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode, + H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name) +{ + const char *fcn_name = "vector_read_test_5()"; + char test_title[120]; + char filename[512]; + haddr_t eoa; + haddr_t base_addr; + hbool_t show_progress = FALSE; + hid_t fapl_id = -1; /* file access property list ID */ + hid_t dxpl_id = -1; /* data access property list ID */ + H5FD_t * lf = NULL; /* VFD struct ptr */ + int cp = 0; + int i; + int j; + int base_index; + uint32_t count = 0; + H5FD_mem_t types[(INTS_PER_RANK / 16) + 1]; + haddr_t addrs[(INTS_PER_RANK / 16) + 1]; + size_t sizes[2]; + void * bufs[(INTS_PER_RANK / 16) + 1]; + + pass = TRUE; + + if (mpi_rank == 0) { + + if (xfer_mode == H5FD_MPIO_INDEPENDENT) { + + sprintf(test_title, "parallel vector read test 5 -- %s / independent", vfd_name); + } + else if (coll_opt_mode == H5FD_MPIO_INDIVIDUAL_IO) { + + sprintf(test_title, "parallel vector read test 5 -- %s / col op / ind I/O", vfd_name); + } + else { + + HDassert(coll_opt_mode == H5FD_MPIO_COLLECTIVE_IO); + + sprintf(test_title, "parallel vector read test 5 -- %s / col op / col I/O", vfd_name); + } + + TESTING(test_title); + } + + show_progress = ((show_progress) && (mpi_rank == 0)); + + if (show_progress) + HDfprintf(stdout, "\n%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 1) Open the test file with the specified VFD, set the eoa, and setup the dxpl */ + if (pass) { + + eoa = (haddr_t)mpi_size * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t)); + + setup_vfd_test_file(file_name_id, filename, mpi_size, xfer_mode, coll_opt_mode, vfd_name, eoa, &lf, + &fapl_id, &dxpl_id); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 2) Using rank zero, write the entire negative_fi_buf to + * the file. + */ + if (pass) { + + size_t image_size = (size_t)mpi_size * (size_t)INTS_PER_RANK * sizeof(int32_t); + + if (mpi_rank == 0) { + + if (H5FDwrite(lf, H5FD_MEM_DRAW, H5P_DEFAULT, (haddr_t)0, image_size, (void *)increasing_fi_buf) < + 0) { + + pass = FALSE; + failure_mssg = "H5FDwrite() on rank 0 failed.\n"; + } + } + } + + /* 3) Barrier */ + + if (pass) { + + MPI_Barrier(MPI_COMM_WORLD); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 4) Set all cells of read_fi_buf to zero. */ + if (pass) { + + for (i = 0; i < mpi_size * INTS_PER_RANK; i++) { + + read_fi_buf[i] = 0; + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 5) For each rank, define base_index equal to: + * + * mpi_rank * INTS_PER_RANK + * + * and define base_addr equal to + * + * base_index * sizeof(int32_t). + * + * Setup a vector read between base_addr and + * base_addr + INTS_PER_RANK * sizeof(int32_t) - 1 + * that reads every 16th integer located in that + * that range starting at base_addr. Use a sizes[] + * array of length 2, with sizes[0] set to sizeof(int32_t), + * and sizes[1] = 0. + * + * Read the integers into the corresponding locations in + * read_fi_buf. + */ + if (pass) { + + base_index = (mpi_rank * INTS_PER_RANK); + base_addr = (haddr_t)base_index * (haddr_t)sizeof(int32_t); + + count = INTS_PER_RANK / 16; + sizes[0] = sizeof(int32_t); + sizes[1] = 0; + + for (i = 0; i < INTS_PER_RANK / 16; i++) { + + types[i] = H5FD_MEM_DRAW; + addrs[i] = base_addr + ((haddr_t)(16 * i) * (haddr_t)sizeof(int32_t)); + bufs[i] = (void *)(&(read_fi_buf[base_index + (i * 16)])); + } + + if (H5FDread_vector(lf, dxpl_id, count, types, addrs, sizes, bufs) < 0) { + + pass = FALSE; + failure_mssg = "H5FDread_vector() failed (1).\n"; + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 6) On each rank, verify that read_fi_buf contains the + * the expected values -- that is the matching values from + * increasing_fi_buf where ever there was a read, and zero + * otherwise. + */ + if (pass) { + + for (i = 0; ((pass) && (i < mpi_size)); i++) { + + base_index = i * INTS_PER_RANK; + + for (j = base_index; j < base_index + INTS_PER_RANK; j++) { + + if ((i == mpi_rank) && (j % 16 == 0)) { + + if (read_fi_buf[j] != increasing_fi_buf[j]) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (1)"; + } + } + else if (read_fi_buf[j] != 0) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (2)"; + } + } /* end for loop */ + } /* end for loop */ + } /* end if */ + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 7) Barrier */ + + if (pass) { + + MPI_Barrier(MPI_COMM_WORLD); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 8) Close the test file and delete it (on rank 0 only). + * Close FAPL and DXPL. + */ + + if (pass) { + + takedown_vfd_test_file(mpi_rank, filename, &lf, &fapl_id, &dxpl_id); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* report results */ + if (mpi_rank == 0) { + + if (pass) { + + PASSED(); + } + else { + + H5_FAILED(); + + if (show_progress) { + HDfprintf(stdout, "%s: failure_mssg = \"%s\"\n", fcn_name, failure_mssg); + } + } + } + + return (!pass); + +} /* vector_read_test_5() */ + +/*------------------------------------------------------------------------- + * Function: vector_write_test_1() + * + * Purpose: Simple vector write test: + * + * 1) Open the test file with the specified VFD, set the eoa, + * and setup the DXPL. + * + * 2) Write the entire increasing_fi_buf to the file, with + * exactly one buffer per vector per rank. Use either + * independant or collective I/O as specified. + * + * 3) Barrier + * + * 4) On each rank, read the entire file into the read_fi_buf, + * and compare against increasing_fi_buf. Report failure + * if any differences are detected. + * + * 5) Close the test file. + * + * 6) On rank 0, delete the test file. + * + * Return: FALSE on success, TRUE if any errors are detected. + * + * Programmer: John Mainzer + * 3/26/21 + * + * Modifications: + * + * None. + * + *------------------------------------------------------------------------- + */ + +static unsigned +vector_write_test_1(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode, + H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name) +{ + const char *fcn_name = "vector_write_test_1()"; + char test_title[120]; + char filename[512]; + haddr_t eoa; + hbool_t show_progress = FALSE; + hid_t fapl_id = -1; /* file access property list ID */ + hid_t dxpl_id = -1; /* data access property list ID */ + H5FD_t * lf = NULL; /* VFD struct ptr */ + int cp = 0; + int i; + uint32_t count; + H5FD_mem_t types[1]; + haddr_t addrs[1]; + size_t sizes[1]; + void * bufs[1]; + + pass = TRUE; + + if (mpi_rank == 0) { + + if (xfer_mode == H5FD_MPIO_INDEPENDENT) { + + sprintf(test_title, "parallel vector write test 1 -- %s / independent", vfd_name); + } + else if (coll_opt_mode == H5FD_MPIO_INDIVIDUAL_IO) { + + sprintf(test_title, "parallel vector write test 1 -- %s / col op / ind I/O", vfd_name); + } + else { + + HDassert(coll_opt_mode == H5FD_MPIO_COLLECTIVE_IO); + + sprintf(test_title, "parallel vector write test 1 -- %s / col op / col I/O", vfd_name); + } + + TESTING(test_title); + } + + show_progress = ((show_progress) && (mpi_rank == 0)); + + if (show_progress) + HDfprintf(stdout, "\n%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 1) Open the test file with the specified VFD, set the eoa, and setup the dxpl */ + if (pass) { + + eoa = (haddr_t)mpi_size * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t)); + + setup_vfd_test_file(file_name_id, filename, mpi_size, xfer_mode, coll_opt_mode, vfd_name, eoa, &lf, + &fapl_id, &dxpl_id); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 2) Write the entire increasing_fi_buf to the file, with + * exactly one buffer per vector per rank. Use either + * independant or collective I/O as specified. + */ + + if (pass) { + + count = 1; + types[0] = H5FD_MEM_DRAW; + addrs[0] = (haddr_t)mpi_rank * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t)); + sizes[0] = (size_t)INTS_PER_RANK * sizeof(int32_t); + bufs[0] = (void *)(&(increasing_fi_buf[mpi_rank * INTS_PER_RANK])); + + if (H5FDwrite_vector(lf, dxpl_id, count, types, addrs, sizes, bufs) < 0) { + + pass = FALSE; + failure_mssg = "H5FDwrite_vector() failed.\n"; + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 3) Barrier + */ + + if (pass) { + + MPI_Barrier(MPI_COMM_WORLD); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 4) On each rank, read the entire file into the read_fi_buf, + * and compare against increasing_fi_buf. Report failure + * if any differences are detected. + */ + + if (pass) { + + size_t image_size = (size_t)mpi_size * (size_t)INTS_PER_RANK * sizeof(int32_t); + + if (H5FDread(lf, H5FD_MEM_DRAW, H5P_DEFAULT, (haddr_t)0, image_size, (void *)read_fi_buf) < 0) { + + pass = FALSE; + failure_mssg = "H5FDread() failed.\n"; + } + + for (i = 0; i < mpi_size * INTS_PER_RANK; i++) { + + if (read_fi_buf[i] != increasing_fi_buf[i]) { + + pass = FALSE; + failure_mssg = "unexpected data read from file"; + break; + } + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 5) Close the test file and delete it (on rank 0 only). + * Close FAPL and DXPL. + */ + + if (pass) { + + takedown_vfd_test_file(mpi_rank, filename, &lf, &fapl_id, &dxpl_id); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* report results */ + if (mpi_rank == 0) { + + if (pass) { + + PASSED(); + } + else { + + H5_FAILED(); + + if (show_progress) { + HDfprintf(stdout, "%s: failure_mssg = \"%s\"\n", fcn_name, failure_mssg); + } + } + } + + return (!pass); + +} /* vector_write_test_1() */ + +/*------------------------------------------------------------------------- + * Function: vector_write_test_2() + * + * Purpose: Test vector I/O writes in which only some ranks participate. + * Depending on the collective parameter, these writes will + * be either collective or independent. + * + * 1) Open the test file with the specified VFD, and set + * the eoa. + * + * 2) Write the odd blocks of the increasing_fi_buf to the file, + * with the odd ranks writting the odd blocks, and the even + * ranks writing an empty vector. + * + * Here, a "block" of the increasing_fi_buf is a sequence + * of integers in increasing_fi_buf of length INTS_PER_RANK, + * and with start index a multiple of INTS_PER_RANK. + * + * 3) Write the even blocks of the negative_fi_buf to the file, + * with the even ranks writting the even blocks, and the odd + * ranks writing an empty vector. + * + * 4) Barrier + * + * 4) On each rank, read the entire file into the read_fi_buf, + * and compare against increasing_fi_buf and negative_fi_buf + * as appropriate. Report failure if any differences are + * detected. + * + * 5) Close the test file. On rank 0, delete the test file. + * + * Return: FALSE on success, TRUE if any errors are detected. + * + * Programmer: John Mainzer + * 3/28/21 + * + * Modifications: + * + * None. + * + *------------------------------------------------------------------------- + */ + +static unsigned +vector_write_test_2(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode, + H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name) +{ + const char *fcn_name = "vector_write_test_2()"; + char test_title[120]; + char filename[512]; + haddr_t eoa; + hbool_t show_progress = FALSE; + hid_t fapl_id = -1; /* file access property list ID */ + hid_t dxpl_id = -1; /* data access property list ID */ + H5FD_t * lf = NULL; /* VFD struct ptr */ + int cp = 0; + int i; + int j; + uint32_t count; + H5FD_mem_t types[1]; + haddr_t addrs[1]; + size_t sizes[1]; + void * bufs[1]; + + pass = TRUE; + + if (mpi_rank == 0) { + + if (xfer_mode == H5FD_MPIO_INDEPENDENT) { + + sprintf(test_title, "parallel vector write test 2 -- %s / independent", vfd_name); + } + else if (coll_opt_mode == H5FD_MPIO_INDIVIDUAL_IO) { + + sprintf(test_title, "parallel vector write test 2 -- %s / col op / ind I/O", vfd_name); + } + else { + + HDassert(coll_opt_mode == H5FD_MPIO_COLLECTIVE_IO); + + sprintf(test_title, "parallel vector write test 2 -- %s / col op / col I/O", vfd_name); + } + + TESTING(test_title); + } + + show_progress = ((show_progress) && (mpi_rank == 0)); + + if (show_progress) + HDfprintf(stdout, "\n%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 1) Open the test file with the specified VFD, set the eoa, and setup the dxpl */ + if (pass) { + + eoa = (haddr_t)mpi_size * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t)); + + setup_vfd_test_file(file_name_id, filename, mpi_size, xfer_mode, coll_opt_mode, vfd_name, eoa, &lf, + &fapl_id, &dxpl_id); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 2) Write the odd blocks of the increasing_fi_buf to the file, + * with the odd ranks writting the odd blocks, and the even + * ranks writing an empty vector. + * + * Here, a "block" of the increasing_fi_buf is a sequence + * of integers in increasing_fi_buf of length INTS_PER_RANK, + * and with start index a multiple of INTS_PER_RANK. + */ + if (pass) { + + if (mpi_rank % 2 == 1) { /* odd ranks */ + + count = 1; + types[0] = H5FD_MEM_DRAW; + addrs[0] = (haddr_t)mpi_rank * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t)); + sizes[0] = (size_t)INTS_PER_RANK * sizeof(int32_t); + bufs[0] = (void *)(&(increasing_fi_buf[mpi_rank * INTS_PER_RANK])); + + if (H5FDwrite_vector(lf, dxpl_id, count, types, addrs, sizes, bufs) < 0) { + + pass = FALSE; + failure_mssg = "H5FDwrite_vector() failed (1).\n"; + } + } + else { /* even ranks */ + + if (H5FDwrite_vector(lf, dxpl_id, 0, NULL, NULL, NULL, NULL) < 0) { + + pass = FALSE; + failure_mssg = "H5FDwrite_vector() failed (2).\n"; + } + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 3) Write the even blocks of the negative_fi_buf to the file, + * with the even ranks writting the even blocks, and the odd + * ranks writing an empty vector. + */ + if (pass) { + + if (mpi_rank % 2 == 1) { /* odd ranks */ + + if (H5FDwrite_vector(lf, dxpl_id, 0, NULL, NULL, NULL, NULL) < 0) { + + pass = FALSE; + failure_mssg = "H5FDwrite_vector() failed (3).\n"; + } + } + else { /* even ranks */ + + count = 1; + types[0] = H5FD_MEM_DRAW; + addrs[0] = (haddr_t)mpi_rank * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t)); + sizes[0] = (size_t)INTS_PER_RANK * sizeof(int32_t); + bufs[0] = (void *)(&(negative_fi_buf[mpi_rank * INTS_PER_RANK])); + + if (H5FDwrite_vector(lf, dxpl_id, count, types, addrs, sizes, bufs) < 0) { + + pass = FALSE; + failure_mssg = "H5FDwrite_vector() failed (4).\n"; + } + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 4) Barrier + */ + + if (pass) { + + MPI_Barrier(MPI_COMM_WORLD); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 5) On each rank, read the entire file into the read_fi_buf, + * and compare against increasing_fi_buf. Report failure + * if any differences are detected. + */ + + if (pass) { + + size_t image_size = (size_t)mpi_size * (size_t)INTS_PER_RANK * sizeof(int32_t); + + if (H5FDread(lf, H5FD_MEM_DRAW, H5P_DEFAULT, (haddr_t)0, image_size, (void *)read_fi_buf) < 0) { + + pass = FALSE; + failure_mssg = "H5FDread() failed.\n"; + } + + for (i = 0; ((pass) && (i < mpi_size)); i++) { + + if (i % 2 == 1) { /* odd block */ + + for (j = i * INTS_PER_RANK; ((pass) && (j < (i + 1) * INTS_PER_RANK)); j++) { + + if (read_fi_buf[j] != increasing_fi_buf[j]) { + + pass = FALSE; + failure_mssg = "unexpected data read from file"; + break; + } + } + } + else { /* even block */ + + for (j = i * INTS_PER_RANK; ((pass) && (j < (i + 1) * INTS_PER_RANK)); j++) { + + if (read_fi_buf[j] != negative_fi_buf[j]) { + + pass = FALSE; + failure_mssg = "unexpected data read from file"; + break; + } + } + } + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 6) Close the test file and delete it (on rank 0 only). + * Close FAPL and DXPL. + */ + + if (pass) { + + takedown_vfd_test_file(mpi_rank, filename, &lf, &fapl_id, &dxpl_id); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* report results */ + if (mpi_rank == 0) { + + if (pass) { + + PASSED(); + } + else { + + H5_FAILED(); + + if (show_progress) { + HDfprintf(stdout, "%s: failure_mssg = \"%s\"\n", fcn_name, failure_mssg); + } + } + } + + return (!pass); + +} /* vector_write_test_2() */ + +/*------------------------------------------------------------------------- + * Function: vector_write_test_3() + * + * Purpose: Test vector I/O writes with vectors of multiple entries. + * For now, keep the vectors sorted in increasing address + * order. + * + * 1) Open the test file with the specified VFD, and set + * the eoa. + * + * 2) For each rank, construct a vector with base address + * (mpi_rank * INTS_PER_RANK) and writing all bytes from + * that address to ((mpi_rank + 1) * INTS_PER_RANK) - 1. + * Draw equal parts from increasing_fi_buf, + * decreasing_fi_buf, negative_fi_buf, and zero_fi_buf. + * + * Write to file. + * + * 3) Barrier + * + * 4) On each rank, read the entire file into the read_fi_buf, + * and compare against increasing_fi_buf, + * decreasing_fi_buf, negative_fi_buf, and zero_fi_buf as + * appropriate. Report failure if any differences are + * detected. + * + * 5) Close the test file. On rank 0, delete the test file. + * + * Return: FALSE on success, TRUE if any errors are detected. + * + * Programmer: John Mainzer + * 3/31/21 + * + * Modifications: + * + * None. + * + *------------------------------------------------------------------------- + */ + +static unsigned +vector_write_test_3(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode, + H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name) +{ + const char *fcn_name = "vector_write_test_3()"; + char test_title[120]; + char filename[512]; + haddr_t base_addr; + int base_index; + int ints_per_write; + size_t bytes_per_write; + haddr_t eoa; + hbool_t show_progress = FALSE; + hid_t fapl_id = -1; /* file access property list ID */ + hid_t dxpl_id = -1; /* data access property list ID */ + H5FD_t * lf = NULL; /* VFD struct ptr */ + int cp = 0; + int i; + int j; + uint32_t count; + H5FD_mem_t types[4]; + haddr_t addrs[4]; + size_t sizes[4]; + void * bufs[4]; + + pass = TRUE; + + if (mpi_rank == 0) { + + if (xfer_mode == H5FD_MPIO_INDEPENDENT) { + + sprintf(test_title, "parallel vector write test 3 -- %s / independent", vfd_name); + } + else if (coll_opt_mode == H5FD_MPIO_INDIVIDUAL_IO) { + + sprintf(test_title, "parallel vector write test 3 -- %s / col op / ind I/O", vfd_name); + } + else { + + HDassert(coll_opt_mode == H5FD_MPIO_COLLECTIVE_IO); + + sprintf(test_title, "parallel vector write test 3 -- %s / col op / col I/O", vfd_name); + } + + TESTING(test_title); + } + + show_progress = ((show_progress) && (mpi_rank == 0)); + + if (show_progress) + HDfprintf(stdout, "\n%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 1) Open the test file with the specified VFD, set the eoa, and setup the dxpl */ + if (pass) { + + eoa = (haddr_t)mpi_size * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t)); + + setup_vfd_test_file(file_name_id, filename, mpi_size, xfer_mode, coll_opt_mode, vfd_name, eoa, &lf, + &fapl_id, &dxpl_id); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 2) For each rank, construct a vector with base address + * (mpi_rank * INTS_PER_RANK) and writing all bytes from + * that address to ((mpi_rank + 1) * INTS_PER_RANK) - 1. + * Draw equal parts from increasing_fi_buf, + * decreasing_fi_buf, negative_fi_buf, and zero_fi_buf. + * + * Write to file. + */ + if (pass) { + + count = 4; + + base_addr = (haddr_t)mpi_rank * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t)); + ints_per_write = INTS_PER_RANK / 4; + bytes_per_write = (size_t)(ints_per_write) * sizeof(int32_t); + + types[0] = H5FD_MEM_DRAW; + addrs[0] = base_addr; + sizes[0] = bytes_per_write; + bufs[0] = (void *)(&(increasing_fi_buf[mpi_rank * INTS_PER_RANK])); + + types[1] = H5FD_MEM_DRAW; + addrs[1] = addrs[0] + (haddr_t)(bytes_per_write); + sizes[1] = bytes_per_write; + bufs[1] = (void *)(&(decreasing_fi_buf[(mpi_rank * INTS_PER_RANK) + (INTS_PER_RANK / 4)])); + + types[2] = H5FD_MEM_DRAW; + addrs[2] = addrs[1] + (haddr_t)(bytes_per_write); + sizes[2] = bytes_per_write; + bufs[2] = (void *)(&(negative_fi_buf[(mpi_rank * INTS_PER_RANK) + (INTS_PER_RANK / 2)])); + + types[3] = H5FD_MEM_DRAW; + addrs[3] = addrs[2] + (haddr_t)(bytes_per_write); + sizes[3] = bytes_per_write; + bufs[3] = (void *)(&(zero_fi_buf[(mpi_rank * INTS_PER_RANK) + (3 * (INTS_PER_RANK / 4))])); + +#if 0 /* JRM */ + HDfprintf(stdout, "addrs = { %lld, %lld, %lld, %lld}\n", + (long long)addrs[0], (long long)addrs[1], (long long)addrs[2], (long long)addrs[3]); + HDfprintf(stdout, "sizes = { %lld, %lld, %lld, %lld}\n", + (long long)sizes[0], (long long)sizes[1], (long long)sizes[2], (long long)sizes[3]); + HDfprintf(stdout, "bufs = { 0x%llx, 0x%llx, 0x%llx, 0x%llx}\n", + (unsigned long long)bufs[0], (unsigned long long)bufs[1], + (unsigned long long)bufs[2], (unsigned long long)bufs[3]); +#endif /* JRM */ + + if (H5FDwrite_vector(lf, dxpl_id, count, types, addrs, sizes, bufs) < 0) { + + pass = FALSE; + failure_mssg = "H5FDwrite_vector() failed (1).\n"; + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 3) Barrier + */ + + if (pass) { + + MPI_Barrier(MPI_COMM_WORLD); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 4) On each rank, read the entire file into the read_fi_buf, + * and compare against increasing_fi_buf, + * decreasing_fi_buf, negative_fi_buf, and zero_fi_buf as + * appropriate. Report failure if any differences are + * detected. + */ + + if (pass) { + + size_t image_size = (size_t)mpi_size * (size_t)INTS_PER_RANK * sizeof(int32_t); + + if (H5FDread(lf, H5FD_MEM_DRAW, H5P_DEFAULT, (haddr_t)0, image_size, (void *)read_fi_buf) < 0) { + + pass = FALSE; + failure_mssg = "H5FDread() failed.\n"; + } + + for (i = 0; ((pass) && (i < mpi_size)); i++) { + + base_index = i * INTS_PER_RANK; + + for (j = base_index; j < base_index + (INTS_PER_RANK / 4); j++) { + + if (read_fi_buf[j] != increasing_fi_buf[j]) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (1)"; + break; + } + } + + base_index += (INTS_PER_RANK / 4); + + for (j = base_index; j < base_index + (INTS_PER_RANK / 4); j++) { + + if (read_fi_buf[j] != decreasing_fi_buf[j]) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (2)"; + break; + } + } + + base_index += (INTS_PER_RANK / 4); + + for (j = base_index; j < base_index + (INTS_PER_RANK / 4); j++) { + + if (read_fi_buf[j] != negative_fi_buf[j]) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (3)"; + break; + } + } + + base_index += (INTS_PER_RANK / 4); + + for (j = base_index; j < base_index + (INTS_PER_RANK / 4); j++) { + + if (read_fi_buf[j] != zero_fi_buf[j]) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (3)"; + break; + } + } + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 5) Close the test file and delete it (on rank 0 only). + * Close FAPL and DXPL. + */ + + if (pass) { + + takedown_vfd_test_file(mpi_rank, filename, &lf, &fapl_id, &dxpl_id); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* report results */ + if (mpi_rank == 0) { + + if (pass) { + + PASSED(); + } + else { + + H5_FAILED(); + + if (show_progress) { + HDfprintf(stdout, "%s: failure_mssg = \"%s\"\n", fcn_name, failure_mssg); + } + } + } + + return (!pass); + +} /* vector_write_test_3() */ + +/*------------------------------------------------------------------------- + * Function: vector_write_test_4() + * + * Purpose: Test vector I/O writes with vectors of multiple entries. + * For now, keep the vectors sorted in increasing address + * order. + * + * This test differs from vector_write_test_3() in the order + * in which the file image buffers appear in the vector + * write. This guarantees that at least one of these + * tests will present buffers with non-increasing addresses + * in RAM. + * + * 1) Open the test file with the specified VFD, and set + * the eoa. + * + * 2) For each rank, construct a vector with base address + * (mpi_rank * INTS_PER_RANK) and writing all bytes from + * that address to ((mpi_rank + 1) * INTS_PER_RANK) - 1. + * Draw equal parts from zero_fi_buf, negative_fi_buf, + * decreasing_fi_buf, and increasing_fi_buf. + * + * Write to file. + * + * 3) Barrier + * + * 4) On each rank, read the entire file into the read_fi_buf, + * and compare against zero_fi_buf, negative_fi_buf, + * decreasing_fi_buf, and increasing_fi_buf as + * appropriate. Report failure if any differences are + * detected. + * + * 5) Close the test file. On rank 0, delete the test file. + * + * Return: FALSE on success, TRUE if any errors are detected. + * + * Programmer: John Mainzer + * 3/31/21 + * + * Modifications: + * + * None. + * + *------------------------------------------------------------------------- + */ + +static unsigned +vector_write_test_4(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode, + H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name) +{ + const char *fcn_name = "vector_write_test_4()"; + char test_title[120]; + char filename[512]; + haddr_t base_addr; + int base_index; + int ints_per_write; + size_t bytes_per_write; + haddr_t eoa; + hbool_t show_progress = FALSE; + hid_t fapl_id = -1; /* file access property list ID */ + hid_t dxpl_id = -1; /* data access property list ID */ + H5FD_t * lf = NULL; /* VFD struct ptr */ + int cp = 0; + int i; + int j; + uint32_t count; + H5FD_mem_t types[4]; + haddr_t addrs[4]; + size_t sizes[4]; + void * bufs[4]; + + pass = TRUE; + + if (mpi_rank == 0) { + + if (xfer_mode == H5FD_MPIO_INDEPENDENT) { + + sprintf(test_title, "parallel vector write test 4 -- %s / independent", vfd_name); + } + else if (coll_opt_mode == H5FD_MPIO_INDIVIDUAL_IO) { + + sprintf(test_title, "parallel vector write test 4 -- %s / col op / ind I/O", vfd_name); + } + else { + + HDassert(coll_opt_mode == H5FD_MPIO_COLLECTIVE_IO); + + sprintf(test_title, "parallel vector write test 4 -- %s / col op / col I/O", vfd_name); + } + + TESTING(test_title); + } + + show_progress = ((show_progress) && (mpi_rank == 0)); + + if (show_progress) + HDfprintf(stdout, "\n%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 1) Open the test file with the specified VFD, set the eoa, and setup the dxpl */ + if (pass) { + + eoa = (haddr_t)mpi_size * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t)); + + setup_vfd_test_file(file_name_id, filename, mpi_size, xfer_mode, coll_opt_mode, vfd_name, eoa, &lf, + &fapl_id, &dxpl_id); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 2) For each rank, construct a vector with base address + * (mpi_rank * INTS_PER_RANK) and writing all bytes from + * that address to ((mpi_rank + 1) * INTS_PER_RANK) - 1. + * Draw equal parts from increasing_fi_buf, + * decreasing_fi_buf, negative_fi_buf, and zero_fi_buf. + * + * Write to file. + */ + if (pass) { + + count = 4; + + base_addr = (haddr_t)mpi_rank * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t)); + ints_per_write = INTS_PER_RANK / 4; + bytes_per_write = (size_t)(ints_per_write) * sizeof(int32_t); + + types[0] = H5FD_MEM_DRAW; + addrs[0] = base_addr; + sizes[0] = bytes_per_write; + bufs[0] = (void *)(&(zero_fi_buf[mpi_rank * INTS_PER_RANK])); + + types[1] = H5FD_MEM_DRAW; + addrs[1] = addrs[0] + (haddr_t)(bytes_per_write); + sizes[1] = bytes_per_write; + bufs[1] = (void *)(&(negative_fi_buf[(mpi_rank * INTS_PER_RANK) + (INTS_PER_RANK / 4)])); + + types[2] = H5FD_MEM_DRAW; + addrs[2] = addrs[1] + (haddr_t)(bytes_per_write); + sizes[2] = bytes_per_write; + bufs[2] = (void *)(&(decreasing_fi_buf[(mpi_rank * INTS_PER_RANK) + (INTS_PER_RANK / 2)])); + + types[3] = H5FD_MEM_DRAW; + addrs[3] = addrs[2] + (haddr_t)(bytes_per_write); + sizes[3] = bytes_per_write; + bufs[3] = (void *)(&(increasing_fi_buf[(mpi_rank * INTS_PER_RANK) + (3 * (INTS_PER_RANK / 4))])); + +#if 0 /* JRM */ + HDfprintf(stdout, "addrs = { %lld, %lld, %lld, %lld}\n", + (long long)addrs[0], (long long)addrs[1], (long long)addrs[2], (long long)addrs[3]); + HDfprintf(stdout, "sizes = { %lld, %lld, %lld, %lld}\n", + (long long)sizes[0], (long long)sizes[1], (long long)sizes[2], (long long)sizes[3]); + HDfprintf(stdout, "bufs = { 0x%llx, 0x%llx, 0x%llx, 0x%llx}\n", + (unsigned long long)bufs[0], (unsigned long long)bufs[1], + (unsigned long long)bufs[2], (unsigned long long)bufs[3]); +#endif /* JRM */ + + if (H5FDwrite_vector(lf, dxpl_id, count, types, addrs, sizes, bufs) < 0) { + + pass = FALSE; + failure_mssg = "H5FDwrite_vector() failed (1).\n"; + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 3) Barrier + */ + + if (pass) { + + MPI_Barrier(MPI_COMM_WORLD); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 4) On each rank, read the entire file into the read_fi_buf, + * and compare against increasing_fi_buf, + * decreasing_fi_buf, negative_fi_buf, and zero_fi_buf as + * appropriate. Report failure if any differences are + * detected. + */ + + if (pass) { + + size_t image_size = (size_t)mpi_size * (size_t)INTS_PER_RANK * sizeof(int32_t); + + if (H5FDread(lf, H5FD_MEM_DRAW, H5P_DEFAULT, (haddr_t)0, image_size, (void *)read_fi_buf) < 0) { + + pass = FALSE; + failure_mssg = "H5FDread() failed.\n"; + } + + for (i = 0; ((pass) && (i < mpi_size)); i++) { + + base_index = i * INTS_PER_RANK; + + for (j = base_index; j < base_index + (INTS_PER_RANK / 4); j++) { + + if (read_fi_buf[j] != zero_fi_buf[j]) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (1)"; + break; + } + } + + base_index += (INTS_PER_RANK / 4); + + for (j = base_index; j < base_index + (INTS_PER_RANK / 4); j++) { + + if (read_fi_buf[j] != negative_fi_buf[j]) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (2)"; + break; + } + } + + base_index += (INTS_PER_RANK / 4); + + for (j = base_index; j < base_index + (INTS_PER_RANK / 4); j++) { + + if (read_fi_buf[j] != decreasing_fi_buf[j]) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (3)"; + break; + } + } + + base_index += (INTS_PER_RANK / 4); + + for (j = base_index; j < base_index + (INTS_PER_RANK / 4); j++) { + + if (read_fi_buf[j] != increasing_fi_buf[j]) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (3)"; + break; + } + } + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 5) Close the test file and delete it (on rank 0 only). + * Close FAPL and DXPL. + */ + + if (pass) { + + takedown_vfd_test_file(mpi_rank, filename, &lf, &fapl_id, &dxpl_id); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* report results */ + if (mpi_rank == 0) { + + if (pass) { + + PASSED(); + } + else { + + H5_FAILED(); + + if (show_progress) { + HDfprintf(stdout, "%s: failure_mssg = \"%s\"\n", fcn_name, failure_mssg); + } + } + } + + return (!pass); + +} /* vector_write_test_4() */ + +/*------------------------------------------------------------------------- + * Function: vector_write_test_5() + * + * Purpose: Test vector I/O writes with vectors of different lengths + * and entry sizes across the ranks. Vectors are not, in + * general, sorted in increasing address order. Further, + * writes are not, in general, contiguous. + * + * 1) Open the test file with the specified VFD, and set + * the eoa. + * + * 2) Set the test file in a known state by writing zeros + * to all bytes in the test file. Since we have already + * tested this, do this via a vector write of zero_fi_buf. + * + * 3) Barrier + * + * 4) For each rank, define base_index equal to: + * + * mpi_rank * INTS_PER_RANK + * + * and define base_addr equal to + * + * base_index * sizeof(int32_t). + * + * Setup a vector write between base_addr and + * base_addr + INTS_PER_RANK * sizeof(int32_t) - 1 + * as follows: + * + * if ( rank % 4 == 0 ) construct a vector that writes: + * + * negative_fi_buf starting at base_index + + * INTS_PER_RANK / 2 and running for INTS_PER_RANK / 4 + * entries, + * + * decreasing_fi_buf starting at base_index + + * INTS_PER_RANK / 4 and running for INTS_PER_RANK / 8 + * entries, and + * + * increasing_fi_buf starting at base_index + + * INTS_PER_RANK / 16 and running for INTS_PER_RANK / 16 + * entries + * + * to the equivalent locations in the file. + * + * if ( rank % 4 == 1 ) construct a vector that writes: + * + * increasing_fi_buf starting at base_index + 1 and + * running for (INTS_PER_RANK / 2) - 2 entries, and + * + * decreasing_fi_buf startomg at base_index + + * INTS_PER_RANK / 2 + 1 and running for (INTS_PER_RANK / 2) + * - 2 entries + * + * if ( rank % 4 == 2 ) construct a vector that writes: + * + * negative_fi_buf starting at base_index + + * INTS_PER_RANK / 2 and running for one entry. + * + * if ( rank % 4 == 3 ) construct and write the empty vector + * + * 5) Barrier + * + * 6) On each rank, read the entire file into the read_fi_buf, + * and compare against zero_fi_buf, negative_fi_buf, + * decreasing_fi_buf, and increasing_fi_buf as + * appropriate. Report failure if any differences are + * detected. + * + * 7) Close the test file. On rank 0, delete the test file. + * + * Return: FALSE on success, TRUE if any errors are detected. + * + * Programmer: John Mainzer + * 3/31/21 + * + * Modifications: + * + * None. + * + *------------------------------------------------------------------------- + */ + +static unsigned +vector_write_test_5(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode, + H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name) +{ + const char *fcn_name = "vector_write_test_5()"; + char test_title[120]; + char filename[512]; + haddr_t base_addr; + int base_index; + haddr_t eoa; + hbool_t show_progress = FALSE; + hid_t fapl_id = -1; /* file access property list ID */ + hid_t dxpl_id = -1; /* data access property list ID */ + H5FD_t * lf = NULL; /* VFD struct ptr */ + int cp = 0; + int i; + int j; + int k; + uint32_t count; + H5FD_mem_t types[4]; + haddr_t addrs[4]; + size_t sizes[4]; + void * bufs[4]; + + pass = TRUE; + + if (mpi_rank == 0) { + + if (xfer_mode == H5FD_MPIO_INDEPENDENT) { + + sprintf(test_title, "parallel vector write test 5 -- %s / independent", vfd_name); + } + else if (coll_opt_mode == H5FD_MPIO_INDIVIDUAL_IO) { + + sprintf(test_title, "parallel vector write test 5 -- %s / col op / ind I/O", vfd_name); + } + else { + + HDassert(coll_opt_mode == H5FD_MPIO_COLLECTIVE_IO); + + sprintf(test_title, "parallel vector write test 5 -- %s / col op / col I/O", vfd_name); + } + + TESTING(test_title); + } + + show_progress = ((show_progress) && (mpi_rank == 0)); + + if (show_progress) + HDfprintf(stdout, "\n%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 1) Open the test file with the specified VFD, set the eoa, and setup the dxpl */ + if (pass) { + + eoa = (haddr_t)mpi_size * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t)); + + setup_vfd_test_file(file_name_id, filename, mpi_size, xfer_mode, coll_opt_mode, vfd_name, eoa, &lf, + &fapl_id, &dxpl_id); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 2) Set the test file in a known state by writing zeros + * to all bytes in the test file. Since we have already + * tested this, do this via a vector write of zero_fi_buf. + */ + if (pass) { + + count = 1; + types[0] = H5FD_MEM_DRAW; + addrs[0] = (haddr_t)mpi_rank * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t)); + sizes[0] = (size_t)INTS_PER_RANK * sizeof(int32_t); + bufs[0] = (void *)(&(zero_fi_buf[mpi_rank * INTS_PER_RANK])); + + if (H5FDwrite_vector(lf, dxpl_id, count, types, addrs, sizes, bufs) < 0) { + + pass = FALSE; + failure_mssg = "H5FDwrite_vector() failed.\n"; + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 3) Barrier + */ + + if (pass) { + + MPI_Barrier(MPI_COMM_WORLD); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 4) For each rank, define base_index equal to: + * + * mpi_rank * INTS_PER_RANK + * + * and define base_addr equal to + * + * base_index * sizeof(int32_t). + * + * Setup a vector write between base_addr and + * base_addr + INTS_PER_RANK * sizeof(int32_t) - 1 + * as follows: + */ + if (pass) { + + base_index = mpi_rank * INTS_PER_RANK; + base_addr = (haddr_t)((size_t)base_index * sizeof(int32_t)); + + if ((mpi_rank % 4) == 0) { + + /* if ( rank % 4 == 0 ) construct a vector that writes: + * + * negative_fi_buf starting at base_index + + * INTS_PER_RANK / 2 and running for INTS_PER_RANK / 4 + * entries, + * + * decreasing_fi_buf starting at base_index + + * INTS_PER_RANK / 4 and running for INTS_PER_RANK / 8 + * entries, and + * + * increasing_fi_buf starting at base_index + + * INTS_PER_RANK / 16 and running for INTS_PER_RANK / 16 + * entries + * + * to the equivalent locations in the file. + */ + count = 3; + + types[0] = H5FD_MEM_DRAW; + addrs[0] = base_addr + (haddr_t)((size_t)(INTS_PER_RANK / 2) * sizeof(int32_t)); + sizes[0] = (size_t)(INTS_PER_RANK / 4) * sizeof(int32_t); + bufs[0] = (void *)(&(negative_fi_buf[base_index + (INTS_PER_RANK / 2)])); + + types[1] = H5FD_MEM_DRAW; + addrs[1] = base_addr + (haddr_t)((size_t)(INTS_PER_RANK / 4) * sizeof(int32_t)); + sizes[1] = (size_t)(INTS_PER_RANK / 8) * sizeof(int32_t); + bufs[1] = (void *)(&(decreasing_fi_buf[base_index + (INTS_PER_RANK / 4)])); + + types[2] = H5FD_MEM_DRAW; + addrs[2] = base_addr + (haddr_t)((size_t)(INTS_PER_RANK / 16) * sizeof(int32_t)); + sizes[2] = (size_t)(INTS_PER_RANK / 16) * sizeof(int32_t); + bufs[2] = (void *)(&(increasing_fi_buf[base_index + (INTS_PER_RANK / 16)])); + } + else if ((mpi_rank % 4) == 1) { + + /* if ( rank % 4 == 1 ) construct a vector that writes: + * + * increasing_fi_buf starting at base_index + 1 and + * running for (INTS_PER_RANK / 2) - 2 entries, and + * + * decreasing_fi_buf startomg at base_addr + + * INTS_PER_RANK / 2 + 1 and running for (INTS_PER_RANK / 2) + * - 2 entries + * + * to the equivalent locations in the file. + */ + count = 2; + + types[0] = H5FD_MEM_DRAW; + addrs[0] = base_addr + (haddr_t)(sizeof(int32_t)); + sizes[0] = (size_t)((INTS_PER_RANK / 2) - 2) * sizeof(int32_t); + bufs[0] = (void *)(&(increasing_fi_buf[base_index + 1])); + + types[1] = H5FD_MEM_DRAW; + addrs[1] = base_addr + (haddr_t)((size_t)((INTS_PER_RANK / 2) + 1) * sizeof(int32_t)); + sizes[1] = (size_t)((INTS_PER_RANK / 2) - 2) * sizeof(int32_t); + bufs[1] = (void *)(&(decreasing_fi_buf[base_index + (INTS_PER_RANK / 2) + 1])); + } + else if ((mpi_rank % 4) == 2) { + + /* if ( rank % 4 == 2 ) construct a vector that writes: + * + * negative_fi_buf starting at base_index + + * INTS_PER_RANK / 2 and running for one entry. + * + * to the equivalent location in the file. + */ + count = 1; + + types[0] = H5FD_MEM_DRAW; + addrs[0] = base_addr + (haddr_t)((size_t)(INTS_PER_RANK / 2) * sizeof(int32_t)); + sizes[0] = sizeof(int32_t); + bufs[0] = (void *)(&(negative_fi_buf[base_index + (INTS_PER_RANK / 2)])); + } + else if ((mpi_rank % 4) == 3) { + + /* if ( rank % 4 == 3 ) construct and write the empty vector */ + + count = 0; + } + + if (H5FDwrite_vector(lf, dxpl_id, count, types, addrs, sizes, bufs) < 0) { + + pass = FALSE; + failure_mssg = "H5FDwrite_vector() failed (1).\n"; + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 5) Barrier */ + + if (pass) { + + MPI_Barrier(MPI_COMM_WORLD); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 6) On each rank, read the entire file into the read_fi_buf, + * and compare against increasing_fi_buf, + * decreasing_fi_buf, negative_fi_buf, and zero_fi_buf as + * appropriate. Report failure if any differences are + * detected. + */ + + if (pass) { + + size_t image_size = (size_t)mpi_size * (size_t)INTS_PER_RANK * sizeof(int32_t); + + if (H5FDread(lf, H5FD_MEM_DRAW, H5P_DEFAULT, (haddr_t)0, image_size, (void *)read_fi_buf) < 0) { + + pass = FALSE; + failure_mssg = "H5FDread() failed.\n"; + } + + for (i = 0; ((pass) && (i < mpi_size)); i++) { + + base_index = i * INTS_PER_RANK; + + for (j = base_index; j < base_index + INTS_PER_RANK; j++) { + + k = j - base_index; + + switch (i % 4) { + + case 0: + if (((INTS_PER_RANK / 2) <= k) && (k < (3 * (INTS_PER_RANK / 4)))) { + + if (read_fi_buf[j] != negative_fi_buf[j]) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (1.1)"; + } + } + else if (((INTS_PER_RANK / 4) <= k) && (k < (3 * (INTS_PER_RANK / 8)))) { + + if (read_fi_buf[j] != decreasing_fi_buf[j]) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (1.2)"; + } + } + else if (((INTS_PER_RANK / 16) <= k) && (k < (INTS_PER_RANK / 8))) { + + if (read_fi_buf[j] != increasing_fi_buf[j]) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (1.3)"; + } + } + else { + + if (read_fi_buf[j] != 0) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (1.4)"; + } + } + break; + + case 1: + if ((1 <= k) && (k <= ((INTS_PER_RANK / 2) - 2))) { + + if (read_fi_buf[j] != increasing_fi_buf[j]) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (2.1)"; + } + } + else if ((((INTS_PER_RANK / 2) + 1) <= k) && (k <= (INTS_PER_RANK - 2))) { + + if (read_fi_buf[j] != decreasing_fi_buf[j]) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (2.2)"; + } + } + else { + + if (read_fi_buf[j] != 0) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (2.3)"; + } + } + break; + + case 2: + if (k == INTS_PER_RANK / 2) { + + if (read_fi_buf[j] != negative_fi_buf[j]) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (3.1)"; + } + } + else { + + if (read_fi_buf[j] != 0) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (3.2)"; + } + } + break; + + case 3: + if (read_fi_buf[j] != 0) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (4)"; + } + break; + + default: + HDassert(FALSE); /* should be un-reachable */ + break; + } + } + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 7) Close the test file and delete it (on rank 0 only). + * Close FAPL and DXPL. + */ + + if (pass) { + + takedown_vfd_test_file(mpi_rank, filename, &lf, &fapl_id, &dxpl_id); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* report results */ + if (mpi_rank == 0) { + + if (pass) { + + PASSED(); + } + else { + + H5_FAILED(); + + if (show_progress) { + HDfprintf(stdout, "%s: failure_mssg = \"%s\"\n", fcn_name, failure_mssg); + } + } + } + + return (!pass); + +} /* vector_write_test_5() */ + +/*------------------------------------------------------------------------- + * Function: vector_write_test_6() + * + * Purpose: Test correct management of the sizes[] array optimization, + * where, if sizes[i] == 0, we use sizes[i - 1] as the value + * of size[j], for j >= i. + * + * 1) Open the test file with the specified VFD, set the eoa. + * and setup the DXPL. + * + * 2) Using rank zero, write the entire zero_fi_buf to + * the file. + * + * 3) Barrier + * + * 4) For each rank, define base_index equal to: + * + * mpi_rank * INTS_PER_RANK + * + * and define base_addr equal to + * + * base_index * sizeof(int32_t). + * + * Setup a vector write from increasing_fi_buf between + * base_addr and base_addr + INTS_PER_RANK * + * sizeof(int32_t) - 1 that writes every 16th integer + * located in that range starting at base_addr. + * Use a sizes[] array of length 2, with sizes[0] set + * to sizeof(int32_t), and sizes[1] = 0. + * + * Write the integers into the corresponding locations in + * the file. + * + * 5) Barrier + * + * 6) On each rank, read the entire file into the read_fi_buf, + * and compare against zero_fi_buf, and increasing_fi_buf + * as appropriate. Report failure if any differences are + * detected. + * + * 7) Barrier. + * + * 8) Close the test file. + * + * 9) On rank 0, delete the test file. + * + * Return: FALSE on success, TRUE if any errors are detected. + * + * Programmer: John Mainzer + * 3/26/21 + * + * Modifications: + * + * None. + * + *------------------------------------------------------------------------- + */ + +static unsigned +vector_write_test_6(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode, + H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name) +{ + const char *fcn_name = "vector_write_test_6()"; + char test_title[120]; + char filename[512]; + haddr_t eoa; + haddr_t base_addr; + hbool_t show_progress = FALSE; + hid_t fapl_id = -1; /* file access property list ID */ + hid_t dxpl_id = -1; /* data access property list ID */ + H5FD_t * lf = NULL; /* VFD struct ptr */ + int cp = 0; + int i; + int base_index; + uint32_t count = 0; + H5FD_mem_t types[(INTS_PER_RANK / 16) + 1]; + haddr_t addrs[(INTS_PER_RANK / 16) + 1]; + size_t sizes[2]; + void * bufs[(INTS_PER_RANK / 16) + 1]; + + pass = TRUE; + + if (mpi_rank == 0) { + + if (xfer_mode == H5FD_MPIO_INDEPENDENT) { + + sprintf(test_title, "parallel vector write test 6 -- %s / independent", vfd_name); + } + else if (coll_opt_mode == H5FD_MPIO_INDIVIDUAL_IO) { + + sprintf(test_title, "parallel vector write test 6 -- %s / col op / ind I/O", vfd_name); + } + else { + + HDassert(coll_opt_mode == H5FD_MPIO_COLLECTIVE_IO); + + sprintf(test_title, "parallel vector write test 6 -- %s / col op / col I/O", vfd_name); + } + + TESTING(test_title); + } + + show_progress = ((show_progress) && (mpi_rank == 0)); + + if (show_progress) + HDfprintf(stdout, "\n%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 1) Open the test file with the specified VFD, set the eoa, and setup the dxpl */ + if (pass) { + + eoa = (haddr_t)mpi_size * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t)); + + setup_vfd_test_file(file_name_id, filename, mpi_size, xfer_mode, coll_opt_mode, vfd_name, eoa, &lf, + &fapl_id, &dxpl_id); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 2) Using rank zero, write the entire negative_fi_buf to + * the file. + */ + if (pass) { + + size_t image_size = (size_t)mpi_size * (size_t)INTS_PER_RANK * sizeof(int32_t); + + if (mpi_rank == 0) { + + if (H5FDwrite(lf, H5FD_MEM_DRAW, H5P_DEFAULT, (haddr_t)0, image_size, (void *)zero_fi_buf) < 0) { + + pass = FALSE; + failure_mssg = "H5FDwrite() on rank 0 failed.\n"; + } + } + } + + /* 3) Barrier */ + + if (pass) { + + MPI_Barrier(MPI_COMM_WORLD); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 4) For each rank, define base_index equal to: + * + * mpi_rank * INTS_PER_RANK + * + * and define base_addr equal to + * + * base_index * sizeof(int32_t). + * + * Setup a vector write from increasing_fi_buf between + * base_addr and base_addr + INTS_PER_RANK * + * sizeof(int32_t) - 1 that writes every 16th integer + * located in that range starting at base_addr. + * Use a sizes[] array of length 2, with sizes[0] set + * to sizeof(int32_t), and sizes[1] = 0. + * + * Write the integers into the corresponding locations in + * the file. + */ + if (pass) { + + base_index = (mpi_rank * INTS_PER_RANK); + base_addr = (haddr_t)base_index * (haddr_t)sizeof(int32_t); + + count = INTS_PER_RANK / 16; + sizes[0] = sizeof(int32_t); + sizes[1] = 0; + + for (i = 0; i < INTS_PER_RANK / 16; i++) { + + types[i] = H5FD_MEM_DRAW; + addrs[i] = base_addr + ((haddr_t)(16 * i) * (haddr_t)sizeof(int32_t)); + bufs[i] = (void *)(&(increasing_fi_buf[base_index + (i * 16)])); + } + + if (H5FDwrite_vector(lf, dxpl_id, count, types, addrs, sizes, bufs) < 0) { + + pass = FALSE; + failure_mssg = "H5FDwrite_vector() failed (1).\n"; + } + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 5) Barrier */ + + if (pass) { + + MPI_Barrier(MPI_COMM_WORLD); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 6) On each rank, read the entire file into the read_fi_buf, + * and compare against zero_fi_buf, and increasing_fi_buf + * as appropriate. Report failure if any differences are + * detected. + */ + if (pass) { + + size_t image_size = (size_t)mpi_size * (size_t)INTS_PER_RANK * sizeof(int32_t); + + if (H5FDread(lf, H5FD_MEM_DRAW, H5P_DEFAULT, (haddr_t)0, image_size, (void *)read_fi_buf) < 0) { + + pass = FALSE; + failure_mssg = "H5FDread() failed.\n"; + } + + for (i = 0; ((pass) && (i < mpi_size * INTS_PER_RANK)); i++) { + + if (i % 16 == 0) { + + if (read_fi_buf[i] != increasing_fi_buf[i]) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (1)"; + } + } + else if (read_fi_buf[i] != zero_fi_buf[i]) { + + pass = FALSE; + failure_mssg = "unexpected data read from file (2)"; + } + } + } /* end if */ + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 7) Barrier */ + + if (pass) { + + MPI_Barrier(MPI_COMM_WORLD); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 8) Close the test file and delete it (on rank 0 only). + * Close FAPL and DXPL. + */ + + if (pass) { + + takedown_vfd_test_file(mpi_rank, filename, &lf, &fapl_id, &dxpl_id); + } + + if (show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* report results */ + if (mpi_rank == 0) { + + if (pass) { + + PASSED(); + } + else { + + H5_FAILED(); + + if (show_progress) { + HDfprintf(stdout, "%s: failure_mssg = \"%s\"\n", fcn_name, failure_mssg); + } + } + } + + return (!pass); + +} /* vector_write_test_6() */ + +/*------------------------------------------------------------------------- + * Function: main + * + * Purpose: Run parallel VFD tests. + * + * Return: Success: 0 + * + * Failure: 1 + * + * Programmer: John Mainzer + * 3/2621/ + * + * Modifications: + * + *------------------------------------------------------------------------- + */ + +int +main(int argc, char **argv) +{ + unsigned nerrs = 0; + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Info info = MPI_INFO_NULL; + int mpi_size; + int mpi_rank; + + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + + /* Attempt to turn off atexit post processing so that in case errors + * occur during the test and the process is aborted, it will not hang + * in the atexit post processing. If it does, it may try to make MPI + * calls which may not work. + */ + if (H5dont_atexit() < 0) + HDprintf("%d:Failed to turn off atexit processing. Continue.\n", mpi_rank); + + H5open(); + + if (mpi_rank == 0) { + HDprintf("=========================================\n"); + HDprintf("Parallel virtual file driver (VFD) tests\n"); + HDprintf(" mpi_size = %d\n", mpi_size); + HDprintf("=========================================\n"); + } + + if (mpi_size < 2) { + if (mpi_rank == 0) + HDprintf(" Need at least 2 processes. Exiting.\n"); + goto finish; + } + + alloc_and_init_file_images(mpi_size); + + if (!pass) { + + HDprintf("\nAllocation and initialze of file image buffers failed. Test aborted.\n"); + } + + MPI_Barrier(MPI_COMM_WORLD); + + // sleep(60); + + nerrs += + vector_read_test_1(0, mpi_rank, mpi_size, H5FD_MPIO_INDEPENDENT, H5FD_MPIO_INDIVIDUAL_IO, "mpio"); + nerrs += vector_read_test_1(0, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_INDIVIDUAL_IO, "mpio"); + nerrs += vector_read_test_1(0, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, "mpio"); + + nerrs += + vector_read_test_2(1, mpi_rank, mpi_size, H5FD_MPIO_INDEPENDENT, H5FD_MPIO_INDIVIDUAL_IO, "mpio"); + nerrs += vector_read_test_2(1, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_INDIVIDUAL_IO, "mpio"); + nerrs += vector_read_test_2(1, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, "mpio"); + + nerrs += + vector_read_test_3(2, mpi_rank, mpi_size, H5FD_MPIO_INDEPENDENT, H5FD_MPIO_INDIVIDUAL_IO, "mpio"); + nerrs += vector_read_test_3(2, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_INDIVIDUAL_IO, "mpio"); + nerrs += vector_read_test_3(2, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, "mpio"); + + nerrs += + vector_read_test_4(3, mpi_rank, mpi_size, H5FD_MPIO_INDEPENDENT, H5FD_MPIO_INDIVIDUAL_IO, "mpio"); + nerrs += vector_read_test_4(3, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_INDIVIDUAL_IO, "mpio"); + nerrs += vector_read_test_4(3, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, "mpio"); + + nerrs += + vector_read_test_5(4, mpi_rank, mpi_size, H5FD_MPIO_INDEPENDENT, H5FD_MPIO_INDIVIDUAL_IO, "mpio"); + nerrs += vector_read_test_5(4, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_INDIVIDUAL_IO, "mpio"); + nerrs += vector_read_test_5(4, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, "mpio"); + + nerrs += + vector_write_test_1(0, mpi_rank, mpi_size, H5FD_MPIO_INDEPENDENT, H5FD_MPIO_INDIVIDUAL_IO, "mpio"); + nerrs += + vector_write_test_1(0, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_INDIVIDUAL_IO, "mpio"); + nerrs += + vector_write_test_1(0, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, "mpio"); + + nerrs += + vector_write_test_2(1, mpi_rank, mpi_size, H5FD_MPIO_INDEPENDENT, H5FD_MPIO_INDIVIDUAL_IO, "mpio"); + nerrs += + vector_write_test_2(1, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_INDIVIDUAL_IO, "mpio"); + nerrs += + vector_write_test_2(1, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, "mpio"); + + nerrs += + vector_write_test_3(2, mpi_rank, mpi_size, H5FD_MPIO_INDEPENDENT, H5FD_MPIO_INDIVIDUAL_IO, "mpio"); + nerrs += + vector_write_test_3(2, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_INDIVIDUAL_IO, "mpio"); + nerrs += + vector_write_test_3(2, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, "mpio"); + + nerrs += + vector_write_test_4(3, mpi_rank, mpi_size, H5FD_MPIO_INDEPENDENT, H5FD_MPIO_INDIVIDUAL_IO, "mpio"); + nerrs += + vector_write_test_4(3, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_INDIVIDUAL_IO, "mpio"); + nerrs += + vector_write_test_4(3, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, "mpio"); + + nerrs += + vector_write_test_5(4, mpi_rank, mpi_size, H5FD_MPIO_INDEPENDENT, H5FD_MPIO_INDIVIDUAL_IO, "mpio"); + nerrs += + vector_write_test_5(4, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_INDIVIDUAL_IO, "mpio"); + nerrs += + vector_write_test_5(4, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, "mpio"); + + nerrs += + vector_write_test_6(5, mpi_rank, mpi_size, H5FD_MPIO_INDEPENDENT, H5FD_MPIO_INDIVIDUAL_IO, "mpio"); + nerrs += + vector_write_test_6(5, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_INDIVIDUAL_IO, "mpio"); + nerrs += + vector_write_test_6(5, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, "mpio"); + +finish: + + /* make sure all processes are finished before final report, cleanup + * and exit. + */ + MPI_Barrier(MPI_COMM_WORLD); + + if (mpi_rank == 0) { /* only process 0 reports */ + HDprintf("===================================\n"); + if (nerrs > 0) + HDprintf("***parallel vfd tests detected %d failures***\n", nerrs); + else + HDprintf("parallel vfd tests finished with no failures\n"); + HDprintf("===================================\n"); + } + + /* discard the file image buffers */ + free_file_images(); + + /* close HDF5 library */ + H5close(); + + /* MPI_Finalize must be called AFTER H5close which may use MPI calls */ + MPI_Finalize(); + + /* cannot just return (nerrs) because exit code is limited to 1byte */ + return (nerrs > 0); + +} /* main() */ -- cgit v0.12