summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authormainzer <mainzer#hdfgroup.org>2021-04-20 23:40:45 (GMT)
committermainzer <mainzer#hdfgroup.org>2021-04-20 23:40:45 (GMT)
commit5a61dfb242371f233c54c391f6919055a020df5e (patch)
tree87043d0363cc0017692dd7753acddbb719d5d2ec
parent0c4922865fb9715b75f658fc308e2d7d6c847b77 (diff)
downloadhdf5-5a61dfb242371f233c54c391f6919055a020df5e.zip
hdf5-5a61dfb242371f233c54c391f6919055a020df5e.tar.gz
hdf5-5a61dfb242371f233c54c391f6919055a020df5e.tar.bz2
Added vector read / write support to the MPIO VFD, with associated
test code (see testpar/t_vfd.c). Note that this implementation does NOT support vector entries of size greater than 2 GB. This must be repaired before release, but it should be good enough for correctness testing. As MPIO requires vector I/O requests to be sorted in increasing address order, also added a vector sort utility in H5FDint.c This function is tested in passing by the MPIO vector I/O extension. In passing, repaired a bug in size / type vector extension management in H5FD_read/write_vector() Tested parallel debug and production on charis and Jelly.
-rw-r--r--MANIFEST1
-rw-r--r--src/H5FD.c4
-rw-r--r--src/H5FDint.c354
-rw-r--r--src/H5FDmpio.c908
-rw-r--r--src/H5FDprivate.h4
-rw-r--r--testpar/Makefile.am2
-rw-r--r--testpar/t_vfd.c4057
7 files changed, 5282 insertions, 48 deletions
diff --git a/MANIFEST b/MANIFEST
index f386724..4d3c027 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -1462,6 +1462,7 @@
./testpar/t_pshutdown.c
./testpar/t_prestart.c
./testpar/t_span_tree.c
+./testpar/t_vfd.c
./testpar/t_init_term.c
./testpar/t_2Gio.c
./testpar/testpar.h
diff --git a/src/H5FD.c b/src/H5FD.c
index 1e9a812..5a853e3 100644
--- a/src/H5FD.c
+++ b/src/H5FD.c
@@ -1467,7 +1467,7 @@ H5FDread_vector(H5FD_t *file, hid_t dxpl_id, uint32_t count, H5FD_mem_t types[],
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_API(FAIL)
- H5TRACE7("e", "*xiIu*Mt*a*zx", file, dxpl_id, count, types, addrs, sizes, bufs);
+ H5TRACE7("e", "*#iIu*Mt*a*zx", file, dxpl_id, count, types, addrs, sizes, bufs);
/* Check arguments */
if (!file)
@@ -1548,7 +1548,7 @@ H5FDwrite_vector(H5FD_t *file, hid_t dxpl_id, uint32_t count, H5FD_mem_t types[]
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_API(FAIL)
- H5TRACE7("e", "*xiIu*Mt*a*z**x", file, dxpl_id, count, types, addrs, sizes, bufs);
+ H5TRACE7("e", "*#iIu*Mt*a*z**x", file, dxpl_id, count, types, addrs, sizes, bufs);
/* Check arguments */
if (!file)
diff --git a/src/H5FDint.c b/src/H5FDint.c
index 141e7ab..add69f3 100644
--- a/src/H5FDint.c
+++ b/src/H5FDint.c
@@ -44,6 +44,33 @@
/* Local Typedefs */
/******************/
+/*************************************************************************
+ *
+ * H5FD_vsrt_tmp_t
+ *
+ * Structure used to store vector I/O request addresses and the associated
+ * indexes in the addrs[] array for the purpose of determine the sorted
+ * order.
+ *
+ * This is done by allocating an array of H5FD_vsrt_tmp_t of length
+ * count, loading it with the contents of the addrs[] array and the
+ * associated indicies, and then sorting it.
+ *
+ * This sorted array of H5FD_vsrt_tmp_t is then used to populate sorted
+ * versions of the types[], addrs[], sizes[] and bufs[] vectors.
+ *
+ * addr: haddr_t containing the value of addrs[i],
+ *
+ * index: integer containing the value of i used to obtain the
+ * value of the addr field from the addrs[] vector.
+ *
+ *************************************************************************/
+
+typedef struct H5FD_vsrt_tmp_t {
+ haddr_t addr;
+ int index;
+} H5FD_vsrt_tmp_t;
+
/********************/
/* Package Typedefs */
/********************/
@@ -345,19 +372,48 @@ H5FD_read_vector(H5FD_t *file, uint32_t count, H5FD_mem_t types[], haddr_t addrs
* objects being written within the file by the application performing
* SWMR write operations.
*/
- if (!(file->access_flags & H5F_ACC_SWMR_READ)) {
+ if ((!(file->access_flags & H5F_ACC_SWMR_READ)) && (count > 0)) {
haddr_t eoa;
+ extend_sizes = FALSE;
+ extend_types = FALSE;
+
for (i = 0; i < count; i++) {
- if (HADDR_UNDEF == (eoa = (file->cls->get_eoa)(file, types[i])))
+ if (!extend_sizes) {
+
+ if (sizes[i] == 0) {
+
+ extend_sizes = TRUE;
+ size = sizes[i - 1];
+ }
+ else {
+
+ size = sizes[i];
+ }
+ }
+
+ if (!extend_types) {
+
+ if (types[i] == H5FD_MEM_NOLIST) {
+
+ extend_types = TRUE;
+ type = types[i - 1];
+ }
+ else {
+
+ type = types[i];
+ }
+ }
+
+ if (HADDR_UNDEF == (eoa = (file->cls->get_eoa)(file, type)))
HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver get_eoa request failed")
- if ((addrs[i] + sizes[i]) > eoa)
+ if ((addrs[i] + size) > eoa)
HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL,
"addr overflow, addrs[%d] = %llu, sizes[%d] = %llu, eoa = %llu", (int)i,
- (unsigned long long)(addrs[i]), (int)i, (unsigned long long)sizes[i],
+ (unsigned long long)(addrs[i]), (int)i, (unsigned long long)size,
(unsigned long long)eoa)
}
}
@@ -522,17 +578,46 @@ H5FD_write_vector(H5FD_t *file, uint32_t count, H5FD_mem_t types[], haddr_t addr
addrs_cooked = TRUE;
}
+ extend_sizes = FALSE;
+ extend_types = FALSE;
+
for (i = 0; i < count; i++) {
- if (HADDR_UNDEF == (eoa = (file->cls->get_eoa)(file, types[i])))
+ if (!extend_sizes) {
+
+ if (sizes[i] == 0) {
+
+ extend_sizes = TRUE;
+ size = sizes[i - 1];
+ }
+ else {
+
+ size = sizes[i];
+ }
+ }
+
+ if (!extend_types) {
+
+ if (types[i] == H5FD_MEM_NOLIST) {
+
+ extend_types = TRUE;
+ type = types[i - 1];
+ }
+ else {
+
+ type = types[i];
+ }
+ }
+
+ if (HADDR_UNDEF == (eoa = (file->cls->get_eoa)(file, type)))
HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver get_eoa request failed")
- if ((addrs[i] + sizes[i]) > eoa)
+ if ((addrs[i] + size) > eoa)
HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "addr overflow, addrs[%d] = %llu, sizes[%d] = %llu, \
eoa = %llu",
- (int)i, (unsigned long long)(addrs[i]), (int)i, (unsigned long long)sizes[i],
+ (int)i, (unsigned long long)(addrs[i]), (int)i, (unsigned long long)size,
(unsigned long long)eoa)
}
@@ -739,3 +824,258 @@ H5FD_driver_query(const H5FD_class_t *driver, unsigned long *flags /*out*/)
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_driver_query() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_sort_vector_io_req
+ *
+ * Purpose: Determine whether the supplied vector I/O request is
+ * sorted.
+ *
+ * if is is, set *vector_was_sorted to TRUE, set:
+ *
+ * *s_types_ptr = types
+ * *s_addrs_ptr = addrs
+ * *s_sizes_ptr = sizes
+ * *s_bufs_ptr = bufs
+ *
+ * and return.
+ *
+ * If it is not sorted, duplicate the type, addrs, sizes,
+ * and bufs vectors, storing the base addresses of the new
+ * vectors in *s_types_ptr, *s_addrs_ptr, *s_sizes_ptr, and
+ * *s_bufs_ptr respectively. Determine the sorted order
+ * of the vector I/O request, and load it into the new
+ * vectors in sorted order.
+ *
+ * Note that in this case, it is the callers responsibility
+ * to free the sorted vectors.
+ *
+ * JRM -- 3/15/21
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static int
+H5FD__vsrt_tmp_cmp(const void *element_1, const void *element_2)
+{
+ haddr_t addr_1 = ((const H5FD_vsrt_tmp_t *)element_1)->addr;
+ haddr_t addr_2 = ((const H5FD_vsrt_tmp_t *)element_2)->addr;
+ int ret_value = 0; /* Return value */
+
+ FUNC_ENTER_STATIC_NOERR
+
+ /* Sanity checks */
+ HDassert(H5F_addr_defined(addr_1));
+ HDassert(H5F_addr_defined(addr_2));
+
+ if (H5F_addr_gt(addr_1, addr_2)) {
+
+ ret_value = 1;
+ }
+ else if (H5F_addr_lt(addr_1, addr_2)) {
+
+ ret_value = -1;
+ }
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5FD__vsrt_tmp_cmp() */
+
+herr_t
+H5FD_sort_vector_io_req(hbool_t *vector_was_sorted, uint32_t count, H5FD_mem_t types[], haddr_t addrs[],
+ size_t sizes[], void *bufs[], H5FD_mem_t **s_types_ptr, haddr_t **s_addrs_ptr,
+ size_t **s_sizes_ptr, void ***s_bufs_ptr)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+ int i;
+ struct H5FD_vsrt_tmp_t *srt_tmp = NULL;
+
+ FUNC_ENTER_NOAPI(FAIL)
+
+ /* Sanity checks */
+ HDassert(vector_was_sorted);
+
+ HDassert((types) || (count == 0));
+ HDassert((addrs) || (count == 0));
+ HDassert((sizes) || (count == 0));
+ HDassert((bufs) || (count == 0));
+
+ /* verify that the first elements of the sizes and types arrays are
+ * valid.
+ */
+ HDassert((count == 0) || (sizes[0] != 0));
+ HDassert((count == 0) || (types[0] != H5FD_MEM_NOLIST));
+
+ HDassert((count == 0) || ((s_types_ptr) && (NULL == *s_types_ptr)));
+ HDassert((count == 0) || ((s_addrs_ptr) && (NULL == *s_addrs_ptr)));
+ HDassert((count == 0) || ((s_sizes_ptr) && (NULL == *s_sizes_ptr)));
+ HDassert((count == 0) || ((s_bufs_ptr) && (NULL == *s_bufs_ptr)));
+
+ *vector_was_sorted = TRUE;
+
+ /* if count <= 1, vector is sorted by definition */
+ if (count > 1) {
+
+ /* scan the addrs array to see if it is sorted */
+ i = 1;
+
+ while ((*vector_was_sorted) && (i < (int)(count - 1))) {
+
+ if (H5F_addr_gt(addrs[i - 1], addrs[i])) {
+
+ *vector_was_sorted = FALSE;
+ }
+ else if (H5F_addr_eq(addrs[i - 1], addrs[i])) {
+
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "duplicate addr in vector")
+ }
+ i++;
+ }
+ }
+
+ if (*vector_was_sorted) {
+
+ *s_types_ptr = types;
+ *s_addrs_ptr = addrs;
+ *s_sizes_ptr = sizes;
+ *s_bufs_ptr = bufs;
+ }
+ else {
+
+ /* must sort the addrs array in increasing addr order, while
+ * maintaining the association between each addr, and the
+ * sizes[], types[], and bufs[] values at the same index.
+ *
+ * Do this by allocating an array of struct H5FD_vsrt_tmp_t, where
+ * each instance of H5FD_vsrt_tmp_t has two fields, addr and index.
+ * Load the array with the contents of the addrs array and
+ * the index of the associated entry. Sort the array, allocate
+ * the s_types_ptr, s_addrs_ptr, s_sizes_ptr, and s_bufs_ptr
+ * arrays and populate them using the mapping provided by
+ * the sorted array of H5FD_vsrt_tmp_t.
+ */
+ int j;
+ int fixed_size_index = (int)count;
+ int fixed_type_index = (int)count;
+ size_t srt_tmp_size;
+
+ srt_tmp_size = ((size_t)count * sizeof(struct H5FD_vsrt_tmp_t));
+
+ if (NULL == (srt_tmp = (H5FD_vsrt_tmp_t *)HDmalloc(srt_tmp_size)))
+
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't alloc srt_tmp")
+
+ for (i = 0; i < (int)count; i++) {
+
+ srt_tmp[i].addr = addrs[i];
+ srt_tmp[i].index = i;
+ }
+
+ /* sort the srt_tmp array */
+ HDqsort(srt_tmp, (size_t)count, sizeof(struct H5FD_vsrt_tmp_t), H5FD__vsrt_tmp_cmp);
+
+ /* verify no duplicate entries */
+ i = 1;
+
+ while ((*vector_was_sorted) && (i < (int)(count - 1))) {
+
+ HDassert(H5F_addr_lt(srt_tmp[i - 1].addr, srt_tmp[i].addr));
+
+ if (H5F_addr_eq(addrs[i - 1], addrs[i])) {
+
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "duplicate addr in vector")
+ }
+ i++;
+ }
+
+ if ((NULL == (*s_types_ptr = (H5FD_mem_t *)HDmalloc((size_t)count * sizeof(H5FD_mem_t)))) ||
+ (NULL == (*s_addrs_ptr = (haddr_t *)HDmalloc((size_t)count * sizeof(haddr_t)))) ||
+ (NULL == (*s_sizes_ptr = (size_t *)HDmalloc((size_t)count * sizeof(size_t)))) ||
+ (NULL == (*s_bufs_ptr = (void *)HDmalloc((size_t)count * sizeof(void *))))) {
+
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't alloc sorted vector(s)")
+ }
+
+ HDassert(sizes[0] != 0);
+ HDassert(types[0] != H5FD_MEM_NOLIST);
+
+ /* scan the sizes and types vectors to determine if the fixed size / type
+ * optimization is in use, and if so, to determine the index of the last
+ * valid value on each vector.
+ */
+ i = 0;
+ while ((i < (int)count) && ((fixed_size_index == (int)count) || (fixed_type_index == (int)count))) {
+
+ if ((fixed_size_index == (int)count) && (sizes[i] == 0)) {
+
+ fixed_size_index = i - 1;
+ }
+
+ if ((fixed_type_index == (int)count) && (types[i] == H5FD_MEM_NOLIST)) {
+
+ fixed_type_index = i - 1;
+ }
+
+ i++;
+ }
+
+ HDassert((fixed_size_index >= 0) && (fixed_size_index <= (int)count));
+ HDassert((fixed_type_index >= 0) && (fixed_size_index <= (int)count));
+
+ /* populate the sorted vectors */
+ for (i = 0; i < (int)count; i++) {
+
+ j = srt_tmp[i].index;
+
+ (*s_types_ptr)[j] = types[MIN(i, fixed_type_index)];
+ (*s_addrs_ptr)[j] = addrs[i];
+ (*s_sizes_ptr)[j] = sizes[MIN(i, fixed_size_index)];
+ (*s_bufs_ptr)[j] = bufs[i];
+ }
+ }
+
+done:
+ if (srt_tmp) {
+
+ HDfree(srt_tmp);
+ srt_tmp = NULL;
+ }
+
+ /* On failure, free the sorted vectors if they were allocated.
+ * Note that we only allocate these vectors if the original array
+ * was not sorted -- thus we check both for failure, and for
+ * the flag indicating that the original vector was not sorted
+ * in increasing address order.
+ */
+ if ((ret_value != SUCCEED) && (!(*vector_was_sorted))) {
+
+ /* free space allocated for sorted vectors */
+ if (*s_types_ptr) {
+
+ HDfree(*s_types_ptr);
+ *s_types_ptr = NULL;
+ }
+
+ if (*s_addrs_ptr) {
+
+ HDfree(*s_addrs_ptr);
+ *s_addrs_ptr = NULL;
+ }
+
+ if (*s_sizes_ptr) {
+
+ HDfree(*s_sizes_ptr);
+ *s_sizes_ptr = NULL;
+ }
+
+ if (*s_bufs_ptr) {
+
+ HDfree(*s_bufs_ptr);
+ *s_bufs_ptr = NULL;
+ }
+ }
+
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5FD_sort_vector_io_req() */
diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c
index b5c2684..3ec6802 100644
--- a/src/H5FDmpio.c
+++ b/src/H5FDmpio.c
@@ -84,6 +84,10 @@ static herr_t H5FD__mpio_read(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, h
void *buf);
static herr_t H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size,
const void *buf);
+static herr_t H5FD__mpio_read_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t count,
+ H5FD_mem_t types[], haddr_t addrs[], size_t sizes[], void *bufs[]);
+static herr_t H5FD__mpio_write_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t count,
+ H5FD_mem_t types[], haddr_t addrs[], size_t sizes[], void *bufs[]);
static herr_t H5FD__mpio_flush(H5FD_t *_file, hid_t dxpl_id, hbool_t closing);
static herr_t H5FD__mpio_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing);
static int H5FD__mpio_mpi_rank(const H5FD_t *_file);
@@ -94,44 +98,44 @@ static MPI_Comm H5FD__mpio_communicator(const H5FD_t *_file);
static const H5FD_class_mpi_t H5FD_mpio_g = {
{
/* Start of superclass information */
- "mpio", /*name */
- HADDR_MAX, /*maxaddr */
- H5F_CLOSE_SEMI, /*fc_degree */
- H5FD__mpio_term, /*terminate */
- NULL, /*sb_size */
- NULL, /*sb_encode */
- NULL, /*sb_decode */
- 0, /*fapl_size */
- NULL, /*fapl_get */
- NULL, /*fapl_copy */
- NULL, /*fapl_free */
- 0, /*dxpl_size */
- NULL, /*dxpl_copy */
- NULL, /*dxpl_free */
- H5FD__mpio_open, /*open */
- H5FD__mpio_close, /*close */
- NULL, /*cmp */
- H5FD__mpio_query, /*query */
- NULL, /*get_type_map */
- NULL, /*alloc */
- NULL, /*free */
- H5FD__mpio_get_eoa, /*get_eoa */
- H5FD__mpio_set_eoa, /*set_eoa */
- H5FD__mpio_get_eof, /*get_eof */
- H5FD__mpio_get_handle, /*get_handle */
- H5FD__mpio_read, /*read */
- H5FD__mpio_write, /*write */
- NULL, /*read_vector */
- NULL, /*write_vector */
- H5FD__mpio_flush, /*flush */
- H5FD__mpio_truncate, /*truncate */
- NULL, /*lock */
- NULL, /*unlock */
- H5FD_FLMAP_DICHOTOMY /*fl_map */
- }, /* End of superclass information */
- H5FD__mpio_mpi_rank, /*get_rank */
- H5FD__mpio_mpi_size, /*get_size */
- H5FD__mpio_communicator /*get_comm */
+ "mpio", /*name */
+ HADDR_MAX, /*maxaddr */
+ H5F_CLOSE_SEMI, /*fc_degree */
+ H5FD__mpio_term, /*terminate */
+ NULL, /*sb_size */
+ NULL, /*sb_encode */
+ NULL, /*sb_decode */
+ 0, /*fapl_size */
+ NULL, /*fapl_get */
+ NULL, /*fapl_copy */
+ NULL, /*fapl_free */
+ 0, /*dxpl_size */
+ NULL, /*dxpl_copy */
+ NULL, /*dxpl_free */
+ H5FD__mpio_open, /*open */
+ H5FD__mpio_close, /*close */
+ NULL, /*cmp */
+ H5FD__mpio_query, /*query */
+ NULL, /*get_type_map */
+ NULL, /*alloc */
+ NULL, /*free */
+ H5FD__mpio_get_eoa, /*get_eoa */
+ H5FD__mpio_set_eoa, /*set_eoa */
+ H5FD__mpio_get_eof, /*get_eof */
+ H5FD__mpio_get_handle, /*get_handle */
+ H5FD__mpio_read, /*read */
+ H5FD__mpio_write, /*write */
+ H5FD__mpio_read_vector, /*read_vector */
+ H5FD__mpio_write_vector, /*write_vector */
+ H5FD__mpio_flush, /*flush */
+ H5FD__mpio_truncate, /*truncate */
+ NULL, /*lock */
+ NULL, /*unlock */
+ H5FD_FLMAP_DICHOTOMY /*fl_map */
+ }, /* End of superclass information */
+ H5FD__mpio_mpi_rank, /*get_rank */
+ H5FD__mpio_mpi_size, /*get_size */
+ H5FD__mpio_communicator /*get_comm */
};
#ifdef H5FDmpio_DEBUG
@@ -1521,6 +1525,834 @@ done:
} /* end H5FD__mpio_write() */
/*-------------------------------------------------------------------------
+ * Function: H5FD__mpio_read_vector()
+ *
+ * Purpose: The behaviour of this function dependes on the value of
+ * the io_xfer_mode obtained from the context.
+ *
+ * If it is H5FD_MPIO_COLLECTIVE, this is a collective
+ * operation, which allows us to use MPI_File_set_view, and
+ * then perform the entire vector read in a single MPI call.
+ *
+ * Do this (if count is positive), by constructing memory
+ * and file derived types from the supplied vector, using
+ * file type to set the file view, and then reading the
+ * the memory type from file. Note that this read is
+ * either independent or collective depending on the
+ * value of mpio_coll_opt -- again obtained from the context.
+ *
+ * If count is zero, participate in the collective read
+ * (if so configured) with an empty read.
+ *
+ * Finally, set the file view back to its default state.
+ *
+ * In contrast, if io_xfer_mode is H5FD_MPIO_INDEPENDENT,
+ * this call is independent, and thus we cannot use
+ * MPI_File_set_view().
+ *
+ * In this case, simply walk the vector, and issue an
+ * independent read for each entry.
+ *
+ * WARNING: At present, this function makes no provision
+ * entries of size greater than 2 GB in the vector. This
+ * will have to be fixed before release.
+ *
+ * Return: Success: SUCCEED.
+ * Failure: FAIL.
+ *
+ * Programmer: John Mainzer
+ * March 15, 2021
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD__mpio_read_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t count, H5FD_mem_t types[],
+ haddr_t addrs[], size_t sizes[], void *bufs[])
+{
+ H5FD_mpio_t * file = (H5FD_mpio_t *)_file;
+ hbool_t vector_was_sorted = TRUE;
+ hbool_t fixed_size = FALSE;
+ size_t size;
+ H5FD_mem_t * s_types = NULL;
+ haddr_t * s_addrs = NULL;
+ size_t * s_sizes = NULL;
+ void ** s_bufs = NULL;
+ int * mpi_block_lengths = NULL;
+ char unused = 0; /* Unused, except for non-NULL pointer value */
+ void * mpi_bufs_base = NULL;
+ MPI_Aint mpi_bufs_base_Aint;
+ MPI_Aint * mpi_bufs = NULL;
+ MPI_Aint * mpi_displacments = NULL;
+ MPI_Datatype buf_type = MPI_BYTE; /* MPI description of the selection in memory */
+ hbool_t buf_type_created = FALSE;
+ MPI_Datatype file_type = MPI_BYTE; /* MPI description of the selection in file */
+ hbool_t file_type_created = FALSE;
+ int i;
+ int j;
+ int mpi_code; /* MPI return code */
+ MPI_Offset mpi_off = 0;
+ MPI_Status mpi_stat; /* Status from I/O operation */
+ H5FD_mpio_xfer_t xfer_mode; /* I/O transfer mode */
+ H5FD_mpio_collective_opt_t coll_opt_mode; /* whether we are doing collective or independent I/O */
+ int size_i;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_STATIC
+
+#ifdef H5FDmpio_DEBUG
+ if (H5FD_mpio_Debug[(int)'t'])
+ HDfprintf(stdout, "%s: Entering\n", FUNC);
+#endif
+
+ /* Sanity checks */
+ HDassert(file);
+ HDassert(H5FD_MPIO == file->pub.driver_id);
+ HDassert((types) || (count == 0));
+ HDassert((addrs) || (count == 0));
+ HDassert((sizes) || (count == 0));
+ HDassert((bufs) || (count == 0));
+
+ /* verify that the first elements of the sizes and types arrays are
+ * valid.
+ */
+ HDassert((count == 0) || (sizes[0] != 0));
+ HDassert((count == 0) || (types[0] != H5FD_MEM_NOLIST));
+
+
+ /* sort the vector I/O request into increasing address order if required
+ *
+ * If the vector is already sorted, the base addresses of types, addrs, sizes,
+ * and bufs will be returned in s_types, s_addrs, s_sizes, and s_bufs respectively.
+ *
+ * If the vector was not already sorted, new, sorted versions of types, addrs, sizes, and bufs
+ * are allocated, populated, and returned in s_types, s_addrs, s_sizes, and s_bufs respectively.
+ * In this case, this function must free the memory allocated for the sorted vectors.
+ */
+ if (H5FD_sort_vector_io_req(&vector_was_sorted, count, types, addrs, sizes, bufs, &s_types, &s_addrs,
+ &s_sizes, &s_bufs) < 0)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "can't sort vector I/O request")
+
+
+ /* Get the transfer mode from the API context
+ *
+ * This flag is set to H5FD_MPIO_COLLECTIVE if the API call is
+ * collective, and to H5FD_MPIO_INDEPENDENT if it is not.
+ *
+ * While this doesn't mean that we are actually about to do a collective
+ * read, it does mean that all ranks are here, so we can use MPI_File_set_view().
+ */
+ if (H5CX_get_io_xfer_mode(&xfer_mode) < 0)
+ HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't get MPI-I/O transfer mode")
+
+ if (xfer_mode == H5FD_MPIO_COLLECTIVE) {
+
+ if (count > 0) { /* create MPI derived types describing the vector write */
+
+ if ((NULL == (mpi_block_lengths = (int *)HDmalloc((size_t)count * sizeof(int)))) ||
+ (NULL == (mpi_displacments = (MPI_Aint *)HDmalloc((size_t)count * sizeof(MPI_Aint)))) ||
+ (NULL == (mpi_bufs = (MPI_Aint *)HDmalloc((size_t)count * sizeof(MPI_Aint))))) {
+
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't alloc mpi block lengths / displacement")
+ }
+
+ /* when we setup mpi_bufs[] below, all addresses are offsets from
+ * mpi_bufs_base.
+ *
+ * Since these offsets must all be positive, we must scan through
+ * s_bufs[] to find the smallest value, and choose that for
+ * mpi_bufs_base.
+ */
+
+ j = 0; /* guess at the index of the smallest value of s_bufs[] */
+
+ for (i = 1; i < (int)count; i++) {
+
+ if (s_bufs[i] < s_bufs[j]) {
+
+ j = i;
+ }
+ }
+
+ mpi_bufs_base = s_bufs[j];
+
+ if (MPI_SUCCESS != (mpi_code = MPI_Get_address(mpi_bufs_base, &mpi_bufs_base_Aint)))
+
+ HMPI_GOTO_ERROR(FAIL, "MPI_Get_address for s_bufs[] to mpi_bufs_base failed", mpi_code)
+
+ size_i = 1;
+
+ fixed_size = FALSE;
+
+ /* load the mpi_block_lengths and mpi_displacements arrays */
+ for (i = 0; i < (int)count; i++) {
+
+ if (!fixed_size) {
+
+ if (sizes[i] == 0) {
+
+ fixed_size = TRUE;
+ size = sizes[i - 1];
+ }
+ else {
+
+ size = s_sizes[i];
+ }
+ }
+
+ /* There is an obvious possibility of an overflow here, as size_t
+ * will typically be 64 bits, where as int will typically be 32 bits.
+ * This must be fixed, but it should be good enough for initial
+ * correctness testing.
+ * JRM -- 3/17/21
+ */
+ mpi_block_lengths[i] = (int)size;
+ mpi_displacments[i] = (MPI_Aint)s_addrs[i];
+
+ /* convert s_bufs[i] to MPI_Aint... */
+ if (MPI_SUCCESS != (mpi_code = MPI_Get_address(s_bufs[i], &(mpi_bufs[i]))))
+
+ HMPI_GOTO_ERROR(FAIL, "MPI_Get_address for s_bufs[] - mpi_bufs_base failed", mpi_code)
+
+ /*... and then subtract mpi_bufs_base_Aint from it. */
+#if ((MPI_VERSION > 3) || ((MPI_VERSION == 3) && (MPI_SUBVERSION >= 1)))
+ mpi_bufs[i] = MPI_Aint_diff(mpi_bufs[i], mpi_bufs_base_Aint);
+#else
+ mpi_bufs[i] = mpi_bufs[i] - mpi_bufs_base_Aint;
+#endif
+ }
+
+
+ /* create the memory MPI derived types */
+ if (MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int)count, mpi_block_lengths, mpi_bufs,
+ MPI_BYTE, &buf_type)))
+
+ HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed for buf_type failed", mpi_code)
+
+ buf_type_created = TRUE;
+
+ if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(&buf_type)))
+
+ HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit for buf_type failed", mpi_code)
+
+
+ /* create the file MPI derived type */
+ if (MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int)count, mpi_block_lengths,
+ mpi_displacments, MPI_BYTE, &file_type)))
+
+ HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed for file_type failed", mpi_code)
+
+ file_type_created = TRUE;
+
+ if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(&file_type)))
+
+ HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit for file_type failed", mpi_code)
+ }
+ else {
+
+ /* setup for null participation in the collective operation. */
+
+ buf_type = MPI_BYTE;
+ file_type = MPI_BYTE;
+
+ /* Set non-NULL pointer for I/O operation */
+ mpi_bufs_base = (void *)(&unused);
+
+ /* MPI count to read */
+ size_i = 0;
+ }
+
+ /* Portably initialize MPI status variable */
+ HDmemset(&mpi_stat, 0, sizeof(MPI_Status));
+
+ /* some numeric conversions */
+ if (H5FD_mpi_haddr_to_MPIOff((haddr_t)0, &mpi_off) < 0)
+ HGOTO_ERROR(H5E_INTERNAL, H5E_BADRANGE, FAIL, "can't set MPI off to 0")
+
+#ifdef H5FDmpio_DEBUG
+ if (H5FD_mpio_Debug[(int)'r'])
+ HDfprintf(stdout, "%s: mpi_off = %ld size_i = %d\n", FUNC, (long)mpi_off, size_i);
+#endif
+
+ /* Setup the file view. */
+ if (MPI_SUCCESS != (mpi_code = MPI_File_set_view(file->f, mpi_off, MPI_BYTE, file_type,
+ H5FD_mpi_native_g, file->info)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_File_set_view failed", mpi_code)
+
+ /* Get the collective_opt property to check whether the application wants to do IO individually.
+ */
+ if (H5CX_get_mpio_coll_opt(&coll_opt_mode) < 0)
+
+ HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't get MPI-I/O collective_op property")
+
+ /* Read the data. */
+#ifdef H5FDmpio_DEBUG
+ if (H5FD_mpio_Debug[(int)'r'])
+ HDfprintf(stdout, "%s: using MPIO collective mode\n", FUNC);
+#endif
+
+ if (coll_opt_mode == H5FD_MPIO_COLLECTIVE_IO) {
+#ifdef H5FDmpio_DEBUG
+ if (H5FD_mpio_Debug[(int)'r'])
+ HDfprintf(stdout, "%s: doing MPI collective IO\n", FUNC);
+#endif
+
+ if (MPI_SUCCESS != (mpi_code = MPI_File_read_at_all(file->f, mpi_off, mpi_bufs_base, size_i,
+ buf_type, &mpi_stat)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_File_read_at_all failed", mpi_code)
+ } /* end if */
+ else if (size_i > 0) {
+#ifdef H5FDmpio_DEBUG
+ if (H5FD_mpio_Debug[(int)'r'])
+ HDfprintf(stdout, "%s: doing MPI independent IO\n", FUNC);
+#endif
+
+ if (MPI_SUCCESS !=
+ (mpi_code = MPI_File_read_at(file->f, mpi_off, mpi_bufs_base, size_i, buf_type, &mpi_stat)))
+
+ HMPI_GOTO_ERROR(FAIL, "MPI_File_read_at failed", mpi_code)
+
+ } /* end else */
+
+ /* Reset the file view */
+ if (MPI_SUCCESS != (mpi_code = MPI_File_set_view(file->f, (MPI_Offset)0, MPI_BYTE, MPI_BYTE,
+ H5FD_mpi_native_g, file->info)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_File_set_view failed", mpi_code)
+ }
+ else if (count > 0) {
+
+ /* The read is part of an independent operation. As a result,
+ * we can't use MPI_File_set_view() (since it it a collective operation),
+ * and thus there is no point in setting up an MPI derived type, as
+ * (to the best of my knowlege) MPI I/O doesn't have support for
+ * non-contiguous I/O in independent mode.
+ *
+ * Thus we have to read in each element of the vector in a separate
+ * MPI_File_read_at() call.
+ */
+
+ fixed_size = FALSE;
+
+#ifdef H5FDmpio_DEBUG
+ if (H5FD_mpio_Debug[(int)'r'])
+ HDfprintf(stdout, "%s: doing MPI independent IO\n", FUNC);
+#endif
+
+ for (i = 0; i < (int)count; i++) {
+
+ if (H5FD_mpi_haddr_to_MPIOff(s_addrs[i], &mpi_off) < 0)
+
+ HGOTO_ERROR(H5E_INTERNAL, H5E_BADRANGE, FAIL, "can't convert from haddr to MPI off")
+
+ if (!fixed_size) {
+
+ if (sizes[i] == 0) {
+
+ fixed_size = TRUE;
+ size = sizes[i - 1];
+ }
+ else {
+
+ size = s_sizes[i];
+ }
+ }
+
+ size_i = (int)size; /* todo: fix potential for overflow */
+
+ if (MPI_SUCCESS !=
+ (mpi_code = MPI_File_read_at(file->f, mpi_off, s_bufs[i], size_i, MPI_BYTE, &mpi_stat)))
+
+ HMPI_GOTO_ERROR(FAIL, "MPI_File_read_at failed", mpi_code)
+ }
+ }
+
+done:
+
+ if (!vector_was_sorted) { /* free sorted vectors if they exist */
+
+ if (s_types) {
+
+ HDfree(s_types);
+ s_types = NULL;
+ }
+
+ if (s_addrs) {
+
+ HDfree(s_addrs);
+ s_addrs = NULL;
+ }
+
+ if (s_sizes) {
+
+ HDfree(s_sizes);
+ s_sizes = NULL;
+ }
+
+ if (s_bufs) {
+
+ HDfree(s_bufs);
+ s_bufs = NULL;
+ }
+ }
+
+ if (mpi_block_lengths) {
+
+ HDfree(mpi_block_lengths);
+ mpi_block_lengths = NULL;
+ }
+
+ if (mpi_displacments) {
+
+ HDfree(mpi_displacments);
+ mpi_displacments = NULL;
+ }
+
+ if (mpi_bufs) {
+
+ HDfree(mpi_bufs);
+ mpi_bufs = NULL;
+ }
+
+ if (buf_type_created) {
+ MPI_Type_free(&buf_type);
+ }
+
+ if (file_type_created) {
+ MPI_Type_free(&file_type);
+ }
+
+#ifdef H5FDmpio_DEBUG
+ if (H5FD_mpio_Debug[(int)'t'])
+ HDfprintf(stdout, "%s: Leaving, proc %d: ret_value = %d\n", FUNC, file->mpi_rank, ret_value);
+#endif
+
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5FD__mpio_read_vector() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD__mpio_write_vector
+ *
+ * Purpose: The behaviour of this function dependes on the value of
+ * the io_xfer_mode obtained from the context.
+ *
+ * If it is H5FD_MPIO_COLLECTIVE, this is a collective
+ * operation, which allows us to use MPI_File_set_view, and
+ * then perform the entire vector write in a single MPI call.
+ *
+ * Do this (if count is positive), by constructing memory
+ * and file derived types from the supplied vector, using
+ * file type to set the file view, and then writing the
+ * the memory type to file. Note that this write is
+ * either independent or collective depending on the
+ * value of mpio_coll_opt -- again obtained from the context.
+ *
+ * If count is zero, participate in the collective write
+ * (if so configured) with an empty write.
+ *
+ * Finally, set the file view back to its default state.
+ *
+ * In contrast, if io_xfer_mode is H5FD_MPIO_INDEPENDENT,
+ * this call is independent, and thus we cannot use
+ * MPI_File_set_view().
+ *
+ * In this case, simply walk the vector, and issue an
+ * independent write for each entry.
+ *
+ * WARNING: At present, this function makes no provision
+ * entries of size greater than 2 GB in the vector. This
+ * will have to be fixed before release.
+ *
+ * Return: Success: SUCCEED.
+ * Failure: FAIL.
+ *
+ * Programmer: John Mainzer
+ * March 15, 2021
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD__mpio_write_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t count, H5FD_mem_t types[],
+ haddr_t addrs[], size_t sizes[], void *bufs[])
+{
+ H5FD_mpio_t * file = (H5FD_mpio_t *)_file;
+ hbool_t vector_was_sorted = TRUE;
+ hbool_t fixed_size = FALSE;
+ size_t size;
+ H5FD_mem_t * s_types = NULL;
+ haddr_t * s_addrs = NULL;
+ size_t * s_sizes = NULL;
+ void ** s_bufs = NULL;
+ int * mpi_block_lengths = NULL;
+ char unused = 0; /* Unused, except for non-NULL pointer value */
+ void * mpi_bufs_base = NULL;
+ MPI_Aint mpi_bufs_base_Aint;
+ MPI_Aint * mpi_bufs = NULL;
+ MPI_Aint * mpi_displacments = NULL;
+ MPI_Datatype buf_type = MPI_BYTE; /* MPI description of the selection in memory */
+ hbool_t buf_type_created = FALSE;
+ MPI_Datatype file_type = MPI_BYTE; /* MPI description of the selection in file */
+ hbool_t file_type_created = FALSE;
+ int i;
+ int j;
+ int mpi_code; /* MPI return code */
+ MPI_Offset mpi_off = 0;
+ MPI_Status mpi_stat; /* Status from I/O operation */
+ H5FD_mpio_xfer_t xfer_mode; /* I/O transfer mode */
+ H5FD_mpio_collective_opt_t coll_opt_mode; /* whether we are doing collective or independent I/O */
+ int size_i;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_STATIC
+
+#ifdef H5FDmpio_DEBUG
+ if (H5FD_mpio_Debug[(int)'t'])
+ HDfprintf(stdout, "%s: Entering\n", FUNC);
+#endif
+
+ /* Sanity checks */
+ HDassert(file);
+ HDassert(H5FD_MPIO == file->pub.driver_id);
+ HDassert((types) || (count == 0));
+ HDassert((addrs) || (count == 0));
+ HDassert((sizes) || (count == 0));
+ HDassert((bufs) || (count == 0));
+
+ /* verify that the first elements of the sizes and types arrays are
+ * valid.
+ */
+ HDassert((count == 0) || (sizes[0] != 0));
+ HDassert((count == 0) || (types[0] != H5FD_MEM_NOLIST));
+
+
+ /* Verify that no data is written when between MPI_Barrier()s during file flush */
+
+ HDassert(!H5CX_get_mpi_file_flushing());
+
+ /* sort the vector I/O request into increasing address order if required
+ *
+ * If the vector is already sorted, the base addresses of types, addrs, sizes,
+ * and bufs will be returned in s_types, s_addrs, s_sizes, and s_bufs respectively.
+ *
+ * If the vector was not already sorted, new, sorted versions of types, addrs, sizes, and bufs
+ * are allocated, populated, and returned in s_types, s_addrs, s_sizes, and s_bufs respectively.
+ * In this case, this function must free the memory allocated for the sorted vectors.
+ */
+ if (H5FD_sort_vector_io_req(&vector_was_sorted, count, types, addrs, sizes, bufs, &s_types, &s_addrs,
+ &s_sizes, &s_bufs) < 0)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "can't sort vector I/O request")
+
+
+ /* Get the transfer mode from the API context
+ *
+ * This flag is set to H5FD_MPIO_COLLECTIVE if the API call is
+ * collective, and to H5FD_MPIO_INDEPENDENT if it is not.
+ *
+ * While this doesn't mean that we are actually about to do a collective
+ * write, it does mean that all ranks are here, so we can use MPI_File_set_view().
+ */
+ if (H5CX_get_io_xfer_mode(&xfer_mode) < 0)
+ HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't get MPI-I/O transfer mode")
+
+ if (xfer_mode == H5FD_MPIO_COLLECTIVE) {
+
+ if (count > 0) { /* create MPI derived types describing the vector write */
+
+ if ((NULL == (mpi_block_lengths = (int *)HDmalloc((size_t)count * sizeof(int)))) ||
+ (NULL == (mpi_displacments = (MPI_Aint *)HDmalloc((size_t)count * sizeof(MPI_Aint)))) ||
+ (NULL == (mpi_bufs = (MPI_Aint *)HDmalloc((size_t)count * sizeof(MPI_Aint))))) {
+
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't alloc mpi block lengths / displacement")
+ }
+
+ /* when we setup mpi_bufs[] below, all addresses are offsets from
+ * mpi_bufs_base.
+ *
+ * Since these offsets must all be positive, we must scan through
+ * s_bufs[] to find the smallest value, and choose that for
+ * mpi_bufs_base.
+ */
+
+ j = 0; /* guess at the index of the smallest value of s_bufs[] */
+
+ for (i = 1; i < (int)count; i++) {
+
+ if (s_bufs[i] < s_bufs[j]) {
+
+ j = i;
+ }
+ }
+
+ mpi_bufs_base = s_bufs[j];
+
+ if (MPI_SUCCESS != (mpi_code = MPI_Get_address(mpi_bufs_base, &mpi_bufs_base_Aint)))
+
+ HMPI_GOTO_ERROR(FAIL, "MPI_Get_address for s_bufs[] to mpi_bufs_base failed", mpi_code)
+
+ size_i = 1;
+
+ fixed_size = FALSE;
+
+ /* load the mpi_block_lengths and mpi_displacements arrays */
+ for (i = 0; i < (int)count; i++) {
+
+ if (!fixed_size) {
+
+ if (sizes[i] == 0) {
+
+ fixed_size = TRUE;
+ size = sizes[i - 1];
+ }
+ else {
+
+ size = s_sizes[i];
+ }
+ }
+
+ /* There is an obvious possibility of an overflow here, as size_t
+ * will typically be 64 bits, where as int will typically be 32 bits.
+ * This must be fixed, but it should be good enough for initial
+ * correctness testing.
+ * JRM -- 3/17/21
+ */
+ mpi_block_lengths[i] = (int)size;
+ mpi_displacments[i] = (MPI_Aint)s_addrs[i];
+
+ /* convert s_bufs[i] to MPI_Aint... */
+ if (MPI_SUCCESS != (mpi_code = MPI_Get_address(s_bufs[i], &(mpi_bufs[i]))))
+
+ HMPI_GOTO_ERROR(FAIL, "MPI_Get_address for s_bufs[] - mpi_bufs_base failed", mpi_code)
+
+ /*... and then subtract mpi_bufs_base_Aint from it. */
+#if ((MPI_VERSION > 3) || ((MPI_VERSION == 3) && (MPI_SUBVERSION >= 1)))
+ mpi_bufs[i] = MPI_Aint_diff(mpi_bufs[i], mpi_bufs_base_Aint);
+#else
+ mpi_bufs[i] = mpi_bufs[i] - mpi_bufs_base_Aint;
+#endif
+ }
+
+
+ /* create the memory MPI derived types */
+ if (MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int)count, mpi_block_lengths, mpi_bufs,
+ MPI_BYTE, &buf_type)))
+
+ HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed for buf_type failed", mpi_code)
+
+ buf_type_created = TRUE;
+
+ if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(&buf_type)))
+
+ HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit for buf_type failed", mpi_code)
+
+ /* create the file MPI derived type */
+ if (MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int)count, mpi_block_lengths,
+ mpi_displacments, MPI_BYTE, &file_type)))
+
+ HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed for file_type failed", mpi_code)
+
+ file_type_created = TRUE;
+
+ if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(&file_type)))
+
+ HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit for file_type failed", mpi_code)
+ }
+ else {
+
+ /* setup for null participation in the collective operation. */
+
+ buf_type = MPI_BYTE;
+ file_type = MPI_BYTE;
+
+ /* Set non-NULL pointer for I/O operation */
+ mpi_bufs_base = (void *)(&unused);
+
+ /* MPI count to write */
+ size_i = 0;
+ }
+
+ /* Portably initialize MPI status variable */
+ HDmemset(&mpi_stat, 0, sizeof(MPI_Status));
+
+ /* some numeric conversions */
+ if (H5FD_mpi_haddr_to_MPIOff((haddr_t)0, &mpi_off) < 0)
+ HGOTO_ERROR(H5E_INTERNAL, H5E_BADRANGE, FAIL, "can't set MPI off to 0")
+
+#ifdef H5FDmpio_DEBUG
+ if (H5FD_mpio_Debug[(int)'w'])
+ HDfprintf(stdout, "%s: mpi_off = %ld size_i = %d\n", FUNC, (long)mpi_off, size_i);
+#endif
+
+ /* Setup the file view. */
+ if (MPI_SUCCESS != (mpi_code = MPI_File_set_view(file->f, mpi_off, MPI_BYTE, file_type,
+ H5FD_mpi_native_g, file->info)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_File_set_view failed", mpi_code)
+
+ /* Get the collective_opt property to check whether the application wants to do IO individually.
+ */
+ if (H5CX_get_mpio_coll_opt(&coll_opt_mode) < 0)
+
+ HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't get MPI-I/O collective_op property")
+
+ /* Write the data. */
+#ifdef H5FDmpio_DEBUG
+ if (H5FD_mpio_Debug[(int)'w'])
+ HDfprintf(stdout, "%s: using MPIO collective mode\n", FUNC);
+#endif
+
+ if (coll_opt_mode == H5FD_MPIO_COLLECTIVE_IO) {
+#ifdef H5FDmpio_DEBUG
+ if (H5FD_mpio_Debug[(int)'w'])
+ HDfprintf(stdout, "%s: doing MPI collective IO\n", FUNC);
+#endif
+
+ if (MPI_SUCCESS != (mpi_code = MPI_File_write_at_all(file->f, mpi_off, mpi_bufs_base, size_i,
+ buf_type, &mpi_stat)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_File_write_at_all failed", mpi_code)
+ } /* end if */
+ else if (size_i > 0) {
+#ifdef H5FDmpio_DEBUG
+ if (H5FD_mpio_Debug[(int)'w'])
+ HDfprintf(stdout, "%s: doing MPI independent IO\n", FUNC);
+#endif
+
+ if (MPI_SUCCESS !=
+ (mpi_code = MPI_File_write_at(file->f, mpi_off, mpi_bufs_base, size_i, buf_type, &mpi_stat)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_File_write_at failed", mpi_code)
+ } /* end else */
+
+ /* Reset the file view */
+ if (MPI_SUCCESS != (mpi_code = MPI_File_set_view(file->f, (MPI_Offset)0, MPI_BYTE, MPI_BYTE,
+ H5FD_mpi_native_g, file->info)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_File_set_view failed", mpi_code)
+ }
+ else if (count > 0) {
+
+ /* The write is part of an independent operation. As a result,
+ * we can't use MPI_File_set_view() (since it it a collective operation),
+ * and thus there is no point in setting up an MPI derived type, as
+ * (to the best of my knowlege) MPI I/O doesn't have support for
+ * non-contiguous I/O in independent mode.
+ *
+ * Thus we have to write out each element of the vector in a separate
+ * MPI_File_write_at() call.
+ */
+
+ fixed_size = FALSE;
+
+#ifdef H5FDmpio_DEBUG
+ if (H5FD_mpio_Debug[(int)'w'])
+ HDfprintf(stdout, "%s: doing MPI independent IO\n", FUNC);
+#endif
+
+ for (i = 0; i < (int)count; i++) {
+
+ if (H5FD_mpi_haddr_to_MPIOff(s_addrs[i], &mpi_off) < 0)
+
+ HGOTO_ERROR(H5E_INTERNAL, H5E_BADRANGE, FAIL, "can't convert from haddr to MPI off")
+
+ if (!fixed_size) {
+
+ if (sizes[i] == 0) {
+
+ fixed_size = TRUE;
+ size = sizes[i - 1];
+ }
+ else {
+
+ size = s_sizes[i];
+ }
+ }
+
+ size_i = (int)size; /* todo: fix potential for overflow */
+
+ if (MPI_SUCCESS !=
+ (mpi_code = MPI_File_write_at(file->f, mpi_off, s_bufs[i], size_i, MPI_BYTE, &mpi_stat)))
+
+ HMPI_GOTO_ERROR(FAIL, "MPI_File_write_at failed", mpi_code)
+ }
+ }
+
+ /* Each process will keep track of its perceived EOF value locally, and
+ * ultimately we will reduce this value to the maximum amongst all
+ * processes, but until then keep the actual eof at HADDR_UNDEF just in
+ * case something bad happens before that point. (rather have a value
+ * we know is wrong sitting around rather than one that could only
+ * potentially be wrong.)
+ */
+ file->eof = HADDR_UNDEF;
+
+ /* check to see if the local eof has changed been extended, and update if so.
+ * Since the vector write request has been sorted in increasing address order,
+ * we need only look at the address and size of the last element in the vector.
+ */
+ if ((count > 0) && ((s_addrs[count - 1] + (haddr_t)(s_sizes[count - 1])) > file->local_eof)) {
+
+ file->local_eof = (s_addrs[count - 1] + (haddr_t)(s_sizes[count - 1]));
+ }
+
+done:
+
+ if (!vector_was_sorted) { /* free sorted vectors if they exist */
+
+ if (s_types) {
+
+ HDfree(s_types);
+ s_types = NULL;
+ }
+
+ if (s_addrs) {
+
+ HDfree(s_addrs);
+ s_addrs = NULL;
+ }
+
+ if (s_sizes) {
+
+ HDfree(s_sizes);
+ s_sizes = NULL;
+ }
+
+ if (s_bufs) {
+
+ HDfree(s_bufs);
+ s_bufs = NULL;
+ }
+ }
+
+ if (mpi_block_lengths) {
+
+ HDfree(mpi_block_lengths);
+ mpi_block_lengths = NULL;
+ }
+
+ if (mpi_displacments) {
+
+ HDfree(mpi_displacments);
+ mpi_displacments = NULL;
+ }
+
+ if (mpi_bufs) {
+
+ HDfree(mpi_bufs);
+ mpi_bufs = NULL;
+ }
+
+ if (buf_type_created) {
+ MPI_Type_free(&buf_type);
+ }
+
+ if (file_type_created) {
+ MPI_Type_free(&file_type);
+ }
+
+#ifdef H5FDmpio_DEBUG
+ if (H5FD_mpio_Debug[(int)'t'])
+ HDfprintf(stdout, "%s: Leaving, proc %d: ret_value = %d\n", FUNC, file->mpi_rank, ret_value);
+#endif
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD__mpio_write_vector() */
+
+/*-------------------------------------------------------------------------
* Function: H5FD__mpio_flush
*
* Purpose: Makes sure that all data is on disk. This is collective.
diff --git a/src/H5FDprivate.h b/src/H5FDprivate.h
index d485ea9..82f51e8 100644
--- a/src/H5FDprivate.h
+++ b/src/H5FDprivate.h
@@ -145,6 +145,10 @@ H5_DLL herr_t H5FD_set_base_addr(H5FD_t *file, haddr_t base_addr);
H5_DLL haddr_t H5FD_get_base_addr(const H5FD_t *file);
H5_DLL herr_t H5FD_set_paged_aggr(H5FD_t *file, hbool_t paged);
+H5_DLL herr_t H5FD_sort_vector_io_req(hbool_t *vector_was_sorted, uint32_t count, H5FD_mem_t types[],
+ haddr_t addrs[], size_t sizes[], void *bufs[], H5FD_mem_t **s_types_ptr,
+ haddr_t **s_addrs_ptr, size_t **s_sizes_ptr, void ***s_bufs_ptr);
+
/* Function prototypes for MPI based VFDs*/
#ifdef H5_HAVE_PARALLEL
/* General routines */
diff --git a/testpar/Makefile.am b/testpar/Makefile.am
index 6a8cc2b..cbde0c1 100644
--- a/testpar/Makefile.am
+++ b/testpar/Makefile.am
@@ -30,7 +30,7 @@ check_SCRIPTS = $(TEST_SCRIPT_PARA)
# Test programs. These are our main targets.
#
-TEST_PROG_PARA=t_mpi t_bigio testphdf5 t_cache t_cache_image t_pread t_pshutdown t_prestart t_init_term t_shapesame t_filters_parallel t_2Gio
+TEST_PROG_PARA=t_mpi t_bigio testphdf5 t_cache t_cache_image t_pread t_pshutdown t_prestart t_init_term t_shapesame t_filters_parallel t_2Gio t_vfd
# t_pflush1 and t_pflush2 are used by testpflush.sh
check_PROGRAMS = $(TEST_PROG_PARA) t_pflush1 t_pflush2
diff --git a/testpar/t_vfd.c b/testpar/t_vfd.c
new file mode 100644
index 0000000..880cc96
--- /dev/null
+++ b/testpar/t_vfd.c
@@ -0,0 +1,4057 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * Copyright by the Board of Trustees of the University of Illinois. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://www.hdfgroup.org/licenses. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/* Programmer: John Mainzer
+ * 7/13/15
+ *
+ * This file is a catchall for parallel VFD tests.
+ */
+
+#include "testphdf5.h"
+
+/* Must be a power of 2. Reducing it below 1024 may cause problems */
+#define INTS_PER_RANK 1024
+
+/* global variable declarations: */
+
+hbool_t pass = TRUE; /* set to FALSE on error */
+const char *failure_mssg = NULL;
+
+const char *FILENAMES[] = {"mpio_vfd_test_file_0", /*0*/
+ "mpio_vfd_test_file_1", /*1*/
+ "mpio_vfd_test_file_2", /*2*/
+ "mpio_vfd_test_file_3", /*3*/
+ "mpio_vfd_test_file_4", /*4*/
+ "mpio_vfd_test_file_5", /*5*/
+ NULL};
+
+/* File Test Images
+ *
+ * Pointers to dynamically allocated buffers of size
+ * INTS_PER_RANK * sizeof(int32_t) * mpi_size(). These
+ * buffers are used to put the test file in a known
+ * state, and to test if the test file contains the
+ * expected data.
+ */
+
+int32_t *increasing_fi_buf = NULL;
+int32_t *decreasing_fi_buf = NULL;
+int32_t *negative_fi_buf = NULL;
+int32_t *zero_fi_buf = NULL;
+int32_t *read_fi_buf = NULL;
+
+/* local utility function declarations */
+
+static unsigned alloc_and_init_file_images(int mpi_size);
+static void free_file_images(void);
+static void setup_vfd_test_file(int file_name_id, char *file_name, int mpi_size, H5FD_mpio_xfer_t xfer_mode,
+ H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name, haddr_t eoa,
+ H5FD_t **lf_ptr, hid_t *fapl_id_ptr, hid_t *dxpl_id_ptr);
+static void takedown_vfd_test_file(int mpi_rank, char *filename, H5FD_t **lf_ptr, hid_t *fapl_id_ptr,
+ hid_t *dxpl_id_ptr);
+
+/* test functions */
+static unsigned vector_read_test_1(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode,
+ H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name);
+static unsigned vector_read_test_2(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode,
+ H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name);
+static unsigned vector_read_test_3(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode,
+ H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name);
+static unsigned vector_read_test_4(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode,
+ H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name);
+static unsigned vector_read_test_5(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode,
+ H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name);
+
+static unsigned vector_write_test_1(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode,
+ H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name);
+static unsigned vector_write_test_2(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode,
+ H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name);
+static unsigned vector_write_test_3(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode,
+ H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name);
+static unsigned vector_write_test_4(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode,
+ H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name);
+static unsigned vector_write_test_5(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode,
+ H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name);
+static unsigned vector_write_test_6(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode,
+ H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name);
+
+/****************************************************************************/
+/***************************** Utility Functions ****************************/
+/****************************************************************************/
+
+/*-------------------------------------------------------------------------
+ * Function: alloc_and_init_file_images
+ *
+ * Purpose: Allocate and initialize the global buffers used to construct,
+ * load and verify test file contents.
+ *
+ * Return: void
+ *
+ * Programmer: John Mainzer
+ * 3/25/26
+ *
+ * Modifications:
+ *
+ * None.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static unsigned
+alloc_and_init_file_images(int mpi_size)
+{
+ const char *fcn_name = "alloc_and_init_file_images()";
+ int cp = 0;
+ int buf_len;
+ size_t buf_size;
+ int i;
+ hbool_t show_progress = FALSE;
+
+ pass = TRUE;
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* allocate the file image buffers */
+ if (pass) {
+
+ buf_len = INTS_PER_RANK * mpi_size;
+ buf_size = sizeof(int32_t) * (size_t)INTS_PER_RANK * (size_t)mpi_size;
+
+ increasing_fi_buf = (int32_t *)HDmalloc(buf_size);
+ decreasing_fi_buf = (int32_t *)HDmalloc(buf_size);
+ negative_fi_buf = (int32_t *)HDmalloc(buf_size);
+ zero_fi_buf = (int32_t *)HDmalloc(buf_size);
+ read_fi_buf = (int32_t *)HDmalloc(buf_size);
+
+ if ((!increasing_fi_buf) || (!decreasing_fi_buf) || (!negative_fi_buf) || (!zero_fi_buf) ||
+ (!read_fi_buf)) {
+
+ pass = FALSE;
+ failure_mssg = "Can't allocate one or more file image buffers.";
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* initialize the file image buffers */
+ if (pass) {
+
+ for (i = 0; i < buf_len; i++) {
+
+ increasing_fi_buf[i] = i;
+ decreasing_fi_buf[i] = buf_len - i;
+ negative_fi_buf[i] = -i;
+ zero_fi_buf[i] = 0;
+ read_fi_buf[i] = 0;
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* discard file image buffers if there was an error */
+ if (!pass) {
+
+ free_file_images();
+ }
+
+ return !pass;
+
+} /* alloc_and_init_file_images() */
+
+/*-------------------------------------------------------------------------
+ * Function: free_file_images
+ *
+ * Purpose: Deallocate any glogal file image buffers that exist, and
+ * set their associated pointers to NULL.
+ *
+ * Return: void
+ *
+ * Programmer: John Mainzer
+ * 1/25/17
+ *
+ * Modifications:
+ *
+ * None.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static void
+free_file_images(void)
+{
+ if (increasing_fi_buf) {
+
+ HDfree(increasing_fi_buf);
+ increasing_fi_buf = NULL;
+ }
+
+ if (decreasing_fi_buf) {
+
+ HDfree(decreasing_fi_buf);
+ decreasing_fi_buf = NULL;
+ }
+
+ if (negative_fi_buf) {
+
+ HDfree(negative_fi_buf);
+ negative_fi_buf = NULL;
+ }
+
+ if (zero_fi_buf) {
+
+ HDfree(zero_fi_buf);
+ zero_fi_buf = NULL;
+ }
+
+ if (read_fi_buf) {
+
+ HDfree(read_fi_buf);
+ read_fi_buf = NULL;
+ }
+
+ return;
+
+} /* free_file_images() */
+
+/*-------------------------------------------------------------------------
+ * Function: setup_vfd_test_file
+ *
+ * Purpose: Create / open the specified test file with the specified
+ * VFD, and set the EOA to the specified value.
+ *
+ * Setup the dxpl for subsequent I/O via the target VFD.
+ *
+ * Return a pointer to the instance of H5FD_t created on
+ * file open in *lf_ptr, and the FAPL and DXPL ids in
+ * *fapl_id_ptr and *dxpl_id_ptr. Similarly, copy the
+ * "fixed" file name into file_name on exit.
+ *
+ * Return: void
+ *
+ * Programmer: John Mainzer
+ * 3/25/26
+ *
+ * Modifications:
+ *
+ * None.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static void
+setup_vfd_test_file(int file_name_id, char *file_name, int mpi_size, H5FD_mpio_xfer_t xfer_mode,
+ H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name, haddr_t eoa,
+ H5FD_t **lf_ptr, hid_t *fapl_id_ptr, hid_t *dxpl_id_ptr)
+{
+ const char *fcn_name = "setup_vfd_test_file()";
+ char filename[512];
+ int cp = 0;
+ hbool_t show_progress = FALSE;
+ hid_t fapl_id = -1; /* file access property list ID */
+ hid_t dxpl_id = -1; /* data access property list ID */
+ unsigned flags = 0; /* file open flags */
+ H5FD_t * lf = NULL; /* VFD struct ptr */
+
+ HDassert(vfd_name);
+ HDassert(lf_ptr);
+ HDassert(fapl_id_ptr);
+ HDassert(dxpl_id_ptr);
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* setupf fapl for target VFD */
+ if (pass) {
+
+ if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "Can't create fapl.";
+ }
+ }
+
+ if (pass) {
+
+ if (strcmp(vfd_name, "mpio") == 0) {
+
+ if (H5Pset_fapl_mpio(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "Can't set mpio fapl.";
+ }
+ }
+ else {
+
+ pass = FALSE;
+ failure_mssg = "un-supported VFD";
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* setup the file name */
+ if (pass) {
+
+ if (h5_fixname(FILENAMES[file_name_id], H5P_DEFAULT, filename, sizeof(filename)) == NULL) {
+
+ pass = FALSE;
+ failure_mssg = "h5_fixname() failed.\n";
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* Open the VFD test file with the specified VFD. */
+
+ if (pass) {
+
+ flags = H5F_ACC_RDWR | H5F_ACC_CREAT | H5F_ACC_TRUNC;
+
+ if (NULL == (lf = H5FDopen(filename, flags, fapl_id, HADDR_UNDEF))) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDopen() failed.\n";
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* set eoa as specified */
+
+ if (pass) {
+
+ eoa = (haddr_t)mpi_size * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t));
+
+ if (H5FDset_eoa(lf, H5FD_MEM_DEFAULT, eoa) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDset_eoa() failed.\n";
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ if (pass) { /* setup dxpl */
+
+ dxpl_id = H5Pcreate(H5P_DATASET_XFER);
+
+ if (dxpl_id < 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5Pcreate(H5P_DATASET_XFER) failed.";
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ if (pass) {
+
+ if (H5Pset_dxpl_mpio(dxpl_id, xfer_mode) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5Pset_dxpl_mpio() failed.";
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ if (pass) {
+
+ if (H5Pset_dxpl_mpio_collective_opt(dxpl_id, coll_opt_mode) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5Pset_dxpl_mpio() failed.";
+ }
+ }
+
+ if (pass) { /* setup pointers with return values */
+
+ strncpy(file_name, filename, 512);
+ *lf_ptr = lf;
+ *fapl_id_ptr = fapl_id;
+ *dxpl_id_ptr = dxpl_id;
+ }
+ else { /* tidy up from failure as posible */
+
+ if (lf)
+ H5FDclose(lf);
+
+ if (fapl_id != -1)
+ H5Pclose(fapl_id);
+
+ if (dxpl_id != -1)
+ H5Pclose(dxpl_id);
+ }
+
+ return;
+
+} /* setup_vfd_test_file() */
+
+/*-------------------------------------------------------------------------
+ * Function: takedown_vfd_test_file
+ *
+ * Purpose: Close and delete the specified test file. Close the
+ * FAPL & DXPL.
+ *
+ * Return: void
+ *
+ * Programmer: John Mainzer
+ * 3/25/26
+ *
+ * Modifications:
+ *
+ * None.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static void
+takedown_vfd_test_file(int mpi_rank, char *filename, H5FD_t **lf_ptr, hid_t *fapl_id_ptr, hid_t *dxpl_id_ptr)
+{
+ const char *fcn_name = "takedown_vfd_test_file()";
+ int cp = 0;
+ hbool_t show_progress = FALSE;
+
+ HDassert(lf_ptr);
+ HDassert(fapl_id_ptr);
+ HDassert(dxpl_id_ptr);
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* Close the test file if it is open, regardless of the value of pass.
+ * This should let the test program shut down more cleanly.
+ */
+
+ if (*lf_ptr) {
+
+ if (H5FDclose(*lf_ptr) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDclose() failed.\n";
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 6) On rank 0, delete the test file.
+ */
+
+ if (pass) {
+
+ /* wait for everyone to close the file */
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ if ((mpi_rank == 0) && (HDremove(filename) < 0)) {
+
+ pass = FALSE;
+ failure_mssg = "HDremove() failed.\n";
+ }
+
+ /* wait for the file delete to complete */
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* Close the fapl */
+ if (H5Pclose(*fapl_id_ptr) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "can't close fapl.\n";
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* Close the dxpl */
+ if (H5Pclose(*dxpl_id_ptr) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "can't close dxpl.\n";
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ return;
+
+} /* takedown_vfd_test_file() */
+
+/****************************************************************************/
+/******************************* Test Functions *****************************/
+/****************************************************************************/
+
+/*-------------------------------------------------------------------------
+ * Function: vector_read_test_1()
+ *
+ * Purpose: Simple vector read test:
+ *
+ * 1) Open the test file with the specified VFD, set the eoa,
+ * and setup the DXPL.
+ *
+ * 2) Using rank zero, write the entire increasing_fi_buf to
+ * the file.
+ *
+ * 3) Barrier
+ *
+ * 4) On each rank, zero the read buffer, and then read
+ * INTS_PER_RANK * sizeof(int32) bytes from the file
+ * starting at offset mpi_rank * INTS_PER_RANK *
+ * sizeof(int32_t) in both the file and read_fi_buf.
+ * Do this with a vector read containing a single
+ * element.
+ *
+ * Verify that read_fi_buf contains zeros for all
+ * indices less than mpi_rank * INTS_PER_RANK, or
+ * greater than or equal to (mpi_rank + 1) * INTS_PER_RANK.
+ * For all other indicies, read_fi_buf should equal
+ * increasing_fi_buf.
+ *
+ * 5) Barrier
+ *
+ * 6) Close the test file.
+ *
+ * 7) On rank 0, delete the test file.
+ *
+ * Return: FALSE on success, TRUE if any errors are detected.
+ *
+ * Programmer: John Mainzer
+ * 3/26/21
+ *
+ * Modifications:
+ *
+ * None.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static unsigned
+vector_read_test_1(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode,
+ H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name)
+{
+ const char *fcn_name = "vector_read_test_1()";
+ char test_title[120];
+ char filename[512];
+ haddr_t eoa;
+ hbool_t show_progress = FALSE;
+ hid_t fapl_id = -1; /* file access property list ID */
+ hid_t dxpl_id = -1; /* data access property list ID */
+ H5FD_t * lf = NULL; /* VFD struct ptr */
+ int cp = 0;
+ int i;
+ uint32_t count;
+ H5FD_mem_t types[1];
+ haddr_t addrs[1];
+ size_t sizes[1];
+ void * bufs[1];
+
+ pass = TRUE;
+
+ if (mpi_rank == 0) {
+
+ if (xfer_mode == H5FD_MPIO_INDEPENDENT) {
+
+ sprintf(test_title, "parallel vector read test 1 -- %s / independent", vfd_name);
+ }
+ else if (coll_opt_mode == H5FD_MPIO_INDIVIDUAL_IO) {
+
+ sprintf(test_title, "parallel vector read test 1 -- %s / col op / ind I/O", vfd_name);
+ }
+ else {
+
+ HDassert(coll_opt_mode == H5FD_MPIO_COLLECTIVE_IO);
+
+ sprintf(test_title, "parallel vector read test 1 -- %s / col op / col I/O", vfd_name);
+ }
+
+ TESTING(test_title);
+ }
+
+ show_progress = ((show_progress) && (mpi_rank == 0));
+
+ if (show_progress)
+ HDfprintf(stdout, "\n%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 1) Open the test file with the specified VFD, set the eoa, and setup the dxpl */
+ if (pass) {
+
+ eoa = (haddr_t)mpi_size * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t));
+
+ setup_vfd_test_file(file_name_id, filename, mpi_size, xfer_mode, coll_opt_mode, vfd_name, eoa, &lf,
+ &fapl_id, &dxpl_id);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 2) Using rank zero, write the entire increasing_fi_buf to
+ * the file.
+ */
+ if (pass) {
+
+ size_t image_size = (size_t)mpi_size * (size_t)INTS_PER_RANK * sizeof(int32_t);
+
+ if (mpi_rank == 0) {
+
+ if (H5FDwrite(lf, H5FD_MEM_DRAW, H5P_DEFAULT, (haddr_t)0, image_size, (void *)increasing_fi_buf) <
+ 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDwrite() on rank 0 failed.\n";
+ }
+ }
+ }
+
+ /* 3) Barrier */
+
+ if (pass) {
+
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 4) On each rank, zero the read buffer, and then read
+ * INTS_PER_RANK * sizeof(int32) bytes from the file
+ * starting at offset mpi_rank * INTS_PER_RANK *
+ * sizeof(int32_t) in both the file and read_fi_buf.
+ * Do this with a vector read containing a single
+ * element.
+ *
+ * Verify that read_fi_buf contains zeros for all
+ * indices less than mpi_rank * INTS_PER_RANK, or
+ * greater than or equal to (mpi_rank + 1) * INTS_PER_RANK.
+ * For all other indicies, read_fi_buf should equal
+ * increasing_fi_buf.
+ */
+ if (pass) {
+
+ for (i = 0; i < mpi_size * INTS_PER_RANK; i++) {
+
+ read_fi_buf[i] = 0;
+ }
+
+ count = 1;
+ types[0] = H5FD_MEM_DRAW;
+ addrs[0] = (haddr_t)mpi_rank * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t));
+ sizes[0] = (size_t)INTS_PER_RANK * sizeof(int32_t);
+ bufs[0] = (void *)(&(read_fi_buf[mpi_rank * INTS_PER_RANK]));
+
+ if (H5FDread_vector(lf, dxpl_id, count, types, addrs, sizes, bufs) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDread_vector() failed.\n";
+ }
+
+ for (i = 0; i < mpi_size * INTS_PER_RANK; i++) {
+
+ if ((i < mpi_rank * INTS_PER_RANK) || (i >= (mpi_rank + 1) * INTS_PER_RANK)) {
+
+ if (read_fi_buf[i] != 0) {
+
+ pass = FALSE;
+ failure_mssg = "Unexpected value in read_fi_buf (1).\n";
+ break;
+ }
+ }
+ else {
+
+ if (read_fi_buf[i] != increasing_fi_buf[i]) {
+
+ pass = FALSE;
+ failure_mssg = "Unexpected value in read_fi_buf (2).\n";
+ break;
+ }
+ }
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 5) Barrier */
+
+ if (pass) {
+
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 6) Close the test file and delete it (on rank 0 only).
+ * Close FAPL and DXPL.
+ */
+
+ if (pass) {
+
+ takedown_vfd_test_file(mpi_rank, filename, &lf, &fapl_id, &dxpl_id);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* report results */
+ if (mpi_rank == 0) {
+
+ if (pass) {
+
+ PASSED();
+ }
+ else {
+
+ H5_FAILED();
+
+ if (show_progress) {
+ HDfprintf(stdout, "%s: failure_mssg = \"%s\"\n", fcn_name, failure_mssg);
+ }
+ }
+ }
+
+ return (!pass);
+
+} /* vector_read_test_1() */
+
+/*-------------------------------------------------------------------------
+ * Function: vector_read_test_2()
+ *
+ * Purpose: Simple vector read test with only half of ranks
+ * participating in each vector read.
+ *
+ * 1) Open the test file with the specified VFD, set the eoa,
+ * and setup the DXPL.
+ *
+ * 2) Using rank zero, write the entire decreasing_fi_buf to
+ * the file.
+ *
+ * 3) Barrier
+ *
+ * 4) On each rank, zero the read buffer.
+ *
+ * 5) On even ranks, read INTS_PER_RANK * sizeof(int32)
+ * bytes from the file starting at offset mpi_rank *
+ * INTS_PER_RANK * sizeof(int32_t) in both the file and
+ * read_fi_buf. Do this with a vector read containing
+ * a single element.
+ *
+ * Odd ranks perform an empty read.
+ *
+ * 6) Barrier.
+ *
+ * 7) On odd ranks, read INTS_PER_RANK * sizeof(int32)
+ * bytes from the file starting at offset mpi_rank *
+ * INTS_PER_RANK * sizeof(int32_t) in both the file and
+ * read_fi_buf. Do this with a vector read containing
+ * a single element.
+ *
+ * Even ranks perform an empty read.
+ *
+ * 8) Verify that read_fi_buf contains zeros for all
+ * indices less than mpi_rank * INTS_PER_RANK, or
+ * greater than or equal to (mpi_rank + 1) * INTS_PER_RANK.
+ * For all other indicies, read_fi_buf should equal
+ * decreasing_fi_buf.
+ *
+ * 9) Barrier
+ *
+ * 10) Close the test file.
+ *
+ * 11) On rank 0, delete the test file.
+ *
+ * Return: FALSE on success, TRUE if any errors are detected.
+ *
+ * Programmer: John Mainzer
+ * 3/26/21
+ *
+ * Modifications:
+ *
+ * None.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static unsigned
+vector_read_test_2(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode,
+ H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name)
+{
+ const char *fcn_name = "vector_read_test_2()";
+ char test_title[120];
+ char filename[512];
+ haddr_t eoa;
+ hbool_t show_progress = FALSE;
+ hid_t fapl_id = -1; /* file access property list ID */
+ hid_t dxpl_id = -1; /* data access property list ID */
+ H5FD_t * lf = NULL; /* VFD struct ptr */
+ int cp = 0;
+ int i;
+ uint32_t count;
+ H5FD_mem_t types[1];
+ haddr_t addrs[1];
+ size_t sizes[1];
+ void * bufs[1];
+
+ pass = TRUE;
+
+ if (mpi_rank == 0) {
+
+ if (xfer_mode == H5FD_MPIO_INDEPENDENT) {
+
+ sprintf(test_title, "parallel vector read test 2 -- %s / independent", vfd_name);
+ }
+ else if (coll_opt_mode == H5FD_MPIO_INDIVIDUAL_IO) {
+
+ sprintf(test_title, "parallel vector read test 2 -- %s / col op / ind I/O", vfd_name);
+ }
+ else {
+
+ HDassert(coll_opt_mode == H5FD_MPIO_COLLECTIVE_IO);
+
+ sprintf(test_title, "parallel vector read test 2 -- %s / col op / col I/O", vfd_name);
+ }
+
+ TESTING(test_title);
+ }
+
+ show_progress = ((show_progress) && (mpi_rank == 0));
+
+ if (show_progress)
+ HDfprintf(stdout, "\n%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 1) Open the test file with the specified VFD, set the eoa, and setup the dxpl */
+ if (pass) {
+
+ eoa = (haddr_t)mpi_size * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t));
+
+ setup_vfd_test_file(file_name_id, filename, mpi_size, xfer_mode, coll_opt_mode, vfd_name, eoa, &lf,
+ &fapl_id, &dxpl_id);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 2) Using rank zero, write the entire decreasing_fi_buf to
+ * the file.
+ */
+ if (pass) {
+
+ size_t image_size = (size_t)mpi_size * (size_t)INTS_PER_RANK * sizeof(int32_t);
+
+ if (mpi_rank == 0) {
+
+ if (H5FDwrite(lf, H5FD_MEM_DRAW, H5P_DEFAULT, (haddr_t)0, image_size, (void *)decreasing_fi_buf) <
+ 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDwrite() on rank 0 failed.\n";
+ }
+ }
+ }
+
+ /* 3) Barrier */
+
+ if (pass) {
+
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 4) On each rank, zero the read buffer. */
+ if (pass) {
+
+ for (i = 0; i < mpi_size * INTS_PER_RANK; i++) {
+
+ read_fi_buf[i] = 0;
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 5) On even ranks, read INTS_PER_RANK * sizeof(int32)
+ * bytes from the file starting at offset mpi_rank *
+ * INTS_PER_RANK * sizeof(int32_t) in both the file and
+ * read_fi_buf. Do this with a vector read containing
+ * a single element.
+ *
+ * Odd ranks perform an empty read.
+ */
+ if (pass) {
+
+ if (mpi_rank % 2 == 0) {
+
+ count = 1;
+ types[0] = H5FD_MEM_DRAW;
+ addrs[0] = (haddr_t)mpi_rank * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t));
+ sizes[0] = (size_t)INTS_PER_RANK * sizeof(int32_t);
+ bufs[0] = (void *)(&(read_fi_buf[mpi_rank * INTS_PER_RANK]));
+ }
+ else {
+
+ count = 0;
+ }
+
+ if (H5FDread_vector(lf, dxpl_id, count, types, addrs, sizes, bufs) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDread_vector() failed.\n";
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 6) Barrier */
+
+ if (pass) {
+
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 7) On odd ranks, read INTS_PER_RANK * sizeof(int32)
+ * bytes from the file starting at offset mpi_rank *
+ * INTS_PER_RANK * sizeof(int32_t) in both the file and
+ * read_fi_buf. Do this with a vector read containing
+ * a single element.
+ *
+ * Even ranks perform an empty read.
+ */
+ if (pass) {
+
+ if (mpi_rank % 2 == 1) {
+
+ count = 1;
+ types[0] = H5FD_MEM_DRAW;
+ addrs[0] = (haddr_t)mpi_rank * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t));
+ sizes[0] = (size_t)INTS_PER_RANK * sizeof(int32_t);
+ bufs[0] = (void *)(&(read_fi_buf[mpi_rank * INTS_PER_RANK]));
+ }
+ else {
+
+ count = 0;
+ }
+
+ if (H5FDread_vector(lf, dxpl_id, count, types, addrs, sizes, bufs) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDread_vector() failed.\n";
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 8) Verify that read_fi_buf contains zeros for all
+ * indices less than mpi_rank * INTS_PER_RANK, or
+ * greater than or equal to (mpi_rank + 1) * INTS_PER_RANK.
+ * For all other indicies, read_fi_buf should equal
+ * decreasing_fi_buf.
+ */
+
+ if (pass) {
+
+ for (i = 0; i < mpi_size * INTS_PER_RANK; i++) {
+
+ if ((i < mpi_rank * INTS_PER_RANK) || (i >= (mpi_rank + 1) * INTS_PER_RANK)) {
+
+ if (read_fi_buf[i] != 0) {
+
+ pass = FALSE;
+ failure_mssg = "Unexpected value in read_fi_buf (1).\n";
+ break;
+ }
+ }
+ else {
+
+ if (read_fi_buf[i] != decreasing_fi_buf[i]) {
+
+ pass = FALSE;
+ failure_mssg = "Unexpected value in read_fi_buf (2).\n";
+ break;
+ }
+ }
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 9) Barrier */
+
+ if (pass) {
+
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 10) Close the test file and delete it (on rank 0 only).
+ * Close FAPL and DXPL.
+ */
+
+ if (pass) {
+
+ takedown_vfd_test_file(mpi_rank, filename, &lf, &fapl_id, &dxpl_id);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* report results */
+ if (mpi_rank == 0) {
+
+ if (pass) {
+
+ PASSED();
+ }
+ else {
+
+ H5_FAILED();
+
+ if (show_progress) {
+ HDfprintf(stdout, "%s: failure_mssg = \"%s\"\n", fcn_name, failure_mssg);
+ }
+ }
+ }
+
+ return (!pass);
+
+} /* vector_read_test_2() */
+
+/*-------------------------------------------------------------------------
+ * Function: vector_read_test_3()
+ *
+ * Purpose: Verify that vector read works with multiple entries in
+ * the vector in each read, and that read buffers need not
+ * be in increasing (memory) address order.
+ *
+ * 1) Open the test file with the specified VFD, set the eoa,
+ * and setup the DXPL.
+ *
+ * 2) Using rank zero, write the entire negative_fi_buf to
+ * the file.
+ *
+ * 3) Barrier
+ *
+ * 4) On each rank, zero the four read buffers.
+ *
+ * 5) On each rank, do a vector read from the file, with
+ * each rank's vector having four elements, with each
+ * element reading INTS_PER_RANK / 4 * sizeof(int32)
+ * bytes, and the reads starting at address:
+ *
+ * (mpi_rank * INTS_PER_RANK) * sizeof(int32_t)
+ *
+ * (mpi_rank * INTS_PER_RANK + INTS_PER_RANK / 4) *
+ * sizeof(int32_t)
+ *
+ * (mpi_rank * INTS_PER_RANK + INTS_PER_RANK / 2) *
+ * sizeof(int32_t)
+ *
+ * (mpi_rank * INTS_PER_RANK + 3 * INTS_PER_RANK / 2) *
+ * sizeof(int32_t)
+ *
+ * On even ranks, the targets of the reads should be
+ * buf_0, buf_1, buf_2, and buf_3 respectively.
+ *
+ * On odd ranks, the targets of the reads should be
+ * buf_3, buf_2, buf_1, and buf_0 respectively.
+ *
+ * This has the effect of ensuring that on at least
+ * some ranks, the read buffers are not in increasing
+ * address order.
+ *
+ * 6) Verify that buf_0, buf_1, buf_2, and buf_3 contain
+ * the expected data. Note that this will be different
+ * on even vs. odd ranks.
+ *
+ * 7) Barrier.
+ *
+ * 8) Close the test file.
+ *
+ * 9) On rank 0, delete the test file.
+ *
+ * Return: FALSE on success, TRUE if any errors are detected.
+ *
+ * Programmer: John Mainzer
+ * 3/26/21
+ *
+ * Modifications:
+ *
+ * None.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static unsigned
+vector_read_test_3(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode,
+ H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name)
+{
+ const char *fcn_name = "vector_read_test_3()";
+ char test_title[120];
+ char filename[512];
+ int32_t buf_0[(INTS_PER_RANK / 4) + 1];
+ int32_t buf_1[(INTS_PER_RANK / 4) + 1];
+ int32_t buf_2[(INTS_PER_RANK / 4) + 1];
+ int32_t buf_3[(INTS_PER_RANK / 4) + 1];
+ haddr_t eoa;
+ hbool_t show_progress = FALSE;
+ hid_t fapl_id = -1; /* file access property list ID */
+ hid_t dxpl_id = -1; /* data access property list ID */
+ H5FD_t * lf = NULL; /* VFD struct ptr */
+ int cp = 0;
+ int i;
+ uint32_t count;
+ H5FD_mem_t types[4];
+ haddr_t addrs[4];
+ size_t sizes[4];
+ void * bufs[4];
+
+ pass = TRUE;
+
+ if (mpi_rank == 0) {
+
+ if (xfer_mode == H5FD_MPIO_INDEPENDENT) {
+
+ sprintf(test_title, "parallel vector read test 3 -- %s / independent", vfd_name);
+ }
+ else if (coll_opt_mode == H5FD_MPIO_INDIVIDUAL_IO) {
+
+ sprintf(test_title, "parallel vector read test 3 -- %s / col op / ind I/O", vfd_name);
+ }
+ else {
+
+ HDassert(coll_opt_mode == H5FD_MPIO_COLLECTIVE_IO);
+
+ sprintf(test_title, "parallel vector read test 3 -- %s / col op / col I/O", vfd_name);
+ }
+
+ TESTING(test_title);
+ }
+
+ show_progress = ((show_progress) && (mpi_rank == 0));
+
+ if (show_progress)
+ HDfprintf(stdout, "\n%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 1) Open the test file with the specified VFD, set the eoa, and setup the dxpl */
+ if (pass) {
+
+ eoa = (haddr_t)mpi_size * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t));
+
+ setup_vfd_test_file(file_name_id, filename, mpi_size, xfer_mode, coll_opt_mode, vfd_name, eoa, &lf,
+ &fapl_id, &dxpl_id);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 2) Using rank zero, write the entire negative_fi_buf to
+ * the file.
+ */
+ if (pass) {
+
+ size_t image_size = (size_t)mpi_size * (size_t)INTS_PER_RANK * sizeof(int32_t);
+
+ if (mpi_rank == 0) {
+
+ if (H5FDwrite(lf, H5FD_MEM_DRAW, H5P_DEFAULT, (haddr_t)0, image_size, (void *)negative_fi_buf) <
+ 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDwrite() on rank 0 failed.\n";
+ }
+ }
+ }
+
+ /* 3) Barrier */
+
+ if (pass) {
+
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 4) On each rank, zero the four read buffers. */
+ if (pass) {
+
+ for (i = 0; i <= INTS_PER_RANK / 4; i++) {
+
+ buf_0[i] = 0;
+ buf_1[i] = 0;
+ buf_2[i] = 0;
+ buf_3[i] = 0;
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 5) On each rank, do a vector read from the file, with
+ * each rank's vector having four elements, with each
+ * element reading INTS_PER_RANK / 4 * sizeof(int32)
+ * bytes, and the reads starting at address:
+ *
+ * (mpi_rank * INTS_PER_RANK) * sizeof(int32_t)
+ *
+ * (mpi_rank * INTS_PER_RANK + INTS_PER_RANK / 4) *
+ * sizeof(int32_t)
+ *
+ * (mpi_rank * INTS_PER_RANK + INTS_PER_RANK / 2) *
+ * sizeof(int32_t)
+ *
+ * (mpi_rank * INTS_PER_RANK + 3 * INTS_PER_RANK / 2) *
+ * sizeof(int32_t)
+ *
+ * On even ranks, the targets of the reads should be
+ * buf_0, buf_1, buf_2, and buf_3 respectively.
+ *
+ * On odd ranks, the targets of the reads should be
+ * buf_3, buf_2, buf_1, and buf_0 respectively.
+ *
+ * This has the effect of ensuring that on at least
+ * some ranks, the read buffers are not in increasing
+ * address order.
+ */
+ if (pass) {
+
+ haddr_t base_addr = (haddr_t)mpi_rank * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t));
+
+ count = 4;
+
+ types[0] = H5FD_MEM_DRAW;
+ addrs[0] = base_addr;
+ sizes[0] = (size_t)(INTS_PER_RANK / 4) * sizeof(int32_t);
+
+ types[1] = H5FD_MEM_DRAW;
+ addrs[1] = base_addr + ((haddr_t)(INTS_PER_RANK / 4) * (haddr_t)(sizeof(int32_t)));
+ sizes[1] = (size_t)(INTS_PER_RANK / 4) * sizeof(int32_t);
+
+ types[2] = H5FD_MEM_DRAW;
+ addrs[2] = base_addr + ((haddr_t)(INTS_PER_RANK / 2) * (haddr_t)(sizeof(int32_t)));
+ sizes[2] = (size_t)(INTS_PER_RANK / 4) * sizeof(int32_t);
+
+ types[3] = H5FD_MEM_DRAW;
+ addrs[3] = base_addr + ((haddr_t)(3 * INTS_PER_RANK / 4) * (haddr_t)(sizeof(int32_t)));
+ sizes[3] = (size_t)INTS_PER_RANK / 4 * sizeof(int32_t);
+
+ if (mpi_rank % 2 == 0) {
+
+ bufs[0] = (void *)(&(buf_0[0]));
+ bufs[1] = (void *)(buf_1);
+ bufs[2] = (void *)(buf_2);
+ bufs[3] = (void *)(buf_3);
+ }
+ else {
+
+ bufs[0] = (void *)(&(buf_3[0]));
+ bufs[1] = (void *)(buf_2);
+ bufs[2] = (void *)(buf_1);
+ bufs[3] = (void *)(buf_0);
+ }
+
+ if (H5FDread_vector(lf, dxpl_id, count, types, addrs, sizes, bufs) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDread_vector() failed.\n";
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 6) Verify that buf_0, buf_1, buf_2, and buf_3 contain
+ * the expected data. Note that this will be different
+ * on even vs. odd ranks.
+ */
+ if (pass) {
+
+ int base_index = mpi_rank * INTS_PER_RANK;
+
+ for (i = 0; ((pass) && (i < INTS_PER_RANK / 4)); i++) {
+
+ if (((mpi_rank % 2 == 0) && (buf_0[i] != negative_fi_buf[base_index + i])) ||
+ ((mpi_rank % 2 == 1) && (buf_3[i] != negative_fi_buf[base_index + i]))) {
+
+ pass = FALSE;
+ failure_mssg = "Unexpected value in buf (1).\n";
+ }
+ }
+
+ base_index += INTS_PER_RANK / 4;
+
+ for (i = 0; ((pass) && (i < INTS_PER_RANK / 4)); i++) {
+
+ if (((mpi_rank % 2 == 0) && (buf_1[i] != negative_fi_buf[base_index + i])) ||
+ ((mpi_rank % 2 == 1) && (buf_2[i] != negative_fi_buf[base_index + i]))) {
+
+ pass = FALSE;
+ failure_mssg = "Unexpected value in buf (2).\n";
+ }
+ }
+
+ base_index += INTS_PER_RANK / 4;
+
+ for (i = 0; ((pass) && (i < INTS_PER_RANK / 4)); i++) {
+
+ if (((mpi_rank % 2 == 0) && (buf_2[i] != negative_fi_buf[base_index + i])) ||
+ ((mpi_rank % 2 == 1) && (buf_1[i] != negative_fi_buf[base_index + i]))) {
+
+ pass = FALSE;
+ failure_mssg = "Unexpected value in buf (3).\n";
+ }
+ }
+
+ base_index += INTS_PER_RANK / 4;
+
+ for (i = 0; ((pass) && (i < INTS_PER_RANK / 4)); i++) {
+
+ if (((mpi_rank % 2 == 0) && (buf_3[i] != negative_fi_buf[base_index + i])) ||
+ ((mpi_rank % 2 == 1) && (buf_0[i] != negative_fi_buf[base_index + i]))) {
+
+ pass = FALSE;
+ failure_mssg = "Unexpected value in buf (4).\n";
+ }
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 7) Barrier */
+
+ if (pass) {
+
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 8) Close the test file and delete it (on rank 0 only).
+ * Close FAPL and DXPL.
+ */
+
+ if (pass) {
+
+ takedown_vfd_test_file(mpi_rank, filename, &lf, &fapl_id, &dxpl_id);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* report results */
+ if (mpi_rank == 0) {
+
+ if (pass) {
+
+ PASSED();
+ }
+ else {
+
+ H5_FAILED();
+
+ if (show_progress) {
+ HDfprintf(stdout, "%s: failure_mssg = \"%s\"\n", fcn_name, failure_mssg);
+ }
+ }
+ }
+
+ return (!pass);
+
+} /* vector_read_test_3() */
+
+/*-------------------------------------------------------------------------
+ * Function: vector_read_test_4()
+ *
+ * Purpose: Test vector I/O reads with vectors of different lengths
+ * and entry sizes across the ranks. Vectors are not, in
+ * general, sorted in increasing address order. Further,
+ * reads are not, in general, contiguous.
+ *
+ * 1) Open the test file with the specified VFD, set the eoa.
+ * and setup the DXPL.
+ *
+ * 2) Using rank zero, write the entire increasing_fi_buf to
+ * the file.
+ *
+ * 3) Barrier
+ *
+ * 4) Set all cells of read_fi_buf to zero.
+ *
+ * 5) For each rank, define base_index equal to:
+ *
+ * mpi_rank * INTS_PER_RANK
+ *
+ * and define base_addr equal to
+ *
+ * base_index * sizeof(int32_t).
+ *
+ * Setup a vector read between base_addr and
+ * base_addr + INTS_PER_RANK * sizeof(int32_t) - 1
+ * as follows:
+ *
+ * if ( rank % 4 == 0 ) construct a vector that reads:
+ *
+ * INTS_PER_RANK / 4 * sizeof(int32_t) bytes
+ * starting at base_addr + INTS_PER_RANK / 2 *
+ * sizeof(int32_t),
+ *
+ * INTS_PER_RANK / 8 * sizeof(int32_t) bytes
+ * starting at base_addr + INTS_PER_RANK / 4 *
+ * sizeof(int32_t), and
+ *
+ * INTS_PER_RANK / 16 * sizeof(int32_t) butes
+ * starting at base_addr + INTS_PER_RANK / 16 *
+ * sizeof(int32_t)
+ *
+ * to the equivalent locations in read_fi_buf
+ *
+ * if ( rank % 4 == 1 ) construct a vector that reads:
+ *
+ * ((INTS_PER_RANK / 2) - 2) * sizeof(int32_t)
+ * bytes starting at base_addr + sizeof(int32_t), and
+ *
+ * ((INTS_PER_RANK / 2) - 2) * sizeof(int32_t) bytes
+ * starting at base_addr + (INTS_PER_RANK / 2 + 1) *
+ * sizeof(int32_t).
+ *
+ * to the equivalent locations in read_fi_buf
+ *
+ * if ( rank % 4 == 2 ) construct a vector that reads:
+ *
+ * sizeof(int32_t) bytes starting at base_index +
+ * (INTS_PER_RANK / 2) * sizeof int32_t.
+ *
+ * to the equivalent locations in read_fi_buf
+ *
+ * if ( rank % 4 == 3 ) construct and read the empty vector
+ *
+ * 6) On each rank, verify that read_fi_buf contains the
+ * the expected values -- that is the matching values from
+ * increasing_fi_buf where ever there was a read, and zero
+ * otherwise.
+ *
+ * 7) Barrier.
+ *
+ * 8) Close the test file.
+ *
+ * 9) On rank 0, delete the test file.
+ *
+ * Return: FALSE on success, TRUE if any errors are detected.
+ *
+ * Programmer: John Mainzer
+ * 3/26/21
+ *
+ * Modifications:
+ *
+ * None.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static unsigned
+vector_read_test_4(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode,
+ H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name)
+{
+ const char *fcn_name = "vector_read_test_4()";
+ char test_title[120];
+ char filename[512];
+ haddr_t eoa;
+ haddr_t base_addr;
+ hbool_t show_progress = FALSE;
+ hid_t fapl_id = -1; /* file access property list ID */
+ hid_t dxpl_id = -1; /* data access property list ID */
+ H5FD_t * lf = NULL; /* VFD struct ptr */
+ int cp = 0;
+ int i;
+ int j;
+ int k;
+ int base_index;
+ uint32_t count = 0;
+ H5FD_mem_t types[4];
+ haddr_t addrs[4];
+ size_t sizes[4];
+ void * bufs[4];
+
+ pass = TRUE;
+
+ if (mpi_rank == 0) {
+
+ if (xfer_mode == H5FD_MPIO_INDEPENDENT) {
+
+ sprintf(test_title, "parallel vector read test 4 -- %s / independent", vfd_name);
+ }
+ else if (coll_opt_mode == H5FD_MPIO_INDIVIDUAL_IO) {
+
+ sprintf(test_title, "parallel vector read test 4 -- %s / col op / ind I/O", vfd_name);
+ }
+ else {
+
+ HDassert(coll_opt_mode == H5FD_MPIO_COLLECTIVE_IO);
+
+ sprintf(test_title, "parallel vector read test 4 -- %s / col op / col I/O", vfd_name);
+ }
+
+ TESTING(test_title);
+ }
+
+ show_progress = ((show_progress) && (mpi_rank == 0));
+
+ if (show_progress)
+ HDfprintf(stdout, "\n%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 1) Open the test file with the specified VFD, set the eoa, and setup the dxpl */
+ if (pass) {
+
+ eoa = (haddr_t)mpi_size * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t));
+
+ setup_vfd_test_file(file_name_id, filename, mpi_size, xfer_mode, coll_opt_mode, vfd_name, eoa, &lf,
+ &fapl_id, &dxpl_id);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 2) Using rank zero, write the entire negative_fi_buf to
+ * the file.
+ */
+ if (pass) {
+
+ size_t image_size = (size_t)mpi_size * (size_t)INTS_PER_RANK * sizeof(int32_t);
+
+ if (mpi_rank == 0) {
+
+ if (H5FDwrite(lf, H5FD_MEM_DRAW, H5P_DEFAULT, (haddr_t)0, image_size, (void *)increasing_fi_buf) <
+ 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDwrite() on rank 0 failed.\n";
+ }
+ }
+ }
+
+ /* 3) Barrier */
+
+ if (pass) {
+
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 4) Set all cells of read_fi_buf to zero. */
+ if (pass) {
+
+ for (i = 0; i < mpi_size * INTS_PER_RANK; i++) {
+
+ read_fi_buf[i] = 0;
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 5) For each rank, define base_index equal to:
+ *
+ * mpi_rank * INTS_PER_RANK
+ *
+ * and define base_addr equal to
+ *
+ * base_index * sizeof(int32_t).
+ *
+ * Setup a vector read between base_addr and
+ * base_addr + INTS_PER_RANK * sizeof(int32_t) - 1
+ * as follows:
+ */
+ if (pass) {
+
+ base_index = mpi_rank * INTS_PER_RANK;
+ base_addr = (haddr_t)base_index * (haddr_t)sizeof(int32_t);
+
+ if ((mpi_rank % 4) == 0) {
+
+ /* if ( rank % 4 == 0 ) construct a vector that reads:
+ *
+ * INTS_PER_RANK / 4 * sizeof(int32_t) bytes
+ * starting at base_addr + INTS_PER_RANK / 2 *
+ * sizeof(int32_t),
+ *
+ * INTS_PER_RANK / 8 * sizeof(int32_t) bytes
+ * starting at base_addr + INTS_PER_RANK / 4 *
+ * sizeof(int32_t), and
+ *
+ * INTS_PER_RANK / 16 * sizeof(int32_t) butes
+ * starting at base_addr + INTS_PER_RANK / 16 *
+ * sizeof(int32_t)
+ *
+ * to the equivalent locations in read_fi_buf
+ */
+
+ count = 3;
+
+ types[0] = H5FD_MEM_DRAW;
+ addrs[0] = base_addr + (haddr_t)((size_t)(INTS_PER_RANK / 2) * sizeof(int32_t));
+ sizes[0] = (size_t)(INTS_PER_RANK / 4) * sizeof(int32_t);
+ bufs[0] = (void *)(&(read_fi_buf[base_index + (INTS_PER_RANK / 2)]));
+
+ types[1] = H5FD_MEM_DRAW;
+ addrs[1] = base_addr + (haddr_t)((size_t)(INTS_PER_RANK / 4) * sizeof(int32_t));
+ sizes[1] = (size_t)(INTS_PER_RANK / 8) * sizeof(int32_t);
+ bufs[1] = (void *)(&(read_fi_buf[base_index + (INTS_PER_RANK / 4)]));
+
+ types[2] = H5FD_MEM_DRAW;
+ addrs[2] = base_addr + (haddr_t)((size_t)(INTS_PER_RANK / 16) * sizeof(int32_t));
+ sizes[2] = (size_t)(INTS_PER_RANK / 16) * sizeof(int32_t);
+ bufs[2] = (void *)(&(read_fi_buf[base_index + (INTS_PER_RANK / 16)]));
+ }
+ else if ((mpi_rank % 4) == 1) {
+
+ /* if ( rank % 4 == 1 ) construct a vector that reads:
+ *
+ * ((INTS_PER_RANK / 2) - 2) * sizeof(int32_t)
+ * bytes starting at base_addr + sizeof(int32_t), and
+ *
+ * ((INTS_PER_RANK / 2) - 2) * sizeof(int32_t) bytes
+ * starting at base_addr + (INTS_PER_RANK / 2 + 1) *
+ * sizeof(int32_t).
+ *
+ * to the equivalent locations in read_fi_buf
+ */
+ count = 2;
+
+ types[0] = H5FD_MEM_DRAW;
+ addrs[0] = base_addr + (haddr_t)(sizeof(int32_t));
+ sizes[0] = (size_t)((INTS_PER_RANK / 2) - 2) * sizeof(int32_t);
+ bufs[0] = (void *)(&(read_fi_buf[base_index + 1]));
+
+ types[1] = H5FD_MEM_DRAW;
+ addrs[1] = base_addr + (haddr_t)((size_t)((INTS_PER_RANK / 2) + 1) * sizeof(int32_t));
+ sizes[1] = (size_t)((INTS_PER_RANK / 2) - 2) * sizeof(int32_t);
+ bufs[1] = (void *)(&(read_fi_buf[base_index + (INTS_PER_RANK / 2) + 1]));
+ }
+ else if ((mpi_rank % 4) == 2) {
+
+ /* if ( rank % 4 == 2 ) construct a vector that reads:
+ *
+ * sizeof(int32_t) bytes starting at base_index +
+ * (INTS_PER_RANK / 2) * sizeof int32_t.
+ *
+ * to the equivalent locations in read_fi_buf
+ */
+ count = 1;
+
+ types[0] = H5FD_MEM_DRAW;
+ addrs[0] = base_addr + (haddr_t)((size_t)(INTS_PER_RANK / 2) * sizeof(int32_t));
+ sizes[0] = sizeof(int32_t);
+ bufs[0] = (void *)(&(read_fi_buf[base_index + (INTS_PER_RANK / 2)]));
+ }
+ else if ((mpi_rank % 4) == 3) {
+
+ /* if ( rank % 4 == 3 ) construct and read the empty vector */
+
+ count = 0;
+ }
+
+ if (H5FDread_vector(lf, dxpl_id, count, types, addrs, sizes, bufs) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDread_vector() failed (1).\n";
+ }
+ }
+
+ /* 6) On each rank, verify that read_fi_buf contains the
+ * the expected values -- that is the matching values from
+ * increasing_fi_buf where ever there was a read, and zero
+ * otherwise.
+ */
+ if (pass) {
+
+ for (i = 0; ((pass) && (i < mpi_size)); i++) {
+
+ base_index = i * INTS_PER_RANK;
+#if 1
+ for (j = base_index; j < base_index + INTS_PER_RANK; j++) {
+
+ k = j - base_index;
+#else
+ for (k = 0; k < INTS_PER_RANK; k++) {
+
+ j = k + base_index;
+#endif
+
+ if (i == mpi_rank) {
+
+ switch (i % 4) {
+
+ case 0:
+ if (((INTS_PER_RANK / 2) <= k) && (k < (3 * (INTS_PER_RANK / 4)))) {
+
+ if (read_fi_buf[j] != increasing_fi_buf[j]) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (1.1)";
+ HDfprintf(stdout, "\nread_fi_buf[%d] = %d, increasing_fi_buf[%d] = %d\n",
+ j, read_fi_buf[j], j, increasing_fi_buf[j]);
+ }
+ }
+ else if (((INTS_PER_RANK / 4) <= k) && (k < (3 * (INTS_PER_RANK / 8)))) {
+
+ if (read_fi_buf[j] != increasing_fi_buf[j]) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (1.2)";
+ }
+ }
+ else if (((INTS_PER_RANK / 16) <= k) && (k < (INTS_PER_RANK / 8))) {
+
+ if (read_fi_buf[j] != increasing_fi_buf[j]) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (1.3)";
+ }
+ }
+ else {
+
+ if (read_fi_buf[j] != 0) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (1.4)";
+ }
+ }
+ break;
+
+ case 1:
+ if ((1 <= k) && (k <= ((INTS_PER_RANK / 2) - 2))) {
+
+ if (read_fi_buf[j] != increasing_fi_buf[j]) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (2.1)";
+ }
+ }
+ else if ((((INTS_PER_RANK / 2) + 1) <= k) && (k <= (INTS_PER_RANK - 2))) {
+
+ if (read_fi_buf[j] != increasing_fi_buf[j]) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (2.2)";
+ }
+ }
+ else {
+
+ if (read_fi_buf[j] != 0) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (2.3)";
+ }
+ }
+ break;
+
+ case 2:
+ if (k == INTS_PER_RANK / 2) {
+
+ if (read_fi_buf[j] != increasing_fi_buf[j]) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (3.1)";
+ }
+ }
+ else {
+
+ if (read_fi_buf[j] != 0) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (3.2)";
+ }
+ }
+ break;
+
+ case 3:
+ if (read_fi_buf[j] != 0) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (4)";
+ }
+ break;
+
+ default:
+ HDassert(FALSE); /* should be un-reachable */
+ break;
+ }
+ }
+ else if (read_fi_buf[j] != 0) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (5)";
+ }
+ } /* end for loop */
+ } /* end for loop */
+ } /* end if */
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 7) Barrier */
+
+ if (pass) {
+
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 8) Close the test file and delete it (on rank 0 only).
+ * Close FAPL and DXPL.
+ */
+
+ if (pass) {
+
+ takedown_vfd_test_file(mpi_rank, filename, &lf, &fapl_id, &dxpl_id);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* report results */
+ if (mpi_rank == 0) {
+
+ if (pass) {
+
+ PASSED();
+ }
+ else {
+
+ H5_FAILED();
+
+ if (show_progress) {
+ HDfprintf(stdout, "%s: failure_mssg = \"%s\"\n", fcn_name, failure_mssg);
+ }
+ }
+ }
+
+ return (!pass);
+
+} /* vector_read_test_4() */
+
+/*-------------------------------------------------------------------------
+ * Function: vector_read_test_5()
+ *
+ * Purpose: Test correct management of the sizes[] array optimization,
+ * where, if sizes[i] == 0, we use sizes[i - 1] as the value
+ * of size[j], for j >= i.
+ *
+ * 1) Open the test file with the specified VFD, set the eoa.
+ * and setup the DXPL.
+ *
+ * 2) Using rank zero, write the entire increasing_fi_buf to
+ * the file.
+ *
+ * 3) Barrier
+ *
+ * 4) Set all cells of read_fi_buf to zero.
+ *
+ * 5) For each rank, define base_index equal to:
+ *
+ * mpi_rank * INTS_PER_RANK
+ *
+ * and define base_addr equal to
+ *
+ * base_index * sizeof(int32_t).
+ *
+ * Setup a vector read between base_addr and
+ * base_addr + INTS_PER_RANK * sizeof(int32_t) - 1
+ * that reads every 16th integer located in that
+ * that range starting at base_addr. Use a sizes[]
+ * array of length 2, with sizes[0] set to sizeof(int32_t),
+ * and sizes[1] = 0.
+ *
+ * Read the integers into the corresponding locations in
+ * read_fi_buf.
+ *
+ * 6) On each rank, verify that read_fi_buf contains the
+ * the expected values -- that is the matching values from
+ * increasing_fi_buf where ever there was a read, and zero
+ * otherwise.
+ *
+ * 7) Barrier.
+ *
+ * 8) Close the test file.
+ *
+ * 9) On rank 0, delete the test file.
+ *
+ * Return: FALSE on success, TRUE if any errors are detected.
+ *
+ * Programmer: John Mainzer
+ * 3/26/21
+ *
+ * Modifications:
+ *
+ * None.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static unsigned
+vector_read_test_5(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode,
+ H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name)
+{
+ const char *fcn_name = "vector_read_test_5()";
+ char test_title[120];
+ char filename[512];
+ haddr_t eoa;
+ haddr_t base_addr;
+ hbool_t show_progress = FALSE;
+ hid_t fapl_id = -1; /* file access property list ID */
+ hid_t dxpl_id = -1; /* data access property list ID */
+ H5FD_t * lf = NULL; /* VFD struct ptr */
+ int cp = 0;
+ int i;
+ int j;
+ int base_index;
+ uint32_t count = 0;
+ H5FD_mem_t types[(INTS_PER_RANK / 16) + 1];
+ haddr_t addrs[(INTS_PER_RANK / 16) + 1];
+ size_t sizes[2];
+ void * bufs[(INTS_PER_RANK / 16) + 1];
+
+ pass = TRUE;
+
+ if (mpi_rank == 0) {
+
+ if (xfer_mode == H5FD_MPIO_INDEPENDENT) {
+
+ sprintf(test_title, "parallel vector read test 5 -- %s / independent", vfd_name);
+ }
+ else if (coll_opt_mode == H5FD_MPIO_INDIVIDUAL_IO) {
+
+ sprintf(test_title, "parallel vector read test 5 -- %s / col op / ind I/O", vfd_name);
+ }
+ else {
+
+ HDassert(coll_opt_mode == H5FD_MPIO_COLLECTIVE_IO);
+
+ sprintf(test_title, "parallel vector read test 5 -- %s / col op / col I/O", vfd_name);
+ }
+
+ TESTING(test_title);
+ }
+
+ show_progress = ((show_progress) && (mpi_rank == 0));
+
+ if (show_progress)
+ HDfprintf(stdout, "\n%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 1) Open the test file with the specified VFD, set the eoa, and setup the dxpl */
+ if (pass) {
+
+ eoa = (haddr_t)mpi_size * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t));
+
+ setup_vfd_test_file(file_name_id, filename, mpi_size, xfer_mode, coll_opt_mode, vfd_name, eoa, &lf,
+ &fapl_id, &dxpl_id);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 2) Using rank zero, write the entire negative_fi_buf to
+ * the file.
+ */
+ if (pass) {
+
+ size_t image_size = (size_t)mpi_size * (size_t)INTS_PER_RANK * sizeof(int32_t);
+
+ if (mpi_rank == 0) {
+
+ if (H5FDwrite(lf, H5FD_MEM_DRAW, H5P_DEFAULT, (haddr_t)0, image_size, (void *)increasing_fi_buf) <
+ 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDwrite() on rank 0 failed.\n";
+ }
+ }
+ }
+
+ /* 3) Barrier */
+
+ if (pass) {
+
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 4) Set all cells of read_fi_buf to zero. */
+ if (pass) {
+
+ for (i = 0; i < mpi_size * INTS_PER_RANK; i++) {
+
+ read_fi_buf[i] = 0;
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 5) For each rank, define base_index equal to:
+ *
+ * mpi_rank * INTS_PER_RANK
+ *
+ * and define base_addr equal to
+ *
+ * base_index * sizeof(int32_t).
+ *
+ * Setup a vector read between base_addr and
+ * base_addr + INTS_PER_RANK * sizeof(int32_t) - 1
+ * that reads every 16th integer located in that
+ * that range starting at base_addr. Use a sizes[]
+ * array of length 2, with sizes[0] set to sizeof(int32_t),
+ * and sizes[1] = 0.
+ *
+ * Read the integers into the corresponding locations in
+ * read_fi_buf.
+ */
+ if (pass) {
+
+ base_index = (mpi_rank * INTS_PER_RANK);
+ base_addr = (haddr_t)base_index * (haddr_t)sizeof(int32_t);
+
+ count = INTS_PER_RANK / 16;
+ sizes[0] = sizeof(int32_t);
+ sizes[1] = 0;
+
+ for (i = 0; i < INTS_PER_RANK / 16; i++) {
+
+ types[i] = H5FD_MEM_DRAW;
+ addrs[i] = base_addr + ((haddr_t)(16 * i) * (haddr_t)sizeof(int32_t));
+ bufs[i] = (void *)(&(read_fi_buf[base_index + (i * 16)]));
+ }
+
+ if (H5FDread_vector(lf, dxpl_id, count, types, addrs, sizes, bufs) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDread_vector() failed (1).\n";
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 6) On each rank, verify that read_fi_buf contains the
+ * the expected values -- that is the matching values from
+ * increasing_fi_buf where ever there was a read, and zero
+ * otherwise.
+ */
+ if (pass) {
+
+ for (i = 0; ((pass) && (i < mpi_size)); i++) {
+
+ base_index = i * INTS_PER_RANK;
+
+ for (j = base_index; j < base_index + INTS_PER_RANK; j++) {
+
+ if ((i == mpi_rank) && (j % 16 == 0)) {
+
+ if (read_fi_buf[j] != increasing_fi_buf[j]) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (1)";
+ }
+ }
+ else if (read_fi_buf[j] != 0) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (2)";
+ }
+ } /* end for loop */
+ } /* end for loop */
+ } /* end if */
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 7) Barrier */
+
+ if (pass) {
+
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 8) Close the test file and delete it (on rank 0 only).
+ * Close FAPL and DXPL.
+ */
+
+ if (pass) {
+
+ takedown_vfd_test_file(mpi_rank, filename, &lf, &fapl_id, &dxpl_id);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* report results */
+ if (mpi_rank == 0) {
+
+ if (pass) {
+
+ PASSED();
+ }
+ else {
+
+ H5_FAILED();
+
+ if (show_progress) {
+ HDfprintf(stdout, "%s: failure_mssg = \"%s\"\n", fcn_name, failure_mssg);
+ }
+ }
+ }
+
+ return (!pass);
+
+} /* vector_read_test_5() */
+
+/*-------------------------------------------------------------------------
+ * Function: vector_write_test_1()
+ *
+ * Purpose: Simple vector write test:
+ *
+ * 1) Open the test file with the specified VFD, set the eoa,
+ * and setup the DXPL.
+ *
+ * 2) Write the entire increasing_fi_buf to the file, with
+ * exactly one buffer per vector per rank. Use either
+ * independant or collective I/O as specified.
+ *
+ * 3) Barrier
+ *
+ * 4) On each rank, read the entire file into the read_fi_buf,
+ * and compare against increasing_fi_buf. Report failure
+ * if any differences are detected.
+ *
+ * 5) Close the test file.
+ *
+ * 6) On rank 0, delete the test file.
+ *
+ * Return: FALSE on success, TRUE if any errors are detected.
+ *
+ * Programmer: John Mainzer
+ * 3/26/21
+ *
+ * Modifications:
+ *
+ * None.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static unsigned
+vector_write_test_1(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode,
+ H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name)
+{
+ const char *fcn_name = "vector_write_test_1()";
+ char test_title[120];
+ char filename[512];
+ haddr_t eoa;
+ hbool_t show_progress = FALSE;
+ hid_t fapl_id = -1; /* file access property list ID */
+ hid_t dxpl_id = -1; /* data access property list ID */
+ H5FD_t * lf = NULL; /* VFD struct ptr */
+ int cp = 0;
+ int i;
+ uint32_t count;
+ H5FD_mem_t types[1];
+ haddr_t addrs[1];
+ size_t sizes[1];
+ void * bufs[1];
+
+ pass = TRUE;
+
+ if (mpi_rank == 0) {
+
+ if (xfer_mode == H5FD_MPIO_INDEPENDENT) {
+
+ sprintf(test_title, "parallel vector write test 1 -- %s / independent", vfd_name);
+ }
+ else if (coll_opt_mode == H5FD_MPIO_INDIVIDUAL_IO) {
+
+ sprintf(test_title, "parallel vector write test 1 -- %s / col op / ind I/O", vfd_name);
+ }
+ else {
+
+ HDassert(coll_opt_mode == H5FD_MPIO_COLLECTIVE_IO);
+
+ sprintf(test_title, "parallel vector write test 1 -- %s / col op / col I/O", vfd_name);
+ }
+
+ TESTING(test_title);
+ }
+
+ show_progress = ((show_progress) && (mpi_rank == 0));
+
+ if (show_progress)
+ HDfprintf(stdout, "\n%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 1) Open the test file with the specified VFD, set the eoa, and setup the dxpl */
+ if (pass) {
+
+ eoa = (haddr_t)mpi_size * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t));
+
+ setup_vfd_test_file(file_name_id, filename, mpi_size, xfer_mode, coll_opt_mode, vfd_name, eoa, &lf,
+ &fapl_id, &dxpl_id);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 2) Write the entire increasing_fi_buf to the file, with
+ * exactly one buffer per vector per rank. Use either
+ * independant or collective I/O as specified.
+ */
+
+ if (pass) {
+
+ count = 1;
+ types[0] = H5FD_MEM_DRAW;
+ addrs[0] = (haddr_t)mpi_rank * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t));
+ sizes[0] = (size_t)INTS_PER_RANK * sizeof(int32_t);
+ bufs[0] = (void *)(&(increasing_fi_buf[mpi_rank * INTS_PER_RANK]));
+
+ if (H5FDwrite_vector(lf, dxpl_id, count, types, addrs, sizes, bufs) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDwrite_vector() failed.\n";
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 3) Barrier
+ */
+
+ if (pass) {
+
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 4) On each rank, read the entire file into the read_fi_buf,
+ * and compare against increasing_fi_buf. Report failure
+ * if any differences are detected.
+ */
+
+ if (pass) {
+
+ size_t image_size = (size_t)mpi_size * (size_t)INTS_PER_RANK * sizeof(int32_t);
+
+ if (H5FDread(lf, H5FD_MEM_DRAW, H5P_DEFAULT, (haddr_t)0, image_size, (void *)read_fi_buf) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDread() failed.\n";
+ }
+
+ for (i = 0; i < mpi_size * INTS_PER_RANK; i++) {
+
+ if (read_fi_buf[i] != increasing_fi_buf[i]) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file";
+ break;
+ }
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 5) Close the test file and delete it (on rank 0 only).
+ * Close FAPL and DXPL.
+ */
+
+ if (pass) {
+
+ takedown_vfd_test_file(mpi_rank, filename, &lf, &fapl_id, &dxpl_id);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* report results */
+ if (mpi_rank == 0) {
+
+ if (pass) {
+
+ PASSED();
+ }
+ else {
+
+ H5_FAILED();
+
+ if (show_progress) {
+ HDfprintf(stdout, "%s: failure_mssg = \"%s\"\n", fcn_name, failure_mssg);
+ }
+ }
+ }
+
+ return (!pass);
+
+} /* vector_write_test_1() */
+
+/*-------------------------------------------------------------------------
+ * Function: vector_write_test_2()
+ *
+ * Purpose: Test vector I/O writes in which only some ranks participate.
+ * Depending on the collective parameter, these writes will
+ * be either collective or independent.
+ *
+ * 1) Open the test file with the specified VFD, and set
+ * the eoa.
+ *
+ * 2) Write the odd blocks of the increasing_fi_buf to the file,
+ * with the odd ranks writting the odd blocks, and the even
+ * ranks writing an empty vector.
+ *
+ * Here, a "block" of the increasing_fi_buf is a sequence
+ * of integers in increasing_fi_buf of length INTS_PER_RANK,
+ * and with start index a multiple of INTS_PER_RANK.
+ *
+ * 3) Write the even blocks of the negative_fi_buf to the file,
+ * with the even ranks writting the even blocks, and the odd
+ * ranks writing an empty vector.
+ *
+ * 4) Barrier
+ *
+ * 4) On each rank, read the entire file into the read_fi_buf,
+ * and compare against increasing_fi_buf and negative_fi_buf
+ * as appropriate. Report failure if any differences are
+ * detected.
+ *
+ * 5) Close the test file. On rank 0, delete the test file.
+ *
+ * Return: FALSE on success, TRUE if any errors are detected.
+ *
+ * Programmer: John Mainzer
+ * 3/28/21
+ *
+ * Modifications:
+ *
+ * None.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static unsigned
+vector_write_test_2(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode,
+ H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name)
+{
+ const char *fcn_name = "vector_write_test_2()";
+ char test_title[120];
+ char filename[512];
+ haddr_t eoa;
+ hbool_t show_progress = FALSE;
+ hid_t fapl_id = -1; /* file access property list ID */
+ hid_t dxpl_id = -1; /* data access property list ID */
+ H5FD_t * lf = NULL; /* VFD struct ptr */
+ int cp = 0;
+ int i;
+ int j;
+ uint32_t count;
+ H5FD_mem_t types[1];
+ haddr_t addrs[1];
+ size_t sizes[1];
+ void * bufs[1];
+
+ pass = TRUE;
+
+ if (mpi_rank == 0) {
+
+ if (xfer_mode == H5FD_MPIO_INDEPENDENT) {
+
+ sprintf(test_title, "parallel vector write test 2 -- %s / independent", vfd_name);
+ }
+ else if (coll_opt_mode == H5FD_MPIO_INDIVIDUAL_IO) {
+
+ sprintf(test_title, "parallel vector write test 2 -- %s / col op / ind I/O", vfd_name);
+ }
+ else {
+
+ HDassert(coll_opt_mode == H5FD_MPIO_COLLECTIVE_IO);
+
+ sprintf(test_title, "parallel vector write test 2 -- %s / col op / col I/O", vfd_name);
+ }
+
+ TESTING(test_title);
+ }
+
+ show_progress = ((show_progress) && (mpi_rank == 0));
+
+ if (show_progress)
+ HDfprintf(stdout, "\n%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 1) Open the test file with the specified VFD, set the eoa, and setup the dxpl */
+ if (pass) {
+
+ eoa = (haddr_t)mpi_size * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t));
+
+ setup_vfd_test_file(file_name_id, filename, mpi_size, xfer_mode, coll_opt_mode, vfd_name, eoa, &lf,
+ &fapl_id, &dxpl_id);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 2) Write the odd blocks of the increasing_fi_buf to the file,
+ * with the odd ranks writting the odd blocks, and the even
+ * ranks writing an empty vector.
+ *
+ * Here, a "block" of the increasing_fi_buf is a sequence
+ * of integers in increasing_fi_buf of length INTS_PER_RANK,
+ * and with start index a multiple of INTS_PER_RANK.
+ */
+ if (pass) {
+
+ if (mpi_rank % 2 == 1) { /* odd ranks */
+
+ count = 1;
+ types[0] = H5FD_MEM_DRAW;
+ addrs[0] = (haddr_t)mpi_rank * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t));
+ sizes[0] = (size_t)INTS_PER_RANK * sizeof(int32_t);
+ bufs[0] = (void *)(&(increasing_fi_buf[mpi_rank * INTS_PER_RANK]));
+
+ if (H5FDwrite_vector(lf, dxpl_id, count, types, addrs, sizes, bufs) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDwrite_vector() failed (1).\n";
+ }
+ }
+ else { /* even ranks */
+
+ if (H5FDwrite_vector(lf, dxpl_id, 0, NULL, NULL, NULL, NULL) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDwrite_vector() failed (2).\n";
+ }
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 3) Write the even blocks of the negative_fi_buf to the file,
+ * with the even ranks writting the even blocks, and the odd
+ * ranks writing an empty vector.
+ */
+ if (pass) {
+
+ if (mpi_rank % 2 == 1) { /* odd ranks */
+
+ if (H5FDwrite_vector(lf, dxpl_id, 0, NULL, NULL, NULL, NULL) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDwrite_vector() failed (3).\n";
+ }
+ }
+ else { /* even ranks */
+
+ count = 1;
+ types[0] = H5FD_MEM_DRAW;
+ addrs[0] = (haddr_t)mpi_rank * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t));
+ sizes[0] = (size_t)INTS_PER_RANK * sizeof(int32_t);
+ bufs[0] = (void *)(&(negative_fi_buf[mpi_rank * INTS_PER_RANK]));
+
+ if (H5FDwrite_vector(lf, dxpl_id, count, types, addrs, sizes, bufs) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDwrite_vector() failed (4).\n";
+ }
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 4) Barrier
+ */
+
+ if (pass) {
+
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 5) On each rank, read the entire file into the read_fi_buf,
+ * and compare against increasing_fi_buf. Report failure
+ * if any differences are detected.
+ */
+
+ if (pass) {
+
+ size_t image_size = (size_t)mpi_size * (size_t)INTS_PER_RANK * sizeof(int32_t);
+
+ if (H5FDread(lf, H5FD_MEM_DRAW, H5P_DEFAULT, (haddr_t)0, image_size, (void *)read_fi_buf) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDread() failed.\n";
+ }
+
+ for (i = 0; ((pass) && (i < mpi_size)); i++) {
+
+ if (i % 2 == 1) { /* odd block */
+
+ for (j = i * INTS_PER_RANK; ((pass) && (j < (i + 1) * INTS_PER_RANK)); j++) {
+
+ if (read_fi_buf[j] != increasing_fi_buf[j]) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file";
+ break;
+ }
+ }
+ }
+ else { /* even block */
+
+ for (j = i * INTS_PER_RANK; ((pass) && (j < (i + 1) * INTS_PER_RANK)); j++) {
+
+ if (read_fi_buf[j] != negative_fi_buf[j]) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file";
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 6) Close the test file and delete it (on rank 0 only).
+ * Close FAPL and DXPL.
+ */
+
+ if (pass) {
+
+ takedown_vfd_test_file(mpi_rank, filename, &lf, &fapl_id, &dxpl_id);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* report results */
+ if (mpi_rank == 0) {
+
+ if (pass) {
+
+ PASSED();
+ }
+ else {
+
+ H5_FAILED();
+
+ if (show_progress) {
+ HDfprintf(stdout, "%s: failure_mssg = \"%s\"\n", fcn_name, failure_mssg);
+ }
+ }
+ }
+
+ return (!pass);
+
+} /* vector_write_test_2() */
+
+/*-------------------------------------------------------------------------
+ * Function: vector_write_test_3()
+ *
+ * Purpose: Test vector I/O writes with vectors of multiple entries.
+ * For now, keep the vectors sorted in increasing address
+ * order.
+ *
+ * 1) Open the test file with the specified VFD, and set
+ * the eoa.
+ *
+ * 2) For each rank, construct a vector with base address
+ * (mpi_rank * INTS_PER_RANK) and writing all bytes from
+ * that address to ((mpi_rank + 1) * INTS_PER_RANK) - 1.
+ * Draw equal parts from increasing_fi_buf,
+ * decreasing_fi_buf, negative_fi_buf, and zero_fi_buf.
+ *
+ * Write to file.
+ *
+ * 3) Barrier
+ *
+ * 4) On each rank, read the entire file into the read_fi_buf,
+ * and compare against increasing_fi_buf,
+ * decreasing_fi_buf, negative_fi_buf, and zero_fi_buf as
+ * appropriate. Report failure if any differences are
+ * detected.
+ *
+ * 5) Close the test file. On rank 0, delete the test file.
+ *
+ * Return: FALSE on success, TRUE if any errors are detected.
+ *
+ * Programmer: John Mainzer
+ * 3/31/21
+ *
+ * Modifications:
+ *
+ * None.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static unsigned
+vector_write_test_3(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode,
+ H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name)
+{
+ const char *fcn_name = "vector_write_test_3()";
+ char test_title[120];
+ char filename[512];
+ haddr_t base_addr;
+ int base_index;
+ int ints_per_write;
+ size_t bytes_per_write;
+ haddr_t eoa;
+ hbool_t show_progress = FALSE;
+ hid_t fapl_id = -1; /* file access property list ID */
+ hid_t dxpl_id = -1; /* data access property list ID */
+ H5FD_t * lf = NULL; /* VFD struct ptr */
+ int cp = 0;
+ int i;
+ int j;
+ uint32_t count;
+ H5FD_mem_t types[4];
+ haddr_t addrs[4];
+ size_t sizes[4];
+ void * bufs[4];
+
+ pass = TRUE;
+
+ if (mpi_rank == 0) {
+
+ if (xfer_mode == H5FD_MPIO_INDEPENDENT) {
+
+ sprintf(test_title, "parallel vector write test 3 -- %s / independent", vfd_name);
+ }
+ else if (coll_opt_mode == H5FD_MPIO_INDIVIDUAL_IO) {
+
+ sprintf(test_title, "parallel vector write test 3 -- %s / col op / ind I/O", vfd_name);
+ }
+ else {
+
+ HDassert(coll_opt_mode == H5FD_MPIO_COLLECTIVE_IO);
+
+ sprintf(test_title, "parallel vector write test 3 -- %s / col op / col I/O", vfd_name);
+ }
+
+ TESTING(test_title);
+ }
+
+ show_progress = ((show_progress) && (mpi_rank == 0));
+
+ if (show_progress)
+ HDfprintf(stdout, "\n%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 1) Open the test file with the specified VFD, set the eoa, and setup the dxpl */
+ if (pass) {
+
+ eoa = (haddr_t)mpi_size * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t));
+
+ setup_vfd_test_file(file_name_id, filename, mpi_size, xfer_mode, coll_opt_mode, vfd_name, eoa, &lf,
+ &fapl_id, &dxpl_id);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 2) For each rank, construct a vector with base address
+ * (mpi_rank * INTS_PER_RANK) and writing all bytes from
+ * that address to ((mpi_rank + 1) * INTS_PER_RANK) - 1.
+ * Draw equal parts from increasing_fi_buf,
+ * decreasing_fi_buf, negative_fi_buf, and zero_fi_buf.
+ *
+ * Write to file.
+ */
+ if (pass) {
+
+ count = 4;
+
+ base_addr = (haddr_t)mpi_rank * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t));
+ ints_per_write = INTS_PER_RANK / 4;
+ bytes_per_write = (size_t)(ints_per_write) * sizeof(int32_t);
+
+ types[0] = H5FD_MEM_DRAW;
+ addrs[0] = base_addr;
+ sizes[0] = bytes_per_write;
+ bufs[0] = (void *)(&(increasing_fi_buf[mpi_rank * INTS_PER_RANK]));
+
+ types[1] = H5FD_MEM_DRAW;
+ addrs[1] = addrs[0] + (haddr_t)(bytes_per_write);
+ sizes[1] = bytes_per_write;
+ bufs[1] = (void *)(&(decreasing_fi_buf[(mpi_rank * INTS_PER_RANK) + (INTS_PER_RANK / 4)]));
+
+ types[2] = H5FD_MEM_DRAW;
+ addrs[2] = addrs[1] + (haddr_t)(bytes_per_write);
+ sizes[2] = bytes_per_write;
+ bufs[2] = (void *)(&(negative_fi_buf[(mpi_rank * INTS_PER_RANK) + (INTS_PER_RANK / 2)]));
+
+ types[3] = H5FD_MEM_DRAW;
+ addrs[3] = addrs[2] + (haddr_t)(bytes_per_write);
+ sizes[3] = bytes_per_write;
+ bufs[3] = (void *)(&(zero_fi_buf[(mpi_rank * INTS_PER_RANK) + (3 * (INTS_PER_RANK / 4))]));
+
+#if 0 /* JRM */
+ HDfprintf(stdout, "addrs = { %lld, %lld, %lld, %lld}\n",
+ (long long)addrs[0], (long long)addrs[1], (long long)addrs[2], (long long)addrs[3]);
+ HDfprintf(stdout, "sizes = { %lld, %lld, %lld, %lld}\n",
+ (long long)sizes[0], (long long)sizes[1], (long long)sizes[2], (long long)sizes[3]);
+ HDfprintf(stdout, "bufs = { 0x%llx, 0x%llx, 0x%llx, 0x%llx}\n",
+ (unsigned long long)bufs[0], (unsigned long long)bufs[1],
+ (unsigned long long)bufs[2], (unsigned long long)bufs[3]);
+#endif /* JRM */
+
+ if (H5FDwrite_vector(lf, dxpl_id, count, types, addrs, sizes, bufs) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDwrite_vector() failed (1).\n";
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 3) Barrier
+ */
+
+ if (pass) {
+
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 4) On each rank, read the entire file into the read_fi_buf,
+ * and compare against increasing_fi_buf,
+ * decreasing_fi_buf, negative_fi_buf, and zero_fi_buf as
+ * appropriate. Report failure if any differences are
+ * detected.
+ */
+
+ if (pass) {
+
+ size_t image_size = (size_t)mpi_size * (size_t)INTS_PER_RANK * sizeof(int32_t);
+
+ if (H5FDread(lf, H5FD_MEM_DRAW, H5P_DEFAULT, (haddr_t)0, image_size, (void *)read_fi_buf) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDread() failed.\n";
+ }
+
+ for (i = 0; ((pass) && (i < mpi_size)); i++) {
+
+ base_index = i * INTS_PER_RANK;
+
+ for (j = base_index; j < base_index + (INTS_PER_RANK / 4); j++) {
+
+ if (read_fi_buf[j] != increasing_fi_buf[j]) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (1)";
+ break;
+ }
+ }
+
+ base_index += (INTS_PER_RANK / 4);
+
+ for (j = base_index; j < base_index + (INTS_PER_RANK / 4); j++) {
+
+ if (read_fi_buf[j] != decreasing_fi_buf[j]) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (2)";
+ break;
+ }
+ }
+
+ base_index += (INTS_PER_RANK / 4);
+
+ for (j = base_index; j < base_index + (INTS_PER_RANK / 4); j++) {
+
+ if (read_fi_buf[j] != negative_fi_buf[j]) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (3)";
+ break;
+ }
+ }
+
+ base_index += (INTS_PER_RANK / 4);
+
+ for (j = base_index; j < base_index + (INTS_PER_RANK / 4); j++) {
+
+ if (read_fi_buf[j] != zero_fi_buf[j]) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (3)";
+ break;
+ }
+ }
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 5) Close the test file and delete it (on rank 0 only).
+ * Close FAPL and DXPL.
+ */
+
+ if (pass) {
+
+ takedown_vfd_test_file(mpi_rank, filename, &lf, &fapl_id, &dxpl_id);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* report results */
+ if (mpi_rank == 0) {
+
+ if (pass) {
+
+ PASSED();
+ }
+ else {
+
+ H5_FAILED();
+
+ if (show_progress) {
+ HDfprintf(stdout, "%s: failure_mssg = \"%s\"\n", fcn_name, failure_mssg);
+ }
+ }
+ }
+
+ return (!pass);
+
+} /* vector_write_test_3() */
+
+/*-------------------------------------------------------------------------
+ * Function: vector_write_test_4()
+ *
+ * Purpose: Test vector I/O writes with vectors of multiple entries.
+ * For now, keep the vectors sorted in increasing address
+ * order.
+ *
+ * This test differs from vector_write_test_3() in the order
+ * in which the file image buffers appear in the vector
+ * write. This guarantees that at least one of these
+ * tests will present buffers with non-increasing addresses
+ * in RAM.
+ *
+ * 1) Open the test file with the specified VFD, and set
+ * the eoa.
+ *
+ * 2) For each rank, construct a vector with base address
+ * (mpi_rank * INTS_PER_RANK) and writing all bytes from
+ * that address to ((mpi_rank + 1) * INTS_PER_RANK) - 1.
+ * Draw equal parts from zero_fi_buf, negative_fi_buf,
+ * decreasing_fi_buf, and increasing_fi_buf.
+ *
+ * Write to file.
+ *
+ * 3) Barrier
+ *
+ * 4) On each rank, read the entire file into the read_fi_buf,
+ * and compare against zero_fi_buf, negative_fi_buf,
+ * decreasing_fi_buf, and increasing_fi_buf as
+ * appropriate. Report failure if any differences are
+ * detected.
+ *
+ * 5) Close the test file. On rank 0, delete the test file.
+ *
+ * Return: FALSE on success, TRUE if any errors are detected.
+ *
+ * Programmer: John Mainzer
+ * 3/31/21
+ *
+ * Modifications:
+ *
+ * None.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static unsigned
+vector_write_test_4(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode,
+ H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name)
+{
+ const char *fcn_name = "vector_write_test_4()";
+ char test_title[120];
+ char filename[512];
+ haddr_t base_addr;
+ int base_index;
+ int ints_per_write;
+ size_t bytes_per_write;
+ haddr_t eoa;
+ hbool_t show_progress = FALSE;
+ hid_t fapl_id = -1; /* file access property list ID */
+ hid_t dxpl_id = -1; /* data access property list ID */
+ H5FD_t * lf = NULL; /* VFD struct ptr */
+ int cp = 0;
+ int i;
+ int j;
+ uint32_t count;
+ H5FD_mem_t types[4];
+ haddr_t addrs[4];
+ size_t sizes[4];
+ void * bufs[4];
+
+ pass = TRUE;
+
+ if (mpi_rank == 0) {
+
+ if (xfer_mode == H5FD_MPIO_INDEPENDENT) {
+
+ sprintf(test_title, "parallel vector write test 4 -- %s / independent", vfd_name);
+ }
+ else if (coll_opt_mode == H5FD_MPIO_INDIVIDUAL_IO) {
+
+ sprintf(test_title, "parallel vector write test 4 -- %s / col op / ind I/O", vfd_name);
+ }
+ else {
+
+ HDassert(coll_opt_mode == H5FD_MPIO_COLLECTIVE_IO);
+
+ sprintf(test_title, "parallel vector write test 4 -- %s / col op / col I/O", vfd_name);
+ }
+
+ TESTING(test_title);
+ }
+
+ show_progress = ((show_progress) && (mpi_rank == 0));
+
+ if (show_progress)
+ HDfprintf(stdout, "\n%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 1) Open the test file with the specified VFD, set the eoa, and setup the dxpl */
+ if (pass) {
+
+ eoa = (haddr_t)mpi_size * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t));
+
+ setup_vfd_test_file(file_name_id, filename, mpi_size, xfer_mode, coll_opt_mode, vfd_name, eoa, &lf,
+ &fapl_id, &dxpl_id);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 2) For each rank, construct a vector with base address
+ * (mpi_rank * INTS_PER_RANK) and writing all bytes from
+ * that address to ((mpi_rank + 1) * INTS_PER_RANK) - 1.
+ * Draw equal parts from increasing_fi_buf,
+ * decreasing_fi_buf, negative_fi_buf, and zero_fi_buf.
+ *
+ * Write to file.
+ */
+ if (pass) {
+
+ count = 4;
+
+ base_addr = (haddr_t)mpi_rank * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t));
+ ints_per_write = INTS_PER_RANK / 4;
+ bytes_per_write = (size_t)(ints_per_write) * sizeof(int32_t);
+
+ types[0] = H5FD_MEM_DRAW;
+ addrs[0] = base_addr;
+ sizes[0] = bytes_per_write;
+ bufs[0] = (void *)(&(zero_fi_buf[mpi_rank * INTS_PER_RANK]));
+
+ types[1] = H5FD_MEM_DRAW;
+ addrs[1] = addrs[0] + (haddr_t)(bytes_per_write);
+ sizes[1] = bytes_per_write;
+ bufs[1] = (void *)(&(negative_fi_buf[(mpi_rank * INTS_PER_RANK) + (INTS_PER_RANK / 4)]));
+
+ types[2] = H5FD_MEM_DRAW;
+ addrs[2] = addrs[1] + (haddr_t)(bytes_per_write);
+ sizes[2] = bytes_per_write;
+ bufs[2] = (void *)(&(decreasing_fi_buf[(mpi_rank * INTS_PER_RANK) + (INTS_PER_RANK / 2)]));
+
+ types[3] = H5FD_MEM_DRAW;
+ addrs[3] = addrs[2] + (haddr_t)(bytes_per_write);
+ sizes[3] = bytes_per_write;
+ bufs[3] = (void *)(&(increasing_fi_buf[(mpi_rank * INTS_PER_RANK) + (3 * (INTS_PER_RANK / 4))]));
+
+#if 0 /* JRM */
+ HDfprintf(stdout, "addrs = { %lld, %lld, %lld, %lld}\n",
+ (long long)addrs[0], (long long)addrs[1], (long long)addrs[2], (long long)addrs[3]);
+ HDfprintf(stdout, "sizes = { %lld, %lld, %lld, %lld}\n",
+ (long long)sizes[0], (long long)sizes[1], (long long)sizes[2], (long long)sizes[3]);
+ HDfprintf(stdout, "bufs = { 0x%llx, 0x%llx, 0x%llx, 0x%llx}\n",
+ (unsigned long long)bufs[0], (unsigned long long)bufs[1],
+ (unsigned long long)bufs[2], (unsigned long long)bufs[3]);
+#endif /* JRM */
+
+ if (H5FDwrite_vector(lf, dxpl_id, count, types, addrs, sizes, bufs) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDwrite_vector() failed (1).\n";
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 3) Barrier
+ */
+
+ if (pass) {
+
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 4) On each rank, read the entire file into the read_fi_buf,
+ * and compare against increasing_fi_buf,
+ * decreasing_fi_buf, negative_fi_buf, and zero_fi_buf as
+ * appropriate. Report failure if any differences are
+ * detected.
+ */
+
+ if (pass) {
+
+ size_t image_size = (size_t)mpi_size * (size_t)INTS_PER_RANK * sizeof(int32_t);
+
+ if (H5FDread(lf, H5FD_MEM_DRAW, H5P_DEFAULT, (haddr_t)0, image_size, (void *)read_fi_buf) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDread() failed.\n";
+ }
+
+ for (i = 0; ((pass) && (i < mpi_size)); i++) {
+
+ base_index = i * INTS_PER_RANK;
+
+ for (j = base_index; j < base_index + (INTS_PER_RANK / 4); j++) {
+
+ if (read_fi_buf[j] != zero_fi_buf[j]) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (1)";
+ break;
+ }
+ }
+
+ base_index += (INTS_PER_RANK / 4);
+
+ for (j = base_index; j < base_index + (INTS_PER_RANK / 4); j++) {
+
+ if (read_fi_buf[j] != negative_fi_buf[j]) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (2)";
+ break;
+ }
+ }
+
+ base_index += (INTS_PER_RANK / 4);
+
+ for (j = base_index; j < base_index + (INTS_PER_RANK / 4); j++) {
+
+ if (read_fi_buf[j] != decreasing_fi_buf[j]) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (3)";
+ break;
+ }
+ }
+
+ base_index += (INTS_PER_RANK / 4);
+
+ for (j = base_index; j < base_index + (INTS_PER_RANK / 4); j++) {
+
+ if (read_fi_buf[j] != increasing_fi_buf[j]) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (3)";
+ break;
+ }
+ }
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 5) Close the test file and delete it (on rank 0 only).
+ * Close FAPL and DXPL.
+ */
+
+ if (pass) {
+
+ takedown_vfd_test_file(mpi_rank, filename, &lf, &fapl_id, &dxpl_id);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* report results */
+ if (mpi_rank == 0) {
+
+ if (pass) {
+
+ PASSED();
+ }
+ else {
+
+ H5_FAILED();
+
+ if (show_progress) {
+ HDfprintf(stdout, "%s: failure_mssg = \"%s\"\n", fcn_name, failure_mssg);
+ }
+ }
+ }
+
+ return (!pass);
+
+} /* vector_write_test_4() */
+
+/*-------------------------------------------------------------------------
+ * Function: vector_write_test_5()
+ *
+ * Purpose: Test vector I/O writes with vectors of different lengths
+ * and entry sizes across the ranks. Vectors are not, in
+ * general, sorted in increasing address order. Further,
+ * writes are not, in general, contiguous.
+ *
+ * 1) Open the test file with the specified VFD, and set
+ * the eoa.
+ *
+ * 2) Set the test file in a known state by writing zeros
+ * to all bytes in the test file. Since we have already
+ * tested this, do this via a vector write of zero_fi_buf.
+ *
+ * 3) Barrier
+ *
+ * 4) For each rank, define base_index equal to:
+ *
+ * mpi_rank * INTS_PER_RANK
+ *
+ * and define base_addr equal to
+ *
+ * base_index * sizeof(int32_t).
+ *
+ * Setup a vector write between base_addr and
+ * base_addr + INTS_PER_RANK * sizeof(int32_t) - 1
+ * as follows:
+ *
+ * if ( rank % 4 == 0 ) construct a vector that writes:
+ *
+ * negative_fi_buf starting at base_index +
+ * INTS_PER_RANK / 2 and running for INTS_PER_RANK / 4
+ * entries,
+ *
+ * decreasing_fi_buf starting at base_index +
+ * INTS_PER_RANK / 4 and running for INTS_PER_RANK / 8
+ * entries, and
+ *
+ * increasing_fi_buf starting at base_index +
+ * INTS_PER_RANK / 16 and running for INTS_PER_RANK / 16
+ * entries
+ *
+ * to the equivalent locations in the file.
+ *
+ * if ( rank % 4 == 1 ) construct a vector that writes:
+ *
+ * increasing_fi_buf starting at base_index + 1 and
+ * running for (INTS_PER_RANK / 2) - 2 entries, and
+ *
+ * decreasing_fi_buf startomg at base_index +
+ * INTS_PER_RANK / 2 + 1 and running for (INTS_PER_RANK / 2)
+ * - 2 entries
+ *
+ * if ( rank % 4 == 2 ) construct a vector that writes:
+ *
+ * negative_fi_buf starting at base_index +
+ * INTS_PER_RANK / 2 and running for one entry.
+ *
+ * if ( rank % 4 == 3 ) construct and write the empty vector
+ *
+ * 5) Barrier
+ *
+ * 6) On each rank, read the entire file into the read_fi_buf,
+ * and compare against zero_fi_buf, negative_fi_buf,
+ * decreasing_fi_buf, and increasing_fi_buf as
+ * appropriate. Report failure if any differences are
+ * detected.
+ *
+ * 7) Close the test file. On rank 0, delete the test file.
+ *
+ * Return: FALSE on success, TRUE if any errors are detected.
+ *
+ * Programmer: John Mainzer
+ * 3/31/21
+ *
+ * Modifications:
+ *
+ * None.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static unsigned
+vector_write_test_5(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode,
+ H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name)
+{
+ const char *fcn_name = "vector_write_test_5()";
+ char test_title[120];
+ char filename[512];
+ haddr_t base_addr;
+ int base_index;
+ haddr_t eoa;
+ hbool_t show_progress = FALSE;
+ hid_t fapl_id = -1; /* file access property list ID */
+ hid_t dxpl_id = -1; /* data access property list ID */
+ H5FD_t * lf = NULL; /* VFD struct ptr */
+ int cp = 0;
+ int i;
+ int j;
+ int k;
+ uint32_t count;
+ H5FD_mem_t types[4];
+ haddr_t addrs[4];
+ size_t sizes[4];
+ void * bufs[4];
+
+ pass = TRUE;
+
+ if (mpi_rank == 0) {
+
+ if (xfer_mode == H5FD_MPIO_INDEPENDENT) {
+
+ sprintf(test_title, "parallel vector write test 5 -- %s / independent", vfd_name);
+ }
+ else if (coll_opt_mode == H5FD_MPIO_INDIVIDUAL_IO) {
+
+ sprintf(test_title, "parallel vector write test 5 -- %s / col op / ind I/O", vfd_name);
+ }
+ else {
+
+ HDassert(coll_opt_mode == H5FD_MPIO_COLLECTIVE_IO);
+
+ sprintf(test_title, "parallel vector write test 5 -- %s / col op / col I/O", vfd_name);
+ }
+
+ TESTING(test_title);
+ }
+
+ show_progress = ((show_progress) && (mpi_rank == 0));
+
+ if (show_progress)
+ HDfprintf(stdout, "\n%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 1) Open the test file with the specified VFD, set the eoa, and setup the dxpl */
+ if (pass) {
+
+ eoa = (haddr_t)mpi_size * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t));
+
+ setup_vfd_test_file(file_name_id, filename, mpi_size, xfer_mode, coll_opt_mode, vfd_name, eoa, &lf,
+ &fapl_id, &dxpl_id);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 2) Set the test file in a known state by writing zeros
+ * to all bytes in the test file. Since we have already
+ * tested this, do this via a vector write of zero_fi_buf.
+ */
+ if (pass) {
+
+ count = 1;
+ types[0] = H5FD_MEM_DRAW;
+ addrs[0] = (haddr_t)mpi_rank * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t));
+ sizes[0] = (size_t)INTS_PER_RANK * sizeof(int32_t);
+ bufs[0] = (void *)(&(zero_fi_buf[mpi_rank * INTS_PER_RANK]));
+
+ if (H5FDwrite_vector(lf, dxpl_id, count, types, addrs, sizes, bufs) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDwrite_vector() failed.\n";
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 3) Barrier
+ */
+
+ if (pass) {
+
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 4) For each rank, define base_index equal to:
+ *
+ * mpi_rank * INTS_PER_RANK
+ *
+ * and define base_addr equal to
+ *
+ * base_index * sizeof(int32_t).
+ *
+ * Setup a vector write between base_addr and
+ * base_addr + INTS_PER_RANK * sizeof(int32_t) - 1
+ * as follows:
+ */
+ if (pass) {
+
+ base_index = mpi_rank * INTS_PER_RANK;
+ base_addr = (haddr_t)((size_t)base_index * sizeof(int32_t));
+
+ if ((mpi_rank % 4) == 0) {
+
+ /* if ( rank % 4 == 0 ) construct a vector that writes:
+ *
+ * negative_fi_buf starting at base_index +
+ * INTS_PER_RANK / 2 and running for INTS_PER_RANK / 4
+ * entries,
+ *
+ * decreasing_fi_buf starting at base_index +
+ * INTS_PER_RANK / 4 and running for INTS_PER_RANK / 8
+ * entries, and
+ *
+ * increasing_fi_buf starting at base_index +
+ * INTS_PER_RANK / 16 and running for INTS_PER_RANK / 16
+ * entries
+ *
+ * to the equivalent locations in the file.
+ */
+ count = 3;
+
+ types[0] = H5FD_MEM_DRAW;
+ addrs[0] = base_addr + (haddr_t)((size_t)(INTS_PER_RANK / 2) * sizeof(int32_t));
+ sizes[0] = (size_t)(INTS_PER_RANK / 4) * sizeof(int32_t);
+ bufs[0] = (void *)(&(negative_fi_buf[base_index + (INTS_PER_RANK / 2)]));
+
+ types[1] = H5FD_MEM_DRAW;
+ addrs[1] = base_addr + (haddr_t)((size_t)(INTS_PER_RANK / 4) * sizeof(int32_t));
+ sizes[1] = (size_t)(INTS_PER_RANK / 8) * sizeof(int32_t);
+ bufs[1] = (void *)(&(decreasing_fi_buf[base_index + (INTS_PER_RANK / 4)]));
+
+ types[2] = H5FD_MEM_DRAW;
+ addrs[2] = base_addr + (haddr_t)((size_t)(INTS_PER_RANK / 16) * sizeof(int32_t));
+ sizes[2] = (size_t)(INTS_PER_RANK / 16) * sizeof(int32_t);
+ bufs[2] = (void *)(&(increasing_fi_buf[base_index + (INTS_PER_RANK / 16)]));
+ }
+ else if ((mpi_rank % 4) == 1) {
+
+ /* if ( rank % 4 == 1 ) construct a vector that writes:
+ *
+ * increasing_fi_buf starting at base_index + 1 and
+ * running for (INTS_PER_RANK / 2) - 2 entries, and
+ *
+ * decreasing_fi_buf startomg at base_addr +
+ * INTS_PER_RANK / 2 + 1 and running for (INTS_PER_RANK / 2)
+ * - 2 entries
+ *
+ * to the equivalent locations in the file.
+ */
+ count = 2;
+
+ types[0] = H5FD_MEM_DRAW;
+ addrs[0] = base_addr + (haddr_t)(sizeof(int32_t));
+ sizes[0] = (size_t)((INTS_PER_RANK / 2) - 2) * sizeof(int32_t);
+ bufs[0] = (void *)(&(increasing_fi_buf[base_index + 1]));
+
+ types[1] = H5FD_MEM_DRAW;
+ addrs[1] = base_addr + (haddr_t)((size_t)((INTS_PER_RANK / 2) + 1) * sizeof(int32_t));
+ sizes[1] = (size_t)((INTS_PER_RANK / 2) - 2) * sizeof(int32_t);
+ bufs[1] = (void *)(&(decreasing_fi_buf[base_index + (INTS_PER_RANK / 2) + 1]));
+ }
+ else if ((mpi_rank % 4) == 2) {
+
+ /* if ( rank % 4 == 2 ) construct a vector that writes:
+ *
+ * negative_fi_buf starting at base_index +
+ * INTS_PER_RANK / 2 and running for one entry.
+ *
+ * to the equivalent location in the file.
+ */
+ count = 1;
+
+ types[0] = H5FD_MEM_DRAW;
+ addrs[0] = base_addr + (haddr_t)((size_t)(INTS_PER_RANK / 2) * sizeof(int32_t));
+ sizes[0] = sizeof(int32_t);
+ bufs[0] = (void *)(&(negative_fi_buf[base_index + (INTS_PER_RANK / 2)]));
+ }
+ else if ((mpi_rank % 4) == 3) {
+
+ /* if ( rank % 4 == 3 ) construct and write the empty vector */
+
+ count = 0;
+ }
+
+ if (H5FDwrite_vector(lf, dxpl_id, count, types, addrs, sizes, bufs) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDwrite_vector() failed (1).\n";
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 5) Barrier */
+
+ if (pass) {
+
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 6) On each rank, read the entire file into the read_fi_buf,
+ * and compare against increasing_fi_buf,
+ * decreasing_fi_buf, negative_fi_buf, and zero_fi_buf as
+ * appropriate. Report failure if any differences are
+ * detected.
+ */
+
+ if (pass) {
+
+ size_t image_size = (size_t)mpi_size * (size_t)INTS_PER_RANK * sizeof(int32_t);
+
+ if (H5FDread(lf, H5FD_MEM_DRAW, H5P_DEFAULT, (haddr_t)0, image_size, (void *)read_fi_buf) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDread() failed.\n";
+ }
+
+ for (i = 0; ((pass) && (i < mpi_size)); i++) {
+
+ base_index = i * INTS_PER_RANK;
+
+ for (j = base_index; j < base_index + INTS_PER_RANK; j++) {
+
+ k = j - base_index;
+
+ switch (i % 4) {
+
+ case 0:
+ if (((INTS_PER_RANK / 2) <= k) && (k < (3 * (INTS_PER_RANK / 4)))) {
+
+ if (read_fi_buf[j] != negative_fi_buf[j]) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (1.1)";
+ }
+ }
+ else if (((INTS_PER_RANK / 4) <= k) && (k < (3 * (INTS_PER_RANK / 8)))) {
+
+ if (read_fi_buf[j] != decreasing_fi_buf[j]) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (1.2)";
+ }
+ }
+ else if (((INTS_PER_RANK / 16) <= k) && (k < (INTS_PER_RANK / 8))) {
+
+ if (read_fi_buf[j] != increasing_fi_buf[j]) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (1.3)";
+ }
+ }
+ else {
+
+ if (read_fi_buf[j] != 0) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (1.4)";
+ }
+ }
+ break;
+
+ case 1:
+ if ((1 <= k) && (k <= ((INTS_PER_RANK / 2) - 2))) {
+
+ if (read_fi_buf[j] != increasing_fi_buf[j]) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (2.1)";
+ }
+ }
+ else if ((((INTS_PER_RANK / 2) + 1) <= k) && (k <= (INTS_PER_RANK - 2))) {
+
+ if (read_fi_buf[j] != decreasing_fi_buf[j]) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (2.2)";
+ }
+ }
+ else {
+
+ if (read_fi_buf[j] != 0) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (2.3)";
+ }
+ }
+ break;
+
+ case 2:
+ if (k == INTS_PER_RANK / 2) {
+
+ if (read_fi_buf[j] != negative_fi_buf[j]) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (3.1)";
+ }
+ }
+ else {
+
+ if (read_fi_buf[j] != 0) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (3.2)";
+ }
+ }
+ break;
+
+ case 3:
+ if (read_fi_buf[j] != 0) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (4)";
+ }
+ break;
+
+ default:
+ HDassert(FALSE); /* should be un-reachable */
+ break;
+ }
+ }
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 7) Close the test file and delete it (on rank 0 only).
+ * Close FAPL and DXPL.
+ */
+
+ if (pass) {
+
+ takedown_vfd_test_file(mpi_rank, filename, &lf, &fapl_id, &dxpl_id);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* report results */
+ if (mpi_rank == 0) {
+
+ if (pass) {
+
+ PASSED();
+ }
+ else {
+
+ H5_FAILED();
+
+ if (show_progress) {
+ HDfprintf(stdout, "%s: failure_mssg = \"%s\"\n", fcn_name, failure_mssg);
+ }
+ }
+ }
+
+ return (!pass);
+
+} /* vector_write_test_5() */
+
+/*-------------------------------------------------------------------------
+ * Function: vector_write_test_6()
+ *
+ * Purpose: Test correct management of the sizes[] array optimization,
+ * where, if sizes[i] == 0, we use sizes[i - 1] as the value
+ * of size[j], for j >= i.
+ *
+ * 1) Open the test file with the specified VFD, set the eoa.
+ * and setup the DXPL.
+ *
+ * 2) Using rank zero, write the entire zero_fi_buf to
+ * the file.
+ *
+ * 3) Barrier
+ *
+ * 4) For each rank, define base_index equal to:
+ *
+ * mpi_rank * INTS_PER_RANK
+ *
+ * and define base_addr equal to
+ *
+ * base_index * sizeof(int32_t).
+ *
+ * Setup a vector write from increasing_fi_buf between
+ * base_addr and base_addr + INTS_PER_RANK *
+ * sizeof(int32_t) - 1 that writes every 16th integer
+ * located in that range starting at base_addr.
+ * Use a sizes[] array of length 2, with sizes[0] set
+ * to sizeof(int32_t), and sizes[1] = 0.
+ *
+ * Write the integers into the corresponding locations in
+ * the file.
+ *
+ * 5) Barrier
+ *
+ * 6) On each rank, read the entire file into the read_fi_buf,
+ * and compare against zero_fi_buf, and increasing_fi_buf
+ * as appropriate. Report failure if any differences are
+ * detected.
+ *
+ * 7) Barrier.
+ *
+ * 8) Close the test file.
+ *
+ * 9) On rank 0, delete the test file.
+ *
+ * Return: FALSE on success, TRUE if any errors are detected.
+ *
+ * Programmer: John Mainzer
+ * 3/26/21
+ *
+ * Modifications:
+ *
+ * None.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static unsigned
+vector_write_test_6(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode,
+ H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name)
+{
+ const char *fcn_name = "vector_write_test_6()";
+ char test_title[120];
+ char filename[512];
+ haddr_t eoa;
+ haddr_t base_addr;
+ hbool_t show_progress = FALSE;
+ hid_t fapl_id = -1; /* file access property list ID */
+ hid_t dxpl_id = -1; /* data access property list ID */
+ H5FD_t * lf = NULL; /* VFD struct ptr */
+ int cp = 0;
+ int i;
+ int base_index;
+ uint32_t count = 0;
+ H5FD_mem_t types[(INTS_PER_RANK / 16) + 1];
+ haddr_t addrs[(INTS_PER_RANK / 16) + 1];
+ size_t sizes[2];
+ void * bufs[(INTS_PER_RANK / 16) + 1];
+
+ pass = TRUE;
+
+ if (mpi_rank == 0) {
+
+ if (xfer_mode == H5FD_MPIO_INDEPENDENT) {
+
+ sprintf(test_title, "parallel vector write test 6 -- %s / independent", vfd_name);
+ }
+ else if (coll_opt_mode == H5FD_MPIO_INDIVIDUAL_IO) {
+
+ sprintf(test_title, "parallel vector write test 6 -- %s / col op / ind I/O", vfd_name);
+ }
+ else {
+
+ HDassert(coll_opt_mode == H5FD_MPIO_COLLECTIVE_IO);
+
+ sprintf(test_title, "parallel vector write test 6 -- %s / col op / col I/O", vfd_name);
+ }
+
+ TESTING(test_title);
+ }
+
+ show_progress = ((show_progress) && (mpi_rank == 0));
+
+ if (show_progress)
+ HDfprintf(stdout, "\n%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 1) Open the test file with the specified VFD, set the eoa, and setup the dxpl */
+ if (pass) {
+
+ eoa = (haddr_t)mpi_size * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t));
+
+ setup_vfd_test_file(file_name_id, filename, mpi_size, xfer_mode, coll_opt_mode, vfd_name, eoa, &lf,
+ &fapl_id, &dxpl_id);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 2) Using rank zero, write the entire negative_fi_buf to
+ * the file.
+ */
+ if (pass) {
+
+ size_t image_size = (size_t)mpi_size * (size_t)INTS_PER_RANK * sizeof(int32_t);
+
+ if (mpi_rank == 0) {
+
+ if (H5FDwrite(lf, H5FD_MEM_DRAW, H5P_DEFAULT, (haddr_t)0, image_size, (void *)zero_fi_buf) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDwrite() on rank 0 failed.\n";
+ }
+ }
+ }
+
+ /* 3) Barrier */
+
+ if (pass) {
+
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 4) For each rank, define base_index equal to:
+ *
+ * mpi_rank * INTS_PER_RANK
+ *
+ * and define base_addr equal to
+ *
+ * base_index * sizeof(int32_t).
+ *
+ * Setup a vector write from increasing_fi_buf between
+ * base_addr and base_addr + INTS_PER_RANK *
+ * sizeof(int32_t) - 1 that writes every 16th integer
+ * located in that range starting at base_addr.
+ * Use a sizes[] array of length 2, with sizes[0] set
+ * to sizeof(int32_t), and sizes[1] = 0.
+ *
+ * Write the integers into the corresponding locations in
+ * the file.
+ */
+ if (pass) {
+
+ base_index = (mpi_rank * INTS_PER_RANK);
+ base_addr = (haddr_t)base_index * (haddr_t)sizeof(int32_t);
+
+ count = INTS_PER_RANK / 16;
+ sizes[0] = sizeof(int32_t);
+ sizes[1] = 0;
+
+ for (i = 0; i < INTS_PER_RANK / 16; i++) {
+
+ types[i] = H5FD_MEM_DRAW;
+ addrs[i] = base_addr + ((haddr_t)(16 * i) * (haddr_t)sizeof(int32_t));
+ bufs[i] = (void *)(&(increasing_fi_buf[base_index + (i * 16)]));
+ }
+
+ if (H5FDwrite_vector(lf, dxpl_id, count, types, addrs, sizes, bufs) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDwrite_vector() failed (1).\n";
+ }
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 5) Barrier */
+
+ if (pass) {
+
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 6) On each rank, read the entire file into the read_fi_buf,
+ * and compare against zero_fi_buf, and increasing_fi_buf
+ * as appropriate. Report failure if any differences are
+ * detected.
+ */
+ if (pass) {
+
+ size_t image_size = (size_t)mpi_size * (size_t)INTS_PER_RANK * sizeof(int32_t);
+
+ if (H5FDread(lf, H5FD_MEM_DRAW, H5P_DEFAULT, (haddr_t)0, image_size, (void *)read_fi_buf) < 0) {
+
+ pass = FALSE;
+ failure_mssg = "H5FDread() failed.\n";
+ }
+
+ for (i = 0; ((pass) && (i < mpi_size * INTS_PER_RANK)); i++) {
+
+ if (i % 16 == 0) {
+
+ if (read_fi_buf[i] != increasing_fi_buf[i]) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (1)";
+ }
+ }
+ else if (read_fi_buf[i] != zero_fi_buf[i]) {
+
+ pass = FALSE;
+ failure_mssg = "unexpected data read from file (2)";
+ }
+ }
+ } /* end if */
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 7) Barrier */
+
+ if (pass) {
+
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* 8) Close the test file and delete it (on rank 0 only).
+ * Close FAPL and DXPL.
+ */
+
+ if (pass) {
+
+ takedown_vfd_test_file(mpi_rank, filename, &lf, &fapl_id, &dxpl_id);
+ }
+
+ if (show_progress)
+ HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+ /* report results */
+ if (mpi_rank == 0) {
+
+ if (pass) {
+
+ PASSED();
+ }
+ else {
+
+ H5_FAILED();
+
+ if (show_progress) {
+ HDfprintf(stdout, "%s: failure_mssg = \"%s\"\n", fcn_name, failure_mssg);
+ }
+ }
+ }
+
+ return (!pass);
+
+} /* vector_write_test_6() */
+
+/*-------------------------------------------------------------------------
+ * Function: main
+ *
+ * Purpose: Run parallel VFD tests.
+ *
+ * Return: Success: 0
+ *
+ * Failure: 1
+ *
+ * Programmer: John Mainzer
+ * 3/2621/
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+
+int
+main(int argc, char **argv)
+{
+ unsigned nerrs = 0;
+ MPI_Comm comm = MPI_COMM_WORLD;
+ MPI_Info info = MPI_INFO_NULL;
+ int mpi_size;
+ int mpi_rank;
+
+ MPI_Init(&argc, &argv);
+ MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
+ MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
+
+ /* Attempt to turn off atexit post processing so that in case errors
+ * occur during the test and the process is aborted, it will not hang
+ * in the atexit post processing. If it does, it may try to make MPI
+ * calls which may not work.
+ */
+ if (H5dont_atexit() < 0)
+ HDprintf("%d:Failed to turn off atexit processing. Continue.\n", mpi_rank);
+
+ H5open();
+
+ if (mpi_rank == 0) {
+ HDprintf("=========================================\n");
+ HDprintf("Parallel virtual file driver (VFD) tests\n");
+ HDprintf(" mpi_size = %d\n", mpi_size);
+ HDprintf("=========================================\n");
+ }
+
+ if (mpi_size < 2) {
+ if (mpi_rank == 0)
+ HDprintf(" Need at least 2 processes. Exiting.\n");
+ goto finish;
+ }
+
+ alloc_and_init_file_images(mpi_size);
+
+ if (!pass) {
+
+ HDprintf("\nAllocation and initialze of file image buffers failed. Test aborted.\n");
+ }
+
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ // sleep(60);
+
+ nerrs +=
+ vector_read_test_1(0, mpi_rank, mpi_size, H5FD_MPIO_INDEPENDENT, H5FD_MPIO_INDIVIDUAL_IO, "mpio");
+ nerrs += vector_read_test_1(0, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_INDIVIDUAL_IO, "mpio");
+ nerrs += vector_read_test_1(0, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, "mpio");
+
+ nerrs +=
+ vector_read_test_2(1, mpi_rank, mpi_size, H5FD_MPIO_INDEPENDENT, H5FD_MPIO_INDIVIDUAL_IO, "mpio");
+ nerrs += vector_read_test_2(1, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_INDIVIDUAL_IO, "mpio");
+ nerrs += vector_read_test_2(1, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, "mpio");
+
+ nerrs +=
+ vector_read_test_3(2, mpi_rank, mpi_size, H5FD_MPIO_INDEPENDENT, H5FD_MPIO_INDIVIDUAL_IO, "mpio");
+ nerrs += vector_read_test_3(2, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_INDIVIDUAL_IO, "mpio");
+ nerrs += vector_read_test_3(2, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, "mpio");
+
+ nerrs +=
+ vector_read_test_4(3, mpi_rank, mpi_size, H5FD_MPIO_INDEPENDENT, H5FD_MPIO_INDIVIDUAL_IO, "mpio");
+ nerrs += vector_read_test_4(3, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_INDIVIDUAL_IO, "mpio");
+ nerrs += vector_read_test_4(3, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, "mpio");
+
+ nerrs +=
+ vector_read_test_5(4, mpi_rank, mpi_size, H5FD_MPIO_INDEPENDENT, H5FD_MPIO_INDIVIDUAL_IO, "mpio");
+ nerrs += vector_read_test_5(4, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_INDIVIDUAL_IO, "mpio");
+ nerrs += vector_read_test_5(4, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, "mpio");
+
+ nerrs +=
+ vector_write_test_1(0, mpi_rank, mpi_size, H5FD_MPIO_INDEPENDENT, H5FD_MPIO_INDIVIDUAL_IO, "mpio");
+ nerrs +=
+ vector_write_test_1(0, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_INDIVIDUAL_IO, "mpio");
+ nerrs +=
+ vector_write_test_1(0, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, "mpio");
+
+ nerrs +=
+ vector_write_test_2(1, mpi_rank, mpi_size, H5FD_MPIO_INDEPENDENT, H5FD_MPIO_INDIVIDUAL_IO, "mpio");
+ nerrs +=
+ vector_write_test_2(1, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_INDIVIDUAL_IO, "mpio");
+ nerrs +=
+ vector_write_test_2(1, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, "mpio");
+
+ nerrs +=
+ vector_write_test_3(2, mpi_rank, mpi_size, H5FD_MPIO_INDEPENDENT, H5FD_MPIO_INDIVIDUAL_IO, "mpio");
+ nerrs +=
+ vector_write_test_3(2, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_INDIVIDUAL_IO, "mpio");
+ nerrs +=
+ vector_write_test_3(2, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, "mpio");
+
+ nerrs +=
+ vector_write_test_4(3, mpi_rank, mpi_size, H5FD_MPIO_INDEPENDENT, H5FD_MPIO_INDIVIDUAL_IO, "mpio");
+ nerrs +=
+ vector_write_test_4(3, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_INDIVIDUAL_IO, "mpio");
+ nerrs +=
+ vector_write_test_4(3, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, "mpio");
+
+ nerrs +=
+ vector_write_test_5(4, mpi_rank, mpi_size, H5FD_MPIO_INDEPENDENT, H5FD_MPIO_INDIVIDUAL_IO, "mpio");
+ nerrs +=
+ vector_write_test_5(4, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_INDIVIDUAL_IO, "mpio");
+ nerrs +=
+ vector_write_test_5(4, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, "mpio");
+
+ nerrs +=
+ vector_write_test_6(5, mpi_rank, mpi_size, H5FD_MPIO_INDEPENDENT, H5FD_MPIO_INDIVIDUAL_IO, "mpio");
+ nerrs +=
+ vector_write_test_6(5, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_INDIVIDUAL_IO, "mpio");
+ nerrs +=
+ vector_write_test_6(5, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, "mpio");
+
+finish:
+
+ /* make sure all processes are finished before final report, cleanup
+ * and exit.
+ */
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ if (mpi_rank == 0) { /* only process 0 reports */
+ HDprintf("===================================\n");
+ if (nerrs > 0)
+ HDprintf("***parallel vfd tests detected %d failures***\n", nerrs);
+ else
+ HDprintf("parallel vfd tests finished with no failures\n");
+ HDprintf("===================================\n");
+ }
+
+ /* discard the file image buffers */
+ free_file_images();
+
+ /* close HDF5 library */
+ H5close();
+
+ /* MPI_Finalize must be called AFTER H5close which may use MPI calls */
+ MPI_Finalize();
+
+ /* cannot just return (nerrs) because exit code is limited to 1byte */
+ return (nerrs > 0);
+
+} /* main() */