path: root/testpar/t_shapesame.c

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
 * Copyright by The HDF Group.                                               *
 * All rights reserved.                                                      *
 *                                                                           *
 * This file is part of HDF5.  The full HDF5 copyright notice, including     *
 * terms governing use, modification, and redistribution, is contained in    *
 * the COPYING file, which can be found at the root of the source code       *
 * distribution tree, or in https://www.hdfgroup.org/licenses.               *
 * If you do not have access to either file, you may request a copy from     *
 * help@hdfgroup.org.                                                        *
 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

/*
   This program will test independent and collective reads and writes between
   selections of different rank that non-the-less are deemed as having the
   same shape by H5Sselect_shape_same().
 */

#define H5S_FRIEND /*suppress error about including H5Spkg   */

/* Define this macro to indicate that the testing APIs should be available */
#define H5S_TESTING

#include "H5Spkg.h" /* Dataspaces                           */
#include "testphdf5.h"

#ifndef PATH_MAX
#define PATH_MAX 512
#endif

/* FILENAME and filenames must have the same number of names.
 * Use PARATESTFILE in general and use a separated filename only if the file
 * created in one test is accessed by a different test.
 * filenames[0] is reserved as the file name for PARATESTFILE.
 */
#define NFILENAME    2
#define PARATESTFILE filenames[0]
const char *FILENAME[NFILENAME] = {"ShapeSameTest", NULL};
char       *filenames[NFILENAME];
hid_t       fapl; /* file access property list */

/* On Lustre (and perhaps other parallel file systems?), we have severe
 * slow downs if two or more processes attempt to access the same file system
 * block.  To minimize this problem, we set alignment in the shape same tests
 * to the default Lustre block size -- which greatly reduces contention in
 * the chunked dataset case.
 */

#define SHAPE_SAME_TEST_ALIGNMENT ((hsize_t)(4 * 1024 * 1024))

#define PAR_SS_DR_MAX_RANK 5 /* must update code if this changes */

struct hs_dr_pio_test_vars_t {
    int       mpi_size;
    int       mpi_rank;
    MPI_Comm  mpi_comm;
    MPI_Info  mpi_info;
    int       test_num;
    int       edge_size;
    int       checker_edge_size;
    int       chunk_edge_size;
    int       small_rank;
    int       large_rank;
    hid_t     dset_type;
    uint32_t *small_ds_buf_0;
    uint32_t *small_ds_buf_1;
    uint32_t *small_ds_buf_2;
    uint32_t *small_ds_slice_buf;
    uint32_t *large_ds_buf_0;
    uint32_t *large_ds_buf_1;
    uint32_t *large_ds_buf_2;
    uint32_t *large_ds_slice_buf;
    int       small_ds_offset;
    int       large_ds_offset;
    hid_t     fid; /* HDF5 file ID */
    hid_t     xfer_plist;
    hid_t     full_mem_small_ds_sid;
    hid_t     full_file_small_ds_sid;
    hid_t     mem_small_ds_sid;
    hid_t     file_small_ds_sid_0;
    hid_t     file_small_ds_sid_1;
    hid_t     small_ds_slice_sid;
    hid_t     full_mem_large_ds_sid;
    hid_t     full_file_large_ds_sid;
    hid_t     mem_large_ds_sid;
    hid_t     file_large_ds_sid_0;
    hid_t     file_large_ds_sid_1;
    hid_t     file_large_ds_process_slice_sid;
    hid_t     mem_large_ds_process_slice_sid;
    hid_t     large_ds_slice_sid;
    hid_t     small_dataset; /* Dataset ID */
    hid_t     large_dataset; /* Dataset ID */
    size_t    small_ds_size;
    size_t    small_ds_slice_size;
    size_t    large_ds_size;
    size_t    large_ds_slice_size;
    hsize_t   dims[PAR_SS_DR_MAX_RANK];
    hsize_t   chunk_dims[PAR_SS_DR_MAX_RANK];
    hsize_t   start[PAR_SS_DR_MAX_RANK];
    hsize_t   stride[PAR_SS_DR_MAX_RANK];
    hsize_t   count[PAR_SS_DR_MAX_RANK];
    hsize_t   block[PAR_SS_DR_MAX_RANK];
    hsize_t  *start_ptr;
    hsize_t  *stride_ptr;
    hsize_t  *count_ptr;
    hsize_t  *block_ptr;
    int       skips;
    int       max_skips;
    int64_t   total_tests;
    int64_t   tests_run;
    int64_t   tests_skipped;
};

/*-------------------------------------------------------------------------
 * Function:    hs_dr_pio_test__setup()
 *
 * Purpose:    Do setup for tests of I/O to/from hyperslab selections of
 *         different rank in the parallel case.
 *
 * Return:    void
 *
 *-------------------------------------------------------------------------
 */

#define CONTIG_HS_DR_PIO_TEST__SETUP__DEBUG 0

static void
hs_dr_pio_test__setup(const int test_num, const int edge_size, const int checker_edge_size,
                      const int chunk_edge_size, const int small_rank, const int large_rank,
                      const bool use_collective_io, const hid_t dset_type, const int express_test,
                      struct hs_dr_pio_test_vars_t *tv_ptr)
{
#if CONTIG_HS_DR_PIO_TEST__SETUP__DEBUG
    const char *fcnName = "hs_dr_pio_test__setup()";
#endif /* CONTIG_HS_DR_PIO_TEST__SETUP__DEBUG */
    const char *filename;
    bool        mis_match = false;
    int         i;
    int         mrc;
    int         mpi_rank; /* needed by the VRFY macro */
    uint32_t    expected_value;
    uint32_t   *ptr_0;
    uint32_t   *ptr_1;
    hid_t       acc_tpl; /* File access templates */
    hid_t       small_ds_dcpl_id = H5P_DEFAULT;
    hid_t       large_ds_dcpl_id = H5P_DEFAULT;
    herr_t      ret; /* Generic return value */

    assert(edge_size >= 6);
    assert(edge_size >= chunk_edge_size);
    assert((chunk_edge_size == 0) || (chunk_edge_size >= 3));
    assert(1 < small_rank);
    assert(small_rank < large_rank);
    assert(large_rank <= PAR_SS_DR_MAX_RANK);

    tv_ptr->test_num          = test_num;
    tv_ptr->edge_size         = edge_size;
    tv_ptr->checker_edge_size = checker_edge_size;
    tv_ptr->chunk_edge_size   = chunk_edge_size;
    tv_ptr->small_rank        = small_rank;
    tv_ptr->large_rank        = large_rank;
    tv_ptr->dset_type         = dset_type;

    MPI_Comm_size(MPI_COMM_WORLD, &(tv_ptr->mpi_size));
    MPI_Comm_rank(MPI_COMM_WORLD, &(tv_ptr->mpi_rank));
    /* the VRFY() macro needs the local variable mpi_rank -- set it up now */
    mpi_rank = tv_ptr->mpi_rank;

    assert(tv_ptr->mpi_size >= 1);

    tv_ptr->mpi_comm = MPI_COMM_WORLD;
    tv_ptr->mpi_info = MPI_INFO_NULL;

    for (i = 0; i < tv_ptr->small_rank - 1; i++) {
        tv_ptr->small_ds_size *= (size_t)(tv_ptr->edge_size);
        tv_ptr->small_ds_slice_size *= (size_t)(tv_ptr->edge_size);
    }
    tv_ptr->small_ds_size *= (size_t)(tv_ptr->mpi_size + 1);

    /* used by checker board tests only */
    tv_ptr->small_ds_offset = PAR_SS_DR_MAX_RANK - tv_ptr->small_rank;

    assert(0 < tv_ptr->small_ds_offset);
    assert(tv_ptr->small_ds_offset < PAR_SS_DR_MAX_RANK);

    for (i = 0; i < tv_ptr->large_rank - 1; i++) {

        tv_ptr->large_ds_size *= (size_t)(tv_ptr->edge_size);
        tv_ptr->large_ds_slice_size *= (size_t)(tv_ptr->edge_size);
    }
    tv_ptr->large_ds_size *= (size_t)(tv_ptr->mpi_size + 1);

    /* used by checker board tests only */
    tv_ptr->large_ds_offset = PAR_SS_DR_MAX_RANK - tv_ptr->large_rank;

    assert(0 <= tv_ptr->large_ds_offset);
    assert(tv_ptr->large_ds_offset < PAR_SS_DR_MAX_RANK);

    /* set up the start, stride, count, and block pointers */
    /* used by contiguous tests only */
    tv_ptr->start_ptr  = &(tv_ptr->start[PAR_SS_DR_MAX_RANK - tv_ptr->large_rank]);
    tv_ptr->stride_ptr = &(tv_ptr->stride[PAR_SS_DR_MAX_RANK - tv_ptr->large_rank]);
    tv_ptr->count_ptr  = &(tv_ptr->count[PAR_SS_DR_MAX_RANK - tv_ptr->large_rank]);
    tv_ptr->block_ptr  = &(tv_ptr->block[PAR_SS_DR_MAX_RANK - tv_ptr->large_rank]);

    /* Allocate buffers */
    tv_ptr->small_ds_buf_0 = (uint32_t *)malloc(sizeof(uint32_t) * tv_ptr->small_ds_size);
    VRFY((tv_ptr->small_ds_buf_0 != NULL), "malloc of small_ds_buf_0 succeeded");

    tv_ptr->small_ds_buf_1 = (uint32_t *)malloc(sizeof(uint32_t) * tv_ptr->small_ds_size);
    VRFY((tv_ptr->small_ds_buf_1 != NULL), "malloc of small_ds_buf_1 succeeded");

    tv_ptr->small_ds_buf_2 = (uint32_t *)malloc(sizeof(uint32_t) * tv_ptr->small_ds_size);
    VRFY((tv_ptr->small_ds_buf_2 != NULL), "malloc of small_ds_buf_2 succeeded");

    tv_ptr->small_ds_slice_buf = (uint32_t *)malloc(sizeof(uint32_t) * tv_ptr->small_ds_slice_size);
    VRFY((tv_ptr->small_ds_slice_buf != NULL), "malloc of small_ds_slice_buf succeeded");

    tv_ptr->large_ds_buf_0 = (uint32_t *)malloc(sizeof(uint32_t) * tv_ptr->large_ds_size);
    VRFY((tv_ptr->large_ds_buf_0 != NULL), "malloc of large_ds_buf_0 succeeded");

    tv_ptr->large_ds_buf_1 = (uint32_t *)malloc(sizeof(uint32_t) * tv_ptr->large_ds_size);
    VRFY((tv_ptr->large_ds_buf_1 != NULL), "malloc of large_ds_buf_1 succeeded");

    tv_ptr->large_ds_buf_2 = (uint32_t *)malloc(sizeof(uint32_t) * tv_ptr->large_ds_size);
    VRFY((tv_ptr->large_ds_buf_2 != NULL), "malloc of large_ds_buf_2 succeeded");

    tv_ptr->large_ds_slice_buf = (uint32_t *)malloc(sizeof(uint32_t) * tv_ptr->large_ds_slice_size);
    VRFY((tv_ptr->large_ds_slice_buf != NULL), "malloc of large_ds_slice_buf succeeded");

    /* initialize the buffers */

    ptr_0 = tv_ptr->small_ds_buf_0;
    for (i = 0; i < (int)(tv_ptr->small_ds_size); i++)
        *ptr_0++ = (uint32_t)i;
    memset(tv_ptr->small_ds_buf_1, 0, sizeof(uint32_t) * tv_ptr->small_ds_size);
    memset(tv_ptr->small_ds_buf_2, 0, sizeof(uint32_t) * tv_ptr->small_ds_size);

    memset(tv_ptr->small_ds_slice_buf, 0, sizeof(uint32_t) * tv_ptr->small_ds_slice_size);

    ptr_0 = tv_ptr->large_ds_buf_0;
    for (i = 0; i < (int)(tv_ptr->large_ds_size); i++)
        *ptr_0++ = (uint32_t)i;
    memset(tv_ptr->large_ds_buf_1, 0, sizeof(uint32_t) * tv_ptr->large_ds_size);
    memset(tv_ptr->large_ds_buf_2, 0, sizeof(uint32_t) * tv_ptr->large_ds_size);

    memset(tv_ptr->large_ds_slice_buf, 0, sizeof(uint32_t) * tv_ptr->large_ds_slice_size);

    filename = (const char *)GetTestParameters();
    assert(filename != NULL);
#if CONTIG_HS_DR_PIO_TEST__SETUP__DEBUG
    if (MAINPROCESS) {

        fprintf(stdout, "%d: test num = %d.\n", tv_ptr->mpi_rank, tv_ptr->test_num);
        fprintf(stdout, "%d: mpi_size = %d.\n", tv_ptr->mpi_rank, tv_ptr->mpi_size);
        fprintf(stdout, "%d: small/large rank = %d/%d, use_collective_io = %d.\n", tv_ptr->mpi_rank,
                tv_ptr->small_rank, tv_ptr->large_rank, (int)use_collective_io);
        fprintf(stdout, "%d: edge_size = %d, chunk_edge_size = %d.\n", tv_ptr->mpi_rank, tv_ptr->edge_size,
                tv_ptr->chunk_edge_size);
        fprintf(stdout, "%d: checker_edge_size = %d.\n", tv_ptr->mpi_rank, tv_ptr->checker_edge_size);
        fprintf(stdout, "%d: small_ds_size = %d, large_ds_size = %d.\n", tv_ptr->mpi_rank,
                (int)(tv_ptr->small_ds_size), (int)(tv_ptr->large_ds_size));
        fprintf(stdout, "%d: filename = %s.\n", tv_ptr->mpi_rank, filename);
    }
#endif /* CONTIG_HS_DR_PIO_TEST__SETUP__DEBUG */
    /* ----------------------------------------
     * CREATE AN HDF5 FILE WITH PARALLEL ACCESS
     * ---------------------------------------*/
    /* setup file access template */
    acc_tpl = create_faccess_plist(tv_ptr->mpi_comm, tv_ptr->mpi_info, facc_type);
    VRFY((acc_tpl >= 0), "create_faccess_plist() succeeded");

    /* set the alignment -- need it large so that we aren't always hitting the
     * the same file system block.  Do this only if express_test is greater
     * than zero.
     */
    if (express_test > 0) {

        ret = H5Pset_alignment(acc_tpl, (hsize_t)0, SHAPE_SAME_TEST_ALIGNMENT);
        VRFY((ret != FAIL), "H5Pset_alignment() succeeded");
    }

    /* create the file collectively */
    tv_ptr->fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, acc_tpl);
    VRFY((tv_ptr->fid >= 0), "H5Fcreate succeeded");

    MESG("File opened.");

    /* Release file-access template */
    ret = H5Pclose(acc_tpl);
    VRFY((ret >= 0), "H5Pclose(acc_tpl) succeeded");

    /* setup dims: */
    tv_ptr->dims[0] = (hsize_t)(tv_ptr->mpi_size + 1);
    tv_ptr->dims[1] = tv_ptr->dims[2] = tv_ptr->dims[3] = tv_ptr->dims[4] = (hsize_t)(tv_ptr->edge_size);

    /* Create small ds dataspaces */
    tv_ptr->full_mem_small_ds_sid = H5Screate_simple(tv_ptr->small_rank, tv_ptr->dims, NULL);
    VRFY((tv_ptr->full_mem_small_ds_sid != 0), "H5Screate_simple() full_mem_small_ds_sid succeeded");

    tv_ptr->full_file_small_ds_sid = H5Screate_simple(tv_ptr->small_rank, tv_ptr->dims, NULL);
    VRFY((tv_ptr->full_file_small_ds_sid != 0), "H5Screate_simple() full_file_small_ds_sid succeeded");

    tv_ptr->mem_small_ds_sid = H5Screate_simple(tv_ptr->small_rank, tv_ptr->dims, NULL);
    VRFY((tv_ptr->mem_small_ds_sid != 0), "H5Screate_simple() mem_small_ds_sid succeeded");

    tv_ptr->file_small_ds_sid_0 = H5Screate_simple(tv_ptr->small_rank, tv_ptr->dims, NULL);
    VRFY((tv_ptr->file_small_ds_sid_0 != 0), "H5Screate_simple() file_small_ds_sid_0 succeeded");

    /* used by checker board tests only */
    tv_ptr->file_small_ds_sid_1 = H5Screate_simple(tv_ptr->small_rank, tv_ptr->dims, NULL);
    VRFY((tv_ptr->file_small_ds_sid_1 != 0), "H5Screate_simple() file_small_ds_sid_1 succeeded");

    tv_ptr->small_ds_slice_sid = H5Screate_simple(tv_ptr->small_rank - 1, &(tv_ptr->dims[1]), NULL);
    VRFY((tv_ptr->small_ds_slice_sid != 0), "H5Screate_simple() small_ds_slice_sid succeeded");

    /* Create large ds dataspaces */
    tv_ptr->full_mem_large_ds_sid = H5Screate_simple(tv_ptr->large_rank, tv_ptr->dims, NULL);
    VRFY((tv_ptr->full_mem_large_ds_sid != 0), "H5Screate_simple() full_mem_large_ds_sid succeeded");

    tv_ptr->full_file_large_ds_sid = H5Screate_simple(tv_ptr->large_rank, tv_ptr->dims, NULL);
    VRFY((tv_ptr->full_file_large_ds_sid != FAIL), "H5Screate_simple() full_file_large_ds_sid succeeded");

    tv_ptr->mem_large_ds_sid = H5Screate_simple(tv_ptr->large_rank, tv_ptr->dims, NULL);
    VRFY((tv_ptr->mem_large_ds_sid != FAIL), "H5Screate_simple() mem_large_ds_sid succeeded");

    tv_ptr->file_large_ds_sid_0 = H5Screate_simple(tv_ptr->large_rank, tv_ptr->dims, NULL);
    VRFY((tv_ptr->file_large_ds_sid_0 != FAIL), "H5Screate_simple() file_large_ds_sid_0 succeeded");

    /* used by checker board tests only */
    tv_ptr->file_large_ds_sid_1 = H5Screate_simple(tv_ptr->large_rank, tv_ptr->dims, NULL);
    VRFY((tv_ptr->file_large_ds_sid_1 != FAIL), "H5Screate_simple() file_large_ds_sid_1 succeeded");

    tv_ptr->mem_large_ds_process_slice_sid = H5Screate_simple(tv_ptr->large_rank, tv_ptr->dims, NULL);
    VRFY((tv_ptr->mem_large_ds_process_slice_sid != FAIL),
         "H5Screate_simple() mem_large_ds_process_slice_sid succeeded");

    tv_ptr->file_large_ds_process_slice_sid = H5Screate_simple(tv_ptr->large_rank, tv_ptr->dims, NULL);
    VRFY((tv_ptr->file_large_ds_process_slice_sid != FAIL),
         "H5Screate_simple() file_large_ds_process_slice_sid succeeded");

    tv_ptr->large_ds_slice_sid = H5Screate_simple(tv_ptr->large_rank - 1, &(tv_ptr->dims[1]), NULL);
    VRFY((tv_ptr->large_ds_slice_sid != 0), "H5Screate_simple() large_ds_slice_sid succeeded");

    /* if chunk edge size is greater than zero, set up the small and
     * large data set creation property lists to specify chunked
     * datasets.
     */
    if (tv_ptr->chunk_edge_size > 0) {

        /* Under Lustre (and perhaps other parallel file systems?) we get
         * locking delays when two or more processes attempt to access the
         * same file system block.
         *
         * To minimize this problem, I have changed chunk_dims[0]
         * from (mpi_size + 1) to just when any sort of express test is
         * selected.  Given the structure of the test, and assuming we
         * set the alignment large enough, this avoids the contention
         * issue by seeing to it that each chunk is only accessed by one
         * process.
         *
         * One can argue as to whether this is a good thing to do in our
         * tests, but for now it is necessary if we want the test to complete
         * in a reasonable amount of time.
         *
         *                                         JRM -- 9/16/10
         */

        tv_ptr->chunk_dims[0] = 1;

        tv_ptr->chunk_dims[1] = tv_ptr->chunk_dims[2] = tv_ptr->chunk_dims[3] = tv_ptr->chunk_dims[4] =
            (hsize_t)(tv_ptr->chunk_edge_size);

        small_ds_dcpl_id = H5Pcreate(H5P_DATASET_CREATE);
        VRFY((ret != FAIL), "H5Pcreate() small_ds_dcpl_id succeeded");

        ret = H5Pset_layout(small_ds_dcpl_id, H5D_CHUNKED);
        VRFY((ret != FAIL), "H5Pset_layout() small_ds_dcpl_id succeeded");

        ret = H5Pset_chunk(small_ds_dcpl_id, tv_ptr->small_rank, tv_ptr->chunk_dims);
        VRFY((ret != FAIL), "H5Pset_chunk() small_ds_dcpl_id succeeded");

        large_ds_dcpl_id = H5Pcreate(H5P_DATASET_CREATE);
        VRFY((ret != FAIL), "H5Pcreate() large_ds_dcpl_id succeeded");

        ret = H5Pset_layout(large_ds_dcpl_id, H5D_CHUNKED);
        VRFY((ret != FAIL), "H5Pset_layout() large_ds_dcpl_id succeeded");

        ret = H5Pset_chunk(large_ds_dcpl_id, tv_ptr->large_rank, tv_ptr->chunk_dims);
        VRFY((ret != FAIL), "H5Pset_chunk() large_ds_dcpl_id succeeded");
    }

    /* create the small dataset */
    tv_ptr->small_dataset =
        H5Dcreate2(tv_ptr->fid, "small_dataset", tv_ptr->dset_type, tv_ptr->file_small_ds_sid_0, H5P_DEFAULT,
                   small_ds_dcpl_id, H5P_DEFAULT);
    VRFY((ret != FAIL), "H5Dcreate2() small_dataset succeeded");

    /* create the large dataset */
    tv_ptr->large_dataset =
        H5Dcreate2(tv_ptr->fid, "large_dataset", tv_ptr->dset_type, tv_ptr->file_large_ds_sid_0, H5P_DEFAULT,
                   large_ds_dcpl_id, H5P_DEFAULT);
    VRFY((ret != FAIL), "H5Dcreate2() large_dataset succeeded");

    /* setup xfer property list */
    tv_ptr->xfer_plist = H5Pcreate(H5P_DATASET_XFER);
    VRFY((tv_ptr->xfer_plist >= 0), "H5Pcreate(H5P_DATASET_XFER) succeeded");

    if (use_collective_io) {
        ret = H5Pset_dxpl_mpio(tv_ptr->xfer_plist, H5FD_MPIO_COLLECTIVE);
        VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded");
    }

    /* setup selection to write initial data to the small and large data sets */
    tv_ptr->start[0]  = (hsize_t)(tv_ptr->mpi_rank);
    tv_ptr->stride[0] = (hsize_t)(2 * (tv_ptr->mpi_size + 1));
    tv_ptr->count[0]  = 1;
    tv_ptr->block[0]  = 1;

    for (i = 1; i < tv_ptr->large_rank; i++) {

        tv_ptr->start[i]  = 0;
        tv_ptr->stride[i] = (hsize_t)(2 * tv_ptr->edge_size);
        tv_ptr->count[i]  = 1;
        tv_ptr->block[i]  = (hsize_t)(tv_ptr->edge_size);
    }

    /* setup selections for writing initial data to the small data set */
    ret = H5Sselect_hyperslab(tv_ptr->mem_small_ds_sid, H5S_SELECT_SET, tv_ptr->start, tv_ptr->stride,
                              tv_ptr->count, tv_ptr->block);
    VRFY((ret >= 0), "H5Sselect_hyperslab(mem_small_ds_sid, set) succeeded");

    ret = H5Sselect_hyperslab(tv_ptr->file_small_ds_sid_0, H5S_SELECT_SET, tv_ptr->start, tv_ptr->stride,
                              tv_ptr->count, tv_ptr->block);
    VRFY((ret >= 0), "H5Sselect_hyperslab(file_small_ds_sid_0, set) succeeded");

    if (MAINPROCESS) { /* add an additional slice to the selections */

        tv_ptr->start[0] = (hsize_t)(tv_ptr->mpi_size);

        ret = H5Sselect_hyperslab(tv_ptr->mem_small_ds_sid, H5S_SELECT_OR, tv_ptr->start, tv_ptr->stride,
                                  tv_ptr->count, tv_ptr->block);
        VRFY((ret >= 0), "H5Sselect_hyperslab(mem_small_ds_sid, or) succeeded");

        ret = H5Sselect_hyperslab(tv_ptr->file_small_ds_sid_0, H5S_SELECT_OR, tv_ptr->start, tv_ptr->stride,
                                  tv_ptr->count, tv_ptr->block);
        VRFY((ret >= 0), "H5Sselect_hyperslab(file_small_ds_sid_0, or) succeeded");
    }

    /* write the initial value of the small data set to file */
    ret = H5Dwrite(tv_ptr->small_dataset, tv_ptr->dset_type, tv_ptr->mem_small_ds_sid,
                   tv_ptr->file_small_ds_sid_0, tv_ptr->xfer_plist, tv_ptr->small_ds_buf_0);

    VRFY((ret >= 0), "H5Dwrite() small_dataset initial write succeeded");

    /* sync with the other processes before checking data */
    mrc = MPI_Barrier(MPI_COMM_WORLD);
    VRFY((mrc == MPI_SUCCESS), "Sync after small dataset writes");

    /* read the small data set back to verify that it contains the
     * expected data.  Note that each process reads in the entire
     * data set and verifies it.
     */
    ret = H5Dread(tv_ptr->small_dataset, H5T_NATIVE_UINT32, tv_ptr->full_mem_small_ds_sid,
                  tv_ptr->full_file_small_ds_sid, tv_ptr->xfer_plist, tv_ptr->small_ds_buf_1);
    VRFY((ret >= 0), "H5Dread() small_dataset initial read succeeded");

    /* verify that the correct data was written to the small data set */
    expected_value = 0;
    mis_match      = false;
    ptr_1          = tv_ptr->small_ds_buf_1;

    i = 0;
    for (i = 0; i < (int)(tv_ptr->small_ds_size); i++) {

        if (*ptr_1 != expected_value) {

            mis_match = true;
        }
        ptr_1++;
        expected_value++;
    }
    VRFY((mis_match == false), "small ds init data good.");

    /* setup selections for writing initial data to the large data set */

    tv_ptr->start[0] = (hsize_t)(tv_ptr->mpi_rank);

    ret = H5Sselect_hyperslab(tv_ptr->mem_large_ds_sid, H5S_SELECT_SET, tv_ptr->start, tv_ptr->stride,
                              tv_ptr->count, tv_ptr->block);
    VRFY((ret >= 0), "H5Sselect_hyperslab(mem_large_ds_sid, set) succeeded");

    ret = H5Sselect_hyperslab(tv_ptr->file_large_ds_sid_0, H5S_SELECT_SET, tv_ptr->start, tv_ptr->stride,
                              tv_ptr->count, tv_ptr->block);
    VRFY((ret >= 0), "H5Sselect_hyperslab(file_large_ds_sid_0, set) succeeded");

    /* In passing, setup the process slice dataspaces as well */

    ret = H5Sselect_hyperslab(tv_ptr->mem_large_ds_process_slice_sid, H5S_SELECT_SET, tv_ptr->start,
                              tv_ptr->stride, tv_ptr->count, tv_ptr->block);
    VRFY((ret >= 0), "H5Sselect_hyperslab(mem_large_ds_process_slice_sid, set) succeeded");

    ret = H5Sselect_hyperslab(tv_ptr->file_large_ds_process_slice_sid, H5S_SELECT_SET, tv_ptr->start,
                              tv_ptr->stride, tv_ptr->count, tv_ptr->block);
    VRFY((ret >= 0), "H5Sselect_hyperslab(file_large_ds_process_slice_sid, set) succeeded");

    if (MAINPROCESS) { /* add an additional slice to the selections */

        tv_ptr->start[0] = (hsize_t)(tv_ptr->mpi_size);

        ret = H5Sselect_hyperslab(tv_ptr->mem_large_ds_sid, H5S_SELECT_OR, tv_ptr->start, tv_ptr->stride,
                                  tv_ptr->count, tv_ptr->block);
        VRFY((ret >= 0), "H5Sselect_hyperslab(mem_large_ds_sid, or) succeeded");

        ret = H5Sselect_hyperslab(tv_ptr->file_large_ds_sid_0, H5S_SELECT_OR, tv_ptr->start, tv_ptr->stride,
                                  tv_ptr->count, tv_ptr->block);
        VRFY((ret >= 0), "H5Sselect_hyperslab(file_large_ds_sid_0, or) succeeded");
    }

    /* write the initial value of the large data set to file */
    ret = H5Dwrite(tv_ptr->large_dataset, tv_ptr->dset_type, tv_ptr->mem_large_ds_sid,
                   tv_ptr->file_large_ds_sid_0, tv_ptr->xfer_plist, tv_ptr->large_ds_buf_0);
    if (ret < 0)
        H5Eprint2(H5E_DEFAULT, stderr);
    VRFY((ret >= 0), "H5Dwrite() large_dataset initial write succeeded");

    /* sync with the other processes before checking data */
    mrc = MPI_Barrier(MPI_COMM_WORLD);
    VRFY((mrc == MPI_SUCCESS), "Sync after large dataset writes");

    /* read the large data set back to verify that it contains the
     * expected data.  Note that each process reads in the entire
     * data set.
     */
    ret = H5Dread(tv_ptr->large_dataset, H5T_NATIVE_UINT32, tv_ptr->full_mem_large_ds_sid,
                  tv_ptr->full_file_large_ds_sid, tv_ptr->xfer_plist, tv_ptr->large_ds_buf_1);
    VRFY((ret >= 0), "H5Dread() large_dataset initial read succeeded");

    /* verify that the correct data was written to the large data set */
    expected_value = 0;
    mis_match      = false;
    ptr_1          = tv_ptr->large_ds_buf_1;

    i = 0;
    for (i = 0; i < (int)(tv_ptr->large_ds_size); i++) {

        if (*ptr_1 != expected_value) {

            mis_match = true;
        }
        ptr_1++;
        expected_value++;
    }
    VRFY((mis_match == false), "large ds init data good.");

    /* sync with the other processes before changing data */
    mrc = MPI_Barrier(MPI_COMM_WORLD);
    VRFY((mrc == MPI_SUCCESS), "Sync initial values check");

    return;

} /* hs_dr_pio_test__setup() */

/*-------------------------------------------------------------------------
 * Function:    hs_dr_pio_test__takedown()
 *
 * Purpose:    Do takedown after tests of I/O to/from hyperslab selections
 *        of different rank in the parallel case.
 *
 * Return:    void
 *
 *-------------------------------------------------------------------------
 */

#define HS_DR_PIO_TEST__TAKEDOWN__DEBUG 0

static void
hs_dr_pio_test__takedown(struct hs_dr_pio_test_vars_t *tv_ptr)
{
#if HS_DR_PIO_TEST__TAKEDOWN__DEBUG
    const char *fcnName = "hs_dr_pio_test__takedown()";
#endif               /* HS_DR_PIO_TEST__TAKEDOWN__DEBUG */
    int    mpi_rank; /* needed by the VRFY macro */
    herr_t ret;      /* Generic return value */

    /* initialize the local copy of mpi_rank */
    mpi_rank = tv_ptr->mpi_rank;

    /* Close property lists */
    if (tv_ptr->xfer_plist != H5P_DEFAULT) {
        ret = H5Pclose(tv_ptr->xfer_plist);
        VRFY((ret != FAIL), "H5Pclose(xfer_plist) succeeded");
    }

    /* Close dataspaces */
    ret = H5Sclose(tv_ptr->full_mem_small_ds_sid);
    VRFY((ret != FAIL), "H5Sclose(full_mem_small_ds_sid) succeeded");

    ret = H5Sclose(tv_ptr->full_file_small_ds_sid);
    VRFY((ret != FAIL), "H5Sclose(full_file_small_ds_sid) succeeded");

    ret = H5Sclose(tv_ptr->mem_small_ds_sid);
    VRFY((ret != FAIL), "H5Sclose(mem_small_ds_sid) succeeded");

    ret = H5Sclose(tv_ptr->file_small_ds_sid_0);
    VRFY((ret != FAIL), "H5Sclose(file_small_ds_sid_0) succeeded");

    ret = H5Sclose(tv_ptr->file_small_ds_sid_1);
    VRFY((ret != FAIL), "H5Sclose(file_small_ds_sid_1) succeeded");

    ret = H5Sclose(tv_ptr->small_ds_slice_sid);
    VRFY((ret != FAIL), "H5Sclose(small_ds_slice_sid) succeeded");

    ret = H5Sclose(tv_ptr->full_mem_large_ds_sid);
    VRFY((ret != FAIL), "H5Sclose(full_mem_large_ds_sid) succeeded");

    ret = H5Sclose(tv_ptr->full_file_large_ds_sid);
    VRFY((ret != FAIL), "H5Sclose(full_file_large_ds_sid) succeeded");

    ret = H5Sclose(tv_ptr->mem_large_ds_sid);
    VRFY((ret != FAIL), "H5Sclose(mem_large_ds_sid) succeeded");

    ret = H5Sclose(tv_ptr->file_large_ds_sid_0);
    VRFY((ret != FAIL), "H5Sclose(file_large_ds_sid_0) succeeded");

    ret = H5Sclose(tv_ptr->file_large_ds_sid_1);
    VRFY((ret != FAIL), "H5Sclose(file_large_ds_sid_1) succeeded");

    ret = H5Sclose(tv_ptr->mem_large_ds_process_slice_sid);
    VRFY((ret != FAIL), "H5Sclose(mem_large_ds_process_slice_sid) succeeded");

    ret = H5Sclose(tv_ptr->file_large_ds_process_slice_sid);
    VRFY((ret != FAIL), "H5Sclose(file_large_ds_process_slice_sid) succeeded");

    ret = H5Sclose(tv_ptr->large_ds_slice_sid);
    VRFY((ret != FAIL), "H5Sclose(large_ds_slice_sid) succeeded");

    /* Close Datasets */
    ret = H5Dclose(tv_ptr->small_dataset);
    VRFY((ret != FAIL), "H5Dclose(small_dataset) succeeded");

    ret = H5Dclose(tv_ptr->large_dataset);
    VRFY((ret != FAIL), "H5Dclose(large_dataset) succeeded");

    /* close the file collectively */
    MESG("about to close file.");
    ret = H5Fclose(tv_ptr->fid);
    VRFY((ret != FAIL), "file close succeeded");

    /* Free memory buffers */

    if (tv_ptr->small_ds_buf_0 != NULL)
        free(tv_ptr->small_ds_buf_0);
    if (tv_ptr->small_ds_buf_1 != NULL)
        free(tv_ptr->small_ds_buf_1);
    if (tv_ptr->small_ds_buf_2 != NULL)
        free(tv_ptr->small_ds_buf_2);
    if (tv_ptr->small_ds_slice_buf != NULL)
        free(tv_ptr->small_ds_slice_buf);

    if (tv_ptr->large_ds_buf_0 != NULL)
        free(tv_ptr->large_ds_buf_0);
    if (tv_ptr->large_ds_buf_1 != NULL)
        free(tv_ptr->large_ds_buf_1);
    if (tv_ptr->large_ds_buf_2 != NULL)
        free(tv_ptr->large_ds_buf_2);
    if (tv_ptr->large_ds_slice_buf != NULL)
        free(tv_ptr->large_ds_slice_buf);

    return;

} /* hs_dr_pio_test__takedown() */

/*-------------------------------------------------------------------------
 * Function:    contig_hs_dr_pio_test__d2m_l2s()
 *
 * Purpose:    Part one of a series of tests of I/O to/from hyperslab
 *        selections of different rank in the parallel.
 *
 *        Verify that we can read from disk correctly using
 *        selections of different rank that H5Sselect_shape_same()
 *        views as being of the same shape.
 *
 *              In this function, we test this by reading small_rank - 1
 *        slices from the on disk large cube, and verifying that the
 *        data read is correct.  Verify that H5Sselect_shape_same()
 *        returns true on the memory and file selections.
 *
 * Return:    void
 *
 *-------------------------------------------------------------------------
 */

#define CONTIG_HS_DR_PIO_TEST__D2M_L2S__DEBUG 0

static void
contig_hs_dr_pio_test__d2m_l2s(struct hs_dr_pio_test_vars_t *tv_ptr)
{
#if CONTIG_HS_DR_PIO_TEST__D2M_L2S__DEBUG
    const char *fcnName = "contig_hs_dr_pio_test__run_test()";
#endif /* CONTIG_HS_DR_PIO_TEST__D2M_L2S__DEBUG */
    bool      mis_match = false;
    int       i, j, k, l;
    size_t    n;
    int       mpi_rank; /* needed by the VRFY macro */
    uint32_t  expected_value;
    uint32_t *ptr_1;
    htri_t    check; /* Shape comparison return value */
    herr_t    ret;   /* Generic return value */

    /* initialize the local copy of mpi_rank */
    mpi_rank = tv_ptr->mpi_rank;

    /* We have already done a H5Sselect_all() on the dataspace
     * small_ds_slice_sid in the initialization phase, so no need to
     * call H5Sselect_all() again.
     */

    /* set up start, stride, count, and block -- note that we will
     * change start[] so as to read slices of the large cube.
     */
    for (i = 0; i < PAR_SS_DR_MAX_RANK; i++) {

        tv_ptr->start[i]  = 0;
        tv_ptr->stride[i] = (hsize_t)(2 * tv_ptr->edge_size);
        tv_ptr->count[i]  = 1;
        if ((PAR_SS_DR_MAX_RANK - i) > (tv_ptr->small_rank - 1)) {

            tv_ptr->block[i] = 1;
        }
        else {

            tv_ptr->block[i] = (hsize_t)(tv_ptr->edge_size);
        }
    }

    /* zero out the buffer we will be reading into */
    memset(tv_ptr->small_ds_slice_buf, 0, sizeof(uint32_t) * tv_ptr->small_ds_slice_size);

#if CONTIG_HS_DR_PIO_TEST__D2M_L2S__DEBUG
    fprintf(stdout, "%s reading slices from big cube on disk into small cube slice.\n", fcnName);
#endif /* CONTIG_HS_DR_PIO_TEST__D2M_L2S__DEBUG */

    /* in serial versions of this test, we loop through all the dimensions
     * of the large data set.  However, in the parallel version, each
     * process only works with that slice of the large cube indicated
     * by its rank -- hence we set the most slowly changing index to
     * mpi_rank, and don't iterate over it.
     */

    if (PAR_SS_DR_MAX_RANK - tv_ptr->large_rank == 0) {

        i = tv_ptr->mpi_rank;
    }
    else {

        i = 0;
    }

    /* since large_rank is at most PAR_SS_DR_MAX_RANK, no need to
     * loop over it -- either we are setting i to mpi_rank, or
     * we are setting it to zero.  It will not change during the
     * test.
     */

    if (PAR_SS_DR_MAX_RANK - tv_ptr->large_rank == 1) {

        j = tv_ptr->mpi_rank;
    }
    else {

        j = 0;
    }

    do {
        if (PAR_SS_DR_MAX_RANK - tv_ptr->large_rank == 2) {

            k = tv_ptr->mpi_rank;
        }
        else {

            k = 0;
        }

        do {
            /* since small rank >= 2 and large_rank > small_rank, we
             * have large_rank >= 3.  Since PAR_SS_DR_MAX_RANK == 5
             * (baring major re-orgaization), this gives us:
             *
             *     (PAR_SS_DR_MAX_RANK - large_rank) <= 2
             *
             * so no need to repeat the test in the outer loops --
             * just set l = 0.
             */

            l = 0;
            do {
                if ((tv_ptr->skips)++ < tv_ptr->max_skips) { /* skip the test */

                    (tv_ptr->tests_skipped)++;
                }
                else { /* run the test */

                    tv_ptr->skips = 0; /* reset the skips counter */

                    /* we know that small_rank - 1 >= 1 and that
                     * large_rank > small_rank by the assertions at the head
                     * of this function.  Thus no need for another inner loop.
                     */
                    tv_ptr->start[0] = (hsize_t)i;
                    tv_ptr->start[1] = (hsize_t)j;
                    tv_ptr->start[2] = (hsize_t)k;
                    tv_ptr->start[3] = (hsize_t)l;
                    tv_ptr->start[4] = 0;

                    ret = H5Sselect_hyperslab(tv_ptr->file_large_ds_sid_0, H5S_SELECT_SET, tv_ptr->start_ptr,
                                              tv_ptr->stride_ptr, tv_ptr->count_ptr, tv_ptr->block_ptr);
                    VRFY((ret != FAIL), "H5Sselect_hyperslab(file_large_cube_sid) succeeded");

                    /* verify that H5Sselect_shape_same() reports the two
                     * selections as having the same shape.
                     */
                    check = H5Sselect_shape_same(tv_ptr->small_ds_slice_sid, tv_ptr->file_large_ds_sid_0);
                    VRFY((check == true), "H5Sselect_shape_same passed");

                    /* Read selection from disk */
#if CONTIG_HS_DR_PIO_TEST__D2M_L2S__DEBUG
                    fprintf(stdout, "%s:%d: start = %d %d %d %d %d.\n", fcnName, (int)(tv_ptr->mpi_rank),
                            (int)(tv_ptr->start[0]), (int)(tv_ptr->start[1]), (int)(tv_ptr->start[2]),
                            (int)(tv_ptr->start[3]), (int)(tv_ptr->start[4]));
                    fprintf(stdout, "%s slice/file extent dims = %d/%d.\n", fcnName,
                            H5Sget_simple_extent_ndims(tv_ptr->small_ds_slice_sid),
                            H5Sget_simple_extent_ndims(tv_ptr->file_large_ds_sid_0));
#endif /* CONTIG_HS_DR_PIO_TEST__D2M_L2S__DEBUG */
                    ret =
                        H5Dread(tv_ptr->large_dataset, H5T_NATIVE_UINT32, tv_ptr->small_ds_slice_sid,
                                tv_ptr->file_large_ds_sid_0, tv_ptr->xfer_plist, tv_ptr->small_ds_slice_buf);
                    VRFY((ret >= 0), "H5Dread() slice from large ds succeeded.");

                    /* verify that expected data is retrieved */

                    mis_match = false;
                    ptr_1     = tv_ptr->small_ds_slice_buf;
                    expected_value =
                        (uint32_t)((i * tv_ptr->edge_size * tv_ptr->edge_size * tv_ptr->edge_size *
                                    tv_ptr->edge_size) +
                                   (j * tv_ptr->edge_size * tv_ptr->edge_size * tv_ptr->edge_size) +
                                   (k * tv_ptr->edge_size * tv_ptr->edge_size) + (l * tv_ptr->edge_size));

                    for (n = 0; n < tv_ptr->small_ds_slice_size; n++) {

                        if (*ptr_1 != expected_value) {

                            mis_match = true;
                        }

                        *ptr_1 = 0; /* zero data for next use */

                        ptr_1++;
                        expected_value++;
                    }

                    VRFY((mis_match == false), "small slice read from large ds data good.");

                    (tv_ptr->tests_run)++;
                }

                l++;

                (tv_ptr->total_tests)++;

            } while ((tv_ptr->large_rank > 2) && ((tv_ptr->small_rank - 1) <= 1) && (l < tv_ptr->edge_size));
            k++;
        } while ((tv_ptr->large_rank > 3) && ((tv_ptr->small_rank - 1) <= 2) && (k < tv_ptr->edge_size));
        j++;
    } while ((tv_ptr->large_rank > 4) && ((tv_ptr->small_rank - 1) <= 3) && (j < tv_ptr->edge_size));

    return;

} /* contig_hs_dr_pio_test__d2m_l2s() */

/*-------------------------------------------------------------------------
 * Function:    contig_hs_dr_pio_test__d2m_s2l()
 *
 * Purpose:    Part two of a series of tests of I/O to/from hyperslab
 *        selections of different rank in the parallel.
 *
 *        Verify that we can read from disk correctly using
 *        selections of different rank that H5Sselect_shape_same()
 *        views as being of the same shape.
 *
 *        In this function, we test this by reading slices of the
 *        on disk small data set into slices through the in memory
 *        large data set, and verify that the correct data (and
 *        only the correct data) is read.
 *
 * Return:    void
 *
 *-------------------------------------------------------------------------
 */

#define CONTIG_HS_DR_PIO_TEST__D2M_S2L__DEBUG 0

static void
contig_hs_dr_pio_test__d2m_s2l(struct hs_dr_pio_test_vars_t *tv_ptr)
{
#if CONTIG_HS_DR_PIO_TEST__D2M_S2L__DEBUG
    const char *fcnName = "contig_hs_dr_pio_test__d2m_s2l()";
#endif /* CONTIG_HS_DR_PIO_TEST__D2M_S2L__DEBUG */
    bool      mis_match = false;
    int       i, j, k, l;
    size_t    n;
    int       mpi_rank; /* needed by the VRFY macro */
    size_t    start_index;
    size_t    stop_index;
    uint32_t  expected_value;
    uint32_t *ptr_1;
    htri_t    check; /* Shape comparison return value */
    herr_t    ret;   /* Generic return value */

    /* initialize the local copy of mpi_rank */
    mpi_rank = tv_ptr->mpi_rank;

    /* Read slices of the on disk small data set into slices
     * through the in memory large data set, and verify that the correct
     * data (and only the correct data) is read.
     */

    tv_ptr->start[0]  = (hsize_t)(tv_ptr->mpi_rank);
    tv_ptr->stride[0] = (hsize_t)(2 * (tv_ptr->mpi_size + 1));
    tv_ptr->count[0]  = 1;
    tv_ptr->block[0]  = 1;

    for (i = 1; i < tv_ptr->large_rank; i++) {

        tv_ptr->start[i]  = 0;
        tv_ptr->stride[i] = (hsize_t)(2 * tv_ptr->edge_size);
        tv_ptr->count[i]  = 1;
        tv_ptr->block[i]  = (hsize_t)(tv_ptr->edge_size);
    }

    ret = H5Sselect_hyperslab(tv_ptr->file_small_ds_sid_0, H5S_SELECT_SET, tv_ptr->start, tv_ptr->stride,
                              tv_ptr->count, tv_ptr->block);
    VRFY((ret >= 0), "H5Sselect_hyperslab(file_small_ds_sid_0, set) succeeded");

#if CONTIG_HS_DR_PIO_TEST__D2M_S2L__DEBUG
    fprintf(stdout, "%s reading slices of on disk small data set into slices of big data set.\n", fcnName);
#endif /* CONTIG_HS_DR_PIO_TEST__D2M_S2L__DEBUG */

    /* zero out the in memory large ds */
    memset(tv_ptr->large_ds_buf_1, 0, sizeof(uint32_t) * tv_ptr->large_ds_size);

    /* set up start, stride, count, and block -- note that we will
     * change start[] so as to read slices of the large cube.
     */
    for (i = 0; i < PAR_SS_DR_MAX_RANK; i++) {

        tv_ptr->start[i]  = 0;
        tv_ptr->stride[i] = (hsize_t)(2 * tv_ptr->edge_size);
        tv_ptr->count[i]  = 1;
        if ((PAR_SS_DR_MAX_RANK - i) > (tv_ptr->small_rank - 1)) {

            tv_ptr->block[i] = 1;
        }
        else {

            tv_ptr->block[i] = (hsize_t)(tv_ptr->edge_size);
        }
    }

    /* in serial versions of this test, we loop through all the dimensions
     * of the large data set that don't appear in the small data set.
     *
     * However, in the parallel version, each process only works with that
     * slice of the large (and small) data set indicated by its rank -- hence
     * we set the most slowly changing index to mpi_rank, and don't iterate
     * over it.
     */

    if (PAR_SS_DR_MAX_RANK - tv_ptr->large_rank == 0) {

        i = tv_ptr->mpi_rank;
    }
    else {

        i = 0;
    }

    /* since large_rank is at most PAR_SS_DR_MAX_RANK, no need to
     * loop over it -- either we are setting i to mpi_rank, or
     * we are setting it to zero.  It will not change during the
     * test.
     */

    if (PAR_SS_DR_MAX_RANK - tv_ptr->large_rank == 1) {

        j = tv_ptr->mpi_rank;
    }
    else {

        j = 0;
    }

    do {
        if (PAR_SS_DR_MAX_RANK - tv_ptr->large_rank == 2) {

            k = tv_ptr->mpi_rank;
        }
        else {

            k = 0;
        }

        do {
            /* since small rank >= 2 and large_rank > small_rank, we
             * have large_rank >= 3.  Since PAR_SS_DR_MAX_RANK == 5
             * (baring major re-orgaization), this gives us:
             *
             *     (PAR_SS_DR_MAX_RANK - large_rank) <= 2
             *
             * so no need to repeat the test in the outer loops --
             * just set l = 0.
             */

            l = 0;
            do {
                if ((tv_ptr->skips)++ < tv_ptr->max_skips) { /* skip the test */

                    (tv_ptr->tests_skipped)++;
                }
                else { /* run the test */

                    tv_ptr->skips = 0; /* reset the skips counter */

                    /* we know that small_rank >= 1 and that large_rank > small_rank
                     * by the assertions at the head of this function.  Thus no
                     * need for another inner loop.
                     */
                    tv_ptr->start[0] = (hsize_t)i;
                    tv_ptr->start[1] = (hsize_t)j;
                    tv_ptr->start[2] = (hsize_t)k;
                    tv_ptr->start[3] = (hsize_t)l;
                    tv_ptr->start[4] = 0;

                    ret = H5Sselect_hyperslab(tv_ptr->mem_large_ds_sid, H5S_SELECT_SET, tv_ptr->start_ptr,
                                              tv_ptr->stride_ptr, tv_ptr->count_ptr, tv_ptr->block_ptr);
                    VRFY((ret != FAIL), "H5Sselect_hyperslab(mem_large_ds_sid) succeeded");

                    /* verify that H5Sselect_shape_same() reports the two
                     * selections as having the same shape.
                     */
                    check = H5Sselect_shape_same(tv_ptr->file_small_ds_sid_0, tv_ptr->mem_large_ds_sid);
                    VRFY((check == true), "H5Sselect_shape_same passed");

                    /* Read selection from disk */
#if CONTIG_HS_DR_PIO_TEST__D2M_S2L__DEBUG
                    fprintf(stdout, "%s:%d: start = %d %d %d %d %d.\n", fcnName, (int)(tv_ptr->mpi_rank),
                            (int)(tv_ptr->start[0]), (int)(tv_ptr->start[1]), (int)(tv_ptr->start[2]),
                            (int)(tv_ptr->start[3]), (int)(tv_ptr->start[4]));
                    fprintf(stdout, "%s:%d: mem/file extent dims = %d/%d.\n", fcnName, tv_ptr->mpi_rank,
                            H5Sget_simple_extent_ndims(tv_ptr->mem_large_ds_sid),
                            H5Sget_simple_extent_ndims(tv_ptr->file_small_ds_sid_0));
#endif /* CONTIG_HS_DR_PIO_TEST__D2M_S2L__DEBUG */
                    ret = H5Dread(tv_ptr->small_dataset, H5T_NATIVE_UINT32, tv_ptr->mem_large_ds_sid,
                                  tv_ptr->file_small_ds_sid_0, tv_ptr->xfer_plist, tv_ptr->large_ds_buf_1);
                    VRFY((ret >= 0), "H5Dread() slice from small ds succeeded.");

                    /* verify that the expected data and only the
                     * expected data was read.
                     */
                    ptr_1          = tv_ptr->large_ds_buf_1;
                    expected_value = (uint32_t)((size_t)(tv_ptr->mpi_rank) * tv_ptr->small_ds_slice_size);
                    start_index =
                        (size_t)((i * tv_ptr->edge_size * tv_ptr->edge_size * tv_ptr->edge_size *
                                  tv_ptr->edge_size) +
                                 (j * tv_ptr->edge_size * tv_ptr->edge_size * tv_ptr->edge_size) +
                                 (k * tv_ptr->edge_size * tv_ptr->edge_size) + (l * tv_ptr->edge_size));
                    stop_index = start_index + tv_ptr->small_ds_slice_size - 1;

                    assert(start_index < stop_index);
                    assert(stop_index <= tv_ptr->large_ds_size);

                    for (n = 0; n < tv_ptr->large_ds_size; n++) {

                        if ((n >= start_index) && (n <= stop_index)) {

                            if (*ptr_1 != expected_value) {

                                mis_match = true;
                            }
                            expected_value++;
                        }
                        else {

                            if (*ptr_1 != 0) {

                                mis_match = true;
                            }
                        }
                        /* zero out the value for the next pass */
                        *ptr_1 = 0;

                        ptr_1++;
                    }

                    VRFY((mis_match == false), "small slice read from large ds data good.");

                    (tv_ptr->tests_run)++;
                }

                l++;

                (tv_ptr->total_tests)++;

            } while ((tv_ptr->large_rank > 2) && ((tv_ptr->small_rank - 1) <= 1) && (l < tv_ptr->edge_size));
            k++;
        } while ((tv_ptr->large_rank > 3) && ((tv_ptr->small_rank - 1) <= 2) && (k < tv_ptr->edge_size));
        j++;
    } while ((tv_ptr->large_rank > 4) && ((tv_ptr->small_rank - 1) <= 3) && (j < tv_ptr->edge_size));

    return;

} /* contig_hs_dr_pio_test__d2m_s2l() */

/*-------------------------------------------------------------------------
 * Function:    contig_hs_dr_pio_test__m2d_l2s()
 *
 * Purpose:    Part three of a series of tests of I/O to/from hyperslab
 *        selections of different rank in the parallel.
 *
 *        Verify that we can write from memory to file using
 *        selections of different rank that H5Sselect_shape_same()
 *        views as being of the same shape.
 *
 *        Do this by writing small_rank - 1 dimensional slices from
 *        the in memory large data set to the on disk small cube
 *        dataset.  After each write, read the slice of the small
 *        dataset back from disk, and verify that it contains
 *        the expected data. Verify that H5Sselect_shape_same()
 *        returns true on the memory and file selections.
 *
 * Return:    void
 *
 *-------------------------------------------------------------------------
 */

#define CONTIG_HS_DR_PIO_TEST__M2D_L2S__DEBUG 0

static void
contig_hs_dr_pio_test__m2d_l2s(struct hs_dr_pio_test_vars_t *tv_ptr)
{
#if CONTIG_HS_DR_PIO_TEST__M2D_L2S__DEBUG
    const char *fcnName = "contig_hs_dr_pio_test__m2d_l2s()";
#endif /* CONTIG_HS_DR_PIO_TEST__M2D_L2S__DEBUG */
    bool      mis_match = false;
    int       i, j, k, l;
    size_t    n;
    int       mpi_rank; /* needed by the VRFY macro */
    size_t    start_index;
    size_t    stop_index;
    uint32_t  expected_value;
    uint32_t *ptr_1;
    htri_t    check; /* Shape comparison return value */
    herr_t    ret;   /* Generic return value */

    /* initialize the local copy of mpi_rank */
    mpi_rank = tv_ptr->mpi_rank;

    /* now we go in the opposite direction, verifying that we can write
     * from memory to file using selections of different rank that
     * H5Sselect_shape_same() views as being of the same shape.
     *
     * Start by writing small_rank - 1 dimensional slices from the in memory large
     * data set to the on disk small cube dataset.  After each write, read the
     * slice of the small dataset back from disk, and verify that it contains
     * the expected data. Verify that H5Sselect_shape_same() returns true on
     * the memory and file selections.
     */

    tv_ptr->start[0]  = (hsize_t)(tv_ptr->mpi_rank);
    tv_ptr->stride[0] = (hsize_t)(2 * (tv_ptr->mpi_size + 1));
    tv_ptr->count[0]  = 1;
    tv_ptr->block[0]  = 1;

    for (i = 1; i < tv_ptr->large_rank; i++) {

        tv_ptr->start[i]  = 0;
        tv_ptr->stride[i] = (hsize_t)(2 * tv_ptr->edge_size);
        tv_ptr->count[i]  = 1;
        tv_ptr->block[i]  = (hsize_t)(tv_ptr->edge_size);
    }

    ret = H5Sselect_hyperslab(tv_ptr->file_small_ds_sid_0, H5S_SELECT_SET, tv_ptr->start, tv_ptr->stride,
                              tv_ptr->count, tv_ptr->block);
    VRFY((ret >= 0), "H5Sselect_hyperslab(file_small_ds_sid_0, set) succeeded");

    ret = H5Sselect_hyperslab(tv_ptr->mem_small_ds_sid, H5S_SELECT_SET, tv_ptr->start, tv_ptr->stride,
                              tv_ptr->count, tv_ptr->block);
    VRFY((ret >= 0), "H5Sselect_hyperslab(mem_small_ds_sid, set) succeeded");

    /* set up start, stride, count, and block -- note that we will
     * change start[] so as to read slices of the large cube.
     */
    for (i = 0; i < PAR_SS_DR_MAX_RANK; i++) {

        tv_ptr->start[i]  = 0;
        tv_ptr->stride[i] = (hsize_t)(2 * tv_ptr->edge_size);
        tv_ptr->count[i]  = 1;
        if ((PAR_SS_DR_MAX_RANK - i) > (tv_ptr->small_rank - 1)) {

            tv_ptr->block[i] = 1;
        }
        else {

            tv_ptr->block[i] = (hsize_t)(tv_ptr->edge_size);
        }
    }

    /* zero out the in memory small ds */
    memset(tv_ptr->small_ds_buf_1, 0, sizeof(uint32_t) * tv_ptr->small_ds_size);

#if CONTIG_HS_DR_PIO_TEST__M2D_L2S__DEBUG
    fprintf(stdout, "%s writing slices from big ds to slices of small ds on disk.\n", fcnName);
#endif /* CONTIG_HS_DR_PIO_TEST__M2D_L2S__DEBUG */

    /* in serial versions of this test, we loop through all the dimensions
     * of the large data set that don't appear in the small data set.
     *
     * However, in the parallel version, each process only works with that
     * slice of the large (and small) data set indicated by its rank -- hence
     * we set the most slowly changing index to mpi_rank, and don't iterate
     * over it.
     */

    if (PAR_SS_DR_MAX_RANK - tv_ptr->large_rank == 0) {

        i = tv_ptr->mpi_rank;
    }
    else {

        i = 0;
    }

    /* since large_rank is at most PAR_SS_DR_MAX_RANK, no need to
     * loop over it -- either we are setting i to mpi_rank, or
     * we are setting it to zero.  It will not change during the
     * test.
     */

    if (PAR_SS_DR_MAX_RANK - tv_ptr->large_rank == 1) {

        j = tv_ptr->mpi_rank;
    }
    else {

        j = 0;
    }

    j = 0;
    do {
        if (PAR_SS_DR_MAX_RANK - tv_ptr->large_rank == 2) {

            k = tv_ptr->mpi_rank;
        }
        else {

            k = 0;
        }

        do {
            /* since small rank >= 2 and large_rank > small_rank, we
             * have large_rank >= 3.  Since PAR_SS_DR_MAX_RANK == 5
             * (baring major re-orgaization), this gives us:
             *
             *     (PAR_SS_DR_MAX_RANK - large_rank) <= 2
             *
             * so no need to repeat the test in the outer loops --
             * just set l = 0.
             */

            l = 0;
            do {
                if ((tv_ptr->skips)++ < tv_ptr->max_skips) { /* skip the test */

                    (tv_ptr->tests_skipped)++;
                }
                else { /* run the test */

                    tv_ptr->skips = 0; /* reset the skips counter */

                    /* we know that small_rank >= 1 and that large_rank > small_rank
                     * by the assertions at the head of this function.  Thus no
                     * need for another inner loop.
                     */

                    /* zero out this rank's slice of the on disk small data set */
                    ret = H5Dwrite(tv_ptr->small_dataset, H5T_NATIVE_UINT32, tv_ptr->mem_small_ds_sid,
                                   tv_ptr->file_small_ds_sid_0, tv_ptr->xfer_plist, tv_ptr->small_ds_buf_2);
                    VRFY((ret >= 0), "H5Dwrite() zero slice to small ds succeeded.");

                    /* select the portion of the in memory large cube from which we
                     * are going to write data.
                     */
                    tv_ptr->start[0] = (hsize_t)i;
                    tv_ptr->start[1] = (hsize_t)j;
                    tv_ptr->start[2] = (hsize_t)k;
                    tv_ptr->start[3] = (hsize_t)l;
                    tv_ptr->start[4] = 0;

                    ret = H5Sselect_hyperslab(tv_ptr->mem_large_ds_sid, H5S_SELECT_SET, tv_ptr->start_ptr,
                                              tv_ptr->stride_ptr, tv_ptr->count_ptr, tv_ptr->block_ptr);
                    VRFY((ret >= 0), "H5Sselect_hyperslab() mem_large_ds_sid succeeded.");

                    /* verify that H5Sselect_shape_same() reports the in
                     * memory slice through the cube selection and the
                     * on disk full square selections as having the same shape.
                     */
                    check = H5Sselect_shape_same(tv_ptr->file_small_ds_sid_0, tv_ptr->mem_large_ds_sid);
                    VRFY((check == true), "H5Sselect_shape_same passed.");

                    /* write the slice from the in memory large data set to the
                     * slice of the on disk small dataset. */
#if CONTIG_HS_DR_PIO_TEST__M2D_L2S__DEBUG
                    fprintf(stdout, "%s:%d: start = %d %d %d %d %d.\n", fcnName, (int)(tv_ptr->mpi_rank),
                            (int)(tv_ptr->start[0]), (int)(tv_ptr->start[1]), (int)(tv_ptr->start[2]),
                            (int)(tv_ptr->start[3]), (int)(tv_ptr->start[4]));
                    fprintf(stdout, "%s:%d: mem/file extent dims = %d/%d.\n", fcnName, tv_ptr->mpi_rank,
                            H5Sget_simple_extent_ndims(tv_ptr->mem_large_ds_sid),
                            H5Sget_simple_extent_ndims(tv_ptr->file_small_ds_sid_0));
#endif /* CONTIG_HS_DR_PIO_TEST__M2D_L2S__DEBUG */
                    ret = H5Dwrite(tv_ptr->small_dataset, H5T_NATIVE_UINT32, tv_ptr->mem_large_ds_sid,
                                   tv_ptr->file_small_ds_sid_0, tv_ptr->xfer_plist, tv_ptr->large_ds_buf_0);
                    VRFY((ret >= 0), "H5Dwrite() slice to large ds succeeded.");

                    /* read the on disk square into memory */
                    ret = H5Dread(tv_ptr->small_dataset, H5T_NATIVE_UINT32, tv_ptr->mem_small_ds_sid,
                                  tv_ptr->file_small_ds_sid_0, tv_ptr->xfer_plist, tv_ptr->small_ds_buf_1);
                    VRFY((ret >= 0), "H5Dread() slice from small ds succeeded.");

                    /* verify that expected data is retrieved */

                    mis_match = false;
                    ptr_1     = tv_ptr->small_ds_buf_1;

                    expected_value =
                        (uint32_t)((i * tv_ptr->edge_size * tv_ptr->edge_size * tv_ptr->edge_size *
                                    tv_ptr->edge_size) +
                                   (j * tv_ptr->edge_size * tv_ptr->edge_size * tv_ptr->edge_size) +
                                   (k * tv_ptr->edge_size * tv_ptr->edge_size) + (l * tv_ptr->edge_size));

                    start_index = (size_t)(tv_ptr->mpi_rank) * tv_ptr->small_ds_slice_size;
                    stop_index  = start_index + tv_ptr->small_ds_slice_size - 1;

                    assert(start_index < stop_index);
                    assert(stop_index <= tv_ptr->small_ds_size);

                    for (n = 0; n < tv_ptr->small_ds_size; n++) {

                        if ((n >= start_index) && (n <= stop_index)) {

                            if (*ptr_1 != expected_value) {

                                mis_match = true;
                            }
                            expected_value++;
                        }
                        else {

                            if (*ptr_1 != 0) {

                                mis_match = true;
                            }
                        }
                        /* zero out the value for the next pass */
                        *ptr_1 = 0;

                        ptr_1++;
                    }

                    VRFY((mis_match == false), "small slice write from large ds data good.");

                    (tv_ptr->tests_run)++;
                }

                l++;

                (tv_ptr->total_tests)++;

            } while ((tv_ptr->large_rank > 2) && ((tv_ptr->small_rank - 1) <= 1) && (l < tv_ptr->edge_size));
            k++;
        } while ((tv_ptr->large_rank > 3) && ((tv_ptr->small_rank - 1) <= 2) && (k < tv_ptr->edge_size));
        j++;
    } while ((tv_ptr->large_rank > 4) && ((tv_ptr->small_rank - 1) <= 3) && (j < tv_ptr->edge_size));

    return;

} /* contig_hs_dr_pio_test__m2d_l2s() */

/*-------------------------------------------------------------------------
 * Function:    contig_hs_dr_pio_test__m2d_s2l()
 *
 * Purpose:    Part four of a series of tests of I/O to/from hyperslab
 *        selections of different rank in the parallel.
 *
 *        Verify that we can write from memory to file using
 *        selections of different rank that H5Sselect_shape_same()
 *        views as being of the same shape.
 *
 *        Do this by writing the contents of the process's slice of
 *        the in memory small data set to slices of the on disk
 *        large data set.  After each write, read the process's
 *        slice of the large data set back into memory, and verify
 *        that it contains the expected data.
 *
 *        Verify that H5Sselect_shape_same() returns true on the
 *        memory and file selections.
 *
 * Return:    void
 *
 *-------------------------------------------------------------------------
 */

#define CONTIG_HS_DR_PIO_TEST__M2D_S2L__DEBUG 0

static void
contig_hs_dr_pio_test__m2d_s2l(struct hs_dr_pio_test_vars_t *tv_ptr)
{
#if CONTIG_HS_DR_PIO_TEST__M2D_S2L__DEBUG
    const char *fcnName = "contig_hs_dr_pio_test__m2d_s2l()";
#endif /* CONTIG_HS_DR_PIO_TEST__M2D_S2L__DEBUG */
    bool      mis_match = false;
    int       i, j, k, l;
    size_t    n;
    int       mpi_rank; /* needed by the VRFY macro */
    size_t    start_index;
    size_t    stop_index;
    uint32_t  expected_value;
    uint32_t *ptr_1;
    htri_t    check; /* Shape comparison return value */
    herr_t    ret;   /* Generic return value */

    /* initialize the local copy of mpi_rank */
    mpi_rank = tv_ptr->mpi_rank;

    /* Now write the contents of the process's slice of the in memory
     * small data set to slices of the on disk large data set.  After
     * each write, read the process's slice of the large data set back
     * into memory, and verify that it contains the expected data.
     * Verify that H5Sselect_shape_same() returns true on the memory
     * and file selections.
     */

    /* select the slice of the in memory small data set associated with
     * the process's mpi rank.
     */
    tv_ptr->start[0]  = (hsize_t)(tv_ptr->mpi_rank);
    tv_ptr->stride[0] = (hsize_t)(2 * (tv_ptr->mpi_size + 1));
    tv_ptr->count[0]  = 1;
    tv_ptr->block[0]  = 1;

    for (i = 1; i < tv_ptr->large_rank; i++) {

        tv_ptr->start[i]  = 0;
        tv_ptr->stride[i] = (hsize_t)(2 * tv_ptr->edge_size);
        tv_ptr->count[i]  = 1;
        tv_ptr->block[i]  = (hsize_t)(tv_ptr->edge_size);
    }

    ret = H5Sselect_hyperslab(tv_ptr->mem_small_ds_sid, H5S_SELECT_SET, tv_ptr->start, tv_ptr->stride,
                              tv_ptr->count, tv_ptr->block);
    VRFY((ret >= 0), "H5Sselect_hyperslab(mem_small_ds_sid, set) succeeded");

    /* set up start, stride, count, and block -- note that we will
     * change start[] so as to write slices of the small data set to
     * slices of the large data set.
     */
    for (i = 0; i < PAR_SS_DR_MAX_RANK; i++) {

        tv_ptr->start[i]  = 0;
        tv_ptr->stride[i] = (hsize_t)(2 * tv_ptr->edge_size);
        tv_ptr->count[i]  = 1;
        if ((PAR_SS_DR_MAX_RANK - i) > (tv_ptr->small_rank - 1)) {

            tv_ptr->block[i] = 1;
        }
        else {

            tv_ptr->block[i] = (hsize_t)(tv_ptr->edge_size);
        }
    }

    /* zero out the in memory large ds */
    memset(tv_ptr->large_ds_buf_1, 0, sizeof(uint32_t) * tv_ptr->large_ds_size);

#if CONTIG_HS_DR_PIO_TEST__M2D_S2L__DEBUG
    fprintf(stdout, "%s writing process slices of small ds to slices of large ds on disk.\n", fcnName);
#endif /* CONTIG_HS_DR_PIO_TEST__M2D_S2L__DEBUG */

    if (PAR_SS_DR_MAX_RANK - tv_ptr->large_rank == 0) {

        i = tv_ptr->mpi_rank;
    }
    else {

        i = 0;
    }

    /* since large_rank is at most PAR_SS_DR_MAX_RANK, no need to
     * loop over it -- either we are setting i to mpi_rank, or
     * we are setting it to zero.  It will not change during the
     * test.
     */

    if (PAR_SS_DR_MAX_RANK - tv_ptr->large_rank == 1) {

        j = tv_ptr->mpi_rank;
    }
    else {

        j = 0;
    }

    do {
        if (PAR_SS_DR_MAX_RANK - tv_ptr->large_rank == 2) {

            k = tv_ptr->mpi_rank;
        }
        else {

            k = 0;
        }

        do {
            /* since small rank >= 2 and large_rank > small_rank, we
             * have large_rank >= 3.  Since PAR_SS_DR_MAX_RANK == 5
             * (baring major re-orgaization), this gives us:
             *
             *     (PAR_SS_DR_MAX_RANK - large_rank) <= 2
             *
             * so no need to repeat the test in the outer loops --
             * just set l = 0.
             */

            l = 0;
            do {
                if ((tv_ptr->skips)++ < tv_ptr->max_skips) { /* skip the test */

                    (tv_ptr->tests_skipped)++;

#if CONTIG_HS_DR_PIO_TEST__M2D_S2L__DEBUG
                    tv_ptr->start[0] = (hsize_t)i;
                    tv_ptr->start[1] = (hsize_t)j;
                    tv_ptr->start[2] = (hsize_t)k;
                    tv_ptr->start[3] = (hsize_t)l;
                    tv_ptr->start[4] = 0;

                    fprintf(stdout, "%s:%d: skipping test with start = %d %d %d %d %d.\n", fcnName,
                            (int)(tv_ptr->mpi_rank), (int)(tv_ptr->start[0]), (int)(tv_ptr->start[1]),
                            (int)(tv_ptr->start[2]), (int)(tv_ptr->start[3]), (int)(tv_ptr->start[4]));
                    fprintf(stdout, "%s:%d: mem/file extent dims = %d/%d.\n", fcnName, tv_ptr->mpi_rank,
                            H5Sget_simple_extent_ndims(tv_ptr->mem_small_ds_sid),
                            H5Sget_simple_extent_ndims(tv_ptr->file_large_ds_sid_0));
#endif /* CONTIG_HS_DR_PIO_TEST__M2D_S2L__DEBUG */
                }
                else { /* run the test */

                    tv_ptr->skips = 0; /* reset the skips counter */

                    /* we know that small_rank >= 1 and that large_rank > small_rank
                     * by the assertions at the head of this function.  Thus no
                     * need for another inner loop.
                     */

                    /* Zero out this processes slice of the on disk large data set.
                     * Note that this will leave one slice with its original data
                     * as there is one more slice than processes.
                     */
                    ret = H5Dwrite(tv_ptr->large_dataset, H5T_NATIVE_UINT32, tv_ptr->large_ds_slice_sid,
                                   tv_ptr->file_large_ds_process_slice_sid, tv_ptr->xfer_plist,
                                   tv_ptr->large_ds_buf_2);
                    VRFY((ret != FAIL), "H5Dwrite() to zero large ds succeeded");

                    /* select the portion of the in memory large cube to which we
                     * are going to write data.
                     */
                    tv_ptr->start[0] = (hsize_t)i;
                    tv_ptr->start[1] = (hsize_t)j;
                    tv_ptr->start[2] = (hsize_t)k;
                    tv_ptr->start[3] = (hsize_t)l;
                    tv_ptr->start[4] = 0;

                    ret = H5Sselect_hyperslab(tv_ptr->file_large_ds_sid_0, H5S_SELECT_SET, tv_ptr->start_ptr,
                                              tv_ptr->stride_ptr, tv_ptr->count_ptr, tv_ptr->block_ptr);
                    VRFY((ret != FAIL), "H5Sselect_hyperslab() target large ds slice succeeded");

                    /* verify that H5Sselect_shape_same() reports the in
                     * memory small data set slice selection and the
                     * on disk slice through the large data set selection
                     * as having the same shape.
                     */
                    check = H5Sselect_shape_same(tv_ptr->mem_small_ds_sid, tv_ptr->file_large_ds_sid_0);
                    VRFY((check == true), "H5Sselect_shape_same passed");

                    /* write the small data set slice from memory to the
                     * target slice of the disk data set
                     */
#if CONTIG_HS_DR_PIO_TEST__M2D_S2L__DEBUG
                    fprintf(stdout, "%s:%d: start = %d %d %d %d %d.\n", fcnName, (int)(tv_ptr->mpi_rank),
                            (int)(tv_ptr->start[0]), (int)(tv_ptr->start[1]), (int)(tv_ptr->start[2]),
                            (int)(tv_ptr->start[3]), (int)(tv_ptr->start[4]));
                    fprintf(stdout, "%s:%d: mem/file extent dims = %d/%d.\n", fcnName, tv_ptr->mpi_rank,
                            H5Sget_simple_extent_ndims(tv_ptr->mem_small_ds_sid),
                            H5Sget_simple_extent_ndims(tv_ptr->file_large_ds_sid_0));
#endif /* CONTIG_HS_DR_PIO_TEST__M2D_S2L__DEBUG */
                    ret = H5Dwrite(tv_ptr->large_dataset, H5T_NATIVE_UINT32, tv_ptr->mem_small_ds_sid,
                                   tv_ptr->file_large_ds_sid_0, tv_ptr->xfer_plist, tv_ptr->small_ds_buf_0);
                    VRFY((ret != FAIL), "H5Dwrite of small ds slice to large ds succeeded");

                    /* read this processes slice on the on disk large
                     * data set into memory.
                     */

                    ret = H5Dread(
                        tv_ptr->large_dataset, H5T_NATIVE_UINT32, tv_ptr->mem_large_ds_process_slice_sid,
                        tv_ptr->file_large_ds_process_slice_sid, tv_ptr->xfer_plist, tv_ptr->large_ds_buf_1);
                    VRFY((ret != FAIL), "H5Dread() of process slice of large ds succeeded");

                    /* verify that the expected data and only the
                     * expected data was read.
                     */
                    ptr_1          = tv_ptr->large_ds_buf_1;
                    expected_value = (uint32_t)((size_t)(tv_ptr->mpi_rank) * tv_ptr->small_ds_slice_size);

                    start_index =
                        (size_t)((i * tv_ptr->edge_size * tv_ptr->edge_size * tv_ptr->edge_size *
                                  tv_ptr->edge_size) +
                                 (j * tv_ptr->edge_size * tv_ptr->edge_size * tv_ptr->edge_size) +
                                 (k * tv_ptr->edge_size * tv_ptr->edge_size) + (l * tv_ptr->edge_size));
                    stop_index = start_index + tv_ptr->small_ds_slice_size - 1;

                    assert(start_index < stop_index);
                    assert(stop_index < tv_ptr->large_ds_size);

                    for (n = 0; n < tv_ptr->large_ds_size; n++) {

                        if ((n >= start_index) && (n <= stop_index)) {

                            if (*ptr_1 != expected_value) {

                                mis_match = true;
                            }

                            expected_value++;
                        }
                        else {

                            if (*ptr_1 != 0) {

                                mis_match = true;
                            }
                        }
                        /* zero out buffer for next test */
                        *ptr_1 = 0;
                        ptr_1++;
                    }

                    VRFY((mis_match == false), "small ds slice write to large ds slice data good.");

                    (tv_ptr->tests_run)++;
                }

                l++;

                (tv_ptr->total_tests)++;

            } while ((tv_ptr->large_rank > 2) && ((tv_ptr->small_rank - 1) <= 1) && (l < tv_ptr->edge_size));
            k++;
        } while ((tv_ptr->large_rank > 3) && ((tv_ptr->small_rank - 1) <= 2) && (k < tv_ptr->edge_size));
        j++;
    } while ((tv_ptr->large_rank > 4) && ((tv_ptr->small_rank - 1) <= 3) && (j < tv_ptr->edge_size));

    return;

} /* contig_hs_dr_pio_test__m2d_s2l() */

/*-------------------------------------------------------------------------
 * Function:    contig_hs_dr_pio_test__run_test()
 *
 * Purpose:    Test I/O to/from hyperslab selections of different rank in
 *        the parallel.
 *
 * Return:    void
 *
 *-------------------------------------------------------------------------
 */

#define CONTIG_HS_DR_PIO_TEST__RUN_TEST__DEBUG 0

static void
contig_hs_dr_pio_test__run_test(const int test_num, const int edge_size, const int chunk_edge_size,
                                const int small_rank, const int large_rank, const bool use_collective_io,
                                const hid_t dset_type, int express_test, int *skips_ptr, int max_skips,
                                int64_t *total_tests_ptr, int64_t *tests_run_ptr, int64_t *tests_skipped_ptr,
                                int mpi_rank)
{
#if CONTIG_HS_DR_PIO_TEST__RUN_TEST__DEBUG
    const char *fcnName = "contig_hs_dr_pio_test__run_test()";
#endif /* CONTIG_HS_DR_PIO_TEST__RUN_TEST__DEBUG */
    struct hs_dr_pio_test_vars_t test_vars = {
        /* int           mpi_size                        = */ -1,
        /* int         mpi_rank                        = */ -1,
        /* MPI_Comm    mpi_comm                        = */ MPI_COMM_NULL,
        /* MPI_Inf     mpi_info                        = */ MPI_INFO_NULL,
        /* int         test_num                        = */ -1,
        /* int         edge_size                       = */ -1,
        /* int         checker_edge_size               = */ -1,
        /* int         chunk_edge_size                 = */ -1,
        /* int         small_rank                      = */ -1,
        /* int         large_rank                      = */ -1,
        /* hid_t       dset_type                       = */ H5I_INVALID_HID,
        /* uint32_t  * small_ds_buf_0                  = */ NULL,
        /* uint32_t  * small_ds_buf_1                  = */ NULL,
        /* uint32_t  * small_ds_buf_2                  = */ NULL,
        /* uint32_t  * small_ds_slice_buf              = */ NULL,
        /* uint32_t  * large_ds_buf_0                  = */ NULL,
        /* uint32_t  * large_ds_buf_1                  = */ NULL,
        /* uint32_t  * large_ds_buf_2                  = */ NULL,
        /* uint32_t  * large_ds_slice_buf              = */ NULL,
        /* int         small_ds_offset                 = */ -1,
        /* int         large_ds_offset                 = */ -1,
        /* hid_t       fid                             = */ H5I_INVALID_HID, /* HDF5 file ID */
        /* hid_t       xfer_plist                      = */ H5P_DEFAULT,
        /* hid_t       full_mem_small_ds_sid           = */ H5I_INVALID_HID,
        /* hid_t       full_file_small_ds_sid          = */ H5I_INVALID_HID,
        /* hid_t       mem_small_ds_sid                = */ H5I_INVALID_HID,
        /* hid_t       file_small_ds_sid_0             = */ H5I_INVALID_HID,
        /* hid_t       file_small_ds_sid_1             = */ H5I_INVALID_HID,
        /* hid_t       small_ds_slice_sid              = */ H5I_INVALID_HID,
        /* hid_t       full_mem_large_ds_sid           = */ H5I_INVALID_HID,
        /* hid_t       full_file_large_ds_sid          = */ H5I_INVALID_HID,
        /* hid_t       mem_large_ds_sid                = */ H5I_INVALID_HID,
        /* hid_t       file_large_ds_sid_0             = */ H5I_INVALID_HID,
        /* hid_t       file_large_ds_sid_1             = */ H5I_INVALID_HID,
        /* hid_t       file_large_ds_process_slice_sid = */ H5I_INVALID_HID,
        /* hid_t       mem_large_ds_process_slice_sid  = */ H5I_INVALID_HID,
        /* hid_t       large_ds_slice_sid              = */ H5I_INVALID_HID,
        /* hid_t       small_dataset                   = */ H5I_INVALID_HID, /* Dataset ID */
        /* hid_t       large_dataset                   = */ H5I_INVALID_HID, /* Dataset ID */
        /* size_t      small_ds_size                   = */ 1,
        /* size_t      small_ds_slice_size             = */ 1,
        /* size_t      large_ds_size                   = */ 1,
        /* size_t      large_ds_slice_size             = */ 1,
        /* hsize_t     dims[PAR_SS_DR_MAX_RANK]        = */ {0, 0, 0, 0, 0},
        /* hsize_t     chunk_dims[PAR_SS_DR_MAX_RANK]  = */ {0, 0, 0, 0, 0},
        /* hsize_t     start[PAR_SS_DR_MAX_RANK]       = */ {0, 0, 0, 0, 0},
        /* hsize_t     stride[PAR_SS_DR_MAX_RANK]      = */ {0, 0, 0, 0, 0},
        /* hsize_t     count[PAR_SS_DR_MAX_RANK]       = */ {0, 0, 0, 0, 0},
        /* hsize_t     block[PAR_SS_DR_MAX_RANK]       = */ {0, 0, 0, 0, 0},
        /* hsize_t   * start_ptr                       = */ NULL,
        /* hsize_t   * stride_ptr                      = */ NULL,
        /* hsize_t   * count_ptr                       = */ NULL,
        /* hsize_t   * block_ptr                       = */ NULL,
        /* int            skips                           = */ 0,
        /* int            max_skips                       = */ 0,
        /* int64_t     total_tests                     = */ 0,
        /* int64_t     tests_run                       = */ 0,
        /* int64_t     tests_skipped                   = */ 0};
    struct hs_dr_pio_test_vars_t *tv_ptr = &test_vars;

    if (MAINPROCESS)
        printf("\r - running test #%lld: small rank = %d, large rank = %d", (long long)(test_num + 1),
               small_rank, large_rank);

    hs_dr_pio_test__setup(test_num, edge_size, -1, chunk_edge_size, small_rank, large_rank, use_collective_io,
                          dset_type, express_test, tv_ptr);

    /* initialize skips & max_skips */
    tv_ptr->skips     = *skips_ptr;
    tv_ptr->max_skips = max_skips;

#if CONTIG_HS_DR_PIO_TEST__RUN_TEST__DEBUG
    if (MAINPROCESS) {
        fprintf(stdout, "test %d: small rank = %d, large rank = %d.\n", test_num, small_rank, large_rank);
        fprintf(stdout, "test %d: Initialization complete.\n", test_num);
    }
#endif /* CONTIG_HS_DR_PIO_TEST__RUN_TEST__DEBUG */

    /* first, verify that we can read from disk correctly using selections
     * of different rank that H5Sselect_shape_same() views as being of the
     * same shape.
     *
     * Start by reading small_rank - 1 dimensional slice from the on disk
     * large cube, and verifying that the data read is correct.  Verify that
     * H5Sselect_shape_same() returns true on the memory and file selections.
     */

#if CONTIG_HS_DR_PIO_TEST__RUN_TEST__DEBUG
    if (MAINPROCESS) {
        fprintf(stdout, "test %d: running contig_hs_dr_pio_test__d2m_l2s.\n", test_num);
    }
#endif /* CONTIG_HS_DR_PIO_TEST__RUN_TEST__DEBUG */
    contig_hs_dr_pio_test__d2m_l2s(tv_ptr);

    /* Second, read slices of the on disk small data set into slices
     * through the in memory large data set, and verify that the correct
     * data (and only the correct data) is read.
     */

#if CONTIG_HS_DR_PIO_TEST__RUN_TEST__DEBUG
    if (MAINPROCESS) {
        fprintf(stdout, "test %d: running contig_hs_dr_pio_test__d2m_s2l.\n", test_num);
    }
#endif /* CONTIG_HS_DR_PIO_TEST__RUN_TEST__DEBUG */
    contig_hs_dr_pio_test__d2m_s2l(tv_ptr);

    /* now we go in the opposite direction, verifying that we can write
     * from memory to file using selections of different rank that
     * H5Sselect_shape_same() views as being of the same shape.
     *
     * Start by writing small_rank - 1 D slices from the in memory large data
     * set to the on disk small cube dataset.  After each write, read the
     * slice of the small dataset back from disk, and verify that it contains
     * the expected data. Verify that H5Sselect_shape_same() returns true on
     * the memory and file selections.
     */

#if CONTIG_HS_DR_PIO_TEST__RUN_TEST__DEBUG
    if (MAINPROCESS) {
        fprintf(stdout, "test %d: running contig_hs_dr_pio_test__m2d_l2s.\n", test_num);
    }
#endif /* CONTIG_HS_DR_PIO_TEST__RUN_TEST__DEBUG */
    contig_hs_dr_pio_test__m2d_l2s(tv_ptr);

    /* Now write the contents of the process's slice of the in memory
     * small data set to slices of the on disk large data set.  After
     * each write, read the process's slice of the large data set back
     * into memory, and verify that it contains the expected data.
     * Verify that H5Sselect_shape_same() returns true on the memory
     * and file selections.
     */

#if CONTIG_HS_DR_PIO_TEST__RUN_TEST__DEBUG
    if (MAINPROCESS) {
        fprintf(stdout, "test %d: running contig_hs_dr_pio_test__m2d_s2l.\n", test_num);
    }
#endif /* CONTIG_HS_DR_PIO_TEST__RUN_TEST__DEBUG */
    contig_hs_dr_pio_test__m2d_s2l(tv_ptr);

#if CONTIG_HS_DR_PIO_TEST__RUN_TEST__DEBUG
    if (MAINPROCESS) {
        fprintf(stdout, "test %d: Subtests complete -- tests run/skipped/total = %lld/%lld/%lld.\n", test_num,
                (long long)(tv_ptr->tests_run), (long long)(tv_ptr->tests_skipped),
                (long long)(tv_ptr->total_tests));
    }
#endif /* CONTIG_HS_DR_PIO_TEST__RUN_TEST__DEBUG */

    hs_dr_pio_test__takedown(tv_ptr);

#if CONTIG_HS_DR_PIO_TEST__RUN_TEST__DEBUG
    if (MAINPROCESS) {
        fprintf(stdout, "test %d: Takedown complete.\n", test_num);
    }
#endif /* CONTIG_HS_DR_PIO_TEST__RUN_TEST__DEBUG */

    *skips_ptr = tv_ptr->skips;
    *total_tests_ptr += tv_ptr->total_tests;
    *tests_run_ptr += tv_ptr->tests_run;
    *tests_skipped_ptr += tv_ptr->tests_skipped;

    return;

} /* contig_hs_dr_pio_test__run_test() */

/*-------------------------------------------------------------------------
 * Function:    contig_hs_dr_pio_test(ShapeSameTestMethods sstest_type)
 *
 * Purpose:    Test I/O to/from hyperslab selections of different rank in
 *        the parallel case.
 *
 * Return:    void
 *
 *-------------------------------------------------------------------------
 */

#define CONTIG_HS_DR_PIO_TEST__DEBUG 0

static void
contig_hs_dr_pio_test(ShapeSameTestMethods sstest_type)
{
    int express_test;
    int local_express_test;
    int mpi_rank = -1;
    int mpi_size;
    int test_num = 0;
    int edge_size;
    int chunk_edge_size = 0;
    int small_rank;
    int large_rank;
    int mpi_result;
    int skips     = 0;
    int max_skips = 0;
    /* The following table list the number of sub-tests skipped between
     * each test that is actually executed as a function of the express
     * test level.  Note that any value in excess of 4880 will cause all
     * sub tests to be skipped.
     */
    int     max_skips_tbl[4] = {0, 4, 64, 1024};
    hid_t   dset_type        = H5T_NATIVE_UINT;
    int64_t total_tests      = 0;
    int64_t tests_run        = 0;
    int64_t tests_skipped    = 0;

    HDcompile_assert(sizeof(uint32_t) == sizeof(unsigned));

    MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
    MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);

    edge_size = (mpi_size > 6 ? mpi_size : 6);

    local_express_test = GetTestExpress();

    mpi_result = MPI_Allreduce((void *)&local_express_test, (void *)&express_test, 1, MPI_INT, MPI_MAX,
                               MPI_COMM_WORLD);

    VRFY((mpi_result == MPI_SUCCESS), "MPI_Allreduce(0) succeeded");

    if (local_express_test < 0) {
        max_skips = max_skips_tbl[0];
    }
    else if (local_express_test > 3) {
        max_skips = max_skips_tbl[3];
    }
    else {
        max_skips = max_skips_tbl[local_express_test];
    }

    for (large_rank = 3; large_rank <= PAR_SS_DR_MAX_RANK; large_rank++) {

        for (small_rank = 2; small_rank < large_rank; small_rank++) {

            switch (sstest_type) {
                case IND_CONTIG:
                    /* contiguous data set, independent I/O */
                    chunk_edge_size = 0;

                    contig_hs_dr_pio_test__run_test(
                        test_num, edge_size, chunk_edge_size, small_rank, large_rank, false, dset_type,
                        express_test, &skips, max_skips, &total_tests, &tests_run, &tests_skipped, mpi_rank);
                    test_num++;
                    break;
                    /* end of case IND_CONTIG */

                case COL_CONTIG:
                    /* contiguous data set, collective I/O */
                    chunk_edge_size = 0;

                    contig_hs_dr_pio_test__run_test(
                        test_num, edge_size, chunk_edge_size, small_rank, large_rank, true, dset_type,
                        express_test, &skips, max_skips, &total_tests, &tests_run, &tests_skipped, mpi_rank);
                    test_num++;
                    break;
                    /* end of case COL_CONTIG */

                case IND_CHUNKED:
                    /* chunked data set, independent I/O */
                    chunk_edge_size = 5;

                    contig_hs_dr_pio_test__run_test(
                        test_num, edge_size, chunk_edge_size, small_rank, large_rank, false, dset_type,
                        express_test, &skips, max_skips, &total_tests, &tests_run, &tests_skipped, mpi_rank);
                    test_num++;
                    break;
                    /* end of case IND_CHUNKED */

                case COL_CHUNKED:
                    /* chunked data set, collective I/O */
                    chunk_edge_size = 5;

                    contig_hs_dr_pio_test__run_test(
                        test_num, edge_size, chunk_edge_size, small_rank, large_rank, true, dset_type,
                        express_test, &skips, max_skips, &total_tests, &tests_run, &tests_skipped, mpi_rank);
                    test_num++;
                    break;
                    /* end of case COL_CHUNKED */

                default:
                    VRFY((false), "unknown test type");
                    break;

            } /* end of switch(sstest_type) */
#if CONTIG_HS_DR_PIO_TEST__DEBUG
            if ((MAINPROCESS) && (tests_skipped > 0)) {
                fprintf(stdout, "    run/skipped/total = %lld/%lld/%lld.\n", tests_run, tests_skipped,
                        total_tests);
            }
#endif /* CONTIG_HS_DR_PIO_TEST__DEBUG */
        }
    }

    if (MAINPROCESS) {
        if (tests_skipped > 0) {
            fprintf(stdout, "    %" PRId64 " of %" PRId64 " subtests skipped to expedite testing.\n",
                    tests_skipped, total_tests);
        }
        else
            printf("\n");
    }

    return;

} /* contig_hs_dr_pio_test() */

/****************************************************************
**
**  ckrbrd_hs_dr_pio_test__slct_ckrbrd():
**    Given a dataspace of tgt_rank, and dimensions:
**
**        (mpi_size + 1), edge_size, ... , edge_size
**
**    edge_size, and a checker_edge_size, select a checker
**    board selection of a sel_rank (sel_rank < tgt_rank)
**    dimensional slice through the dataspace parallel to the
**      sel_rank fastest changing indices, with origin (in the
**    higher indices) as indicated by the start array.
**
**    Note that this function, like all its relatives, is
**    hard coded to presume a maximum dataspace rank of 5.
**    While this maximum is declared as a constant, increasing
**    it will require extensive coding in addition to changing
**      the value of the constant.
**
**                    JRM -- 10/8/09
**
****************************************************************/

#define CKRBRD_HS_DR_PIO_TEST__SELECT_CHECKER_BOARD__DEBUG 0

static void
ckrbrd_hs_dr_pio_test__slct_ckrbrd(const int mpi_rank, const hid_t tgt_sid, const int tgt_rank,
                                   const int edge_size, const int checker_edge_size, const int sel_rank,
                                   hsize_t sel_start[])
{
#if CKRBRD_HS_DR_PIO_TEST__SELECT_CHECKER_BOARD__DEBUG
    const char *fcnName = "ckrbrd_hs_dr_pio_test__slct_ckrbrd():";
#endif
    bool      first_selection = true;
    int       i, j, k, l, m;
    int       n_cube_offset;
    int       sel_offset;
    const int test_max_rank = PAR_SS_DR_MAX_RANK; /* must update code if */
                                                  /* this changes        */
    hsize_t base_count;
    hsize_t offset_count;
    hsize_t start[PAR_SS_DR_MAX_RANK];
    hsize_t stride[PAR_SS_DR_MAX_RANK];
    hsize_t count[PAR_SS_DR_MAX_RANK];
    hsize_t block[PAR_SS_DR_MAX_RANK];
    herr_t  ret; /* Generic return value */

    assert(edge_size >= 6);
    assert(0 < checker_edge_size);
    assert(checker_edge_size <= edge_size);
    assert(0 < sel_rank);
    assert(sel_rank <= tgt_rank);
    assert(tgt_rank <= test_max_rank);
    assert(test_max_rank <= PAR_SS_DR_MAX_RANK);

    sel_offset = test_max_rank - sel_rank;
    assert(sel_offset >= 0);

    n_cube_offset = test_max_rank - tgt_rank;
    assert(n_cube_offset >= 0);
    assert(n_cube_offset <= sel_offset);

#if CKRBRD_HS_DR_PIO_TEST__SELECT_CHECKER_BOARD__DEBUG
    fprintf(stdout, "%s:%d: edge_size/checker_edge_size = %d/%d\n", fcnName, mpi_rank, edge_size,
            checker_edge_size);
    fprintf(stdout, "%s:%d: sel_rank/sel_offset = %d/%d.\n", fcnName, mpi_rank, sel_rank, sel_offset);
    fprintf(stdout, "%s:%d: tgt_rank/n_cube_offset = %d/%d.\n", fcnName, mpi_rank, tgt_rank, n_cube_offset);
#endif /* CKRBRD_HS_DR_PIO_TEST__SELECT_CHECKER_BOARD__DEBUG */

    /* First, compute the base count (which assumes start == 0
     * for the associated offset) and offset_count (which
     * assumes start == checker_edge_size for the associated
     * offset).
     *
     * Note that the following computation depends on the C99
     * requirement that integer division discard any fraction
     * (truncation towards zero) to function correctly. As we
     * now require C99, this shouldn't be a problem, but noting
     * it may save us some pain if we are ever obliged to support
     * pre-C99 compilers again.
     */

    base_count = (hsize_t)(edge_size / (checker_edge_size * 2));

    if ((edge_size % (checker_edge_size * 2)) > 0) {

        base_count++;
    }

    offset_count = (hsize_t)((edge_size - checker_edge_size) / (checker_edge_size * 2));

    if (((edge_size - checker_edge_size) % (checker_edge_size * 2)) > 0) {

        offset_count++;
    }

    /* Now set up the stride and block arrays, and portions of the start
     * and count arrays that will not be altered during the selection of
     * the checker board.
     */
    i = 0;
    while (i < n_cube_offset) {

        /* these values should never be used */
        start[i]  = 0;
        stride[i] = 0;
        count[i]  = 0;
        block[i]  = 0;

        i++;
    }

    while (i < sel_offset) {

        start[i]  = sel_start[i];
        stride[i] = (hsize_t)(2 * edge_size);
        count[i]  = 1;
        block[i]  = 1;

        i++;
    }

    while (i < test_max_rank) {

        stride[i] = (hsize_t)(2 * checker_edge_size);
        block[i]  = (hsize_t)checker_edge_size;

        i++;
    }

    i = 0;
    do {
        if (0 >= sel_offset) {

            if (i == 0) {

                start[0] = 0;
                count[0] = base_count;
            }
            else {

                start[0] = (hsize_t)checker_edge_size;
                count[0] = offset_count;
            }
        }

        j = 0;
        do {
            if (1 >= sel_offset) {

                if (j == 0) {

                    start[1] = 0;
                    count[1] = base_count;
                }
                else {

                    start[1] = (hsize_t)checker_edge_size;
                    count[1] = offset_count;
                }
            }

            k = 0;
            do {
                if (2 >= sel_offset) {

                    if (k == 0) {

                        start[2] = 0;
                        count[2] = base_count;
                    }
                    else {

                        start[2] = (hsize_t)checker_edge_size;
                        count[2] = offset_count;
                    }
                }

                l = 0;
                do {
                    if (3 >= sel_offset) {

                        if (l == 0) {

                            start[3] = 0;
                            count[3] = base_count;
                        }
                        else {

                            start[3] = (hsize_t)checker_edge_size;
                            count[3] = offset_count;
                        }
                    }

                    m = 0;
                    do {
                        if (4 >= sel_offset) {

                            if (m == 0) {

                                start[4] = 0;
                                count[4] = base_count;
                            }
                            else {

                                start[4] = (hsize_t)checker_edge_size;
                                count[4] = offset_count;
                            }
                        }

                        if (((i + j + k + l + m) % 2) == 0) {

#if CKRBRD_HS_DR_PIO_TEST__SELECT_CHECKER_BOARD__DEBUG
                            fprintf(stdout, "%s%d: *** first_selection = %d ***\n", fcnName, mpi_rank,
                                    (int)first_selection);
                            fprintf(stdout, "%s:%d: i/j/k/l/m = %d/%d/%d/%d/%d\n", fcnName, mpi_rank, i, j, k,
                                    l, m);
                            fprintf(stdout, "%s:%d: start = %d %d %d %d %d.\n", fcnName, mpi_rank,
                                    (int)start[0], (int)start[1], (int)start[2], (int)start[3],
                                    (int)start[4]);
                            fprintf(stdout, "%s:%d: stride = %d %d %d %d %d.\n", fcnName, mpi_rank,
                                    (int)stride[0], (int)stride[1], (int)stride[2], (int)stride[3],
                                    (int)stride[4]);
                            fprintf(stdout, "%s:%d: count = %d %d %d %d %d.\n", fcnName, mpi_rank,
                                    (int)count[0], (int)count[1], (int)count[2], (int)count[3],
                                    (int)count[4]);
                            fprintf(stdout, "%s:%d: block = %d %d %d %d %d.\n", fcnName, mpi_rank,
                                    (int)block[0], (int)block[1], (int)block[2], (int)block[3],
                                    (int)block[4]);
                            fprintf(stdout, "%s:%d: n-cube extent dims = %d.\n", fcnName, mpi_rank,
                                    H5Sget_simple_extent_ndims(tgt_sid));
                            fprintf(stdout, "%s:%d: selection rank = %d.\n", fcnName, mpi_rank, sel_rank);
#endif

                            if (first_selection) {

                                first_selection = false;

                                ret = H5Sselect_hyperslab(tgt_sid, H5S_SELECT_SET, &(start[n_cube_offset]),
                                                          &(stride[n_cube_offset]), &(count[n_cube_offset]),
                                                          &(block[n_cube_offset]));

                                VRFY((ret != FAIL), "H5Sselect_hyperslab(SET) succeeded");
                            }
                            else {

                                ret = H5Sselect_hyperslab(tgt_sid, H5S_SELECT_OR, &(start[n_cube_offset]),
                                                          &(stride[n_cube_offset]), &(count[n_cube_offset]),
                                                          &(block[n_cube_offset]));

                                VRFY((ret != FAIL), "H5Sselect_hyperslab(OR) succeeded");
                            }
                        }

                        m++;

                    } while ((m <= 1) && (4 >= sel_offset));

                    l++;

                } while ((l <= 1) && (3 >= sel_offset));

                k++;

            } while ((k <= 1) && (2 >= sel_offset));

            j++;

        } while ((j <= 1) && (1 >= sel_offset));

        i++;

    } while ((i <= 1) && (0 >= sel_offset));

#if CKRBRD_HS_DR_PIO_TEST__SELECT_CHECKER_BOARD__DEBUG
    fprintf(stdout, "%s%d: H5Sget_select_npoints(tgt_sid) = %d.\n", fcnName, mpi_rank,
            (int)H5Sget_select_npoints(tgt_sid));
#endif /* CKRBRD_HS_DR_PIO_TEST__SELECT_CHECKER_BOARD__DEBUG */

    /* Clip the selection back to the dataspace proper. */

    for (i = 0; i < test_max_rank; i++) {

        start[i]  = 0;
        stride[i] = (hsize_t)edge_size;
        count[i]  = 1;
        block[i]  = (hsize_t)edge_size;
    }

    ret = H5Sselect_hyperslab(tgt_sid, H5S_SELECT_AND, start, stride, count, block);

    VRFY((ret != FAIL), "H5Sselect_hyperslab(AND) succeeded");

#if CKRBRD_HS_DR_PIO_TEST__SELECT_CHECKER_BOARD__DEBUG
    fprintf(stdout, "%s%d: H5Sget_select_npoints(tgt_sid) = %d.\n", fcnName, mpi_rank,
            (int)H5Sget_select_npoints(tgt_sid));
    fprintf(stdout, "%s%d: done.\n", fcnName, mpi_rank);
#endif /* CKRBRD_HS_DR_PIO_TEST__SELECT_CHECKER_BOARD__DEBUG */

    return;

} /* ckrbrd_hs_dr_pio_test__slct_ckrbrd() */

/****************************************************************
**
**  ckrbrd_hs_dr_pio_test__verify_data():
**
**    Examine the supplied buffer to see if it contains the
**    expected data.  Return true if it does, and false
**      otherwise.
**
**    The supplied buffer is presumed to this process's slice
**    of the target data set.  Each such slice will be an
**    n-cube of rank (rank -1) and the supplied edge_size with
**    origin (mpi_rank, 0, ... , 0) in the target data set.
**
**    Further, the buffer is presumed to be the result of reading
**    or writing a checker board selection of an m (1 <= m <
**      rank) dimensional slice through this processes slice
**    of the target data set.  Also, this slice must be parallel
**    to the fastest changing indices.
**
**    It is further presumed that the buffer was zeroed before
**    the read/write, and that the full target data set (i.e.
**    the buffer/data set for all processes) was initialized
**      with the natural numbers listed in order from the origin
**    along the fastest changing axis.
**
**      Thus for a 20x10x10 dataset, the value stored in location
**    (x, y, z) (assuming that z is the fastest changing index
**    and x the slowest) is assumed to be:
**
**        (10 * 10 * x) + (10 * y) + z
**
**    Further, supposing that this is process 10, this process's
**    slice of the dataset would be a 10 x 10 2-cube with origin
**    (10, 0, 0) in the data set, and would be initialize (prior
**    to the checkerboard selection) as follows:
**
**        1000, 1001, 1002, ... 1008, 1009
**        1010, 1011, 1012, ... 1018, 1019
**          .     .     .         .     .
**          .     .     .         .     .
**          .     .     .         .     .
**        1090, 1091, 1092, ... 1098, 1099
**
**    In the case of a read from the processors slice of another
**    data set of different rank, the values expected will have
**    to be adjusted accordingly.  This is done via the
**    first_expected_val parameter.
**
**    Finally, the function presumes that the first element
**    of the buffer resides either at the origin of either
**    a selected or an unselected checker.  (Translation:
**    if partial checkers appear in the buffer, they will
**    intersect the edges of the n-cube opposite the origin.)
**
****************************************************************/

#define CKRBRD_HS_DR_PIO_TEST__VERIFY_DATA__DEBUG 0

static bool
ckrbrd_hs_dr_pio_test__verify_data(uint32_t *buf_ptr, const int rank, const int edge_size,
                                   const int checker_edge_size, uint32_t first_expected_val,
                                   bool buf_starts_in_checker)
{
#if CKRBRD_HS_DR_PIO_TEST__VERIFY_DATA__DEBUG
    const char *fcnName = "ckrbrd_hs_dr_pio_test__verify_data():";
#endif
    bool      good_data = true;
    bool      in_checker;
    bool      start_in_checker[5];
    uint32_t  expected_value;
    uint32_t *val_ptr;
    int       i, j, k, l, m;     /* to track position in n-cube */
    int       v, w, x, y, z;     /* to track position in checker */
    const int test_max_rank = 5; /* code changes needed if this is increased */

    assert(buf_ptr != NULL);
    assert(0 < rank);
    assert(rank <= test_max_rank);
    assert(edge_size >= 6);
    assert(0 < checker_edge_size);
    assert(checker_edge_size <= edge_size);
    assert(test_max_rank <= PAR_SS_DR_MAX_RANK);

#if CKRBRD_HS_DR_PIO_TEST__VERIFY_DATA__DEBUG

    int mpi_rank;

    MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
    fprintf(stdout, "%s mpi_rank = %d.\n", fcnName, mpi_rank);
    fprintf(stdout, "%s rank = %d.\n", fcnName, rank);
    fprintf(stdout, "%s edge_size = %d.\n", fcnName, edge_size);
    fprintf(stdout, "%s checker_edge_size = %d.\n", fcnName, checker_edge_size);
    fprintf(stdout, "%s first_expected_val = %d.\n", fcnName, (int)first_expected_val);
    fprintf(stdout, "%s starts_in_checker = %d.\n", fcnName, (int)buf_starts_in_checker);
}
#endif

val_ptr        = buf_ptr;
expected_value = first_expected_val;

i                   = 0;
v                   = 0;
start_in_checker[0] = buf_starts_in_checker;
do {
    if (v >= checker_edge_size) {

        start_in_checker[0] = !start_in_checker[0];
        v                   = 0;
    }

    j                   = 0;
    w                   = 0;
    start_in_checker[1] = start_in_checker[0];
    do {
        if (w >= checker_edge_size) {

            start_in_checker[1] = !start_in_checker[1];
            w                   = 0;
        }

        k                   = 0;
        x                   = 0;
        start_in_checker[2] = start_in_checker[1];
        do {
            if (x >= checker_edge_size) {

                start_in_checker[2] = !start_in_checker[2];
                x                   = 0;
            }

            l                   = 0;
            y                   = 0;
            start_in_checker[3] = start_in_checker[2];
            do {
                if (y >= checker_edge_size) {

                    start_in_checker[3] = !start_in_checker[3];
                    y                   = 0;
                }

                m = 0;
                z = 0;
#if CKRBRD_HS_DR_PIO_TEST__VERIFY_DATA__DEBUG
                fprintf(stdout, "%d, %d, %d, %d, %d:", i, j, k, l, m);
#endif
                in_checker = start_in_checker[3];
                do {
#if CKRBRD_HS_DR_PIO_TEST__VERIFY_DATA__DEBUG
                    fprintf(stdout, " %d", (int)(*val_ptr));
#endif
                    if (z >= checker_edge_size) {

                        in_checker = !in_checker;
                        z          = 0;
                    }

                    if (in_checker) {

                        if (*val_ptr != expected_value) {

                            good_data = false;
                        }

                        /* zero out buffer for reuse */
                        *val_ptr = 0;
                    }
                    else if (*val_ptr != 0) {

                        good_data = false;

                        /* zero out buffer for reuse */
                        *val_ptr = 0;
                    }

                    val_ptr++;
                    expected_value++;
                    m++;
                    z++;

                } while ((rank >= (test_max_rank - 4)) && (m < edge_size));
#if CKRBRD_HS_DR_PIO_TEST__VERIFY_DATA__DEBUG
                fprintf(stdout, "\n");
#endif
                l++;
                y++;
            } while ((rank >= (test_max_rank - 3)) && (l < edge_size));
            k++;
            x++;
        } while ((rank >= (test_max_rank - 2)) && (k < edge_size));
        j++;
        w++;
    } while ((rank >= (test_max_rank - 1)) && (j < edge_size));
    i++;
    v++;
} while ((rank >= test_max_rank) && (i < edge_size));

return (good_data);

} /* ckrbrd_hs_dr_pio_test__verify_data() */

/*-------------------------------------------------------------------------
 * Function:    ckrbrd_hs_dr_pio_test__d2m_l2s()
 *
 * Purpose:    Part one of a series of tests of I/O to/from hyperslab
 *        selections of different rank in the parallel.
 *
 *        Verify that we can read from disk correctly using checker
 *        board selections of different rank that
 *        H5Sselect_shape_same() views as being of the same shape.
 *
 *        In this function, we test this by reading small_rank - 1
 *        checker board slices from the on disk large cube, and
 *        verifying that the data read is correct.  Verify that
 *        H5Sselect_shape_same() returns true on the memory and
 *        file selections.
 *
 * Return:    void
 *
 *-------------------------------------------------------------------------
 */

#define CHECKER_BOARD_HS_DR_PIO_TEST__D2M_L2S__DEBUG 0

static void
ckrbrd_hs_dr_pio_test__d2m_l2s(struct hs_dr_pio_test_vars_t *tv_ptr)
{
#if CHECKER_BOARD_HS_DR_PIO_TEST__D2M_L2S__DEBUG
    const char *fcnName = "ckrbrd_hs_dr_pio_test__d2m_l2s()";
    uint32_t   *ptr_0;
#endif /* CHECKER_BOARD_HS_DR_PIO_TEST__D2M_L2S__DEBUG */
    bool     data_ok = false;
    int      i, j, k, l;
    uint32_t expected_value;
    int      mpi_rank; /* needed by VRFY */
    hsize_t  sel_start[PAR_SS_DR_MAX_RANK];
    htri_t   check; /* Shape comparison return value */
    herr_t   ret;   /* Generic return value */

    /* initialize the local copy of mpi_rank */
    mpi_rank = tv_ptr->mpi_rank;

    /* first, verify that we can read from disk correctly using selections
     * of different rank that H5Sselect_shape_same() views as being of the
     * same shape.
     *
     * Start by reading a (small_rank - 1)-D checker board slice from this
     * processes slice of the on disk large data set, and verifying that the
     * data read is correct.  Verify that H5Sselect_shape_same() returns
     * true on the memory and file selections.
     *
     * The first step is to set up the needed checker board selection in the
     * in memory small small cube
     */

    sel_start[0] = sel_start[1] = sel_start[2] = sel_start[3] = sel_start[4] = 0;
    sel_start[tv_ptr->small_ds_offset]                                       = (hsize_t)(tv_ptr->mpi_rank);

    ckrbrd_hs_dr_pio_test__slct_ckrbrd(tv_ptr->mpi_rank, tv_ptr->small_ds_slice_sid, tv_ptr->small_rank - 1,
                                       tv_ptr->edge_size, tv_ptr->checker_edge_size, tv_ptr->small_rank - 1,
                                       sel_start);

    /* zero out the buffer we will be reading into */
    memset(tv_ptr->small_ds_slice_buf, 0, sizeof(uint32_t) * tv_ptr->small_ds_slice_size);

#if CHECKER_BOARD_HS_DR_PIO_TEST__D2M_L2S__DEBUG
    fprintf(stdout, "%s:%d: initial small_ds_slice_buf = ", fcnName, tv_ptr->mpi_rank);
    ptr_0 = tv_ptr->small_ds_slice_buf;
    for (i = 0; i < (int)(tv_ptr->small_ds_slice_size); i++) {
        fprintf(stdout, "%d ", (int)(*ptr_0));
        ptr_0++;
    }
    fprintf(stdout, "\n");
#endif /* CHECKER_BOARD_HS_DR_PIO_TEST__D2M_L2S__DEBUG */

    /* set up start, stride, count, and block -- note that we will
     * change start[] so as to read slices of the large cube.
     */
    for (i = 0; i < PAR_SS_DR_MAX_RANK; i++) {

        tv_ptr->start[i]  = 0;
        tv_ptr->stride[i] = (hsize_t)(2 * tv_ptr->edge_size);
        tv_ptr->count[i]  = 1;
        if ((PAR_SS_DR_MAX_RANK - i) > (tv_ptr->small_rank - 1)) {

            tv_ptr->block[i] = 1;
        }
        else {

            tv_ptr->block[i] = (hsize_t)(tv_ptr->edge_size);
        }
    }

#if CHECKER_BOARD_HS_DR_PIO_TEST__D2M_L2S__DEBUG
    fprintf(stdout, "%s:%d: reading slice from big ds on disk into small ds slice.\n", fcnName,
            tv_ptr->mpi_rank);
#endif /* CHECKER_BOARD_HS_DR_PIO_TEST__D2M_L2S__DEBUG */
    /* in serial versions of this test, we loop through all the dimensions
     * of the large data set.  However, in the parallel version, each
     * process only works with that slice of the large cube indicated
     * by its rank -- hence we set the most slowly changing index to
     * mpi_rank, and don't iterate over it.
     */

    if (PAR_SS_DR_MAX_RANK - tv_ptr->large_rank == 0) {

        i = tv_ptr->mpi_rank;
    }
    else {

        i = 0;
    }

    /* since large_rank is at most PAR_SS_DR_MAX_RANK, no need to
     * loop over it -- either we are setting i to mpi_rank, or
     * we are setting it to zero.  It will not change during the
     * test.
     */

    if (PAR_SS_DR_MAX_RANK - tv_ptr->large_rank == 1) {

        j = tv_ptr->mpi_rank;
    }
    else {

        j = 0;
    }

    do {
        if (PAR_SS_DR_MAX_RANK - tv_ptr->large_rank == 2) {

            k = tv_ptr->mpi_rank;
        }
        else {

            k = 0;
        }

        do {
            /* since small rank >= 2 and large_rank > small_rank, we
             * have large_rank >= 3.  Since PAR_SS_DR_MAX_RANK == 5
             * (baring major re-orgaization), this gives us:
             *
             *     (PAR_SS_DR_MAX_RANK - large_rank) <= 2
             *
             * so no need to repeat the test in the outer loops --
             * just set l = 0.
             */

            l = 0;
            do {
                if ((tv_ptr->skips)++ < tv_ptr->max_skips) { /* skip the test */

                    (tv_ptr->tests_skipped)++;
                }
                else { /* run the test */

                    tv_ptr->skips = 0; /* reset the skips counter */

                    /* we know that small_rank - 1 >= 1 and that
                     * large_rank > small_rank by the assertions at the head
                     * of this function.  Thus no need for another inner loop.
                     */
                    tv_ptr->start[0] = (hsize_t)i;
                    tv_ptr->start[1] = (hsize_t)j;
                    tv_ptr->start[2] = (hsize_t)k;
                    tv_ptr->start[3] = (hsize_t)l;
                    tv_ptr->start[4] = 0;

                    assert((tv_ptr->start[0] == 0) || (0 < tv_ptr->small_ds_offset + 1));
                    assert((tv_ptr->start[1] == 0) || (1 < tv_ptr->small_ds_offset + 1));
                    assert((tv_ptr->start[2] == 0) || (2 < tv_ptr->small_ds_offset + 1));
                    assert((tv_ptr->start[3] == 0) || (3 < tv_ptr->small_ds_offset + 1));
                    assert((tv_ptr->start[4] == 0) || (4 < tv_ptr->small_ds_offset + 1));

                    ckrbrd_hs_dr_pio_test__slct_ckrbrd(
                        tv_ptr->mpi_rank, tv_ptr->file_large_ds_sid_0, tv_ptr->large_rank, tv_ptr->edge_size,
                        tv_ptr->checker_edge_size, tv_ptr->small_rank - 1, tv_ptr->start);

                    /* verify that H5Sselect_shape_same() reports the two
                     * selections as having the same shape.
                     */
                    check = H5Sselect_shape_same(tv_ptr->small_ds_slice_sid, tv_ptr->file_large_ds_sid_0);
                    VRFY((check == true), "H5Sselect_shape_same passed");

                    /* Read selection from disk */
#if CHECKER_BOARD_HS_DR_PIO_TEST__D2M_L2S__DEBUG
                    fprintf(stdout, "%s:%d: start = %d %d %d %d %d.\n", fcnName, tv_ptr->mpi_rank,
                            tv_ptr->start[0], tv_ptr->start[1], tv_ptr->start[2], tv_ptr->start[3],
                            tv_ptr->start[4]);
                    fprintf(stdout, "%s slice/file extent dims = %d/%d.\n", fcnName,
                            H5Sget_simple_extent_ndims(tv_ptr->small_ds_slice_sid),
                            H5Sget_simple_extent_ndims(tv_ptr->file_large_ds_sid_0));
#endif /* CHECKER_BOARD_HS_DR_PIO_TEST__D2M_L2S__DEBUG */

                    ret =
                        H5Dread(tv_ptr->large_dataset, H5T_NATIVE_UINT32, tv_ptr->small_ds_slice_sid,
                                tv_ptr->file_large_ds_sid_0, tv_ptr->xfer_plist, tv_ptr->small_ds_slice_buf);
                    VRFY((ret >= 0), "H5Dread() slice from large ds succeeded.");

#if CHECKER_BOARD_HS_DR_PIO_TEST__D2M_L2S__DEBUG
                    fprintf(stdout, "%s:%d: H5Dread() returns.\n", fcnName, tv_ptr->mpi_rank);
#endif /* CHECKER_BOARD_HS_DR_PIO_TEST__D2M_L2S__DEBUG */

                    /* verify that expected data is retrieved */

                    expected_value =
                        (uint32_t)((i * tv_ptr->edge_size * tv_ptr->edge_size * tv_ptr->edge_size *
                                    tv_ptr->edge_size) +
                                   (j * tv_ptr->edge_size * tv_ptr->edge_size * tv_ptr->edge_size) +
                                   (k * tv_ptr->edge_size * tv_ptr->edge_size) + (l * tv_ptr->edge_size));

                    data_ok = ckrbrd_hs_dr_pio_test__verify_data(
                        tv_ptr->small_ds_slice_buf, tv_ptr->small_rank - 1, tv_ptr->edge_size,
                        tv_ptr->checker_edge_size, expected_value, (bool)true);

                    VRFY((data_ok == true), "small slice read from large ds data good.");

                    (tv_ptr->tests_run)++;
                }

                l++;

                (tv_ptr->total_tests)++;

            } while ((tv_ptr->large_rank > 2) && ((tv_ptr->small_rank - 1) <= 1) && (l < tv_ptr->edge_size));
            k++;
        } while ((tv_ptr->large_rank > 3) && ((tv_ptr->small_rank - 1) <= 2) && (k < tv_ptr->edge_size));
        j++;
    } while ((tv_ptr->large_rank > 4) && ((tv_ptr->small_rank - 1) <= 3) && (j < tv_ptr->edge_size));

    return;

} /* ckrbrd_hs_dr_pio_test__d2m_l2s() */

/*-------------------------------------------------------------------------
 * Function:    ckrbrd_hs_dr_pio_test__d2m_s2l()
 *
 * Purpose:    Part two of a series of tests of I/O to/from hyperslab
 *        selections of different rank in the parallel.
 *
 *        Verify that we can read from disk correctly using
 *        selections of different rank that H5Sselect_shape_same()
 *        views as being of the same shape.
 *
 *        In this function, we test this by reading checker board
 *        slices of the on disk small data set into slices through
 *        the in memory large data set, and verify that the correct
 *        data (and only the correct data) is read.
 *
 * Return:    void
 *
 *-------------------------------------------------------------------------
 */

#define CHECKER_BOARD_HS_DR_PIO_TEST__D2M_S2L__DEBUG 0

static void
ckrbrd_hs_dr_pio_test__d2m_s2l(struct hs_dr_pio_test_vars_t *tv_ptr)
{
#if CHECKER_BOARD_HS_DR_PIO_TEST__D2M_S2L__DEBUG
    const char *fcnName = "ckrbrd_hs_dr_pio_test__d2m_s2l()";
#endif /* CHECKER_BOARD_HS_DR_PIO_TEST__D2M_S2L__DEBUG */
    bool      data_ok = false;
    int       i, j, k, l;
    size_t    u;
    size_t    start_index;
    size_t    stop_index;
    uint32_t  expected_value;
    uint32_t *ptr_1;
    int       mpi_rank; /* needed by VRFY */
    hsize_t   sel_start[PAR_SS_DR_MAX_RANK];
    htri_t    check; /* Shape comparison return value */
    herr_t    ret;   /* Generic return value */

    /* initialize the local copy of mpi_rank */
    mpi_rank = tv_ptr->mpi_rank;

    /* similarly, read slices of the on disk small data set into slices
     * through the in memory large data set, and verify that the correct
     * data (and only the correct data) is read.
     */

    sel_start[0] = sel_start[1] = sel_start[2] = sel_start[3] = sel_start[4] = 0;
    sel_start[tv_ptr->small_ds_offset]                                       = (hsize_t)(tv_ptr->mpi_rank);

    ckrbrd_hs_dr_pio_test__slct_ckrbrd(tv_ptr->mpi_rank, tv_ptr->file_small_ds_sid_0, tv_ptr->small_rank,
                                       tv_ptr->edge_size, tv_ptr->checker_edge_size, tv_ptr->small_rank - 1,
                                       sel_start);

#if CHECKER_BOARD_HS_DR_PIO_TEST__D2M_S2L__DEBUG
    fprintf(stdout, "%s reading slices of on disk small data set into slices of big data set.\n", fcnName);
#endif /* CHECKER_BOARD_HS_DR_PIO_TEST__D2M_S2L__DEBUG */

    /* zero out the buffer we will be reading into */
    memset(tv_ptr->large_ds_buf_1, 0, sizeof(uint32_t) * tv_ptr->large_ds_size);

    /* set up start, stride, count, and block -- note that we will
     * change start[] so as to read the slice of the small data set
     * into different slices of the process slice of the large data
     * set.
     */
    for (i = 0; i < PAR_SS_DR_MAX_RANK; i++) {

        tv_ptr->start[i]  = 0;
        tv_ptr->stride[i] = (hsize_t)(2 * tv_ptr->edge_size);
        tv_ptr->count[i]  = 1;
        if ((PAR_SS_DR_MAX_RANK - i) > (tv_ptr->small_rank - 1)) {

            tv_ptr->block[i] = 1;
        }
        else {

            tv_ptr->block[i] = (hsize_t)(tv_ptr->edge_size);
        }
    }

    /* in serial versions of this test, we loop through all the dimensions
     * of the large data set that don't appear in the small data set.
     *
     * However, in the parallel version, each process only works with that
     * slice of the large (and small) data set indicated by its rank -- hence
     * we set the most slowly changing index to mpi_rank, and don't iterate
     * over it.
     */

    if (PAR_SS_DR_MAX_RANK - tv_ptr->large_rank == 0) {

        i = tv_ptr->mpi_rank;
    }
    else {

        i = 0;
    }

    /* since large_rank is at most PAR_SS_DR_MAX_RANK, no need to
     * loop over it -- either we are setting i to mpi_rank, or
     * we are setting it to zero.  It will not change during the
     * test.
     */

    if (PAR_SS_DR_MAX_RANK - tv_ptr->large_rank == 1) {

        j = tv_ptr->mpi_rank;
    }
    else {

        j = 0;
    }

    do {
        if (PAR_SS_DR_MAX_RANK - tv_ptr->large_rank == 2) {

            k = tv_ptr->mpi_rank;
        }
        else {

            k = 0;
        }

        do {
            /* since small rank >= 2 and large_rank > small_rank, we
             * have large_rank >= 3.  Since PAR_SS_DR_MAX_RANK == 5
             * (baring major re-orgaization), this gives us:
             *
             *     (PAR_SS_DR_MAX_RANK - large_rank) <= 2
             *
             * so no need to repeat the test in the outer loops --
             * just set l = 0.
             */

            l = 0;
            do {
                if ((tv_ptr->skips)++ < tv_ptr->max_skips) { /* skip the test */

                    (tv_ptr->tests_skipped)++;
                }
                else { /* run the test */

                    tv_ptr->skips = 0; /* reset the skips counter */

                    /* we know that small_rank >= 1 and that large_rank > small_rank
                     * by the assertions at the head of this function.  Thus no
                     * need for another inner loop.
                     */
                    tv_ptr->start[0] = (hsize_t)i;
                    tv_ptr->start[1] = (hsize_t)j;
                    tv_ptr->start[2] = (hsize_t)k;
                    tv_ptr->start[3] = (hsize_t)l;
                    tv_ptr->start[4] = 0;

                    assert((tv_ptr->start[0] == 0) || (0 < tv_ptr->small_ds_offset + 1));
                    assert((tv_ptr->start[1] == 0) || (1 < tv_ptr->small_ds_offset + 1));
                    assert((tv_ptr->start[2] == 0) || (2 < tv_ptr->small_ds_offset + 1));
                    assert((tv_ptr->start[3] == 0) || (3 < tv_ptr->small_ds_offset + 1));
                    assert((tv_ptr->start[4] == 0) || (4 < tv_ptr->small_ds_offset + 1));

                    ckrbrd_hs_dr_pio_test__slct_ckrbrd(
                        tv_ptr->mpi_rank, tv_ptr->mem_large_ds_sid, tv_ptr->large_rank, tv_ptr->edge_size,
                        tv_ptr->checker_edge_size, tv_ptr->small_rank - 1, tv_ptr->start);

                    /* verify that H5Sselect_shape_same() reports the two
                     * selections as having the same shape.
                     */
                    check = H5Sselect_shape_same(tv_ptr->file_small_ds_sid_0, tv_ptr->mem_large_ds_sid);
                    VRFY((check == true), "H5Sselect_shape_same passed");

                    /* Read selection from disk */
#if CHECKER_BOARD_HS_DR_PIO_TEST__D2M_S2L__DEBUG
                    fprintf(stdout, "%s:%d: start = %d %d %d %d %d.\n", fcnName, tv_ptr->mpi_rank,
                            tv_ptr->start[0], tv_ptr->start[1], tv_ptr->start[2], tv_ptr->start[3],
                            tv_ptr->start[4]);
                    fprintf(stdout, "%s:%d: mem/file extent dims = %d/%d.\n", fcnName, tv_ptr->mpi_rank,
                            H5Sget_simple_extent_ndims(tv_ptr->large_ds_slice_sid),
                            H5Sget_simple_extent_ndims(tv_ptr->file_small_ds_sid_0));
#endif /* CHECKER_BOARD_HS_DR_PIO_TEST__D2M_S2L__DEBUG */
                    ret = H5Dread(tv_ptr->small_dataset, H5T_NATIVE_UINT32, tv_ptr->mem_large_ds_sid,
                                  tv_ptr->file_small_ds_sid_0, tv_ptr->xfer_plist, tv_ptr->large_ds_buf_1);
                    VRFY((ret >= 0), "H5Dread() slice from small ds succeeded.");

                    /* verify that the expected data and only the
                     * expected data was read.
                     */
                    data_ok        = true;
                    ptr_1          = tv_ptr->large_ds_buf_1;
                    expected_value = (uint32_t)((size_t)(tv_ptr->mpi_rank) * tv_ptr->small_ds_slice_size);
                    start_index =
                        (size_t)((i * tv_ptr->edge_size * tv_ptr->edge_size * tv_ptr->edge_size *
                                  tv_ptr->edge_size) +
                                 (j * tv_ptr->edge_size * tv_ptr->edge_size * tv_ptr->edge_size) +
                                 (k * tv_ptr->edge_size * tv_ptr->edge_size) + (l * tv_ptr->edge_size));
                    stop_index = start_index + tv_ptr->small_ds_slice_size - 1;

#if CHECKER_BOARD_HS_DR_PIO_TEST__D2M_S2L__DEBUG
                    {
                        int m, n;

                        fprintf(stdout, "%s:%d: expected_value = %d.\n", fcnName, tv_ptr->mpi_rank,
                                expected_value);
                        fprintf(stdout, "%s:%d: start/stop index = %d/%d.\n", fcnName, tv_ptr->mpi_rank,
                                start_index, stop_index);
                        n = 0;
                        for (m = 0; (unsigned)m < tv_ptr->large_ds_size; m++) {
                            fprintf(stdout, "%d ", (int)(*ptr_1));
                            ptr_1++;
                            n++;
                            if (n >= tv_ptr->edge_size) {
                                fprintf(stdout, "\n");
                                n = 0;
                            }
                        }
                        fprintf(stdout, "\n");
                        ptr_1 = tv_ptr->large_ds_buf_1;
                    }
#endif /* CHECKER_BOARD_HS_DR_PIO_TEST__D2M_S2L__DEBUG */

                    assert(start_index < stop_index);
                    assert(stop_index <= tv_ptr->large_ds_size);

                    for (u = 0; u < start_index; u++) {

                        if (*ptr_1 != 0) {

                            data_ok = false;
                        }

                        /* zero out the value for the next pass */
                        *ptr_1 = 0;

                        ptr_1++;
                    }

                    VRFY((data_ok == true), "slice read from small to large ds data good(1).");

                    data_ok = ckrbrd_hs_dr_pio_test__verify_data(ptr_1, tv_ptr->small_rank - 1,
                                                                 tv_ptr->edge_size, tv_ptr->checker_edge_size,
                                                                 expected_value, (bool)true);

                    VRFY((data_ok == true), "slice read from small to large ds data good(2).");

                    ptr_1 = tv_ptr->large_ds_buf_1 + stop_index + 1;

                    for (u = stop_index + 1; u < tv_ptr->large_ds_size; u++) {

                        if (*ptr_1 != 0) {

                            data_ok = false;
                        }

                        /* zero out the value for the next pass */
                        *ptr_1 = 0;

                        ptr_1++;
                    }

                    VRFY((data_ok == true), "slice read from small to large ds data good(3).");

                    (tv_ptr->tests_run)++;
                }

                l++;

                (tv_ptr->total_tests)++;

            } while ((tv_ptr->large_rank > 2) && ((tv_ptr->small_rank - 1) <= 1) && (l < tv_ptr->edge_size));
            k++;
        } while ((tv_ptr->large_rank > 3) && ((tv_ptr->small_rank - 1) <= 2) && (k < tv_ptr->edge_size));
        j++;
    } while ((tv_ptr->large_rank > 4) && ((tv_ptr->small_rank - 1) <= 3) && (j < tv_ptr->edge_size));

    return;

} /* ckrbrd_hs_dr_pio_test__d2m_s2l() */

/*-------------------------------------------------------------------------
 * Function:    ckrbrd_hs_dr_pio_test__m2d_l2s()
 *
 * Purpose:    Part three of a series of tests of I/O to/from checker
 *        board hyperslab selections of different rank in the
 *        parallel.
 *
 *        Verify that we can write from memory to file using checker
 *        board selections of different rank that
 *        H5Sselect_shape_same() views as being of the same shape.
 *
 *        Do this by writing small_rank - 1 dimensional checker
 *        board slices from the in memory large data set to the on
 *        disk small cube dataset.  After each write, read the
 *        slice of the small dataset back from disk, and verify
 *        that it contains the expected data. Verify that
 *        H5Sselect_shape_same() returns true on the memory and
 *        file selections.
 *
 * Return:    void
 *
 *-------------------------------------------------------------------------
 */

#define CHECKER_BOARD_HS_DR_PIO_TEST__M2D_L2S__DEBUG 0

static void
ckrbrd_hs_dr_pio_test__m2d_l2s(struct hs_dr_pio_test_vars_t *tv_ptr)
{
#if CHECKER_BOARD_HS_DR_PIO_TEST__M2D_L2S__DEBUG
    const char *fcnName = "ckrbrd_hs_dr_pio_test__m2d_l2s()";
#endif /* CHECKER_BOARD_HS_DR_PIO_TEST__M2D_L2S__DEBUG */
    bool      data_ok = false;
    int       i, j, k, l;
    size_t    u;
    size_t    start_index;
    size_t    stop_index;
    uint32_t  expected_value;
    uint32_t *ptr_1;
    int       mpi_rank; /* needed by VRFY */
    hsize_t   sel_start[PAR_SS_DR_MAX_RANK];
    htri_t    check; /* Shape comparison return value */
    herr_t    ret;   /* Generic return value */

    /* initialize the local copy of mpi_rank */
    mpi_rank = tv_ptr->mpi_rank;

    /* now we go in the opposite direction, verifying that we can write
     * from memory to file using selections of different rank that
     * H5Sselect_shape_same() views as being of the same shape.
     *
     * Start by writing small_rank - 1 D slices from the in memory large data
     * set to the on disk small dataset.  After each write, read the slice of
     * the small dataset back from disk, and verify that it contains the
     * expected data. Verify that H5Sselect_shape_same() returns true on
     * the memory and file selections.
     */

    tv_ptr->start[0]  = (hsize_t)(tv_ptr->mpi_rank);
    tv_ptr->stride[0] = (hsize_t)(2 * (tv_ptr->mpi_size + 1));
    tv_ptr->count[0]  = 1;
    tv_ptr->block[0]  = 1;

    for (i = 1; i < tv_ptr->large_rank; i++) {

        tv_ptr->start[i]  = 0;
        tv_ptr->stride[i] = (hsize_t)(2 * tv_ptr->edge_size);
        tv_ptr->count[i]  = 1;
        tv_ptr->block[i]  = (hsize_t)(tv_ptr->edge_size);
    }

    ret = H5Sselect_hyperslab(tv_ptr->file_small_ds_sid_0, H5S_SELECT_SET, tv_ptr->start, tv_ptr->stride,
                              tv_ptr->count, tv_ptr->block);
    VRFY((ret >= 0), "H5Sselect_hyperslab(file_small_ds_sid_0, set) succeeded");

    ret = H5Sselect_hyperslab(tv_ptr->mem_small_ds_sid, H5S_SELECT_SET, tv_ptr->start, tv_ptr->stride,
                              tv_ptr->count, tv_ptr->block);
    VRFY((ret >= 0), "H5Sselect_hyperslab(mem_small_ds_sid, set) succeeded");

    sel_start[0] = sel_start[1] = sel_start[2] = sel_start[3] = sel_start[4] = 0;
    sel_start[tv_ptr->small_ds_offset]                                       = (hsize_t)(tv_ptr->mpi_rank);

    ckrbrd_hs_dr_pio_test__slct_ckrbrd(tv_ptr->mpi_rank, tv_ptr->file_small_ds_sid_1, tv_ptr->small_rank,
                                       tv_ptr->edge_size, tv_ptr->checker_edge_size, tv_ptr->small_rank - 1,
                                       sel_start);

    /* set up start, stride, count, and block -- note that we will
     * change start[] so as to read slices of the large cube.
     */
    for (i = 0; i < PAR_SS_DR_MAX_RANK; i++) {

        tv_ptr->start[i]  = 0;
        tv_ptr->stride[i] = (hsize_t)(2 * tv_ptr->edge_size);
        tv_ptr->count[i]  = 1;
        if ((PAR_SS_DR_MAX_RANK - i) > (tv_ptr->small_rank - 1)) {

            tv_ptr->block[i] = 1;
        }
        else {

            tv_ptr->block[i] = (hsize_t)(tv_ptr->edge_size);
        }
    }

    /* zero out the in memory small ds */
    memset(tv_ptr->small_ds_buf_1, 0, sizeof(uint32_t) * tv_ptr->small_ds_size);

#if CHECKER_BOARD_HS_DR_PIO_TEST__M2D_L2S__DEBUG
    fprintf(stdout,
            "%s writing checker boards selections of slices from big ds to slices of small ds on disk.\n",
            fcnName);
#endif /* CHECKER_BOARD_HS_DR_PIO_TEST__M2D_L2S__DEBUG */

    /* in serial versions of this test, we loop through all the dimensions
     * of the large data set that don't appear in the small data set.
     *
     * However, in the parallel version, each process only works with that
     * slice of the large (and small) data set indicated by its rank -- hence
     * we set the most slowly changing index to mpi_rank, and don't iterate
     * over it.
     */

    if (PAR_SS_DR_MAX_RANK - tv_ptr->large_rank == 0) {

        i = tv_ptr->mpi_rank;
    }
    else {

        i = 0;
    }

    /* since large_rank is at most PAR_SS_DR_MAX_RANK, no need to
     * loop over it -- either we are setting i to mpi_rank, or
     * we are setting it to zero.  It will not change during the
     * test.
     */

    if (PAR_SS_DR_MAX_RANK - tv_ptr->large_rank == 1) {

        j = tv_ptr->mpi_rank;
    }
    else {

        j = 0;
    }

    j = 0;
    do {
        if (PAR_SS_DR_MAX_RANK - tv_ptr->large_rank == 2) {

            k = tv_ptr->mpi_rank;
        }
        else {

            k = 0;
        }

        do {
            /* since small rank >= 2 and large_rank > small_rank, we
             * have large_rank >= 3.  Since PAR_SS_DR_MAX_RANK == 5
             * (baring major re-orgaization), this gives us:
             *
             *     (PAR_SS_DR_MAX_RANK - large_rank) <= 2
             *
             * so no need to repeat the test in the outer loops --
             * just set l = 0.
             */

            l = 0;
            do {
                if ((tv_ptr->skips)++ < tv_ptr->max_skips) { /* skip the test */

                    (tv_ptr->tests_skipped)++;
                }
                else { /* run the test */

                    tv_ptr->skips = 0; /* reset the skips counter */

                    /* we know that small_rank >= 1 and that large_rank > small_rank
                     * by the assertions at the head of this function.  Thus no
                     * need for another inner loop.
                     */

                    /* zero out this rank's slice of the on disk small data set */
                    ret = H5Dwrite(tv_ptr->small_dataset, H5T_NATIVE_UINT32, tv_ptr->mem_small_ds_sid,
                                   tv_ptr->file_small_ds_sid_0, tv_ptr->xfer_plist, tv_ptr->small_ds_buf_2);
                    VRFY((ret >= 0), "H5Dwrite() zero slice to small ds succeeded.");

                    /* select the portion of the in memory large cube from which we
                     * are going to write data.
                     */
                    tv_ptr->start[0] = (hsize_t)i;
                    tv_ptr->start[1] = (hsize_t)j;
                    tv_ptr->start[2] = (hsize_t)k;
                    tv_ptr->start[3] = (hsize_t)l;
                    tv_ptr->start[4] = 0;

                    assert((tv_ptr->start[0] == 0) || (0 < tv_ptr->small_ds_offset + 1));
                    assert((tv_ptr->start[1] == 0) || (1 < tv_ptr->small_ds_offset + 1));
                    assert((tv_ptr->start[2] == 0) || (2 < tv_ptr->small_ds_offset + 1));
                    assert((tv_ptr->start[3] == 0) || (3 < tv_ptr->small_ds_offset + 1));
                    assert((tv_ptr->start[4] == 0) || (4 < tv_ptr->small_ds_offset + 1));

                    ckrbrd_hs_dr_pio_test__slct_ckrbrd(
                        tv_ptr->mpi_rank, tv_ptr->mem_large_ds_sid, tv_ptr->large_rank, tv_ptr->edge_size,
                        tv_ptr->checker_edge_size, tv_ptr->small_rank - 1, tv_ptr->start);

                    /* verify that H5Sselect_shape_same() reports the in
                     * memory checkerboard selection of the slice through the
                     * large dataset and the checkerboard selection of the process
                     * slice of the small data set as having the same shape.
                     */
                    check = H5Sselect_shape_same(tv_ptr->file_small_ds_sid_1, tv_ptr->mem_large_ds_sid);
                    VRFY((check == true), "H5Sselect_shape_same passed.");

                    /* write the checker board selection of the slice from the in
                     * memory large data set to the slice of the on disk small
                     * dataset.
                     */
#if CHECKER_BOARD_HS_DR_PIO_TEST__M2D_L2S__DEBUG
                    fprintf(stdout, "%s:%d: start = %d %d %d %d %d.\n", fcnName, tv_ptr->mpi_rank,
                            tv_ptr->start[0], tv_ptr->start[1], tv_ptr->start[2], tv_ptr->start[3],
                            tv_ptr->start[4]);
                    fprintf(stdout, "%s:%d: mem/file extent dims = %d/%d.\n", fcnName, tv_ptr->mpi_rank,
                            H5Sget_simple_extent_ndims(tv_ptr->mem_large_ds_sid),
                            H5Sget_simple_extent_ndims(tv_ptr->file_small_ds_sid_1));
#endif /* CHECKER_BOARD_HS_DR_PIO_TEST__M2D_L2S__DEBUG */
                    ret = H5Dwrite(tv_ptr->small_dataset, H5T_NATIVE_UINT32, tv_ptr->mem_large_ds_sid,
                                   tv_ptr->file_small_ds_sid_1, tv_ptr->xfer_plist, tv_ptr->large_ds_buf_0);
                    VRFY((ret >= 0), "H5Dwrite() slice to large ds succeeded.");

                    /* read the on disk process slice of the small dataset into memory */
                    ret = H5Dread(tv_ptr->small_dataset, H5T_NATIVE_UINT32, tv_ptr->mem_small_ds_sid,
                                  tv_ptr->file_small_ds_sid_0, tv_ptr->xfer_plist, tv_ptr->small_ds_buf_1);
                    VRFY((ret >= 0), "H5Dread() slice from small ds succeeded.");

                    /* verify that expected data is retrieved */

                    expected_value =
                        (uint32_t)((i * tv_ptr->edge_size * tv_ptr->edge_size * tv_ptr->edge_size *
                                    tv_ptr->edge_size) +
                                   (j * tv_ptr->edge_size * tv_ptr->edge_size * tv_ptr->edge_size) +
                                   (k * tv_ptr->edge_size * tv_ptr->edge_size) + (l * tv_ptr->edge_size));

                    start_index = (size_t)(tv_ptr->mpi_rank) * tv_ptr->small_ds_slice_size;
                    stop_index  = start_index + tv_ptr->small_ds_slice_size - 1;

                    assert(start_index < stop_index);
                    assert(stop_index <= tv_ptr->small_ds_size);

                    data_ok = true;

                    ptr_1 = tv_ptr->small_ds_buf_1;
                    for (u = 0; u < start_index; u++, ptr_1++) {

                        if (*ptr_1 != 0) {

                            data_ok = false;
                            *ptr_1  = 0;
                        }
                    }

                    data_ok &= ckrbrd_hs_dr_pio_test__verify_data(
                        tv_ptr->small_ds_buf_1 + start_index, tv_ptr->small_rank - 1, tv_ptr->edge_size,
                        tv_ptr->checker_edge_size, expected_value, (bool)true);

                    ptr_1 = tv_ptr->small_ds_buf_1;
                    for (u = stop_index; u < tv_ptr->small_ds_size; u++, ptr_1++) {

                        if (*ptr_1 != 0) {

                            data_ok = false;
                            *ptr_1  = 0;
                        }
                    }

                    VRFY((data_ok == true), "large slice write slice to small slice data good.");

                    (tv_ptr->tests_run)++;
                }

                l++;

                (tv_ptr->total_tests)++;

            } while ((tv_ptr->large_rank > 2) && ((tv_ptr->small_rank - 1) <= 1) && (l < tv_ptr->edge_size));
            k++;
        } while ((tv_ptr->large_rank > 3) && ((tv_ptr->small_rank - 1) <= 2) && (k < tv_ptr->edge_size));
        j++;
    } while ((tv_ptr->large_rank > 4) && ((tv_ptr->small_rank - 1) <= 3) && (j < tv_ptr->edge_size));

    return;

} /* ckrbrd_hs_dr_pio_test__m2d_l2s() */

/*-------------------------------------------------------------------------
 * Function:    ckrbrd_hs_dr_pio_test__m2d_s2l()
 *
 * Purpose:    Part four of a series of tests of I/O to/from checker
 *        board hyperslab selections of different rank in the parallel.
 *
 *        Verify that we can write from memory to file using
 *        selections of different rank that H5Sselect_shape_same()
 *        views as being of the same shape.
 *
 *        Do this by writing checker board selections of the contents
 *        of the process's slice of the in memory small data set to
 *        slices of the on disk large data set.  After each write,
 *        read the process's slice of the large data set back into
 *        memory, and verify that it contains the expected data.
 *
 *        Verify that H5Sselect_shape_same() returns true on the
 *        memory and file selections.
 *
 * Return:    void
 *
 *-------------------------------------------------------------------------
 */

#define CHECKER_BOARD_HS_DR_PIO_TEST__M2D_S2L__DEBUG 0

static void
ckrbrd_hs_dr_pio_test__m2d_s2l(struct hs_dr_pio_test_vars_t *tv_ptr)
{
#if CHECKER_BOARD_HS_DR_PIO_TEST__M2D_S2L__DEBUG
    const char *fcnName = "ckrbrd_hs_dr_pio_test__m2d_s2l()";
#endif /* CONTIG_HS_DR_PIO_TEST__M2D_S2L__DEBUG */
    bool      data_ok = false;
    int       i, j, k, l;
    size_t    u;
    size_t    start_index;
    size_t    stop_index;
    uint32_t  expected_value;
    uint32_t *ptr_1;
    int       mpi_rank; /* needed by VRFY */
    hsize_t   sel_start[PAR_SS_DR_MAX_RANK];
    htri_t    check; /* Shape comparison return value */
    herr_t    ret;   /* Generic return value */

    /* initialize the local copy of mpi_rank */
    mpi_rank = tv_ptr->mpi_rank;

    /* Now write the contents of the process's slice of the in memory
     * small data set to slices of the on disk large data set.  After
     * each write, read the process's slice of the large data set back
     * into memory, and verify that it contains the expected data.
     * Verify that H5Sselect_shape_same() returns true on the memory
     * and file selections.
     */

    tv_ptr->start[0]  = (hsize_t)(tv_ptr->mpi_rank);
    tv_ptr->stride[0] = (hsize_t)(2 * (tv_ptr->mpi_size + 1));
    tv_ptr->count[0]  = 1;
    tv_ptr->block[0]  = 1;

    for (i = 1; i < tv_ptr->large_rank; i++) {

        tv_ptr->start[i]  = 0;
        tv_ptr->stride[i] = (hsize_t)(2 * tv_ptr->edge_size);
        tv_ptr->count[i]  = 1;
        tv_ptr->block[i]  = (hsize_t)(tv_ptr->edge_size);
    }

    ret = H5Sselect_hyperslab(tv_ptr->file_large_ds_sid_0, H5S_SELECT_SET, tv_ptr->start, tv_ptr->stride,
                              tv_ptr->count, tv_ptr->block);
    VRFY((ret >= 0), "H5Sselect_hyperslab(file_large_ds_sid_0, set) succeeded");

    ret = H5Sselect_hyperslab(tv_ptr->mem_large_ds_sid, H5S_SELECT_SET, tv_ptr->start, tv_ptr->stride,
                              tv_ptr->count, tv_ptr->block);
    VRFY((ret >= 0), "H5Sselect_hyperslab(tv_ptr->mem_large_ds_sid, set) succeeded");

    /* setup a checkerboard selection of the slice of the in memory small
     * data set associated with the process's mpi rank.
     */

    sel_start[0] = sel_start[1] = sel_start[2] = sel_start[3] = sel_start[4] = 0;
    sel_start[tv_ptr->small_ds_offset]                                       = (hsize_t)(tv_ptr->mpi_rank);

    ckrbrd_hs_dr_pio_test__slct_ckrbrd(tv_ptr->mpi_rank, tv_ptr->mem_small_ds_sid, tv_ptr->small_rank,
                                       tv_ptr->edge_size, tv_ptr->checker_edge_size, tv_ptr->small_rank - 1,
                                       sel_start);

    /* set up start, stride, count, and block -- note that we will
     * change start[] so as to write checkerboard selections of slices
     * of the small data set to slices of the large data set.
     */
    for (i = 0; i < PAR_SS_DR_MAX_RANK; i++) {

        tv_ptr->start[i]  = 0;
        tv_ptr->stride[i] = (hsize_t)(2 * tv_ptr->edge_size);
        tv_ptr->count[i]  = 1;
        if ((PAR_SS_DR_MAX_RANK - i) > (tv_ptr->small_rank - 1)) {

            tv_ptr->block[i] = 1;
        }
        else {

            tv_ptr->block[i] = (hsize_t)(tv_ptr->edge_size);
        }
    }

    /* zero out the in memory large ds */
    memset(tv_ptr->large_ds_buf_1, 0, sizeof(uint32_t) * tv_ptr->large_ds_size);

#if CHECKER_BOARD_HS_DR_PIO_TEST__M2D_S2L__DEBUG
    fprintf(stdout,
            "%s writing process checkerboard selections of slices of small ds to process slices of large "
            "ds on disk.\n",
            fcnName);
#endif /* CHECKER_BOARD_HS_DR_PIO_TEST__M2D_S2L__DEBUG */

    if (PAR_SS_DR_MAX_RANK - tv_ptr->large_rank == 0) {

        i = tv_ptr->mpi_rank;
    }
    else {

        i = 0;
    }

    /* since large_rank is at most PAR_SS_DR_MAX_RANK, no need to
     * loop over it -- either we are setting i to mpi_rank, or
     * we are setting it to zero.  It will not change during the
     * test.
     */

    if (PAR_SS_DR_MAX_RANK - tv_ptr->large_rank == 1) {

        j = tv_ptr->mpi_rank;
    }
    else {

        j = 0;
    }

    do {
        if (PAR_SS_DR_MAX_RANK - tv_ptr->large_rank == 2) {

            k = tv_ptr->mpi_rank;
        }
        else {

            k = 0;
        }

        do {
            /* since small rank >= 2 and large_rank > small_rank, we
             * have large_rank >= 3.  Since PAR_SS_DR_MAX_RANK == 5
             * (baring major re-orgaization), this gives us:
             *
             *     (PAR_SS_DR_MAX_RANK - large_rank) <= 2
             *
             * so no need to repeat the test in the outer loops --
             * just set l = 0.
             */

            l = 0;
            do {
                if ((tv_ptr->skips)++ < tv_ptr->max_skips) { /* skip the test */

                    (tv_ptr->tests_skipped)++;
                }
                else { /* run the test */

                    tv_ptr->skips = 0; /* reset the skips counter */

                    /* we know that small_rank >= 1 and that large_rank > small_rank
                     * by the assertions at the head of this function.  Thus no
                     * need for another inner loop.
                     */

                    /* Zero out this processes slice of the on disk large data set.
                     * Note that this will leave one slice with its original data
                     * as there is one more slice than processes.
                     */
                    ret = H5Dwrite(tv_ptr->large_dataset, H5T_NATIVE_UINT32, tv_ptr->mem_large_ds_sid,
                                   tv_ptr->file_large_ds_sid_0, tv_ptr->xfer_plist, tv_ptr->large_ds_buf_2);
                    VRFY((ret != FAIL), "H5Dwrite() to zero large ds succeeded");

                    /* select the portion of the in memory large cube to which we
                     * are going to write data.
                     */
                    tv_ptr->start[0] = (hsize_t)i;
                    tv_ptr->start[1] = (hsize_t)j;
                    tv_ptr->start[2] = (hsize_t)k;
                    tv_ptr->start[3] = (hsize_t)l;
                    tv_ptr->start[4] = 0;

                    assert((tv_ptr->start[0] == 0) || (0 < tv_ptr->small_ds_offset + 1));
                    assert((tv_ptr->start[1] == 0) || (1 < tv_ptr->small_ds_offset + 1));
                    assert((tv_ptr->start[2] == 0) || (2 < tv_ptr->small_ds_offset + 1));
                    assert((tv_ptr->start[3] == 0) || (3 < tv_ptr->small_ds_offset + 1));
                    assert((tv_ptr->start[4] == 0) || (4 < tv_ptr->small_ds_offset + 1));

                    ckrbrd_hs_dr_pio_test__slct_ckrbrd(
                        tv_ptr->mpi_rank, tv_ptr->file_large_ds_sid_1, tv_ptr->large_rank, tv_ptr->edge_size,
                        tv_ptr->checker_edge_size, tv_ptr->small_rank - 1, tv_ptr->start);

                    /* verify that H5Sselect_shape_same() reports the in
                     * memory small data set slice selection and the
                     * on disk slice through the large data set selection
                     * as having the same shape.
                     */
                    check = H5Sselect_shape_same(tv_ptr->mem_small_ds_sid, tv_ptr->file_large_ds_sid_1);
                    VRFY((check == true), "H5Sselect_shape_same passed");

                    /* write the small data set slice from memory to the
                     * target slice of the disk data set
                     */
#if CHECKER_BOARD_HS_DR_PIO_TEST__M2D_S2L__DEBUG
                    fprintf(stdout, "%s:%d: start = %d %d %d %d %d.\n", fcnName, tv_ptr->mpi_rank,
                            tv_ptr->start[0], tv_ptr->start[1], tv_ptr->start[2], tv_ptr->start[3],
                            tv_ptr->start[4]);
                    fprintf(stdout, "%s:%d: mem/file extent dims = %d/%d.\n", fcnName, tv_ptr->mpi_rank,
                            H5Sget_simple_extent_ndims(tv_ptr->mem_small_ds_sid),
                            H5Sget_simple_extent_ndims(tv_ptr->file_large_ds_sid_1));
#endif /* CHECKER_BOARD_HS_DR_PIO_TEST__M2D_S2L__DEBUG */
                    ret = H5Dwrite(tv_ptr->large_dataset, H5T_NATIVE_UINT32, tv_ptr->mem_small_ds_sid,
                                   tv_ptr->file_large_ds_sid_1, tv_ptr->xfer_plist, tv_ptr->small_ds_buf_0);
                    VRFY((ret != FAIL), "H5Dwrite of small ds slice to large ds succeeded");

                    /* read this processes slice on the on disk large
                     * data set into memory.
                     */

                    ret = H5Dread(tv_ptr->large_dataset, H5T_NATIVE_UINT32, tv_ptr->mem_large_ds_sid,
                                  tv_ptr->file_large_ds_sid_0, tv_ptr->xfer_plist, tv_ptr->large_ds_buf_1);
                    VRFY((ret != FAIL), "H5Dread() of process slice of large ds succeeded");

                    /* verify that the expected data and only the
                     * expected data was read.
                     */
                    expected_value = (uint32_t)((size_t)(tv_ptr->mpi_rank) * tv_ptr->small_ds_slice_size);

                    start_index =
                        (size_t)((i * tv_ptr->edge_size * tv_ptr->edge_size * tv_ptr->edge_size *
                                  tv_ptr->edge_size) +
                                 (j * tv_ptr->edge_size * tv_ptr->edge_size * tv_ptr->edge_size) +
                                 (k * tv_ptr->edge_size * tv_ptr->edge_size) + (l * tv_ptr->edge_size));
                    stop_index = start_index + tv_ptr->small_ds_slice_size - 1;

                    assert(start_index < stop_index);
                    assert(stop_index < tv_ptr->large_ds_size);

                    data_ok = true;

                    ptr_1 = tv_ptr->large_ds_buf_1;
                    for (u = 0; u < start_index; u++, ptr_1++) {

                        if (*ptr_1 != 0) {

                            data_ok = false;
                            *ptr_1  = 0;
                        }
                    }

                    data_ok &= ckrbrd_hs_dr_pio_test__verify_data(
                        tv_ptr->large_ds_buf_1 + start_index, tv_ptr->small_rank - 1, tv_ptr->edge_size,
                        tv_ptr->checker_edge_size, expected_value, (bool)true);

                    ptr_1 = tv_ptr->large_ds_buf_1;
                    for (u = stop_index; u < tv_ptr->small_ds_size; u++, ptr_1++) {

                        if (*ptr_1 != 0) {

                            data_ok = false;
                            *ptr_1  = 0;
                        }
                    }

                    VRFY((data_ok == true), "small ds cb slice write to large ds slice data good.");

                    (tv_ptr->tests_run)++;
                }

                l++;

                (tv_ptr->total_tests)++;

            } while ((tv_ptr->large_rank > 2) && ((tv_ptr->small_rank - 1) <= 1) && (l < tv_ptr->edge_size));
            k++;
        } while ((tv_ptr->large_rank > 3) && ((tv_ptr->small_rank - 1) <= 2) && (k < tv_ptr->edge_size));
        j++;
    } while ((tv_ptr->large_rank > 4) && ((tv_ptr->small_rank - 1) <= 3) && (j < tv_ptr->edge_size));

    return;

} /* ckrbrd_hs_dr_pio_test__m2d_s2l() */

/*-------------------------------------------------------------------------
 * Function:    ckrbrd_hs_dr_pio_test__run_test()
 *
 * Purpose:    Test I/O to/from checkerboard selections of hyperslabs of
 *        different rank in the parallel.
 *
 * Return:    void
 *
 *-------------------------------------------------------------------------
 */

#define CKRBRD_HS_DR_PIO_TEST__RUN_TEST__DEBUG 0

static void
ckrbrd_hs_dr_pio_test__run_test(const int test_num, const int edge_size, const int checker_edge_size,
                                const int chunk_edge_size, const int small_rank, const int large_rank,
                                const bool use_collective_io, const hid_t dset_type, const int express_test,
                                int *skips_ptr, int max_skips, int64_t *total_tests_ptr,
                                int64_t *tests_run_ptr, int64_t *tests_skipped_ptr, int mpi_rank)

{
#if CKRBRD_HS_DR_PIO_TEST__RUN_TEST__DEBUG
    const char *fcnName = "ckrbrd_hs_dr_pio_test__run_test()";
#endif /* CKRBRD_HS_DR_PIO_TEST__RUN_TEST__DEBUG */
    struct hs_dr_pio_test_vars_t test_vars = {
        /* int           mpi_size                        = */ -1,
        /* int         mpi_rank                        = */ -1,
        /* MPI_Comm    mpi_comm                        = */ MPI_COMM_NULL,
        /* MPI_Inf     mpi_info                        = */ MPI_INFO_NULL,
        /* int         test_num                        = */ -1,
        /* int         edge_size                       = */ -1,
        /* int         checker_edge_size               = */ -1,
        /* int         chunk_edge_size                 = */ -1,
        /* int         small_rank                      = */ -1,
        /* int         large_rank                      = */ -1,
        /* hid_t       dset_type                       = */ H5I_INVALID_HID,
        /* uint32_t  * small_ds_buf_0                  = */ NULL,
        /* uint32_t  * small_ds_buf_1                  = */ NULL,
        /* uint32_t  * small_ds_buf_2                  = */ NULL,
        /* uint32_t  * small_ds_slice_buf              = */ NULL,
        /* uint32_t  * large_ds_buf_0                  = */ NULL,
        /* uint32_t  * large_ds_buf_1                  = */ NULL,
        /* uint32_t  * large_ds_buf_2                  = */ NULL,
        /* uint32_t  * large_ds_slice_buf              = */ NULL,
        /* int         small_ds_offset                 = */ -1,
        /* int         large_ds_offset                 = */ -1,
        /* hid_t       fid                             = */ H5I_INVALID_HID, /* HDF5 file ID */
        /* hid_t       xfer_plist                      = */ H5P_DEFAULT,
        /* hid_t       full_mem_small_ds_sid           = */ H5I_INVALID_HID,
        /* hid_t       full_file_small_ds_sid          = */ H5I_INVALID_HID,
        /* hid_t       mem_small_ds_sid                = */ H5I_INVALID_HID,
        /* hid_t       file_small_ds_sid_0             = */ H5I_INVALID_HID,
        /* hid_t       file_small_ds_sid_1             = */ H5I_INVALID_HID,
        /* hid_t       small_ds_slice_sid              = */ H5I_INVALID_HID,
        /* hid_t       full_mem_large_ds_sid           = */ H5I_INVALID_HID,
        /* hid_t       full_file_large_ds_sid          = */ H5I_INVALID_HID,
        /* hid_t       mem_large_ds_sid                = */ H5I_INVALID_HID,
        /* hid_t       file_large_ds_sid_0             = */ H5I_INVALID_HID,
        /* hid_t       file_large_ds_sid_1             = */ H5I_INVALID_HID,
        /* hid_t       file_large_ds_process_slice_sid = */ H5I_INVALID_HID,
        /* hid_t       mem_large_ds_process_slice_sid  = */ H5I_INVALID_HID,
        /* hid_t       large_ds_slice_sid              = */ H5I_INVALID_HID,
        /* hid_t       small_dataset                   = */ H5I_INVALID_HID, /* Dataset ID */
        /* hid_t       large_dataset                   = */ H5I_INVALID_HID, /* Dataset ID */
        /* size_t      small_ds_size                   = */ 1,
        /* size_t      small_ds_slice_size             = */ 1,
        /* size_t      large_ds_size                   = */ 1,
        /* size_t      large_ds_slice_size             = */ 1,
        /* hsize_t     dims[PAR_SS_DR_MAX_RANK]        = */ {0, 0, 0, 0, 0},
        /* hsize_t     chunk_dims[PAR_SS_DR_MAX_RANK]  = */ {0, 0, 0, 0, 0},
        /* hsize_t     start[PAR_SS_DR_MAX_RANK]       = */ {0, 0, 0, 0, 0},
        /* hsize_t     stride[PAR_SS_DR_MAX_RANK]      = */ {0, 0, 0, 0, 0},
        /* hsize_t     count[PAR_SS_DR_MAX_RANK]       = */ {0, 0, 0, 0, 0},
        /* hsize_t     block[PAR_SS_DR_MAX_RANK]       = */ {0, 0, 0, 0, 0},
        /* hsize_t   * start_ptr                       = */ NULL,
        /* hsize_t   * stride_ptr                      = */ NULL,
        /* hsize_t   * count_ptr                       = */ NULL,
        /* hsize_t   * block_ptr                       = */ NULL,
        /* int            skips                           = */ 0,
        /* int            max_skips                       = */ 0,
        /* int64_t     total_tests                     = */ 0,
        /* int64_t     tests_run                       = */ 0,
        /* int64_t     tests_skipped                   = */ 0};
    struct hs_dr_pio_test_vars_t *tv_ptr = &test_vars;

    if (MAINPROCESS)
        printf("\r - running test #%lld: small rank = %d, large rank = %d", (long long)(test_num + 1),
               small_rank, large_rank);

    hs_dr_pio_test__setup(test_num, edge_size, checker_edge_size, chunk_edge_size, small_rank, large_rank,
                          use_collective_io, dset_type, express_test, tv_ptr);

    /* initialize skips & max_skips */
    tv_ptr->skips     = *skips_ptr;
    tv_ptr->max_skips = max_skips;

#if CKRBRD_HS_DR_PIO_TEST__RUN_TEST__DEBUG
    if (MAINPROCESS) {
        fprintf(stdout, "test %d: small rank = %d, large rank = %d.\n", test_num, small_rank, large_rank);
        fprintf(stdout, "test %d: Initialization complete.\n", test_num);
    }
#endif /* CKRBRD_HS_DR_PIO_TEST__RUN_TEST__DEBUG */

    /* first, verify that we can read from disk correctly using selections
     * of different rank that H5Sselect_shape_same() views as being of the
     * same shape.
     *
     * Start by reading a (small_rank - 1)-D slice from this processes slice
     * of the on disk large data set, and verifying that the data read is
     * correct.  Verify that H5Sselect_shape_same() returns true on the
     * memory and file selections.
     *
     * The first step is to set up the needed checker board selection in the
     * in memory small small cube
     */

    ckrbrd_hs_dr_pio_test__d2m_l2s(tv_ptr);

    /* similarly, read slices of the on disk small data set into slices
     * through the in memory large data set, and verify that the correct
     * data (and only the correct data) is read.
     */

    ckrbrd_hs_dr_pio_test__d2m_s2l(tv_ptr);

    /* now we go in the opposite direction, verifying that we can write
     * from memory to file using selections of different rank that
     * H5Sselect_shape_same() views as being of the same shape.
     *
     * Start by writing small_rank - 1 D slices from the in memory large data
     * set to the on disk small dataset.  After each write, read the slice of
     * the small dataset back from disk, and verify that it contains the
     * expected data. Verify that H5Sselect_shape_same() returns true on
     * the memory and file selections.
     */

    ckrbrd_hs_dr_pio_test__m2d_l2s(tv_ptr);

    /* Now write the contents of the process's slice of the in memory
     * small data set to slices of the on disk large data set.  After
     * each write, read the process's slice of the large data set back
     * into memory, and verify that it contains the expected data.
     * Verify that H5Sselect_shape_same() returns true on the memory
     * and file selections.
     */

    ckrbrd_hs_dr_pio_test__m2d_s2l(tv_ptr);

#if CKRBRD_HS_DR_PIO_TEST__RUN_TEST__DEBUG
    if (MAINPROCESS) {
        fprintf(stdout, "test %d: Subtests complete -- tests run/skipped/total = %lld/%lld/%lld.\n", test_num,
                (long long)(tv_ptr->tests_run), (long long)(tv_ptr->tests_skipped),
                (long long)(tv_ptr->total_tests));
    }
#endif /* CKRBRD_HS_DR_PIO_TEST__RUN_TEST__DEBUG */

    hs_dr_pio_test__takedown(tv_ptr);

#if CKRBRD_HS_DR_PIO_TEST__RUN_TEST__DEBUG
    if (MAINPROCESS) {
        fprintf(stdout, "test %d: Takedown complete.\n", test_num);
    }
#endif /* CKRBRD_HS_DR_PIO_TEST__RUN_TEST__DEBUG */

    *skips_ptr = tv_ptr->skips;
    *total_tests_ptr += tv_ptr->total_tests;
    *tests_run_ptr += tv_ptr->tests_run;
    *tests_skipped_ptr += tv_ptr->tests_skipped;

    return;

} /* ckrbrd_hs_dr_pio_test__run_test() */

/*-------------------------------------------------------------------------
 * Function:    ckrbrd_hs_dr_pio_test()
 *
 * Purpose:    Test I/O to/from hyperslab selections of different rank in
 *        the parallel case.
 *
 * Return:    void
 *
 *-------------------------------------------------------------------------
 */

static void
ckrbrd_hs_dr_pio_test(ShapeSameTestMethods sstest_type)
{
    int   express_test;
    int   local_express_test;
    int   mpi_size = -1;
    int   mpi_rank = -1;
    int   test_num = 0;
    int   edge_size;
    int   checker_edge_size = 3;
    int   chunk_edge_size   = 0;
    int   small_rank        = 3;
    int   large_rank        = 4;
    int   mpi_result;
    hid_t dset_type = H5T_NATIVE_UINT;
    int   skips     = 0;
    int   max_skips = 0;
    /* The following table list the number of sub-tests skipped between
     * each test that is actually executed as a function of the express
     * test level.  Note that any value in excess of 4880 will cause all
     * sub tests to be skipped.
     */
    int     max_skips_tbl[4] = {0, 4, 64, 1024};
    int64_t total_tests      = 0;
    int64_t tests_run        = 0;
    int64_t tests_skipped    = 0;

    MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
    MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);

    edge_size = (mpi_size > 6 ? mpi_size : 6);

    local_express_test = GetTestExpress();

    HDcompile_assert(sizeof(uint32_t) == sizeof(unsigned));

    mpi_result = MPI_Allreduce((void *)&local_express_test, (void *)&express_test, 1, MPI_INT, MPI_MAX,
                               MPI_COMM_WORLD);

    VRFY((mpi_result == MPI_SUCCESS), "MPI_Allreduce(0) succeeded");

    if (local_express_test < 0) {
        max_skips = max_skips_tbl[0];
    }
    else if (local_express_test > 3) {
        max_skips = max_skips_tbl[3];
    }
    else {
        max_skips = max_skips_tbl[local_express_test];
    }

#if 0
    {
        int DebugWait = 1;

        while (DebugWait) ;
    }
#endif

    for (large_rank = 3; large_rank <= PAR_SS_DR_MAX_RANK; large_rank++) {

        for (small_rank = 2; small_rank < large_rank; small_rank++) {
            switch (sstest_type) {
                case IND_CONTIG:
                    /* contiguous data set, independent I/O */
                    chunk_edge_size = 0;
                    ckrbrd_hs_dr_pio_test__run_test(test_num, edge_size, checker_edge_size, chunk_edge_size,
                                                    small_rank, large_rank, false, dset_type, express_test,
                                                    &skips, max_skips, &total_tests, &tests_run,
                                                    &tests_skipped, mpi_rank);
                    test_num++;
                    break;
                    /* end of case IND_CONTIG */

                case COL_CONTIG:
                    /* contiguous data set, collective I/O */
                    chunk_edge_size = 0;
                    ckrbrd_hs_dr_pio_test__run_test(test_num, edge_size, checker_edge_size, chunk_edge_size,
                                                    small_rank, large_rank, true, dset_type, express_test,
                                                    &skips, max_skips, &total_tests, &tests_run,
                                                    &tests_skipped, mpi_rank);
                    test_num++;
                    break;
                    /* end of case COL_CONTIG */

                case IND_CHUNKED:
                    /* chunked data set, independent I/O */
                    chunk_edge_size = 5;
                    ckrbrd_hs_dr_pio_test__run_test(test_num, edge_size, checker_edge_size, chunk_edge_size,
                                                    small_rank, large_rank, false, dset_type, express_test,
                                                    &skips, max_skips, &total_tests, &tests_run,
                                                    &tests_skipped, mpi_rank);
                    test_num++;
                    break;
                    /* end of case IND_CHUNKED */

                case COL_CHUNKED:
                    /* chunked data set, collective I/O */
                    chunk_edge_size = 5;
                    ckrbrd_hs_dr_pio_test__run_test(test_num, edge_size, checker_edge_size, chunk_edge_size,
                                                    small_rank, large_rank, true, dset_type, express_test,
                                                    &skips, max_skips, &total_tests, &tests_run,
                                                    &tests_skipped, mpi_rank);
                    test_num++;
                    break;
                    /* end of case COL_CHUNKED */

                default:
                    VRFY((false), "unknown test type");
                    break;

            } /* end of switch(sstest_type) */
#if CONTIG_HS_DR_PIO_TEST__DEBUG
            if ((MAINPROCESS) && (tests_skipped > 0)) {
                fprintf(stdout, "     run/skipped/total = %" PRId64 "/%" PRId64 "/%" PRId64 ".\n", tests_run,
                        tests_skipped, total_tests);
            }
#endif /* CONTIG_HS_DR_PIO_TEST__DEBUG */
        }
    }

    if (MAINPROCESS) {
        if (tests_skipped > 0) {
            fprintf(stdout, "     %" PRId64 " of %" PRId64 " subtests skipped to expedite testing.\n",
                    tests_skipped, total_tests);
        }
        else
            printf("\n");
    }

    return;

} /* ckrbrd_hs_dr_pio_test() */

/* Main Body. Here for now, may have to move them to a separated file later. */

/*
 * Main driver of the Parallel HDF5 tests
 */

/* global variables */
int dim0;
int dim1;
int chunkdim0;
int chunkdim1;
int nerrors   = 0;               /* errors count */
int ndatasets = 300;             /* number of datasets to create*/
int ngroups   = 512;             /* number of groups to create in root
                                  * group. */
int facc_type       = FACC_MPIO; /*Test file access type */
int dxfer_coll_type = DXFER_COLLECTIVE_IO;

H5E_auto2_t old_func;        /* previous error handler */
void       *old_client_data; /* previous error handler arg.*/

/* other option flags */

#ifdef USE_PAUSE
/* pause the process for a moment to allow debugger to attach if desired. */
/* Will pause more if greenlight file is not present but will eventually */
/* continue. */
#include <sys/types.h>
#include <sys/stat.h>

void
pause_proc(void)
{

    int       pid;
    h5_stat_t statbuf;
    char      greenlight[] = "go";
    int       maxloop      = 10;
    int       loops        = 0;
    int       time_int     = 10;

    /* mpi variables */
    int  mpi_size, mpi_rank;
    int  mpi_namelen;
    char mpi_name[MPI_MAX_PROCESSOR_NAME];

    pid = getpid();
    MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
    MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
    MPI_Get_processor_name(mpi_name, &mpi_namelen);

    if (MAINPROCESS)
        while ((HDstat(greenlight, &statbuf) == -1) && loops < maxloop) {
            if (!loops++) {
                printf("Proc %d (%*s, %d): to debug, attach %d\n", mpi_rank, mpi_namelen, mpi_name, pid, pid);
            }
            printf("waiting(%ds) for file %s ...\n", time_int, greenlight);
            fflush(stdout);
            HDsleep(time_int);
        }
    MPI_Barrier(MPI_COMM_WORLD);
}

/* Use the Profile feature of MPI to call the pause_proc() */
int
MPI_Init(int *argc, char ***argv)
{
    int ret_code;
    ret_code = PMPI_Init(argc, argv);
    pause_proc();
    return (ret_code);
}
#endif /* USE_PAUSE */

/*
 * Show command usage
 */
static void
usage(void)
{
    printf("    [-r] [-w] [-m<n_datasets>] [-n<n_groups>] "
           "[-o] [-f <prefix>] [-d <dim0> <dim1>]\n");
    printf("\t-m<n_datasets>"
           "\tset number of datasets for the multiple dataset test\n");
    printf("\t-n<n_groups>"
           "\tset number of groups for the multiple group test\n");
    printf("\t-f <prefix>\tfilename prefix\n");
    printf("\t-2\t\tuse Split-file together with MPIO\n");
    printf("\t-d <factor0> <factor1>\tdataset dimensions factors. Defaults (%d,%d)\n", ROW_FACTOR,
           COL_FACTOR);
    printf("\t-c <dim0> <dim1>\tdataset chunk dimensions. Defaults (dim0/10,dim1/10)\n");
    printf("\n");
}

/*
 * parse the command line options
 */
static int
parse_options(int argc, char **argv)
{
    int mpi_size, mpi_rank; /* mpi variables */

    MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
    MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);

    /* setup default chunk-size. Make sure sizes are > 0 */

    chunkdim0 = (dim0 + 9) / 10;
    chunkdim1 = (dim1 + 9) / 10;

    while (--argc) {
        if (**(++argv) != '-') {
            break;
        }
        else {
            switch (*(*argv + 1)) {
                case 'm':
                    ndatasets = atoi((*argv + 1) + 1);
                    if (ndatasets < 0) {
                        nerrors++;
                        return (1);
                    }
                    break;
                case 'n':
                    ngroups = atoi((*argv + 1) + 1);
                    if (ngroups < 0) {
                        nerrors++;
                        return (1);
                    }
                    break;
                case 'f':
                    if (--argc < 1) {
                        nerrors++;
                        return (1);
                    }
                    if (**(++argv) == '-') {
                        nerrors++;
                        return (1);
                    }
                    paraprefix = *argv;
                    break;
                case 'i': /* Collective MPI-IO access with independent IO  */
                    dxfer_coll_type = DXFER_INDEPENDENT_IO;
                    break;
                case '2': /* Use the split-file driver with MPIO access */
                    /* Can use $HDF5_METAPREFIX to define the */
                    /* meta-file-prefix. */
                    facc_type = FACC_MPIO | FACC_SPLIT;
                    break;
                case 'd': /* dimensizes */
                    if (--argc < 2) {
                        nerrors++;
                        return (1);
                    }
                    dim0 = atoi(*(++argv)) * mpi_size;
                    argc--;
                    dim1 = atoi(*(++argv)) * mpi_size;
                    /* set default chunkdim sizes too */
                    chunkdim0 = (dim0 + 9) / 10;
                    chunkdim1 = (dim1 + 9) / 10;
                    break;
                case 'c': /* chunk dimensions */
                    if (--argc < 2) {
                        nerrors++;
                        return (1);
                    }
                    chunkdim0 = atoi(*(++argv));
                    argc--;
                    chunkdim1 = atoi(*(++argv));
                    break;
                case 'h': /* print help message--return with nerrors set */
                    return (1);
                default:
                    if (MAINPROCESS)
                        printf("Illegal option(%s)\n", *argv);
                    nerrors++;
                    return (1);
            }
        }
    } /*while*/

    /* check validity of dimension and chunk sizes */
    if (dim0 <= 0 || dim1 <= 0) {
        if (MAINPROCESS)
            printf("Illegal dim sizes (%d, %d)\n", dim0, dim1);
        nerrors++;
        return (1);
    }
    if (chunkdim0 <= 0 || chunkdim1 <= 0) {
        if (MAINPROCESS)
            printf("Illegal chunkdim sizes (%d, %d)\n", chunkdim0, chunkdim1);
        nerrors++;
        return (1);
    }

    /* Make sure datasets can be divided into equal portions by the processes */
    if ((dim0 % mpi_size) || (dim1 % mpi_size)) {
        if (MAINPROCESS)
            printf("dim0(%d) and dim1(%d) must be multiples of processes(%d)\n", dim0, dim1, mpi_size);
        nerrors++;
        return (1);
    }

    /* compose the test filenames */
    {
        int i, n;

        n = sizeof(FILENAME) / sizeof(FILENAME[0]) - 1; /* exclude the NULL */

        for (i = 0; i < n; i++)
            if (h5_fixname(FILENAME[i], fapl, filenames[i], PATH_MAX) == NULL) {
                printf("h5_fixname failed\n");
                nerrors++;
                return (1);
            }
        if (MAINPROCESS) {
            printf("Test filenames are:\n");
            for (i = 0; i < n; i++)
                printf("    %s\n", filenames[i]);
        }
    }

    return (0);
}

/*
 * Create the appropriate File access property list
 */
hid_t
create_faccess_plist(MPI_Comm comm, MPI_Info info, int l_facc_type)
{
    hid_t  ret_pl = H5I_INVALID_HID;
    herr_t ret;      /* generic return value */
    int    mpi_rank; /* mpi variables */

    /* need the rank for error checking macros */
    MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);

    ret_pl = H5Pcreate(H5P_FILE_ACCESS);
    VRFY((ret_pl >= 0), "H5P_FILE_ACCESS");

    if (l_facc_type == FACC_DEFAULT)
        return (ret_pl);

    if (l_facc_type == FACC_MPIO) {
        /* set Parallel access with communicator */
        ret = H5Pset_fapl_mpio(ret_pl, comm, info);
        VRFY((ret >= 0), "");
        ret = H5Pset_all_coll_metadata_ops(ret_pl, true);
        VRFY((ret >= 0), "");
        ret = H5Pset_coll_metadata_write(ret_pl, true);
        VRFY((ret >= 0), "");
        return (ret_pl);
    }

    if (l_facc_type == (FACC_MPIO | FACC_SPLIT)) {
        hid_t mpio_pl;

        mpio_pl = H5Pcreate(H5P_FILE_ACCESS);
        VRFY((mpio_pl >= 0), "");
        /* set Parallel access with communicator */
        ret = H5Pset_fapl_mpio(mpio_pl, comm, info);
        VRFY((ret >= 0), "");

        /* setup file access template */
        ret_pl = H5Pcreate(H5P_FILE_ACCESS);
        VRFY((ret_pl >= 0), "");
        /* set Parallel access with communicator */
        ret = H5Pset_fapl_split(ret_pl, ".meta", mpio_pl, ".raw", mpio_pl);
        VRFY((ret >= 0), "H5Pset_fapl_split succeeded");
        H5Pclose(mpio_pl);
        return (ret_pl);
    }

    /* unknown file access types */
    return (ret_pl);
}

/* Shape Same test using contiguous hyperslab using independent IO on contiguous datasets */
static void
sscontig1(void)
{
    contig_hs_dr_pio_test(IND_CONTIG);
}

/* Shape Same test using contiguous hyperslab using collective IO on contiguous datasets */
static void
sscontig2(void)
{
    contig_hs_dr_pio_test(COL_CONTIG);
}

/* Shape Same test using contiguous hyperslab using independent IO on chunked datasets */
static void
sscontig3(void)
{
    contig_hs_dr_pio_test(IND_CHUNKED);
}

/* Shape Same test using contiguous hyperslab using collective IO on chunked datasets */
static void
sscontig4(void)
{
    contig_hs_dr_pio_test(COL_CHUNKED);
}

/* Shape Same test using checker hyperslab using independent IO on contiguous datasets */
static void
sschecker1(void)
{
    ckrbrd_hs_dr_pio_test(IND_CONTIG);
}

/* Shape Same test using checker hyperslab using collective IO on contiguous datasets */
static void
sschecker2(void)
{
    ckrbrd_hs_dr_pio_test(COL_CONTIG);
}

/* Shape Same test using checker hyperslab using independent IO on chunked datasets */
static void
sschecker3(void)
{
    ckrbrd_hs_dr_pio_test(IND_CHUNKED);
}

/* Shape Same test using checker hyperslab using collective IO on chunked datasets */
static void
sschecker4(void)
{
    ckrbrd_hs_dr_pio_test(COL_CHUNKED);
}

int
main(int argc, char **argv)
{
    int mpi_size, mpi_rank; /* mpi variables */

#ifndef H5_HAVE_WIN32_API
    /* Un-buffer the stdout and stderr */
    HDsetbuf(stderr, NULL);
    HDsetbuf(stdout, NULL);
#endif

    MPI_Init(&argc, &argv);
    MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
    MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);

    mpi_rank_framework_g = mpi_rank;

    dim0 = ROW_FACTOR * mpi_size;
    dim1 = COL_FACTOR * mpi_size;

    if (MAINPROCESS) {
        printf("===================================\n");
        printf("Shape Same Tests Start\n");
        printf("    express_test = %d.\n", GetTestExpress());
        printf("===================================\n");
    }

    /* Attempt to turn off atexit post processing so that in case errors
     * happen during the test and the process is aborted, it will not get
     * hung in the atexit post processing in which it may try to make MPI
     * calls.  By then, MPI calls may not work.
     */
    if (H5dont_atexit() < 0) {
        if (MAINPROCESS)
            printf("%d: Failed to turn off atexit processing. Continue.\n", mpi_rank);
    };
    H5open();
    h5_show_hostname();

    fapl = H5Pcreate(H5P_FILE_ACCESS);

    /* Get the capability flag of the VOL connector being used */
    if (H5Pget_vol_cap_flags(fapl, &vol_cap_flags_g) < 0) {
        if (MAINPROCESS)
            printf("Failed to get the capability flag of the VOL connector being used\n");

        MPI_Finalize();
        return -1;
    }

    /* Make sure the connector supports the API functions being tested.  This test only
     * uses a few API functions, such as H5Fcreate/close/delete, H5Dcreate/write/read/close,
     */
    if (!(vol_cap_flags_g & H5VL_CAP_FLAG_FILE_BASIC) || !(vol_cap_flags_g & H5VL_CAP_FLAG_DATASET_BASIC)) {
        if (MAINPROCESS)
            printf("API functions for basic file and dataset aren't supported with this connector\n");

        MPI_Finalize();
        return 0;
    }

    memset(filenames, 0, sizeof(filenames));
    for (int i = 0; i < NFILENAME; i++) {
        if (NULL == (filenames[i] = malloc(PATH_MAX))) {
            printf("couldn't allocate filename array\n");
            MPI_Abort(MPI_COMM_WORLD, -1);
        }
    }

    /* Initialize testing framework */
    TestInit(argv[0], usage, parse_options);

    /* Shape Same tests using contiguous hyperslab */
    AddTest("sscontig1", sscontig1, NULL, "Cntg hslab, ind IO, cntg dsets", PARATESTFILE);
    AddTest("sscontig2", sscontig2, NULL, "Cntg hslab, col IO, cntg dsets", PARATESTFILE);
    AddTest("sscontig3", sscontig3, NULL, "Cntg hslab, ind IO, chnk dsets", PARATESTFILE);
    AddTest("sscontig4", sscontig4, NULL, "Cntg hslab, col IO, chnk dsets", PARATESTFILE);

    /* Shape Same tests using checker board hyperslab */
    AddTest("sschecker1", sschecker1, NULL, "Check hslab, ind IO, cntg dsets", PARATESTFILE);
    AddTest("sschecker2", sschecker2, NULL, "Check hslab, col IO, cntg dsets", PARATESTFILE);
    AddTest("sschecker3", sschecker3, NULL, "Check hslab, ind IO, chnk dsets", PARATESTFILE);
    AddTest("sschecker4", sschecker4, NULL, "Check hslab, col IO, chnk dsets", PARATESTFILE);

    /* Display testing information */
    TestInfo(argv[0]);

    /* setup file access property list */
    H5Pset_fapl_mpio(fapl, MPI_COMM_WORLD, MPI_INFO_NULL);

    /* Parse command line arguments */
    TestParseCmdLine(argc, argv);

    if (dxfer_coll_type == DXFER_INDEPENDENT_IO && MAINPROCESS) {
        printf("===================================\n"
               "   Using Independent I/O with file set view to replace collective I/O \n"
               "===================================\n");
    }

    /* Perform requested testing */
    PerformTests();

    /* make sure all processes are finished before final report, cleanup
     * and exit.
     */
    MPI_Barrier(MPI_COMM_WORLD);

    /* Display test summary, if requested */
    if (MAINPROCESS && GetTestSummary())
        TestSummary();

    /* Clean up test files */
    h5_clean_files(FILENAME, fapl);

    H5Pclose(fapl);

    nerrors += GetTestNumErrs();

    /* Gather errors from all processes */
    {
        int temp;
        MPI_Allreduce(&nerrors, &temp, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD);
        nerrors = temp;
    }

    if (MAINPROCESS) { /* only process 0 reports */
        printf("===================================\n");
        if (nerrors)
            printf("***Shape Same tests detected %d errors***\n", nerrors);
        else
            printf("Shape Same tests finished with no errors\n");
        printf("===================================\n");
    }

    for (int i = 0; i < NFILENAME; i++) {
        free(filenames[i]);
        filenames[i] = NULL;
    }

    /* close HDF5 library */
    H5close();

    /* Release test infrastructure */
    TestShutdown();

    MPI_Finalize();

    /* cannot just return (nerrors) because exit code is limited to 1byte */
    return (nerrors != 0);
}