Merge examples and workflows from develop (#3918)

1 files changed, 1100 insertions, 0 deletions

diff --git a/HDF5Examples/C/H5PAR/ph5example.c b/HDF5Examples/C/H5PAR/ph5example.c
new file mode 100644
index 0000000..5ec2cdc
--- /dev/null
+++ b/HDF5Examples/C/H5PAR/ph5example.c
@@ -0,0 +1,1100 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group.                                               *
+ * All rights reserved.                                                      *
+ *                                                                           *
+ * This file is part of HDF5.  The full HDF5 copyright notice, including     *
+ * terms governing use, modification, and redistribution, is contained in    *
+ * the COPYING file, which can be found at the root of the source code       *
+ * distribution tree, or in https://www.hdfgroup.org/licenses.               *
+ * If you do not have access to either file, you may request a copy from     *
+ * help@hdfgroup.org.                                                        *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Example of using the parallel HDF5 library to access datasets.
+ * Last revised: April 24, 2001.
+ *
+ * This program contains two parts.  In the first part, the mpi processes
+ * collectively create a new parallel HDF5 file and create two fixed
+ * dimension datasets in it.  Then each process writes a hyperslab into
+ * each dataset in an independent mode.  All processes collectively
+ * close the datasets and the file.
+ * In the second part, the processes collectively open the created file
+ * and the two datasets in it.  Then each process reads a hyperslab from
+ * each dataset in an independent mode and prints them out.
+ * All processes collectively close the datasets and the file.
+ *
+ * The need of requirement of parallel file prefix is that in general
+ * the current working directory in which compiling is done, is not suitable
+ * for parallel I/O and there is no standard pathname for parallel file
+ * systems.  In some cases, the parallel file name may even needs some
+ * parallel file type prefix such as: "pfs:/GF/...".  Therefore, this
+ * example requires an explicit parallel file prefix.  See the usage
+ * for more detail.
+ */
+
+#include <assert.h>
+#include "hdf5.h"
+#include <string.h>
+#include <stdlib.h>
+
+#ifdef H5_HAVE_PARALLEL
+/* Temporary source code */
+#define FAIL -1
+/* temporary code end */
+
+/* Define some handy debugging shorthands, routines, ... */
+/* debugging tools */
+#define MESG(x)                                                                                              \
+    do {                                                                                                     \
+        if (verbose)                                                                                         \
+            printf("%s\n", x);                                                                               \
+    } while (0)
+
+#define MPI_BANNER(mesg)                                                                                     \
+    do {                                                                                                     \
+        printf("--------------------------------\n");                                                        \
+        printf("Proc %d: ", mpi_rank);                                                                       \
+        printf("*** %s\n", mesg);                                                                            \
+        printf("--------------------------------\n");                                                        \
+    } while (0)
+
+#define SYNC(comm)                                                                                           \
+    do {                                                                                                     \
+        MPI_BANNER("doing a SYNC");                                                                          \
+        MPI_Barrier(comm);                                                                                   \
+        MPI_BANNER("SYNC DONE");                                                                             \
+    } while (0)
+/* End of Define some handy debugging shorthands, routines, ... */
+
+/* Constants definitions */
+/* 24 is a multiple of 2, 3, 4, 6, 8, 12.  Neat for parallel tests. */
+#define SPACE1_DIM1  24
+#define SPACE1_DIM2  24
+#define SPACE1_RANK  2
+#define DATASETNAME1 "Data1"
+#define DATASETNAME2 "Data2"
+#define DATASETNAME3 "Data3"
+/* hyperslab layout styles */
+#define BYROW 1 /* divide into slabs of rows */
+#define BYCOL 2 /* divide into blocks of columns */
+
+#define PARAPREFIX "HDF5_PARAPREFIX" /* file prefix environment variable name */
+
+/* dataset data type.  Int's can be easily octo dumped. */
+typedef int DATATYPE;
+
+/* global variables */
+int nerrors = 0; /* errors count */
+#ifndef PATH_MAX
+#define PATH_MAX 512
+#endif /* !PATH_MAX */
+char testfiles[2][PATH_MAX];
+
+int mpi_size, mpi_rank; /* mpi variables */
+
+/* option flags */
+int verbose   = 0; /* verbose, default as no. */
+int doread    = 1; /* read test */
+int dowrite   = 1; /* write test */
+int docleanup = 1; /* cleanup */
+
+/* Prototypes */
+void slab_set(hsize_t start[], hsize_t count[], hsize_t stride[], int mode);
+void dataset_fill(hsize_t start[], hsize_t count[], hsize_t stride[], DATATYPE *dataset);
+void dataset_print(hsize_t start[], hsize_t count[], hsize_t stride[], DATATYPE *dataset);
+int  dataset_vrfy(hsize_t start[], hsize_t count[], hsize_t stride[], DATATYPE *dataset, DATATYPE *original);
+void phdf5writeInd(char *filename);
+void phdf5readInd(char *filename);
+void phdf5writeAll(char *filename);
+void phdf5readAll(char *filename);
+void test_split_comm_access(char filenames[][PATH_MAX]);
+int  parse_options(int argc, char **argv);
+void usage(void);
+int  mkfilenames(char *prefix);
+void cleanup(void);
+
+/*
+ * Setup the dimensions of the hyperslab.
+ * Two modes--by rows or by columns.
+ * Assume dimension rank is 2.
+ */
+void
+slab_set(hsize_t start[], hsize_t count[], hsize_t stride[], int mode)
+{
+    switch (mode) {
+        case BYROW:
+            /* Each process takes a slabs of rows. */
+            stride[0] = 1;
+            stride[1] = 1;
+            count[0]  = SPACE1_DIM1 / mpi_size;
+            count[1]  = SPACE1_DIM2;
+            start[0]  = mpi_rank * count[0];
+            start[1]  = 0;
+            break;
+        case BYCOL:
+            /* Each process takes a block of columns. */
+            stride[0] = 1;
+            stride[1] = 1;
+            count[0]  = SPACE1_DIM1;
+            count[1]  = SPACE1_DIM2 / mpi_size;
+            start[0]  = 0;
+            start[1]  = mpi_rank * count[1];
+            break;
+        default:
+            /* Unknown mode.  Set it to cover the whole dataset. */
+            printf("unknown slab_set mode (%d)\n", mode);
+            stride[0] = 1;
+            stride[1] = 1;
+            count[0]  = SPACE1_DIM1;
+            count[1]  = SPACE1_DIM2;
+            start[0]  = 0;
+            start[1]  = 0;
+            break;
+    }
+}
+
+/*
+ * Fill the dataset with trivial data for testing.
+ * Assume dimension rank is 2 and data is stored contiguous.
+ */
+void
+dataset_fill(hsize_t start[], hsize_t count[], hsize_t stride[], DATATYPE *dataset)
+{
+    DATATYPE *dataptr = dataset;
+    hsize_t   i, j;
+
+    /* put some trivial data in the data_array */
+    for (i = 0; i < count[0]; i++) {
+        for (j = 0; j < count[1]; j++) {
+            *dataptr++ = (i * stride[0] + start[0]) * 100 + (j * stride[1] + start[1] + 1);
+        }
+    }
+}
+
+/*
+ * Print the content of the dataset.
+ */
+void
+dataset_print(hsize_t start[], hsize_t count[], hsize_t stride[], DATATYPE *dataset)
+{
+    DATATYPE *dataptr = dataset;
+    hsize_t   i, j;
+
+    /* print the slab read */
+    for (i = 0; i < count[0]; i++) {
+        printf("Row %lu: ", (unsigned long)(i * stride[0] + start[0]));
+        for (j = 0; j < count[1]; j++) {
+            printf("%03d ", *dataptr++);
+        }
+        printf("\n");
+    }
+}
+
+/*
+ * Print the content of the dataset.
+ */
+int
+dataset_vrfy(hsize_t start[], hsize_t count[], hsize_t stride[], DATATYPE *dataset, DATATYPE *original)
+{
+#define MAX_ERR_REPORT 10 /* Maximum number of errors reported */
+
+    hsize_t i, j;
+    int     nerr;
+
+    /* print it if verbose */
+    if (verbose)
+        dataset_print(start, count, stride, dataset);
+
+    nerr = 0;
+    for (i = 0; i < count[0]; i++) {
+        for (j = 0; j < count[1]; j++) {
+            if (*dataset++ != *original++) {
+                nerr++;
+                if (nerr <= MAX_ERR_REPORT) {
+                    printf("Dataset Verify failed at [%lu][%lu](row %lu, col %lu): expect %d, got %d\n",
+                           (unsigned long)i, (unsigned long)j, (unsigned long)(i * stride[0] + start[0]),
+                           (unsigned long)(j * stride[1] + start[1]), *(dataset - 1), *(original - 1));
+                }
+            }
+        }
+    }
+    if (nerr > MAX_ERR_REPORT)
+        printf("[more errors ...]\n");
+    if (nerr)
+        printf("%d errors found in dataset_vrfy\n", nerr);
+    return (nerr);
+}
+
+/*
+ * Example of using the parallel HDF5 library to create two datasets
+ * in one HDF5 files with parallel MPIO access support.
+ * The Datasets are of sizes (number-of-mpi-processes x DIM1) x DIM2.
+ * Each process controls only a slab of size DIM1 x DIM2 within each
+ * dataset.
+ */
+
+void
+phdf5writeInd(char *filename)
+{
+    hid_t    fid1;                                            /* HDF5 file IDs */
+    hid_t    acc_tpl1;                                        /* File access templates */
+    hid_t    sid1;                                            /* Dataspace ID */
+    hid_t    file_dataspace;                                  /* File dataspace ID */
+    hid_t    mem_dataspace;                                   /* memory dataspace ID */
+    hid_t    dataset1, dataset2;                              /* Dataset ID */
+    hsize_t  dims1[SPACE1_RANK] = {SPACE1_DIM1, SPACE1_DIM2}; /* dataspace dim sizes */
+    DATATYPE data_array1[SPACE1_DIM1][SPACE1_DIM2];           /* data buffer */
+
+    hsize_t start[SPACE1_RANK];                      /* for hyperslab setting */
+    hsize_t count[SPACE1_RANK], stride[SPACE1_RANK]; /* for hyperslab setting */
+
+    herr_t ret; /* Generic return value */
+
+    MPI_Comm comm = MPI_COMM_WORLD;
+    MPI_Info info = MPI_INFO_NULL;
+
+    if (verbose)
+        printf("Independent write test on file %s\n", filename);
+
+    /* -------------------
+     * START AN HDF5 FILE
+     * -------------------*/
+    /* setup file access template with parallel IO access. */
+    acc_tpl1 = H5Pcreate(H5P_FILE_ACCESS);
+    assert(acc_tpl1 != FAIL);
+    MESG("H5Pcreate access succeed");
+    /* set Parallel access with communicator */
+    ret = H5Pset_fapl_mpio(acc_tpl1, comm, info);
+    assert(ret != FAIL);
+    MESG("H5Pset_fapl_mpio succeed");
+
+    /* create the file collectively */
+    fid1 = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, acc_tpl1);
+    assert(fid1 != FAIL);
+    MESG("H5Fcreate succeed");
+
+    /* Release file-access template */
+    ret = H5Pclose(acc_tpl1);
+    assert(ret != FAIL);
+
+    /* --------------------------
+     * Define the dimensions of the overall datasets
+     * and the slabs local to the MPI process.
+     * ------------------------- */
+    /* setup dimensionality object */
+    sid1 = H5Screate_simple(SPACE1_RANK, dims1, NULL);
+    assert(sid1 != FAIL);
+    MESG("H5Screate_simple succeed");
+
+    /* create a dataset collectively */
+    dataset1 = H5Dcreate2(fid1, DATASETNAME1, H5T_NATIVE_INT, sid1, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+    assert(dataset1 != FAIL);
+    MESG("H5Dcreate2 succeed");
+
+    /* create another dataset collectively */
+    dataset2 = H5Dcreate2(fid1, DATASETNAME2, H5T_NATIVE_INT, sid1, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+    assert(dataset2 != FAIL);
+    MESG("H5Dcreate2 succeed");
+
+    /* set up dimensions of the slab this process accesses */
+    start[0]  = mpi_rank * SPACE1_DIM1 / mpi_size;
+    start[1]  = 0;
+    count[0]  = SPACE1_DIM1 / mpi_size;
+    count[1]  = SPACE1_DIM2;
+    stride[0] = 1;
+    stride[1] = 1;
+    if (verbose)
+        printf("start[]=(%lu,%lu), count[]=(%lu,%lu), total datapoints=%lu\n", (unsigned long)start[0],
+               (unsigned long)start[1], (unsigned long)count[0], (unsigned long)count[1],
+               (unsigned long)(count[0] * count[1]));
+
+    /* put some trivial data in the data_array */
+    dataset_fill(start, count, stride, &data_array1[0][0]);
+    MESG("data_array initialized");
+
+    /* create a file dataspace independently */
+    file_dataspace = H5Dget_space(dataset1);
+    assert(file_dataspace != FAIL);
+    MESG("H5Dget_space succeed");
+    ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, NULL);
+    assert(ret != FAIL);
+    MESG("H5Sset_hyperslab succeed");
+
+    /* create a memory dataspace independently */
+    mem_dataspace = H5Screate_simple(SPACE1_RANK, count, NULL);
+    assert(mem_dataspace != FAIL);
+
+    /* write data independently */
+    ret = H5Dwrite(dataset1, H5T_NATIVE_INT, mem_dataspace, file_dataspace, H5P_DEFAULT, data_array1);
+    assert(ret != FAIL);
+    MESG("H5Dwrite succeed");
+
+    /* write data independently */
+    ret = H5Dwrite(dataset2, H5T_NATIVE_INT, mem_dataspace, file_dataspace, H5P_DEFAULT, data_array1);
+    assert(ret != FAIL);
+    MESG("H5Dwrite succeed");
+
+    /* release dataspace ID */
+    H5Sclose(file_dataspace);
+
+    /* close dataset collectively */
+    ret = H5Dclose(dataset1);
+    assert(ret != FAIL);
+    MESG("H5Dclose1 succeed");
+    ret = H5Dclose(dataset2);
+    assert(ret != FAIL);
+    MESG("H5Dclose2 succeed");
+
+    /* release all IDs created */
+    H5Sclose(sid1);
+
+    /* close the file collectively */
+    H5Fclose(fid1);
+}
+
+/* Example of using the parallel HDF5 library to read a dataset */
+void
+phdf5readInd(char *filename)
+{
+    hid_t    fid1;                                   /* HDF5 file IDs */
+    hid_t    acc_tpl1;                               /* File access templates */
+    hid_t    file_dataspace;                         /* File dataspace ID */
+    hid_t    mem_dataspace;                          /* memory dataspace ID */
+    hid_t    dataset1, dataset2;                     /* Dataset ID */
+    DATATYPE data_array1[SPACE1_DIM1][SPACE1_DIM2];  /* data buffer */
+    DATATYPE data_origin1[SPACE1_DIM1][SPACE1_DIM2]; /* expected data buffer */
+
+    hsize_t start[SPACE1_RANK];                      /* for hyperslab setting */
+    hsize_t count[SPACE1_RANK], stride[SPACE1_RANK]; /* for hyperslab setting */
+
+    herr_t ret; /* Generic return value */
+
+    MPI_Comm comm = MPI_COMM_WORLD;
+    MPI_Info info = MPI_INFO_NULL;
+
+    if (verbose)
+        printf("Independent read test on file %s\n", filename);
+
+    /* setup file access template */
+    acc_tpl1 = H5Pcreate(H5P_FILE_ACCESS);
+    assert(acc_tpl1 != FAIL);
+    /* set Parallel access with communicator */
+    ret = H5Pset_fapl_mpio(acc_tpl1, comm, info);
+    assert(ret != FAIL);
+
+    /* open the file collectively */
+    fid1 = H5Fopen(filename, H5F_ACC_RDWR, acc_tpl1);
+    assert(fid1 != FAIL);
+
+    /* Release file-access template */
+    ret = H5Pclose(acc_tpl1);
+    assert(ret != FAIL);
+
+    /* open the dataset1 collectively */
+    dataset1 = H5Dopen2(fid1, DATASETNAME1, H5P_DEFAULT);
+    assert(dataset1 != FAIL);
+
+    /* open another dataset collectively */
+    dataset2 = H5Dopen2(fid1, DATASETNAME1, H5P_DEFAULT);
+    assert(dataset2 != FAIL);
+
+    /* set up dimensions of the slab this process accesses */
+    start[0]  = mpi_rank * SPACE1_DIM1 / mpi_size;
+    start[1]  = 0;
+    count[0]  = SPACE1_DIM1 / mpi_size;
+    count[1]  = SPACE1_DIM2;
+    stride[0] = 1;
+    stride[1] = 1;
+    if (verbose)
+        printf("start[]=(%lu,%lu), count[]=(%lu,%lu), total datapoints=%lu\n", (unsigned long)start[0],
+               (unsigned long)start[1], (unsigned long)count[0], (unsigned long)count[1],
+               (unsigned long)(count[0] * count[1]));
+
+    /* create a file dataspace independently */
+    file_dataspace = H5Dget_space(dataset1);
+    assert(file_dataspace != FAIL);
+    ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, NULL);
+    assert(ret != FAIL);
+
+    /* create a memory dataspace independently */
+    mem_dataspace = H5Screate_simple(SPACE1_RANK, count, NULL);
+    assert(mem_dataspace != FAIL);
+
+    /* fill dataset with test data */
+    dataset_fill(start, count, stride, &data_origin1[0][0]);
+
+    /* read data independently */
+    ret = H5Dread(dataset1, H5T_NATIVE_INT, mem_dataspace, file_dataspace, H5P_DEFAULT, data_array1);
+    assert(ret != FAIL);
+
+    /* verify the read data with original expected data */
+    ret = dataset_vrfy(start, count, stride, &data_array1[0][0], &data_origin1[0][0]);
+    assert(ret != FAIL);
+
+    /* read data independently */
+    ret = H5Dread(dataset2, H5T_NATIVE_INT, mem_dataspace, file_dataspace, H5P_DEFAULT, data_array1);
+    assert(ret != FAIL);
+
+    /* verify the read data with original expected data */
+    ret = dataset_vrfy(start, count, stride, &data_array1[0][0], &data_origin1[0][0]);
+    assert(ret == 0);
+
+    /* close dataset collectively */
+    ret = H5Dclose(dataset1);
+    assert(ret != FAIL);
+    ret = H5Dclose(dataset2);
+    assert(ret != FAIL);
+
+    /* release all IDs created */
+    H5Sclose(file_dataspace);
+
+    /* close the file collectively */
+    H5Fclose(fid1);
+}
+
+/*
+ * Example of using the parallel HDF5 library to create two datasets
+ * in one HDF5 file with collective parallel access support.
+ * The Datasets are of sizes (number-of-mpi-processes x DIM1) x DIM2.
+ * Each process controls only a slab of size DIM1 x DIM2 within each
+ * dataset. [Note: not so yet.  Datasets are of sizes DIM1xDIM2 and
+ * each process controls a hyperslab within.]
+ */
+
+void
+phdf5writeAll(char *filename)
+{
+    hid_t    fid1;                                            /* HDF5 file IDs */
+    hid_t    acc_tpl1;                                        /* File access templates */
+    hid_t    xfer_plist;                                      /* Dataset transfer properties list */
+    hid_t    sid1;                                            /* Dataspace ID */
+    hid_t    file_dataspace;                                  /* File dataspace ID */
+    hid_t    mem_dataspace;                                   /* memory dataspace ID */
+    hid_t    dataset1, dataset2;                              /* Dataset ID */
+    hsize_t  dims1[SPACE1_RANK] = {SPACE1_DIM1, SPACE1_DIM2}; /* dataspace dim sizes */
+    DATATYPE data_array1[SPACE1_DIM1][SPACE1_DIM2];           /* data buffer */
+
+    hsize_t start[SPACE1_RANK];                      /* for hyperslab setting */
+    hsize_t count[SPACE1_RANK], stride[SPACE1_RANK]; /* for hyperslab setting */
+
+    herr_t ret; /* Generic return value */
+
+    MPI_Comm comm = MPI_COMM_WORLD;
+    MPI_Info info = MPI_INFO_NULL;
+
+    if (verbose)
+        printf("Collective write test on file %s\n", filename);
+
+    /* -------------------
+     * START AN HDF5 FILE
+     * -------------------*/
+    /* setup file access template with parallel IO access. */
+    acc_tpl1 = H5Pcreate(H5P_FILE_ACCESS);
+    assert(acc_tpl1 != FAIL);
+    MESG("H5Pcreate access succeed");
+    /* set Parallel access with communicator */
+    ret = H5Pset_fapl_mpio(acc_tpl1, comm, info);
+    assert(ret != FAIL);
+    MESG("H5Pset_fapl_mpio succeed");
+
+    /* create the file collectively */
+    fid1 = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, acc_tpl1);
+    assert(fid1 != FAIL);
+    MESG("H5Fcreate succeed");
+
+    /* Release file-access template */
+    ret = H5Pclose(acc_tpl1);
+    assert(ret != FAIL);
+
+    /* --------------------------
+     * Define the dimensions of the overall datasets
+     * and create the dataset
+     * ------------------------- */
+    /* setup dimensionality object */
+    sid1 = H5Screate_simple(SPACE1_RANK, dims1, NULL);
+    assert(sid1 != FAIL);
+    MESG("H5Screate_simple succeed");
+
+    /* create a dataset collectively */
+    dataset1 = H5Dcreate2(fid1, DATASETNAME1, H5T_NATIVE_INT, sid1, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+    assert(dataset1 != FAIL);
+    MESG("H5Dcreate2 succeed");
+
+    /* create another dataset collectively */
+    dataset2 = H5Dcreate2(fid1, DATASETNAME2, H5T_NATIVE_INT, sid1, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+    assert(dataset2 != FAIL);
+    MESG("H5Dcreate2 2 succeed");
+
+    /*
+     * Set up dimensions of the slab this process accesses.
+     */
+
+    /* Dataset1: each process takes a block of rows. */
+    slab_set(start, count, stride, BYROW);
+    if (verbose)
+        printf("start[]=(%lu,%lu), count[]=(%lu,%lu), total datapoints=%lu\n", (unsigned long)start[0],
+               (unsigned long)start[1], (unsigned long)count[0], (unsigned long)count[1],
+               (unsigned long)(count[0] * count[1]));
+
+    /* create a file dataspace independently */
+    file_dataspace = H5Dget_space(dataset1);
+    assert(file_dataspace != FAIL);
+    MESG("H5Dget_space succeed");
+    ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, NULL);
+    assert(ret != FAIL);
+    MESG("H5Sset_hyperslab succeed");
+
+    /* create a memory dataspace independently */
+    mem_dataspace = H5Screate_simple(SPACE1_RANK, count, NULL);
+    assert(mem_dataspace != FAIL);
+
+    /* fill the local slab with some trivial data */
+    dataset_fill(start, count, stride, &data_array1[0][0]);
+    MESG("data_array initialized");
+    if (verbose) {
+        MESG("data_array created");
+        dataset_print(start, count, stride, &data_array1[0][0]);
+    }
+
+    /* set up the collective transfer properties list */
+    xfer_plist = H5Pcreate(H5P_DATASET_XFER);
+    assert(xfer_plist != FAIL);
+    ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE);
+    assert(ret != FAIL);
+    MESG("H5Pcreate xfer succeed");
+
+    /* write data collectively */
+    ret = H5Dwrite(dataset1, H5T_NATIVE_INT, mem_dataspace, file_dataspace, xfer_plist, data_array1);
+    assert(ret != FAIL);
+    MESG("H5Dwrite succeed");
+
+    /* release all temporary handles. */
+    /* Could have used them for dataset2 but it is cleaner */
+    /* to create them again.*/
+    H5Sclose(file_dataspace);
+    H5Sclose(mem_dataspace);
+    H5Pclose(xfer_plist);
+
+    /* Dataset2: each process takes a block of columns. */
+    slab_set(start, count, stride, BYCOL);
+    if (verbose)
+        printf("start[]=(%lu,%lu), count[]=(%lu,%lu), total datapoints=%lu\n", (unsigned long)start[0],
+               (unsigned long)start[1], (unsigned long)count[0], (unsigned long)count[1],
+               (unsigned long)(count[0] * count[1]));
+
+    /* put some trivial data in the data_array */
+    dataset_fill(start, count, stride, &data_array1[0][0]);
+    MESG("data_array initialized");
+    if (verbose) {
+        MESG("data_array created");
+        dataset_print(start, count, stride, &data_array1[0][0]);
+    }
+
+    /* create a file dataspace independently */
+    file_dataspace = H5Dget_space(dataset1);
+    assert(file_dataspace != FAIL);
+    MESG("H5Dget_space succeed");
+    ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, NULL);
+    assert(ret != FAIL);
+    MESG("H5Sset_hyperslab succeed");
+
+    /* create a memory dataspace independently */
+    mem_dataspace = H5Screate_simple(SPACE1_RANK, count, NULL);
+    assert(mem_dataspace != FAIL);
+
+    /* fill the local slab with some trivial data */
+    dataset_fill(start, count, stride, &data_array1[0][0]);
+    MESG("data_array initialized");
+    if (verbose) {
+        MESG("data_array created");
+        dataset_print(start, count, stride, &data_array1[0][0]);
+    }
+
+    /* set up the collective transfer properties list */
+    xfer_plist = H5Pcreate(H5P_DATASET_XFER);
+    assert(xfer_plist != FAIL);
+    ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE);
+    assert(ret != FAIL);
+    MESG("H5Pcreate xfer succeed");
+
+    /* write data independently */
+    ret = H5Dwrite(dataset2, H5T_NATIVE_INT, mem_dataspace, file_dataspace, xfer_plist, data_array1);
+    assert(ret != FAIL);
+    MESG("H5Dwrite succeed");
+
+    /* release all temporary handles. */
+    H5Sclose(file_dataspace);
+    H5Sclose(mem_dataspace);
+    H5Pclose(xfer_plist);
+
+    /*
+     * All writes completed.  Close datasets collectively
+     */
+    ret = H5Dclose(dataset1);
+    assert(ret != FAIL);
+    MESG("H5Dclose1 succeed");
+    ret = H5Dclose(dataset2);
+    assert(ret != FAIL);
+    MESG("H5Dclose2 succeed");
+
+    /* release all IDs created */
+    H5Sclose(sid1);
+
+    /* close the file collectively */
+    H5Fclose(fid1);
+}
+
+/*
+ * Example of using the parallel HDF5 library to read two datasets
+ * in one HDF5 file with collective parallel access support.
+ * The Datasets are of sizes (number-of-mpi-processes x DIM1) x DIM2.
+ * Each process controls only a slab of size DIM1 x DIM2 within each
+ * dataset. [Note: not so yet.  Datasets are of sizes DIM1xDIM2 and
+ * each process controls a hyperslab within.]
+ */
+
+void
+phdf5readAll(char *filename)
+{
+    hid_t    fid1;                                   /* HDF5 file IDs */
+    hid_t    acc_tpl1;                               /* File access templates */
+    hid_t    xfer_plist;                             /* Dataset transfer properties list */
+    hid_t    file_dataspace;                         /* File dataspace ID */
+    hid_t    mem_dataspace;                          /* memory dataspace ID */
+    hid_t    dataset1, dataset2;                     /* Dataset ID */
+    DATATYPE data_array1[SPACE1_DIM1][SPACE1_DIM2];  /* data buffer */
+    DATATYPE data_origin1[SPACE1_DIM1][SPACE1_DIM2]; /* expected data buffer */
+
+    hsize_t start[SPACE1_RANK];                      /* for hyperslab setting */
+    hsize_t count[SPACE1_RANK], stride[SPACE1_RANK]; /* for hyperslab setting */
+
+    herr_t ret; /* Generic return value */
+
+    MPI_Comm comm = MPI_COMM_WORLD;
+    MPI_Info info = MPI_INFO_NULL;
+
+    if (verbose)
+        printf("Collective read test on file %s\n", filename);
+
+    /* -------------------
+     * OPEN AN HDF5 FILE
+     * -------------------*/
+    /* setup file access template with parallel IO access. */
+    acc_tpl1 = H5Pcreate(H5P_FILE_ACCESS);
+    assert(acc_tpl1 != FAIL);
+    MESG("H5Pcreate access succeed");
+    /* set Parallel access with communicator */
+    ret = H5Pset_fapl_mpio(acc_tpl1, comm, info);
+    assert(ret != FAIL);
+    MESG("H5Pset_fapl_mpio succeed");
+
+    /* open the file collectively */
+    fid1 = H5Fopen(filename, H5F_ACC_RDWR, acc_tpl1);
+    assert(fid1 != FAIL);
+    MESG("H5Fopen succeed");
+
+    /* Release file-access template */
+    ret = H5Pclose(acc_tpl1);
+    assert(ret != FAIL);
+
+    /* --------------------------
+     * Open the datasets in it
+     * ------------------------- */
+    /* open the dataset1 collectively */
+    dataset1 = H5Dopen2(fid1, DATASETNAME1, H5P_DEFAULT);
+    assert(dataset1 != FAIL);
+    MESG("H5Dopen2 succeed");
+
+    /* open another dataset collectively */
+    dataset2 = H5Dopen2(fid1, DATASETNAME1, H5P_DEFAULT);
+    assert(dataset2 != FAIL);
+    MESG("H5Dopen2 2 succeed");
+
+    /*
+     * Set up dimensions of the slab this process accesses.
+     */
+
+    /* Dataset1: each process takes a block of columns. */
+    slab_set(start, count, stride, BYCOL);
+    if (verbose)
+        printf("start[]=(%lu,%lu), count[]=(%lu,%lu), total datapoints=%lu\n", (unsigned long)start[0],
+               (unsigned long)start[1], (unsigned long)count[0], (unsigned long)count[1],
+               (unsigned long)(count[0] * count[1]));
+
+    /* create a file dataspace independently */
+    file_dataspace = H5Dget_space(dataset1);
+    assert(file_dataspace != FAIL);
+    MESG("H5Dget_space succeed");
+    ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, NULL);
+    assert(ret != FAIL);
+    MESG("H5Sset_hyperslab succeed");
+
+    /* create a memory dataspace independently */
+    mem_dataspace = H5Screate_simple(SPACE1_RANK, count, NULL);
+    assert(mem_dataspace != FAIL);
+
+    /* fill dataset with test data */
+    dataset_fill(start, count, stride, &data_origin1[0][0]);
+    MESG("data_array initialized");
+    if (verbose) {
+        MESG("data_array created");
+        dataset_print(start, count, stride, &data_array1[0][0]);
+    }
+
+    /* set up the collective transfer properties list */
+    xfer_plist = H5Pcreate(H5P_DATASET_XFER);
+    assert(xfer_plist != FAIL);
+    ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE);
+    assert(ret != FAIL);
+    MESG("H5Pcreate xfer succeed");
+
+    /* read data collectively */
+    ret = H5Dread(dataset1, H5T_NATIVE_INT, mem_dataspace, file_dataspace, xfer_plist, data_array1);
+    assert(ret != FAIL);
+    MESG("H5Dread succeed");
+
+    /* verify the read data with original expected data */
+    ret = dataset_vrfy(start, count, stride, &data_array1[0][0], &data_origin1[0][0]);
+    assert(ret != FAIL);
+
+    /* release all temporary handles. */
+    /* Could have used them for dataset2 but it is cleaner */
+    /* to create them again.*/
+    H5Sclose(file_dataspace);
+    H5Sclose(mem_dataspace);
+    H5Pclose(xfer_plist);
+
+    /* Dataset2: each process takes a block of rows. */
+    slab_set(start, count, stride, BYROW);
+    if (verbose)
+        printf("start[]=(%lu,%lu), count[]=(%lu,%lu), total datapoints=%lu\n", (unsigned long)start[0],
+               (unsigned long)start[1], (unsigned long)count[0], (unsigned long)count[1],
+               (unsigned long)(count[0] * count[1]));
+
+    /* create a file dataspace independently */
+    file_dataspace = H5Dget_space(dataset1);
+    assert(file_dataspace != FAIL);
+    MESG("H5Dget_space succeed");
+    ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, NULL);
+    assert(ret != FAIL);
+    MESG("H5Sset_hyperslab succeed");
+
+    /* create a memory dataspace independently */
+    mem_dataspace = H5Screate_simple(SPACE1_RANK, count, NULL);
+    assert(mem_dataspace != FAIL);
+
+    /* fill dataset with test data */
+    dataset_fill(start, count, stride, &data_origin1[0][0]);
+    MESG("data_array initialized");
+    if (verbose) {
+        MESG("data_array created");
+        dataset_print(start, count, stride, &data_array1[0][0]);
+    }
+
+    /* set up the collective transfer properties list */
+    xfer_plist = H5Pcreate(H5P_DATASET_XFER);
+    assert(xfer_plist != FAIL);
+    ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE);
+    assert(ret != FAIL);
+    MESG("H5Pcreate xfer succeed");
+
+    /* read data independently */
+    ret = H5Dread(dataset2, H5T_NATIVE_INT, mem_dataspace, file_dataspace, xfer_plist, data_array1);
+    assert(ret != FAIL);
+    MESG("H5Dread succeed");
+
+    /* verify the read data with original expected data */
+    ret = dataset_vrfy(start, count, stride, &data_array1[0][0], &data_origin1[0][0]);
+    assert(ret != FAIL);
+
+    /* release all temporary handles. */
+    H5Sclose(file_dataspace);
+    H5Sclose(mem_dataspace);
+    H5Pclose(xfer_plist);
+
+    /*
+     * All reads completed.  Close datasets collectively
+     */
+    ret = H5Dclose(dataset1);
+    assert(ret != FAIL);
+    MESG("H5Dclose1 succeed");
+    ret = H5Dclose(dataset2);
+    assert(ret != FAIL);
+    MESG("H5Dclose2 succeed");
+
+    /* close the file collectively */
+    H5Fclose(fid1);
+}
+
+/*
+ * test file access by communicator besides COMM_WORLD.
+ * Split COMM_WORLD into two, one (even_comm) contains the original
+ * processes of even ranks.  The other (odd_comm) contains the original
+ * processes of odd ranks.  Processes in even_comm creates a file, then
+ * cloose it, using even_comm.  Processes in old_comm just do a barrier
+ * using odd_comm.  Then they all do a barrier using COMM_WORLD.
+ * If the file creation and cloose does not do correct collective action
+ * according to the communicator argument, the processes will freeze up
+ * sooner or later due to barrier mixed up.
+ */
+void
+test_split_comm_access(char filenames[][PATH_MAX])
+{
+    MPI_Comm comm;
+    MPI_Info info = MPI_INFO_NULL;
+    int      color, mrc;
+    int      newrank, newprocs;
+    hid_t    fid;     /* file IDs */
+    hid_t    acc_tpl; /* File access properties */
+    herr_t   ret;     /* generic return value */
+
+    if (verbose)
+        printf("Independent write test on file %s %s\n", filenames[0], filenames[1]);
+
+    color = mpi_rank % 2;
+    mrc   = MPI_Comm_split(MPI_COMM_WORLD, color, mpi_rank, &comm);
+    assert(mrc == MPI_SUCCESS);
+    MPI_Comm_size(comm, &newprocs);
+    MPI_Comm_rank(comm, &newrank);
+
+    if (color) {
+        /* odd-rank processes */
+        mrc = MPI_Barrier(comm);
+        assert(mrc == MPI_SUCCESS);
+    }
+    else {
+        /* even-rank processes */
+        /* setup file access template */
+        acc_tpl = H5Pcreate(H5P_FILE_ACCESS);
+        assert(acc_tpl != FAIL);
+
+        /* set Parallel access with communicator */
+        ret = H5Pset_fapl_mpio(acc_tpl, comm, info);
+        assert(ret != FAIL);
+
+        /* create the file collectively */
+        fid = H5Fcreate(filenames[color], H5F_ACC_TRUNC, H5P_DEFAULT, acc_tpl);
+        assert(fid != FAIL);
+        MESG("H5Fcreate succeed");
+
+        /* Release file-access template */
+        ret = H5Pclose(acc_tpl);
+        assert(ret != FAIL);
+
+        ret = H5Fclose(fid);
+        assert(ret != FAIL);
+    }
+    if (mpi_rank == 0) {
+        mrc = MPI_File_delete(filenames[color], info);
+        assert(mrc == MPI_SUCCESS);
+    }
+    MPI_Comm_free(&comm);
+}
+
+/*
+ * Show command usage
+ */
+void
+usage(void)
+{
+    printf("Usage: testphdf5 [-f <prefix>] [-r] [-w] [-v]\n");
+    printf("\t-f\tfile prefix for parallel test files.\n");
+    printf("\t  \t e.g. pfs:/PFS/myname\n");
+    printf("\t  \tcan be set via $" PARAPREFIX ".\n");
+    printf("\t  \tDefault is current directory.\n");
+    printf("\t-c\tno cleanup\n");
+    printf("\t-r\tno read\n");
+    printf("\t-w\tno write\n");
+    printf("\t-v\tverbose on\n");
+    printf("\tdefault do write then read\n");
+    printf("\n");
+}
+
+/*
+ * compose the test filename with the prefix supplied.
+ * return code: 0 if no error
+ *              1 otherwise.
+ */
+int
+mkfilenames(char *prefix)
+{
+    int    i, n;
+    size_t strsize;
+
+    /* filename will be prefix/ParaEgN.h5 where N is 0 to 9. */
+    /* So, string must be big enough to hold the prefix, / and 10 more chars */
+    /* and the terminating null. */
+    strsize = strlen(prefix) + 12;
+    if (strsize > PATH_MAX) {
+        printf("File prefix too long;  Use a short path name.\n");
+        return (1);
+    }
+    n = sizeof(testfiles) / sizeof(testfiles[0]);
+    if (n > 9) {
+        printf("Warning: Too many entries in testfiles. "
+               "Need to adjust the code to accommodate the large size.\n");
+    }
+    for (i = 0; i < n; i++) {
+        snprintf(testfiles[i], PATH_MAX, "%s/ParaEg%d.h5", prefix, i);
+    }
+    return (0);
+}
+
+/*
+ * parse the command line options
+ */
+int
+parse_options(int argc, char **argv)
+{
+    int i, n;
+
+    /* initialize testfiles to nulls */
+    n = sizeof(testfiles) / sizeof(testfiles[0]);
+    for (i = 0; i < n; i++) {
+        testfiles[i][0] = '\0';
+    }
+
+    while (--argc) {
+        if (**(++argv) != '-') {
+            break;
+        }
+        else {
+            switch (*(*argv + 1)) {
+                case 'f':
+                    ++argv;
+                    if (--argc < 1) {
+                        usage();
+                        nerrors++;
+                        return (1);
+                    }
+                    if (mkfilenames(*argv)) {
+                        nerrors++;
+                        return (1);
+                    }
+                    break;
+                case 'c':
+                    docleanup = 0; /* no cleanup */
+                    break;
+                case 'r':
+                    doread = 0;
+                    break;
+                case 'w':
+                    dowrite = 0;
+                    break;
+                case 'v':
+                    verbose = 1;
+                    break;
+                default:
+                    usage();
+                    nerrors++;
+                    return (1);
+            }
+        }
+    }
+
+    /* check the file prefix */
+    if (testfiles[0][0] == '\0') {
+        /* try get it from environment variable HDF5_PARAPREFIX */
+        char *env;
+        char *env_default = "."; /* default to current directory */
+        if ((env = getenv(PARAPREFIX)) == NULL) {
+            env = env_default;
+        }
+        mkfilenames(env);
+    }
+    return (0);
+}
+
+/*
+ * cleanup test files created
+ */
+void
+cleanup(void)
+{
+    int i, n;
+
+    n = sizeof(testfiles) / sizeof(testfiles[0]);
+    for (i = 0; i < n; i++) {
+        MPI_File_delete(testfiles[i], MPI_INFO_NULL);
+    }
+}
+
+/* Main Program */
+int
+main(int argc, char **argv)
+{
+    int  mpi_namelen;
+    char mpi_name[MPI_MAX_PROCESSOR_NAME];
+    int  i, n;
+
+    MPI_Init(&argc, &argv);
+    MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
+    MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
+    MPI_Get_processor_name(mpi_name, &mpi_namelen);
+    /* Make sure datasets can be divided into equal chunks by the processes */
+    if ((SPACE1_DIM1 % mpi_size) || (SPACE1_DIM2 % mpi_size)) {
+        printf("DIM1(%d) and DIM2(%d) must be multiples of processes (%d)\n", SPACE1_DIM1, SPACE1_DIM2,
+               mpi_size);
+        nerrors++;
+        goto finish;
+    }
+
+    if (parse_options(argc, argv) != 0)
+        goto finish;
+
+    /* show test file names */
+    if (mpi_rank == 0) {
+        n = sizeof(testfiles) / sizeof(testfiles[0]);
+        printf("Parallel test files are:\n");
+        for (i = 0; i < n; i++) {
+            printf("   %s\n", testfiles[i]);
+        }
+    }
+
+    if (dowrite) {
+        MPI_BANNER("testing PHDF5 dataset using split communicators...");
+        test_split_comm_access(testfiles);
+        MPI_BANNER("testing PHDF5 dataset independent write...");
+        phdf5writeInd(testfiles[0]);
+        MPI_BANNER("testing PHDF5 dataset collective write...");
+        phdf5writeAll(testfiles[1]);
+    }
+    if (doread) {
+        MPI_BANNER("testing PHDF5 dataset independent read...");
+        phdf5readInd(testfiles[0]);
+        MPI_BANNER("testing PHDF5 dataset collective read...");
+        phdf5readAll(testfiles[1]);
+    }
+
+    if (!(dowrite || doread)) {
+        usage();
+        nerrors++;
+    }
+
+finish:
+    if (mpi_rank == 0) { /* only process 0 reports */
+        if (nerrors)
+            printf("***PHDF5 example detected %d errors***\n", nerrors);
+        else {
+            printf("=====================================\n");
+            printf("PHDF5 example finished with no errors\n");
+            printf("=====================================\n");
+        }
+    }
+    if (docleanup)
+        cleanup();
+    MPI_Finalize();
+
+    return (nerrors);
+}
+
+#else  /* H5_HAVE_PARALLEL */
+/* dummy program since H5_HAVE_PARALLE is not configured in */
+int
+main(void)
+{
+    printf("No PHDF5 example because parallel is not configured in\n");
+    return (0);
+}
+#endif /* H5_HAVE_PARALLEL */