[svn-r24177] update testff for ctests (did not try out)

add bash script examples/run_ff_tests.sh to run all the FF tests

12 files changed, 2587 insertions, 619 deletions

diff --git a/testff/CMakeLists.txt b/testff/CMakeLists.txt
index 4eb2d08..3af9576 100644
--- a/testff/CMakeLists.txt
+++ b/testff/CMakeLists.txt
@@ -64,7 +64,15 @@ ENDMACRO (ADD_H5FF_DRIVER_TEST)
 #-----------------------------------------------------------------------------
 
 ADD_H5FF_DRIVER_TEST(hello hello_server hello_client)
-ADD_H5FF_DRIVER_TEST(client_server h5ff_test_server h5ff_test_client)
+ADD_H5FF_DRIVER_TEST(client_server h5ff_server h5ff_client_attr)
+ADD_H5FF_DRIVER_TEST(client_server h5ff_server h5ff_client_dset)
+ADD_H5FF_DRIVER_TEST(client_server h5ff_server h5ff_client_links)
+ADD_H5FF_DRIVER_TEST(client_server h5ff_server h5ff_client_map)
+ADD_H5FF_DRIVER_TEST(client_server h5ff_server h5ff_client_multiple_cont)
+ADD_H5FF_DRIVER_TEST(client_server h5ff_server h5ff_client_obj)
+ADD_H5FF_DRIVER_TEST(client_server h5ff_server h5ff_client_open)
+ADD_H5FF_DRIVER_TEST(client_server h5ff_server h5ff_client_paths)
+ADD_H5FF_DRIVER_TEST(client_server h5ff_server h5ff_client_trans)
 #SET (H5FF_TESTS
 #    t_mpi
 #    ...
diff --git a/testff/h5ff_client_attr.c b/testff/h5ff_client_attr.c
new file mode 100644
index 0000000..537fb81
--- /dev/null
+++ b/testff/h5ff_client_attr.c
@@ -0,0 +1,334 @@
+/* 
+ * h5ff_client_attr.c: Client side test for attribute routines.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+#include "mpi.h"
+#include "hdf5.h"
+
+int main(int argc, char **argv) {
+    const char file_name[]="eff_file.h5";
+
+    hid_t file_id;
+    hid_t gid1;
+    hid_t sid, dtid;
+    hid_t did1, map;
+    hid_t aid1, aid2, aid3, aid4, aid5;
+    hid_t tid1, tid2, rid1, rid2;
+    hid_t fapl_id, trspl_id;
+    hid_t event_q;
+    hbool_t exists1;
+    hbool_t exists2;
+
+    uint64_t version;
+    uint64_t trans_num;
+
+    int *wdata1 = NULL, *wdata2 = NULL;
+    int *rdata1 = NULL, *rdata2 = NULL;
+    const unsigned int nelem=60;
+    hsize_t dims[1];
+
+    int my_rank, my_size;
+    int provided;
+    MPI_Request mpi_req;
+
+    H5_status_t *status = NULL;
+    int num_requests = 0;
+    H5_request_t req1;
+    H5_status_t status1;
+    unsigned int i = 0;
+    herr_t ret;
+
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
+    if(MPI_THREAD_MULTIPLE != provided) {
+        fprintf(stderr, "MPI does not have MPI_THREAD_MULTIPLE support\n");
+        exit(1);
+    }
+
+    /* Call EFF_init to initialize the EFF stack. */
+    EFF_init(MPI_COMM_WORLD, MPI_INFO_NULL);
+
+    MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &my_size);
+    fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank);
+
+    /* Choose the IOD VOL plugin to use with this file. */
+    fapl_id = H5Pcreate (H5P_FILE_ACCESS);
+    H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);
+
+    /* allocate and initialize arrays for dataset I/O */
+    wdata1 = malloc (sizeof(int32_t)*nelem);
+    wdata2 = malloc (sizeof(int32_t)*nelem);
+    rdata1 = malloc (sizeof(int32_t)*nelem);
+    rdata2 = malloc (sizeof(int32_t)*nelem);
+    for(i=0;i<nelem;++i) {
+        rdata1[i] = 0;
+        rdata2[i] = 0;
+        wdata1[i]=i;
+        wdata2[i]=i*2;
+    }
+
+    /* create an event Queue for managing asynchronous requests. */
+    event_q = H5EQcreate(fapl_id);
+    assert(event_q);
+
+    /* create the file. */
+    file_id = H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);
+    assert(file_id > 0);
+
+    /* create 1-D dataspace with 60 elements */
+    dims [0] = nelem;
+    sid = H5Screate_simple(1, dims, NULL);
+
+    dtid = H5Tcopy(H5T_STD_I32LE);
+
+    /* acquire container version 0 - EXACT */
+    version = 0;
+    rid1 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_QUEUE_NULL);
+    assert(0 == version);
+
+    /* create transaction object */
+    tid1 = H5TRcreate(file_id, rid1, (uint64_t)1);
+    assert(tid1);
+
+    /* start transaction 1 with default num_peers (= 0). 
+       This is asynchronous. */
+    if(0 == my_rank) {
+        ret = H5TRstart(tid1, H5P_DEFAULT, H5_EVENT_QUEUE_NULL);
+        assert(0 == ret);
+    }
+
+    /* Tell other procs that transaction 1 is started */
+    trans_num = 1;
+    MPI_Ibcast(&trans_num, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD, &mpi_req);
+
+    /* Process 0 can continue writing to transaction 1, 
+       while others wait for the bcast to complete */
+    if(0 != my_rank)
+        MPI_Wait(&mpi_req, MPI_STATUS_IGNORE);
+
+    /* create group /G1 */
+    gid1 = H5Gcreate_ff(file_id, "G1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);
+    assert(gid1 > 0);
+
+    /* create dataset /G1/D1 */
+    did1 = H5Dcreate_ff(gid1, "D1", dtid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);
+    assert(did1 > 0);
+
+    /* Commit the datatype dtid to the file. */
+    ret = H5Tcommit_ff(file_id, "DT1", dtid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, 
+                       tid1, event_q);
+    assert(ret == 0);
+
+    map = H5Mcreate_ff(file_id, "MAP1", H5T_STD_I32LE, H5T_STD_I32LE, 
+                        H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT, tid1, event_q);
+    assert(map > 0);
+
+    /* create an attribute on root group */
+    aid1 = H5Acreate_ff(file_id, "ROOT_ATTR", dtid, sid, 
+                        H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);
+    assert(aid1);
+
+    /* create an attribute on group G1. */
+    aid2 = H5Acreate_ff(gid1, "GROUP_ATTR", dtid, sid, 
+                        H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);
+    assert(aid2);
+
+    /* write data to attributes */
+    ret = H5Awrite_ff(aid1, dtid, wdata1, tid1, event_q);
+    assert(ret == 0);
+    ret = H5Awrite_ff(aid2, dtid, wdata2, tid1, event_q);
+    assert(ret == 0);
+
+    /* none leader procs have to complete operations before notifying the leader */
+    if(0 != my_rank) {
+        H5EQwait(event_q, &num_requests, &status);
+        printf("%d requests in event queue. Completions: ", num_requests);
+        for(i=0 ; i<num_requests; i++)
+            fprintf(stderr, "%d ",status[i]);
+        fprintf(stderr, "\n");
+        free(status);
+    }
+
+    /* Barrier to make sure all processes are done writing so Process
+       0 can finish transaction 1 and acquire a read context on it. */
+    MPI_Barrier(MPI_COMM_WORLD);
+
+    if(0 == my_rank) {
+        MPI_Wait(&mpi_req, MPI_STATUS_IGNORE);
+
+        ret = H5TRfinish(tid1, H5P_DEFAULT, &rid2, H5_EVENT_QUEUE_NULL);
+        assert(0 == ret);
+    }
+
+    /* another barrier so other processes know that container version is acquried */
+    MPI_Barrier(MPI_COMM_WORLD);
+
+    /* Local op */
+    ret = H5TRclose(tid1);
+    assert(0 == ret);
+
+    /* close attribute objects */
+    ret = H5Aclose_ff(aid1, event_q);
+    assert(ret == 0);
+    ret = H5Aclose_ff(aid2, event_q);
+    assert(ret == 0);
+
+    /* release container version 0. This is async. */
+    ret = H5RCrelease(rid1, event_q);
+    assert(0 == ret);
+
+    H5EQwait(event_q, &num_requests, &status);
+    printf("%d requests in event queue. Completions: ", num_requests);
+    for(i=0 ; i<num_requests; i++)
+        fprintf(stderr, "%d ",status[i]);
+    fprintf(stderr, "\n");
+    free(status);
+
+    /* Tell other procs that container version 1 is acquired */
+    version = 1;
+    MPI_Bcast(&version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD);
+
+    /* other processes just create a read context object; no need to
+       acquire it */
+    if(0 != my_rank) {
+        rid2 = H5RCcreate(file_id, version);
+        assert(rid2 > 0);
+    }
+
+    ret = H5Aexists_ff(file_id, "ROOT_ATTR", &exists1, rid2, event_q);
+    assert(ret == 0);
+    ret = H5Aexists_by_name_ff(file_id, "G1", "GROUP_ATTR", H5P_DEFAULT, &exists2, rid2, event_q);
+    assert(ret == 0);
+
+    aid1 = H5Aopen_ff(file_id, "ROOT_ATTR", H5P_DEFAULT, rid2, event_q);
+    aid2 = H5Aopen_ff(gid1, "GROUP_ATTR", H5P_DEFAULT, rid2, event_q);
+
+    /* read data from datasets with read version 1. */
+    ret = H5Aread_ff(aid1, dtid, rdata1, rid2, event_q);
+    assert(ret == 0);
+    ret = H5Aread_ff(aid2, dtid, rdata2, rid2, event_q);
+    assert(ret == 0);
+
+    /* close attribute objects */
+    ret = H5Aclose_ff(aid1, event_q);
+    assert(ret == 0);
+    ret = H5Aclose_ff(aid2, event_q);
+    assert(ret == 0);
+
+    /* create & start transaction 2 with num_peers = n */
+    tid2 = H5TRcreate(file_id, rid2, (uint64_t)2);
+    assert(tid2);
+    trspl_id = H5Pcreate (H5P_TR_START);
+    ret = H5Pset_trspl_num_peers(trspl_id, my_size);
+    assert(0 == ret);
+    ret = H5TRstart(tid2, trspl_id, event_q);
+    assert(0 == ret);
+    ret = H5Pclose(trspl_id);
+    assert(0 == ret);
+
+    /* Delete an attribute */
+    ret = H5Adelete_ff(file_id, "ROOT_ATTR", tid2, event_q);
+    assert(ret == 0);
+
+    /* rename an attribute */
+    ret = H5Arename_ff(gid1, "GROUP_ATTR", "RENAMED_GROUP_ATTR", tid2, event_q);
+    assert(ret == 0);
+
+    /* create an attribute on dataset */
+    aid3 = H5Acreate_ff(did1, "DSET_ATTR", dtid, sid, 
+                        H5P_DEFAULT, H5P_DEFAULT, tid2, event_q);
+    assert(aid1);
+
+    /* create an attribute on datatype */
+    aid4 = H5Acreate_ff(dtid, "DTYPE_ATTR", dtid, sid, 
+                        H5P_DEFAULT, H5P_DEFAULT, tid2, event_q);
+    assert(aid2);
+
+    /* create an attribute on dataset */
+    aid5 = H5Acreate_ff(map, "MAP_ATTR", dtid, sid, 
+                        H5P_DEFAULT, H5P_DEFAULT, tid2, event_q);
+    assert(aid1);
+
+    /* finish transaction 2 */
+    ret = H5TRfinish(tid2, H5P_DEFAULT, NULL, event_q);
+    assert(0 == ret);
+
+    /* release container version 1. This is async. */
+    ret = H5RCrelease(rid2, event_q);
+    assert(0 == ret);
+
+    /* close objects */
+    ret = H5Aclose_ff(aid3, event_q);
+    assert(ret == 0);
+    ret = H5Aclose_ff(aid4, event_q);
+    assert(ret == 0);
+    ret = H5Aclose_ff(aid5, event_q);
+    assert(ret == 0);
+    ret = H5Dclose_ff(did1, event_q);
+    assert(ret == 0);
+    ret = H5Mclose_ff(map, event_q);
+    assert(ret == 0);
+    ret = H5Gclose_ff(gid1, event_q);
+    assert(ret == 0);
+
+    ret = H5Sclose(sid);
+    assert(ret == 0);
+    ret = H5Tclose(dtid);
+    assert(ret == 0);
+    ret = H5Pclose(fapl_id);
+    assert(ret == 0);
+
+    H5Fclose_ff(file_id, event_q);
+
+    H5EQwait(event_q, &num_requests, &status);
+    printf("%d requests in event queue. Completions: ", num_requests);
+    for(i=0 ; i<num_requests; i++)
+        fprintf(stderr, "%d ",status[i]);
+    fprintf(stderr, "\n");
+    free(status);
+
+    ret = H5RCclose(rid1);
+    assert(0 == ret);
+    ret = H5RCclose(rid2);
+    assert(0 == ret);
+    ret = H5TRclose(tid2);
+    assert(0 == ret);
+
+    /* Now we can check operations that were issued previously */
+    if(exists1)
+        printf("Attribute ROOT_ATTR exists!\n");
+    else
+        printf("Errr Attribute ROOT_ATTR does NOT exist. \n");
+
+    if(exists2)
+        printf("Attribute GROUP_ATTR exists!\n");
+    else
+        printf("Errr Attribute GROUP_ATTR does NOT exist. \n");
+
+    fprintf(stderr, "Read Data1: ");
+    for(i=0;i<nelem;++i)
+        fprintf(stderr, "%d ",rdata1[i]);
+    fprintf(stderr, "\n");
+
+    fprintf(stderr, "Read Data2: ");
+    for(i=0;i<nelem;++i)
+        fprintf(stderr, "%d ",rdata2[i]);
+    fprintf(stderr, "\n");
+
+    ret = H5EQclose(event_q);
+    assert(ret == 0);
+
+    free(wdata1);
+    free(wdata2);
+    free(rdata1);
+    free(rdata2);
+
+    EFF_finalize();
+    MPI_Finalize();
+
+    return 0;
+}
diff --git a/testff/h5ff_client_dset.c b/testff/h5ff_client_dset.c
new file mode 100644
index 0000000..a595b4c
--- /dev/null
+++ b/testff/h5ff_client_dset.c
@@ -0,0 +1,417 @@
+/* 
+ * h5ff_client_dset.c: Client side test for Dataset routines.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+#include "mpi.h"
+#include "hdf5.h"
+
+int main(int argc, char **argv) {
+    const char file_name[]="eff_file.h5";
+
+    hid_t file_id;
+    hid_t gid1, gid2, gid3;
+    hid_t sid, dtid;
+    hid_t did1, did2, did3;
+    hid_t tid1, tid2, rid1, rid2;
+    hid_t fapl_id, trspl_id, dxpl_id;
+    hid_t event_q;
+
+    uint64_t version;
+    uint64_t trans_num;
+
+    int32_t *wdata1 = NULL, *wdata2 = NULL;
+    int16_t *wdata3 = NULL;
+    int32_t *rdata1 = NULL, *rdata2 = NULL;
+    int16_t *rdata3 = NULL;
+    const unsigned int nelem=60;
+    hsize_t dims[1];
+    hsize_t extent;
+
+    int my_rank, my_size;
+    int provided;
+    MPI_Request mpi_req;
+
+    H5_status_t *status = NULL;
+    int num_requests = 0;
+    unsigned int i = 0;
+    uint32_t cs = 0,read1_cs = 0, read2_cs = 0;
+    uint32_t cs_scope = 0;
+    herr_t ret;
+
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
+    if(MPI_THREAD_MULTIPLE != provided) {
+        fprintf(stderr, "MPI does not have MPI_THREAD_MULTIPLE support\n");
+        exit(1);
+    }
+
+    /* Call EFF_init to initialize the EFF stack. */
+    EFF_init(MPI_COMM_WORLD, MPI_INFO_NULL);
+
+    MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &my_size);
+    fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank);
+
+    /* Choose the IOD VOL plugin to use with this file. */
+    fapl_id = H5Pcreate (H5P_FILE_ACCESS);
+    H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);
+
+    /* allocate and initialize arrays for dataset I/O */
+    wdata1 = malloc (sizeof(int32_t)*nelem);
+    wdata2 = malloc (sizeof(int32_t)*nelem);
+    wdata3 = malloc (sizeof(int16_t)*nelem);
+    rdata1 = malloc (sizeof(int32_t)*nelem);
+    rdata2 = malloc (sizeof(int32_t)*nelem);
+    rdata3 = malloc (sizeof(int16_t)*nelem);
+    for(i=0;i<nelem;++i) {
+        rdata1[i] = 0;
+        rdata2[i] = 0;
+        rdata3[i] = 0;
+        wdata1[i]=i;
+        wdata2[i]=i;
+        wdata3[i]=i;
+    }
+
+    /* create an event Queue for managing asynchronous requests. */
+    event_q = H5EQcreate(fapl_id);
+    assert(event_q);
+
+    /* set the metada data integrity checks to happend at transfer through mercury */
+    cs_scope |= H5_CHECKSUM_TRANSFER;
+    ret = H5Pset_metadata_integrity_scope(fapl_id, cs_scope);
+    assert(ret == 0);
+
+    /* create the file. */
+    file_id = H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);
+    assert(file_id > 0);
+
+    /* create 1-D dataspace with 60 elements */
+    dims [0] = nelem;
+    sid = H5Screate_simple(1, dims, NULL);
+
+    dtid = H5Tcopy(H5T_STD_I32LE);
+
+    /* acquire container version 0 - EXACT */
+    version = 0;
+    rid1 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_QUEUE_NULL);
+    assert(0 == version);
+
+    /* create transaction object */
+    tid1 = H5TRcreate(file_id, rid1, (uint64_t)1);
+    assert(tid1);
+
+    /* start transaction 1 with default num_peers (= 0). */
+    if(0 == my_rank) {
+        ret = H5TRstart(tid1, H5P_DEFAULT, H5_EVENT_QUEUE_NULL);
+        assert(0 == ret);
+    }
+
+    /* Tell other procs that transaction 1 is started */
+    trans_num = 1;
+    MPI_Ibcast(&trans_num, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD, &mpi_req);
+
+    /* Process 0 can continue writing to transaction 1, 
+       while others wait for the bcast to complete */
+    if(0 != my_rank)
+        MPI_Wait(&mpi_req, MPI_STATUS_IGNORE);
+
+    /* create group hierarchy /G1/G2/G3 */
+    gid1 = H5Gcreate_ff(file_id, "G1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);
+    assert(gid1 > 0);
+    gid2 = H5Gcreate_ff(gid1, "G2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);
+    assert(gid2 > 0);
+    gid3 = H5Gcreate_ff(gid2, "G3", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);
+    assert(gid3 > 0);
+
+    /* create datasets */
+    did1 = H5Dcreate_ff(gid1, "D1", dtid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);
+    assert(did1 > 0);
+    did2 = H5Dcreate_ff(gid2, "D2", dtid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);
+    assert(did2 > 0);
+    did3 = H5Dcreate_ff(gid3, "D3", dtid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);
+    assert(did3 > 0);
+
+    /* write data to datasets */
+
+    /* Attach a checksum to the dxpl which is verified all the way
+       down at the server */
+    dxpl_id = H5Pcreate (H5P_DATASET_XFER);
+    cs = H5checksum(wdata1, sizeof(int32_t) * nelem, NULL);
+    H5Pset_dxpl_checksum(dxpl_id, cs);
+
+    /* tell HDF5 to disable all data integrity checks for this write */
+    cs_scope = 0;
+    ret = H5Pset_rawdata_integrity_scope(dxpl_id, cs_scope);
+    assert(ret == 0);
+
+    ret = H5Dwrite_ff(did1, dtid, sid, sid, dxpl_id, wdata1, tid1, event_q);
+    assert(ret == 0);
+
+    /* Raw data write on D2. same as previous, but here we indicate
+       through the property list that we want to inject a
+       corruption. */
+    cs = H5checksum(wdata2, sizeof(int32_t) * nelem, NULL);
+    H5Pset_dxpl_checksum(dxpl_id, cs);
+    H5Pset_dxpl_inject_corruption(dxpl_id, 1);
+
+    /* tell HDF5 to disable data integrity checks stored at IOD for this write;
+       The transfer checksum will still capture the corruption. */
+    cs_scope |= H5_CHECKSUM_IOD;
+    ret = H5Pset_rawdata_integrity_scope(dxpl_id, cs_scope);
+    assert(ret == 0);
+
+    ret = H5Dwrite_ff(did2, dtid, sid, sid, dxpl_id, wdata2, tid1, event_q);
+    assert(ret == 0);
+
+    /* Raw data write on D3. Same as previous; however we specify that
+       the data in the buffer is in BE byte order. Type conversion will
+       happen at the server when we detect that the dataset type is of
+       LE order and the datatype here is in BE order. */
+    ret = H5Dwrite_ff(did3, H5T_STD_I16BE, sid, sid, H5P_DEFAULT, wdata3, tid1, event_q);
+    assert(ret == 0);
+
+    H5Pclose(dxpl_id);
+
+    /* none leader procs have to complete operations before notifying the leader */
+    if(0 != my_rank) {
+        H5EQwait(event_q, &num_requests, &status);
+        printf("%d requests in event queue. Completions: ", num_requests);
+        for(i=0 ; i<num_requests; i++)
+            fprintf(stderr, "%d ",status[i]);
+        fprintf(stderr, "\n");
+        free(status);
+    }
+
+    /* Barrier to make sure all processes are done writing so Process
+       0 can finish transaction 1 and acquire a read context on it. */
+    MPI_Barrier(MPI_COMM_WORLD);
+
+    if(0 == my_rank) {
+        MPI_Wait(&mpi_req, MPI_STATUS_IGNORE);
+
+        /* make this synchronous so we know the container version has been acquired */
+        ret = H5TRfinish(tid1, H5P_DEFAULT, &rid2, H5_EVENT_QUEUE_NULL);
+        assert(0 == ret);
+    }
+
+    /* another barrier so other processes know that container version is acquried */
+    MPI_Barrier(MPI_COMM_WORLD);
+
+    /* Local op */
+    ret = H5TRclose(tid1);
+    assert(0 == ret);
+
+    /* close some objects */
+    ret = H5Dclose_ff(did1, event_q);
+    assert(ret == 0);
+    ret = H5Gclose_ff(gid1, event_q);
+    assert(ret == 0);
+
+    /* release container version 0. This is async. */
+    ret = H5RCrelease(rid1, event_q);
+    assert(0 == ret);
+
+    H5EQwait(event_q, &num_requests, &status);
+    printf("%d requests in event queue. Completions: ", num_requests);
+    for(i=0 ; i<num_requests; i++)
+        fprintf(stderr, "%d ",status[i]);
+    fprintf(stderr, "\n");
+    free(status);
+
+    /* Tell other procs that container version 1 is acquired */
+    version = 1;
+    MPI_Bcast(&version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD);
+
+    /* other processes just create a read context object; no need to
+       acquire it */
+    if(0 != my_rank) {
+        rid2 = H5RCcreate(file_id, version);
+        assert(rid2 > 0);
+    }
+
+    /* Open objects closed before */
+    gid1 = H5Gopen_ff(file_id, "G1", H5P_DEFAULT, rid2, event_q);
+    did1 = H5Dopen_ff(file_id, "G1/D1", H5P_DEFAULT, rid2, event_q);
+
+    /* read data from datasets with read version 1. */
+
+    dxpl_id = H5Pcreate (H5P_DATASET_XFER);
+    /* Give a location to the DXPL to store the checksum once the read has completed */
+    H5Pset_dxpl_checksum_ptr(dxpl_id, &read1_cs);
+    ret = H5Dread_ff(did1, dtid, sid, sid, dxpl_id, rdata1, rid2, event_q);
+    assert(ret == 0);
+    H5Pclose(dxpl_id);
+
+    /* Here we demo that we can pass hints down to the IOD server. 
+       We create a new property, for demo purposes, to tell the server to inject 
+       corrupted data into the received array, and hence an incorrect checksum. 
+       This also detects that we are passing checksum values in both directions for 
+       raw data to ensure data integrity. The read should fail when we wait on it in
+       the H5Dclose(D2) later, but for the demo purposes we are not actually going to 
+       fail the close, but just print a Fatal error. */
+    dxpl_id = H5Pcreate (H5P_DATASET_XFER);
+    H5Pset_dxpl_inject_corruption(dxpl_id, 1);
+    /* Give a location to the DXPL to store the checksum once the read has completed */
+    H5Pset_dxpl_checksum_ptr(dxpl_id, &read2_cs);
+
+    ret = H5Dread_ff(did2, dtid, sid, sid, dxpl_id, rdata2, rid2, event_q);
+    assert(ret == 0);
+    H5Pclose(dxpl_id);
+
+    /* Raw data read on D3. This is asynchronous.  Note that the type
+       is different than the dataset type. */
+    ret = H5Dread_ff(did3, H5T_STD_I16BE, sid, sid, H5P_DEFAULT, rdata3, rid2, event_q);
+    assert(ret == 0);
+
+
+
+    /* Raw data read on D1. This is asynchronous.  The read is done into a 
+       noncontiguous memory dataspace selection */
+    {
+        hid_t mem_space;
+        hsize_t start = 0;
+        hsize_t stride = 2;
+        hsize_t count = nelem;
+        hsize_t block = 1;
+        H5_request_t req1;
+        H5_status_t status1;
+        int *buf = NULL;
+
+        buf = calloc (nelem*2, sizeof(int));
+
+        /* create a dataspace. This is a local Bookeeping operation that 
+           does not touch the file */
+        dims [0] = nelem*2;
+        mem_space = H5Screate_simple(1, dims, NULL);
+        H5Sselect_hyperslab(mem_space, H5S_SELECT_SET, &start,&stride,&count,&block);
+
+        ret = H5Dread_ff(did1, H5T_STD_I32LE, mem_space, sid, H5P_DEFAULT, buf, 
+                         rid2, event_q);
+        assert(ret == 0);
+        H5Sclose(mem_space);
+
+        if(H5EQpop(event_q, &req1) < 0)
+            exit(1);
+        /* wait synchronously on the operation */
+        assert(H5AOwait(req1, &status1) == 0);
+        assert (status1);
+
+        fprintf(stderr, "Printing all Dataset values. We should have a 0 after each element: ");
+        for(i=0;i<120;++i)
+            fprintf(stderr, "%d ", buf[i]);
+        fprintf(stderr, "\n");
+
+        free(buf);
+    }
+
+
+    /* create & start transaction 2 with num_peers = n */
+    tid2 = H5TRcreate(file_id, rid2, (uint64_t)2);
+    assert(tid2);
+    trspl_id = H5Pcreate (H5P_TR_START);
+    ret = H5Pset_trspl_num_peers(trspl_id, my_size);
+    assert(0 == ret);
+    ret = H5TRstart(tid2, trspl_id, event_q);
+    assert(0 == ret);
+    ret = H5Pclose(trspl_id);
+    assert(0 == ret);
+
+    extent = 10;
+    ret = H5Dset_extent_ff(did1, &extent, tid2, event_q);
+    assert(ret == 0);
+
+    extent = 30;
+    ret = H5Dset_extent_ff(did2, &extent, tid2, event_q);
+    assert(ret == 0);
+
+    extent = 60;
+    ret = H5Dset_extent_ff(did3, &extent, tid2, event_q);
+    assert(ret == 0);
+
+    /* finish transaction 2 */
+    ret = H5TRfinish(tid2, H5P_DEFAULT, NULL, event_q);
+    assert(0 == ret);
+
+    /* release container version 1. This is async. */
+    ret = H5RCrelease(rid2, event_q);
+    assert(0 == ret);
+
+    /* close objects */
+    ret = H5Dclose_ff(did1, event_q);
+    assert(ret == 0);
+    ret = H5Dclose_ff(did2, event_q);
+    assert(ret == 0);
+    ret = H5Dclose_ff(did3, event_q);
+    assert(ret == 0);
+    ret = H5Gclose_ff(gid1, event_q);
+    assert(ret == 0);
+    ret = H5Gclose_ff(gid2, event_q);
+    assert(ret == 0);
+    ret = H5Gclose_ff(gid3, event_q);
+    assert(ret == 0);
+
+    ret = H5Sclose(sid);
+    assert(ret == 0);
+    ret = H5Tclose(dtid);
+    assert(ret == 0);
+    ret = H5Pclose(fapl_id);
+    assert(ret == 0);
+
+    H5Fclose_ff(file_id, event_q);
+
+    H5EQwait(event_q, &num_requests, &status);
+    printf("%d requests in event queue. Completions: ", num_requests);
+    for(i=0 ; i<num_requests; i++)
+        fprintf(stderr, "%d ",status[i]);
+    fprintf(stderr, "\n");
+    free(status);
+
+    ret = H5RCclose(rid1);
+    assert(0 == ret);
+    ret = H5RCclose(rid2);
+    assert(0 == ret);
+    ret = H5TRclose(tid2);
+    assert(0 == ret);
+
+    fprintf(stderr, "Read Data1: ");
+    for(i=0;i<nelem;++i)
+        fprintf(stderr, "%d ",rdata1[i]);
+    fprintf(stderr, "\n");
+    fprintf(stderr, 
+            "Checksum Receieved = %u  Checksum Computed = %u (Should be Equal)\n", 
+            read1_cs, cs);
+
+    fprintf(stderr, "Read Data2 (corrupted): ");
+    for(i=0;i<nelem;++i)
+        fprintf(stderr, "%d ",rdata2[i]);
+    fprintf(stderr, "\n");
+    fprintf(stderr, 
+            "Checksum Receieved = %u  Checksum Computed = %u (Should NOT be Equal)\n", 
+            read2_cs, cs);
+
+    assert(read1_cs == cs);
+    assert(read2_cs != cs);
+
+    fprintf(stderr, "Read Data3 (32 LE converted to 16 bit BE order): ");
+    for(i=0;i<nelem;++i)
+        fprintf(stderr, "%d ",rdata3[i]);
+    fprintf(stderr, "\n");
+
+    ret = H5EQclose(event_q);
+    assert(ret == 0);
+
+    free(wdata1);
+    free(wdata2);
+    free(wdata3);
+    free(rdata1);
+    free(rdata2);
+    free(rdata3);
+
+    EFF_finalize();
+    MPI_Finalize();
+
+    return 0;
+}
diff --git a/testff/h5ff_client_links.c b/testff/h5ff_client_links.c
new file mode 100644
index 0000000..261f9f8
--- /dev/null
+++ b/testff/h5ff_client_links.c
@@ -0,0 +1,297 @@
+/* 

+ * test_client_map.c: Client side of Map Object

+ */

+

+#include <stdio.h>

+#include <stdlib.h>

+#include <assert.h>

+#include <string.h>

+#include "mpi.h"

+#include "hdf5.h"

+

+int main(int argc, char **argv) {

+    const char file_name[]="map_file.h5";

+    hid_t file_id;

+    hid_t gid1, gid2, gid3, gid4, gid5;

+    hid_t did1, did2, did3;

+    hid_t sid, dtid;

+    hid_t tid1, tid2, rid1, rid2, rid3;

+    hid_t fapl_id, dxpl_id, trspl_id;

+    hid_t event_q;

+

+    const unsigned int nelem=60;

+    hsize_t dims[1];

+

+    uint64_t version;

+    uint64_t trans_num;

+

+    int my_rank, my_size;

+    int provided;

+    MPI_Request mpi_req;

+

+    H5_status_t *status = NULL;

+    int num_requests = 0, i;

+    herr_t ret;

+

+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);

+    if(MPI_THREAD_MULTIPLE != provided) {

+        fprintf(stderr, "MPI does not have MPI_THREAD_MULTIPLE support\n");

+        exit(1);

+    }

+

+    /* Call EFF_init to initialize the EFF stack.  

+       As a result of this call, the Function Shipper client is started, 

+       and HDF5 VOL calls are registered with the function shipper.

+       An "IOD init" call is forwarded from the FS client to the FS server 

+       which should already be running. */

+    EFF_init(MPI_COMM_WORLD, MPI_INFO_NULL);

+

+    MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);

+    MPI_Comm_size(MPI_COMM_WORLD, &my_size);

+    fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank);

+

+    fprintf(stderr, "Create the FAPL to set the IOD VOL plugin and create the file\n");

+    /* Choose the IOD VOL plugin to use with this file. 

+       First we create a file access property list. Then we call a new routine to set

+       the IOD plugin to use with this fapl */

+    fapl_id = H5Pcreate (H5P_FILE_ACCESS);

+    H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);

+

+    /* create an event Queue for managing asynchronous requests.

+

+       Event Queues will releive the use from managing and completing

+       individual requests for every operation. Instead of passing a

+       request for every operation, the event queue is passed and

+       internally the HDF5 library creates a request and adds it to

+       the event queue.

+

+       Multiple Event queue can be created used by the application. */

+    event_q = H5EQcreate(fapl_id);

+    assert(event_q);

+

+    /* create the file. This is asynchronous, but the file_id can be used. */

+    file_id = H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);

+    assert(file_id);

+

+    /* create 1-D dataspace with 60 elements */

+    dims [0] = nelem;

+    sid = H5Screate_simple(1, dims, NULL);

+    dtid = H5Tcopy(H5T_STD_I32LE);

+

+    /* acquire container version 0 - EXACT */

+    version = 0;

+    rid1 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_QUEUE_NULL);

+    assert(0 == version);

+

+    /* create transaction object */

+    tid1 = H5TRcreate(file_id, rid1, (uint64_t)1);

+    assert(tid1);

+

+    /* start transaction 1 with default num_peers (= 0). */

+    if(0 == my_rank) {

+        ret = H5TRstart(tid1, H5P_DEFAULT, H5_EVENT_QUEUE_NULL);

+        assert(0 == ret);

+    }

+

+    /* Tell other procs that transaction 1 is started */

+    trans_num = 1;

+    MPI_Ibcast(&trans_num, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD, &mpi_req);

+

+    /* Process 0 can continue writing to transaction 1, 

+       while others wait for the bcast to complete */

+    if(0 != my_rank)

+        MPI_Wait(&mpi_req, MPI_STATUS_IGNORE);

+

+    /* create objects */

+    gid1 = H5Gcreate_ff(file_id, "G1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);

+    gid2 = H5Gcreate_ff(gid1, "G2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);

+    gid3 = H5Gcreate_ff(gid2, "G3", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);

+    did1 = H5Dcreate_ff(gid3, "D1", dtid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);

+    did2 = H5Dcreate_ff(gid3, "D2", dtid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);

+    did3 = H5Dcreate_ff(gid3, "D3", dtid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);

+

+    gid4 = H5Gcreate_ff(file_id, "G4", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);

+    gid5 = H5Gcreate_ff(gid4, "G5", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);

+

+    /* create a hard link to D1 (created previosuly) so that it can be

+       accessed from G5/newD1. */

+    ret = H5Lcreate_hard_ff(did1, ".", gid5, "newD1", H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);

+    assert(ret == 0);

+

+    /* none leader procs have to complete operations before notifying the leader */

+    if(0 != my_rank) {

+        H5EQwait(event_q, &num_requests, &status);

+        printf("%d requests in event queue. Completions: ", num_requests);

+        for(i=0 ; i<num_requests; i++)

+            fprintf(stderr, "%d ",status[i]);

+        fprintf(stderr, "\n");

+        free(status);

+    }

+

+    /* Barrier to make sure all processes are done writing so Process

+       0 can finish transaction 1 and acquire a read context on it. */

+    MPI_Barrier(MPI_COMM_WORLD);

+

+    if(0 == my_rank) {

+        MPI_Wait(&mpi_req, MPI_STATUS_IGNORE);

+

+        ret = H5TRfinish(tid1, H5P_DEFAULT, &rid2, H5_EVENT_QUEUE_NULL);

+        assert(0 == ret);

+    }

+

+    /* another barrier so other processes know that container version is acquried */

+    MPI_Barrier(MPI_COMM_WORLD);

+

+    /* Close transaction object. Local op */

+    ret = H5TRclose(tid1);

+    assert(0 == ret);

+

+    /* release container version 0. This is async. */

+    ret = H5RCrelease(rid1, event_q);

+    assert(0 == ret);

+

+    assert(H5Dclose_ff(did2, event_q) == 0);

+

+    H5EQwait(event_q, &num_requests, &status);

+    printf("%d requests in event queue. Completions: ", num_requests);

+    for(i=0 ; i<num_requests; i++)

+        fprintf(stderr, "%d ",status[i]);

+    fprintf(stderr, "\n");

+    free(status);

+

+    /* Process 0 tells other procs that container version 1 is acquired */

+    version = 1;

+    MPI_Bcast(&version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD);

+

+    /* other processes just create a read context object; no need to

+       acquire it */

+    if(0 != my_rank) {

+        rid2 = H5RCcreate(file_id, version);

+        assert(rid2 > 0);

+    }

+

+    /* Try and open the dataset. This is asynchronous. */

+    did2 = H5Dopen_ff(file_id,"G4/G5/newD1", H5P_DEFAULT, rid2, event_q);

+    assert(did2);

+

+    /* create & start transaction 2 with num_peers = n */

+    tid2 = H5TRcreate(file_id, rid2, (uint64_t)2);

+    assert(tid2);

+    trspl_id = H5Pcreate (H5P_TR_START);

+    ret = H5Pset_trspl_num_peers(trspl_id, my_size);

+    assert(0 == ret);

+    ret = H5TRstart(tid2, trspl_id, event_q);

+    assert(0 == ret);

+    ret = H5Pclose(trspl_id);

+    assert(0 == ret);

+

+    ret = H5Lcreate_soft_ff("/G1/G2/G3/D4", gid4, "G5/newD2", 

+                            H5P_DEFAULT, H5P_DEFAULT, tid2, event_q);

+    assert(ret == 0);

+

+    ret = H5Lmove_ff(gid3, "D3", file_id, "/G4/G5/D3moved", 

+                     H5P_DEFAULT, H5P_DEFAULT, tid2, event_q);

+    assert(ret == 0);

+

+    ret = H5Ldelete_ff(file_id, "/G1/G2/G3/D2", H5P_DEFAULT, tid2, event_q);

+    assert(ret == 0);

+

+    /* finish transaction 2 */

+    ret = H5TRfinish(tid2, H5P_DEFAULT, NULL, event_q);

+    assert(0 == ret);

+

+    /* release container version 1. This is async. */

+    ret = H5RCrelease(rid2, event_q);

+    assert(0 == ret);

+

+    assert(H5Gclose_ff(gid1, event_q) == 0);

+    assert(H5Gclose_ff(gid2, event_q) == 0);

+    assert(H5Gclose_ff(gid3, event_q) == 0);

+    assert(H5Gclose_ff(gid5, event_q) == 0);

+

+    assert(H5Dclose_ff(did1, event_q) == 0);

+    assert(H5Dclose_ff(did2, event_q) == 0);

+    assert(H5Dclose_ff(did3, event_q) == 0);

+

+    H5EQwait(event_q, &num_requests, &status);

+    printf("%d requests in event queue. Completions: ", num_requests);

+    for(i=0 ; i<num_requests; i++)

+        fprintf(stderr, "%d ",status[i]);

+    fprintf(stderr, "\n");

+    free(status);

+

+    ret = H5RCclose(rid1);

+    assert(0 == ret);

+    ret = H5RCclose(rid2);

+    assert(0 == ret);

+    ret = H5TRclose(tid2);

+    assert(0 == ret);

+

+    /* acquire container version 2 - EXACT */

+    version = 2;

+    rid3 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_QUEUE_NULL);

+    assert(2 == version);

+

+    {

+        H5L_ff_info_t linfo;

+        char *link_buf;

+

+        ret = H5Lget_info_ff(gid4, "G5/newD2", &linfo, H5P_DEFAULT, rid3, H5_EVENT_QUEUE_NULL);

+        assert(ret == 0);

+

+        switch(linfo.type) {

+        case H5L_TYPE_SOFT:

+            fprintf(stderr, 

+                    "LINK GET INFO return a SOFT LINK with value size: %zu\n", 

+                    linfo.u.val_size);

+            break;

+        default:

+            fprintf(stderr, "Unexpected result from lget_info\n");

+            exit(1);

+        }

+

+        link_buf = malloc(linfo.u.val_size);

+

+        ret = H5Lget_val_ff(gid4, "G5/newD2", link_buf, linfo.u.val_size, 

+                            H5P_DEFAULT, rid3, H5_EVENT_QUEUE_NULL);

+        assert(ret == 0);

+

+        fprintf(stderr, "Link value = %s\n", link_buf);

+

+        free(link_buf);

+    }

+

+    /* release container version 1. This is async. */

+    ret = H5RCrelease(rid3, event_q);

+    assert(0 == ret);

+

+    assert(H5Gclose_ff(gid4, event_q) == 0);

+

+    /* closing the container also acts as a wait all on all pending requests 

+       on the container. */

+    assert(H5Fclose_ff(file_id, event_q) == 0);

+

+    /* wait on all requests and print completion status */

+    H5EQwait(event_q, &num_requests, &status);

+    fprintf(stderr, "%d requests in event queue. Completions: ", num_requests);

+    for(i=0 ; i<num_requests; i++)

+        fprintf(stderr, "%d ",status[i]);

+    fprintf(stderr, "\n");

+    free(status);

+

+    ret = H5RCclose(rid3);

+    assert(0 == ret);

+

+    H5Sclose(sid);

+    H5Tclose(dtid);

+    H5Pclose(fapl_id);

+    H5EQclose(event_q);

+

+    /* This finalizes the EFF stack. ships a terminate and IOD finalize to the server 

+       and shutsdown the FS server (when all clients send the terminate request) 

+       and client */

+    EFF_finalize();

+

+    MPI_Finalize();

+    return 0;

+}

diff --git a/testff/h5ff_client_map.c b/testff/h5ff_client_map.c
new file mode 100644
index 0000000..b471e63
--- /dev/null
+++ b/testff/h5ff_client_map.c
@@ -0,0 +1,395 @@
+/* 

+ * test_client_map.c: Client side of Map Object

+ */

+

+#include <stdio.h>

+#include <stdlib.h>

+#include <assert.h>

+#include <string.h>

+#include "mpi.h"

+#include "hdf5.h"

+

+int main(int argc, char **argv) {

+    const char file_name[]="map_file.h5";

+    hid_t file_id;

+    hid_t gid1, gid2, dtid1, dtid2;

+    hid_t map1, map2, map3;

+    hid_t tid1, tid2, rid1, rid2, rid3;

+    hid_t fapl_id, dxpl_id, trspl_id;

+    hid_t event_q;

+

+    const char *str_wdata[5]= {

+        "Four score and seven years ago our forefathers brought forth on this continent a new nation,",

+        "conceived in liberty and dedicated to the proposition that all men are created equal.",

+        "Now we are engaged in a great civil war,",

+        "testing whether that nation or any nation so conceived and so dedicated can long endure.",

+        "President Abraham Lincoln"

+    };   /* Information to write */

+    char *str_rdata[5];   /* Information read in */

+    hvl_t wdata[5];   /* Information to write */

+    hvl_t rdata[5];   /* Information to write */

+    int i;

+

+    uint64_t version;

+    uint64_t trans_num;

+

+    int my_rank, my_size;

+    int provided;

+    MPI_Request mpi_req;

+

+    H5_status_t *status = NULL;

+    int num_requests = 0;

+    herr_t ret;

+

+    hsize_t count = -1;

+    int key, value;

+    hbool_t exists;

+    H5_request_t req1;

+    H5_status_t status1;

+

+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);

+    if(MPI_THREAD_MULTIPLE != provided) {

+        fprintf(stderr, "MPI does not have MPI_THREAD_MULTIPLE support\n");

+        exit(1);

+    }

+

+    /* Call EFF_init to initialize the EFF stack.  

+       As a result of this call, the Function Shipper client is started, 

+       and HDF5 VOL calls are registered with the function shipper.

+       An "IOD init" call is forwarded from the FS client to the FS server 

+       which should already be running. */

+    EFF_init(MPI_COMM_WORLD, MPI_INFO_NULL);

+

+    MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);

+    MPI_Comm_size(MPI_COMM_WORLD, &my_size);

+    fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank);

+

+    fprintf(stderr, "Create the FAPL to set the IOD VOL plugin and create the file\n");

+    /* Choose the IOD VOL plugin to use with this file. 

+       First we create a file access property list. Then we call a new routine to set

+       the IOD plugin to use with this fapl */

+    fapl_id = H5Pcreate (H5P_FILE_ACCESS);

+    H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);

+

+    /* create an event Queue for managing asynchronous requests.

+

+       Event Queues will releive the use from managing and completing

+       individual requests for every operation. Instead of passing a

+       request for every operation, the event queue is passed and

+       internally the HDF5 library creates a request and adds it to

+       the event queue.

+

+       Multiple Event queue can be created used by the application. */

+    event_q = H5EQcreate(fapl_id);

+    assert(event_q);

+

+    /* create the file. This is asynchronous, but the file_id can be used. */

+    file_id = H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);

+    assert(file_id);

+

+    /* acquire container version 0 - EXACT */

+    version = 0;

+    rid1 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_QUEUE_NULL);

+    assert(0 == version);

+

+    /* create transaction object */

+    tid1 = H5TRcreate(file_id, rid1, (uint64_t)1);

+    assert(tid1);

+

+    /* start transaction 1 with default num_peers (= 0). */

+    if(0 == my_rank) {

+        ret = H5TRstart(tid1, H5P_DEFAULT, H5_EVENT_QUEUE_NULL);

+        assert(0 == ret);

+    }

+

+    /* Tell other procs that transaction 1 is started */

+    trans_num = 1;

+    MPI_Ibcast(&trans_num, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD, &mpi_req);

+

+    /* Process 0 can continue writing to transaction 1, 

+       while others wait for the bcast to complete */

+    if(0 != my_rank)

+        MPI_Wait(&mpi_req, MPI_STATUS_IGNORE);

+

+    /* create two groups */

+    gid1 = H5Gcreate_ff(file_id, "G1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);

+    gid2 = H5Gcreate_ff(gid1, "G2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);

+

+    /* Create Map objects with the key type being 32 bit LE integer */

+    /* Val type, 32 bit LE integer */

+    map1 = H5Mcreate_ff(file_id, "MAP_1", H5T_STD_I32LE, H5T_STD_I32LE, 

+                        H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT, tid1, event_q);

+

+    /* Val type: VL datatype */

+    {

+        int increment, j, n;

+

+        n = 0;

+        increment = 4;

+        /* Allocate and initialize VL data to write */

+        for(i = 0; i < 5; i++) {

+            int temp = i*increment + increment;

+

+            wdata[i].p = malloc(temp * sizeof(unsigned int));

+            wdata[i].len = temp;

+            for(j = 0; j < temp; j++)

+                ((unsigned int *)wdata[i].p)[j] = n ++;

+        } /* end for */

+

+        /* Create a datatype to refer to */

+        dtid1 = H5Tvlen_create (H5T_NATIVE_UINT);

+

+        map2 = H5Mcreate_ff(gid1, "MAP_2", H5T_STD_I32LE, dtid1, 

+                            H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT, tid1, event_q);

+    }

+

+    /* Val type: VL string */

+    {

+        hid_t dtid;

+

+        /* Create a datatype to refer to */

+        dtid2 = H5Tcopy(H5T_C_S1);

+        H5Tset_size(dtid2,H5T_VARIABLE);

+

+        /* Val type, VL string */

+        map3 = H5Mcreate_ff(gid2, "MAP_3", H5T_STD_I32LE, dtid1, 

+                            H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT, tid1, event_q);

+    }

+

+    {

+        key = 1;

+        value = 1000;

+        ret = H5Mset_ff(map1, H5T_STD_I32LE, &key, H5T_STD_I32LE, &value,

+                        H5P_DEFAULT, tid1, event_q);

+        assert(ret == 0);

+

+        for(i=0 ; i<5 ; i++) {

+            key = i;

+            ret = H5Mset_ff(map2, H5T_STD_I32LE, &key, dtid1, &wdata[i],

+                            H5P_DEFAULT, tid1, event_q);

+            assert(ret == 0);

+        }

+

+        for(i=0 ; i<5 ; i++) {

+            key = i;

+            ret = H5Mset_ff(map3, H5T_STD_I32LE, &key, dtid2, str_wdata[i],

+                            H5P_DEFAULT, tid1, event_q);

+            assert(ret == 0);

+        }

+    }

+

+    /* none leader procs have to complete operations before notifying the leader */

+    if(0 != my_rank) {

+        H5EQwait(event_q, &num_requests, &status);

+        printf("%d requests in event queue. Completions: ", num_requests);

+        for(i=0 ; i<num_requests; i++)

+            fprintf(stderr, "%d ",status[i]);

+        fprintf(stderr, "\n");

+        free(status);

+    }

+

+    /* Barrier to make sure all processes are done writing so Process

+       0 can finish transaction 1 and acquire a read context on it. */

+    MPI_Barrier(MPI_COMM_WORLD);

+

+    if(0 == my_rank) {

+        MPI_Wait(&mpi_req, MPI_STATUS_IGNORE);

+

+        ret = H5TRfinish(tid1, H5P_DEFAULT, &rid2, H5_EVENT_QUEUE_NULL);

+        assert(0 == ret);

+    }

+

+    /* another barrier so other processes know that container version is acquried */

+    MPI_Barrier(MPI_COMM_WORLD);

+

+    /* Close transaction object. Local op */

+    ret = H5TRclose(tid1);

+    assert(0 == ret);

+

+    /* release container version 0. This is async. */

+    ret = H5RCrelease(rid1, event_q);

+    assert(0 == ret);

+

+    H5EQwait(event_q, &num_requests, &status);

+    printf("%d requests in event queue. Completions: ", num_requests);

+    for(i=0 ; i<num_requests; i++)

+        fprintf(stderr, "%d ",status[i]);

+    fprintf(stderr, "\n");

+    free(status);

+

+    /* Process 0 tells other procs that container version 1 is acquired */

+    version = 1;

+    MPI_Bcast(&version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD);

+

+    /* other processes just create a read context object; no need to

+       acquire it */

+    if(0 != my_rank) {

+        rid2 = H5RCcreate(file_id, version);

+        assert(rid2 > 0);

+    }

+

+    ret = H5Mget_count_ff(map3, &count, rid2, event_q);

+    assert(ret == 0);

+

+    key = 1;

+    ret = H5Mexists_ff(map3, H5T_STD_I32LE, &key, &exists, rid2, event_q);

+    assert(ret == 0);

+

+    /* create & start transaction 2 with num_peers = n */

+    tid2 = H5TRcreate(file_id, rid2, (uint64_t)2);

+    assert(tid2);

+    trspl_id = H5Pcreate (H5P_TR_START);

+    ret = H5Pset_trspl_num_peers(trspl_id, my_size);

+    assert(0 == ret);

+    ret = H5TRstart(tid2, trspl_id, event_q);

+    assert(0 == ret);

+    ret = H5Pclose(trspl_id);

+    assert(0 == ret);

+

+    key = 1;

+    ret = H5Mdelete_ff(map3, H5T_STD_I32LE, &key, tid2, event_q);

+    assert(ret == 0);

+

+    /* finish transaction 2 */

+    ret = H5TRfinish(tid2, H5P_DEFAULT, NULL, event_q);

+    assert(0 == ret);

+

+    /* release container version 1. This is async. */

+    ret = H5RCrelease(rid2, event_q);

+    assert(0 == ret);

+

+    assert(H5Gclose_ff(gid1, event_q) == 0);

+    assert(H5Gclose_ff(gid2, event_q) == 0);

+    assert(H5Mclose_ff(map1, event_q) == 0);

+    assert(H5Mclose_ff(map2, event_q) == 0);

+    assert(H5Mclose_ff(map3, event_q) == 0);

+

+    H5EQwait(event_q, &num_requests, &status);

+    printf("%d requests in event queue. Completions: ", num_requests);

+    for(i=0 ; i<num_requests; i++)

+        fprintf(stderr, "%d ",status[i]);

+    fprintf(stderr, "\n");

+    free(status);

+

+    ret = H5RCclose(rid1);

+    assert(0 == ret);

+    ret = H5RCclose(rid2);

+    assert(0 == ret);

+    ret = H5TRclose(tid2);

+    assert(0 == ret);

+

+    /* acquire container version 0 - EXACT */

+    version = 2;

+    rid3 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_QUEUE_NULL);

+    assert(2 == version);

+

+    map1 = H5Mopen_ff(file_id, "MAP_1", H5P_DEFAULT, rid3, event_q);

+    map2 = H5Mopen_ff(file_id, "G1/MAP_2", H5P_DEFAULT, rid3, event_q);

+    map3 = H5Mopen_ff(file_id, "G1/G2/MAP_3", H5P_DEFAULT, rid3, event_q);

+

+    {

+        int key, value;

+        int increment=4, j=0;

+

+        key = 1;

+        value = -1;

+        ret = H5Mget_ff(map1, H5T_STD_I32LE, &key, H5T_STD_I32LE, &value,

+                        H5P_DEFAULT, rid3, event_q);

+

+        if(H5EQpop(event_q, &req1) < 0)

+            exit(1);

+        assert(H5AOwait(req1, &status1) == 0);

+        assert (status1);

+

+        printf("Value recieved = %d\n", value);

+        /* this is the fake value we sent from the server */

+        assert(value == 1024);

+

+        if(my_size == 1) {

+            for(i=0 ; i<5 ; i++) {

+                key = i;

+                ret = H5Mget_ff(map2, H5T_STD_I32LE, &key, dtid1, &rdata[i],

+                                H5P_DEFAULT, rid3, H5_EVENT_QUEUE_NULL);

+                assert(ret == 0);

+            }

+

+            /* Print VL DATA */

+            fprintf(stderr, "Reading VL Data: \n");

+            for(i = 0; i < 5; i++) {

+                int temp = i*increment + increment;

+

+                fprintf(stderr, "Key %d  size %zu: ", i, rdata[i].len);

+                for(j = 0; j < temp; j++)

+                    fprintf(stderr, "%d ",((unsigned int *)rdata[i].p)[j]);

+                fprintf(stderr, "\n");

+            } /* end for */

+

+            for(i=0 ; i<5 ; i++) {

+                key = i;

+                ret = H5Mget_ff(map3, H5T_STD_I32LE, &key, dtid2, &str_rdata[i],

+                                H5P_DEFAULT, rid3, H5_EVENT_QUEUE_NULL);

+                assert(ret == 0);

+            }

+

+            fprintf(stderr, "Reading VL Strings: \n");

+            for(i=0 ; i<5 ; i++) {

+                fprintf(stderr, "Key %d:  %s\n", i, str_rdata[i]);

+                free(str_rdata[i]);

+            }

+        }

+    }

+

+    /* release container version 1. This is async. */

+    ret = H5RCrelease(rid3, event_q);

+    assert(0 == ret);

+

+    assert(H5Mclose_ff(map1, event_q) == 0);

+    assert(H5Mclose_ff(map2, event_q) == 0);

+    assert(H5Mclose_ff(map3, event_q) == 0);

+

+    H5Tclose(dtid1);

+    H5Tclose(dtid2);

+

+    /* closing the container also acts as a wait all on all pending requests 

+       on the container. */

+    assert(H5Fclose_ff(file_id, event_q) == 0);

+

+    fprintf(stderr, "\n*****************************************************************************************************************\n");

+    fprintf(stderr, "Wait on everything in EQ and check Results of operations in EQ\n");

+    fprintf(stderr, "*****************************************************************************************************************\n");

+

+    /* wait on all requests and print completion status */

+    H5EQwait(event_q, &num_requests, &status);

+    fprintf(stderr, "%d requests in event queue. Completions: ", num_requests);

+    for(i=0 ; i<num_requests; i++)

+        fprintf(stderr, "%d ",status[i]);

+    fprintf(stderr, "\n");

+    free(status);

+

+    ret = H5RCclose(rid3);

+    assert(0 == ret);

+

+    assert (count == 3);

+    assert (exists > 0);

+

+    fprintf(stderr, "\n*****************************************************************************************************************\n");

+    fprintf(stderr, "Finalize EFF stack\n");

+    fprintf(stderr, "*****************************************************************************************************************\n");

+

+    H5EQclose(event_q);

+    H5Pclose(fapl_id);

+

+    for(i=0 ; i<5 ; i++) {

+        free(wdata[i].p);

+        if(my_size == 1)

+            free(rdata[i].p);

+    }

+

+    /* This finalizes the EFF stack. ships a terminate and IOD finalize to the server 

+       and shutsdown the FS server (when all clients send the terminate request) 

+       and client */

+    EFF_finalize();

+

+    MPI_Finalize();

+    return 0;

+}

diff --git a/testff/h5ff_client_multiple_cont.c b/testff/h5ff_client_multiple_cont.c
new file mode 100644
index 0000000..ad930d0
--- /dev/null
+++ b/testff/h5ff_client_multiple_cont.c
@@ -0,0 +1,304 @@
+/* 

+ * test_client.c: Client side of Milestone 4.2 Asynchronous I/O and initial

+ * IOD VOL plugin demonstration.  This is, in effect, the application program that 

+ * would run on one or more compute nodes and make calls to the HDF5 API.

+ */

+

+#include <stdio.h>

+#include <stdlib.h>

+#include <assert.h>

+#include <string.h>

+#include "mpi.h"

+#include "hdf5.h"

+

+int main(int argc, char **argv) {

+    const char file_name1[]="eff_file_1.h5";

+    const char file_name2[]="eff_file_2.h5";

+    hid_t fid1, fid2;

+    hid_t gid1, gid2, gid3, gid4;

+    hid_t sid;

+    hid_t did1, did2;

+    hid_t dtid1, dtid2;

+

+    hid_t tid1, tid2, rid1, rid2, rid3, rid4;

+    hid_t fapl_id, trspl_id;

+    hid_t event_q;

+

+    uint64_t version;

+    uint64_t trans_num;

+

+    const unsigned int nelem=60;

+    int *wdata1 = NULL, *wdata2 = NULL;

+    int *rdata1 = NULL, *rdata2 = NULL;

+    unsigned int i = 0;

+    hsize_t dims[1];

+    int my_rank, my_size;

+    int provided;

+    MPI_Request mpi_req;

+

+    H5_status_t *status = NULL;

+    int num_requests = 0;

+    herr_t ret;

+

+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);

+    if(MPI_THREAD_MULTIPLE != provided) {

+        fprintf(stderr, "MPI does not have MPI_THREAD_MULTIPLE support\n");

+        exit(1);

+    }

+

+    /* Call EFF_init to initialize the EFF stack.  

+       As a result of this call, the Function Shipper client is started, 

+       and HDF5 VOL calls are registered with the function shipper.

+       An "IOD init" call is forwarded from the FS client to the FS server 

+       which should already be running. */

+    EFF_init(MPI_COMM_WORLD, MPI_INFO_NULL);

+

+    MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);

+    MPI_Comm_size(MPI_COMM_WORLD, &my_size);

+    fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank);

+

+    fprintf(stderr, "Create the FAPL to set the IOD VOL plugin and create the file\n");

+    /* Choose the IOD VOL plugin to use with this file. 

+       First we create a file access property list. Then we call a new routine to set

+       the IOD plugin to use with this fapl */

+    fapl_id = H5Pcreate (H5P_FILE_ACCESS);

+    H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);

+

+    /* allocate and initialize arrays for dataset & attribute writes and reads. 

+       The write arrays are intialized to contain 60 integers (0-59). 

+       The read arrays are intialized to contain 60 integers all 0s. */

+    wdata1 = malloc (sizeof(int)*nelem);

+    wdata2 = malloc (sizeof(int)*nelem);

+    rdata1 = malloc (sizeof(int)*nelem);

+    rdata2 = malloc (sizeof(int)*nelem);

+    for(i=0;i<nelem;++i) {

+        rdata1[i] = 0;

+        wdata1[i]=i;

+        rdata2[i] = 0;

+        wdata2[i]=i;

+    }

+

+    /* create an event Queue for managing asynchronous requests.

+

+       Event Queues will releive the use from managing and completing

+       individual requests for every operation. Instead of passing a

+       request for every operation, the event queue is passed and

+       internally the HDF5 library creates a request and adds it to

+       the event queue.

+

+       Multiple Event queue can be created used by the application. */

+    event_q = H5EQcreate(fapl_id);

+    assert(event_q);

+

+    /* create the file. */

+    fid1 = H5Fcreate(file_name1, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);

+    assert(fid1);

+    /* create second file */

+    fid2 = H5Fcreate(file_name2, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);

+    assert(fid1);

+

+    /* acquire container version 0 on both containers - EXACT */

+    version = 0;

+    rid1 = H5RCacquire(fid1, &version, H5P_DEFAULT, H5_EVENT_QUEUE_NULL);

+    assert(0 == version);

+    rid2 = H5RCacquire(fid2, &version, H5P_DEFAULT, H5_EVENT_QUEUE_NULL);

+    assert(0 == version);

+

+    /* create transaction objects */

+    tid1 = H5TRcreate(fid1, rid1, (uint64_t)1);

+    assert(tid1);

+    tid2 = H5TRcreate(fid2, rid2, (uint64_t)1);

+    assert(tid1);

+

+    /* start transaction 1 with default num_peers (= 0). */

+    if(0 == my_rank) {

+        ret = H5TRstart(tid1, H5P_DEFAULT, H5_EVENT_QUEUE_NULL);

+        assert(0 == ret);

+        ret = H5TRstart(tid2, H5P_DEFAULT, H5_EVENT_QUEUE_NULL);

+        assert(0 == ret);

+    }

+

+    /* Tell other procs that transaction 1 is started */

+    trans_num = 1;

+    MPI_Ibcast(&trans_num, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD, &mpi_req);

+

+    /* Process 0 can continue writing to transaction 1, 

+       while others wait for the bcast to complete */

+    if(0 != my_rank)

+        MPI_Wait(&mpi_req, MPI_STATUS_IGNORE);

+

+    /* create groups */

+    gid1 = H5Gcreate_ff(fid1, "G1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);

+    gid2 = H5Gcreate_ff(gid1, "G2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);

+

+    gid3 = H5Gcreate_ff(fid2, "G1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid2, event_q);

+    gid4 = H5Gcreate_ff(gid3, "G2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid2, event_q);

+

+    /* create a 32 bit integer LE datatype. This is a local operation

+       that does not touch the file */

+    dtid1 = H5Tcopy(H5T_STD_I32LE);

+    dtid2 = H5Tcopy(H5T_STD_I32LE);

+

+    /* Commit the datatype to the file. This is asynchronous & immediate. 

+     * Other Local H5T type operations can be issued before completing this call

+     * because they do not depend on the committed state of the datatype.

+     */

+     H5Tcommit_ff(fid1, "int", dtid1, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, 

+                  tid1, event_q);

+     H5Tcommit_ff(fid2, "int", dtid2, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, 

+                  tid2, event_q);

+

+    /* create a dataspace. This is a local Bookeeping operation that 

+       does not touch the file */

+    dims [0] = 60;

+    sid = H5Screate_simple(1, dims, NULL);

+

+    /* create Datasets */

+    did1 = H5Dcreate_ff(gid2,"D1",dtid1,sid,

+                        H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT, tid1, event_q);

+    did2 = H5Dcreate_ff(gid4,"D1",dtid2,sid,

+                        H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT, tid2, event_q);

+

+    /* Raw data write on D1. */

+    H5Dwrite_ff(did1, dtid1, sid, sid, H5P_DEFAULT, wdata1, tid1, event_q);

+    H5Dwrite_ff(did2, dtid2, sid, sid, H5P_DEFAULT, wdata2, tid2, event_q);

+

+    /* none leader procs have to complete operations before notifying the leader */

+    if(0 != my_rank) {

+        H5EQwait(event_q, &num_requests, &status);

+        printf("%d requests in event queue. Completions: ", num_requests);

+        for(i=0 ; i<num_requests; i++)

+            fprintf(stderr, "%d ",status[i]);

+        fprintf(stderr, "\n");

+        free(status);

+    }

+

+    /* Barrier to make sure all processes are done writing so Process

+       0 can finish transaction 1 and acquire a read context on it. */

+    MPI_Barrier(MPI_COMM_WORLD);

+

+    if(0 == my_rank) {

+        MPI_Wait(&mpi_req, MPI_STATUS_IGNORE);

+

+        /* make this synchronous so we know the container version has been acquired */

+        ret = H5TRfinish(tid1, H5P_DEFAULT, &rid3, H5_EVENT_QUEUE_NULL);

+        assert(0 == ret);

+        /* make this synchronous so we know the container version has been acquired */

+        ret = H5TRfinish(tid2, H5P_DEFAULT, &rid4, H5_EVENT_QUEUE_NULL);

+        assert(0 == ret);

+    }

+

+    /* another barrier so other processes know that container version is acquried */

+    MPI_Barrier(MPI_COMM_WORLD);

+

+    /* Local op */

+    ret = H5TRclose(tid1);

+    assert(0 == ret);

+    /* Local op */

+    ret = H5TRclose(tid2);

+    assert(0 == ret);

+

+    /* release container version 0. This is async. */

+    ret = H5RCrelease(rid1, event_q);

+    assert(0 == ret);

+    ret = H5RCrelease(rid2, event_q);

+    assert(0 == ret);

+

+    /* other processes just create a read context object; no need to

+       acquire it */

+    if(0 != my_rank) {

+        rid3 = H5RCcreate(fid1, version);

+        assert(rid3 > 0);

+        rid4 = H5RCcreate(fid2, version);

+        assert(rid4 > 0);

+    }

+

+    /* read data from datasets with read version 1. */

+    ret = H5Dread_ff(did1, dtid1, sid, sid, H5P_DEFAULT, rdata1, rid3, event_q);

+    assert(ret == 0);

+    ret = H5Dread_ff(did2, dtid2, sid, sid, H5P_DEFAULT, rdata2, rid4, event_q);

+    assert(ret == 0);

+

+    fprintf(stderr, "\n*****************************************************************************************************************\n");

+    fprintf(stderr, "Close Objects\n");

+    fprintf(stderr, "*****************************************************************************************************************\n");

+

+    /* closing did1 acts as a barrier task at the server for all

+       operations dependeing on the dataset. This is asynchronous. */

+    assert(H5Dclose_ff(did1, event_q) == 0);

+    assert(H5Dclose_ff(did2, event_q) == 0);

+

+    assert(H5Tclose_ff(dtid1, event_q) == 0);

+    assert(H5Tclose_ff(dtid2, event_q) == 0);

+

+    assert(H5Gclose_ff(gid1, event_q) == 0);

+    assert(H5Gclose_ff(gid2, event_q) == 0);

+    assert(H5Gclose_ff(gid3, event_q) == 0);

+    assert(H5Gclose_ff(gid4, event_q) == 0);

+

+    /* none leader procs have to complete operations before notifying the leader */

+    if(0 != my_rank) {

+        H5EQwait(event_q, &num_requests, &status);

+        printf("%d requests in event queue. Completions: ", num_requests);

+        for(i=0 ; i<num_requests; i++)

+            fprintf(stderr, "%d ",status[i]);

+        fprintf(stderr, "\n");

+        free(status);

+    }

+

+    MPI_Barrier(MPI_COMM_WORLD);

+

+    /* release container version 1. This is async. */

+    if(0 == my_rank) {

+        ret = H5RCrelease(rid3, event_q);

+        assert(0 == ret);

+        ret = H5RCrelease(rid4, event_q);

+        assert(0 == ret);

+    }

+

+    ret = H5RCclose(rid3);

+    assert(0 == ret);

+    ret = H5RCclose(rid4);

+    assert(0 == ret);

+

+    assert(H5Fclose_ff(fid1, event_q) == 0);

+    assert(H5Fclose_ff(fid2, event_q) == 0);

+

+    /* wait on all requests and print completion status */

+    H5EQwait(event_q, &num_requests, &status);

+    fprintf(stderr, "%d requests in event queue. Completions: ", num_requests);

+    for(i=0 ; i<num_requests; i++)

+        fprintf(stderr, "%d ",status[i]);

+    fprintf(stderr, "\n");

+    free(status);

+

+    fprintf(stderr, "Printing After Waiting for data in first file");

+    for(i=0;i<nelem;++i)

+        fprintf(stderr, "%d ",rdata1[i]);

+    fprintf(stderr, "\n");

+

+    fprintf(stderr, "Printing After Waiting for data in second file");

+    for(i=0;i<nelem;++i)

+        fprintf(stderr, "%d ",rdata2[i]);

+    fprintf(stderr, "\n");

+

+    H5EQclose(event_q);

+    H5Pclose(fapl_id);

+

+    free(wdata1);

+    free(rdata1);

+    free(wdata2);

+    free(rdata2);

+

+    fprintf(stderr, "\n*****************************************************************************************************************\n");

+    fprintf(stderr, "Finalize EFF stack\n");

+    fprintf(stderr, "*****************************************************************************************************************\n");

+

+    /* This finalizes the EFF stack. ships a terminate and IOD finalize to the server 

+       and shutsdown the FS server (when all clients send the terminate request) 

+       and client */

+    EFF_finalize();

+

+    MPI_Finalize();

+    return 0;

+}

diff --git a/testff/h5ff_client_obj.c b/testff/h5ff_client_obj.c
new file mode 100644
index 0000000..ab2c67c
--- /dev/null
+++ b/testff/h5ff_client_obj.c
@@ -0,0 +1,262 @@
+/* 

+ * test_client_obj.c: Client side of H5O routines

+ */

+

+#include <stdio.h>

+#include <stdlib.h>

+#include <assert.h>

+#include <string.h>

+#include "mpi.h"

+#include "hdf5.h"

+

+int main(int argc, char **argv) {

+    const char file_name[]="map_file.h5";

+    hid_t file_id;

+    hid_t gid;

+    hid_t did, map;

+    hid_t sid, dtid;

+    hid_t tid1, rid1, rid2;

+    hid_t fapl_id, dxpl_id;

+    hid_t event_q;

+    hbool_t exists;

+

+    const unsigned int nelem=60;

+    hsize_t dims[1];

+

+    uint64_t version;

+    uint64_t trans_num;

+

+    int my_rank, my_size;

+    int provided;

+    MPI_Request mpi_req;

+

+    H5_status_t *status = NULL;

+    int num_requests = 0, i;

+    herr_t ret;

+

+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);

+    if(MPI_THREAD_MULTIPLE != provided) {

+        fprintf(stderr, "MPI does not have MPI_THREAD_MULTIPLE support\n");

+        exit(1);

+    }

+

+    /* Call EFF_init to initialize the EFF stack.  

+       As a result of this call, the Function Shipper client is started, 

+       and HDF5 VOL calls are registered with the function shipper.

+       An "IOD init" call is forwarded from the FS client to the FS server 

+       which should already be running. */

+    EFF_init(MPI_COMM_WORLD, MPI_INFO_NULL);

+

+    MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);

+    MPI_Comm_size(MPI_COMM_WORLD, &my_size);

+    fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank);

+

+    fprintf(stderr, "Create the FAPL to set the IOD VOL plugin and create the file\n");

+    /* Choose the IOD VOL plugin to use with this file. 

+       First we create a file access property list. Then we call a new routine to set

+       the IOD plugin to use with this fapl */

+    fapl_id = H5Pcreate (H5P_FILE_ACCESS);

+    H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);

+

+    /* create an event Queue for managing asynchronous requests.

+

+       Event Queues will releive the use from managing and completing

+       individual requests for every operation. Instead of passing a

+       request for every operation, the event queue is passed and

+       internally the HDF5 library creates a request and adds it to

+       the event queue.

+

+       Multiple Event queue can be created used by the application. */

+    event_q = H5EQcreate(fapl_id);

+    assert(event_q);

+

+    /* create the file. This is asynchronous, but the file_id can be used. */

+    file_id = H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);

+    assert(file_id);

+

+    /* create 1-D dataspace with 60 elements */

+    dims [0] = nelem;

+    sid = H5Screate_simple(1, dims, NULL);

+    dtid = H5Tcopy(H5T_STD_I32LE);

+

+    /* acquire container version 0 - EXACT */

+    version = 0;

+    rid1 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_QUEUE_NULL);

+    assert(0 == version);

+

+    /* create transaction object */

+    tid1 = H5TRcreate(file_id, rid1, (uint64_t)1);

+    assert(tid1);

+

+    /* start transaction 1 with default num_peers (= 0). */

+    if(0 == my_rank) {

+        ret = H5TRstart(tid1, H5P_DEFAULT, H5_EVENT_QUEUE_NULL);

+        assert(0 == ret);

+    }

+

+    /* Tell other procs that transaction 1 is started */

+    trans_num = 1;

+    MPI_Ibcast(&trans_num, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD, &mpi_req);

+

+    /* Process 0 can continue writing to transaction 1, 

+       while others wait for the bcast to complete */

+    if(0 != my_rank)

+        MPI_Wait(&mpi_req, MPI_STATUS_IGNORE);

+

+    /* create objects */

+    gid = H5Gcreate_ff(file_id, "G1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);

+    ret = H5Oset_comment_ff(gid, "Testing Object Comment", tid1, event_q);

+    assert(ret == 0);

+    did = H5Dcreate_ff(gid, "D1", dtid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);

+    ret = H5Tcommit_ff(file_id, "DT1", dtid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);

+    assert(ret == 0);

+    map = H5Mcreate_ff(file_id, "MAP1", H5T_STD_I32LE, H5T_STD_I32LE, 

+                       H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT, tid1, event_q);

+    assert(map > 0);

+

+    ret = H5Ocopy_ff(gid, ".", file_id, "G1_COPY", 

+                     H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);

+    assert(ret == 0);

+

+    /* none leader procs have to complete operations before notifying the leader */

+    if(0 != my_rank) {

+        H5EQwait(event_q, &num_requests, &status);

+        printf("%d requests in event queue. Completions: ", num_requests);

+        for(i=0 ; i<num_requests; i++)

+            fprintf(stderr, "%d ",status[i]);

+        fprintf(stderr, "\n");

+        free(status);

+    }

+

+    /* Barrier to make sure all processes are done writing so Process

+       0 can finish transaction 1 and acquire a read context on it. */

+    MPI_Barrier(MPI_COMM_WORLD);

+

+    if(0 == my_rank) {

+        MPI_Wait(&mpi_req, MPI_STATUS_IGNORE);

+

+        ret = H5TRfinish(tid1, H5P_DEFAULT, &rid2, H5_EVENT_QUEUE_NULL);

+        assert(0 == ret);

+    }

+

+    /* another barrier so other processes know that container version is acquried */

+    MPI_Barrier(MPI_COMM_WORLD);

+

+    /* Close transaction object. Local op */

+    ret = H5TRclose(tid1);

+    assert(0 == ret);

+

+    /* release container version 0. This is async. */

+    ret = H5RCrelease(rid1, event_q);

+    assert(0 == ret);

+

+    assert(H5Gclose_ff(gid, event_q) == 0);

+    assert(H5Dclose_ff(did, event_q) == 0);

+    assert(H5Tclose_ff(dtid, event_q) == 0);

+    assert(H5Mclose_ff(map, event_q) == 0);

+

+    H5EQwait(event_q, &num_requests, &status);

+    printf("%d requests in event queue. Completions: ", num_requests);

+    for(i=0 ; i<num_requests; i++)

+        fprintf(stderr, "%d ",status[i]);

+    fprintf(stderr, "\n");

+    free(status);

+

+    /* Process 0 tells other procs that container version 1 is acquired */

+    version = 1;

+    MPI_Bcast(&version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD);

+

+    /* other processes just create a read context object; no need to

+       acquire it */

+    if(0 != my_rank) {

+        rid2 = H5RCcreate(file_id, version);

+        assert(rid2 > 0);

+    }

+

+    gid = H5Oopen_ff(file_id, "G1", H5P_DEFAULT, rid2);

+    assert(gid);

+    dtid = H5Oopen_ff(file_id, "DT1", H5P_DEFAULT, rid2);

+    assert(dtid);

+    did = H5Oopen_ff(gid,"D1", H5P_DEFAULT, rid2);

+    assert(did);

+    map = H5Oopen_ff(file_id,"MAP1", H5P_DEFAULT, rid2);

+    assert(did);

+

+    {

+        ssize_t ret_size = 0;

+        char *comment = NULL;

+        H5O_ff_info_t oinfo;

+

+        ret = H5Oget_comment_ff(gid, NULL, 0, &ret_size, rid2, H5_EVENT_QUEUE_NULL);

+        assert(ret == 0);

+

+        fprintf(stderr, "size of comment is %d\n", ret_size);

+

+        comment = malloc((size_t)ret_size + 1);

+

+        ret = H5Oget_comment_ff(gid, comment, (size_t)ret_size + 1, 

+                                &ret_size, rid2, H5_EVENT_QUEUE_NULL);

+        assert(ret == 0);

+

+        fprintf(stderr, "size of comment is %d Comment is %s\n", ret_size, comment);

+        free(comment);

+

+        ret = H5Oget_info_ff(gid, &oinfo, rid2, H5_EVENT_QUEUE_NULL);

+        assert(ret == 0);

+

+        switch(oinfo.type) {

+        case H5O_TYPE_GROUP:

+            fprintf(stderr, 

+                    "OBJECT GET INFO return a GROUP TYPE with IOD ID: %llu, num attrs = %llu, reference count = %d\n", 

+                    oinfo.addr, oinfo.num_attrs, oinfo.rc);

+            break;

+        default:

+            fprintf(stderr, "Unexpected result from oget_info\n");

+            exit(1);

+        }

+    }

+

+    /* check if an object exists. This is asynchronous, so checking

+       the value should be done after the wait */

+    ret = H5Oexists_by_name_ff(file_id, "G1_COPY", &exists, 

+                               H5P_DEFAULT, rid2, event_q);

+    assert(ret == 0);

+

+    /* release container version 1. This is async. */

+    ret = H5RCrelease(rid2, event_q);

+    assert(0 == ret);

+

+    assert(H5Oclose_ff(gid, event_q) == 0);

+    assert(H5Oclose_ff(did, event_q) == 0);

+    assert(H5Oclose_ff(dtid, event_q) == 0);

+    assert(H5Oclose_ff(map, event_q) == 0);

+

+    ret = H5RCclose(rid1);

+    assert(0 == ret);

+    ret = H5RCclose(rid2);

+    assert(0 == ret);

+

+    /* closing the container also acts as a wait all on all pending requests 

+       on the container. */

+    assert(H5Fclose_ff(file_id, event_q) == 0);

+

+    /* wait on all requests and print completion status */

+    H5EQwait(event_q, &num_requests, &status);

+    fprintf(stderr, "%d requests in event queue. Completions: ", num_requests);

+    for(i=0 ; i<num_requests; i++)

+        fprintf(stderr, "%d ",status[i]);

+    fprintf(stderr, "\n");

+    free(status);

+

+    H5Sclose(sid);

+    H5Pclose(fapl_id);

+    H5EQclose(event_q);

+

+    /* This finalizes the EFF stack. ships a terminate and IOD finalize to the server 

+       and shutsdown the FS server (when all clients send the terminate request) 

+       and client */

+    EFF_finalize();

+

+    MPI_Finalize();

+    return 0;

+}

diff --git a/testff/h5ff_client_open.c b/testff/h5ff_client_open.c
new file mode 100644
index 0000000..e4b9240
--- /dev/null
+++ b/testff/h5ff_client_open.c
@@ -0,0 +1,120 @@
+/* 

+ * test_client_obj.c: Client side of H5O routines

+ */

+

+#include <stdio.h>

+#include <stdlib.h>

+#include <assert.h>

+#include <string.h>

+#include "mpi.h"

+#include "hdf5.h"

+

+int main(int argc, char **argv) {

+    const char file_name[]="map_file.h5";

+    hid_t file_id;

+    hid_t gid;

+    hid_t did; 

+    hid_t map;

+    hid_t dtid;

+    hid_t rid1;

+    hid_t aid;

+    hid_t fapl_id;

+    hid_t event_q;

+    uint64_t version;

+    int my_rank, my_size;

+    int provided;

+    H5_status_t *status = NULL;

+    int num_requests = 0, i;

+    herr_t ret;

+

+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);

+    if(MPI_THREAD_MULTIPLE != provided) {

+        fprintf(stderr, "MPI does not have MPI_THREAD_MULTIPLE support\n");

+        exit(1);

+    }

+

+    /* Call EFF_init to initialize the EFF stack.  

+       As a result of this call, the Function Shipper client is started, 

+       and HDF5 VOL calls are registered with the function shipper.

+       An "IOD init" call is forwarded from the FS client to the FS server 

+       which should already be running. */

+    EFF_init(MPI_COMM_WORLD, MPI_INFO_NULL);

+

+    MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);

+    MPI_Comm_size(MPI_COMM_WORLD, &my_size);

+    fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank);

+

+    fprintf(stderr, "Create the FAPL to set the IOD VOL plugin and create the file\n");

+    /* Choose the IOD VOL plugin to use with this file. 

+       First we create a file access property list. Then we call a new routine to set

+       the IOD plugin to use with this fapl */

+    fapl_id = H5Pcreate (H5P_FILE_ACCESS);

+    H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);

+

+    /* create an event Queue for managing asynchronous requests.

+

+       Event Queues will releive the use from managing and completing

+       individual requests for every operation. Instead of passing a

+       request for every operation, the event queue is passed and

+       internally the HDF5 library creates a request and adds it to

+       the event queue.

+

+       Multiple Event queue can be created used by the application. */

+    event_q = H5EQcreate(fapl_id);

+    assert(event_q);

+

+    /* Open the file and ask to acquire the latest readable version */

+    file_id = H5Fopen_ff(file_name, H5F_ACC_RDONLY, fapl_id, &rid1, H5_EVENT_QUEUE_NULL);

+    assert(file_id);

+

+    /* query the latest readable version number */

+    H5RCget_version(rid1, &version);

+    /* MSC - assert the fake version value that is returned for now */

+    assert(version == 1024);

+

+    /* create objects */

+    gid = H5Gopen_ff(file_id, "G1", H5P_DEFAULT, rid1, event_q);

+    assert(gid > 0);

+    did = H5Dopen_ff(gid, "D1", H5P_DEFAULT, rid1, event_q);

+    assert(did > 0);

+    map = H5Mopen_ff(file_id, "MAP1", H5P_DEFAULT, rid1, event_q);

+    assert(map > 0);

+    dtid = H5Topen_ff(file_id, "DT1", H5P_DEFAULT, rid1, event_q);

+    assert(dtid > 0);

+    aid = H5Aopen_ff(did, "ATTR_DSET", H5P_DEFAULT, rid1, event_q);

+    assert(aid > 0);

+

+    assert(H5Gclose_ff(gid, event_q) == 0);

+    assert(H5Dclose_ff(did, event_q) == 0);

+    assert(H5Tclose_ff(dtid, event_q) == 0);

+    assert(H5Mclose_ff(map, event_q) == 0);

+    assert(H5Aclose_ff(aid, event_q) == 0);

+

+    /* release container version 1024. This is async. */

+    ret = H5RCrelease(rid1, event_q);

+    assert(0 == ret);

+

+    ret = H5RCclose(rid1);

+    assert(0 == ret);

+    /* closing the container also acts as a wait all on all pending requests 

+       on the container. */

+    assert(H5Fclose_ff(file_id, event_q) == 0);

+

+    H5EQwait(event_q, &num_requests, &status);

+    printf("%d requests in event queue. Completions: ", num_requests);

+    for(i=0 ; i<num_requests; i++)

+        fprintf(stderr, "%d ",status[i]);

+    fprintf(stderr, "\n");

+    free(status);

+

+    H5Pclose(fapl_id);

+    H5EQclose(event_q);

+

+    /* This finalizes the EFF stack. ships a terminate and IOD finalize to the server 

+       and shutsdown the FS server (when all clients send the terminate request) 

+       and client */

+    EFF_finalize();

+

+    MPI_Finalize();

+    return 0;

+}

diff --git a/testff/h5ff_client_paths.c b/testff/h5ff_client_paths.c
new file mode 100644
index 0000000..57542a1
--- /dev/null
+++ b/testff/h5ff_client_paths.c
@@ -0,0 +1,253 @@
+/* 

+ * test_client_path.c: This example demonstrates what can and can't be

+ * done with access through Paths in the IOD plugin.

+ */

+

+#include <stdio.h>

+#include <stdlib.h>

+#include <assert.h>

+#include <string.h>

+#include "mpi.h"

+#include "hdf5.h"

+

+int main(int argc, char **argv) {

+    const char file_name[]="map_file.h5";

+    hid_t file_id;

+    hid_t gid1, gid2, gid3;

+    hid_t map;

+    hid_t tid1, tid2, rid1, rid2;

+    hid_t fapl_id, dxpl_id, trspl_id;

+    hid_t event_q;

+

+    uint64_t version;

+    uint64_t trans_num;

+

+    int my_rank, my_size;

+    int provided;

+    MPI_Request mpi_req;

+

+    H5_status_t *status = NULL;

+    int num_requests = 0, i;

+    herr_t ret;

+

+    hsize_t count = -1;

+    int key, value;

+    hbool_t exists = -1;

+    H5_request_t req1;

+    H5_status_t status1;

+

+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);

+    if(MPI_THREAD_MULTIPLE != provided) {

+        fprintf(stderr, "MPI does not have MPI_THREAD_MULTIPLE support\n");

+        exit(1);

+    }

+

+    /* Call EFF_init to initialize the EFF stack.  

+       As a result of this call, the Function Shipper client is started, 

+       and HDF5 VOL calls are registered with the function shipper.

+       An "IOD init" call is forwarded from the FS client to the FS server 

+       which should already be running. */

+    EFF_init(MPI_COMM_WORLD, MPI_INFO_NULL);

+

+    MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);

+    MPI_Comm_size(MPI_COMM_WORLD, &my_size);

+    fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank);

+

+    fprintf(stderr, "Create the FAPL to set the IOD VOL plugin and create the file\n");

+    /* Choose the IOD VOL plugin to use with this file. 

+       First we create a file access property list. Then we call a new routine to set

+       the IOD plugin to use with this fapl */

+    fapl_id = H5Pcreate (H5P_FILE_ACCESS);

+    H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);

+

+    /* create an event Queue for managing asynchronous requests.

+

+       Event Queues will releive the use from managing and completing

+       individual requests for every operation. Instead of passing a

+       request for every operation, the event queue is passed and

+       internally the HDF5 library creates a request and adds it to

+       the event queue.

+

+       Multiple Event queue can be created used by the application. */

+    event_q = H5EQcreate(fapl_id);

+    assert(event_q);

+

+    /* create the file. This is asynchronous, but the file_id can be used. */

+    file_id = H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);

+    assert(file_id);

+

+    /* acquire container version 0 - EXACT */

+    version = 0;

+    rid1 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_QUEUE_NULL);

+    assert(0 == version);

+

+    /******************************** Transaction 1 ****************************************/

+    /* create transaction object */

+    tid1 = H5TRcreate(file_id, rid1, (uint64_t)1);

+    assert(tid1);

+

+    /* start transaction 1 with default num_peers (= 0). */

+    if(0 == my_rank) {

+        ret = H5TRstart(tid1, H5P_DEFAULT, H5_EVENT_QUEUE_NULL);

+        assert(0 == ret);

+    }

+

+    /* Tell other procs that transaction 1 is started */

+    trans_num = 1;

+    MPI_Ibcast(&trans_num, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD, &mpi_req);

+

+    /* Process 0 can continue writing to transaction 1, 

+       while others wait for the bcast to complete */

+    if(0 != my_rank)

+        MPI_Wait(&mpi_req, MPI_STATUS_IGNORE);

+

+    /* create a group G1 on the root group */

+    gid1 = H5Gcreate_ff(file_id, "G1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);

+

+    /* to be able to create G2 on G1, we have to use G1's ID as the starting location */

+    gid2 = H5Gcreate_ff(gid1, "G2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);

+

+    /* If we happen to do something like this :

+

+       gid2 = H5Gcreate_ff(file_id, "G1/G2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);

+

+       We will get a failure when the operation completes, because the

+       path G1/G2 indicates that we have to read G1 from IOD in read

+       context 0. Since G1 is created in transaction 1, it is still

+       not yet visible so we can not read from it. We can only write

+       to it, which makes the earlier create directly on G1

+       possible. This operation will succeed in a subsequent

+       transaction that has a read context 1. 

+    */

+

+

+    /* none leader procs have to complete operations before notifying the leader */

+    if(0 != my_rank) {

+        H5EQwait(event_q, &num_requests, &status);

+        printf("%d requests in event queue. Completions: ", num_requests);

+        for(i=0 ; i<num_requests; i++)

+            fprintf(stderr, "%d ",status[i]);

+        fprintf(stderr, "\n");

+        free(status);

+    }

+

+    /* Barrier to make sure all processes are done writing so Process

+       0 can finish transaction 1 and acquire a read context on it. */

+    MPI_Barrier(MPI_COMM_WORLD);

+

+    if(0 == my_rank) {

+        MPI_Wait(&mpi_req, MPI_STATUS_IGNORE);

+

+        ret = H5TRfinish(tid1, H5P_DEFAULT, &rid2, H5_EVENT_QUEUE_NULL);

+        assert(0 == ret);

+    }

+

+    /* another barrier so other processes know that container version is acquried */

+    MPI_Barrier(MPI_COMM_WORLD);

+

+    /* Close transaction object. Local op */

+    ret = H5TRclose(tid1);

+    assert(0 == ret);

+

+    /******************************** END Transaction 1 ****************************************/

+

+    /* release container version 0. This is async. */

+    ret = H5RCrelease(rid1, event_q);

+    assert(0 == ret);

+

+    H5EQwait(event_q, &num_requests, &status);

+    printf("%d requests in event queue. Completions: ", num_requests);

+    for(i=0 ; i<num_requests; i++)

+        fprintf(stderr, "%d ",status[i]);

+    fprintf(stderr, "\n");

+    free(status);

+

+    /* Process 0 tells other procs that container version 1 is acquired */

+    version = 1;

+    MPI_Bcast(&version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD);

+

+    /* other processes just create a read context object; no need to

+       acquire it */

+    if(0 != my_rank) {

+        rid2 = H5RCcreate(file_id, version);

+        assert(rid2 > 0);

+    }

+

+

+    /******************************** Transaction 2 ****************************************/

+

+    /* create & start transaction 2 with num_peers = n */

+    tid2 = H5TRcreate(file_id, rid2, (uint64_t)2);

+    assert(tid2);

+    trspl_id = H5Pcreate (H5P_TR_START);

+    ret = H5Pset_trspl_num_peers(trspl_id, my_size);

+    assert(0 == ret);

+    ret = H5TRstart(tid2, trspl_id, event_q);

+    assert(0 == ret);

+    ret = H5Pclose(trspl_id);

+    assert(0 == ret);

+

+    /* create G3 under /G1/G2. This can be done by either providing

+       the file_id as the starting location, and the entire path to

+       where G3 needs to be created (G1/G2/G3), since the intermediate

+       groups G1 and G2 are readable, or gid2 as the starting

+       location. The latter would be more performant, since we already

+       have G2 open, so it will avoid extra access to get to G2 as

+       would be in the former case. */

+    gid3 = H5Gcreate_ff(gid2, "G3", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid2, event_q);

+    /* This will work too:

+       gid3 = H5Gcreate_ff(file_id, "G1/G2/G3", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid2, event_q);

+    */

+

+    /* create a Map object under G3. Since G3 is created in the same

+       transaction as the map create will occur, the direct location

+       for G3 is needed for the create to succeed. */

+    map = H5Mcreate_ff(gid3, "MAP", H5T_STD_I32LE, H5T_STD_I32LE, 

+                       H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT, tid2, event_q);

+

+    /* finish transaction 2 */

+    ret = H5TRfinish(tid2, H5P_DEFAULT, NULL, event_q);

+    assert(0 == ret);

+

+    ret = H5TRclose(tid2);

+    assert(0 == ret);

+

+    /******************************** END Transaction 2 ****************************************/

+

+    /* release container version 1. This is async. */

+    ret = H5RCrelease(rid2, event_q);

+    assert(0 == ret);

+

+    assert(H5Gclose_ff(gid1, event_q) == 0);

+    assert(H5Gclose_ff(gid2, event_q) == 0);

+    assert(H5Gclose_ff(gid3, event_q) == 0);

+    assert(H5Mclose_ff(map, event_q) == 0);

+

+    ret = H5RCclose(rid1);

+    assert(0 == ret);

+    ret = H5RCclose(rid2);

+    assert(0 == ret);

+

+    /* closing the container also acts as a wait all on all pending requests 

+       on the container. */

+    assert(H5Fclose_ff(file_id, event_q) == 0);

+

+    /* wait on all requests and print completion status */

+    H5EQwait(event_q, &num_requests, &status);

+    fprintf(stderr, "%d requests in event queue. Completions: ", num_requests);

+    for(i=0 ; i<num_requests; i++)

+        fprintf(stderr, "%d ",status[i]);

+    fprintf(stderr, "\n");

+    free(status);

+

+    H5EQclose(event_q);

+    H5Pclose(fapl_id);

+

+    /* This finalizes the EFF stack. ships a terminate and IOD finalize to the server 

+       and shutsdown the FS server (when all clients send the terminate request) 

+       and client */

+    EFF_finalize();

+

+    MPI_Finalize();

+    return 0;

+}

diff --git a/testff/h5ff_client_trans.c b/testff/h5ff_client_trans.c
new file mode 100644
index 0000000..7ae0c1c
--- /dev/null
+++ b/testff/h5ff_client_trans.c
@@ -0,0 +1,194 @@
+/* 

+ * test_client.c: Client side of Milestone 4.2 Asynchronous I/O and initial

+ * IOD VOL plugin demonstration.  This is, in effect, the application program that 

+ * would run on one or more compute nodes and make calls to the HDF5 API.

+ */

+

+#include <stdio.h>

+#include <stdlib.h>

+#include <assert.h>

+#include <string.h>

+#include "mpi.h"

+#include "hdf5.h"

+

+int main(int argc, char **argv) {

+    const char file_name[]="eff_file.h5";

+    hid_t file_id;

+    hid_t fapl_id, trspl_id;

+    hid_t tid1, tid2, tid3;

+    hid_t rid1, rid2, rid3;

+    uint64_t version;

+    int my_rank, my_size;

+    int provided;

+    hid_t event_q;

+    H5_status_t *status = NULL;

+    int i, num_requests = 0;

+    herr_t ret;

+    H5_request_t req1;

+    H5_status_t status1;

+

+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);

+    if(MPI_THREAD_MULTIPLE != provided) {

+        fprintf(stderr, "MPI does not have MPI_THREAD_MULTIPLE support\n");

+        exit(1);

+    }

+

+    fprintf(stderr, "\n*****************************************************************************************************************\n");

+    fprintf(stderr, "Initialize EFF stack\n");

+    fprintf(stderr, "*****************************************************************************************************************\n");

+

+    /* Call EFF_init to initialize the EFF stack.  

+       As a result of this call, the Function Shipper client is started, 

+       and HDF5 VOL calls are registered with the function shipper.

+       An "IOD init" call is forwarded from the FS client to the FS server 

+       which should already be running. */

+    EFF_init(MPI_COMM_WORLD, MPI_INFO_NULL);

+

+    MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);

+    MPI_Comm_size(MPI_COMM_WORLD, &my_size);

+    fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank);

+

+    fprintf(stderr, "Create the FAPL to set the IOD VOL plugin and create the file\n");

+    /* Choose the IOD VOL plugin to use with this file. 

+       First we create a file access property list. Then we call a new routine to set

+       the IOD plugin to use with this fapl */

+    fapl_id = H5Pcreate (H5P_FILE_ACCESS);

+    H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);

+

+    event_q = H5EQcreate(fapl_id);

+    assert(event_q);

+

+    /* create the file */

+    file_id = H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);

+    assert(file_id);

+

+    version = 0;

+    /* acquire container version 0 - EXACT */

+    rid1 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_QUEUE_NULL);

+

+    /* create 2 transactions objects (does not start transactions). Local call. */

+    tid1 = H5TRcreate(file_id, rid1, (uint64_t)1);

+    assert(tid1);

+    tid2 = H5TRcreate(file_id, rid1, (uint64_t)555);

+    assert(tid2);

+

+    /* start transaction 1 with default num_peers (= 0). 

+       This is asynchronous. */

+    if(my_rank == 0) {

+        hid_t gid1, gid2;

+

+        ret = H5TRstart(tid1, H5P_DEFAULT, event_q);

+        assert(0 == ret);

+

+        gid1 = H5Gcreate_ff(file_id, "G1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);

+        assert(gid1);

+        gid2 = H5Gcreate_ff(gid1, "G2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, event_q);

+        assert(gid2);

+

+        assert(H5Gclose_ff(gid1, event_q) == 0);

+        assert(H5Gclose_ff(gid2, event_q) == 0);

+    }

+

+    /* skip transactions 2 till 554. This is asynchronous. */

+    ret = H5TRskip(file_id, (uint64_t)2, (uint64_t)553, event_q);

+    assert(0 == ret);

+

+    /* start transaction 555 with num_peers = n */

+    trspl_id = H5Pcreate (H5P_TR_START);

+    ret = H5Pset_trspl_num_peers(trspl_id, my_size);

+    assert(0 == ret);

+    ret = H5TRstart(tid2, trspl_id, event_q);

+    assert(0 == ret);

+    ret = H5Pclose(trspl_id);

+    assert(0 == ret);

+

+    /* set dependency from transaction 555 on 2. 

+       This is asynchronous but has a dependency on H5TRstart() of tid2. */

+    ret = H5TRset_dependency(tid2, (uint64_t)1, event_q);

+    assert(0 == ret);

+

+    /* finish transaction 1. 

+       This is asynchronous, but has a dependency on H5TRstart() of tid1. */

+    if(my_rank == 0) {

+        ret = H5TRfinish(tid1, H5P_DEFAULT, NULL, event_q);

+        assert(0 == ret);

+    }

+

+    /* Local op */

+    ret = H5TRclose(tid1);

+    assert(0 == ret);

+

+    /* finish transaction 555 and acquire a read context for it */

+    ret = H5TRfinish(tid2, H5P_DEFAULT, &rid2, event_q);

+    assert(0 == ret);

+    ret = H5TRclose(tid2);

+    assert(0 == ret);

+

+    /* release container version 0. This is async. */

+    ret = H5RCrelease(rid1, event_q);

+    assert(0 == ret);

+

+    ret = H5RCclose(rid1);

+    assert(0 == ret);

+

+    /* wait on all requests in event queue */

+    H5EQwait(event_q, &num_requests, &status);

+    fprintf(stderr, "%d requests in event queue. Completions: ", num_requests);

+    for(i=0 ; i<num_requests; i++)

+        fprintf(stderr, "%d ",status[i]);

+    fprintf(stderr, "\n");

+    free(status);

+

+    if(my_rank == 0) {

+        /* Start transaction 556 from read context 555 */

+        tid3 = H5TRcreate(file_id, rid2, (uint64_t)556);

+        assert(tid1);

+

+        ret = H5TRstart(tid3, H5P_DEFAULT, event_q);

+        assert(0 == ret);

+

+        /* abort transaction 556 */

+        ret = H5TRabort(tid3, event_q);

+        assert(tid1);

+

+        ret = H5TRclose(tid3);

+        assert(0 == ret);

+    }

+

+    /* persist version 555 */

+    ret = H5RCpersist(rid2, event_q);

+    assert(0 == ret);

+

+    /* snapshot version 555 */

+    ret = H5RCsnapshot(rid2, "container_555", event_q);

+    assert(0 == ret);

+

+    ret = H5RCclose(rid2);

+    assert(0 == ret);

+

+    ret = H5Fclose(file_id);

+    assert(0 == ret);

+

+    /* wait on all requests in event queue */

+    H5EQwait(event_q, &num_requests, &status);

+    fprintf(stderr, "%d requests in event queue. Completions: ", num_requests);

+    for(i=0 ; i<num_requests; i++)

+        fprintf(stderr, "%d ",status[i]);

+    fprintf(stderr, "\n");

+    free(status);

+

+    H5EQclose(event_q);

+    H5Pclose(fapl_id);

+

+    fprintf(stderr, "\n*****************************************************************************************************************\n");

+    fprintf(stderr, "Finalize EFF stack\n");

+    fprintf(stderr, "*****************************************************************************************************************\n");

+

+    /* This finalizes the EFF stack. ships a terminate and IOD finalize to the server 

+       and shutsdown the FS server (when all clients send the terminate request) 

+       and client */

+    EFF_finalize();

+

+    MPI_Finalize();

+    return 0;

+}

diff --git a/testff/h5ff_test_server.c b/testff/h5ff_server.c
index 31be7ce..dc5f4a4 100644
--- a/testff/h5ff_test_server.c
+++ b/testff/h5ff_server.c
@@ -24,9 +24,7 @@ int main(int argc, char **argv) {
     MPI_Comm_size(MPI_COMM_WORLD, &my_size);

     printf("Number of server processes = %d, my rank is %d\n", my_size, my_rank);

 

-    H5open();

-

-    printf("Waiting for client\n");

+    //H5open();

     /* This call initiliazes the FS for the server processes (create metadata and

        bulk data handles). It also registers with the Function shipper the 

        HDF5 VOL server routines that will be executed when the clients ship the VOL 

@@ -42,8 +40,7 @@ int main(int argc, char **argv) {
        Finally, when all clients send a terminate call, the function shipper interface

        is finalized the operation returns. */

     H5VLiod_start_handler(MPI_COMM_WORLD, MPI_INFO_NULL);

-

-    H5close();

+    //H5close();

     MPI_Finalize();

 

     return 0;

diff --git a/testff/h5ff_test_client.c b/testff/h5ff_test_client.c
deleted file mode 100644
index a23188f..0000000
--- a/testff/h5ff_test_client.c
+++ /dev/null
@@ -1,613 +0,0 @@
-/* 

- * test_client.c: Client side of Milestone 3.3 Asynchronous I/O and initial

- * IOD VOL plugin demonstration.  This is, in effect, the application program that 

- * would run on one or more compute nodes and make calls to the HDF5 API.

- */

-

-#include <stdio.h>

-#include <stdlib.h>

-#include <assert.h>

-#include <string.h>

-#include "mpi.h"

-#include "hdf5.h"

-

-int main(int argc, char **argv) {

-    const char file_name[]="eff_file.h5";

-    hid_t file_id;

-    hid_t gid1, gid2, gid3;

-    hid_t dataspaceId;

-    hid_t did1, did2, did3;

-    hid_t aid1, aid2, aid3;

-    hid_t fapl_id, dxpl_id;

-    const unsigned int nelem=60;

-    int *data = NULL, *r_data = NULL, *r2_data = NULL, *data2 = NULL;

-    int16_t *data3 = NULL;

-    int16_t *r3_data = NULL;

-    int *a_data = NULL, *ra_data = NULL;

-    unsigned int i = 0;

-    hsize_t dims[1];

-    int my_rank, my_size;

-    int provided;

-    hid_t event_q;

-    hid_t int_id;

-    H5_status_t *status = NULL;

-    int num_requests = 0;

-    hsize_t extent = 20;

-    hbool_t exists;

-    uint32_t cs;

-    H5_request_t req1;

-    H5_status_t status1;

-

-    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);

-    if(MPI_THREAD_MULTIPLE != provided) {

-        fprintf(stderr, "MPI does not have MPI_THREAD_MULTIPLE support\n");

-        exit(1);

-    }

-

-    /* Call EFF_init to initialize the EFF stack.  

-       As a result of this call, the Function Shipper client is started, 

-       and HDF5 VOL calls are registered with the function shipper.

-       An "IOD init" call is forwarded from the FS client to the FS server 

-       which should already be running. */

-    EFF_init(MPI_COMM_WORLD, MPI_INFO_NULL);

-

-    MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);

-    MPI_Comm_size(MPI_COMM_WORLD, &my_size);

-    fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank);

-

-    /* Choose the IOD VOL plugin to use with this file. 

-       First we create a file access property list. Then we call a new routine to set

-       the IOD plugin to use with this fapl */

-    fapl_id = H5Pcreate (H5P_FILE_ACCESS);

-    H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);

-

-    /* allocate and initialize arrays for dataset & attribute writes and reads. 

-       The write arrays are intialized to contain 60 integers (0-59). 

-       The read arrays are intialized to contain 60 integers all 0s. */

-    data = malloc (sizeof(int)*nelem);

-    data2 = malloc (sizeof(int)*nelem);

-    data3 = malloc (sizeof(int16_t)*nelem);

-    a_data = malloc (sizeof(int)*nelem);

-    ra_data = malloc (sizeof(int)*nelem);

-    r_data = malloc (sizeof(int)*nelem);

-    r2_data = malloc (sizeof(int)*nelem);

-    r3_data = malloc (sizeof(int16_t)*nelem);

-    for(i=0;i<nelem;++i) {

-        r_data[i] = 0;

-        ra_data[i] = 0;

-        r2_data[i] = 0;

-        r3_data[i] = 0;

-        data[i]=i;

-        data2[i]=i;

-        data3[i]=i;

-        a_data[i]=i;

-    }

-

-    /* create an event Queue for managing asynchronous requests.

-

-       Event Queues will releive the use from managing and completing

-       individual requests for every operation. Instead of passing a

-       request for every operation, the event queue is passed and

-       internally the HDF5 library creates a request and adds it to

-       the event queue.

-

-       Multiple Event queue can be created used by the application. */

-    event_q = H5EQcreate(fapl_id);

-    assert(event_q);

-

-    /* create the file. This is asynchronous, but the file_id can be used. */

-    file_id = H5Fcreate_ff(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id, event_q);

-

-    {

-        char temp_name[50];

-        H5Fget_name(file_id, temp_name, 50);

-        fprintf(stderr, "File name %s   %s\n", temp_name, file_name);

-    }

-

-    /* create 3 groups in the file. All calls are completely

-       asychronous. Even though they depend one each other here and

-       they also depend on the file, they will be shipped immdeiately

-       and asynchronously to the server and return to the user. At the

-       server, dependencies are noted in the AXE and functions execute

-       in the order they are supposed to execute. */

-    gid1 = H5Gcreate_ff(file_id, "G1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, 0, event_q);

-    H5Oset_comment_ff(gid1, "Testing Object Comment", 0, event_q);

-    gid2 = H5Gcreate_ff(file_id, "G1/G2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, 0, event_q);

-    gid3 = H5Gcreate_ff(file_id, "G1/G2/G3", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, 0, event_q);

-

-    assert(gid1);

-    assert(gid2);

-    assert(gid3);

-

-    /* create a 32 bit integer LE datatype. This is a local operation

-       that does not touch the file */

-    int_id = H5Tcopy(H5T_STD_I32LE);

-

-    /* Commit the datatype to the file. This is asynchronous & immediate. 

-     * Other Local H5T type operations can be issued before completing this call

-     * because they do not depend on the committed state of the datatype.

-     */

-    H5Tcommit_ff(file_id, "int", int_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, 

-                 0, event_q);

-

-    /* create a dataspace. This is a local Bookeeping operation that 

-       does not touch the file */

-    dims [0] = 60;

-    dataspaceId = H5Screate_simple(1, dims, NULL);

-

-    /* create an attribute on group G1. This is asynchronous and

-       returns immediately, however at the server it won't start until

-       the group G1 creation op is completed */

-    aid1 = H5Acreate_ff(gid1, "ATTR1", H5T_NATIVE_INT, dataspaceId, 

-                        H5P_DEFAULT, H5P_DEFAULT, 0, event_q);

-    assert(aid1);

-

-    /* close the attribute right after the create. This is

-       asynchronous. The attribute id can not be used anymore, however

-       the creation operation will occur first at the server before

-       the close */

-    H5Aclose_ff(aid1, event_q);

-

-    /* Check if the attribute on group G1 exists. This is

-       asynchronous, so the exists status is correct to examine after

-       the completion of the operation */

-    assert(H5Aexists_by_name_ff(file_id,"G1","ATTR1", H5P_DEFAULT, &exists, 0, event_q) == 0);

-

-    /* Pop the request for the last operation from the event

-       queue. The Event queue does not manage it anymore and it is the

-       application's responsibilty to wait/test and ensure

-       completion */

-    if(H5EQpop(event_q, &req1) < 0)

-        exit(1);

-    /* wait synchronously on the operation */

-    assert(H5AOwait(req1, &status1) == 0);

-    assert (status1);

-

-    if(exists)

-        printf("Attribute ATTR1 exists!\n");

-    else

-        printf("Attribute ATTR1 does NOT exist. This must be the test without a native backend\n");

-

-    /* Delete the attribute just created, this is asynchronous */

-    assert(H5Adelete_by_name_ff(file_id, "G1", "ATTR1", H5P_DEFAULT, 0, event_q) == 0);

-

-    /* check if it exists now. This is asynchronous. Can't check the

-       exists status until the operation completes. */

-    assert(0 == H5Aexists_ff(gid1, "ATTR1", &exists, 0, event_q));

-

-    /* create a Dataset D1 on the file, in group /G1/G2/G3. This is

-       asynchronous.  Internally to the IOD-VOL this call traverses

-       the path G1/G2/G3.  it forwards the call asynchronously to the

-       server, with a dependency on Group G3 creation operation */

-    did1 = H5Dcreate_ff(file_id,"G1/G2/G3/D1",int_id,dataspaceId,

-                        H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT, 0, event_q);

-    assert(did1);

-

-    /* create an attribute on dataset D1. This is asynchronous, but

-       executes at the server after D1 is created */

-    aid2 = H5Acreate_by_name_ff(file_id, "G1/G2/G3/D1", "ATTR2_tmp", H5T_NATIVE_INT, 

-                                dataspaceId, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, 0, event_q);

-    assert(aid2);

-

-    /* rename the attribute. This is asynchronous with a dependency on

-       the create at the server*/

-    H5Arename_ff(did1, "ATTR2_tmp", "ATTR2", 0, event_q);

-

-    /* write data to the attribute. This is asynchronous but will wait

-       for the creation and rename calls at the server to complete */

-    H5Awrite_ff(aid2, int_id, a_data, 0, event_q);

-

-    /* similar to the previous H5Dcreate. */

-    did2 = H5Dcreate_ff(file_id,"G1/G2/G3/D2",int_id,dataspaceId,

-                        H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT, 0, event_q);

-    assert(did2);

-

-    /* similar to the previous H5Dcreate. */

-    did3 = H5Dcreate_ff(file_id,"G1/G2/G3/D3",int_id,dataspaceId,

-                        H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT, 0, event_q);

-    assert(did3);

-

-    /* Attach a checksum to the dxpl which is verified all the way

-       down at the server */

-    dxpl_id = H5Pcreate (H5P_DATASET_XFER);

-    cs = H5_checksum_fletcher32(data, sizeof(int) * nelem);

-    H5Pset_dxpl_checksum(dxpl_id, cs);

-

-    /* Raw data write on D1. This is asynchronous, but it is delayed

-       internally at the server until the create for D1

-       is completed.  Internal to the IOD-VOL plugin client we

-       generate a checksum for data and ship it with the write bulk

-       data function shipper handle to the server. */

-    H5Dwrite_ff(did1, int_id, dataspaceId, dataspaceId, dxpl_id, data, 

-                0, event_q);

-

-    cs = H5_checksum_fletcher32(data2, sizeof(int) * nelem);

-    H5Pset_dxpl_checksum(dxpl_id, cs);

-

-    /* Raw data write on D2. same as previous. */

-    H5Dwrite_ff(did2, int_id, dataspaceId, dataspaceId, dxpl_id, data2, 

-                0, event_q);

-

-    H5Pclose(dxpl_id);

-

-    /* Raw data write on D3. Same as previous; however we specify that

-       the data in the buffer is in BE byte order and of smaller

-       extent than the datatype of the dataset. Type conversion will

-       happen at the server when we detect that the dataset type is of

-       LE order and the datatype here is in BE order and the

-       difference in size. The buffer is resized at the server also to

-       encompass the increase. */

-    H5Dwrite_ff(did3, H5T_STD_I16LE, dataspaceId, dataspaceId, H5P_DEFAULT, data3, 

-                0, event_q);

-

-    /* NOTE: all raw data reads/writes execute concurrently at the

-       server if they get scheduled by the Asynchronous eXecution

-       Engine, AXE, (i.e. no dependencies on each other). */

-

-

-    /* Pop the request from the event queue to wait on it next. This

-       will return the request associated with the last H5Dwrite_ff

-       call on D3. Note that this also removes the request from the

-       event queue, so the user now owns the request and is

-       responsible to ensure its completion. */

-    if(H5EQpop(event_q, &req1) < 0)

-        exit(1);

-

-    /* Wait on the write request. This is not required to be done now.

-       But we are demoing how a particular request of interest can be

-       obtained and waited on. */

-    assert(H5AOwait(req1, &status1) == 0);

-    assert (status1);

-

-    /* Raw data read on D1. This is asynchronous.

-

-       At the server side, since the IOD library is "skeletal" and no

-       data exists, I am creating an array with the same size and

-       elements as the data that is written.

-

-       The server returns, along with the data array, a checksum for

-       the data that should be stored, but for now generated.  */

-    H5Dread_ff(did1, int_id, dataspaceId, dataspaceId, H5P_DEFAULT, r_data, 

-               0, event_q);

-

-    /* Pop head request from the event queue to test it next. This is the 

-       request that belongs to the previous H5Dread_ff call. */

-    if(H5EQpop(event_q, &req1) < 0)

-        exit(1);

-

-    /* Test if the Read operation has completed. Since it is asynchronous, It is

-       most likely that the operation is pending */

-    assert(H5AOtest(req1, &status1) == 0);

-    (status1 == H5AO_PENDING) ? fprintf(stderr, "Read is still pending\n") : fprintf(stderr, "Read has completed\n");

-

-    /* Print the received buffer before a completion call on the read is 

-       issued. This should print 0s or partial data recieved. */

-    fprintf(stderr, "Printing Just after Test (before waiting) ");

-    for(i=0;i<nelem;++i)

-        fprintf(stderr, "%d ",r_data[i]);

-    fprintf(stderr, "\n");

-

-    /* Here we demo that we can pass hints down to the IOD server. 

-       We create a new property, for demo purposes, to tell the server to inject 

-       corrupted data into the received array, and hence an incorrect checksum. 

-       This also detects that we are passing checksum values in both directions for 

-       raw data to ensure data integrity. The read should fail when we wait on it in

-       the H5Dclose(D1) later, but for the demo purposes we are not actually going to 

-       fail the close, but just print a Fatal error. */

-    dxpl_id = H5Pcreate (H5P_DATASET_XFER);

-    H5Pset_dxpl_inject_bad_checksum(dxpl_id, 1);

-    H5Dread_ff(did1, int_id, dataspaceId, dataspaceId, dxpl_id, r2_data, 

-               0, event_q);

-    H5Pclose(dxpl_id);

-

-    /* Issue other operations to query certain metadata values or

-       update previously created objects */

-    {

-        unsigned intent;

-        char temp_name[50];

-        hid_t plist_id;

-

-        H5Fget_intent(file_id, &intent);

-        fprintf(stderr, "Intent %d   %d\n", intent, H5F_ACC_RDWR);

-        H5Fget_name(gid1, temp_name, 50);

-        fprintf(stderr, "File name %s   %s\n", temp_name, file_name);

-

-        plist_id = H5Fget_access_plist(file_id);

-        assert(plist_id);

-        H5Pclose(plist_id);

-

-        plist_id = H5Fget_create_plist(file_id);

-        assert(plist_id);

-        H5Pclose(plist_id);

-

-        /* change the dataset dimensions for Dataset D1. */

-        assert(H5Dset_extent_ff(did1, &extent, event_q) == 0);

-    }

-

-    /* closing did1 acts as a barrier task at the server for all

-       operations dependeing on the dataset. This is asynchronous. */

-    assert(H5Dclose_ff(did1, event_q) == 0);

-

-    assert(H5Dclose_ff(did2, event_q) == 0);

-    assert(H5Dclose_ff(did3, event_q) == 0);

-    assert(H5Aclose_ff(aid2, event_q) == 0);

-    assert(H5Tclose_ff(int_id, event_q) == 0);

-    assert(H5Gclose_ff(gid1, event_q) == 0);

-    assert(H5Gclose_ff(gid2, event_q) == 0);

-    assert(H5Gclose_ff(gid3, event_q) == 0);

-

-    /* Test Links */

-

-    /* create two new groups /G4 and /G4/G5 */

-    gid1 = H5Gcreate_ff(file_id, "G4", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, 0, event_q);

-    gid2 = H5Gcreate_ff(file_id, "G4/G5", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, 0, event_q);

-

-    /* create a hard link to D1 (created previosuly) so that it can be

-       accessed from G5/newD1. This is asynchronous; however all

-       operation that depend on G5 now will have this operation as a

-       parent at the AXE on the server. */

-    H5Lcreate_hard_ff(file_id, "G1/G2/G3/D1", gid1, "G5/newD1", H5P_DEFAULT, H5P_DEFAULT, 0, event_q);

-

-    /* Try and open the dataset. This is asynchronous. */

-    did1 = H5Dopen_ff(file_id,"G4/G5/newD1", H5P_DEFAULT, 0, event_q);

-

-    H5Lcreate_soft_ff("/G1/G2/G3/D4", gid1, "G5/newD2", H5P_DEFAULT, H5P_DEFAULT, 0, event_q);

-    H5Lmove_ff(file_id, "/G1/G2/G3/D3", file_id, "/G4/G5/D3moved", H5P_DEFAULT, H5P_DEFAULT, 0, event_q);

-    H5Ldelete_ff(file_id, "/G1/G2/G3/D2", H5P_DEFAULT, 0, event_q);

-

-    assert(H5Dclose_ff(did1, event_q) == 0);

-    assert(H5Gclose_ff(gid1, event_q) == 0);

-    assert(H5Gclose_ff(gid2, event_q) == 0);

-

-    /* flush all the contents of file to disk. This is a barrier task

-       at the server. This is asynchronous. */

-    assert(H5Fflush_ff(file_id, H5F_SCOPE_GLOBAL, event_q) == 0);

-

-    /* If the read request did no complete earlier when we tested it, Wait on it now.

-       We have to do this since we popped it earlier from the event queue */

-    if(H5AO_PENDING == status1) {

-        assert(H5AOwait(req1, &status1) == 0);

-        assert (status1);

-    }

-    else

-        assert(H5AO_SUCCEEDED == status1);

-

-    /* closing the container also acts as a wait all on all pending requests 

-       on the container. */

-    assert(H5Fclose_ff(file_id, event_q) == 0);

-

-    /* wait on all requests and print completion status */

-    H5EQwait(event_q, &num_requests, &status);

-    fprintf(stderr, "%d requests in event queue. Completions: ", num_requests);

-    for(i=0 ; i<num_requests; i++)

-        fprintf(stderr, "%d ",status[i]);

-    fprintf(stderr, "\n");

-    free(status);

-

-    /* wait again.. event queue should be empty now */

-    H5EQwait(event_q, &num_requests, &status);

-    fprintf(stderr, "%d requests in event queue. Expecting 0. Completions: ", num_requests);

-    for(i=0 ; i<num_requests; i++)

-        fprintf(stderr, "%d ",status[i]);

-    fprintf(stderr, "\n");

-    free(status);

-

-    /* Now we can check operations that were issued previously */

-    if(exists)

-        printf("Attribute ATTR1 exists after being removed! Something is wrong!\n");

-    else

-        printf("Attribute ATTR1 does NOT exist. Good, it was removed!\n");

-

-    /* Print the data that has been read, after we have issued a wait 

-       (in the H5Dclose).

-       This should printf the correct array (0-59) */

-    fprintf(stderr, "Printing After Waiting ");

-    for(i=0;i<nelem;++i)

-        fprintf(stderr, "%d ",r_data[i]);

-    fprintf(stderr, "\n");

-

-    /* Print the data that has been read with an injected fault,

-       This should print the array similar to the previous one, but with the 

-       first value modified to be 10 (the injected error) */

-    fprintf(stderr, "Printing Corrupted Data ");

-    for(i=0;i<nelem;++i)

-        fprintf(stderr, "%d ",r2_data[i]);

-    fprintf(stderr, "\n");

-

-    /* Now we test the Open routines. Since there is no underneath

-       container, the underlying VOL server is just going to "fake"

-       open calls and assume they exist. However there is no metadata

-       information returned since nothing is stored on disk for

-       now. */

-

-    /* Open the file. This is asynchronous. Waiting on requests can be

-       left for the IOD VOL plugin to handle as necessary, as we do

-       here. We also can wait on a request using the new H5AOwait()

-       routine */

-    file_id = H5Fopen_ff(file_name, H5F_ACC_RDONLY, fapl_id, event_q);

-    assert(file_id);

-

-    /* test object copy. This is asynchronous */

-    assert(H5Ocopy_ff(file_id, "/G1/G2/G3/D1", file_id, "D1_copy", 

-                      H5P_DEFAULT, H5P_DEFAULT, 0, event_q) == 0);

-

-    /* check if an object exists. This is asynchronous, so checking

-       the value should be done after the wait */

-    H5Oexists_by_name_ff(file_id, "G1/G2/G3", &exists, H5P_DEFAULT, 0, event_q);

-

-    /* Open objects using the general H5O routines. This has an

-       asynchronous interface, but right now we implement it

-       synchronously, because we don't the object type to be able to

-       generate the ID. */

-    gid1 = H5Oopen_ff(file_id, "G1", H5P_DEFAULT, 0, event_q);

-    assert(gid1);

-    int_id = H5Oopen_ff(file_id, "int", H5P_DEFAULT, 0, event_q);

-    assert(int_id);

-    did1 = H5Oopen_ff(file_id,"G1/G2/G3/D1", H5P_DEFAULT, 0, event_q);

-    assert(did1);

-

-    /* open attribute by name. This is asynchronous. */

-    aid2 = H5Aopen_by_name_ff(file_id, "G1/G2/G3/D1", "ATTR2", 

-                              H5P_DEFAULT, H5P_DEFAULT, 0, event_q);

-    assert(aid2);

-

-    assert(H5Aclose_ff(aid2, event_q) == 0);

-    assert(H5Oclose_ff(did1, event_q) == 0);

-    assert(H5Oclose_ff(int_id, event_q) == 0);

-    assert(H5Oclose_ff(gid1, event_q) == 0);

-

-    /* Open a group G1 on the file. This is asynchronous */

-    gid1 = H5Gopen_ff(file_id, "G1", H5P_DEFAULT, 0, event_q);

-    assert(gid1);

-

-    /* get operations on the group */

-    {

-        ssize_t ret_size;

-        char *comment = NULL;

-

-        H5Oget_comment_ff(gid1, NULL, 0, &ret_size, 0, event_q);

-        if(H5EQpop(event_q, &req1) < 0)

-            exit(1);

-        assert(H5AOwait(req1, &status1) == 0);

-        assert (status1);

-        fprintf(stderr, "size of comment is %d\n", ret_size);

-

-        comment = malloc((size_t)ret_size);

-

-        H5Oget_comment_ff(gid1, comment, (size_t)ret_size + 1, &ret_size, 0, event_q);

-        if(H5EQpop(event_q, &req1) < 0)

-            exit(1);

-        assert(H5AOwait(req1, &status1) == 0);

-        assert (status1);

-        fprintf(stderr, "size of comment is %d Comment is %s\n", ret_size, comment);

-        free(comment);

-    }

-

-    /* Open a named datatype in the file. 

-     * This is implemented synchronously for now. */

-    int_id = H5Topen_ff(file_id, "int", H5P_DEFAULT, 0, event_q);

-    assert(int_id);

-

-    /* Open a dataset D1 on the file in a group hierarchy. 

-       This is asynchronous */

-    did1 = H5Dopen_ff(file_id,"G1/G2/G3/D1", H5P_DEFAULT, 0, event_q);

-    assert(did1);

-

-    /* Raw data read on D1. This is asynchronous.  Note that the type

-       is different than the dataset type. The dataset was created in

-       int LE order with data written to it earlier in the same type

-       (0 - 60).

-

-       Reading this now will read those values, and convert them to BE

-       16 bit integers at the server and send them to the

-       client. Printing this data will result in 0 - 60 in 16 bit BE byte

-       order. */

-    memset(r_data, 0, nelem*sizeof(int));

-    H5Dread_ff(did1, H5T_STD_I16BE, dataspaceId, dataspaceId, H5P_DEFAULT, r_data, 

-               0, event_q);

-    if(H5EQpop(event_q, &req1) < 0)

-        exit(1);

-    assert(H5AOwait(req1, &status1) == 0);

-    assert (status1);

-    fprintf(stderr, "Printing Dataset value read in 16 bit BE order. This should be interesting: ");

-    for(i=0;i<nelem;++i)

-        fprintf(stderr, "%d ",r_data[i]);

-    fprintf(stderr, "\n");

-

-    /* Raw data read on D1. This is asynchronous.  The read is done into a 

-       noncontiguous memory dataspace selection */

-    {

-        hid_t mem_space;

-        hsize_t start = 0;

-        hsize_t stride = 2;

-        hsize_t count = 60;

-        hsize_t block = 1;

-        int *buf = NULL;

-

-        buf = calloc (120, sizeof(int));

-

-        /* create a dataspace. This is a local Bookeeping operation that 

-           does not touch the file */

-        dims [0] = 120;

-        mem_space = H5Screate_simple(1, dims, NULL);

-        H5Sselect_hyperslab(mem_space, H5S_SELECT_SET, &start,&stride,&count,&block);

-

-        H5Dread_ff(did1, H5T_STD_I32LE, mem_space, dataspaceId, H5P_DEFAULT, buf, 

-                   0, event_q);

-

-        if(H5EQpop(event_q, &req1) < 0)

-            exit(1);

-        assert(H5AOwait(req1, &status1) == 0);

-        assert (status1);

-

-        fprintf(stderr, "Printing all Dataset values. We should have a 0 after each element: ");

-        for(i=0;i<120;++i)

-            fprintf(stderr, "%d ", buf[i]);

-        fprintf(stderr, "\n");

-

-        H5Sclose(mem_space);

-        free(buf);

-    }

-

-    assert(H5Dclose(did1) == 0);

-

-    /* open attribute on dataset D1. This is asynchronous */

-    aid2 = H5Aopen_by_name_ff(file_id, "G1/G2/G3/D1", "ATTR2", H5P_DEFAULT, H5P_DEFAULT, 0, event_q);

-    assert(aid2);

-

-    /* read data from attribute. this is asynchronous */

-    H5Aread_ff(aid2, int_id, ra_data, 0, event_q);

-    assert(H5Aclose(aid2) == 0);

-

-    assert(H5Tclose(int_id) == 0);

-    assert(H5Gclose(gid1) == 0);

-    assert(H5Fclose(file_id) == 0);

-

-    /* wait on all requests in event queue */

-    H5EQwait(event_q, &num_requests, &status);

-    fprintf(stderr, "%d requests in event queue. Completions: ", num_requests);

-    for(i=0 ; i<num_requests; i++)

-        fprintf(stderr, "%d ",status[i]);

-    fprintf(stderr, "\n");

-    free(status);

-

-    fprintf(stderr, "Printing Attribute data (after EQ wait): ");

-    for(i=0;i<nelem;++i)

-        fprintf(stderr, "%d ",ra_data[i]);

-    fprintf(stderr, "\n");

-

-    if(exists)

-        printf("Group G3 exists!\n");

-    else

-        printf("Group G3 does NOT exist. This must be the test without a native backend\n");

-

-    H5EQclose(event_q);

-    H5Pclose(fapl_id);

-    H5Sclose(dataspaceId);

-

-    /*

-    assert(H5AOwait(&req1, &status1) == 0);

-    assert (status1);

-    assert(H5AOwait(&req2, &status2) == 0);

-    assert (status2);

-    assert(H5AOwait(&req3, &status3) == 0);

-    assert (status3);

-    */

-    free(data);

-    free(r_data);

-    free(ra_data);

-    free(a_data);

-    free(data2);

-    free(r2_data);

-    free(r3_data);

-    free(data3);

-

-    /* This finalizes the EFF stack. ships a terminate and IOD finalize to the server 

-       and shutsdown the FS server (when all clients send the terminate request) 

-       and client */

-    EFF_finalize();

-

-    MPI_Finalize();

-    return 0;

-}

-