[svn-r24603] - capture oid index metadata information on every tr_finish.

- add special bootsrapping option on H5Fopen when file does not shut down with H5Fclose
- add fixes to allow library to build properly without EFF third party libs
- fix several bugs
- fix issues reported by valgrind

13 files changed, 429 insertions, 163 deletions

diff --git a/src/H5.c b/src/H5.c
index a1bfbf8..98b1c13 100644
--- a/src/H5.c
+++ b/src/H5.c
@@ -255,12 +255,16 @@ H5_term_library(void)
 	pending += DOWN(D);
 	pending += DOWN(L);
 	pending += DOWN(G);
+#ifdef H5_HAVE_EFF
 	pending += DOWN(M);
+#endif /* H5_HAVE_EFF */
 	pending += DOWN(A);
 	pending += DOWN(S);
 	pending += DOWN(T);
+#ifdef H5_HAVE_EFF
 	pending += DOWN(TR);
 	pending += DOWN(RC);
+#endif /* H5_HAVE_EFF */
 	pending += DOWN(ES);
 	pending += DOWN(Q);
 
diff --git a/src/H5Eprivate.h b/src/H5Eprivate.h
index 8c6548d..1bb55aa 100644
--- a/src/H5Eprivate.h
+++ b/src/H5Eprivate.h
@@ -69,6 +69,17 @@ typedef struct H5E_t H5E_t;
    HGOTO_DONE(ret_val)						              \
 }
 
+#define HGOTO_ERROR_IOD(iod_ret, ret_val, string) {			      \
+   fprintf(stderr, "%s\n", string);                                           \
+   fprintf(stderr, "%d (%s).\n", iod_ret, strerror(-iod_ret));                \
+   HGOTO_DONE(ret_val)						              \
+}
+
+#define HGOTO_ERROR_FF(ret_val, string) {			              \
+   fprintf(stderr, "%s\n", string);                                           \
+   HGOTO_DONE(ret_val)						              \
+}
+
 #define HGOTO_ERROR2(maj, min, ret_val, string) {			      \
    fprintf(stderr, "%s\n", string);                                           \
    err_occurred = TRUE;                                                       \
diff --git a/src/H5M.c b/src/H5M.c
index bbf5f48..e916b44 100644
--- a/src/H5M.c
+++ b/src/H5M.c
@@ -33,6 +33,8 @@
 #include "H5VLprivate.h"	/* VOL plugins				*/
 #include "H5VLiod_client.h"	/* IOD VOL plugin			*/
 
+#ifdef H5_HAVE_EFF
+
 /****************/
 /* Local Macros */
 /****************/
@@ -944,3 +946,5 @@ H5M_close_map(void *map, H5VL_t *vol_plugin)
 done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5M_close_map() */
+
+#endif /* H5_HAVE_EFF */
diff --git a/src/H5Ptrfpl.c b/src/H5Ptrfpl.c
index d072eb5..3029ff1 100644
--- a/src/H5Ptrfpl.c
+++ b/src/H5Ptrfpl.c
@@ -129,3 +129,4 @@ H5P__trfcc_reg_prop(H5P_genclass_t *pclass)
 done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5P__trfcc_reg_prop() */
+
diff --git a/src/H5Sselect.c b/src/H5Sselect.c
index 94d87fb..eabfc09 100644
--- a/src/H5Sselect.c
+++ b/src/H5Sselect.c
@@ -31,7 +31,9 @@
 #include "H5Spkg.h"		/* Dataspaces 				*/
 #include "H5Vprivate.h"		/* Vector and array functions		*/
 #include "H5WBprivate.h"        /* Wrapped Buffers                      */
+#ifdef H5_HAVE_EFF
 #include "mchecksum.h"          /* Mercury Checksum library             */
+#endif /* H5_HAVE_EFF */
 
 /* Local functions */
 #ifdef LATER
@@ -2145,6 +2147,7 @@ done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5S_get_offsets() */
 
+#ifdef H5_HAVE_EFF
 
 /*-------------------------------------------------------------------------
  * Function:	H5S_checksum
@@ -2237,3 +2240,4 @@ done:
             HDONE_ERROR(H5E_DATASPACE, H5E_CANTRELEASE, 0, "unable to release selection iterator")
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5S_checksum */
+#endif /* H5_HAVE_EFF */
diff --git a/src/H5TRprivate.h b/src/H5TRprivate.h
index 4c20073..70f5e5e 100644
--- a/src/H5TRprivate.h
+++ b/src/H5TRprivate.h
@@ -24,13 +24,13 @@
 /* Private headers needed by this file */
 #include "H5RCprivate.h"	/* Read Contexts			*/
 
-#ifdef H5_HAVE_EFF
-
 /**************************/
 /* Library Private Macros */
 /**************************/
 #define H5TR_FINISH_ACQUIRE_NAME "acquire_after_finishing"
 
+#ifdef H5_HAVE_EFF
+
 /****************************/
 /* Library Private Typedefs */
 /****************************/
diff --git a/src/H5VLiod.c b/src/H5VLiod.c
index 6586c26..b59bbc4 100644
--- a/src/H5VLiod.c
+++ b/src/H5VLiod.c
@@ -1542,8 +1542,21 @@ H5VL_iod_file_create(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl
     file->remote_file.root_oh.wr_oh.cookie = IOD_OH_UNDEFINED;
     file->remote_file.root_id = IOD_OBJ_INVALID;
     file->remote_file.c_version = 0;
+
     MPI_Comm_rank(fa->comm, &file->my_rank);
     MPI_Comm_size(fa->comm, &file->num_procs);
+
+    if(0 == file->my_rank) {
+        file->remote_file.kv_oid_index = 4;
+        file->remote_file.array_oid_index = 0;
+        file->remote_file.blob_oid_index = 0;
+    }
+    else {
+        file->remote_file.kv_oid_index = 0;
+        file->remote_file.array_oid_index = 0;
+        file->remote_file.blob_oid_index = 0;
+    }
+
     /* Duplicate communicator and Info object. */
     if(FAIL == H5FD_mpi_comm_info_dup(fa->comm, fa->info, &file->comm, &file->info))
 	HGOTO_ERROR(H5E_INTERNAL, H5E_CANTCOPY, NULL, "Communicator/Info duplicate failed");
@@ -1560,6 +1573,10 @@ H5VL_iod_file_create(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl
     H5VL_iod_gen_obj_id(0, file->num_procs, (uint64_t)2, IOD_OBJ_KV, &input.attrkv_id);
     file->remote_file.attrkv_id = input.attrkv_id;
 
+    /* Generate an IOD ID for the OID index array to be created */
+    H5VL_iod_gen_obj_id(0, file->num_procs, (uint64_t)3, IOD_OBJ_KV, &input.oidkv_id);
+    file->remote_file.oidkv_id = input.oidkv_id;
+
     /* set the input structure for the HG encode routine */
     input.name = name;
     input.num_peers = (uint32_t)file->num_procs;
@@ -1687,8 +1704,12 @@ H5VL_iod_file_open(const char *name, unsigned flags, hid_t fapl_id,
     file->remote_file.root_id = IOD_OBJ_INVALID;
     file->remote_file.mdkv_id = IOD_OBJ_INVALID;
     file->remote_file.attrkv_id = IOD_OBJ_INVALID;
+    file->remote_file.oidkv_id = IOD_OBJ_INVALID;
     file->remote_file.fcpl_id = -1;
     file->remote_file.c_version = IOD_TID_UNKNOWN;
+    file->remote_file.kv_oid_index = 0;
+    file->remote_file.array_oid_index = 0;
+    file->remote_file.blob_oid_index = 0;
 
     /* set input paramters in struct to give to the function shipper */
     input.name = name;
@@ -2040,8 +2061,17 @@ H5VL_iod_file_close(void *_file, hid_t UNUSED dxpl_id, void **req)
     printf("File Close Root ID %"PRIu64" axe id %"PRIu64"\n", input.root_id, g_axe_id);
 #endif
 
+    /* 
+     * All ranks must wait for rank 0 to go close the file and write
+     * out the metadata before going on an closing the file. Rank 0
+     * calls the barrier on request completion in
+     * H5VLiod_client.c. 
+
+     * (This is an IOD limitation.) 
+     */
     if(0 != file->my_rank)
         MPI_Barrier (file->comm);
+
     if(H5VL__iod_create_and_forward(H5VL_FILE_CLOSE_ID, HG_FILE_CLOSE, 
                                     (H5VL_iod_object_t *)file, 1,
                                     num_parents, parent_reqs,
@@ -2860,8 +2890,8 @@ H5VL_iod_dataset_read(void *_dset, hid_t mem_type_id, hid_t mem_space_id,
     H5P_genplist_t *plist = NULL;
     hg_bulk_t *bulk_handle = NULL;
     H5VL_iod_read_status_t *status = NULL;
-    const H5S_t *mem_space = NULL;
-    const H5S_t *file_space = NULL;
+    H5S_t *mem_space = NULL;
+    H5S_t *file_space = NULL;
     char fake_char;
     size_t type_size; /* size of mem type */
     hssize_t nelmts;    /* num elements in mem dataspace */
@@ -2875,16 +2905,6 @@ H5VL_iod_dataset_read(void *_dset, hid_t mem_type_id, hid_t mem_space_id,
 
     FUNC_ENTER_NOAPI_NOINIT
 
-    /* get the context ID */
-    if(NULL == (plist = (H5P_genplist_t *)H5I_object(dxpl_id)))
-        HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID");
-    if(H5P_get(plist, H5VL_CONTEXT_ID, &rcxt_id) < 0)
-        HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get property value for trans_id");
-
-    /* get the RC object */
-    if(NULL == (rc = (H5RC_t *)H5I_object_verify(rcxt_id, H5I_RC)))
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a READ CONTEXT ID")
-
     /* If there is information needed about the dataset that is not present locally, wait */
     if(-1 == dset->remote_dset.dcpl_id ||
        -1 == dset->remote_dset.type_id ||
@@ -2895,6 +2915,57 @@ H5VL_iod_dataset_read(void *_dset, hid_t mem_type_id, hid_t mem_space_id,
         dset->common.request = NULL;
     }
 
+    /* check arguments */
+    if(H5S_ALL != mem_space_id) {
+	if(NULL == (mem_space = (H5S_t *)H5I_object_verify(mem_space_id, H5I_DATASPACE)))
+	    HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data space");
+
+	/* Check for valid selection */
+	if(H5S_SELECT_VALID(mem_space) != TRUE)
+	    HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "selection+offset not within extent");
+    }
+    else {
+	if(NULL == (mem_space = (H5S_t *)H5I_object_verify(dset->remote_dset.space_id, 
+                                                                 H5I_DATASPACE)))
+	    HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data space");
+
+        if(H5S_select_all(mem_space, TRUE) < 0)
+            HGOTO_ERROR(H5E_DATASPACE, H5E_CANTDELETE, FAIL, "can't change selection")
+    }
+
+    if(H5S_ALL != file_space_id) {
+	if(NULL == (file_space = (H5S_t *)H5I_object_verify(file_space_id, H5I_DATASPACE)))
+	    HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data space");
+
+	/* Check for valid selection */
+	if(H5S_SELECT_VALID(file_space) != TRUE)
+	    HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "selection+offset not within extent");
+    }
+    else {
+	if(NULL == (file_space = (H5S_t *)H5I_object_verify(dset->remote_dset.space_id, 
+                                                                  H5I_DATASPACE)))
+	    HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data space");
+
+        if(H5S_select_all(file_space, TRUE) < 0)
+            HGOTO_ERROR(H5E_DATASPACE, H5E_CANTDELETE, FAIL, "can't change selection")
+    }
+
+    if(!buf && (NULL == file_space || H5S_GET_SELECT_NPOINTS(file_space) != 0))
+	HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no output buffer");
+
+    if(!buf)
+        buf = &fake_char;
+
+    /* get the context ID */
+    if(NULL == (plist = (H5P_genplist_t *)H5I_object(dxpl_id)))
+        HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID");
+    if(H5P_get(plist, H5VL_CONTEXT_ID, &rcxt_id) < 0)
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get property value for trans_id");
+
+    /* get the RC object */
+    if(NULL == (rc = (H5RC_t *)H5I_object_verify(rcxt_id, H5I_RC)))
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a READ CONTEXT ID")
+
     /* At the IOD server, the array object is actually created with
        MAX dims, not current dims. So here we need to range check the
        filespace against the current dimensions of the dataset to make
@@ -2925,30 +2996,6 @@ H5VL_iod_dataset_read(void *_dset, hid_t mem_type_id, hid_t mem_space_id,
         }
     }
 
-    /* check arguments */
-    if(H5S_ALL != mem_space_id) {
-	if(NULL == (mem_space = (const H5S_t *)H5I_object_verify(mem_space_id, H5I_DATASPACE)))
-	    HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data space");
-
-	/* Check for valid selection */
-	if(H5S_SELECT_VALID(mem_space) != TRUE)
-	    HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "selection+offset not within extent");
-    } /* end if */
-    if(H5S_ALL != file_space_id) {
-	if(NULL == (file_space = (const H5S_t *)H5I_object_verify(file_space_id, H5I_DATASPACE)))
-	    HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data space");
-
-	/* Check for valid selection */
-	if(H5S_SELECT_VALID(file_space) != TRUE)
-	    HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "selection+offset not within extent");
-    } /* end if */
-
-    if(!buf && (NULL == file_space || H5S_GET_SELECT_NPOINTS(file_space) != 0))
-	HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no output buffer");
-
-    if(!buf)
-        buf = &fake_char;
-
     /* get the memory type size */
     {
         H5T_t *dt = NULL;
@@ -3099,8 +3146,8 @@ H5VL_iod_dataset_write(void *_dset, hid_t mem_type_id, hid_t mem_space_id,
     dset_io_in_t input;
     H5P_genplist_t *plist = NULL;
     hg_bulk_t *bulk_handle = NULL;
-    const H5S_t *mem_space = NULL;
-    const H5S_t *file_space = NULL;
+    H5S_t *mem_space = NULL;
+    H5S_t *file_space = NULL;
     char fake_char;
     int *status = NULL;
     H5VL_iod_io_info_t *info; /* info struct used to manage I/O parameters once the operation completes*/
@@ -3126,6 +3173,47 @@ H5VL_iod_dataset_write(void *_dset, hid_t mem_type_id, hid_t mem_space_id,
         dset->common.request = NULL;
     }
 
+    /* check arguments */
+    if(H5S_ALL != mem_space_id) {
+	if(NULL == (mem_space = (H5S_t *)H5I_object_verify(mem_space_id, H5I_DATASPACE)))
+	    HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data space");
+
+	/* Check for valid selection */
+	if(H5S_SELECT_VALID(mem_space) != TRUE)
+	    HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "selection+offset not within extent");
+    }
+    else {
+	if(NULL == (mem_space = (H5S_t *)H5I_object_verify(dset->remote_dset.space_id, 
+                                                           H5I_DATASPACE)))
+	    HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data space");
+
+        if(H5S_select_all(mem_space, TRUE) < 0)
+            HGOTO_ERROR(H5E_DATASPACE, H5E_CANTDELETE, FAIL, "can't change selection")
+    }
+
+    if(H5S_ALL != file_space_id) {
+	if(NULL == (file_space = (H5S_t *)H5I_object_verify(file_space_id, H5I_DATASPACE)))
+	    HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data space");
+
+	/* Check for valid selection */
+	if(H5S_SELECT_VALID(file_space) != TRUE)
+	    HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "selection+offset not within extent");
+    }
+    else {
+	if(NULL == (file_space = (H5S_t *)H5I_object_verify(dset->remote_dset.space_id, 
+                                                                  H5I_DATASPACE)))
+	    HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data space");
+
+        if(H5S_select_all(file_space, TRUE) < 0)
+            HGOTO_ERROR(H5E_DATASPACE, H5E_CANTDELETE, FAIL, "can't change selection")
+    }
+
+    if(!buf && (NULL == file_space || H5S_GET_SELECT_NPOINTS(file_space) != 0))
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no output buffer");
+
+    if(!buf)
+        buf = &fake_char;
+
     /* At the IOD server, the array object is actually created with
        MAX dims, not current dims. So here we need to range check the
        filespace against the current dimensions of the dataset to make
@@ -3156,30 +3244,6 @@ H5VL_iod_dataset_write(void *_dset, hid_t mem_type_id, hid_t mem_space_id,
         }
     }
 
-    /* check arguments */
-    if(H5S_ALL != mem_space_id) {
-	if(NULL == (mem_space = (const H5S_t *)H5I_object_verify(mem_space_id, H5I_DATASPACE)))
-	    HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data space");
-
-	/* Check for valid selection */
-	if(H5S_SELECT_VALID(mem_space) != TRUE)
-	    HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "selection+offset not within extent");
-    } /* end if */
-    if(H5S_ALL != file_space_id) {
-	if(NULL == (file_space = (const H5S_t *)H5I_object_verify(file_space_id, H5I_DATASPACE)))
-	    HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data space");
-
-	/* Check for valid selection */
-	if(H5S_SELECT_VALID(file_space) != TRUE)
-	    HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "selection+offset not within extent");
-    } /* end if */
-
-    if(!buf && (NULL == file_space || H5S_GET_SELECT_NPOINTS(file_space) != 0))
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no output buffer");
-
-    if(!buf)
-        buf = &fake_char;
-
     /* get the plist pointer */
     if(NULL == (plist = (H5P_genplist_t *)H5I_object(dxpl_id)))
         HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID");
@@ -7886,7 +7950,7 @@ done:
 /*-------------------------------------------------------------------------
  * Function:	H5VL_iod_rc_release
  *
- * Purpose:	Forwards an release on an acquired read context to IOD
+ * Purpose:	Forwards a release on an acquired read context to IOD
  *
  * Return:	Success:	SUCCEED
  *		Failure:	FAIL
@@ -8138,6 +8202,7 @@ H5VL_iod_tr_start(H5TR_t *tr, hid_t trspl_id, void **req)
         H5VL_iod_request_t *request = (H5VL_iod_request_t *)(*req);
 
         tr->req_info.request = request;
+        //request->ref_count ++;
     }
 
 done:
@@ -8243,6 +8308,11 @@ H5VL_iod_tr_finish(H5TR_t *tr, hbool_t acquire, hid_t trfpl_id, void **req)
     input.trans_num = tr->trans_num;
     input.trfpl_id = trfpl_id;
     input.acquire = acquire;
+    input.client_rank = tr->file->my_rank;
+    input.oidkv_id = tr->file->num_procs * 3;
+    input.kv_oid_index = tr->file->remote_file.kv_oid_index;
+    input.array_oid_index = tr->file->remote_file.array_oid_index;
+    input.blob_oid_index = tr->file->remote_file.blob_oid_index;
 
     status = (int *)malloc(sizeof(int));
 
diff --git a/src/H5VLiod_client.c b/src/H5VLiod_client.c
index 660ee89..3ac131a 100644
--- a/src/H5VLiod_client.c
+++ b/src/H5VLiod_client.c
@@ -1498,6 +1498,7 @@ H5VL_iod_request_complete(H5VL_iod_file_t *file, H5VL_iod_request_t *req)
 
             rc_info->read_cxt->c_version = rc_info->result.c_version;
             *rc_info->c_version_ptr = rc_info->result.c_version;
+            rc_info->read_cxt->req_info.request = NULL;
             rc_info = (H5VL_iod_rc_info_t *)H5MM_xfree(rc_info);
             req->data = NULL;
             H5VL_iod_request_delete(file, req);
@@ -2084,7 +2085,9 @@ H5VL_iod_get_parent_requests(H5VL_iod_object_t *obj, H5VL_iod_req_info_t *req_in
         count ++;
     }
 
-    if(req_info && req_info->request && req_info->request->status == H5ES_STATUS_IN_PROGRESS) {
+    if(req_info && 
+       req_info->request && 
+       req_info->request->status == H5ES_STATUS_IN_PROGRESS) {
         parent_reqs[count] = req_info->request;
         req_info->request->ref_count ++;
         count ++;
@@ -2104,8 +2107,7 @@ H5VL_iod_gen_obj_id(int myrank, int nranks, uint64_t cur_index,
 
     FUNC_ENTER_NOAPI_NOINIT
 
-    /* determine first the rank of the object with the first 59
-       bits */
+    /* determine first the rank of the object with the first 59 bits */
     tmp_id = (uint32_t)myrank + ((uint32_t)nranks * cur_index);
 
     /* toggle the object type bits */
diff --git a/src/H5VLiod_client.h b/src/H5VLiod_client.h
index c38d53b..c4f9d2a 100644
--- a/src/H5VLiod_client.h
+++ b/src/H5VLiod_client.h
@@ -133,6 +133,7 @@ typedef struct H5VL_iod_remote_file_t {
     iod_obj_id_t root_id;
     iod_obj_id_t mdkv_id;
     iod_obj_id_t attrkv_id;
+    iod_obj_id_t oidkv_id;
     uint64_t c_version;
     hid_t fcpl_id;
 } H5VL_iod_remote_file_t;
diff --git a/src/H5VLiod_common.h b/src/H5VLiod_common.h
index 63bf4aa..dcb01a4 100644
--- a/src/H5VLiod_common.h
+++ b/src/H5VLiod_common.h
@@ -146,15 +146,13 @@ MERCURY_GEN_PROC(file_create_in_t,
                  ((iod_obj_id_t)(root_id))
                  ((iod_obj_id_t)(mdkv_id)) 
                  ((iod_obj_id_t)(attrkv_id)) 
+                 ((iod_obj_id_t)(oidkv_id)) 
                  ((uint32_t)(num_peers))
                  ((hid_t)(fapl_id)) 
                  ((hid_t)(fcpl_id)))
 MERCURY_GEN_PROC(file_create_out_t, 
-                 ((iod_handle_t)(coh)) 
-                 ((iod_handles_t)(root_oh))
-                 ((uint64_t)(kv_oid_index)) 
-                 ((uint64_t)(array_oid_index))
-                 ((uint64_t)(blob_oid_index)))
+                 ((iod_handle_t)(coh))
+                 ((iod_handles_t)(root_oh)))
 MERCURY_GEN_PROC(file_open_in_t, 
                  ((axe_t)(axe_info)) 
                  ((hg_const_string_t)(name)) 
@@ -170,6 +168,7 @@ MERCURY_GEN_PROC(file_open_out_t,
                  ((iod_obj_id_t)(root_id)) 
                  ((iod_obj_id_t)(mdkv_id)) 
                  ((iod_obj_id_t)(attrkv_id)) 
+                 ((iod_obj_id_t)(oidkv_id)) 
                  ((uint64_t)(c_version)) 
                  ((hid_t)(fcpl_id)))
 MERCURY_GEN_PROC(file_close_in_t, 
@@ -178,9 +177,9 @@ MERCURY_GEN_PROC(file_close_in_t,
                  ((iod_handle_t)(coh)) 
                  ((iod_handles_t)(root_oh))
                  ((iod_obj_id_t)(root_id)) 
-                 ((uint64_t)(max_kv_index)) 
-                 ((uint64_t)(max_array_index)) 
-                 ((uint64_t)(max_blob_index)))
+                 ((iod_obj_id_t)(max_kv_index)) 
+                 ((iod_obj_id_t)(max_array_index)) 
+                 ((iod_obj_id_t)(max_blob_index)))
 
 MERCURY_GEN_PROC(attr_create_in_t, 
                  ((axe_t)(axe_info)) 
@@ -658,6 +657,11 @@ MERCURY_GEN_PROC(tr_finish_in_t,
                  ((iod_handle_t)(coh))
                  ((uint64_t)(trans_num))
                  ((hbool_t)(acquire))
+                 ((uint32_t)(client_rank))
+                 ((iod_obj_id_t)(oidkv_id))
+                 ((iod_obj_id_t)(kv_oid_index)) 
+                 ((iod_obj_id_t)(array_oid_index)) 
+                 ((iod_obj_id_t)(blob_oid_index))
                  ((hid_t)(trfpl_id)))
 MERCURY_GEN_PROC(tr_set_depend_in_t, 
                  ((axe_t)(axe_info))
diff --git a/src/H5VLiod_file.c b/src/H5VLiod_file.c
index 8aa6afd..0f94138 100644
--- a/src/H5VLiod_file.c
+++ b/src/H5VLiod_file.c
@@ -51,6 +51,7 @@ H5VL_iod_server_file_create_cb(AXE_engine_t UNUSED axe_engine,
     iod_obj_id_t root_id = input->root_id;
     iod_obj_id_t mdkv_id = input->mdkv_id;
     iod_obj_id_t attrkv_id = input->attrkv_id;
+    iod_obj_id_t oidkv_id = input->oidkv_id;
     unsigned int mode; /* create mode */
     iod_handle_t coh; /* container handle */
     iod_handles_t root_oh; /* root object handle */
@@ -66,6 +67,7 @@ H5VL_iod_server_file_create_cb(AXE_engine_t UNUSED axe_engine,
     fprintf(stderr, "Start file create %s ", input->name);
     fprintf(stderr, "with MDKV %"PRIx64" ", mdkv_id), 
     fprintf(stderr, "with attrKV %"PRIx64"\n", attrkv_id), 
+    fprintf(stderr, "with OIDKV %"PRIx64"\n", oidkv_id), 
 #endif
 
     /* convert HDF5 flags to IOD flags */
@@ -77,31 +79,28 @@ H5VL_iod_server_file_create_cb(AXE_engine_t UNUSED axe_engine,
         HGOTO_ERROR2(H5E_PLIST, H5E_CANTGET, FAIL, "can't get scope for data integrity checks");
 
     /* Create the Container */
-    if((ret = iod_container_open(input->name, NULL /*hints*/, mode, &coh, NULL /*event*/)) < 0) {
-        fprintf(stderr, "%d (%s).\n", ret, strerror(-ret));
-        HGOTO_ERROR2(H5E_FILE, H5E_CANTINIT, FAIL, "can't create container");
-    }
+    ret = iod_container_open(input->name, NULL, mode, &coh, NULL);
+    if(ret < 0)
+        HGOTO_ERROR_IOD(ret, FAIL, "can't create container");
 
-    if(iod_trans_start(coh, &first_tid, NULL, num_peers, IOD_TRANS_W, NULL) < 0)
-        HGOTO_ERROR2(H5E_SYM, H5E_CANTSET, FAIL, "can't start transaction");
+    ret = iod_trans_start(coh, &first_tid, NULL, num_peers, IOD_TRANS_W, NULL);
+    if(ret < 0)
+        HGOTO_ERROR_IOD(ret, FAIL, "can't start transaction");
 
     /* create the root group */
     root_ret = iod_obj_create(coh, first_tid, NULL, IOD_OBJ_KV, NULL, NULL, 
                          &root_id, NULL);
     if(0 == root_ret || -EEXIST == root_ret) {
-        //fprintf(stderr, "created Root group %"PRIx64"\n", root_id);
         /* root group has been created, open it */
-        if ((ret = iod_obj_open_write(coh, root_id, first_tid, NULL, &root_oh.wr_oh, NULL)) < 0) {
-            fprintf(stderr, "%d (%s).\n", ret, strerror(-ret));
-            HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "can't open root group for write");
-        }
-        if ((ret = iod_obj_open_read(coh, root_id, first_tid, NULL, &root_oh.rd_oh, NULL)) < 0) {
-            fprintf(stderr, "%d (%s).\n", ret, strerror(-ret));
-            HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "can't open root group for read");
-        }
+        ret = iod_obj_open_write(coh, root_id, first_tid, NULL, &root_oh.wr_oh, NULL);
+        if(ret < 0)
+            HGOTO_ERROR_IOD(ret, FAIL, "can't open root group for write");
+        ret = iod_obj_open_read(coh, root_id, first_tid, NULL, &root_oh.rd_oh, NULL);
+        if(ret < 0)
+            HGOTO_ERROR_IOD(ret, FAIL, "can't open root group for read");
     }
     else {
-        HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "can't create root group");
+        HGOTO_ERROR_IOD(ret, FAIL, "can't create root group");
     }
 
     /* for the process that succeeded in creating the group, create
@@ -112,23 +111,26 @@ H5VL_iod_server_file_create_cb(AXE_engine_t UNUSED axe_engine,
         uint64_t value = 1;
         hid_t fcpl_id;
 
-        //fprintf(stderr, "created MDKV and ATTRKV %"PRIx64"  %"PRIx64"\n", 
-        //mdkv_id, attrkv_id);
         /* create the metadata KV object for the root group */
-        if((ret = iod_obj_create(coh, first_tid, NULL, IOD_OBJ_KV, 
-                                 NULL, NULL, &mdkv_id, NULL)) < 0) {
-            fprintf(stderr, "%d (%s).\n", ret, strerror(-ret));
-            HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "can't create metadata KV object");
-        }
+        ret = iod_obj_create(coh, first_tid, NULL, IOD_OBJ_KV, NULL, NULL, &mdkv_id, NULL);
+        if(ret < 0)
+            HGOTO_ERROR_IOD(ret, FAIL, "can't create metadata KV object");
+
         /* create the attribute KV object for the root group */
-        if(iod_obj_create(coh, first_tid, NULL, IOD_OBJ_KV, 
-                          NULL, NULL, &attrkv_id, NULL) < 0)
-            HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "can't create attribute KV object");
+        ret = iod_obj_create(coh, first_tid, NULL, IOD_OBJ_KV, NULL, NULL, &attrkv_id, NULL);
+        if(ret < 0)
+            HGOTO_ERROR_IOD(ret, FAIL, "can't create attribute KV object");
+
+        /* create the KV object to hold each client's indexes for
+           object OIDs after each trans_finish and file_close */
+        ret = iod_obj_create(coh, first_tid, NULL, IOD_OBJ_KV, NULL, NULL, &oidkv_id, NULL);
+        if(ret != 0)
+            HGOTO_ERROR_IOD(ret, FAIL, "can't create array for OID indexes");
 
         /* set values for the scratch pad object */
         sp[0] = mdkv_id;
         sp[1] = attrkv_id;
-        sp[2] = IOD_OBJ_INVALID;
+        sp[2] = oidkv_id;
         sp[3] = IOD_OBJ_INVALID;
 
         if(cs_scope & H5_CHECKSUM_IOD) {
@@ -136,62 +138,63 @@ H5VL_iod_server_file_create_cb(AXE_engine_t UNUSED axe_engine,
 
             sp_cs = H5_checksum_crc64(&sp, sizeof(sp));
             /* set scratch pad in root group */
-            if (iod_obj_set_scratch(root_oh.wr_oh, first_tid, &sp, &sp_cs, NULL) < 0)
-                HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "can't set scratch pad");
+            ret = iod_obj_set_scratch(root_oh.wr_oh, first_tid, &sp, &sp_cs, NULL);
+            if(ret < 0)
+                HGOTO_ERROR_IOD(ret, FAIL, "can't set scratch pad");
         }
         else {
-#if H5VL_IOD_DEBUG
-            fprintf(stderr, "METADATA INTEGRITY DISABLED\n");
-#endif
-            if (iod_obj_set_scratch(root_oh.wr_oh, first_tid, &sp, NULL, NULL) < 0)
-                HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "can't set scratch pad");
+            ret = iod_obj_set_scratch(root_oh.wr_oh, first_tid, &sp, NULL, NULL);
+            if(ret < 0)
+                HGOTO_ERROR_IOD(ret, FAIL, "can't set scratch pad");
         }
 
         /* Store Metadata in scratch pad */
-        if (iod_obj_open_write(coh, input->mdkv_id, first_tid, NULL, &mdkv_oh, NULL) < 0)
-            HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "can't open metadata KV");
+        ret = iod_obj_open_write(coh, input->mdkv_id, first_tid, NULL, &mdkv_oh, NULL);
+        if(ret < 0)
+            HGOTO_ERROR_IOD(ret, FAIL, "can't open metadata KV");
 
         /* store metadata */
         if(H5P_DEFAULT == input->fcpl_id)
             input->fcpl_id = H5Pcopy(H5P_FILE_CREATE_DEFAULT);
         fcpl_id = input->fcpl_id;
         /* insert plist metadata */
-        if(H5VL_iod_insert_plist(mdkv_oh, first_tid, fcpl_id, 
-                                 NULL, NULL, NULL) < 0)
-            HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "can't insert link count KV value");
+        if(H5VL_iod_insert_plist(mdkv_oh, first_tid, fcpl_id, NULL, NULL, NULL) < 0)
+            HGOTO_ERROR2(H5E_SYM, H5E_CANTSET, FAIL, "can't insert link count KV value");
 
         kv.value = &value;
         kv.value_len = sizeof(uint64_t);
+
         kv.key = (void *)H5VL_IOD_KEY_KV_IDS_INDEX;
         kv.key_len = strlen(H5VL_IOD_KEY_KV_IDS_INDEX);
-
-        if (iod_kv_set(mdkv_oh, first_tid, NULL, &kv, NULL, NULL) < 0)
-            HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "can't set KV pair in parent");
+        ret = iod_kv_set(mdkv_oh, first_tid, NULL, &kv, NULL, NULL);
+        if(ret < 0)
+            HGOTO_ERROR_IOD(ret, FAIL, "can't set KV pair in parent");
 
         kv.key = (void *)H5VL_IOD_KEY_ARRAY_IDS_INDEX;
         kv.key_len = strlen(H5VL_IOD_KEY_ARRAY_IDS_INDEX);
-        if (iod_kv_set(mdkv_oh, first_tid, NULL, &kv, NULL, NULL) < 0)
-            HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "can't set KV pair in parent");
+        ret = iod_kv_set(mdkv_oh, first_tid, NULL, &kv, NULL, NULL);
+        if(ret < 0)
+            HGOTO_ERROR_IOD(ret, FAIL, "can't set KV pair in parent");
 
         kv.key = (void *)H5VL_IOD_KEY_BLOB_IDS_INDEX;
         kv.key_len = strlen(H5VL_IOD_KEY_BLOB_IDS_INDEX);
-        if (iod_kv_set(mdkv_oh, first_tid, NULL, &kv, NULL, NULL) < 0)
-            HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "can't set KV pair in parent");
+        ret = iod_kv_set(mdkv_oh, first_tid, NULL, &kv, NULL, NULL);
+        if(ret < 0)
+            HGOTO_ERROR_IOD(ret, FAIL, "can't set KV pair in parent");
 
-        if(iod_obj_close(mdkv_oh, NULL, NULL) < 0)
-            HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "can't close root object handle");
+        ret = iod_obj_close(mdkv_oh, NULL, NULL);
+        if(ret < 0)
+            HGOTO_ERROR_IOD(ret, FAIL, "can't close root object handle");
     }
 
     /* Finish  the transaction */
-    if(iod_trans_finish(coh, first_tid, NULL, 0, NULL) < 0)
-        HGOTO_ERROR2(H5E_SYM, H5E_CANTSET, FAIL, "can't finish transaction 0");
+    ret = iod_trans_finish(coh, first_tid, NULL, 0, NULL);
+    if(ret < 0)
+        HGOTO_ERROR_IOD(ret, FAIL, "can't finish transaction 0");
 
     output.coh.cookie = coh.cookie;
     output.root_oh.rd_oh = root_oh.rd_oh;
     output.root_oh.wr_oh = root_oh.wr_oh;
-    output.kv_oid_index = 3;
-    output.array_oid_index = 0;
-    output.blob_oid_index = 0;
 
 #if H5VL_IOD_DEBUG
     fprintf(stderr, "Done with file create coh: %"PRIu64" root rd: %"PRIu64"  wr: %"PRIu64"\n",
@@ -205,9 +208,6 @@ done:
         output.coh.cookie = IOD_OH_UNDEFINED;
         output.root_oh.rd_oh.cookie = IOD_OH_UNDEFINED;
         output.root_oh.wr_oh.cookie = IOD_OH_UNDEFINED;
-        output.kv_oid_index = 0;
-        output.array_oid_index = 0;
-        output.blob_oid_index = 0;
         HG_Handler_start_output(op_data->hg_handle, &ret_value);
     }
 
@@ -245,6 +245,8 @@ H5VL_iod_server_file_open_cb(AXE_engine_t UNUSED axe_engine,
     iod_handle_t coh; /* container handle */
     iod_handles_t root_oh; /* root object handle */
     iod_handle_t mdkv_oh; /* metadata object handle for KV to store file's metadata */
+    iod_handle_t oidkv_oh; /* object handle for KV to store file's OID indexes*/
+    int num_entries;
     scratch_pad sp;
     iod_checksum_t sp_cs = 0;
     iod_cont_trans_stat_t *tids = NULL;
@@ -271,7 +273,7 @@ H5VL_iod_server_file_open_cb(AXE_engine_t UNUSED axe_engine,
         HGOTO_ERROR2(H5E_PLIST, H5E_CANTGET, FAIL, "can't get scope for data integrity checks");
 
     /* open the container */
-    if(iod_container_open(input->name, NULL /*hints*/, mode, &coh, NULL /*event*/))
+    if(iod_container_open(input->name, NULL, mode, &coh, NULL /*event*/))
         HGOTO_ERROR2(H5E_FILE, H5E_CANTINIT, FAIL, "can't open file");
 
     if(iod_query_cont_trans_stat(coh, &tids, NULL) < 0)
@@ -286,11 +288,11 @@ H5VL_iod_server_file_open_cb(AXE_engine_t UNUSED axe_engine,
         HGOTO_ERROR2(H5E_SYM, H5E_CANTSET, FAIL, "can't start transaction");
 
     /* open the root group */
-    if ((ret = iod_obj_open_read(coh, ROOT_ID, rtid, NULL /*hints*/, &root_oh.rd_oh, NULL)) < 0) {
+    if ((ret = iod_obj_open_read(coh, ROOT_ID, rtid, NULL, &root_oh.rd_oh, NULL)) < 0) {
         fprintf(stderr, "%d (%s).\n", ret, strerror(-ret));
         HGOTO_ERROR2(H5E_FILE, H5E_CANTINIT, FAIL, "can't open root object for read");
     }
-    if ((ret = iod_obj_open_write(coh, ROOT_ID, rtid, NULL /*hints*/, &root_oh.wr_oh, NULL)) < 0) {
+    if ((ret = iod_obj_open_write(coh, ROOT_ID, rtid, NULL, &root_oh.wr_oh, NULL)) < 0) {
         fprintf(stderr, "%d (%s).\n", ret, strerror(-ret));
         HGOTO_ERROR2(H5E_FILE, H5E_CANTINIT, FAIL, "can't open root object for write");
     }
@@ -304,40 +306,105 @@ H5VL_iod_server_file_open_cb(AXE_engine_t UNUSED axe_engine,
             HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "Scratch Pad failed integrity check");
     }
 
-    /* open the metadata scratch pad */
-    if (iod_obj_open_read(coh, sp[0], rtid, NULL /*hints*/, &mdkv_oh, NULL) < 0)
-        HGOTO_ERROR2(H5E_FILE, H5E_CANTINIT, FAIL, "can't open scratch pad");
+    /* open the metadata KV object */
+    if (iod_obj_open_read(coh, sp[0], rtid, NULL, &mdkv_oh, NULL) < 0)
+        HGOTO_ERROR2(H5E_FILE, H5E_CANTINIT, FAIL, "can't open MD KV");
 
     /* retrieve all metadata from scratch pad */
     if(H5VL_iod_get_metadata(mdkv_oh, rtid, H5VL_IOD_PLIST, H5VL_IOD_KEY_OBJ_CPL,
                              NULL, NULL, &output.fcpl_id) < 0)
         HGOTO_ERROR2(H5E_SYM, H5E_CANTGET, FAIL, "failed to retrieve fcpl");
 
+    /* open the OID indexes KV object */
+    if (iod_obj_open_read(coh, sp[2], rtid, NULL, &oidkv_oh, NULL) < 0)
+        HGOTO_ERROR2(H5E_FILE, H5E_CANTINIT, FAIL, "can't open OID KV");
+
+    ret = iod_kv_get_num(oidkv_oh, rtid, &num_entries, NULL);
+    if(ret != 0)
+        HGOTO_ERROR_IOD(ret, FAIL, "can't get number of KV entries");
+
     val_size = sizeof(uint64_t);
-    key_size = strlen(H5VL_IOD_KEY_KV_IDS_INDEX);
 
-    if(iod_kv_get_value(mdkv_oh, rtid, H5VL_IOD_KEY_KV_IDS_INDEX, key_size,
-                        &output.kv_oid_index, &val_size, NULL, NULL) < 0)
-        HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "KV index lookup failed");
+    /* This was not a clean shutdown, so gather all the oid indexes
+       and compute the max */
+    if(0 != num_entries) {
+        iod_kv_params_t *kvs = NULL;
+        iod_kv_t *kv = NULL;
+        iod_checksum_t *oid_cs = NULL;
+        iod_ret_t *oid_ret = NULL;
+        int i;
+
+        kvs = (iod_kv_params_t *)malloc(sizeof(iod_kv_params_t) * (size_t)num_entries);
+        kv = (iod_kv_t *)malloc(sizeof(iod_kv_t) * (size_t)num_entries);
+        oid_cs = (iod_checksum_t *)malloc(sizeof(iod_checksum_t) * (size_t)num_entries);
+        oid_ret = (iod_ret_t *)malloc(sizeof(iod_ret_t) * (size_t)num_entries);
+
+        for(i=0 ; i<num_entries ; i++) {
+            kv[i].key = malloc(sizeof(uint32_t));
+            kv[i].key_len = sizeof(uint32_t);
+            kv[i].value = malloc(val_size * 3);
+            kv[i].value_len = val_size * 3;
+            kvs[i].kv = &kv[i];
+            kvs[i].cs = &oid_cs[i];
+            kvs[i].ret = &oid_ret[i];
+        }
+
+        ret = iod_kv_get_list(oidkv_oh, rtid, NULL, 0, &num_entries, kvs, NULL);
+        if(ret != 0)
+            HGOTO_ERROR_IOD(ret, FAIL, "can't get KV list from OID KV");
+
+        for(i=0 ; i<num_entries ; i++) {
+            uint64_t *oid_index = (uint64_t *)kv[i].value;
+
+            if(output.kv_oid_index < oid_index[0])
+                output.kv_oid_index = oid_index[0];
+            if(output.array_oid_index < oid_index[1])
+                output.array_oid_index = oid_index[1];
+            if(output.blob_oid_index < oid_index[2])
+                output.blob_oid_index = oid_index[2];
+        }
 
-    key_size = strlen(H5VL_IOD_KEY_ARRAY_IDS_INDEX);
-    if(iod_kv_get_value(mdkv_oh, rtid, H5VL_IOD_KEY_ARRAY_IDS_INDEX, key_size,
-                        &output.array_oid_index, &val_size, NULL, NULL) < 0)
-        HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "Array index lookup failed");
+        for(i=0 ; i<num_entries ; i++) {
+            free(kv[i].key);
+            free(kv[i].value);
+        }
 
-    key_size = strlen(H5VL_IOD_KEY_BLOB_IDS_INDEX);
-    if(iod_kv_get_value(mdkv_oh, rtid, H5VL_IOD_KEY_BLOB_IDS_INDEX, key_size,
-                        &output.blob_oid_index, &val_size, NULL, NULL) < 0)
-        HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "BLOB index lookup failed");
+        free(kv);
+        free(oid_cs);
+        free(oid_ret);
+        free(kvs);
+    }
+    /* This was a clean shutdown, so the maximum is already computed
+       and stored in the metadata KV */
+    else {
+        key_size = strlen(H5VL_IOD_KEY_KV_IDS_INDEX);
+        if(iod_kv_get_value(mdkv_oh, rtid, H5VL_IOD_KEY_KV_IDS_INDEX, key_size,
+                            &output.kv_oid_index, &val_size, NULL, NULL) < 0)
+            HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "KV index lookup failed");
+
+        key_size = strlen(H5VL_IOD_KEY_ARRAY_IDS_INDEX);
+        if(iod_kv_get_value(mdkv_oh, rtid, H5VL_IOD_KEY_ARRAY_IDS_INDEX, key_size,
+                            &output.array_oid_index, &val_size, NULL, NULL) < 0)
+            HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "Array index lookup failed");
+
+        key_size = strlen(H5VL_IOD_KEY_BLOB_IDS_INDEX);
+        if(iod_kv_get_value(mdkv_oh, rtid, H5VL_IOD_KEY_BLOB_IDS_INDEX, key_size,
+                            &output.blob_oid_index, &val_size, NULL, NULL) < 0)
+            HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "BLOB index lookup failed");
+    }
 
-    /* close the metadata scratch pad */
+    /* close the oid KV */
+    if(iod_obj_close(oidkv_oh, NULL, NULL) < 0)
+        HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "can't close OID KV");
+    /* close the metadata KV */
     if(iod_obj_close(mdkv_oh, NULL, NULL) < 0)
-        HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "can't close root object handle");
+        HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "can't close MD KV");
 
     output.coh.cookie = coh.cookie;
     output.root_id = ROOT_ID;
     output.mdkv_id = sp[0];
     output.attrkv_id = sp[1];
+    output.oidkv_id = sp[2];
     output.root_oh.rd_oh = root_oh.rd_oh;
     output.root_oh.wr_oh = root_oh.wr_oh;
     output.c_version = rtid;
@@ -448,36 +515,33 @@ H5VL_iod_server_file_close_cb(AXE_engine_t UNUSED axe_engine,
         if(iod_obj_get_scratch(root_oh.rd_oh, rtid, &sp, &sp_cs, NULL) < 0)
             HGOTO_ERROR2(H5E_FILE, H5E_CANTINIT, FAIL, "can't get scratch pad for root object");
 
-        fprintf(stderr, "root OH %"PRIu64" MDKV ID %"PRIx64"\n", 
-                root_oh.rd_oh.cookie, sp[0]);
-
         if(sp_cs && (cs_scope & H5_CHECKSUM_IOD)) {
             /* verify scratch pad integrity */
             if(H5VL_iod_verify_scratch_pad(sp, sp_cs) < 0)
                 HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "Scratch Pad failed integrity check");
         }
 
-        /* open the metadata scratch pad */
-        if (iod_obj_open_write(coh, sp[0], rtid, NULL, &mdkv_oh, NULL) < 0)
+        /* open the metadata KV object */
+        if (iod_obj_open_write(coh, sp[0], trans_num, NULL, &mdkv_oh, NULL) < 0)
             HGOTO_ERROR2(H5E_FILE, H5E_CANTINIT, FAIL, "can't open metadata KV");
 
         /* insert current indexes in the metadata KV object */
         kv.value = &input->max_kv_index;
-        kv.value_len = sizeof(uint64_t);
+        kv.value_len = sizeof(iod_obj_id_t);
         kv.key = (void *)H5VL_IOD_KEY_KV_IDS_INDEX;
         kv.key_len = strlen(H5VL_IOD_KEY_KV_IDS_INDEX);
         if (iod_kv_set(mdkv_oh, trans_num, NULL, &kv, NULL, NULL) < 0)
             HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "can't set KV pair in parent");
 
         kv.value = &input->max_array_index;
-        kv.value_len = sizeof(uint64_t);
+        kv.value_len = sizeof(iod_obj_id_t);
         kv.key = (void *)H5VL_IOD_KEY_ARRAY_IDS_INDEX;
         kv.key_len = strlen(H5VL_IOD_KEY_ARRAY_IDS_INDEX);
         if (iod_kv_set(mdkv_oh, trans_num, NULL, &kv, NULL, NULL) < 0)
             HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "can't set KV pair in parent");
 
         kv.value = &input->max_blob_index;
-        kv.value_len = sizeof(uint64_t);
+        kv.value_len = sizeof(iod_obj_id_t);
         kv.key = (void *)H5VL_IOD_KEY_BLOB_IDS_INDEX;
         kv.key_len = strlen(H5VL_IOD_KEY_BLOB_IDS_INDEX);
         if (iod_kv_set(mdkv_oh, trans_num, NULL, &kv, NULL, NULL) < 0)
@@ -486,6 +550,64 @@ H5VL_iod_server_file_close_cb(AXE_engine_t UNUSED axe_engine,
         if(iod_obj_close(mdkv_oh, NULL, NULL) < 0)
             HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "can't close root object handle");
 
+        /* open the OID kv object and remove all entries since this is
+           a clean shutdown and the summary is stored in the metadata
+           KV. */
+        {
+            iod_handles_t oidkv_oh;
+            int num_entries = 0, i;
+            iod_kv_params_t *kvs = NULL;
+            iod_kv_t *oid_kv = NULL;
+            iod_checksum_t *oid_cs = NULL;
+            iod_ret_t *oid_ret = NULL;
+
+            if (iod_obj_open_write(coh, sp[2], trans_num, NULL, &oidkv_oh.wr_oh, NULL) < 0)
+                HGOTO_ERROR2(H5E_FILE, H5E_CANTINIT, FAIL, "can't open oid KV");
+            if (iod_obj_open_read(coh, sp[2], rtid, NULL, &oidkv_oh.rd_oh, NULL) < 0)
+                HGOTO_ERROR2(H5E_FILE, H5E_CANTINIT, FAIL, "can't open oid KV");
+
+            ret = iod_kv_get_num(oidkv_oh.rd_oh, rtid, &num_entries, NULL);
+            if(ret != 0)
+                HGOTO_ERROR_IOD(ret, FAIL, "can't get number of KV entries");
+
+            fprintf(stderr, "NUM entries in OID index KV = %d\n", num_entries);
+
+            if(num_entries) {
+                kvs = (iod_kv_params_t *)malloc(sizeof(iod_kv_params_t) * (size_t)num_entries);
+                oid_kv = (iod_kv_t *)malloc(sizeof(iod_kv_t) * (size_t)num_entries);
+                oid_cs = (iod_checksum_t *)malloc(sizeof(iod_checksum_t) * (size_t)num_entries);
+                oid_ret = (iod_ret_t *)malloc(sizeof(iod_ret_t) * (size_t)num_entries);
+
+                for(i=0 ; i<num_entries ; i++) {
+                    oid_kv[i].key = malloc(sizeof(uint32_t));
+                    oid_kv[i].key_len = sizeof(uint32_t);
+                    kvs[i].kv = &oid_kv[i];
+                    kvs[i].cs = &oid_cs[i];
+                    kvs[i].ret = &oid_ret[i];
+                }
+
+                ret = iod_kv_list_key(oidkv_oh.rd_oh, rtid, NULL, 0, &num_entries, kvs, NULL);
+                if(ret != 0)
+                    HGOTO_ERROR_IOD(ret, FAIL, "can't get list of keys");
+
+                ret = iod_kv_unlink_keys(oidkv_oh.wr_oh, trans_num, NULL, num_entries, kvs, NULL);
+                if(ret != 0)
+                    HGOTO_ERROR_IOD(ret, FAIL, "can't unlink keys in OID index KV");
+
+                for(i=0 ; i<num_entries ; i++)
+                    free(oid_kv[i].key);
+                free(oid_kv);
+                free(oid_cs);
+                free(oid_ret);
+                free(kvs);
+            }
+
+            if(iod_obj_close(oidkv_oh.rd_oh, NULL, NULL) < 0)
+                HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "can't close object handle");
+            if(iod_obj_close(oidkv_oh.wr_oh, NULL, NULL) < 0)
+                HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "can't close object handle");
+        }
+
         /* finish the transaction */
         if(iod_trans_finish(coh, rtid, NULL, 0, NULL) < 0)
             HGOTO_ERROR2(H5E_SYM, H5E_CANTSET, FAIL, "can't finish transaction");
@@ -496,7 +618,8 @@ H5VL_iod_server_file_close_cb(AXE_engine_t UNUSED axe_engine,
     }
 
 #if H5VL_IOD_DEBUG
-    fprintf(stderr, "Closing ROOT Group: R: %"PRIu64"  W: %"PRIu64"\n", root_oh.rd_oh, root_oh.wr_oh);
+    fprintf(stderr, "Closing ROOT Group: R: %"PRIu64"  W: %"PRIu64"\n", 
+            root_oh.rd_oh.cookie, root_oh.wr_oh.cookie);
 #endif
 
     /* close the root group */
@@ -506,7 +629,7 @@ H5VL_iod_server_file_close_cb(AXE_engine_t UNUSED axe_engine,
         HGOTO_ERROR2(H5E_SYM, H5E_CANTDEC, FAIL, "can't close root object handle");
 
 #if H5VL_IOD_DEBUG
-    fprintf(stderr, "Closing Container: %"PRIu64"\n", coh);
+    fprintf(stderr, "Closing Container: %"PRIu64"\n", coh.cookie);
 #endif
 
     /* close the container */
diff --git a/src/H5VLiod_trans.c b/src/H5VLiod_trans.c
index 4dd3878..8296766 100644
--- a/src/H5VLiod_trans.c
+++ b/src/H5VLiod_trans.c
@@ -400,14 +400,47 @@ H5VL_iod_server_trans_finish_cb(AXE_engine_t UNUSED axe_engine,
     hid_t trfpl_id;
     iod_trans_id_t trans_num = input->trans_num;
     hbool_t acquire = input->acquire;
+    uint32_t client_rank = input->client_rank;
+    uint64_t oid_index[3];
+    iod_obj_id_t oidkv_id = input->oidkv_id;
+    iod_handle_t oidkv_oh;
+    iod_kv_t kv;
+    iod_ret_t ret;
+    int step = 0;
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_NOAPI_NOINIT
 
 #if H5VL_IOD_DEBUG
-    fprintf(stderr, "Transaction Finish %"PRIu64"\n", input->trans_num);
+    fprintf(stderr, "Transaction Finish %"PRIu64"\n", trans_num);
 #endif
 
+    IOD_OBJID_SETTYPE(oidkv_id, IOD_OBJ_KV)
+    IOD_OBJID_SETOWNER_APP(oidkv_id)
+
+    oid_index[0] = input->kv_oid_index;
+    oid_index[1] = input->array_oid_index;
+    oid_index[2] = input->blob_oid_index;
+
+    ret = iod_obj_open_write(coh, oidkv_id, trans_num, NULL, &oidkv_oh, NULL);
+    if(ret != 0)
+        HGOTO_ERROR_IOD(ret, FAIL, "can't open oid KV");
+
+    step ++;
+
+    kv.value = &oid_index;
+    kv.value_len = sizeof(iod_obj_id_t) * 3;
+    kv.key = &client_rank;
+    kv.key_len = sizeof(uint32_t);
+
+    if (iod_kv_set(oidkv_oh, trans_num, NULL, &kv, NULL, NULL) < 0)
+        HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "can't set KV pair in oid KV");
+
+    if(iod_obj_close(oidkv_oh, NULL, NULL) < 0)
+        HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "can't close object handle");
+
+    step --;
+
     /* Finish  the transaction */
     if(iod_trans_finish(coh, trans_num, NULL, 0, NULL) < 0)
         HGOTO_ERROR2(H5E_SYM, H5E_CANTSET, FAIL, "can't finish transaction");
@@ -430,6 +463,11 @@ done:
     if(HG_SUCCESS != HG_Handler_start_output(op_data->hg_handle, &ret_value))
         fprintf(stderr, "Failed to Finish Transaction\n");
 
+    if(step == 1) {
+        if(iod_obj_close(oidkv_oh, NULL, NULL) < 0)
+            HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "can't close object handle");
+    }
+
     input = (tr_finish_in_t *)H5MM_xfree(input);
     op_data = (op_data_t *)H5MM_xfree(op_data);
 
diff --git a/src/H5checksum.c b/src/H5checksum.c
index 2b57d04..153ce4d 100644
--- a/src/H5checksum.c
+++ b/src/H5checksum.c
@@ -33,7 +33,9 @@
 /* Headers */
 /***********/
 #include "H5private.h"		/* Generic Functions			*/
+#ifdef H5_HAVE_EFF
 #include "mchecksum.h"          /* Mercury Checksum library             */
+#endif /* H5_HAVE_EFF */
 
 /****************/
 /* Local Macros */
@@ -664,6 +666,7 @@ H5_hash_string(const char *str)
     FUNC_LEAVE_NOAPI(hash)
 } /* end H5_hash_string() */
 
+#ifdef H5_HAVE_EFF
 uint64_t 
 H5_checksum_crc64(const void *buf, size_t buf_size)
 {
@@ -722,3 +725,4 @@ H5_checksum_crc64_fragments(void **buf, size_t *buf_size, size_t count)
 
     return hash;
 }
+#endif /* H5_HAVE_EFF */