[svn-r6387] Purpose:

    Bug Fix

Description:
    Metadata cache in parallel I/O can cause hangs in applications which
    perform independent I/O on chunked datasets, because the metadata cache
    can attempt to flush out dirty metadata from only a single process, instead
    of collectively from all processes.

Solution:
    Pass a dataset transfer property list down from every API function which
    could possibly trigger metadata I/O.

    Then, split the metadata cache into two sets of entries to allow dirty
    metadata to be set aside when a hash table collision occurs during
    independent I/O.

Platforms tested:
    Tested h5committest {arabica (fortran), eirene (fortran, C++)
        modi4 (parallel, fortran)}

    FreeBSD 4.7 (sleipnir) serial & parallel

Misc. update:
    Updated release_docs/RELEASE

1 files changed, 732 insertions, 121 deletions

diff --git a/src/H5AC.c b/src/H5AC.c
index 6da9b0b..67b88c6 100644
--- a/src/H5AC.c
+++ b/src/H5AC.c
@@ -28,6 +28,7 @@
 
 #include "H5private.h"
 #include "H5ACprivate.h"
+#include "H5Dprivate.h"		/* Dataset functions			*/
 #include "H5Eprivate.h"
 #include "H5Fpkg.h"
 #include "H5FLprivate.h"	/*Free Lists	  */
@@ -36,10 +37,11 @@
 #include "H5Pprivate.h"         /* Property lists */
 
 /*
- * Sorting the cache by address before flushing is sometimes faster
- * than flushing in cache order.
+ * The MPIO & MPIPOSIX drivers are needed because there are places where we
+ * check for the parallel I/O transfer mode.
  */
-#define H5AC_SORT_BY_ADDR
+#include "H5FDmpio.h"
+#include "H5FDmpiposix.h"
 
 /*
  * Private file-scope variables.
@@ -51,12 +53,22 @@ static int             interface_initialize_g = 0;
 #define INTERFACE_INIT H5AC_init_interface
 static herr_t H5AC_init_interface(void);
 
-/* Dataset transfer property list for flush calls */
-static hid_t H5AC_dxpl_id=(-1);
+/* Default dataset transfer property list for metadata I/O calls */
+/* (Collective set, "block before metadata write" set and "library internal" set) */
+/* (Global variable definition, declaration is in H5ACprivate.h also) */
+hid_t H5AC_dxpl_id=(-1);
+
+/* Private dataset transfer property list for metadata I/O calls */
+/* (Collective set and "library internal" set) */
+/* (Static variable definition) */
+static hid_t H5AC_noblock_dxpl_id=(-1);
+
+/* Dataset transfer property list for independent metadata I/O calls */
+/* (just "library internal" set - i.e. independent transfer mode) */
+/* (Global variable definition, declaration is in H5ACprivate.h also) */
+hid_t H5AC_ind_dxpl_id=(-1);
 
-#ifdef H5AC_SORT_BY_ADDR
 static H5AC_t          *current_cache_g = NULL;         /*for sorting */
-#endif
 
 /* Declare a free list to manage the H5AC_t struct */
 H5FL_DEFINE_STATIC(H5AC_t);
@@ -74,6 +86,35 @@ H5FL_ARR_DEFINE_STATIC(H5AC_prot_t,-1);
 
 
 /*-------------------------------------------------------------------------
+ * Function:	H5AC_init
+ *
+ * Purpose:	Initialize the interface from some other layer.
+ *
+ * Return:	Success:	non-negative
+ *
+ *		Failure:	negative
+ *
+ * Programmer:	Quincey Koziol
+ *              Saturday, January 18, 2003
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5AC_init(void)
+{
+    herr_t ret_value=SUCCEED;   /* Return value */
+
+    FUNC_ENTER_NOAPI(H5AC_init, FAIL);
+    /* FUNC_ENTER() does all the work */
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value);
+}
+
+
+/*-------------------------------------------------------------------------
  * Function:	H5AC_init_interface
  *
  * Purpose:	Initialize interface-specific information
@@ -93,7 +134,9 @@ H5AC_init_interface(void)
     H5P_genclass_t  *xfer_pclass;   /* Dataset transfer property list class object */
 #ifdef H5_HAVE_PARALLEL
     H5P_genplist_t  *xfer_plist;    /* Dataset transfer property list object */
-    unsigned block_before_meta_write=1; /* Custom value for "block before meta write" property */
+    unsigned block_before_meta_write; /* "block before meta write" property value */
+    unsigned library_internal=1;    /* "library internal" property value */
+    H5FD_mpio_xfer_t xfer_mode;     /* I/O transfer mode property value */
 #endif /* H5_HAVE_PARALLEL */
     herr_t ret_value=SUCCEED;           /* Return value */
 
@@ -106,7 +149,7 @@ H5AC_init_interface(void)
     if (NULL == (xfer_pclass = H5I_object(H5P_CLS_DATASET_XFER_g)))
         HGOTO_ERROR(H5E_CACHE, H5E_BADATOM, FAIL, "can't get property list class");
 
-    /* Create a new dataset transfer property list */
+    /* Get an ID for the blocking, collective H5AC dxpl */
     if ((H5AC_dxpl_id=H5P_create_id(xfer_pclass)) < 0)
         HGOTO_ERROR(H5E_CACHE, H5E_CANTCREATE, FAIL, "unable to register property list");
 
@@ -116,8 +159,66 @@ H5AC_init_interface(void)
         HGOTO_ERROR(H5E_CACHE, H5E_BADATOM, FAIL, "can't get new property list object");
 
     /* Insert 'block before metadata write' property */
+    block_before_meta_write=1;
     if(H5P_insert(xfer_plist,H5AC_BLOCK_BEFORE_META_WRITE_NAME,H5AC_BLOCK_BEFORE_META_WRITE_SIZE,&block_before_meta_write,NULL,NULL,NULL,NULL,NULL)<0)
         HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't insert metadata cache dxpl property");
+
+    /* Insert 'library internal' property */
+    if(H5P_insert(xfer_plist,H5AC_LIBRARY_INTERNAL_NAME,H5AC_LIBRARY_INTERNAL_SIZE,&library_internal,NULL,NULL,NULL,NULL,NULL)<0)
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't insert metadata cache dxpl property");
+
+    /* Set the transfer mode */
+    xfer_mode=H5FD_MPIO_COLLECTIVE;
+    if (H5P_set(xfer_plist,H5D_XFER_IO_XFER_MODE_NAME,&xfer_mode)<0)
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set value");
+#endif /* H5_HAVE_PARALLEL */
+
+    /* Get an ID for the non-blocking, collective H5AC dxpl */
+    if ((H5AC_noblock_dxpl_id=H5P_create_id(xfer_pclass)) < 0)
+        HGOTO_ERROR(H5E_CACHE, H5E_CANTCREATE, FAIL, "unable to register property list");
+
+#ifdef H5_HAVE_PARALLEL
+    /* Get the property list object */
+    if (NULL == (xfer_plist = H5I_object(H5AC_noblock_dxpl_id)))
+        HGOTO_ERROR(H5E_CACHE, H5E_BADATOM, FAIL, "can't get new property list object");
+
+    /* Insert 'block before metadata write' property */
+    block_before_meta_write=0;
+    if(H5P_insert(xfer_plist,H5AC_BLOCK_BEFORE_META_WRITE_NAME,H5AC_BLOCK_BEFORE_META_WRITE_SIZE,&block_before_meta_write,NULL,NULL,NULL,NULL,NULL)<0)
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't insert metadata cache dxpl property");
+
+    /* Insert 'library internal' property */
+    if(H5P_insert(xfer_plist,H5AC_LIBRARY_INTERNAL_NAME,H5AC_LIBRARY_INTERNAL_SIZE,&library_internal,NULL,NULL,NULL,NULL,NULL)<0)
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't insert metadata cache dxpl property");
+
+    /* Set the transfer mode */
+    xfer_mode=H5FD_MPIO_COLLECTIVE;
+    if (H5P_set(xfer_plist,H5D_XFER_IO_XFER_MODE_NAME,&xfer_mode)<0)
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set value");
+#endif /* H5_HAVE_PARALLEL */
+
+    /* Get an ID for the non-blocking, independent H5AC dxpl */
+    if ((H5AC_ind_dxpl_id=H5P_create_id(xfer_pclass)) < 0)
+        HGOTO_ERROR(H5E_CACHE, H5E_CANTCREATE, FAIL, "unable to register property list");
+
+#ifdef H5_HAVE_PARALLEL
+    /* Get the property list object */
+    if (NULL == (xfer_plist = H5I_object(H5AC_ind_dxpl_id)))
+        HGOTO_ERROR(H5E_CACHE, H5E_BADATOM, FAIL, "can't get new property list object");
+
+    /* Insert 'block before metadata write' property */
+    block_before_meta_write=0;
+    if(H5P_insert(xfer_plist,H5AC_BLOCK_BEFORE_META_WRITE_NAME,H5AC_BLOCK_BEFORE_META_WRITE_SIZE,&block_before_meta_write,NULL,NULL,NULL,NULL,NULL)<0)
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't insert metadata cache dxpl property");
+
+    /* Insert 'library internal' property */
+    if(H5P_insert(xfer_plist,H5AC_LIBRARY_INTERNAL_NAME,H5AC_LIBRARY_INTERNAL_SIZE,&library_internal,NULL,NULL,NULL,NULL,NULL)<0)
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't insert metadata cache dxpl property");
+
+    /* Set the transfer mode */
+    xfer_mode=H5FD_MPIO_INDEPENDENT;
+    if (H5P_set(xfer_plist,H5D_XFER_IO_XFER_MODE_NAME,&xfer_mode)<0)
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set value");
 #endif /* H5_HAVE_PARALLEL */
 
 done:
@@ -150,16 +251,20 @@ H5AC_term_interface(void)
     FUNC_ENTER_NOINIT(H5AC_term_interface);
 
     if (interface_initialize_g) {
-        if(H5AC_dxpl_id>0) {
+        if(H5AC_dxpl_id>0 || H5AC_noblock_dxpl_id>0 || H5AC_ind_dxpl_id>0) {
             /* Indicate more work to do */
             n = 1; /* H5I */
 
             /* Close H5AC dxpl */
-            if (H5Pclose(H5AC_dxpl_id) < 0)
+            if (H5Pclose(H5AC_dxpl_id) < 0 ||
+                    H5Pclose(H5AC_noblock_dxpl_id) < 0 ||
+                    H5Pclose(H5AC_ind_dxpl_id) < 0)
                 H5E_clear(); /*ignore the error*/
             else {
-                /* Reset static ID */
+                /* Reset static IDs */
                 H5AC_dxpl_id=(-1);
+                H5AC_noblock_dxpl_id=(-1);
+                H5AC_ind_dxpl_id=(-1);
 
                 /* Reset interface initialization flag */
                 interface_initialize_g = 0;
@@ -209,8 +314,9 @@ H5AC_create(H5F_t *f, int size_hint)
     if (NULL==(f->shared->cache = cache = H5FL_CALLOC(H5AC_t)))
 	HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed");
     cache->nslots = size_hint;
-    cache->slot = H5FL_ARR_CALLOC(H5AC_info_ptr_t,cache->nslots);
-    if (NULL==cache->slot)
+    if (NULL==( cache->slot = H5FL_ARR_CALLOC(H5AC_info_ptr_t,cache->nslots)))
+        HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed");
+    if (NULL==( cache->dslot = H5FL_ARR_CALLOC(H5AC_info_ptr_t,cache->nslots)))
         HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed");
 #ifdef H5AC_DEBUG
     if ((cache->prot = H5FL_ARR_CALLOC(H5AC_prot_t,cache->nslots))==NULL)
@@ -223,6 +329,8 @@ H5AC_create(H5F_t *f, int size_hint)
 done:
     if(ret_value<0) {
         if(cache!=NULL) {
+            if(cache->dslot !=NULL)
+                cache->dslot = H5FL_ARR_FREE (H5AC_info_ptr_t,cache->dslot);
             if(cache->slot !=NULL)
                 cache->slot = H5FL_ARR_FREE (H5AC_info_ptr_t,cache->slot);
 #ifdef H5AC_DEBUG
@@ -255,7 +363,7 @@ done:
  *-------------------------------------------------------------------------
  */
 herr_t
-H5AC_dest(H5F_t *f)
+H5AC_dest(H5F_t *f, hid_t dxpl_id)
 {
     H5AC_t *cache = NULL;
     herr_t ret_value=SUCCEED;      /* Return value */
@@ -266,7 +374,7 @@ H5AC_dest(H5F_t *f)
     assert(f->shared->cache);
     cache = f->shared->cache;
 
-    if (H5AC_flush(f, NULL, HADDR_UNDEF, TRUE) < 0)
+    if (H5AC_flush(f, dxpl_id, NULL, HADDR_UNDEF, TRUE) < 0)
         HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush cache");
 
 #ifdef H5AC_DEBUG
@@ -281,6 +389,7 @@ H5AC_dest(H5F_t *f)
     }
 #endif
 
+    cache->dslot = H5FL_ARR_FREE(H5AC_info_ptr_t,cache->dslot);
     cache->slot = H5FL_ARR_FREE(H5AC_info_ptr_t,cache->slot);
     cache->nslots = 0;
     f->shared->cache = cache = H5FL_FREE(H5AC_t,cache);
@@ -291,7 +400,7 @@ done:
 
 
 /*-------------------------------------------------------------------------
- * Function:    H5AC_find_f
+ * Function:    H5AC_find
  *
  * Purpose:     Given an object type and the address at which that object
  *              is located in the file, return a pointer to the object.
@@ -336,18 +445,20 @@ done:
  *-------------------------------------------------------------------------
  */
 void *
-H5AC_find_f(H5F_t *f, const H5AC_class_t *type, haddr_t addr,
+H5AC_find(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t addr,
 	    const void *udata1, void *udata2)
 {
     unsigned                idx;
-    herr_t                  status;
     void                   *thing;
     H5AC_flush_func_t       flush;
     H5AC_info_t           **info;
+#ifdef H5_HAVE_PARALLEL
+    H5AC_info_t           **dinfo = NULL;
+#endif /* H5_HAVE_PARALLEL */
     H5AC_t                 *cache;
     void                   *ret_value;          /* Return value */
 
-    FUNC_ENTER_NOAPI(H5AC_find_f, NULL);
+    FUNC_ENTER_NOAPI(H5AC_find, NULL);
 
     assert(f);
     assert(f->shared->cache);
@@ -356,17 +467,55 @@ H5AC_find_f(H5F_t *f, const H5AC_class_t *type, haddr_t addr,
     assert(type->flush);
     assert(H5F_addr_defined(addr));
 
-    /* Get local copies of information */
+    /* Get local pointers to the file's cache information */
     idx = H5AC_HASH(f, addr);
     cache = f->shared->cache;
     info = cache->slot + idx;
 
+#ifdef H5_HAVE_PARALLEL
+    /* If MPIO or MPIPOSIX is used, do special parallel I/O actions */
+    if(IS_H5FD_MPIO(f) || IS_H5FD_MPIPOSIX(f)) {
+        H5AC_dest_func_t        dest;
+
+        /* Get local pointer to file's dirty cache information */
+        dinfo = cache->dslot + idx;
+
+        /* Check if the cache has 'held' information for this cache slot */
+        if (*dinfo) {
+            /* Sanity check that the 'clean' item is really clean */
+            assert(*info);
+            assert((*info)->dirty==0);
+
+            /* Destroy 'current' information */
+            dest = (*info)->type->dest;
+            if ((dest)(f, (*info))<0)
+                HGOTO_ERROR(H5E_CACHE, H5E_CANTFREE, NULL, "unable to free cached object");
+
+            /* Restore 'held' information back to 'current' information */
+            (*info)=(*dinfo);
+
+            /* Clear 'held' information */
+            (*dinfo)=NULL;
+
+#ifdef H5AC_DEBUG
+            cache->diagnostics[type->id].nrestores++;
+#endif /* H5AC_DEBUG */
+        } /* end if */
+    } /* end if */
+#endif /* H5_HAVE_PARALLEL */
+
     /*
      * Return right away if the item is in the cache.
      */
-    if ((*info) && H5F_addr_eq(addr,(*info)->addr)) {
+    if ((*info) && H5F_addr_eq(addr,(*info)->addr)
+#ifdef H5AC_DEBUG
+            && (*info)->type==type
+#endif /* H5AC_DEBUG */
+            ) {
+#ifndef H5AC_DEBUG
         /* Sanity check that the object in the cache is the correct type */
         assert((*info)->type==type);
+#endif /* H5AC_DEBUG */
 
 #ifdef H5AC_DEBUG
         cache->diagnostics[type->id].nhits++;
@@ -398,11 +547,46 @@ H5AC_find_f(H5F_t *f, const H5AC_class_t *type, haddr_t addr,
      * Load a new thing.  If it can't be loaded, then return an error
      * without preempting anything.
      */
-    if (NULL == (thing = (type->load)(f, H5P_DATASET_XFER_DEFAULT, addr, udata1, udata2)))
+    if (NULL == (thing = (type->load)(f, dxpl_id, addr, udata1, udata2)))
         HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, NULL, "unable to load object");
 
+#ifdef H5_HAVE_PARALLEL
+    /* If MPIO or MPIPOSIX is used, do special parallel I/O actions */
+    if(IS_H5FD_MPIO(f) || IS_H5FD_MPIPOSIX(f)) {
+        H5P_genplist_t *dxpl;           /* Dataset transfer property list */
+        H5FD_mpio_xfer_t xfer_mode;     /* I/O transfer mode property value */
+
+        /* Get the dataset transfer property list */
+        if (NULL == (dxpl = H5I_object(dxpl_id)))
+            HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a dataset creation property list");
+
+        /* Get the transfer mode property */
+        if(H5P_get(dxpl, H5D_XFER_IO_XFER_MODE_NAME, &xfer_mode) < 0)
+            HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't retrieve xfer mode");
+
+        /* Make certain there is no 'held' info for this slot */
+        assert((*dinfo)==NULL);
+
+        /* Must be using collective I/O to flush metadata in parallel */
+        if(xfer_mode==H5FD_MPIO_INDEPENDENT) {
+            /* Check if there is dirty metadata in this slot */
+            if((*info) && (*info)->dirty) {
+                /* 'Hold' the current metadata for later */
+                (*dinfo)=(*info);
+
+                /* Reset the 'current' metadata, so it doesn't get flushed */
+                (*info)=NULL;
+
+#ifdef H5AC_DEBUG
+                cache->diagnostics[(*dinfo)->type->id].nholds++;
+#endif /* H5AC_DEBUG */
+            } /* end if */
+        } /* end else */
+    } /* end if */
+#endif /* H5_HAVE_PARALLEL */
+
     /*
-     * Free the previous cache entry if there is one.
+     * Flush & destroy the previous cache entry if there is one.
      */
     if (*info) {
 #ifdef H5AC_DEBUG
@@ -410,20 +594,19 @@ H5AC_find_f(H5F_t *f, const H5AC_class_t *type, haddr_t addr,
 #endif /* H5AC_DEBUG */
 
         flush = (*info)->type->flush;
-        status = (flush)(f, H5AC_dxpl_id, TRUE, (*info)->addr, (*info));
-        if (status < 0) {
+        if ( (flush)(f, dxpl_id, TRUE, (*info)->addr, (*info)) < 0) {
             /*
              * The old thing could not be removed from the stack.
              * Release the new thing and fail.
              */
-            if ((type->flush)(f, H5AC_dxpl_id, TRUE, addr, thing) < 0)
-                HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, NULL, "unable to flush just-loaded object");
+            if ((type->dest)(f, thing) < 0)
+                HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, NULL, "unable to destroy just-loaded object");
             HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, NULL, "unable to flush existing cached object");
         }
 #ifdef H5AC_DEBUG
         cache->diagnostics[type_id].nflushes++;
 #endif /* H5AC_DEBUG */
-    }
+    } /* end if */
 
     /*
      * Make the cache point to the new thing.
@@ -431,6 +614,7 @@ H5AC_find_f(H5F_t *f, const H5AC_class_t *type, haddr_t addr,
     (*info)=thing;
     (*info)->type = type;
     (*info)->addr = addr;
+    assert((*info)->dirty==0);  /* Should be clean after being loaded */
 
     /* Set the return value */
     ret_value=thing;
@@ -458,7 +642,6 @@ done:
  *
  *-------------------------------------------------------------------------
  */
-#ifdef H5AC_SORT_BY_ADDR
 static int
 H5AC_compare(const void *_a, const void *_b)
 {
@@ -490,7 +673,6 @@ H5AC_compare(const void *_a, const void *_b)
 
     FUNC_LEAVE_NOAPI(ret_value);
 }
-#endif
 
 
 /*-------------------------------------------------------------------------
@@ -520,15 +702,13 @@ H5AC_compare(const void *_a, const void *_b)
  *-------------------------------------------------------------------------
  */
 herr_t
-H5AC_flush(H5F_t *f, const H5AC_class_t *type, haddr_t addr, hbool_t destroy)
+H5AC_flush(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t addr, hbool_t destroy)
 {
     unsigned                i;
     herr_t                  status;
     H5AC_flush_func_t       flush=NULL;
     H5AC_info_t           **info;
-#ifdef H5AC_SORT_BY_ADDR
     int                   *map = NULL;
-#endif /* H5AC_SORT_BY_ADDR */
     unsigned               nslots;
     H5AC_t                 *cache;
     herr_t ret_value=SUCCEED;      /* Return value */
@@ -544,42 +724,92 @@ H5AC_flush(H5F_t *f, const H5AC_class_t *type, haddr_t addr, hbool_t destroy)
     if (!H5F_addr_defined(addr)) {
         unsigned first_flush=1;     /* Indicate if this is the first flush */
 
-#ifdef H5AC_SORT_BY_ADDR
         /*
          * Sort the cache entries by address since flushing them in
-         * ascending order by address may be much more efficient.
+         * ascending order by address is much more efficient.
          */
         if (NULL==(map=H5FL_ARR_MALLOC(int,cache->nslots)))
             HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed");
-        for (i = nslots = 0; i < cache->nslots; i++) {
-            if (cache->slot[i]!=NULL)
-                map[nslots++] = i;
-        }
+#ifdef H5_HAVE_PARALLEL
+        /* If MPIO or MPIPOSIX is used, do special parallel I/O actions */
+        if(IS_H5FD_MPIO(f) || IS_H5FD_MPIPOSIX(f)) {
+            H5AC_info_t       **dinfo;
+            H5AC_subid_t        type_id;
+#ifndef NDEBUG
+            H5P_genplist_t *dxpl;           /* Dataset transfer property list */
+            H5FD_mpio_xfer_t xfer_mode;     /* I/O transfer mode property value */
+
+            /* Get the dataset transfer property list */
+            if (NULL == (dxpl = H5I_object(dxpl_id)))
+                HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset creation property list");
+
+            /* Get the transfer mode property */
+            if(H5P_get(dxpl, H5D_XFER_IO_XFER_MODE_NAME, &xfer_mode) < 0)
+                HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't retrieve xfer mode");
+
+            /* Sanity check transfer mode */
+            assert(xfer_mode==H5FD_MPIO_COLLECTIVE);
+#endif /* NDEBUG */
+
+            /* Create the mapping */
+            for (i = nslots = 0; i < cache->nslots; i++) {
+                info = cache->slot + i;
+                dinfo = cache->dslot + i;
+
+                /* Move dirty metadata from 'held' slots into 'regular' slots */
+                if((*dinfo)!=NULL) {
+                    H5AC_dest_func_t        dest;
+
+                    /* Various sanity checks */
+                    assert((*dinfo)->dirty);
+                    assert((*info)!=NULL);
+                    assert((*info)->dirty==0);
+
+                    type_id=(*info)->type->id;  /* Remember this for later */
+
+                    /* Destroy 'current' information */
+                    dest = (*info)->type->dest;
+                    if ((dest)(f, (*info))<0)
+                        HGOTO_ERROR(H5E_CACHE, H5E_CANTFREE, NULL, "unable to free cached object");
+
+                    /* Restore 'held' information back to 'current' information */
+                    (*info)=(*dinfo);
+
+                    /* Clear 'held' information */
+                    (*dinfo)=NULL;
+
+#ifdef H5AC_DEBUG
+                    cache->diagnostics[type_id].nrestores++;
+#endif /* H5AC_DEBUG */
+                } /* end if */
+                if ((*info))
+                    map[nslots++] = i;
+            } /* end for */
+        } /* end if */
+        else {
+#endif /* H5_HAVE_PARALLEL */
+            for (i = nslots = 0; i < cache->nslots; i++) {
+                if (cache->slot[i]!=NULL)
+                    map[nslots++] = i;
+            }
+#ifdef H5_HAVE_PARALLEL
+        } /* end else */
+#endif /* H5_HAVE_PARALLEL */
         assert(NULL == current_cache_g);
         current_cache_g = cache;
         HDqsort(map, nslots, sizeof(int), H5AC_compare);
         current_cache_g = NULL;
-#ifdef NDEBUG
+#ifndef NDEBUG
         for (i = 1; i < nslots; i++)
-            assert(H5F_addr_lt(cache->slot[i - 1]->addr, cache->slot[i]->addr));
+            assert(H5F_addr_lt(cache->slot[map[i - 1]]->addr, cache->slot[map[i]]->addr));
 #endif
-#else /* H5AC_SORT_BY_ADDR */
-        nslots = cache->nslots;
-#endif /* H5AC_SORT_BY_ADDR */
 
         /*
          * Look at all cache entries.
          */
         for (i = 0; i < nslots; i++) {
-#ifdef H5AC_SORT_BY_ADDR
             info = cache->slot + map[i];
-            if (NULL == (*info))
-                 break;          /*the rest are empty */
-#else /* H5AC_SORT_BY_ADDR */
-            info = cache->slot + i;
-            if (NULL == (*info))
-                continue;
-#endif /* H5AC_SORT_BY_ADDR */
+            assert(*info);
             if (!type || type == (*info)->type) {
 #ifdef H5AC_DEBUG
                 H5AC_subid_t type_id=(*info)->type->id;  /* Remember this for later */
@@ -588,12 +818,12 @@ H5AC_flush(H5F_t *f, const H5AC_class_t *type, haddr_t addr, hbool_t destroy)
                 flush = (*info)->type->flush;
 
                 /* Only block for all the processes on the first piece of metadata */
-                if(first_flush) {
-                    status = (flush)(f, H5AC_dxpl_id, destroy, (*info)->addr, (*info));
+                if(first_flush && (*info)->dirty) {
+                    status = (flush)(f, dxpl_id, destroy, (*info)->addr, (*info));
                     first_flush=0;
                 } /* end if */
                 else
-                    status = (flush)(f, H5P_DATASET_XFER_DEFAULT, destroy, (*info)->addr, (*info));
+                    status = (flush)(f, H5AC_noblock_dxpl_id, destroy, (*info)->addr, (*info));
                 if (status < 0)
                     HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush cache");
 
@@ -613,32 +843,81 @@ H5AC_flush(H5F_t *f, const H5AC_class_t *type, haddr_t addr, hbool_t destroy)
             HGOTO_ERROR(H5E_CACHE, H5E_PROTECT, FAIL, "cache has protected items");
     } else {
         i = H5AC_HASH(f, addr);
-        if (cache->slot[i] && (!type || cache->slot[i]->type == type) &&
-                H5F_addr_eq(addr,cache->slot[i]->addr)) {
+        info = cache->slot + i;
+#ifdef H5_HAVE_PARALLEL
+        /* If MPIO or MPIPOSIX is used, do special parallel I/O actions */
+        if(IS_H5FD_MPIO(f) || IS_H5FD_MPIPOSIX(f)) {
+            H5AC_info_t       **dinfo;
+            H5AC_subid_t        type_id;
+#ifndef NDEBUG
+            H5P_genplist_t *dxpl;           /* Dataset transfer property list */
+            H5FD_mpio_xfer_t xfer_mode;     /* I/O transfer mode property value */
+
+            /* Get the dataset transfer property list */
+            if (NULL == (dxpl = H5I_object(dxpl_id)))
+                HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset creation property list");
+
+            /* Get the transfer mode property */
+            if(H5P_get(dxpl, H5D_XFER_IO_XFER_MODE_NAME, &xfer_mode) < 0)
+                HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't retrieve xfer mode");
+
+            /* Sanity check transfer mode */
+            assert(xfer_mode==H5FD_MPIO_COLLECTIVE);
+#endif /* NDEBUG */
+
+            dinfo = cache->dslot + i;
+
+            /* Restore dirty metadata from 'held' slot to 'current' slot */
+            if((*dinfo)!=NULL) {
+                H5AC_dest_func_t        dest;
+
+                /* Various sanity checks */
+                assert((*dinfo)->dirty);
+                assert((*info)!=NULL);
+                assert((*info)->dirty==0);
+
+                type_id=(*info)->type->id;  /* Remember this for later */
+
+                /* Destroy 'current' information */
+                dest = (*info)->type->dest;
+                if ((dest)(f, (*info))<0)
+                    HGOTO_ERROR(H5E_CACHE, H5E_CANTFREE, NULL, "unable to free cached object");
+
+                /* Restore 'held' information back to 'current' information */
+                (*info)=(*dinfo);
+
+                /* Clear 'held' information */
+                (*dinfo)=NULL;
+
 #ifdef H5AC_DEBUG
-            H5AC_subid_t type_id=cache->slot[i]->type->id;  /* Remember this for later */
+                cache->diagnostics[type_id].nrestores++;
+#endif /* H5AC_DEBUG */
+            } /* end if */
+        } /* end if */
+#endif /* H5_HAVE_PARALLEL */
+        if ((*info) && (!type || (*info)->type == type) &&
+                H5F_addr_eq((*info)->addr, addr)) {
+#ifdef H5AC_DEBUG
+            H5AC_subid_t type_id=(*info)->type->id;  /* Remember this for later */
 #endif /* H5AC_DEBUG */
 
             /*
              * Flush just this entry.
              */
-            flush = cache->slot[i]->type->flush;
-            if ((flush)(f, H5AC_dxpl_id, destroy, cache->slot[i]->addr,
-			     cache->slot[i]) < 0)
+            flush = (*info)->type->flush;
+            if((flush)(f, dxpl_id, destroy, (*info)->addr, (*info)) < 0)
                 HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush object");
 #ifdef H5AC_DEBUG
             cache->diagnostics[type_id].nflushes++;
 #endif /* H5AC_DEBUG */
             if (destroy)
-                cache->slot[i]= NULL;
-        }
-    }
+                (*info)= NULL;
+        } /* end if */
+    } /* end else */
 
 done:
-#ifdef H5AC_SORT_BY_ADDR
     if(map!=NULL)
         map = H5FL_ARR_FREE(int,map);
-#endif /* H5AC_SORT_BY_ADDR */
 
     FUNC_LEAVE_NOAPI(ret_value);
 }
@@ -666,7 +945,7 @@ done:
  *-------------------------------------------------------------------------
  */
 herr_t
-H5AC_set(H5F_t *f, const H5AC_class_t *type, haddr_t addr, void *thing)
+H5AC_set(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t addr, void *thing)
 {
     unsigned                idx;
     H5AC_flush_func_t       flush;
@@ -699,6 +978,77 @@ H5AC_set(H5F_t *f, const H5AC_class_t *type, haddr_t addr, void *thing)
     }
 #endif
 
+#ifdef H5_HAVE_PARALLEL
+    /* If MPIO or MPIPOSIX is used, do special parallel I/O actions */
+    if(IS_H5FD_MPIO(f) || IS_H5FD_MPIPOSIX(f)) {
+        H5AC_info_t       **dinfo;
+        H5AC_subid_t        type_id;
+        H5P_genplist_t *dxpl;           /* Dataset transfer property list */
+        H5FD_mpio_xfer_t xfer_mode;     /* I/O transfer mode property value */
+
+        /* Get the dataset transfer property list */
+        if (NULL == (dxpl = H5I_object(dxpl_id)))
+            HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset creation property list");
+
+        /* Get the transfer mode property */
+        if(H5P_get(dxpl, H5D_XFER_IO_XFER_MODE_NAME, &xfer_mode) < 0)
+            HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't retrieve xfer mode");
+
+        /* Get pointer to 'held' information */
+        dinfo = cache->dslot + idx;
+
+        /* Sanity check transfer mode */
+        if(xfer_mode==H5FD_MPIO_COLLECTIVE) {
+            /* Check for dirty metadata */
+            if(*dinfo) {
+                H5AC_dest_func_t        dest;
+
+                /* Various sanity checks */
+                assert((*dinfo)->dirty);
+                assert((*info)!=NULL);
+                assert((*info)->dirty==0);
+
+                type_id=(*info)->type->id;  /* Remember this for later */
+
+                /* Destroy 'current' information */
+                dest = (*info)->type->dest;
+                if ((dest)(f, (*info))<0)
+                    HGOTO_ERROR(H5E_CACHE, H5E_CANTFREE, NULL, "unable to free cached object");
+
+                /* Restore 'held' information back to 'current' information */
+                (*info)=(*dinfo);
+
+                /* Clear 'held' information */
+                (*dinfo)=NULL;
+
+#ifdef H5AC_DEBUG
+                cache->diagnostics[type_id].nrestores++;
+#endif /* H5AC_DEBUG */
+            } /* end if */
+        } /* end if */
+        else {
+            /* Sanity check */
+            assert((*dinfo)==NULL);
+            assert(xfer_mode==H5FD_MPIO_INDEPENDENT);
+
+            /* Make certain there will be no write of dirty metadata */
+            if((*info) && (*info)->dirty) {
+                /* Sanity check new item */
+                assert(((H5AC_info_t*)thing)->dirty==0);
+
+                /* 'Hold' the current metadata for later */
+                (*dinfo)=(*info);
+
+                /* Reset the 'current' metadata, so it doesn't get flushed */
+                (*info)=NULL;
+
+#ifdef H5AC_DEBUG
+                cache->diagnostics[(*dinfo)->type->id].nholds++;
+#endif /* H5AC_DEBUG */
+            } /* end if */
+        } /* end else */
+    } /* end if */
+#endif /* H5_HAVE_PARALLEL */
 
     /* Flush any object already in cache slot */
     if ((*info)) {
@@ -707,13 +1057,14 @@ H5AC_set(H5F_t *f, const H5AC_class_t *type, haddr_t addr, void *thing)
 #endif /* H5AC_DEBUG */
 
         flush = (*info)->type->flush;
-        if ((flush)(f, H5AC_dxpl_id, TRUE, (*info)->addr, (*info)) < 0)
+        if ((flush)(f, dxpl_id, TRUE, (*info)->addr, (*info)) < 0)
             HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush object");
 #ifdef H5AC_DEBUG
         cache->diagnostics[type_id].nflushes++;
 #endif /* H5AC_DEBUG */
-    }
+    } /* end if */
 
+    /* Cache this item */
     (*info)=thing;
     (*info)->type = type;
     (*info)->addr = addr;
@@ -748,12 +1099,14 @@ done:
  *-------------------------------------------------------------------------
  */
 herr_t
-H5AC_rename(H5F_t *f, const H5AC_class_t *type, haddr_t old_addr,
+H5AC_rename(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t old_addr,
 	    haddr_t new_addr)
 {
     unsigned                old_idx, new_idx;
     H5AC_flush_func_t       flush;
     H5AC_t                 *cache;
+    H5AC_info_t           **new_info = NULL;
+    H5AC_info_t           **old_info = NULL;
     herr_t ret_value=SUCCEED;      /* Return value */
 
     FUNC_ENTER_NOAPI(H5AC_rename, FAIL);
@@ -766,6 +1119,8 @@ H5AC_rename(H5F_t *f, const H5AC_class_t *type, haddr_t old_addr,
     old_idx = H5AC_HASH(f, old_addr);
     new_idx = H5AC_HASH(f, new_addr);
     cache = f->shared->cache;
+    new_info = cache->slot + new_idx;
+    old_info = cache->slot + old_idx;
 
 #ifdef H5AC_DEBUG
     {
@@ -785,25 +1140,95 @@ H5AC_rename(H5F_t *f, const H5AC_class_t *type, haddr_t old_addr,
      * We don't need to do anything if the object isn't cached or if the
      * new hash value is the same as the old one.
      */
-    if (cache->slot[old_idx]->type != type ||
-            H5F_addr_ne(cache->slot[old_idx]->addr, old_addr)) {
+    if (H5F_addr_ne((*old_info)->addr, old_addr) || (*old_info)->type!=type)
         HGOTO_DONE(SUCCEED);
-    }
     if (old_idx == new_idx) {
-        cache->slot[old_idx]->addr = new_addr;
+        (*old_info)->addr = new_addr;
         HGOTO_DONE(SUCCEED);
     }
+
+#ifdef H5_HAVE_PARALLEL
+    /* If MPIO or MPIPOSIX is used, do special parallel I/O actions */
+    if(IS_H5FD_MPIO(f) || IS_H5FD_MPIPOSIX(f)) {
+        H5AC_info_t       **new_dinfo;
+        H5AC_subid_t        type_id;
+        H5P_genplist_t *dxpl;           /* Dataset transfer property list */
+        H5FD_mpio_xfer_t xfer_mode;     /* I/O transfer mode property value */
+
+        /* Get the dataset transfer property list */
+        if (NULL == (dxpl = H5I_object(dxpl_id)))
+            HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset creation property list");
+
+        /* Get the transfer mode property */
+        if(H5P_get(dxpl, H5D_XFER_IO_XFER_MODE_NAME, &xfer_mode) < 0)
+            HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't retrieve xfer mode");
+
+        /* Get pointer to new 'held' information */
+        new_dinfo = cache->dslot + new_idx;
+
+        /* Sanity check transfer mode */
+        if(xfer_mode==H5FD_MPIO_COLLECTIVE) {
+            /* Check for dirty metadata */
+            if(*new_dinfo) {
+                H5AC_dest_func_t        dest;
+
+                /* Various sanity checks */
+                assert((*new_dinfo)->dirty);
+                assert((*new_info)!=NULL);
+                assert((*new_info)->dirty==0);
+
+                type_id=(*new_info)->type->id;  /* Remember this for later */
+
+                /* Destroy 'current' information */
+                dest = (*new_info)->type->dest;
+                if ((dest)(f, (*new_info))<0)
+                    HGOTO_ERROR(H5E_CACHE, H5E_CANTFREE, NULL, "unable to free cached object");
+
+                /* Restore 'held' information back to 'current' information */
+                (*new_info)=(*new_dinfo);
+
+                /* Clear 'held' information */
+                (*new_dinfo)=NULL;
+
+#ifdef H5AC_DEBUG
+                cache->diagnostics[type_id].nrestores++;
+#endif /* H5AC_DEBUG */
+            } /* end if */
+        } /* end if */
+        else {
+            /* Sanity check that there will be no write of dirty metadata */
+            assert((*new_dinfo)==NULL);
+            assert(xfer_mode==H5FD_MPIO_INDEPENDENT);
+
+            /* Make certain there will be no write of dirty metadata */
+            if((*new_info) && (*new_info)->dirty) {
+                /* Sanity check that we won't put two pieces of dirty metadata in same cache location */
+                assert((*old_info)->dirty==0);
+
+                /* 'Hold' the current metadata for later */
+                (*new_dinfo)=(*new_info);
+
+                /* Reset the 'current' metadata, so it doesn't get flushed */
+                (*new_info)=NULL;
+
+#ifdef H5AC_DEBUG
+                cache->diagnostics[(*new_dinfo)->type->id].nholds++;
+#endif /* H5AC_DEBUG */
+            } /* end if */
+        } /* end else */
+    } /* end if */
+#endif /* H5_HAVE_PARALLEL */
+
     /*
      * Free the item from the destination cache line.
      */
-    if (cache->slot[new_idx]) {
+    if (*new_info) {
 #ifdef H5AC_DEBUG
-        H5AC_subid_t type_id=cache->slot[new_idx]->type->id;  /* Remember this for later */
+        H5AC_subid_t type_id=(*new_info)->type->id;  /* Remember this for later */
 #endif /* H5AC_DEBUG */
 
-        flush = cache->slot[new_idx]->type->flush;
-        if ((flush)(f, H5AC_dxpl_id, TRUE, cache->slot[new_idx]->addr,
-			 cache->slot[new_idx]) < 0)
+        flush = (*new_info)->type->flush;
+        if ( (flush)(f, dxpl_id, TRUE, (*new_info)->addr, (*new_info)) < 0)
             HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush object");
 #ifdef H5AC_DEBUG
         cache->diagnostics[type_id].nflushes++;
@@ -813,10 +1238,45 @@ H5AC_rename(H5F_t *f, const H5AC_class_t *type, haddr_t old_addr,
     /*
      * Move the source to the destination (it might not be cached)
      */
-    cache->slot[new_idx]= cache->slot[old_idx];
-    cache->slot[new_idx]->type = cache->slot[old_idx]->type;
-    cache->slot[new_idx]->addr = new_addr;
-    cache->slot[old_idx]= NULL;
+    (*new_info)= (*old_info);
+    (*new_info)->addr = new_addr;
+
+#ifdef H5_HAVE_PARALLEL
+    /* If MPIO or MPIPOSIX is used, do special parallel I/O actions */
+    if(IS_H5FD_MPIO(f) || IS_H5FD_MPIPOSIX(f)) {
+        H5AC_info_t       **old_dinfo;
+        H5AC_subid_t        type_id;
+
+        /* Get pointer to new 'held' information */
+        old_dinfo = cache->dslot + old_idx;
+
+        /* Check for 'held' metadata in old location & restore it, if so */
+        if(*old_dinfo) {
+            /* Sanity check */
+            assert((*old_dinfo)->dirty);
+
+            type_id=(*old_info)->type->id;  /* Remember this for later */
+
+            /* Restore 'held' information back to 'current' information */
+            (*old_info)=(*old_dinfo);
+
+            /* Clear 'held' information */
+            (*old_dinfo)=NULL;
+
+#ifdef H5AC_DEBUG
+            cache->diagnostics[type_id].nrestores++;
+#endif /* H5AC_DEBUG */
+        } /* end if */
+        else
+            (*old_info)= NULL;
+    } /* end if */
+    else {
+#endif /* H5_HAVE_PARALLEL */
+
+        (*old_info)= NULL;
+#ifdef H5_HAVE_PARALLEL
+    } /* end else */
+#endif /* H5_HAVE_PARALLEL */
 
 done:
     FUNC_LEAVE_NOAPI(ret_value);
@@ -851,19 +1311,19 @@ done:
  *-------------------------------------------------------------------------
  */
 void *
-H5AC_protect(H5F_t *f, const H5AC_class_t *type, haddr_t addr,
+H5AC_protect(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t addr,
 	     const void *udata1, void *udata2)
 {
     int                     idx;
-    void                   *thing;
+    void                   *thing=NULL;
     H5AC_t                 *cache;
     H5AC_info_t           **info;
     void                   *ret_value;          /* Return value */
 
 #ifdef H5AC_DEBUG
     H5AC_prot_t            *prot = NULL;
-
     static int ncalls = 0;
+
     if (0 == ncalls++) {
 	if (H5DEBUG(AC)) {
 	    fprintf(H5DEBUG(AC), "H5AC: debugging cache (expensive)\n");
@@ -889,43 +1349,120 @@ H5AC_protect(H5F_t *f, const H5AC_class_t *type, haddr_t addr,
     prot = cache->prot + idx;
 #endif /* H5AC_DEBUG */
 
-    if ((*info) && H5F_addr_eq(addr,(*info)->addr)) {
-        /* Sanity check that the object in the cache is the correct type */
-        assert((*info)->type==type);
+#ifdef H5_HAVE_PARALLEL
+    /* If MPIO or MPIPOSIX is used, do special parallel I/O actions */
+    if(IS_H5FD_MPIO(f) || IS_H5FD_MPIPOSIX(f)) {
+        H5AC_info_t       **dinfo;
+
+        /* Get pointer to new 'held' information */
+        dinfo = cache->dslot + idx;
+
+        /* Check for 'held' metadata in location & handle it */
+        if(*dinfo) {
+            /* Sanity checks */
+            assert((*dinfo)->dirty);
+            assert((*info));
+            assert((*info)->dirty==0);
+            assert((*dinfo)->addr!=(*info)->addr);
+
+            /* Is 'held' metadata the metadata we are looking for? */
+            if (H5F_addr_eq((*dinfo)->addr, addr) 
+#ifdef H5AC_DEBUG
+                    && (*dinfo)->type==type
+#endif /* H5AC_DEBUG */
+                    ) {
+#ifndef H5AC_DEBUG
+                /* Sanity check that the object in the cache is the correct type */
+                assert((*dinfo)->type==type);
+#endif /* H5AC_DEBUG */
+                /*
+                 * The object is already cached; simply remove it from the cache.
+                 */
+                thing = (*dinfo);
+                (*dinfo)->type = NULL;
+                (*dinfo)->addr = HADDR_UNDEF;
+                (*dinfo)= NULL;
+#ifdef H5AC_DEBUG
+                cache->diagnostics[(*dinfo)->type->id].nhits++;
+#endif /* H5AC_DEBUG */
+            } /* end if */
+            /* 'held' metadata isn't what we are looking for, but check for 'current' metadata */
+            else {
+                if(H5F_addr_eq((*info)->addr, addr)
+#ifdef H5AC_DEBUG
+                        && (*info)->type==type
+#endif /* H5AC_DEBUG */
+                        ) {
+#ifndef H5AC_DEBUG
+                    /* Sanity check that the object in the cache is the correct type */
+                    assert((*info)->type==type);
+#endif /* H5AC_DEBUG */
+                    /*
+                     * The object is already cached; remove it from the cache.
+                     * and bring the 'held' object into the 'regular' information
+                     */
+                    thing = (*info);
+                    (*info)->type = NULL;
+                    (*info)->addr = HADDR_UNDEF;
+                    (*info)= (*dinfo);
+                    (*dinfo)= NULL;
+#ifdef H5AC_DEBUG
+                    cache->diagnostics[(*info)->type->id].nhits++;
+#endif /* H5AC_DEBUG */
+                } /* end if */
+            } /* end else */
+        } /* end if */
+    } /* end if */
 
-        /*
-         * The object is already cached; simply remove it from the cache.
-         */
+    /* Check if we've already found the object to protect */
+    if(thing==NULL) {
+#endif /* H5_HAVE_PARALLEL */
+        if ((*info) && H5F_addr_eq(addr,(*info)->addr)
 #ifdef H5AC_DEBUG
-        cache->diagnostics[(*info)->type->id].nhits++;
+                && (*info)->type==type
 #endif /* H5AC_DEBUG */
-        thing = (*info);
-        (*info)->type = NULL;
-        (*info)->addr = HADDR_UNDEF;
-        (*info)= NULL;
-    } else {
+                ) {
+#ifndef H5AC_DEBUG
+            /* Sanity check that the object in the cache is the correct type */
+            assert((*info)->type==type);
+#endif /* H5AC_DEBUG */
+
+            /*
+             * The object is already cached; simply remove it from the cache.
+             */
+            thing = (*info);
+            (*info)->type = NULL;
+            (*info)->addr = HADDR_UNDEF;
+            (*info)= NULL;
 #ifdef H5AC_DEBUG
-        /*
-         * Check that the requested thing isn't protected, for protected things
-         * can only be modified through the pointer already handed out by the
-         * H5AC_protect() function.
-         */
-        int                    i;
+            cache->diagnostics[(*info)->type->id].nhits++;
+#endif /* H5AC_DEBUG */
+        } else {
+#ifdef H5AC_DEBUG
+            /*
+             * Check that the requested thing isn't protected, for protected things
+             * can only be modified through the pointer already handed out by the
+             * H5AC_protect() function.
+             */
+            int                    i;
 
-        for (i = 0; i < prot->nprots; i++)
-            assert(H5F_addr_ne(addr, prot->slot[i]->addr));
+            for (i = 0; i < prot->nprots; i++)
+                assert(H5F_addr_ne(addr, prot->slot[i]->addr));
 #endif /* H5AC_DEBUG */
 
-        /*
-         * Load a new thing.  If it can't be loaded, then return an error
-         * without preempting anything.
-         */
+            /*
+             * Load a new thing.  If it can't be loaded, then return an error
+             * without preempting anything.
+             */
+            if (NULL == (thing = (type->load)(f, dxpl_id, addr, udata1, udata2)))
+                HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, NULL, "unable to load object");
 #ifdef H5AC_DEBUG
-        cache->diagnostics[type->id].nmisses++;
+            cache->diagnostics[type->id].nmisses++;
 #endif /* H5AC_DEBUG */
-        if (NULL == (thing = (type->load)(f, H5P_DATASET_XFER_DEFAULT, addr, udata1, udata2)))
-            HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, NULL, "unable to load object");
-    }
+        }
+#ifdef H5_HAVE_PARALLEL
+    } /* end if */
+#endif /* H5_HAVE_PARALLEL */
 
 #ifdef H5AC_DEBUG
     /*
@@ -934,8 +1471,8 @@ H5AC_protect(H5F_t *f, const H5AC_class_t *type, haddr_t addr,
      */
     if (prot->nprots >= prot->aprots) {
         size_t na = prot->aprots + 10;
-        H5AC_info_t **x = H5MM_realloc(prot->slot,
-                          na * sizeof(H5AC_info_t *));
+        H5AC_info_t **x = H5MM_realloc(prot->slot, na * sizeof(H5AC_info_t *));
+
         if (NULL==x)
             HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed");
         prot->aprots = (int)na;
@@ -981,7 +1518,7 @@ done:
  *-------------------------------------------------------------------------
  */
 herr_t
-H5AC_unprotect(H5F_t *f, const H5AC_class_t *type, haddr_t addr, void *thing)
+H5AC_unprotect(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t addr, void *thing)
 {
     unsigned                idx;
     H5AC_flush_func_t       flush;
@@ -1004,6 +1541,78 @@ H5AC_unprotect(H5F_t *f, const H5AC_class_t *type, haddr_t addr, void *thing)
     cache = f->shared->cache;
     info = cache->slot + idx;
 
+#ifdef H5_HAVE_PARALLEL
+    /* If MPIO or MPIPOSIX is used, do special parallel I/O actions */
+    if(IS_H5FD_MPIO(f) || IS_H5FD_MPIPOSIX(f)) {
+        H5AC_info_t       **dinfo;
+        H5AC_subid_t        type_id;
+        H5P_genplist_t *dxpl;           /* Dataset transfer property list */
+        H5FD_mpio_xfer_t xfer_mode;     /* I/O transfer mode property value */
+
+        /* Get the dataset transfer property list */
+        if (NULL == (dxpl = H5I_object(dxpl_id)))
+            HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset creation property list");
+
+        /* Get the transfer mode property */
+        if(H5P_get(dxpl, H5D_XFER_IO_XFER_MODE_NAME, &xfer_mode) < 0)
+            HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't retrieve xfer mode");
+
+        /* Get pointer to 'held' information */
+        dinfo = cache->dslot + idx;
+
+        /* Sanity check transfer mode */
+        if(xfer_mode==H5FD_MPIO_COLLECTIVE) {
+            /* Check for dirty metadata */
+            if(*dinfo) {
+                H5AC_dest_func_t        dest;
+
+                /* Various sanity checks */
+                assert((*dinfo)->dirty);
+                assert((*info)!=NULL);
+                assert((*info)->dirty==0);
+
+                type_id=(*info)->type->id;  /* Remember this for later */
+
+                /* Destroy 'current' information */
+                dest = (*info)->type->dest;
+                if ((dest)(f, (*info))<0)
+                    HGOTO_ERROR(H5E_CACHE, H5E_CANTFREE, FAIL, "unable to free cached object");
+
+                /* Restore 'held' information back to 'current' information */
+                (*info)=(*dinfo);
+
+                /* Clear 'held' information */
+                (*dinfo)=NULL;
+
+#ifdef H5AC_DEBUG
+                cache->diagnostics[type_id].nrestores++;
+#endif /* H5AC_DEBUG */
+            } /* end if */
+        } /* end if */
+        else {
+            /* Sanity check */
+            assert((*dinfo)==NULL);
+            assert(xfer_mode==H5FD_MPIO_INDEPENDENT);
+
+            /* Make certain there will be no write of dirty metadata */
+            if((*info) && (*info)->dirty) {
+                /* Sanity check new item */
+                assert(((H5AC_info_t*)thing)->dirty==0);
+
+                /* 'Hold' the current metadata for later */
+                (*dinfo)=(*info);
+
+                /* Reset the 'current' metadata, so it doesn't get flushed */
+                (*info)=NULL;
+
+#ifdef H5AC_DEBUG
+                cache->diagnostics[(*dinfo)->type->id].nholds++;
+#endif /* H5AC_DEBUG */
+            } /* end if */
+        } /* end else */
+    } /* end if */
+#endif /* H5_HAVE_PARALLEL */
+
     /*
      * Flush any object already in the cache at that location.  It had
      * better not be another copy of the protected object.
@@ -1015,12 +1624,13 @@ H5AC_unprotect(H5F_t *f, const H5AC_class_t *type, haddr_t addr, void *thing)
 
         assert(H5F_addr_ne((*info)->addr, addr));
         flush = (*info)->type->flush;
-        if ((flush)(f, H5AC_dxpl_id, TRUE, (*info)->addr, (*info)) < 0)
+        if ((flush)(f, dxpl_id, TRUE, (*info)->addr, (*info)) < 0)
             HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush object");
 #ifdef H5AC_DEBUG
         cache->diagnostics[type_id].nflushes++;
 #endif /* H5AC_DEBUG */
     }
+
 #ifdef H5AC_DEBUG
     /*
      * Remove the object's protect data to indicate that it is no longer
@@ -1079,7 +1689,7 @@ H5AC_debug(H5F_t UNUSED *f)
     char                    s[32], ascii[32];
     H5AC_t                 *cache = f->shared->cache;
     double                  miss_rate;
-#endif
+#endif /* H5AC_DEBUG */
     herr_t ret_value=SUCCEED;   /* Return value */
 
     FUNC_ENTER_NOAPI(H5AC_debug, FAIL);
@@ -1138,8 +1748,9 @@ H5AC_debug(H5F_t UNUSED *f)
 		     (long)(cache->diagnostics[i].ninits)));
 	}
     }
-#endif
+#endif /* H5AC_DEBUG */
 
 done:
     FUNC_LEAVE_NOAPI(ret_value);
 }
+