[svn-r28649] - add new API to toggle collective metadata write.

- set default to FALSE
- fix some bugs.

10 files changed, 119 insertions, 28 deletions

diff --git a/src/H5Cmpio.c b/src/H5Cmpio.c
index 604a991..0d75ac1 100644
--- a/src/H5Cmpio.c
+++ b/src/H5Cmpio.c
@@ -257,9 +257,11 @@ H5C_apply_candidate_list(H5F_t * f,
     HDfprintf(stdout, "%s", tbl_buf);
 #endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
 
-    /* Create skip list of entries for collective write */
-    if(NULL == (collective_write_list = H5SL_create(H5SL_TYPE_HADDR, NULL)))
-        HGOTO_ERROR(H5E_DATASET, H5E_CANTCREATE, FAIL, "can't create skip list for entries")
+    if(f->coll_md_write) {
+        /* Create skip list of entries for collective write */
+        if(NULL == (collective_write_list = H5SL_create(H5SL_TYPE_HADDR, NULL)))
+            HGOTO_ERROR(H5E_DATASET, H5E_CANTCREATE, FAIL, "can't create skip list for entries")
+    }
 
     n = num_candidates / mpi_size;
     m = num_candidates % mpi_size;
@@ -711,30 +713,34 @@ H5C_apply_candidate_list(H5F_t * f,
     if (delayed_ptr) {
 
         if (delayed_ptr->clear_on_unprotect) {
-            if(H5C__flush_single_entry(f, dxpl_id, delayed_ptr, H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG, 
+            if(H5C__flush_single_entry(f, dxpl_id, delayed_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG, 
                                        NULL, NULL) < 0)
                 HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't flush entry collectively.")
+
             entry_ptr->clear_on_unprotect = FALSE;
             entries_cleared++;
         } else if (delayed_ptr->flush_immediately) {
             if(H5C__flush_single_entry(f, dxpl_id, delayed_ptr, H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG, 
                                        NULL, collective_write_list) < 0)
                 HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't flush entry collectively.")
+
             entry_ptr->flush_immediately = FALSE;
             entries_flushed++;
         } /* end if */
 
-
         entries_flushed_collectively++;
         entries_flushed_or_cleared_last++;
     } /* end if */
 
-    /* Write collective list */
-    if(H5C_collective_write(f,
-                            dxpl_id,
-                            collective_write_list) < 0)
-        HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't write metadata collectively")
+    if(f->coll_md_write) {
+        HDassert(collective_write_list);
 
+        /* Write collective list */
+        if(H5C_collective_write(f,
+                                dxpl_id,
+                                collective_write_list) < 0)
+            HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't write metadata collectively")
+    }
     /* ====================================================================== *
      * Finished flushing everything.                                          *
      * ====================================================================== */
diff --git a/src/H5Fint.c b/src/H5Fint.c
index 33db083..4e44bcb 100644
--- a/src/H5Fint.c
+++ b/src/H5Fint.c
@@ -639,6 +639,8 @@ H5F_new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t
 #ifdef H5_HAVE_PARALLEL
         if(H5P_get(plist, H5_COLL_MD_READ_FLAG_NAME, &(f->coll_md_read)) < 0)
             HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get collective metadata read flag")
+        if(H5P_get(plist, H5F_ACS_COLL_MD_WRITE_FLAG_NAME, &(f->coll_md_write)) < 0)
+            HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get collective metadata write flag")
 #endif /* H5_HAVE_PARALLEL */
 
         /* Get the VFD values to cache */
diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h
index 4fda863..55e322b 100644
--- a/src/H5Fpkg.h
+++ b/src/H5Fpkg.h
@@ -312,7 +312,8 @@ struct H5F_t {
     struct H5F_t        *parent;        /* Parent file that this file is mounted to */
     unsigned            nmounts;        /* Number of children mounted to this file */
 #ifdef H5_HAVE_PARALLEL
-    H5P_coll_md_read_flag_t              coll_md_read;   /* Do all metadata reads collectively */
+    H5P_coll_md_read_flag_t coll_md_read;  /* Do all metadata reads collectively */
+    hbool_t             coll_md_write;  /* Do all metadata writes collectively */
 #endif /* H5_HAVE_PARALLEL */
 };
 
diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h
index 8c0e608..2d61a4c 100644
--- a/src/H5Fprivate.h
+++ b/src/H5Fprivate.h
@@ -454,6 +454,7 @@
 #define H5F_ACS_FILE_IMAGE_INFO_NAME            "file_image_info" /* struct containing initial file image and callback info */
 #define H5F_ACS_CORE_WRITE_TRACKING_FLAG_NAME       "core_write_tracking_flag" /* Whether or not core VFD backing store write tracking is enabled */
 #define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_NAME  "core_write_tracking_page_size" /* The page size in kiB when core VFD write tracking is enabled */
+#define H5F_ACS_COLL_MD_WRITE_FLAG_NAME         "collective_metadata_write" /* property indicating whether metadata writes are done collectively or not */
 
 /* ======================== File Mount properties ====================*/
 #define H5F_MNT_SYM_LOCAL_NAME 		"local"                 /* Whether absolute symlinks local to file. */
diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c
index 7e1e0ee..9cc8fc7 100644
--- a/src/H5Pfapl.c
+++ b/src/H5Pfapl.c
@@ -183,6 +183,11 @@
 #define H5F_ACS_COLL_MD_READ_FLAG_DEF    H5P_USER_FALSE
 #define H5F_ACS_COLL_MD_READ_FLAG_ENC    H5P__encode_coll_md_read_flag_t
 #define H5F_ACS_COLL_MD_READ_FLAG_DEC    H5P__decode_coll_md_read_flag_t
+/* Definition of collective metadata write mode flag */
+#define H5F_ACS_COLL_MD_WRITE_FLAG_SIZE   sizeof(hbool_t)
+#define H5F_ACS_COLL_MD_WRITE_FLAG_DEF    FALSE
+#define H5F_ACS_COLL_MD_WRITE_FLAG_ENC    H5P__encode_hbool_t
+#define H5F_ACS_COLL_MD_WRITE_FLAG_DEC    H5P__decode_hbool_t
 
 
 /******************/
@@ -287,6 +292,7 @@ static const H5FD_file_image_info_t H5F_def_file_image_info_g = H5F_ACS_FILE_IMA
 static const hbool_t H5F_def_core_write_tracking_flag_g = H5F_ACS_CORE_WRITE_TRACKING_FLAG_DEF;              /* Default setting for core VFD write tracking */
 static const size_t H5F_def_core_write_tracking_page_size_g = H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_DEF;     /* Default core VFD write tracking page size */
 static const H5P_coll_md_read_flag_t H5F_def_coll_md_read_flag_g = H5F_ACS_COLL_MD_READ_FLAG_DEF;  /* Default setting for the collective metedata read flag */
+static const hbool_t H5F_def_coll_md_write_flag_g = H5F_ACS_COLL_MD_WRITE_FLAG_DEF;  /* Default setting for the collective metedata write flag */
 
 
 
@@ -451,6 +457,13 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass)
             NULL, NULL, NULL, NULL) < 0)
         HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class")
 
+    /* Register the metadata collective write flag */
+    if(H5P_register_real(pclass, H5F_ACS_COLL_MD_WRITE_FLAG_NAME, H5F_ACS_COLL_MD_WRITE_FLAG_SIZE, 
+            &H5F_def_coll_md_write_flag_g, 
+            NULL, NULL, NULL, H5F_ACS_COLL_MD_WRITE_FLAG_ENC, H5F_ACS_COLL_MD_WRITE_FLAG_DEC, 
+            NULL, NULL, NULL, NULL) < 0)
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class")
+
 done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5P__facc_reg_prop() */
@@ -3651,4 +3664,75 @@ H5Pget_coll_metadata_read(hid_t plist_id, hbool_t *is_collective)
 done:
     FUNC_LEAVE_API(ret_value)
 } /* H5Pget_coll_metadata_read */
+
+
+/*-------------------------------------------------------------------------
+ * Function:	H5Pset_coll_metadata_write
+ *
+ * Purpose: Tell the library whether the metadata write operations will
+ * be done collectively (1) or not (0). Default is collective.
+ *
+ * Return:	Non-negative on success/Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pset_coll_metadata_write(hid_t plist_id, hbool_t is_collective)
+{
+    H5P_genplist_t *plist;        /* Property list pointer */
+    herr_t ret_value = SUCCEED;   /* return value */
+
+    FUNC_ENTER_API(FAIL)
+    H5TRACE2("e", "ib", plist_id, is_collective);
+
+    /* Compare the property list's class against the other class */
+    if(TRUE != H5P_isa_class(plist_id, H5P_FILE_ACCESS))
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "property list is not a file access plist")
+
+    /* Get the plist structure */
+    if(NULL == (plist = (H5P_genplist_t *)H5I_object(plist_id)))
+        HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID")
+
+    /* Set values */
+    if(H5P_set(plist, H5F_ACS_COLL_MD_WRITE_FLAG_NAME, &is_collective) < 0)
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set collective metadata write flag")
+
+done:
+    FUNC_LEAVE_API(ret_value)
+} /* H5Pset_coll_metadata_write */
+
+
+/*-------------------------------------------------------------------------
+ * Function:	H5Pget_coll_metadata_write
+ *
+ * Purpose:	Gets information about collective metadata write mode.
+ *
+ * Return:	Non-negative on success/Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pget_coll_metadata_write(hid_t plist_id, hbool_t *is_collective)
+{
+    H5P_genplist_t *plist;      /* Property list pointer */
+    herr_t ret_value = SUCCEED;   /* return value */
+
+    FUNC_ENTER_API(FAIL)
+    H5TRACE2("e", "i*b", plist_id, is_collective);
+
+    /* Compare the property list's class against the other class */
+    if(TRUE != H5P_isa_class(plist_id, H5P_FILE_ACCESS))
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "property list is not an access plist")
+
+    /* Get the plist structure */
+    if(NULL == (plist = (H5P_genplist_t *)H5I_object(plist_id)))
+        HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID")
+
+    if(H5P_get(plist, H5F_ACS_COLL_MD_WRITE_FLAG_NAME, is_collective) < 0)
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get collective metadata write flag")
+
+done:
+    FUNC_LEAVE_API(ret_value)
+} /* H5Pget_coll_metadata_write */
+
 #endif /* H5_HAVE_PARALLEL */
diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h
index 6a1dbad..a0a684f 100644
--- a/src/H5Ppublic.h
+++ b/src/H5Ppublic.h
@@ -354,6 +354,8 @@ H5_DLL herr_t H5Pget_core_write_tracking(hid_t fapl_id, hbool_t *is_enabled, siz
 #ifdef H5_HAVE_PARALLEL
 H5_DLL herr_t H5Pset_coll_metadata_read(hid_t plist_id, hbool_t is_collective);
 H5_DLL herr_t H5Pget_coll_metadata_read(hid_t plist_id, hbool_t *is_collective);
+H5_DLL herr_t H5Pset_coll_metadata_write(hid_t plist_id, hbool_t is_collective);
+H5_DLL herr_t H5Pget_coll_metadata_write(hid_t plist_id, hbool_t *is_collective);
 #endif /* H5_HAVE_PARALLEL */
 
 /* Dataset creation property list (DCPL) routines */
diff --git a/testpar/t_cache.c b/testpar/t_cache.c
index e91ee4e..5b6b44f 100644
--- a/testpar/t_cache.c
+++ b/testpar/t_cache.c
@@ -7508,15 +7508,6 @@ main(int argc, char **argv)
         }
     }
 
-    if ( H5Pset_coll_metadata_read(fapl, 1) < 0 ) {
-
-        nerrors++;
-        if ( verbose ) {
-            HDfprintf(stdout, "%d:%s: H5Pset_coll_metadata_read() failed 1.\n",
-                      world_mpi_rank, FUNC);
-        }
-    }
-
     /* fix the file names */
     for ( u = 0; u < sizeof(FILENAME) / sizeof(FILENAME[0]) - 1; ++u )
     {
@@ -7561,6 +7552,7 @@ main(int argc, char **argv)
                           world_mpi_rank, FUNC);
             }
         }
+
     }
 
     setup_rand();
@@ -7599,6 +7591,10 @@ main(int argc, char **argv)
     smoke_check_4(H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY);
     smoke_check_4(H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED);
 #endif
+#if 1
+    smoke_check_5(H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY);
+    smoke_check_5(H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED);
+#endif
     /* enable the collective metadata read property */
     if ( world_mpi_rank != world_server_mpi_rank ) {
         if ( H5Pset_coll_metadata_read(fapl, 1) < 0 ) {
@@ -7611,11 +7607,6 @@ main(int argc, char **argv)
         }
     }
 #if 1
-    smoke_check_5(H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY);
-    smoke_check_5(H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED);
-#endif
-
-#if 1
     smoke_check_6(H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY);
     smoke_check_6(H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED);
 #endif
diff --git a/testpar/t_dset.c b/testpar/t_dset.c
index ae022fb..2e1005c 100644
--- a/testpar/t_dset.c
+++ b/testpar/t_dset.c
@@ -457,7 +457,7 @@ dataset_readInd(void)
     VRFY((acc_tpl >= 0), "");
 
     /* open the file collectively */
-    fid=H5Fopen(filename,H5F_ACC_RDONLY,acc_tpl);
+    fid = H5Fopen(filename, H5F_ACC_RDONLY, acc_tpl);
     VRFY((fid >= 0), "");
 
     /* Release file-access template */
diff --git a/testpar/t_shapesame.c b/testpar/t_shapesame.c
index 5152408..1b856d0 100644
--- a/testpar/t_shapesame.c
+++ b/testpar/t_shapesame.c
@@ -4948,7 +4948,9 @@ create_faccess_plist(MPI_Comm comm, MPI_Info info, int l_facc_type)
 	/* set Parallel access with communicator */
 	ret = H5Pset_fapl_mpio(ret_pl, comm, info);
 	VRFY((ret >= 0), "");
-        ret = H5Pset_coll_metadata_read(ret_pl, 1);
+        ret = H5Pset_coll_metadata_read(ret_pl, true);
+	VRFY((ret >= 0), "");
+        ret = H5Pset_coll_metadata_write(ret_pl, true);
 	VRFY((ret >= 0), "");
 	return(ret_pl);
     }
diff --git a/testpar/testphdf5.c b/testpar/testphdf5.c
index b652ef4..7ab479d 100644
--- a/testpar/testphdf5.c
+++ b/testpar/testphdf5.c
@@ -272,7 +272,9 @@ create_faccess_plist(MPI_Comm comm, MPI_Info info, int l_facc_type)
 	/* set Parallel access with communicator */
 	ret = H5Pset_fapl_mpio(ret_pl, comm, info);
 	VRFY((ret >= 0), "");
-        ret = H5Pset_coll_metadata_read(ret_pl, 1);
+        ret = H5Pset_coll_metadata_read(ret_pl, true);
+	VRFY((ret >= 0), "");
+        ret = H5Pset_coll_metadata_write(ret_pl, true);
 	VRFY((ret >= 0), "");
 	return(ret_pl);
     }