Merge branch 'develop' of https://bitbucket.hdfgroup.org/scm/hdffv/hdf5 into merge_hyperslab_updates

15 files changed, 192 insertions, 41 deletions

diff --git a/src/H5AC.c b/src/H5AC.c
index a561852..e2271c5 100644
--- a/src/H5AC.c
+++ b/src/H5AC.c
@@ -611,6 +611,7 @@ H5AC_dest(H5F_t *f, hid_t dxpl_id)
 
     /* Sanity check */
     HDassert(f);
+    HDassert(f->shared);
     HDassert(f->shared->cache);
 
 #if H5AC_DUMP_STATS_ON_CLOSE
@@ -641,9 +642,17 @@ H5AC_dest(H5F_t *f, hid_t dxpl_id)
         /* Sanity check */
         HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC);
 
-    /* Attempt to flush all entries from rank 0 & Bcast clean list to other ranks */
-    if(H5AC__flush_entries(f, dxpl_id) < 0)
+    /* If the file was opened R/W, attempt to flush all entries 
+     * from rank 0 & Bcast clean list to other ranks.
+     *
+     * Must not flush in the R/O case, as this will trigger the 
+     * free space manager settle routines.
+     */
+    if ( ( H5F_ACC_RDWR & H5F_INTENT(f) ) && 
+         ( H5AC__flush_entries(f, dxpl_id) < 0 ) )
+
         HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush")
+
 #endif /* H5_HAVE_PARALLEL */
 
     /* Destroy the cache */
@@ -3301,3 +3310,29 @@ done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* H5AC_remove_entry() */
 
+
+/*-------------------------------------------------------------------------
+ * Function:    H5AC_get_mdc_image_info
+ *
+ * Purpose:     Wrapper function for H5C_get_mdc_image_info().
+ *
+ * Return:      SUCCEED on success, and FAIL on failure.
+ *
+ * Programmer:  Vailin Choi; March 2017
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5AC_get_mdc_image_info(H5AC_t *cache_ptr, haddr_t *image_addr, hsize_t *image_len)
+{
+    herr_t ret_value = SUCCEED;      /* Return value */
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    if(H5C_get_mdc_image_info((H5C_t *)cache_ptr, image_addr, image_len) < 0)
+        HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "can't retrieve cache image info")
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* H5AC_get_mdc_image_info() */
+
diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h
index 1f8299b..a724394 100644
--- a/src/H5ACprivate.h
+++ b/src/H5ACprivate.h
@@ -453,6 +453,8 @@ H5_DLL herr_t H5AC_load_cache_image_on_next_protect(H5F_t *f, haddr_t addr,
 H5_DLL herr_t H5AC_validate_cache_image_config(H5AC_cache_image_config_t *config_ptr);
 H5_DLL hbool_t H5AC_cache_image_pending(const H5F_t *f);
 H5_DLL herr_t H5AC_force_cache_image_load(H5F_t * f, hid_t dxpl_id);
+H5_DLL herr_t H5AC_get_mdc_image_info(H5AC_t *cache_ptr, haddr_t *image_addr,
+    hsize_t *image_len);
 
 /* Tag & Ring routines */
 H5_DLL herr_t H5AC_tag(hid_t dxpl_id, haddr_t metadata_tag, haddr_t *prev_tag);
diff --git a/src/H5C.c b/src/H5C.c
index 120abb8..2ba9f2d 100644
--- a/src/H5C.c
+++ b/src/H5C.c
@@ -761,7 +761,10 @@ H5C_prep_for_file_close(H5F_t *f, hid_t dxpl_id)
         HGOTO_ERROR(H5E_CACHE, H5E_CANTCREATE, FAIL, "can't create cache image")
 
 #ifdef H5_HAVE_PARALLEL
-    if(!image_generated && cache_ptr->aux_ptr != NULL && f->shared->fs_persist) {
+    if ( ( H5F_INTENT(f) & H5F_ACC_RDWR ) &&
+         ( ! image_generated ) && 
+         ( cache_ptr->aux_ptr != NULL ) && 
+         ( f->shared->fs_persist ) ) {
         /* If persistent free space managers are enabled, flushing the
          * metadata cache may result in the deletion, insertion, and/or
          * dirtying of entries.
@@ -7295,14 +7298,20 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed,
 
             prev_ptr = entry_ptr->aux_prev;
 
+            if ( ( !(entry_ptr->prefetched_dirty) ) 
 #ifdef H5_HAVE_PARALLEL
-            if(!(entry_ptr->coll_access)) {
+                 && ( ! (entry_ptr->coll_access) )
 #endif /* H5_HAVE_PARALLEL */
-                if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0)
-                    HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush entry")
-#ifdef H5_HAVE_PARALLEL
+               ) {
+
+                if ( H5C__flush_single_entry(f, dxpl_id, entry_ptr, 
+                                  H5C__FLUSH_INVALIDATE_FLAG | 
+                                  H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0 )
+
+                    HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \
+                                "unable to flush entry")
+
             } /* end if */
-#endif /* H5_HAVE_PARALLEL */
 
 	    /* we are scanning the clean LRU, so the serialize function
 	     * will not be called on any entry -- thus there is no
diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h
index 539dece..bdfb23e 100644
--- a/src/H5Cprivate.h
+++ b/src/H5Cprivate.h
@@ -2321,6 +2321,7 @@ H5_DLL herr_t H5C_remove_entry(void *thing);
 H5_DLL herr_t H5C_cache_image_status(H5F_t * f, hbool_t *load_ci_ptr, 
     hbool_t *write_ci_ptr);
 H5_DLL hbool_t H5C_cache_image_pending(const H5C_t *cache_ptr);
+H5_DLL herr_t H5C_get_mdc_image_info(H5C_t *cache_ptr, haddr_t *image_addr, hsize_t *image_len);
 
 #ifdef H5_HAVE_PARALLEL
 H5_DLL herr_t H5C_apply_candidate_list(H5F_t *f, hid_t dxpl_id,
diff --git a/src/H5Cquery.c b/src/H5Cquery.c
index 33a322d..5108013 100644
--- a/src/H5Cquery.c
+++ b/src/H5Cquery.c
@@ -454,3 +454,33 @@ done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* H5C_get_entry_ring() */
 
+/*-------------------------------------------------------------------------
+ * Function:    H5C_get_mdc_image_info
+ *
+ * Purpose:	    To retrieve the address and size of the cache image in the file.
+ *              
+ * Return:      SUCCEED on success, and FAIL on failure.
+ *
+ * Programmer:  Vailin Choi; March 2017
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5C_get_mdc_image_info(H5C_t * cache_ptr, haddr_t *image_addr, hsize_t *image_len)
+{
+    herr_t ret_value = SUCCEED;      /* Return value */
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    if((cache_ptr == NULL) || (cache_ptr->magic != H5C__H5C_T_MAGIC))
+        HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "bad cache_ptr on entry")
+    if(image_addr == NULL || image_len == NULL)
+        HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "bad image_addr or image_len on entry")
+
+    *image_addr = cache_ptr->image_addr;
+    *image_len = cache_ptr->image_len;
+    
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* H5C_get_mdc_image_info() */
+
diff --git a/src/H5F.c b/src/H5F.c
index a0f7599..aef9d1d 100644
--- a/src/H5F.c
+++ b/src/H5F.c
@@ -2086,3 +2086,43 @@ done:
     FUNC_LEAVE_API(ret_value)
 } /* H5Fget_page_buffering_stats() */
 
+
+/*-------------------------------------------------------------------------
+ * Function:    H5Fget_mdc_image_info
+ *
+ * Purpose:     Retrieves the image_addr and image_len for the cache image in the file.
+ *              image_addr:  --base address of the on disk metadata cache image
+ *                           --HADDR_UNDEF if no cache image
+ *              image_len:   --size of the on disk metadata cache image
+ *                           --zero if no cache image
+ *
+ * Return:      Success:        SUCCEED
+ *              Failure:        FAIL
+ *
+ * Programmer:  Vailin Choi; March 2017
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Fget_mdc_image_info(hid_t file_id, haddr_t *image_addr, hsize_t *image_len)
+{
+    H5F_t      *file;                   /* File object for file ID */
+    herr_t     ret_value = SUCCEED;     /* Return value */
+
+    FUNC_ENTER_API(FAIL)
+    H5TRACE3("e", "i*a*h", file_id, image_addr, image_len);
+
+    /* Check args */
+    if(NULL == (file = (H5F_t *)H5I_object_verify(file_id, H5I_FILE)))
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a file ID")
+    if(NULL == image_addr || NULL == image_len)
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "NULL image addr or image len")
+
+    /* Go get the address and size of the cache image */
+    if(H5AC_get_mdc_image_info(file->shared->cache, image_addr, image_len) < 0)
+        HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "can't retrieve cache image info")
+
+done:
+    FUNC_LEAVE_API(ret_value)
+} /* H5Fget_mdc_image_info() */
+
diff --git a/src/H5Fint.c b/src/H5Fint.c
index 794be50..444d409 100644
--- a/src/H5Fint.c
+++ b/src/H5Fint.c
@@ -1006,8 +1006,15 @@ H5F__dest(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t flush)
             /* Push error, but keep going*/
             HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "problems closing file")
 
+        /* Set up I/O info for operation */
+        fio_info.f = f;
+        if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(meta_dxpl_id)))
+            HDONE_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+        if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id)))
+            HDONE_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+
         /* Shutdown the page buffer cache */
-        if(H5PB_dest(f) < 0)
+        if(H5PB_dest(&fio_info) < 0)
             /* Push error, but keep going*/
             HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "problems closing page buffer cache")
 
@@ -1027,13 +1034,6 @@ H5F__dest(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t flush)
             f->shared->root_grp = NULL;
         } /* end if */
 
-        /* Set up I/O info for operation */
-        fio_info.f = f;
-        if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(meta_dxpl_id)))
-            HDONE_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
-        if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id)))
-            HDONE_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
-
         /* Destroy other components of the file */
         if(H5F__accum_reset(&fio_info, TRUE) < 0)
             /* Push error, but keep going*/
diff --git a/src/H5Fpublic.h b/src/H5Fpublic.h
index f87aaad..84c0d38 100644
--- a/src/H5Fpublic.h
+++ b/src/H5Fpublic.h
@@ -267,6 +267,7 @@ H5_DLL herr_t H5Fformat_convert(hid_t fid);
 H5_DLL herr_t H5Freset_page_buffering_stats(hid_t file_id);
 H5_DLL herr_t H5Fget_page_buffering_stats(hid_t file_id, unsigned accesses[2],
     unsigned hits[2], unsigned misses[2], unsigned evictions[2], unsigned bypasses[2]);
+H5_DLL herr_t H5Fget_mdc_image_info(hid_t file_id, haddr_t *image_addr, hsize_t *image_size);
 
 #ifdef H5_HAVE_PARALLEL
 H5_DLL herr_t H5Fset_mpi_atomicity(hid_t file_id, hbool_t flag);
diff --git a/src/H5MF.c b/src/H5MF.c
index 358e326..87c910c 100644
--- a/src/H5MF.c
+++ b/src/H5MF.c
@@ -1347,6 +1347,12 @@ HDfprintf(stderr, "%s: Entering: alloc_type = %u, addr = %a, size = %Hu, extra_r
     HDassert(f);
     HDassert(H5F_INTENT(f) & H5F_ACC_RDWR);
 
+    if(f->shared->first_alloc_dealloc) {
+        HDassert(! H5AC_cache_image_pending(f));
+        if(H5MF_tidy_self_referential_fsm_hack(f, dxpl_id) < 0)
+            HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "tidy of self referential fsm hack failed")
+    } /* end if */
+
     /* Set mapped type, treating global heap as raw data */
     map_type = (alloc_type == H5FD_MEM_GHEAP) ? H5FD_MEM_DRAW : alloc_type;
 
diff --git a/src/H5PB.c b/src/H5PB.c
index c67ae59..575903d 100644
--- a/src/H5PB.c
+++ b/src/H5PB.c
@@ -468,7 +468,7 @@ H5PB__dest_cb(void *item, void H5_ATTR_UNUSED *key, void *_op_data)
 /*-------------------------------------------------------------------------
  * Function:	H5PB_dest
  *
- * Purpose:	destroy the PB on the file.
+ * Purpose:	Flush and destroy the PB on the file if it exists.
  *
  * Return:	Non-negative on success/Negative on failure
  *
@@ -477,20 +477,26 @@ H5PB__dest_cb(void *item, void H5_ATTR_UNUSED *key, void *_op_data)
  *-------------------------------------------------------------------------
  */
 herr_t
-H5PB_dest(H5F_t *f)
+H5PB_dest(const H5F_io_info2_t *fio_info)
 {
     herr_t  ret_value = SUCCEED;        /* Return value */
+    H5F_t  *f;                          /* file pointer */
 
     FUNC_ENTER_NOAPI(FAIL)
 
     /* Sanity checks */
+    HDassert(fio_info);
+    f = fio_info->f;
     HDassert(f);
 
-    /* Destroy page buffer info, if there is any */
+    /* flush and destroy the page buffer, if it exists */
     if(f->shared->page_buf) {
         H5PB_t *page_buf = f->shared->page_buf;
         H5PB_ud1_t op_data;                 /* Iteration context */
 
+        if(H5PB_flush(fio_info)<0)
+            HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTFLUSH, FAIL, "can't flush page buffer")
+
         /* Set up context info */
         op_data.page_buf = page_buf;
 
diff --git a/src/H5PBprivate.h b/src/H5PBprivate.h
index 3b5dcae..7dd4071 100644
--- a/src/H5PBprivate.h
+++ b/src/H5PBprivate.h
@@ -89,7 +89,7 @@ typedef struct H5PB_t {
 /* General routines */
 H5_DLL herr_t H5PB_create(H5F_t *file, size_t page_buffer_size, unsigned page_buf_min_meta_perc, unsigned page_buf_min_raw_perc);
 H5_DLL herr_t H5PB_flush(const H5F_io_info2_t *fio_info);
-H5_DLL herr_t H5PB_dest(H5F_t *file);
+H5_DLL herr_t H5PB_dest(const H5F_io_info2_t *fio_info);
 H5_DLL herr_t H5PB_add_new_page(H5F_t *f, H5FD_mem_t type, haddr_t page_addr);
 H5_DLL herr_t H5PB_update_entry(H5PB_t *page_buf, haddr_t addr, size_t size, const void *buf);
 H5_DLL herr_t H5PB_remove_entry(const H5F_t *f, haddr_t addr);
diff --git a/src/H5private.h b/src/H5private.h
index 00de96c..61ecc5b 100644
--- a/src/H5private.h
+++ b/src/H5private.h
@@ -1386,7 +1386,13 @@ typedef off_t               h5_stat_size_t;
 #ifndef HDstrtol
     #define HDstrtol(S,R,N)    strtol(S,R,N)
 #endif /* HDstrtol */
-H5_DLL int64_t HDstrtoll (const char *s, const char **rest, int base);
+#ifndef HDstrtoll
+    #ifdef H5_HAVE_STRTOLL
+        #define HDstrtoll(S,R,N)  strtoll(S,R,N)
+    #else
+        H5_DLL int64_t HDstrtoll (const char *s, const char **rest, int base);
+    #endif /* H5_HAVE_STRTOLL */
+#endif /* HDstrtoll */
 #ifndef HDstrtoul
     #define HDstrtoul(S,R,N)  strtoul(S,R,N)
 #endif /* HDstrtoul */
diff --git a/src/H5public.h b/src/H5public.h
index f0eb63a..cab90aa 100644
--- a/src/H5public.h
+++ b/src/H5public.h
@@ -93,11 +93,11 @@ extern "C" {
 
 /* Version numbers */
 #define H5_VERS_MAJOR	1	/* For major interface/format changes  	     */
-#define H5_VERS_MINOR	9	/* For minor interface/format changes  	     */
-#define H5_VERS_RELEASE	236	/* For tweaks, bug-fixes, or development     */
+#define H5_VERS_MINOR	11	/* For minor interface/format changes  	     */
+#define H5_VERS_RELEASE	0	/* For tweaks, bug-fixes, or development     */
 #define H5_VERS_SUBRELEASE ""	/* For pre-releases like snap0       */
 				/* Empty string for real releases.           */
-#define H5_VERS_INFO    "HDF5 library version: 1.9.236"      /* Full version string */
+#define H5_VERS_INFO    "HDF5 library version: 1.11.0"      /* Full version string */
 
 #define H5check()	H5check_version(H5_VERS_MAJOR,H5_VERS_MINOR,	      \
 				        H5_VERS_RELEASE)
diff --git a/src/H5system.c b/src/H5system.c
index ac323c0..1f92e19 100644
--- a/src/H5system.c
+++ b/src/H5system.c
@@ -32,10 +32,10 @@
 /***********/
 /* Headers */
 /***********/
-#include "H5private.h"		/* Generic Functions			*/
-#include "H5Eprivate.h"		/* Error handling		  	*/
-#include "H5Fprivate.h"		/* File access				*/
-#include "H5MMprivate.h"	/* Memory management			*/
+#include "H5private.h"        /* Generic Functions            */
+#include "H5Eprivate.h"        /* Error handling              */
+#include "H5Fprivate.h"        /* File access                */
+#include "H5MMprivate.h"    /* Memory management            */
 
 
 /****************/
@@ -474,6 +474,7 @@ HDfprintf(FILE *stream, const char *fmt, ...)
  *
  *-------------------------------------------------------------------------
  */
+#ifndef HDstrtoll
 int64_t
 HDstrtoll(const char *s, const char **rest, int base)
 {
@@ -549,7 +550,7 @@ HDstrtoll(const char *s, const char **rest, int base)
         *rest = s;
     return acc;
 } /* end HDstrtoll() */
-
+#endif
 
 /*-------------------------------------------------------------------------
  * Function:  HDrand/HDsrand
@@ -604,7 +605,7 @@ void HDsrand(unsigned int seed)
 #ifdef H5_HAVE_FCNTL
 int
 Pflock(int fd, int operation) {
-    
+
     struct flock    flk;
 
     /* Set the lock type */
@@ -649,18 +650,18 @@ Nflock(int H5_ATTR_UNUSED fd, int H5_ATTR_UNUSED operation) {
 
 
 /*-------------------------------------------------------------------------
- * Function:	H5_make_time
+ * Function:    H5_make_time
  *
- * Purpose:	Portability routine to abstract converting a 'tm' struct into
- *		a time_t value.
+ * Purpose:    Portability routine to abstract converting a 'tm' struct into
+ *        a time_t value.
  *
- * Note:	This is a little problematic because mktime() operates on
- *		local times.  We convert to local time and then figure out the
- *		adjustment based on the local time zone and daylight savings
- *		setting.
+ * Note:    This is a little problematic because mktime() operates on
+ *        local times.  We convert to local time and then figure out the
+ *        adjustment based on the local time zone and daylight savings
+ *        setting.
  *
- * Return:	Success:  The value of timezone
- *		Failure:  -1
+ * Return:    Success:  The value of timezone
+ *        Failure:  -1
  *
  * Programmer:  Quincey Koziol
  *              November 18, 2015
@@ -1138,7 +1139,7 @@ H5_combine_path(const char* path1, const char* path2, char **full_name /*out*/)
         if(NULL == (*full_name = (char *)H5MM_strdup(path2)))
             HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed")
 
-    } /* end if */ 
+    } /* end if */
     else if(H5_CHECK_ABS_PATH(path2)) {
 
         /* On windows path2 is a path absolute name */
diff --git a/src/H5win32defs.h b/src/H5win32defs.h
index b419f06..e005b51 100644
--- a/src/H5win32defs.h
+++ b/src/H5win32defs.h
@@ -23,6 +23,11 @@
  *
  */
 
+/*
+ * _MSC_VER = 1900 VS2015
+ * _MSC_VER = 1800 VS2013
+ * _MSC_VER = 1700 VS2012
+ */
 #ifdef H5_HAVE_WIN32_API
 
 typedef struct _stati64     h5_stat_t;
@@ -54,13 +59,22 @@ typedef __int64             h5_stat_size_t;
 #define HDsleep(S)          Sleep(S*1000)
 #define HDstat(S,B)         _stati64(S,B)
 #define HDstrcasecmp(A,B)   _stricmp(A,B)
-#define HDstrtoull(S,R,N)   _strtoui64(S,R,N)
 #define HDstrdup(S)         _strdup(S)
 #define HDtzset()           _tzset()
 #define HDunlink(S)         _unlink(S)
 #define HDwrite(F,M,Z)      _write(F,M,Z)
 
 #ifdef H5_HAVE_VISUAL_STUDIO
+
+#if (_MSC_VER < 1800)
+  #ifndef H5_HAVE_STRTOLL
+    #define HDstrtoll(S,R,N)    _strtoi64(S,R,N)
+  #endif /* H5_HAVE_STRTOLL */
+  #ifndef H5_HAVE_STRTOULL
+    #define HDstrtoull(S,R,N)   _strtoui64(S,R,N)
+  #endif /* H5_HAVE_STRTOULL */
+#endif /* MSC_VER < 1800 */
+
 /*
  * The (void*) cast just avoids a compiler warning in H5_HAVE_VISUAL_STUDIO
  */