Merging in latest from upstream (HDFFV/hdf5:refs/heads/develop)

* commit 'ab3963b28e59419f8e857ec224dd4efa3ea6dd8e': (214 commits)
  Merge SWMR-related testing to existing tests.
  Bring over tweak for missing environment variable.
  Update CMake configuration files with SWMR accumulator changes.
  Add missing accumulator test.
  Merge SWMR-oriented accumulator tests from revise_chunks to develop.
  Bring Java SWMR changes from revise_chunks to develop branch
  Snapshot version 1.9 release 235
  Change dlopen from RTLD_NOW to RTLD_LAZY
  Fix bad implementation of Windows nanosleep equivalent.
  Removed NDEBUG guards from H5AC test functions.
  Bring SWMR-related tools changes from revise_chunks to develop.
  Bring over changes from revise_chunks that cleanup recent SWMR changes from code review feedback.
  Updated the H5FS cache code to grab the correct tag and modified the freespace test to use dxpls that have been tagged with the H5AC__FREESPACE_TAG global tag instead of H5AC_ind_read_dxpl_id. The library code now expects the owner of the free space manager to tag it so the owner-less free space managers in the freespace tag had to be tagged with *something* to avoid cache errors.
  Updated the comment for the valgrind fix.
  Fixed a valgrind problem in file shutdown exposed by the swmr.c test.
  Reduce timeout as normal run time on windows is less then 10 min
  Updated the icc flags (C flags only).
  Another Java oversight (sorry, don't have Java configured on my Mac)
  Correct oversight in Java test and remove direct VFD from SWMR supported drivers.
  Bring SWMR support in to the main development branch.  (Finally!)  More tests and the tool and API wrappers will be coming in over the weekend.
  ...

1 files changed, 162 insertions, 124 deletions

diff --git a/src/H5HGcache.c b/src/H5HGcache.c
index c1eb8d9..6bc9860 100644
--- a/src/H5HGcache.c
+++ b/src/H5HGcache.c
@@ -62,11 +62,12 @@
 /********************/
 
 /* Metadata cache callbacks */
-static herr_t H5HG__cache_heap_get_load_size(const void *udata, size_t *image_len);
+static herr_t H5HG__cache_heap_get_initial_load_size(void *udata, size_t *image_len);
+static herr_t H5HG__cache_heap_get_final_load_size(const void *_image,
+    size_t image_len, void *udata, size_t *actual_len);
 static void *H5HG__cache_heap_deserialize(const void *image, size_t len,
     void *udata, hbool_t *dirty); 
-static herr_t H5HG__cache_heap_image_len(const void *thing, size_t *image_len,
-    hbool_t *compressed_ptr, size_t *compressed_image_len_ptr);
+static herr_t H5HG__cache_heap_image_len(const void *thing, size_t *image_len);
 static herr_t H5HG__cache_heap_serialize(const H5F_t *f, void *image,
     size_t len, void *thing); 
 static herr_t H5HG__cache_heap_free_icr(void *thing);
@@ -82,14 +83,15 @@ const H5AC_class_t H5AC_GHEAP[1] = {{
     "global heap",                      /* Metadata client name (for debugging) */
     H5FD_MEM_GHEAP,                     /* File space memory type for client */
     H5AC__CLASS_SPECULATIVE_LOAD_FLAG,  /* Client class behavior flags */
-    H5HG__cache_heap_get_load_size,     /* 'get_load_size' callback */
+    H5HG__cache_heap_get_initial_load_size,     /* 'get_initial_load_size' callback */
+    H5HG__cache_heap_get_final_load_size, /* 'get_final_load_size' callback */
+    NULL, 				/* 'verify_chksum' callback */
     H5HG__cache_heap_deserialize,       /* 'deserialize' callback */
     H5HG__cache_heap_image_len,         /* 'image_len' callback */
     NULL,                               /* 'pre_serialize' callback */
     H5HG__cache_heap_serialize,         /* 'serialize' callback */
     NULL,                               /* 'notify' callback */
     H5HG__cache_heap_free_icr,          /* 'free_icr' callback */
-    NULL,                               /* 'clear' callback */
     NULL,                               /* 'fsf_size' callback */
 }};
 
@@ -106,13 +108,13 @@ const H5AC_class_t H5AC_GHEAP[1] = {{
 
 
 /*-------------------------------------------------------------------------
- * Function:    H5HG__cache_heap_get_load_size()
+ * Function:    H5HG__cache_heap_get_initial_load_size()
  *
  * Purpose:	Return the initial speculative read size to the metadata 
  *		cache.  This size will be used in the initial attempt to read 
  *		the global heap.  If this read is too small, the cache will 
  *		try again with the correct value obtained from 
- *		H5HG__cache_heap_image_len().
+ *		H5HG__cache_get_final_load_size().
  *
  * Return:      Success:        SUCCEED
  *              Failure:        FAIL
@@ -123,16 +125,74 @@ const H5AC_class_t H5AC_GHEAP[1] = {{
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5HG__cache_heap_get_load_size(const void H5_ATTR_UNUSED *_udata, size_t *image_len)
+H5HG__cache_heap_get_initial_load_size(void H5_ATTR_UNUSED *_udata, size_t *image_len)
 {
     FUNC_ENTER_STATIC_NOERR
 
+    /* Sanity check */
     HDassert(image_len);
 
+    /* Set the image length size */
     *image_len = (size_t)H5HG_MINSIZE;
 
     FUNC_LEAVE_NOAPI(SUCCEED)
-} /* end H5HG__cache_heap_get_load_size() */
+} /* end H5HG__cache_heap_get_initial_load_size() */
+
+
+/*-------------------------------------------------------------------------
+ * Function:    H5HG__cache_heap_get_initial_load_size()
+ *
+ * Purpose:	Return the final read size for a speculatively ready heap to
+ *		the metadata cache.
+ *
+ * Return:      Success:        SUCCEED
+ *              Failure:        FAIL
+ *
+ * Programmer:  Quincey Koziol
+ *              November 18, 2016
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5HG__cache_heap_get_final_load_size(const void *_image, size_t image_len,
+    void *_udata, size_t *actual_len)
+{
+    const uint8_t *image = (const uint8_t *)_image;   	/* Pointer into raw data buffer */
+    H5F_t *f = (H5F_t *)_udata;                 /* File pointer -- obtained from user data */
+    size_t heap_size = 0;                       /* Total size of collection      */
+    herr_t ret_value = SUCCEED;                 /* Return value */
+
+    FUNC_ENTER_STATIC
+
+    /* Sanity check */
+    HDassert(image);
+    HDassert(f);
+    HDassert(actual_len);
+    HDassert(*actual_len == image_len);
+
+    /* Magic number */
+    if(HDmemcmp(image, H5HG_MAGIC, (size_t)H5_SIZEOF_MAGIC))
+        HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad global heap collection signature")
+    image += H5_SIZEOF_MAGIC;
+
+    /* Version */
+    if(H5HG_VERSION != *image++)
+        HGOTO_ERROR(H5E_HEAP, H5E_VERSION, FAIL, "wrong version number in global heap")
+
+    /* Reserved */
+    image += 3;
+
+    /* Size */
+    H5F_DECODE_LENGTH(f, image, heap_size);
+    HDassert(heap_size >= H5HG_MINSIZE);
+    HDassert(image_len == H5HG_MINSIZE);
+
+    /* Set the final size for the cache image */
+    *actual_len = heap_size;
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5HG__cache_heap_get_final_load_size() */
 
 
 /*-------------------------------------------------------------------------
@@ -142,10 +202,6 @@ H5HG__cache_heap_get_load_size(const void H5_ATTR_UNUSED *_udata, size_t *image_
  *		heap, deserialize it, load its contents into a newly allocated
  *		instance of H5HG_heap_t, and return a pointer to the new instance.
  *
- *		Note that this heap client uses speculative reads.  If the supplied
- *		buffer is too small, we simply make note of the correct size, and 
- *		wait for the metadata cache to try again.
- *
  * Return:      Success:        Pointer to in core representation
  *              Failure:        NULL
  *
@@ -161,6 +217,8 @@ H5HG__cache_heap_deserialize(const void *_image, size_t len, void *_udata,
     H5F_t       *f = (H5F_t *)_udata;   /* File pointer -- obtained from user data */
     H5HG_heap_t *heap = NULL;   /* New global heap */
     uint8_t     *image;         /* Pointer to image to decode */
+    size_t       max_idx = 0;   /* Maximum heap object index seen */
+    size_t       nalloc;        /* Number of objects allocated */
     void        *ret_value = NULL;      /* Return value */
 
     FUNC_ENTER_STATIC
@@ -201,109 +259,98 @@ H5HG__cache_heap_deserialize(const void *_image, size_t len, void *_udata,
     HDassert((len == H5HG_MINSIZE) /* first try */ || 
              ((len == heap->size) && (len > H5HG_MINSIZE))); /* second try */
     
-    if(len == heap->size) {     /* proceed with the deserialize */
-        size_t       max_idx = 0;
-        size_t       nalloc;
-
-        /* Decode each object */
-        image = heap->chunk + H5HG_SIZEOF_HDR(f);
-        nalloc = H5HG_NOBJS(f, heap->size);
-
-        /* Calloc the obj array because the file format spec makes no guarantee
-         * about the order of the objects, and unused slots must be set to zero.
-         */
-        if(NULL == (heap->obj = H5FL_SEQ_CALLOC(H5HG_obj_t, nalloc)))
-            HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
-        heap->nalloc = nalloc;
-
-        while(image < (heap->chunk + heap->size)) {
-            if((image + H5HG_SIZEOF_OBJHDR(f)) > (heap->chunk + heap->size)) {
-                /*
-                 * The last bit of space is too tiny for an object header, so
-                 * we assume that it's free space.
-                 */
-                HDassert(NULL == heap->obj[0].begin);
-                heap->obj[0].size = (size_t)(((const uint8_t *)heap->chunk + heap->size) - image);
-                heap->obj[0].begin = image;
-                image += heap->obj[0].size;
+    /* Decode each object */
+    image = heap->chunk + H5HG_SIZEOF_HDR(f);
+    nalloc = H5HG_NOBJS(f, heap->size);
+
+    /* Calloc the obj array because the file format spec makes no guarantee
+     * about the order of the objects, and unused slots must be set to zero.
+     */
+    if(NULL == (heap->obj = H5FL_SEQ_CALLOC(H5HG_obj_t, nalloc)))
+        HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
+    heap->nalloc = nalloc;
+
+    while(image < (heap->chunk + heap->size)) {
+        if((image + H5HG_SIZEOF_OBJHDR(f)) > (heap->chunk + heap->size)) {
+            /*
+             * The last bit of space is too tiny for an object header, so
+             * we assume that it's free space.
+             */
+            HDassert(NULL == heap->obj[0].begin);
+            heap->obj[0].size = (size_t)(((const uint8_t *)heap->chunk + heap->size) - image);
+            heap->obj[0].begin = image;
+            image += heap->obj[0].size;
+        } /* end if */
+        else {
+            size_t need;
+            unsigned idx;
+            uint8_t *begin = image;
+
+            UINT16DECODE(image, idx);
+
+            /* Check if we need more room to store heap objects */
+            if(idx >= heap->nalloc) {
+                size_t new_alloc;       /* New allocation number */
+                H5HG_obj_t *new_obj;    /* New array of object   descriptions          */
+
+                /* Determine the new number of objects to index */
+                new_alloc = MAX(heap->nalloc * 2, (idx + 1));
+                HDassert(idx < new_alloc);
+
+                /* Reallocate array of objects */
+                if(NULL == (new_obj = H5FL_SEQ_REALLOC(H5HG_obj_t, heap->obj, new_alloc)))
+                    HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
+
+                /* Clear newly allocated space */
+                HDmemset(&new_obj[heap->nalloc], 0, (new_alloc - heap->nalloc) * sizeof(heap->obj[0]));
+
+                /* Update heap information */
+                heap->nalloc = new_alloc;
+                heap->obj = new_obj;
+                HDassert(heap->nalloc > heap->nused);
+            } /* end if */
+
+            UINT16DECODE(image, heap->obj[idx].nrefs);
+            image += 4; /*reserved*/
+            H5F_DECODE_LENGTH(f, image, heap->obj[idx].size);
+            heap->obj[idx].begin = begin;
+
+            /*
+             * The total storage size includes the size of the object 
+             * header and is zero padded so the next object header is 
+             * properly aligned. The entire obj array was calloc'ed, 
+             * so no need to zero the space here. The last bit of space 
+             * is the free space object whose size is never padded and 
+             * already includes the object header.
+             */
+            if(idx > 0) {
+                need = H5HG_SIZEOF_OBJHDR(f) + H5HG_ALIGN(heap->obj[idx].size);
+                if(idx > max_idx)
+                    max_idx = idx;
             } /* end if */
-            else {
-                size_t need;
-                unsigned idx;
-                uint8_t *begin = image;
-
-                UINT16DECODE(image, idx);
-
-                /* Check if we need more room to store heap objects */
-                if(idx >= heap->nalloc) {
-                    size_t new_alloc;       /* New allocation number */
-                    H5HG_obj_t *new_obj;    /* New array of object   descriptions          */
-
-                    /* Determine the new number of objects to index */
-                    new_alloc = MAX(heap->nalloc * 2, (idx + 1));
-                    HDassert(idx < new_alloc);
-
-                    /* Reallocate array of objects */
-                    if(NULL == (new_obj = H5FL_SEQ_REALLOC(H5HG_obj_t, heap->obj, new_alloc)))
-                        HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
-
-                    /* Clear newly allocated space */
-                    HDmemset(&new_obj[heap->nalloc], 0, (new_alloc - heap->nalloc) * sizeof(heap->obj[0]));
-
-                    /* Update heap information */
-                    heap->nalloc = new_alloc;
-                    heap->obj = new_obj;
-                    HDassert(heap->nalloc > heap->nused);
-                } /* end if */
-
-                UINT16DECODE(image, heap->obj[idx].nrefs);
-                image += 4; /*reserved*/
-                H5F_DECODE_LENGTH(f, image, heap->obj[idx].size);
-                heap->obj[idx].begin = begin;
-
-                /*
-                 * The total storage size includes the size of the object 
-                 * header and is zero padded so the next object header is 
-                 * properly aligned. The entire obj array was calloc'ed, 
-                 * so no need to zero the space here. The last bit of space 
-                 * is the free space object whose size is never padded and 
-                 * already includes the object header.
-                 */
-                if(idx > 0) {
-                    need = H5HG_SIZEOF_OBJHDR(f) + H5HG_ALIGN(heap->obj[idx].size);
-                    if(idx > max_idx)
-                        max_idx = idx;
-                } /* end if */
-                else
-                    need = heap->obj[idx].size;
-
-                image = begin + need;
-            } /* end else */
-        } /* end while */
-
-        HDassert(image == heap->chunk + heap->size);
-        HDassert(H5HG_ISALIGNED(heap->obj[0].size));
-
-        /* Set the next index value to use */
-        if(max_idx > 0)
-            heap->nused = max_idx + 1;
-        else
-            heap->nused = 1;
-
-        HDassert(max_idx < heap->nused);
-
-        /* Add the new heap to the CWFS list for the file */
-        if(H5F_cwfs_add(f, heap) < 0)
-            HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, NULL, "unable to add global heap collection to file's CWFS")
-    } /* end if ( len == heap->size ) */
+            else
+                need = heap->obj[idx].size;
+
+            image = begin + need;
+        } /* end else */
+    } /* end while */
+
+    /* Sanity checks */
+    HDassert(image == heap->chunk + heap->size);
+    HDassert(H5HG_ISALIGNED(heap->obj[0].size));
+
+    /* Set the next index value to use */
+    if(max_idx > 0)
+        heap->nused = max_idx + 1;
     else
-        /* if len is less than heap size, then the initial speculative 
-         * read was too small.  In this case we return without reporting
-         * failure.  H5C_load_entry() will call H5HG__cache_heap_image_len()
-         * to get the actual read size, and then repeat the read with the 
-         * correct size, and call this function a second time.
-         */
-        HDassert(len < heap->size);
+        heap->nused = 1;
+
+    /* Sanity check */
+    HDassert(max_idx < heap->nused);
+
+    /* Add the new heap to the CWFS list for the file */
+    if(H5F_cwfs_add(f, heap) < 0)
+        HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, NULL, "unable to add global heap collection to file's CWFS")
 
     ret_value = heap;
 
@@ -331,8 +378,7 @@ done:
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5HG__cache_heap_image_len(const void *_thing, size_t *image_len, 
-    hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr)
+H5HG__cache_heap_image_len(const void *_thing, size_t *image_len)
 {
     const H5HG_heap_t *heap = (const H5HG_heap_t *)_thing;
 
@@ -350,10 +396,6 @@ H5HG__cache_heap_image_len(const void *_thing, size_t *image_len,
     FUNC_LEAVE_NOAPI(SUCCEED)
 } /* end H5HG__cache_heap_image_len() */
 
-/**************************************/
-/* no H5HG_cache_heap_pre_serialize() */
-/**************************************/
-
 
 /*-------------------------------------------------------------------------
  * Function:    H5HG__cache_heap_serialize
@@ -393,10 +435,6 @@ H5HG__cache_heap_serialize(const H5F_t *f, void *image, size_t len,
     FUNC_LEAVE_NOAPI(SUCCEED)
 } /* end H5HG__cache_heap_serialize() */
 
-/****************************************/
-/* no H5HG_cache_heap_notify() function */
-/****************************************/
-
 
 /*-------------------------------------------------------------------------
  * Function:    H5HG__cache_heap_free_icr