Merging in latest from upstream (HDFFV/hdf5:refs/heads/develop)

* commit '6428a637752753cc8a9007c0c3c38b9ea9cc97fd': (165 commits)
  Skip HDFFV-10160 filenotclosed test for split/multi driver Temporary skip this test for split/multi driver. Further investigation to resolve the failure is needed. Please enter the commit message for your changes. Lines starting
  More minor whitespace cleanup
  Fixed a minor comment typo in H5R.c
  * Reverted removal of H5I_REFERENCE to avoid breaking binary   compatibility. * Added a quick H5I_REFERENCE smoke check to test_misc19()   in tmisc.c.
  Added missing free() call to test_misc19().
  Purged the unused H5I_REFERENCE from the library.
  Added a test for VFD IDs to testhdf5.
  Revert "Partial removal of NDEBUG ifdefs from the library."
  Partial removal of NDEBUG ifdefs from the library.
  Removed compiler flag cleanup since Solaris chokes on it.
  Fix for HDFFV-10160 Modifications to fix the assertion/abort failure when the application does not close the file.
  Fixed a bug in autotools Fortran processing where a test for the presence of a C long double type always returned true.
  Updated the --enable-debug/production handling in configure.ac so that it more reliably indicates that the configure option can no longer be used. Previously, using --enable-debug="", etc. would not trigger an error (even though such usage is unlikely).
  Added awk processing to remove extraneous whitespace from H5_ flags in configure.
  Fixed a comment in configure.ac
  HDFFV-10219 another mode change
  HDFFV-10219 change subset to use BINARY mode for h5dump
  HDFFV-10219 default input binary to NATIVE
  Set default of var to NATIVE
  HDFFV-10219 - disable h5dump tests until h5import NATIVE is fixed
  ...

1 files changed, 387 insertions, 74 deletions

diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c
index 7e1906b..e26581c 100644
--- a/src/H5Dchunk.c
+++ b/src/H5Dchunk.c
@@ -5,12 +5,10 @@
  *                                                                           *
  * This file is part of HDF5.  The full HDF5 copyright notice, including     *
  * terms governing use, modification, and redistribution, is contained in    *
- * the files COPYING and Copyright.html.  COPYING can be found at the root   *
- * of the source code distribution tree; Copyright.html can be found at the  *
- * root level of an installed copy of the electronic HDF5 document set and   *
- * is linked from the top-level documents page.  It can also be found at     *
- * http://hdfgroup.org/HDF5/doc/Copyright.html.  If you do not have          *
- * access to either file, you may request a copy from help@hdfgroup.org.     *
+ * the COPYING file, which can be found at the root of the source code       *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases.  *
+ * If you do not have access to either file, you may request a copy from     *
+ * help@hdfgroup.org.                                                        *
  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 /* Programmer: 	Quincey Koziol <koziol@hdfgroup.org>
@@ -190,6 +188,10 @@ typedef struct H5D_chunk_it_ud3_t {
 
     /* needed for copy object pointed by refs */
     H5O_copy_t          *cpy_info;              /* Copy options */
+
+    /* needed for getting raw data from chunk cache */
+    hbool_t             chunk_in_cache;
+    uint8_t	            *chunk;		                /* the unfiltered chunk data		*/
 } H5D_chunk_it_ud3_t;
 
 /* Callback info for iteration to dump index */
@@ -302,6 +304,9 @@ static herr_t H5D__chunk_collective_fill(const H5D_t *dset, hid_t dxpl_id,
     H5D_chunk_coll_info_t *chunk_info, size_t chunk_size, const void *fill_buf);
 #endif /* H5_HAVE_PARALLEL */
 
+static int
+H5D__chunk_dump_index_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata);
+
 /*********************/
 /* Package Variables */
 /*********************/
@@ -368,8 +373,7 @@ H5FL_EXTERN(H5S_sel_iter_t);
 /*-------------------------------------------------------------------------
  * Function:	H5D__chunk_direct_write
  *
- * Purpose:	Internal routine to write a chunk 
- *              directly into the file.
+ * Purpose:	Internal routine to write a chunk directly into the file.
  *
  * Return:	Non-negative on success/Negative on failure
  *
@@ -407,7 +411,7 @@ H5D__chunk_direct_write(const H5D_t *dset, hid_t dxpl_id, uint32_t filters,
 
     /* Allocate dataspace and initialize it if it hasn't been. */
     if(!(*layout->ops->is_space_alloc)(&layout->storage)) {
- 	/* Allocate storage */
+        /* Allocate storage */
         if(H5D__alloc_storage(&io_info, H5D_ALLOC_WRITE, FALSE, NULL) < 0)
             HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize storage")
     }
@@ -442,11 +446,17 @@ H5D__chunk_direct_write(const H5D_t *dset, hid_t dxpl_id, uint32_t filters,
     /* Set up the size of chunk for user data */
     udata.chunk_block.length = data_size;
 
-    /* Create the chunk it if it doesn't exist, or reallocate the chunk
-     *  if its size changed.
-     */
-    if(H5D__chunk_file_alloc(&idx_info, &old_chunk, &udata.chunk_block, &need_insert, scaled) < 0)
-        HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate chunk")
+    if (0 == idx_info.pline->nused && H5F_addr_defined(old_chunk.offset)) {
+        /* If there are no filters and we are overwriting the chunk we can just set values */
+        need_insert = FALSE;
+    }
+    else {
+        /* Otherwise, create the chunk it if it doesn't exist, or reallocate the chunk
+         * if its size has changed.
+         */
+        if (H5D__chunk_file_alloc(&idx_info, &old_chunk, &udata.chunk_block, &need_insert, scaled) < 0)
+            HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate chunk")
+    }
 
     /* Make sure the address of the chunk is returned. */
     if(!H5F_addr_defined(udata.chunk_block.offset))
@@ -464,7 +474,7 @@ H5D__chunk_direct_write(const H5D_t *dset, hid_t dxpl_id, uint32_t filters,
             HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't fill dxpl cache")
 
         if(H5D__chunk_cache_evict(dset, io_info.md_dxpl_id, dxpl_cache, rdcc->slot[udata.idx_hint], FALSE) < 0)
-	    HGOTO_ERROR(H5E_DATASET, H5E_CANTREMOVE, FAIL, "unable to evict chunk")
+            HGOTO_ERROR(H5E_DATASET, H5E_CANTREMOVE, FAIL, "unable to evict chunk")
     } /* end if */
 
     /* Write the data to the file */
@@ -490,6 +500,246 @@ done:
 
 
 /*-------------------------------------------------------------------------
+ * Function:    H5D__chunk_direct_read
+ *
+ * Purpose:     Internal routine to read a chunk directly from the file.
+ *
+ * Return:      Non-negative on success/Negative on failure
+ *
+ * Programmer:  Matthew Strong (GE Healthcare)
+ *              14 February 2016
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5D__chunk_direct_read(const H5D_t *dset, hid_t dxpl_id, hsize_t *offset,
+    uint32_t* filters, void *buf)
+{
+    const H5O_layout_t *layout = &(dset->shared->layout);    /* Dataset layout */
+    const H5D_rdcc_t   *rdcc = &(dset->shared->cache.chunk); /* raw data chunk cache */
+    H5D_chunk_ud_t     udata;          /* User data for querying chunk info */
+    hsize_t scaled[H5S_MAX_RANK];       /* Scaled coordinates for this chunk */
+    H5D_io_info_t io_info;              /* to hold the dset and two dxpls (meta and raw data) */
+    H5D_dxpl_cache_t   _dxpl_cache;     /* Data transfer property cache buffer */
+    H5D_dxpl_cache_t   *dxpl_cache = &_dxpl_cache; /* Data transfer property cache */
+    hbool_t md_dxpl_generated = FALSE;  /* bool to indicate whether we should free the md_dxpl_id at exit */
+    herr_t             ret_value = SUCCEED;        /* Return value */
+
+    FUNC_ENTER_STATIC_TAG(dxpl_id, dset->oloc.addr, FAIL)
+
+    /* Check args */
+    HDassert(dset && H5D_CHUNKED == layout->type);
+    HDassert(offset);
+    HDassert(filters);
+    HDassert(buf);
+
+    *filters = 0;
+
+    io_info.dset = dset;
+    io_info.raw_dxpl_id = dxpl_id;
+    io_info.md_dxpl_id = dxpl_id;
+
+    /* set the dxpl IO type for sanity checking at the FD layer */
+#ifdef H5_DEBUG_BUILD
+    if(H5D_set_io_info_dxpls(&io_info, dxpl_id) < 0)
+        HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "can't set metadata and raw data dxpls")
+    md_dxpl_generated = TRUE;
+#endif /* H5_DEBUG_BUILD */
+
+    /* Allocate dataspace and initialize it if it hasn't been. */
+    if(!(*layout->ops->is_space_alloc)(&layout->storage))
+        HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "storage is not initialized")
+
+    /* Calculate the index of this chunk */
+    H5VM_chunk_scaled(dset->shared->ndims, offset, layout->u.chunk.dim, scaled);
+    scaled[dset->shared->ndims] = 0;
+
+    /* Reset fields about the chunk we are looking for */
+    udata.filter_mask = 0;
+    udata.chunk_block.offset = HADDR_UNDEF;
+    udata.chunk_block.length = 0;
+    udata.idx_hint = UINT_MAX;
+
+    /* Find out the file address of the chunk */
+    if(H5D__chunk_lookup(dset, io_info.md_dxpl_id, scaled, &udata) < 0)
+        HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error looking up chunk address")
+
+    /* Sanity check */
+    HDassert((H5F_addr_defined(udata.chunk_block.offset) && udata.chunk_block.length > 0) ||
+            (!H5F_addr_defined(udata.chunk_block.offset) && udata.chunk_block.length == 0));
+
+    /* Check if the requested chunk exists in the chunk cache */
+    if(UINT_MAX != udata.idx_hint) {
+        H5D_rdcc_ent_t *ent = rdcc->slot[udata.idx_hint];
+        hbool_t flush;
+
+        /* Sanity checks  */
+        HDassert(udata.idx_hint < rdcc->nslots);
+        HDassert(rdcc->slot[udata.idx_hint]);
+
+        flush = (ent->dirty == TRUE) ? TRUE : FALSE;
+
+        /* Fill the DXPL cache values for later use */
+        if(H5D__get_dxpl_cache(io_info.raw_dxpl_id, &dxpl_cache) < 0)
+            HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't fill dxpl cache")
+
+        /* Flush the chunk to disk and clear the cache entry */
+        if(H5D__chunk_cache_evict(dset, io_info.md_dxpl_id, dxpl_cache, rdcc->slot[udata.idx_hint], flush) < 0)
+            HGOTO_ERROR(H5E_DATASET, H5E_CANTREMOVE, FAIL, "unable to evict chunk")
+
+        /* Reset fields about the chunk we are looking for */
+        udata.filter_mask = 0;
+        udata.chunk_block.offset = HADDR_UNDEF;
+        udata.chunk_block.length = 0;
+        udata.idx_hint = UINT_MAX;
+
+        /* Get the new file address / chunk size after flushing */
+        if(H5D__chunk_lookup(dset, io_info.md_dxpl_id, scaled, &udata) < 0)
+            HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error looking up chunk address")
+    }
+
+    /* Make sure the address of the chunk is returned. */
+    if(!H5F_addr_defined(udata.chunk_block.offset))
+        HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "chunk address isn't defined")
+
+    /* Read the chunk data into the supplied buffer */
+    if(H5F_block_read(dset->oloc.file, H5FD_MEM_DRAW, udata.chunk_block.offset, udata.chunk_block.length, io_info.raw_dxpl_id, buf) < 0)
+        HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "unable to read raw data chunk")
+    
+    /* Return the filter mask */
+    *filters = udata.filter_mask;
+
+done:
+#ifdef H5_DEBUG_BUILD
+    if(md_dxpl_generated && H5I_dec_ref(io_info.md_dxpl_id) < 0)
+        HDONE_ERROR(H5E_DATASET, H5E_CANTDEC, FAIL, "can't close metadata dxpl")
+#endif /* H5_DEBUG_BUILD */
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__chunk_direct_read() */
+
+
+/*-------------------------------------------------------------------------
+ * Function:    H5D__get_chunk_storage_size
+ *
+ * Purpose:     Internal routine to read the storage size of a chunk on disk.
+ *
+ * Return:      Non-negative on success/Negative on failure
+ *
+ * Programmer:  Matthew Strong (GE Healthcare)
+ *              20 October 2016
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5D__get_chunk_storage_size(H5D_t *dset, hid_t dxpl_id, const hsize_t *offset, hsize_t *storage_size)
+{
+    const H5O_layout_t *layout = &(dset->shared->layout);    /* Dataset layout */
+    const H5D_rdcc_t   *rdcc = &(dset->shared->cache.chunk); /* raw data chunk cache */
+    hsize_t scaled[H5S_MAX_RANK];       /* Scaled coordinates for this chunk */
+    H5D_io_info_t io_info;              /* to hold the dset and two dxpls (meta and raw data) */
+    H5D_chunk_ud_t     udata;           /* User data for querying chunk info */
+    H5D_dxpl_cache_t   _dxpl_cache;     /* Data transfer property cache buffer */
+    H5D_dxpl_cache_t   *dxpl_cache = &_dxpl_cache; /* Data transfer property cache */
+    hbool_t md_dxpl_generated = FALSE;  /* bool to indicate whether we should free the md_dxpl_id at exit */
+    herr_t             ret_value = SUCCEED;        /* Return value */
+
+    FUNC_ENTER_STATIC_TAG(dxpl_id, dset->oloc.addr, FAIL)
+
+    /* Check args */
+    HDassert(dset && H5D_CHUNKED == layout->type);
+    HDassert(offset);
+    HDassert(storage_size);
+
+    *storage_size = 0;
+
+    io_info.dset = dset;
+    io_info.raw_dxpl_id = dxpl_id;
+    io_info.md_dxpl_id = dxpl_id;
+
+    /* set the dxpl IO type for sanity checking at the FD layer */
+#ifdef H5_DEBUG_BUILD
+    if(H5D_set_io_info_dxpls(&io_info, dxpl_id) < 0)
+        HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "can't set metadata and raw data dxpls")
+    md_dxpl_generated = TRUE;
+#endif /* H5_DEBUG_BUILD */
+
+    /* Allocate dataspace and initialize it if it hasn't been. */
+    if(!(*layout->ops->is_space_alloc)(&layout->storage))
+        HGOTO_DONE(SUCCEED)
+
+    /* Calculate the index of this chunk */
+    H5VM_chunk_scaled(dset->shared->ndims, offset, layout->u.chunk.dim, scaled);
+    scaled[dset->shared->ndims] = 0;
+
+    /* Reset fields about the chunk we are looking for */
+    udata.chunk_block.offset = HADDR_UNDEF;
+    udata.chunk_block.length = 0;
+    udata.idx_hint = UINT_MAX;
+
+    /* Find out the file address of the chunk */
+    if(H5D__chunk_lookup(dset, io_info.md_dxpl_id, scaled, &udata) < 0)
+        HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error looking up chunk address")
+
+    /* Sanity check */
+    HDassert((H5F_addr_defined(udata.chunk_block.offset) && udata.chunk_block.length > 0) ||
+            (!H5F_addr_defined(udata.chunk_block.offset) && udata.chunk_block.length == 0));
+
+    /* The requested chunk is not in cache or on disk */
+    if(!H5F_addr_defined(udata.chunk_block.offset) && UINT_MAX == udata.idx_hint)
+        HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "chunk storage is not allocated")
+
+    /* Check if there are filters registered to the dataset */
+    if( dset->shared->dcpl_cache.pline.nused > 0 ) {
+        /* Check if the requested chunk exists in the chunk cache */
+        if(UINT_MAX != udata.idx_hint) {
+            H5D_rdcc_ent_t *ent = rdcc->slot[udata.idx_hint];
+
+            /* Sanity checks  */
+            HDassert(udata.idx_hint < rdcc->nslots);
+            HDassert(rdcc->slot[udata.idx_hint]);
+
+            /* If the cached chunk is dirty, it must be flushed to get accurate size */
+            if( ent->dirty == TRUE ) {
+                /* Fill the DXPL cache values for later use */
+                if(H5D__get_dxpl_cache(dxpl_id, &dxpl_cache) < 0)
+                    HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't fill dxpl cache")
+
+                /* Flush the chunk to disk and clear the cache entry */
+                if(H5D__chunk_cache_evict(dset, io_info.md_dxpl_id, dxpl_cache, rdcc->slot[udata.idx_hint], TRUE) < 0)
+                    HGOTO_ERROR(H5E_DATASET, H5E_CANTREMOVE, FAIL, "unable to evict chunk")
+
+                /* Reset fields about the chunk we are looking for */
+                udata.chunk_block.offset = HADDR_UNDEF;
+                udata.chunk_block.length = 0;
+                udata.idx_hint = UINT_MAX;
+
+                /* Get the new file address / chunk size after flushing */
+                if(H5D__chunk_lookup(dset, io_info.md_dxpl_id, scaled, &udata) < 0)
+                    HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error looking up chunk address")
+            }
+        }
+
+        /* Make sure the address of the chunk is returned. */
+        if(!H5F_addr_defined(udata.chunk_block.offset))
+            HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "chunk address isn't defined")
+
+        /* Return the chunk size on disk */
+        *storage_size = udata.chunk_block.length;
+    }
+    /* There are no filters registered, return the chunk size from the storage layout */
+    else
+        *storage_size = dset->shared->layout.u.chunk.size;
+
+done:
+#ifdef H5_DEBUG_BUILD
+    if(md_dxpl_generated && H5I_dec_ref(io_info.md_dxpl_id) < 0)
+        HDONE_ERROR(H5E_DATASET, H5E_CANTDEC, FAIL, "can't close metadata dxpl")
+#endif /* H5_DEBUG_BUILD */
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* H5D__get_chunk_storage_size */
+
+
+/*-------------------------------------------------------------------------
  * Function:	H5D__chunk_set_info_real
  *
  * Purpose:	Internal routine to set the information about chunks for a dataset
@@ -518,22 +768,22 @@ H5D__chunk_set_info_real(H5O_layout_chunk_t *layout, unsigned ndims,
     /* Compute the # of chunks in dataset dimensions */
     for(u = 0, layout->nchunks = 1, layout->max_nchunks = 1; u < ndims; u++) {
         /* Round up to the next integer # of chunks, to accomodate partial chunks */
-	layout->chunks[u] = ((curr_dims[u] + layout->dim[u]) - 1) / layout->dim[u];
+        layout->chunks[u] = ((curr_dims[u] + layout->dim[u]) - 1) / layout->dim[u];
         if(H5S_UNLIMITED == max_dims[u])
             layout->max_chunks[u] = H5S_UNLIMITED;
         else
             layout->max_chunks[u] = ((max_dims[u] + layout->dim[u]) - 1) / layout->dim[u];
 
         /* Accumulate the # of chunks */
-	layout->nchunks *= layout->chunks[u];
-	layout->max_nchunks *= layout->max_chunks[u];
+        layout->nchunks *= layout->chunks[u];
+        layout->max_nchunks *= layout->max_chunks[u];
     } /* end for */
 
     /* Get the "down" sizes for each dimension */
     if(H5VM_array_down(ndims, layout->chunks, layout->down_chunks) < 0)
         HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't compute 'down' chunk size value")
     if(H5VM_array_down(ndims, layout->max_chunks, layout->max_down_chunks) < 0)
-	HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't compute 'down' chunk size value")
+        HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't compute 'down' chunk size value")
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
@@ -2031,8 +2281,8 @@ H5D__chunk_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
             void *chunk = NULL;             /* Pointer to locked chunk buffer */
             htri_t cacheable;               /* Whether the chunk is cacheable */
 
-	    /* Set chunk's [scaled] coordinates */
-	    io_info->store->chunk.scaled = chunk_info->scaled;
+            /* Set chunk's [scaled] coordinates */
+            io_info->store->chunk.scaled = chunk_info->scaled;
 
             /* Determine if we should use the chunk cache */
             if((cacheable = H5D__chunk_cacheable(io_info, udata.chunk_block.offset, FALSE)) < 0)
@@ -2382,14 +2632,14 @@ H5D__chunk_dest(H5D_t *dset, hid_t dxpl_id)
 
     /* Flush all the cached chunks */
     for(ent = rdcc->head; ent; ent = next) {
-	next = ent->next;
-	if(H5D__chunk_cache_evict(dset, dxpl_id, dxpl_cache, ent, TRUE) < 0)
-	    nerrors++;
+        next = ent->next;
+        if(H5D__chunk_cache_evict(dset, dxpl_id, dxpl_cache, ent, TRUE) < 0)
+            nerrors++;
     } /* end for */
     
     /* Continue even if there are failures. */
     if(nerrors)
-	HDONE_ERROR(H5E_IO, H5E_CANTFLUSH, FAIL, "unable to flush one or more raw data chunks")
+        HDONE_ERROR(H5E_IO, H5E_CANTFLUSH, FAIL, "unable to flush one or more raw data chunks")
 
     /* Release cache structures */
     if(rdcc->slot)
@@ -2547,7 +2797,7 @@ H5D__chunk_cinfo_cache_found(const H5D_chunk_cached_t *last, H5D_chunk_ud_t *uda
         /* Retrieve the information from the cache */
         udata->chunk_block.offset = last->addr;
         udata->chunk_block.length = last->nbytes;
-	udata->chunk_idx = last->chunk_idx;
+        udata->chunk_idx = last->chunk_idx;
         udata->filter_mask = last->filter_mask;
 
         /* Indicate that the data was found */
@@ -2707,7 +2957,7 @@ H5D__chunk_lookup(const H5D_t *dset, hid_t dxpl_id, const hsize_t *scaled,
     /* Check for chunk in cache */
     if(dset->shared->cache.chunk.nslots > 0) {
         /* Determine the chunk's location in the hash table */
-	idx = H5D__chunk_hash_val(dset->shared, scaled);
+        idx = H5D__chunk_hash_val(dset->shared, scaled);
 
         /* Get the chunk cache entry for that location */
         ent = dset->shared->cache.chunk.slot[idx];
@@ -2731,7 +2981,7 @@ H5D__chunk_lookup(const H5D_t *dset, hid_t dxpl_id, const hsize_t *scaled,
         udata->idx_hint = idx;
         udata->chunk_block.offset = ent->chunk_block.offset;
         udata->chunk_block.length = ent->chunk_block.length;;
-	udata->chunk_idx = ent->chunk_idx;
+        udata->chunk_idx = ent->chunk_idx;
     } /* end if */
     else {
         /* Invalidate idx_hint, to signal that the chunk is not in cache */
@@ -2816,19 +3066,19 @@ H5D__chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t
 
     buf = ent->chunk;
     if(ent->dirty) {
-	H5D_chk_idx_info_t idx_info;    /* Chunked index info */
+        H5D_chk_idx_info_t idx_info;    /* Chunked index info */
         H5D_chunk_ud_t 	udata;		/* pass through B-tree		*/
         hbool_t must_alloc = FALSE;     /* Whether the chunk must be allocated */
-	hbool_t need_insert = FALSE;    /* Whether the chunk needs to be inserted into the index */
+        hbool_t need_insert = FALSE;    /* Whether the chunk needs to be inserted into the index */
 
         /* Set up user data for index callbacks */
         udata.common.layout = &dset->shared->layout.u.chunk;
         udata.common.storage = &dset->shared->layout.storage.u.chunk;
-	udata.common.scaled = ent->scaled;
+        udata.common.scaled = ent->scaled;
         udata.chunk_block.offset = ent->chunk_block.offset;
         udata.chunk_block.length = dset->shared->layout.u.chunk.size;
         udata.filter_mask = 0;
-	udata.chunk_idx = ent->chunk_idx;
+        udata.chunk_idx = ent->chunk_idx;
 
         /* Should the chunk be filtered before writing it to disk? */
         if(dset->shared->dcpl_cache.pline.nused
@@ -2906,8 +3156,8 @@ H5D__chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t
             /* Create the chunk it if it doesn't exist, or reallocate the chunk
              *  if its size changed.
              */
-	    if(H5D__chunk_file_alloc(&idx_info, &(ent->chunk_block), &udata.chunk_block, &need_insert, ent->scaled) < 0)
-		HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert/resize chunk on chunk level")
+            if(H5D__chunk_file_alloc(&idx_info, &(ent->chunk_block), &udata.chunk_block, &need_insert, ent->scaled) < 0)
+                HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert/resize chunk on chunk level")
 
             /* Update the chunk entry's info, in case it was allocated or relocated */
             ent->chunk_block.offset = udata.chunk_block.offset;
@@ -2921,7 +3171,7 @@ H5D__chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t
             HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to write raw data to file")
 
         /* Insert the chunk record into the index */
-	if(need_insert && dset->shared->layout.storage.u.chunk.ops->insert)
+        if(need_insert && dset->shared->layout.storage.u.chunk.ops->insert)
             if((dset->shared->layout.storage.u.chunk.ops->insert)(&idx_info, &udata, dset) < 0)
                 HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert chunk addr into index")
 
@@ -2996,27 +3246,27 @@ H5D__chunk_cache_evict(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t
     HDassert(ent->idx < rdcc->nslots);
 
     if(flush) {
-	/* Flush */
-	if(H5D__chunk_flush_entry(dset, dxpl_id, dxpl_cache, ent, TRUE) < 0)
-	    HDONE_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot flush indexed storage buffer")
+        /* Flush */
+        if(H5D__chunk_flush_entry(dset, dxpl_id, dxpl_cache, ent, TRUE) < 0)
+            HDONE_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot flush indexed storage buffer")
     } /* end if */
     else {
         /* Don't flush, just free chunk */
-	if(ent->chunk != NULL)
-	    ent->chunk = (uint8_t *)H5D__chunk_mem_xfree(ent->chunk,
+        if(ent->chunk != NULL)
+            ent->chunk = (uint8_t *)H5D__chunk_mem_xfree(ent->chunk,
                     ((ent->edge_chunk_state & H5D_RDCC_DISABLE_FILTERS) ? NULL
                     : &(dset->shared->dcpl_cache.pline)));
     } /* end else */
 
     /* Unlink from list */
     if(ent->prev)
-	ent->prev->next = ent->next;
+        ent->prev->next = ent->next;
     else
-	rdcc->head = ent->next;
+        rdcc->head = ent->next;
     if(ent->next)
-	ent->next->prev = ent->prev;
+        ent->next->prev = ent->prev;
     else
-	rdcc->tail = ent->prev;
+        rdcc->tail = ent->prev;
     ent->prev = ent->next = NULL;
 
     /* Unlink from temporary list */
@@ -3395,7 +3645,7 @@ H5D__chunk_lock(const H5D_io_info_t *io_info, H5D_chunk_ud_t *udata,
                 if(H5F_block_read(dset->oloc.file, H5FD_MEM_DRAW, chunk_addr, my_chunk_alloc, io_info->raw_dxpl_id, chunk) < 0)
                     HGOTO_ERROR(H5E_IO, H5E_READERROR, NULL, "unable to read raw data chunk")
 
-		if(old_pline && old_pline->nused) {
+                if(old_pline && old_pline->nused) {
                     if(H5Z_pipeline(old_pline, H5Z_FLAG_REVERSE,
                             &(udata->filter_mask),
                             io_info->dxpl_cache->err_detect,
@@ -3484,14 +3734,14 @@ H5D__chunk_lock(const H5D_io_info_t *io_info, H5D_chunk_ud_t *udata,
                 if(NULL == (ent = H5FL_CALLOC(H5D_rdcc_ent_t)))
                     HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, NULL, "can't allocate raw data chunk entry")
 
-		ent->edge_chunk_state = disable_filters ? H5D_RDCC_DISABLE_FILTERS : 0;
-		if(udata->new_unfilt_chunk)
-		    ent->edge_chunk_state |= H5D_RDCC_NEWLY_DISABLED_FILTERS;
+                ent->edge_chunk_state = disable_filters ? H5D_RDCC_DISABLE_FILTERS : 0;
+                if(udata->new_unfilt_chunk)
+                    ent->edge_chunk_state |= H5D_RDCC_NEWLY_DISABLED_FILTERS;
 
                 /* Initialize the new entry */
                 ent->chunk_block.offset = chunk_addr;
                 ent->chunk_block.length = chunk_alloc;
-		ent->chunk_idx = udata->chunk_idx;
+                ent->chunk_idx = udata->chunk_idx;
                 HDmemcpy(ent->scaled, udata->common.scaled, sizeof(hsize_t) * layout->u.chunk.ndims);
                 H5_CHECKED_ASSIGN(ent->rd_count, uint32_t, chunk_size, size_t);
                 H5_CHECKED_ASSIGN(ent->wr_count, uint32_t, chunk_size, size_t);
@@ -3512,8 +3762,8 @@ H5D__chunk_lock(const H5D_io_info_t *io_info, H5D_chunk_ud_t *udata,
                 } /* end if */
                 else
                     rdcc->head = rdcc->tail = ent;
-		ent->tmp_next = NULL;
-		ent->tmp_prev = NULL;
+                ent->tmp_next = NULL;
+                ent->tmp_prev = NULL;
 
             } /* end if */
             else
@@ -5409,11 +5659,22 @@ H5D__chunk_copy_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata)
     H5Z_cb_t                cb_struct;                  /* Filter failure callback struct */
     int                     ret_value = H5_ITER_CONT;   /* Return value */
 
+
     FUNC_ENTER_STATIC
 
     /* Get 'size_t' local value for number of bytes in chunk */
     H5_CHECKED_ASSIGN(nbytes, size_t, chunk_rec->nbytes, uint32_t);
 
+    /* Check for filtered chunks */
+    /* Check for an edge chunk that is not filtered */
+    if(pline && pline->nused) {
+        must_filter = TRUE;
+        if( (udata->common.layout->flags & H5O_LAYOUT_CHUNK_DONT_FILTER_PARTIAL_BOUND_CHUNKS) &&
+            (H5D__chunk_is_partial_edge_chunk(udata->dset_ndims, udata->common.layout->dim, 
+                                              chunk_rec->scaled, udata->dset_dims)) )
+            must_filter = FALSE;
+    }
+
     /* Check parameter for type conversion */
     if(udata->do_convert) {
         if(H5T_detect_class(udata->dt_src, H5T_VLEN, FALSE) > 0)
@@ -5424,19 +5685,6 @@ H5D__chunk_copy_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata)
             HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, H5_ITER_ERROR, "unable to copy dataset elements")
     } /* end if */
 
-    /* Check for filtered chunks */
-    if((is_vlen || fix_ref) && pline && pline->nused) {
-        /* Check if we should disable filters on this chunk */
-        if(udata->common.layout->flags
-                & H5O_LAYOUT_CHUNK_DONT_FILTER_PARTIAL_BOUND_CHUNKS) {
-            /* Check if the chunk is an edge chunk, and disable filters if so */
-            if(!H5D__chunk_is_partial_edge_chunk(udata->dset_ndims, udata->common.layout->dim, chunk_rec->scaled, udata->dset_dims))
-                must_filter = TRUE;
-        } /* end if */
-        else
-            must_filter = TRUE;
-    } /* end if */
-
     /* Resize the buf if it is too small to hold the data */
     if(nbytes > buf_size) {
         void *new_buf;          /* New buffer for data */
@@ -5459,12 +5707,51 @@ H5D__chunk_copy_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata)
         udata->buf_size = buf_size = nbytes;
     } /* end if */
 
-    /* read chunk data from the source file */
-    if(H5F_block_read(udata->file_src, H5FD_MEM_DRAW, chunk_rec->chunk_addr, nbytes, H5AC_rawdata_dxpl_id, buf) < 0)
-        HGOTO_ERROR(H5E_IO, H5E_READERROR, H5_ITER_ERROR, "unable to read raw data chunk")
+    if(udata->chunk_in_cache && udata->chunk) {
+        HDassert(!H5F_addr_defined(chunk_rec->chunk_addr));
+        HDmemcpy(buf, udata->chunk, nbytes);
+        udata->chunk = NULL;
+    } else {
+        H5D_rdcc_ent_t *ent = NULL;    /* Cache entry */
+        unsigned idx;                   /* Index of chunk in cache, if present */
+        unsigned u;                     /* Counter */
+        H5D_shared_t *shared_fo = udata->cpy_info->shared_fo;
+
+        /* See if the written chunk is in the chunk cache */
+        if(shared_fo && shared_fo->cache.chunk.nslots > 0) {
+            /* Determine the chunk's location in the hash table */
+            idx = H5D__chunk_hash_val(shared_fo, chunk_rec->scaled);
+
+            /* Get the chunk cache entry for that location */
+            ent = shared_fo->cache.chunk.slot[idx];
+            if(ent) {
+                /* Speculatively set the 'found' flag */
+                udata->chunk_in_cache = TRUE;
+
+                /* Verify that the cache entry is the correct chunk */
+                for(u = 0; u < shared_fo->ndims; u++)
+                    if(chunk_rec->scaled[u] != ent->scaled[u]) {
+                        udata->chunk_in_cache = FALSE;
+                        break;
+                    } /* end if */
+            } /* end if */
+        } /* end if */
+
+        if(udata->chunk_in_cache) {
+            HDassert(H5F_addr_defined(chunk_rec->chunk_addr));
+            HDassert(H5F_addr_defined(ent->chunk_block.offset));
+
+            H5_CHECKED_ASSIGN(nbytes, size_t, shared_fo->layout.u.chunk.size, uint32_t);
+            HDmemcpy(buf, ent->chunk, nbytes);
+        } else {
+            /* read chunk data from the source file */
+            if(H5F_block_read(udata->file_src, H5FD_MEM_DRAW, chunk_rec->chunk_addr, nbytes, H5AC_rawdata_dxpl_id, buf) < 0)
+                HGOTO_ERROR(H5E_IO, H5E_READERROR, H5_ITER_ERROR, "unable to read raw data chunk")
+        }
+    }
 
-    /* Need to uncompress variable-length & reference data elements */
-    if(must_filter) {
+    /* Need to uncompress filtered variable-length & reference data elements that are not found in chunk cache */
+    if(must_filter && (is_vlen || fix_ref) && !udata->chunk_in_cache) {
         unsigned filter_mask = chunk_rec->filter_mask;
 
         cb_struct.func = NULL; /* no callback function when failed */
@@ -5528,8 +5815,8 @@ H5D__chunk_copy_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata)
     udata_dst.chunk_block.length = chunk_rec->nbytes;
     udata_dst.filter_mask = chunk_rec->filter_mask;
 
-    /* Need to compress variable-length & reference data elements before writing to file */
-    if(must_filter) {
+    /* Need to compress variable-length or reference data elements or a chunk found in cache before writing to file */
+    if(must_filter && (is_vlen || fix_ref || udata->chunk_in_cache) ) {
         if(H5Z_pipeline(pline, 0, &(udata_dst.filter_mask), H5Z_NO_EDC, cb_struct, &nbytes, &buf_size, &buf) < 0)
             HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, H5_ITER_ERROR, "output pipeline failed")
 #if H5_SIZEOF_SIZE_T > 4
@@ -5538,12 +5825,14 @@ H5D__chunk_copy_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata)
             HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, H5_ITER_ERROR, "chunk too large for 32-bit length")
 #endif /* H5_SIZEOF_SIZE_T > 4 */
         H5_CHECKED_ASSIGN(udata_dst.chunk_block.length, uint32_t, nbytes, size_t);
-	udata->buf = buf;
-	udata->buf_size = buf_size;
+        udata->buf = buf;
+        udata->buf_size = buf_size;
     } /* end if */
 
+    udata->chunk_in_cache = FALSE;
+
     udata_dst.chunk_idx = H5VM_array_offset_pre(udata_dst.common.layout->ndims - 1, 
-			    udata_dst.common.layout->max_down_chunks, udata_dst.common.scaled);
+	udata_dst.common.layout->max_down_chunks, udata_dst.common.scaled);
 
     /* Allocate chunk in the file */
     if(H5D__chunk_file_alloc(udata->idx_info_dst, NULL, &udata_dst.chunk_block, &need_insert, udata_dst.common.scaled) < 0)
@@ -5798,11 +6087,35 @@ H5D__chunk_copy(H5F_t *f_src, H5O_storage_chunk_t *storage_src,
     udata.dset_ndims = (unsigned)sndims;
     udata.dset_dims = curr_dims;
     udata.cpy_info = cpy_info;
+    udata.chunk_in_cache = FALSE;
+    udata.chunk = NULL;
 
     /* Iterate over chunks to copy data */
     if((storage_src->ops->iterate)(&idx_info_src, H5D__chunk_copy_cb, &udata) < 0)
         HGOTO_ERROR(H5E_DATASET, H5E_BADITER, FAIL, "unable to iterate over chunk index to copy data")
 
+    /* Iterate over the chunk cache to copy data for chunks with undefined address */
+    if(udata.cpy_info->shared_fo) {
+        H5D_rdcc_ent_t  *ent, *next;
+        H5D_chunk_rec_t chunk_rec;
+        H5D_shared_t *shared_fo = (H5D_shared_t *)udata.cpy_info->shared_fo;
+
+        chunk_rec.nbytes = layout_src->size;
+        chunk_rec.filter_mask = 0;
+        chunk_rec.chunk_addr = HADDR_UNDEF;
+
+        for(ent = shared_fo->cache.chunk.head; ent; ent = next) {
+            if(!H5F_addr_defined(ent->chunk_block.offset)) {
+                HDmemcpy(chunk_rec.scaled, ent->scaled, sizeof(chunk_rec.scaled));
+                udata.chunk = ent->chunk;
+                udata.chunk_in_cache = TRUE;
+                if(H5D__chunk_copy_cb(&chunk_rec, &udata) < 0)
+                    HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy chunk data in cache")
+            }
+            next = ent->next;
+        } /* end for */
+    }
+
     /* I/O buffers may have been re-allocated */
     buf = udata.buf;
     bkg = udata.bkg;