Fix for HDFFV-7853 H5Ocopy to 1.8 branch

Apply the fix for HDFFV-7853 H5Ocopy from develop to 1.8 branch.
This is the similar set of changes as in develop branch.
Some coding is added to do the copying because 1.8 does not flush out the
storage address when writing to a dataset.

6 files changed, 139 insertions, 36 deletions

diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c
index fcc752c..021335f 100644
--- a/src/H5Dchunk.c
+++ b/src/H5Dchunk.c
@@ -150,6 +150,10 @@ typedef struct H5D_chunk_it_ud3_t {
 
     /* needed for copy object pointed by refs */
     H5O_copy_t          *cpy_info;              /* Copy options */
+
+    /* needed for getting raw data from chunk cache */
+    hbool_t             chunk_in_cache;
+    uint8_t             *chunk;                 /* the unfiltered chunk data        */
 } H5D_chunk_it_ud3_t;
 
 /* Callback info for iteration to dump index */
@@ -324,7 +328,6 @@ H5D__chunk_direct_write(const H5D_t *dset, hid_t dxpl_id, uint32_t filters, hsiz
     H5D_dxpl_cache_t *dxpl_cache = &_dxpl_cache;   /* Data transfer property cache */
     const H5D_rdcc_t       *rdcc = &(dset->shared->cache.chunk);    /*raw data chunk cache */
     unsigned    space_ndims;            /* Dataset's space rank */
-    hsize_t     space_dim[H5O_LAYOUT_NDIMS];    /* Dataset's dataspace dimensions */
     herr_t ret_value = SUCCEED;         /* Return value */
 
     FUNC_ENTER_STATIC
@@ -414,7 +417,6 @@ H5D__chunk_direct_read(const H5D_t *dset, hid_t dxpl_id, hsize_t *offset,
     const H5D_rdcc_t   *rdcc = &(dset->shared->cache.chunk); /* raw data chunk cache */
     H5D_chunk_ud_t     udata;          /* User data for querying chunk info */
     unsigned            space_ndims;    /* Dataset's space rank */
-    hsize_t            space_dim[H5O_LAYOUT_NDIMS]; /* Dataset's dataspace dimensions */
     hsize_t            chunk_offset[H5O_LAYOUT_NDIMS]; /* Dataset's chunk offset */
     hsize_t            chunk_idx;      /* Index of chunk cache entry */
     H5D_dxpl_cache_t   _dxpl_cache;    /* Data transfer property cache buffer */
@@ -521,7 +523,6 @@ H5D__get_chunk_storage_size(H5D_t *dset, hid_t dxpl_id, const hsize_t *offset, h
     const H5D_rdcc_t   *rdcc = &(dset->shared->cache.chunk); /* raw data chunk cache */
     H5D_chunk_ud_t     udata;          /* User data for querying chunk info */
     unsigned           space_ndims;    /* Dataset's space rank */
-    hsize_t            space_dim[H5O_LAYOUT_NDIMS]; /* Dataset's dataspace dimensions */
     hsize_t            chunk_offset[H5O_LAYOUT_NDIMS]; /* Dataset's chunk offset */
     hsize_t            chunk_idx;      /* Index of chunk cache entry */
     H5D_dxpl_cache_t   _dxpl_cache;    /* Data transfer property cache buffer */
@@ -4836,12 +4837,57 @@ H5D__chunk_copy_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata)
         udata->buf_size = buf_size = nbytes;
     } /* end if */
 
-    /* read chunk data from the source file */
-    if(H5F_block_read(udata->file_src, H5FD_MEM_DRAW, chunk_rec->chunk_addr, nbytes, udata->idx_info_dst->dxpl_id, buf) < 0)
-        HGOTO_ERROR(H5E_IO, H5E_READERROR, H5_ITER_ERROR, "unable to read raw data chunk")
+    if(udata->chunk_in_cache && udata->chunk) {
+        HDassert(!H5F_addr_defined(chunk_rec->chunk_addr));
+        HDmemcpy(buf, udata->chunk, nbytes);
+        udata->chunk = NULL;
+    } else {
+        H5D_rdcc_ent_t *ent = NULL;    /* Cache entry */
+        unsigned idx;                   /* Index of chunk in cache, if present */
+        unsigned u;                     /* Counter */
+        H5D_shared_t *shared_fo = udata->cpy_info->shared_fo;
+
+        /* See if the written chunk is in the chunk cache */
+        if(shared_fo && shared_fo->cache.chunk.nslots > 0) {
+            hsize_t chunk_idx;                  /* Index of chunk cache entry */
+            H5O_layout_t  *layout = &shared_fo->layout;
+
+            /* Calculate the index of this chunk */
+            if(H5VM_chunk_index(layout->u.chunk.ndims - 1, chunk_rec->offset,
+                                layout->u.chunk.dim, layout->u.chunk.down_chunks, &chunk_idx) < 0)
+                HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't get chunk index")
+            idx = H5D_CHUNK_HASH(shared_fo, chunk_idx);
+
+            /* Get the chunk cache entry for that location */
+            ent = shared_fo->cache.chunk.slot[idx];
+            if(ent) {
+                /* Speculatively set the 'found' flag */
+                udata->chunk_in_cache = TRUE;
+
+                /* Verify that the cache entry is the correct chunk */
+                for(u = 0; u < (layout->u.chunk.ndims -1); u++)
+                    if(chunk_rec->offset[u] != ent->offset[u]) {
+                        udata->chunk_in_cache = FALSE;
+                        break;
+                    } /* end if */
+            } /* end if */
+        } /* end if */
+
+        if(udata->chunk_in_cache) {
+            HDassert(H5F_addr_defined(chunk_rec->chunk_addr));
+            HDassert(H5F_addr_defined(ent->chunk_addr));
+
+            H5_CHECKED_ASSIGN(nbytes, size_t, shared_fo->layout.u.chunk.size, uint32_t);
+            HDmemcpy(buf, ent->chunk, nbytes);
+        } else {
+            /* read chunk data from the source file */
+            if(H5F_block_read(udata->file_src, H5FD_MEM_DRAW, chunk_rec->chunk_addr, nbytes, udata->idx_info_dst->dxpl_id, buf) < 0)
+                HGOTO_ERROR(H5E_IO, H5E_READERROR, H5_ITER_ERROR, "unable to read raw data chunk")
+        }
+    }
 
     /* Need to uncompress variable-length & reference data elements */
-    if(has_filters && (is_vlen || fix_ref)) {
+    if(has_filters && (is_vlen || fix_ref) && !udata->chunk_in_cache) {
         unsigned filter_mask = chunk_rec->filter_mask;
 
         if(H5Z_pipeline(pline, H5Z_FLAG_REVERSE, &filter_mask, H5Z_NO_EDC, cb_struct, &nbytes, &buf_size, &buf) < 0)
@@ -4906,7 +4952,7 @@ H5D__chunk_copy_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata)
     udata_dst.addr = HADDR_UNDEF;
 
     /* Need to compress variable-length & reference data elements before writing to file */
-    if(has_filters && (is_vlen || fix_ref) ) {
+    if(has_filters && (is_vlen || fix_ref || udata->chunk_in_cache) ) {
         if(H5Z_pipeline(pline, 0, &(udata_dst.filter_mask), H5Z_NO_EDC, cb_struct, &nbytes, &buf_size, &buf) < 0)
             HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, H5_ITER_ERROR, "output pipeline failed")
 #if H5_SIZEOF_SIZE_T > 4
@@ -4971,6 +5017,7 @@ H5D__chunk_copy(H5F_t *f_src, H5O_storage_chunk_t *storage_src,
     uint32_t    nelmts = 0;             /* Number of elements in buffer */
     hbool_t     do_convert = FALSE;     /* Indicate that type conversions should be performed */
     hbool_t     copy_setup_done = FALSE;        /* Indicate that 'copy setup' is done */
+    H5D_shared_t *shared_fo = (H5D_shared_t *)cpy_info->shared_fo;
     herr_t      ret_value = SUCCEED;    /* Return value */
 
     FUNC_ENTER_PACKAGE
@@ -5157,11 +5204,39 @@ H5D__chunk_copy(H5F_t *f_src, H5O_storage_chunk_t *storage_src,
     udata.nelmts = nelmts;
     udata.pline = pline;
     udata.cpy_info = cpy_info;
+    udata.chunk_in_cache = FALSE;
+    udata.chunk = NULL;
+
+    if(!H5F_addr_defined(idx_info_src.storage->idx_addr))
+        idx_info_src.storage = &shared_fo->layout.storage.u.chunk;
+
+    HDassert(H5F_addr_defined(idx_info_src.storage->idx_addr));
 
     /* Iterate over chunks to copy data */
     if((storage_src->ops->iterate)(&idx_info_src, H5D__chunk_copy_cb, &udata) < 0)
         HGOTO_ERROR(H5E_DATASET, H5E_BADITER, FAIL, "unable to iterate over chunk index to copy data")
 
+     /* Iterate over the chunk cache to copy data for chunks with undefined address */
+    if(shared_fo) {
+        H5D_rdcc_ent_t  *ent, *next;
+        H5D_chunk_rec_t chunk_rec;
+
+        chunk_rec.nbytes = layout_src->size;
+        chunk_rec.filter_mask = 0;
+        chunk_rec.chunk_addr = HADDR_UNDEF;
+
+        for(ent = shared_fo->cache.chunk.head; ent; ent = next) {
+            if(!H5F_addr_defined(ent->chunk_addr)) {
+                HDmemcpy(chunk_rec.offset, ent->offset, sizeof(chunk_rec.offset));
+                udata.chunk = ent->chunk;
+                udata.chunk_in_cache = TRUE;
+                if(H5D__chunk_copy_cb(&chunk_rec, &udata) < 0)
+                    HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy chunk data in cache")
+            }
+            next = ent->next;
+        } /* end for */
+    }
+
     /* I/O buffers may have been re-allocated */
     buf = udata.buf;
     bkg = udata.bkg;
diff --git a/src/H5Dcompact.c b/src/H5Dcompact.c
index b142f3b..4b61442 100644
--- a/src/H5Dcompact.c
+++ b/src/H5Dcompact.c
@@ -396,7 +396,7 @@ done:
  *-------------------------------------------------------------------------
  */
 herr_t
-H5D__compact_copy(H5F_t *f_src, H5O_storage_compact_t *storage_src, H5F_t *f_dst,
+H5D__compact_copy(H5F_t *f_src, H5O_storage_compact_t *_storage_src, H5F_t *f_dst,
     H5O_storage_compact_t *storage_dst, H5T_t *dt_src, H5O_copy_t *cpy_info,
     hid_t dxpl_id)
 {
@@ -407,6 +407,8 @@ H5D__compact_copy(H5F_t *f_src, H5O_storage_compact_t *storage_src, H5F_t *f_dst
     void       *bkg = NULL;             /* Temporary buffer for copying data */
     void       *reclaim_buf = NULL;     /* Buffer for reclaiming data */
     hid_t       buf_sid = -1;           /* ID for buffer dataspace */
+    H5D_shared_t    *shared_fo = cpy_info->shared_fo;   /* Pointer to the shared struct for dataset object */
+    H5O_storage_compact_t *storage_src = _storage_src;  /* Pointer to storage_src */
     herr_t      ret_value = SUCCEED;    /* Return value */
 
     FUNC_ENTER_PACKAGE
@@ -418,6 +420,10 @@ H5D__compact_copy(H5F_t *f_src, H5O_storage_compact_t *storage_src, H5F_t *f_dst
     HDassert(storage_dst);
     HDassert(dt_src);
 
+    /* If the dataset is open in the file, point to "layout" in the shared struct */
+    if(shared_fo != NULL)
+        storage_src = &(shared_fo->layout.storage.u.compact);
+
     /* Allocate space for destination data */
     if(NULL == (storage_dst->buf = H5MM_malloc(storage_src->size)))
         HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate memory for compact dataset")
diff --git a/src/H5Dcontig.c b/src/H5Dcontig.c
index c6cef57..9516d53 100644
--- a/src/H5Dcontig.c
+++ b/src/H5Dcontig.c
@@ -1298,6 +1298,10 @@ H5D__contig_copy(H5F_t *f_src, const H5O_storage_contig_t *storage_src,
     hsize_t     buf_dim[1] = {0};       /* Dimension for buffer */
     hbool_t     is_vlen = FALSE;        /* Flag to indicate that VL type conversion should occur */
     hbool_t     fix_ref = FALSE;        /* Flag to indicate that ref values should be fixed */
+    H5D_shared_t    *shared_fo = cpy_info->shared_fo;  /* Pointer to the shared struct for dataset object */
+    hbool_t     try_sieve = FALSE;          /* Try to get data from the sieve buffer */
+    haddr_t     sieve_start = HADDR_UNDEF;  /* Start location of sieve buffer */
+    haddr_t     sieve_end = HADDR_UNDEF;    /* End locations of sieve buffer */ 
     herr_t      ret_value = SUCCEED;    /* Return value */
 
     FUNC_ENTER_PACKAGE
@@ -1418,9 +1422,32 @@ H5D__contig_copy(H5F_t *f_src, const H5O_storage_contig_t *storage_src,
             HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for copy buffer")
     } /* end if */
 
-    /* Loop over copying data */
     addr_src = storage_src->addr;
     addr_dst = storage_dst->addr;
+
+    /* If data sieving is enabled and the dataset is open in the file,
+       set up to copy data out of the sieve buffer if deemed possible later */
+    if(H5F_HAS_FEATURE(f_src, H5FD_FEAT_DATA_SIEVE) &&
+       shared_fo && shared_fo->cache.contig.sieve_buf) {
+        try_sieve = TRUE;
+        sieve_start = shared_fo->cache.contig.sieve_loc;
+        sieve_end = sieve_start + shared_fo->cache.contig.sieve_size;
+        /* 
+         * It is possble for addr_src to be undefined when:
+         *  (a) The dataset is created and data is written to it.
+         *  (b) The dataset is then copied via H5Ocopy().
+         *  H5D_mark() in H5Dint.c is different between 
+         *  1.8 and develop branches:
+         *  1.8--it just sets dataset->shared->layout_dirty as TRUE 
+         *       to be flushed later.
+         *  develop--it will flush the layout message if it has been changed.
+         */
+        if(!H5F_addr_defined(addr_src))
+            addr_src = sieve_start;
+    }
+
+    HDassert(H5F_addr_defined(addr_src));
+    /* Loop over copying data */
     while(total_src_nbytes > 0) {
         /* Check if we should reduce the number of bytes to transfer */
         if(total_src_nbytes < src_nbytes) {
@@ -1446,14 +1473,20 @@ H5D__contig_copy(H5F_t *f_src, const H5O_storage_contig_t *storage_src,
                 dst_nbytes = mem_nbytes = src_nbytes;
         } /* end if */
 
-        /* Read raw data from source file */
-        if(H5F_block_read(f_src, H5FD_MEM_DRAW, addr_src, src_nbytes, H5P_DATASET_XFER_DEFAULT, buf) < 0)
-            HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read raw data")
+         /* If the entire copy is within the sieve buffer, copy data from the sieve buffer */
+        if(try_sieve && (addr_src >= sieve_start) && ((addr_src + src_nbytes -1) < sieve_end)) {
+            unsigned char *base_sieve_buf = shared_fo->cache.contig.sieve_buf + (addr_src - sieve_start);
+
+            HDmemcpy(buf, base_sieve_buf, src_nbytes);
+        } else
+            /* Read raw data from source file */
+            if(H5F_block_read(f_src, H5FD_MEM_DRAW, addr_src, src_nbytes, H5P_DATASET_XFER_DEFAULT, buf) < 0)
+                HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read raw data")
 
         /* Perform datatype conversion, if necessary */
         if(is_vlen) {
             /* Convert from source file to memory */
-	    if(H5T_convert(tpath_src_mem, tid_src, tid_mem, nelmts, (size_t)0, (size_t)0, buf, bkg, dxpl_id) < 0)
+            if(H5T_convert(tpath_src_mem, tid_src, tid_mem, nelmts, (size_t)0, (size_t)0, buf, bkg, dxpl_id) < 0)
                 HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "datatype conversion failed")
 
             /* Copy into another buffer, to reclaim memory later */
@@ -1463,13 +1496,13 @@ H5D__contig_copy(H5F_t *f_src, const H5O_storage_contig_t *storage_src,
             HDmemset(bkg, 0, buf_size);
 
             /* Convert from memory to destination file */
-	    if(H5T_convert(tpath_mem_dst, tid_mem, tid_dst, nelmts, (size_t)0, (size_t)0, buf, bkg, dxpl_id) < 0)
+            if(H5T_convert(tpath_mem_dst, tid_mem, tid_dst, nelmts, (size_t)0, (size_t)0, buf, bkg, dxpl_id) < 0)
                 HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "datatype conversion failed")
 
             /* Reclaim space from variable length data */
             if(H5D_vlen_reclaim(tid_mem, buf_space, H5P_DATASET_XFER_DEFAULT, reclaim_buf) < 0)
                 HGOTO_ERROR(H5E_DATASET, H5E_BADITER, FAIL, "unable to reclaim variable-length data")
-	} /* end if */
+        } /* end if */
         else if(fix_ref) {
             /* Check for expanding references */
             if(cpy_info->expand_ref) {
diff --git a/src/H5Ocopy.c b/src/H5Ocopy.c
index 9193a4c..31beb0e 100644
--- a/src/H5Ocopy.c
+++ b/src/H5Ocopy.c
@@ -357,24 +357,8 @@ H5O_copy_header_real(const H5O_loc_t *oloc_src, H5O_loc_t *oloc_dst /*out*/,
     if((obj_class = H5O_obj_class(oloc_src, dxpl_id)) == NULL)
         HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, FAIL, "unable to determine object type")
 
-    /* Check if the object at the address is already open in the file */
-    if(H5FO_opened(oloc_src->file, oloc_src->addr) != NULL) {
-	
-	H5G_loc_t   tmp_loc; 	/* Location of object */
-	H5O_loc_t   tmp_oloc; 	/* Location of object */
-	H5G_name_t  tmp_path;	/* Object's path */
-
-	tmp_loc.oloc = &tmp_oloc;
-	tmp_loc.path = &tmp_path;
-	tmp_oloc.file = oloc_src->file;
-	tmp_oloc.addr = oloc_src->addr;
-	tmp_oloc.holding_file = oloc_src->holding_file;
-	H5G_name_reset(tmp_loc.path);
-
-	/* Flush the object of this class */
-        if(obj_class->flush && obj_class->flush(&tmp_loc, dxpl_id) < 0)
-            HGOTO_ERROR(H5E_OHDR, H5E_CANTFLUSH, FAIL, "unable to flush object")
-    }
+    /* Set the pointer to the shared struct for the object if opened in the file */
+    cpy_info->shared_fo = H5FO_opened(oloc_src->file, oloc_src->addr);
 
     /* Get source object header */
     if(NULL == (oh_src = H5O_protect(oloc_src, dxpl_id, H5AC_READ)))
diff --git a/src/H5Olayout.c b/src/H5Olayout.c
index 95d66ab..17385c2 100644
--- a/src/H5Olayout.c
+++ b/src/H5Olayout.c
@@ -592,6 +592,7 @@ H5O_layout_copy_file(H5F_t *file_src, void *mesg_src, H5F_t *file_dst,
     H5O_layout_t       *layout_src = (H5O_layout_t *) mesg_src;
     H5O_layout_t       *layout_dst = NULL;
     hbool_t             copied = FALSE;                         /* Whether the data was copied */
+    H5D_shared_t *shared_fo = (H5D_shared_t *)cpy_info->shared_fo;
     void               *ret_value;                              /* Return value */
 
     FUNC_ENTER_NOAPI_NOINIT
@@ -628,7 +629,9 @@ H5O_layout_copy_file(H5F_t *file_src, void *mesg_src, H5F_t *file_dst,
                 layout_dst->storage.u.contig.size = H5S_extent_nelem(udata->src_space_extent) *
                                         H5T_get_size(udata->src_dtype);
 
-            if(H5D__contig_is_space_alloc(&layout_src->storage)) {
+            if(H5D__contig_is_space_alloc(&layout_src->storage) || 
+               (H5F_HAS_FEATURE(file_src, H5FD_FEAT_DATA_SIEVE) && 
+                shared_fo && shared_fo->cache.contig.sieve_buf)) {
                 /* copy contiguous raw data */
                 if(H5D__contig_copy(file_src, &layout_src->storage.u.contig, file_dst, &layout_dst->storage.u.contig, udata->src_dtype, cpy_info, dxpl_id) < 0)
                     HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, NULL, "unable to copy contiguous storage")
@@ -637,7 +640,8 @@ H5O_layout_copy_file(H5F_t *file_src, void *mesg_src, H5F_t *file_dst,
             break;
 
         case H5D_CHUNKED:
-            if(H5D__chunk_is_space_alloc(&layout_src->storage)) {
+            if(H5D__chunk_is_space_alloc(&layout_src->storage) || 
+               (shared_fo && H5D__chunk_is_space_alloc(&shared_fo->layout.storage))) {
                 /* Create chunked layout */
                 if(H5D__chunk_copy(file_src, &layout_src->storage.u.chunk, &layout_src->u.chunk, file_dst, &layout_dst->storage.u.chunk, udata->src_space_extent, udata->src_dtype, udata->common.src_pline, cpy_info, dxpl_id) < 0)
                     HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, NULL, "unable to copy chunked storage")
diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h
index 9b24977..ca0fc8e 100644
--- a/src/H5Oprivate.h
+++ b/src/H5Oprivate.h
@@ -169,6 +169,7 @@ typedef struct H5O_copy_t {
     H5SL_t  *dst_dt_list;               /* Skip list to hold committed datatypes in dest file */
     hbool_t dst_dt_list_complete;       /* Whether the destination datatype list is complete (i.e. not only populated with "suggestions" from H5Padd_merge_committed_dtype_path) */
     H5O_t   *oh_dst;                    /* The destination object header */
+    void    *shared_fo;                 /* The shared pointer for the object */
     H5O_mcdt_search_cb_t mcdt_cb;	/* The callback to invoke before searching the global list of committed datatypes at destination */
     void *mcdt_ud;			/* User data passed to callback */
 } H5O_copy_t;