Merge pull request #1594 in HDFFV/hdf5 from ~NFORTNE2/hdf5_naf:develop to develop

Fix issues with direct chunk operations and the single chunk index.

* commit 'c812a6c9ec5ff440dc405167c54b86fdb18dcfa4':
  Fix issue with direct chunk write not updating the "last chunk" index cache.  Fix issues involving datasets being "no allocated" when they contain cached raw data.

9 files changed, 130 insertions, 9 deletions

diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c
index dcd3a8d..27a6125 100644
--- a/src/H5Dchunk.c
+++ b/src/H5Dchunk.c
@@ -313,6 +313,7 @@ const H5D_layout_ops_t H5D_LOPS_CHUNK[1] = {{
     H5D__chunk_construct,
     H5D__chunk_init,
     H5D__chunk_is_space_alloc,
+    H5D__chunk_is_data_cached,
     H5D__chunk_io_init,
     H5D__chunk_read,
     H5D__chunk_write,
@@ -340,6 +341,7 @@ const H5D_layout_ops_t H5D_LOPS_NONEXISTENT[1] = {{
     NULL,
     NULL,
     NULL,
+    NULL,
 #ifdef H5_HAVE_PARALLEL
     NULL,
     NULL,
@@ -394,10 +396,13 @@ H5D__chunk_direct_write(const H5D_t *dset, uint32_t filters, hsize_t *offset,
 
     FUNC_ENTER_PACKAGE_TAG(dset->oloc.addr)
 
+    /* Sanity checks */
+    HDassert(layout->type == H5D_CHUNKED);
+
     io_info.dset = dset;
 
     /* Allocate dataspace and initialize it if it hasn't been. */
-    if(!(*layout->ops->is_space_alloc)(&layout->storage))
+    if(!H5D__chunk_is_space_alloc(&layout->storage))
         /* Allocate storage */
         if(H5D__alloc_storage(&io_info, H5D_ALLOC_WRITE, FALSE, NULL) < 0)
             HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize storage")
@@ -435,13 +440,17 @@ H5D__chunk_direct_write(const H5D_t *dset, uint32_t filters, hsize_t *offset,
     if(0 == idx_info.pline->nused && H5F_addr_defined(old_chunk.offset))
         /* If there are no filters and we are overwriting the chunk we can just set values */
         need_insert = FALSE;
-    else
+    else {
         /* Otherwise, create the chunk it if it doesn't exist, or reallocate the chunk
          * if its size has changed.
          */
         if(H5D__chunk_file_alloc(&idx_info, &old_chunk, &udata.chunk_block, &need_insert, scaled) < 0)
             HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate chunk")
 
+        /* Cache the new chunk information */
+        H5D__chunk_cinfo_cache_update(&dset->shared->cache.chunk.last, &udata);
+    } /* end else */
+
     /* Make sure the address of the chunk is returned. */
     if(!H5F_addr_defined(udata.chunk_block.offset))
         HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "chunk address isn't defined")
@@ -506,7 +515,8 @@ H5D__chunk_direct_read(const H5D_t *dset, hsize_t *offset, uint32_t* filters,
     *filters = 0;
 
     /* Allocate dataspace and initialize it if it hasn't been. */
-    if(!(*layout->ops->is_space_alloc)(&layout->storage))
+    if(!H5D__chunk_is_space_alloc(&layout->storage)
+            && !H5D__chunk_is_data_cached(dset->shared))
         HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "storage is not initialized")
 
     /* Calculate the index of this chunk */
@@ -1019,6 +1029,30 @@ H5D__chunk_is_space_alloc(const H5O_storage_t *storage)
 
 
 /*-------------------------------------------------------------------------
+ * Function:    H5D__chunk_is_data_cached
+ *
+ * Purpose:     Query if raw data is cached for dataset
+ *
+ * Return:      Non-negative on success/Negative on failure
+ *
+ * Programmer:  Neil Fortner
+ *              Wednessday, March 6, 2016
+ *
+ *-------------------------------------------------------------------------
+ */
+hbool_t
+H5D__chunk_is_data_cached(const H5D_shared_t *shared_dset)
+{
+    FUNC_ENTER_PACKAGE_NOERR
+
+    /* Sanity checks */
+    HDassert(shared_dset);
+
+    FUNC_LEAVE_NOAPI(shared_dset->cache.chunk.nused > 0)
+} /* end H5D__chunk_is_data_cached() */
+
+
+/*-------------------------------------------------------------------------
  * Function:	H5D__chunk_io_init
  *
  * Purpose:	Performs initialization before any sort of I/O on the raw data
diff --git a/src/H5Dcompact.c b/src/H5Dcompact.c
index c0c2a80..651aaf4 100644
--- a/src/H5Dcompact.c
+++ b/src/H5Dcompact.c
@@ -80,6 +80,7 @@ const H5D_layout_ops_t H5D_LOPS_COMPACT[1] = {{
     H5D__compact_construct,
     NULL,
     H5D__compact_is_space_alloc,
+    NULL,
     H5D__compact_io_init,
     H5D__contig_read,
     H5D__contig_write,
diff --git a/src/H5Dcontig.c b/src/H5Dcontig.c
index ad12ba0..c2e9bfc 100644
--- a/src/H5Dcontig.c
+++ b/src/H5Dcontig.c
@@ -118,6 +118,7 @@ const H5D_layout_ops_t H5D_LOPS_CONTIG[1] = {{
     H5D__contig_construct,
     H5D__contig_init,
     H5D__contig_is_space_alloc,
+    H5D__contig_is_data_cached,
     H5D__contig_io_init,
     H5D__contig_read,
     H5D__contig_write,
@@ -538,6 +539,30 @@ H5D__contig_is_space_alloc(const H5O_storage_t *storage)
 
 
 /*-------------------------------------------------------------------------
+ * Function:    H5D__contig_is_data_cached
+ *
+ * Purpose:     Query if raw data is cached for dataset
+ *
+ * Return:      Non-negative on success/Negative on failure
+ *
+ * Programmer:  Neil Fortner
+ *              Wednessday, March 6, 2016
+ *
+ *-------------------------------------------------------------------------
+ */
+hbool_t
+H5D__contig_is_data_cached(const H5D_shared_t *shared_dset)
+{
+    FUNC_ENTER_PACKAGE_NOERR
+
+    /* Sanity checks */
+    HDassert(shared_dset);
+
+    FUNC_LEAVE_NOAPI(shared_dset->cache.contig.sieve_size > 0)
+} /* end H5D__contig_is_data_cached() */
+
+
+/*-------------------------------------------------------------------------
  * Function:	H5D__contig_io_init
  *
  * Purpose:	Performs initialization before any sort of I/O on the raw data
diff --git a/src/H5Defl.c b/src/H5Defl.c
index b2f9b29..42b3947 100644
--- a/src/H5Defl.c
+++ b/src/H5Defl.c
@@ -91,6 +91,7 @@ const H5D_layout_ops_t H5D_LOPS_EFL[1] = {{
     H5D__efl_construct,
     NULL,
     H5D__efl_is_space_alloc,
+    NULL,
     H5D__efl_io_init,
     H5D__contig_read,
     H5D__contig_write,
diff --git a/src/H5Dint.c b/src/H5Dint.c
index 384c66b..874e845 100644
--- a/src/H5Dint.c
+++ b/src/H5Dint.c
@@ -2898,7 +2898,8 @@ H5D__set_extent(H5D_t *dset, const hsize_t *size)
          *-------------------------------------------------------------------------
          */
         if(H5D_CHUNKED == dset->shared->layout.type) {
-            if(shrink && (*dset->shared->layout.ops->is_space_alloc)(&dset->shared->layout.storage))
+            if(shrink && ((*dset->shared->layout.ops->is_space_alloc)(&dset->shared->layout.storage)
+                    || (dset->shared->layout.ops->is_data_cached && (*dset->shared->layout.ops->is_data_cached)(dset->shared))))
                 /* Remove excess chunks */
                 if(H5D__chunk_prune_by_extent(dset, curr_dims) < 0)
                     HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to remove chunks")
diff --git a/src/H5Dio.c b/src/H5Dio.c
index 2f87e38..9343b80 100644
--- a/src/H5Dio.c
+++ b/src/H5Dio.c
@@ -518,7 +518,8 @@ H5D__read(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space,
      * has been overwritten.  So just proceed in reading.
      */
     if(nelmts > 0 && dataset->shared->dcpl_cache.efl.nused == 0 &&
-            !(*dataset->shared->layout.ops->is_space_alloc)(&dataset->shared->layout.storage)) {
+            !(*dataset->shared->layout.ops->is_space_alloc)(&dataset->shared->layout.storage) &&
+            !(dataset->shared->layout.ops->is_data_cached && (*dataset->shared->layout.ops->is_data_cached)(dataset->shared))) {
         H5D_fill_value_t fill_status;   /* Whether/How the fill value is defined */
 
         /* Retrieve dataset's fill-value properties */
@@ -550,6 +551,7 @@ H5D__read(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space,
     /* Sanity check that space is allocated, if there are elements */
     if(nelmts > 0)
         HDassert((*dataset->shared->layout.ops->is_space_alloc)(&dataset->shared->layout.storage)
+                || (dataset->shared->layout.ops->is_data_cached && (*dataset->shared->layout.ops->is_data_cached)(dataset->shared))
                 || dataset->shared->dcpl_cache.efl.nused > 0
                 || dataset->shared->layout.type == H5D_COMPACT);
 
diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h
index 2767aa1..e7c623b 100644
--- a/src/H5Dpkg.h
+++ b/src/H5Dpkg.h
@@ -113,12 +113,14 @@ typedef struct H5D_type_info_t {
 /* Forward declaration of structs used below */
 struct H5D_io_info_t;
 struct H5D_chunk_map_t;
+typedef struct H5D_shared_t H5D_shared_t;
 
 /* Function pointers for I/O on particular types of dataset layouts */
 typedef herr_t (*H5D_layout_construct_func_t)(H5F_t *f, H5D_t *dset);
 typedef herr_t (*H5D_layout_init_func_t)(H5F_t *f, const H5D_t *dset,
     hid_t dapl_id);
 typedef hbool_t (*H5D_layout_is_space_alloc_func_t)(const H5O_storage_t *storage);
+typedef hbool_t (*H5D_layout_is_data_cached_func_t)(const H5D_shared_t *shared_dset);
 typedef herr_t (*H5D_layout_io_init_func_t)(const struct H5D_io_info_t *io_info,
     const H5D_type_info_t *type_info,
     hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space,
@@ -144,6 +146,7 @@ typedef struct H5D_layout_ops_t {
     H5D_layout_construct_func_t construct;      /* Layout constructor for new datasets */
     H5D_layout_init_func_t init;        /* Layout initializer for dataset */
     H5D_layout_is_space_alloc_func_t is_space_alloc;    /* Query routine to determine if storage is allocated */
+    H5D_layout_is_data_cached_func_t is_data_cached;    /* Query routine to determine if any raw data is cached.  If routine is not present then the layout type never caches raw data. */
     H5D_layout_io_init_func_t io_init;  /* I/O initialization routine */
     H5D_layout_read_func_t ser_read;    /* High-level I/O routine for reading data in serial */
     H5D_layout_write_func_t ser_write;  /* High-level I/O routine for writing data in serial */
@@ -429,7 +432,7 @@ typedef struct H5D_rdcdc_t {
  * created once for a given dataset.  Thus, if a dataset is opened twice,
  * there will be two IDs and two H5D_t structs, both sharing one H5D_shared_t.
  */
-typedef struct H5D_shared_t {
+struct H5D_shared_t {
     size_t              fo_count;       /* Reference count */
     hbool_t             closing;        /* Flag to indicate dataset is closing */
     hid_t               type_id;        /* ID for dataset's datatype    */
@@ -459,7 +462,7 @@ typedef struct H5D_shared_t {
     H5D_append_flush_t   append_flush;   /* Append flush property information */
     char                *extfile_prefix; /* expanded external file prefix */
     char                *vds_prefix;     /* expanded vds prefix */
-} H5D_shared_t;
+};
 
 struct H5D_t {
     H5O_loc_t           oloc;           /* Object header location       */
@@ -618,6 +621,7 @@ H5_DLL herr_t H5D__layout_oh_write(const H5D_t *dataset, H5O_t *oh, unsigned upd
 /* Functions that operate on contiguous storage */
 H5_DLL herr_t H5D__contig_alloc(H5F_t *f, H5O_storage_contig_t *storage);
 H5_DLL hbool_t H5D__contig_is_space_alloc(const H5O_storage_t *storage);
+H5_DLL hbool_t H5D__contig_is_data_cached(const H5D_shared_t *shared_dset);
 H5_DLL herr_t H5D__contig_fill(const H5D_io_info_t *io_info);
 H5_DLL herr_t H5D__contig_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
     hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space,
@@ -636,6 +640,7 @@ H5_DLL htri_t H5D__chunk_cacheable(const H5D_io_info_t *io_info, haddr_t caddr,
 H5_DLL herr_t H5D__chunk_create(const H5D_t *dset /*in,out*/);
 H5_DLL herr_t H5D__chunk_set_info(const H5D_t *dset);
 H5_DLL hbool_t H5D__chunk_is_space_alloc(const H5O_storage_t *storage);
+H5_DLL hbool_t H5D__chunk_is_data_cached(const H5D_shared_t *shared_dset);
 H5_DLL herr_t H5D__chunk_lookup(const H5D_t *dset, const hsize_t *scaled,
     H5D_chunk_ud_t *udata);
 H5_DLL herr_t H5D__chunk_allocated(const H5D_t *dset, hsize_t *nbytes);
diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c
index c0d49d8..a803cca 100644
--- a/src/H5Dvirtual.c
+++ b/src/H5Dvirtual.c
@@ -78,6 +78,7 @@
 /********************/
 
 /* Layout operation callbacks */
+static hbool_t H5D__virtual_is_data_cached(const H5D_shared_t *shared_dset);
 static herr_t H5D__virtual_read(H5D_io_info_t *io_info, const H5D_type_info_t
     *type_info, hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space,
     H5D_chunk_map_t *fm);
@@ -121,6 +122,7 @@ const H5D_layout_ops_t H5D_LOPS_VIRTUAL[1] = {{
     NULL,
     H5D__virtual_init,
     H5D__virtual_is_space_alloc,
+    H5D__virtual_is_data_cached,
     NULL,
     H5D__virtual_read,
     H5D__virtual_write,
@@ -2207,6 +2209,54 @@ H5D__virtual_is_space_alloc(const H5O_storage_t H5_ATTR_UNUSED *storage)
 
 
 /*-------------------------------------------------------------------------
+ * Function:    H5D__virtual_is_data_cached
+ *
+ * Purpose:     Query if raw data is cached for dataset
+ *
+ * Return:      Non-negative on success/Negative on failure
+ *
+ * Programmer:  Neil Fortner
+ *              Wednessday, March 6, 2016
+ *
+ *-------------------------------------------------------------------------
+ */
+static hbool_t
+H5D__virtual_is_data_cached(const H5D_shared_t *shared_dset)
+{
+    const H5O_storage_virtual_t *storage;   /* Convenience pointer */
+    size_t i, j;                            /* Local index variables */
+    hbool_t ret_value = FALSE;              /* Return value */
+
+    FUNC_ENTER_PACKAGE_NOERR
+
+    /* Sanity checks */
+    HDassert(shared_dset);
+    storage = &shared_dset->layout.storage.u.virt;
+
+    /* Iterate over mappings */
+    for(i = 0; i < storage->list_nused; i++)
+        /* Check for "printf" source dataset resolution */
+        if(storage->list[i].psfn_nsubs || storage->list[i].psdn_nsubs) {
+            /* Iterate over sub-source dsets */
+            for(j = storage->list[i].sub_dset_io_start; j < storage->list[i].sub_dset_io_end; j++)
+                /* Check for cahced data in source dset */
+                if(storage->list[i].sub_dset[j].dset
+                        && storage->list[i].sub_dset[j].dset->shared->layout.ops->is_data_cached
+                        && storage->list[i].sub_dset[j].dset->shared->layout.ops->is_data_cached(storage->list[i].sub_dset[j].dset->shared))
+                    HGOTO_DONE(TRUE);
+        } /* end if */
+        else
+            if(storage->list[i].source_dset.dset
+                    && storage->list[i].source_dset.dset->shared->layout.ops->is_data_cached
+                    && storage->list[i].source_dset.dset->shared->layout.ops->is_data_cached(storage->list[i].source_dset.dset->shared))
+                HGOTO_DONE(TRUE);
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__virtual_is_data_cached() */
+
+
+/*-------------------------------------------------------------------------
  * Function:    H5D__virtual_pre_io
  *
  * Purpose:     Project all virtual mappings onto mem_space, with the
diff --git a/src/H5Olayout.c b/src/H5Olayout.c
index 2b65e0c..86f4c46 100644
--- a/src/H5Olayout.c
+++ b/src/H5Olayout.c
@@ -1039,7 +1039,8 @@ H5O__layout_copy_file(H5F_t *file_src, void *mesg_src, H5F_t *file_dst,
                 layout_dst->storage.u.contig.size = H5S_extent_nelem(udata->src_space_extent) *
                                         H5T_get_size(udata->src_dtype);
 
-            if(H5D__contig_is_space_alloc(&layout_src->storage)) {
+            if(H5D__contig_is_space_alloc(&layout_src->storage)
+                    || (cpy_info->shared_fo && H5D__contig_is_data_cached((const H5D_shared_t *)cpy_info->shared_fo))) {
                 /* copy contiguous raw data */
                 if(H5D__contig_copy(file_src, &layout_src->storage.u.contig, file_dst, &layout_dst->storage.u.contig, udata->src_dtype, cpy_info) < 0)
                     HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, NULL, "unable to copy contiguous storage")
@@ -1048,7 +1049,8 @@ H5O__layout_copy_file(H5F_t *file_src, void *mesg_src, H5F_t *file_dst,
             break;
 
         case H5D_CHUNKED:
-            if(H5D__chunk_is_space_alloc(&layout_src->storage)) {
+            if(H5D__chunk_is_space_alloc(&layout_src->storage)
+                    || (cpy_info->shared_fo && H5D__chunk_is_data_cached((const H5D_shared_t *)cpy_info->shared_fo))) {
                 /* Create chunked layout */
                 if(H5D__chunk_copy(file_src, &layout_src->storage.u.chunk, &layout_src->u.chunk, file_dst, &layout_dst->storage.u.chunk, udata->src_space_extent, udata->src_dtype, udata->common.src_pline, cpy_info) < 0)
                     HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, NULL, "unable to copy chunked storage")