Fix issue with direct chunk write not updating the "last chunk" index

cache.  Fix issues involving datasets being "not allocated" when they
contain cached raw data.

8 files changed, 78 insertions, 11 deletions

diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c
index 4cabd72..8a475b9 100644
--- a/src/H5Dchunk.c
+++ b/src/H5Dchunk.c
@@ -256,6 +256,7 @@ const H5D_layout_ops_t H5D_LOPS_CHUNK[1] = {{
     H5D__chunk_construct,
     H5D__chunk_init,
     H5D__chunk_is_space_alloc,
+    H5D__chunk_is_data_cached,
     H5D__chunk_io_init,
     H5D__chunk_read,
     H5D__chunk_write,
@@ -282,6 +283,7 @@ const H5D_layout_ops_t H5D_LOPS_NONEXISTENT[1] = {{
     NULL,
     NULL,
     NULL,
+    NULL,
 #ifdef H5_HAVE_PARALLEL
     NULL,
     NULL,
@@ -332,8 +334,11 @@ H5D__chunk_direct_write(const H5D_t *dset, hid_t dxpl_id, uint32_t filters, hsiz
 
     FUNC_ENTER_STATIC
 
+    /* Sanity checks */
+    HDassert(layout->type == H5D_CHUNKED);
+
     /* Allocate data space and initialize it if it hasn't been. */
-    if(!(*dset->shared->layout.ops->is_space_alloc)(&dset->shared->layout.storage))
+    if(!H5D__chunk_is_space_alloc(&dset->shared->layout.storage))
         /* Allocate storage */
         if(H5D__alloc_storage(dset, dxpl_id, H5D_ALLOC_WRITE, FALSE, NULL) < 0)
             HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize storage")
@@ -374,6 +379,9 @@ H5D__chunk_direct_write(const H5D_t *dset, hid_t dxpl_id, uint32_t filters, hsiz
         if((dset->shared->layout.storage.u.chunk.ops->insert)(&idx_info, &udata) < 0)
             HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert/resize chunk")
 
+        /* Cache the new chunk information */
+        H5D__chunk_cinfo_cache_update(&dset->shared->cache.chunk.last, &udata);
+
         /* Make sure the address of the chunk is returned. */
         if(!H5F_addr_defined(udata.addr))
             HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "chunk address isn't defined")
@@ -437,7 +445,8 @@ H5D__chunk_direct_read(const H5D_t *dset, hid_t dxpl_id, hsize_t *offset,
     space_ndims = dset->shared->layout.u.chunk.ndims - 1;
 
     /* Check if chunk storage is allocated for dataset */
-    if(H5D__chunk_is_space_alloc(&(layout->storage)) == FALSE)
+    if((H5D__chunk_is_space_alloc(&(layout->storage)) == FALSE)
+            && !H5D__chunk_is_data_cached(dset->shared))
         HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "storage is not initialized")
 
     /* Initialize the chunk offset */
@@ -899,6 +908,30 @@ H5D__chunk_is_space_alloc(const H5O_storage_t *storage)
 
 
 /*-------------------------------------------------------------------------
+ * Function:    H5D__chunk_is_data_cached
+ *
+ * Purpose:     Query if raw data is cached for dataset
+ *
+ * Return:      Non-negative on success/Negative on failure
+ *
+ * Programmer:  Neil Fortner
+ *              Wednessday, March 6, 2016
+ *
+ *-------------------------------------------------------------------------
+ */
+hbool_t
+H5D__chunk_is_data_cached(const H5D_shared_t *shared_dset)
+{
+    FUNC_ENTER_PACKAGE_NOERR
+
+    /* Sanity checks */
+    HDassert(shared_dset);
+
+    FUNC_LEAVE_NOAPI(shared_dset->cache.chunk.nused > 0)
+} /* end H5D__chunk_is_data_cached() */
+
+
+/*-------------------------------------------------------------------------
  * Function:    H5D__chunk_io_init
  *
  * Purpose:    Performs initialization before any sort of I/O on the raw data
diff --git a/src/H5Dcompact.c b/src/H5Dcompact.c
index 4b61442..71866c3 100644
--- a/src/H5Dcompact.c
+++ b/src/H5Dcompact.c
@@ -79,6 +79,7 @@ const H5D_layout_ops_t H5D_LOPS_COMPACT[1] = {{
     H5D__compact_construct,
     NULL,
     H5D__compact_is_space_alloc,
+    NULL,
     H5D__compact_io_init,
     H5D__contig_read,
     H5D__contig_write,
diff --git a/src/H5Dcontig.c b/src/H5Dcontig.c
index 9516d53..49e769b 100644
--- a/src/H5Dcontig.c
+++ b/src/H5Dcontig.c
@@ -119,6 +119,7 @@ const H5D_layout_ops_t H5D_LOPS_CONTIG[1] = {{
     H5D__contig_construct,
     NULL,
     H5D__contig_is_space_alloc,
+    H5D__contig_is_data_cached,
     H5D__contig_io_init,
     H5D__contig_read,
     H5D__contig_write,
@@ -483,6 +484,30 @@ H5D__contig_is_space_alloc(const H5O_storage_t *storage)
 
 
 /*-------------------------------------------------------------------------
+ * Function:    H5D__contig_is_data_cached
+ *
+ * Purpose:     Query if raw data is cached for dataset
+ *
+ * Return:      Non-negative on success/Negative on failure
+ *
+ * Programmer:  Neil Fortner
+ *              Wednessday, March 6, 2016
+ *
+ *-------------------------------------------------------------------------
+ */
+hbool_t
+H5D__contig_is_data_cached(const H5D_shared_t *shared_dset)
+{
+    FUNC_ENTER_PACKAGE_NOERR
+
+    /* Sanity checks */
+    HDassert(shared_dset);
+
+    FUNC_LEAVE_NOAPI(shared_dset->cache.contig.sieve_size > 0)
+} /* end H5D__contig_is_data_cached() */
+
+
+/*-------------------------------------------------------------------------
  * Function:	H5D__contig_io_init
  *
  * Purpose:	Performs initialization before any sort of I/O on the raw data
diff --git a/src/H5Defl.c b/src/H5Defl.c
index 3ebcefb..cda841a 100644
--- a/src/H5Defl.c
+++ b/src/H5Defl.c
@@ -91,6 +91,7 @@ const H5D_layout_ops_t H5D_LOPS_EFL[1] = {{
     H5D__efl_construct,
     NULL,
     H5D__efl_is_space_alloc,
+    NULL,
     H5D__efl_io_init,
     H5D__contig_read,
     H5D__contig_write,
diff --git a/src/H5Dint.c b/src/H5Dint.c
index c34c9a9..1093e13 100644
--- a/src/H5Dint.c
+++ b/src/H5Dint.c
@@ -2333,7 +2333,8 @@ H5D__set_extent(H5D_t *dset, const hsize_t *size, hid_t dxpl_id)
          *-------------------------------------------------------------------------
          */
         if(shrink && H5D_CHUNKED == dset->shared->layout.type &&
-                (*dset->shared->layout.ops->is_space_alloc)(&dset->shared->layout.storage))
+                ((*dset->shared->layout.ops->is_space_alloc)(&dset->shared->layout.storage)
+                || (dset->shared->layout.ops->is_data_cached && (*dset->shared->layout.ops->is_data_cached)(dset->shared))))
             /* Remove excess chunks */
             if(H5D__chunk_prune_by_extent(dset, dxpl_id, curr_dims) < 0)
                 HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to remove chunks")
diff --git a/src/H5Dio.c b/src/H5Dio.c
index cb72437..1973b54 100644
--- a/src/H5Dio.c
+++ b/src/H5Dio.c
@@ -550,7 +550,8 @@ H5D__read(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space,
      * has been overwritten.  So just proceed in reading.
      */
     if(nelmts > 0 && dataset->shared->dcpl_cache.efl.nused == 0 &&
-            !(*dataset->shared->layout.ops->is_space_alloc)(&dataset->shared->layout.storage)) {
+            !(*dataset->shared->layout.ops->is_space_alloc)(&dataset->shared->layout.storage) &&
+            !(dataset->shared->layout.ops->is_data_cached && (*dataset->shared->layout.ops->is_data_cached)(dataset->shared))) {
         H5D_fill_value_t fill_status;   /* Whether/How the fill value is defined */
 
         /* Retrieve dataset's fill-value properties */
@@ -585,6 +586,7 @@ H5D__read(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space,
     /* Sanity check that space is allocated, if there are elements */
     if(nelmts > 0)
         HDassert((*dataset->shared->layout.ops->is_space_alloc)(&dataset->shared->layout.storage)
+                || (dataset->shared->layout.ops->is_data_cached && (*dataset->shared->layout.ops->is_data_cached)(dataset->shared))
                 || dataset->shared->dcpl_cache.efl.nused > 0
                 || dataset->shared->layout.type == H5D_COMPACT);
 
diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h
index 8eaa1a4..b985de1 100644
--- a/src/H5Dpkg.h
+++ b/src/H5Dpkg.h
@@ -96,12 +96,14 @@ typedef struct H5D_type_info_t {
 /* Forward declaration of structs used below */
 struct H5D_io_info_t;
 struct H5D_chunk_map_t;
+typedef struct H5D_shared_t H5D_shared_t;
 
 /* Function pointers for I/O on particular types of dataset layouts */
 typedef herr_t (*H5D_layout_construct_func_t)(H5F_t *f, H5D_t *dset);
 typedef herr_t (*H5D_layout_init_func_t)(H5F_t *f, hid_t dxpl_id, const H5D_t *dset,
     hid_t dapl_id);
 typedef hbool_t (*H5D_layout_is_space_alloc_func_t)(const H5O_storage_t *storage);
+typedef hbool_t (*H5D_layout_is_data_cached_func_t)(const H5D_shared_t *shared_dset);
 typedef herr_t (*H5D_layout_io_init_func_t)(const struct H5D_io_info_t *io_info,
     const H5D_type_info_t *type_info,
     hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space,
@@ -126,6 +128,7 @@ typedef struct H5D_layout_ops_t {
     H5D_layout_construct_func_t construct;      /* Layout constructor for new datasets */
     H5D_layout_init_func_t init;        /* Layout initializer for dataset */
     H5D_layout_is_space_alloc_func_t is_space_alloc;    /* Query routine to determine if storage is allocated */
+    H5D_layout_is_data_cached_func_t is_data_cached;    /* Query routine to determine if any raw data is cached.  If routine is not present then the layout type never caches raw data. */
     H5D_layout_io_init_func_t io_init;  /* I/O initialization routine */
     H5D_layout_read_func_t ser_read;    /* High-level I/O routine for reading data in serial */
     H5D_layout_write_func_t ser_write;  /* High-level I/O routine for writing data in serial */
@@ -405,7 +408,7 @@ typedef struct H5D_rdcdc_t {
  * created once for a given dataset.  Thus, if a dataset is opened twice,
  * there will be two IDs and two H5D_t structs, both sharing one H5D_shared_t.
  */
-typedef struct H5D_shared_t {
+struct H5D_shared_t {
     size_t              fo_count;       /* Reference count */
     hid_t               type_id;        /* ID for dataset's datatype    */
     H5T_t              *type;           /* Datatype for this dataset     */
@@ -428,7 +431,7 @@ typedef struct H5D_shared_t {
     } cache;
 
     char                *extfile_prefix; /* expanded external file prefix */
-} H5D_shared_t;
+};
 
 struct H5D_t {
     H5O_loc_t           oloc;           /* Object header location       */
@@ -587,6 +590,7 @@ H5_DLL herr_t H5D__layout_oh_write(H5D_t *dataset, hid_t dxpl_id, H5O_t *oh,
 H5_DLL herr_t H5D__contig_alloc(H5F_t *f, hid_t dxpl_id,
     H5O_storage_contig_t *storage);
 H5_DLL hbool_t H5D__contig_is_space_alloc(const H5O_storage_t *storage);
+H5_DLL hbool_t H5D__contig_is_data_cached(const H5D_shared_t *shared_dset);
 H5_DLL herr_t H5D__contig_fill(const H5D_t *dset, hid_t dxpl_id);
 H5_DLL herr_t H5D__contig_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
     hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space,
@@ -609,6 +613,7 @@ H5_DLL herr_t H5D__chunk_set_info(const H5D_t *dset);
 H5_DLL herr_t H5D__chunk_init(H5F_t *f, hid_t dxpl_id, const H5D_t *dset,
     hid_t dapl_id);
 H5_DLL hbool_t H5D__chunk_is_space_alloc(const H5O_storage_t *storage);
+H5_DLL hbool_t H5D__chunk_is_data_cached(const H5D_shared_t *shared_dset);
 H5_DLL herr_t H5D__chunk_lookup(const H5D_t *dset, hid_t dxpl_id,
     const hsize_t *chunk_offset, hsize_t chunk_idx, H5D_chunk_ud_t *udata);
 H5_DLL void *H5D__chunk_lock(const H5D_io_info_t *io_info,
diff --git a/src/H5Olayout.c b/src/H5Olayout.c
index 3f5076c..c22a1ba 100644
--- a/src/H5Olayout.c
+++ b/src/H5Olayout.c
@@ -630,9 +630,8 @@ H5O_layout_copy_file(H5F_t *file_src, void *mesg_src, H5F_t *file_dst,
                 layout_dst->storage.u.contig.size = H5S_extent_nelem(udata->src_space_extent) *
                                         H5T_get_size(udata->src_dtype);
 
-            if(H5D__contig_is_space_alloc(&layout_src->storage) || 
-               (H5F_HAS_FEATURE(file_src, H5FD_FEAT_DATA_SIEVE) && 
-                shared_fo && shared_fo->cache.contig.sieve_buf)) {
+            if(H5D__contig_is_space_alloc(&layout_src->storage)
+                    || (cpy_info->shared_fo && H5D__contig_is_data_cached((const H5D_shared_t *)cpy_info->shared_fo))) {
                 /* copy contiguous raw data */
                 if(H5D__contig_copy(file_src, &layout_src->storage.u.contig, file_dst, &layout_dst->storage.u.contig, udata->src_dtype, cpy_info, dxpl_id) < 0)
                     HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, NULL, "unable to copy contiguous storage")
@@ -641,8 +640,8 @@ H5O_layout_copy_file(H5F_t *file_src, void *mesg_src, H5F_t *file_dst,
             break;
 
         case H5D_CHUNKED:
-            if(H5D__chunk_is_space_alloc(&layout_src->storage) || 
-               (shared_fo && H5D__chunk_is_space_alloc(&shared_fo->layout.storage))) {
+            if(H5D__chunk_is_space_alloc(&layout_src->storage)
+                    || (cpy_info->shared_fo && H5D__chunk_is_data_cached((const H5D_shared_t *)cpy_info->shared_fo))) 
                 /* Create chunked layout */
                 if(H5D__chunk_copy(file_src, &layout_src->storage.u.chunk, &layout_src->u.chunk, file_dst, &layout_dst->storage.u.chunk, udata->src_space_extent, udata->src_dtype, udata->common.src_pline, cpy_info, dxpl_id) < 0)
                     HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, NULL, "unable to copy chunked storage")