Backport H5Dchunk_iter to 1.12 branch (#1970)

* Revert "Revert H5Dchunk_iter changes in hdf5_1_12_1 (#733)"

This reverts commit 10abe9a8b4985a18a0c88615686184d233d7b845.

* Apply clang-format

* Reincorporate spelling fix from #1166

* Incorporate H5Dchunk_iter changes from #161

* Backport to 1.12: Adds a quick for for some egregious chunk_info badness (#722)

* Backport to 1.12: Converts testhdf5 macros to h5test macros in chunk_info.c (#1820)

The two macro schemes were not designed to work together. Also
quiets some MSVC warnings about comparing pointers and integers.

* Backport to 1.12: H5Dchunk_iter now passes offsets in units of dataset elements, fix #1419 (#1969)

* H5Dchunk_iter now passes chunk dimension scaled offsets, fix #1419

* Update docs for H5Dchunk_iter, H5Dget_chunk_info*

Modified description for `H5Dchunk_iter`, `H5Dget_chunk_info`, and `H5Dget_chunk_info_by_coord` to the following

 * offset          Logical position of the chunk’s first element in units of dataset elements
 * filter_mask  Bitmask indicating the filters used when the chunk was written
 * size             Chunk size in bytes, 0 if the chunk does not exist

* Sync H5Dchunk_iter documentation with develop

* Remove H5VL_DATASET_WAIT references from 1.12

* Backport to 1.12 #161, #1474, review comments

Co-authored-by: Dana Robinson <43805+derobins@users.noreply.github.com>

6 files changed, 269 insertions, 16 deletions

diff --git a/src/H5D.c b/src/H5D.c
index 2347c93..ceadc64 100644
--- a/src/H5D.c
+++ b/src/H5D.c
@@ -1126,3 +1126,77 @@ H5Dget_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, unsigned *filte
 done:
     FUNC_LEAVE_API(ret_value)
 } /* end H5Dget_chunk_info_by_coord() */
+
+/*-------------------------------------------------------------------------
+ * Function:    H5Dchunk_iter
+ *
+ * Purpose:     Iterates over all chunks in dataset with given callback and user data.
+ *
+ * Parameters:
+ *              hid_t dset_id;          IN: Chunked dataset ID
+ *              hid_t dxpl_id;          IN: Dataset transfer property list ID
+ *              H5D_chunk_iter_op_t cb  IN: User callback function, called for every chunk.
+ *              void *op_data           IN/OUT: Optional user data passed on to user callback.
+ *
+ * Callback information:
+ *      H5D_chunk_iter_op_t is defined as:
+ *
+ *        typedef int (*H5D_chunk_iter_op_t)(
+ *            const hsize_t *offset,
+ *            uint32_t filter_mask,
+ *            haddr_t addr,
+ *            uint32_t size,
+ *            void *op_data);
+ *
+ *      H5D_chunk_iter_op_t parameters:
+ *          hsize_t *offset;        IN/OUT: Array of starting logical coordinates of chunk.
+ *          uint32_t filter_mask;   IN: Filter mask of chunk.
+ *          haddr_t addr;           IN: Offset in file of chunk data.
+ *          uint32_t nbytes;        IN: Size in number of bytes of chunk data in file.
+ *          void *op_data;          IN/OUT: Pointer to any user-defined data
+ *                                  associated with the operation.
+ *
+ *      The return values from an operator are:
+ *          Zero (H5_ITER_CONT) causes the iterator to continue, returning zero when all
+ *              elements have been processed.
+ *          Positive (H5_ITER_STOP) causes the iterator to immediately return that positive
+ *              value, indicating short-circuit success.
+ *          Negative (H5_ITER_ERROR) causes the iterator to immediately return that value,
+ *              indicating failure.
+ *
+ * Return:      Non-negative on success, negative on failure
+ *
+ * Programmer:  Gaute Hope
+ *              August 2020
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Dchunk_iter(hid_t dset_id, hid_t dxpl_id, H5D_chunk_iter_op_t op, void *op_data)
+{
+    H5VL_object_t *vol_obj   = NULL; /* Dataset for this operation */
+    herr_t         ret_value = SUCCEED;
+
+    FUNC_ENTER_API(FAIL)
+    H5TRACE4("e", "iix*x", dset_id, dxpl_id, op, op_data);
+
+    /* Check arguments */
+    if (NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(dset_id, H5I_DATASET)))
+        HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid dataset identifier")
+    if (NULL == op)
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid callback to chunk iteration")
+
+    /* Get the default dataset transfer property list if the user didn't provide one */
+    if (H5P_DEFAULT == dxpl_id)
+        dxpl_id = H5P_DATASET_XFER_DEFAULT;
+    else if (TRUE != H5P_isa_class(dxpl_id, H5P_DATASET_XFER))
+        HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dxpl_id is not a dataset transfer property list ID")
+
+    /* Iterate over the chunks */
+    if ((ret_value = H5VL_dataset_optional(vol_obj, H5VL_NATIVE_DATASET_CHUNK_ITER, dxpl_id, H5_REQUEST_NULL,
+                                           op, op_data)) < 0)
+        HGOTO_ERROR(H5E_BADITER, H5E_BADITER, FAIL, "error iterating over dataset chunks")
+
+done:
+    FUNC_LEAVE_API(ret_value)
+} /* end H5Dchunk_iter() */
diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c
index f58f3ec..e0eaaeb 100644
--- a/src/H5Dchunk.c
+++ b/src/H5Dchunk.c
@@ -249,6 +249,12 @@ typedef struct H5D_chunk_coll_fill_info_t {
 } H5D_chunk_coll_fill_info_t;
 #endif /* H5_HAVE_PARALLEL */
 
+typedef struct H5D_chunk_iter_ud_t {
+    H5D_chunk_iter_op_t op;      /* User defined callback */
+    void               *op_data; /* User data for user defined callback */
+    H5O_layout_chunk_t *chunk;   /* Chunk layout */
+} H5D_chunk_iter_ud_t;
+
 /********************/
 /* Local Prototypes */
 /********************/
@@ -272,6 +278,7 @@ static herr_t H5D__chunk_dest(H5D_t *dset);
 static int H5D__get_num_chunks_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata);
 static int H5D__get_chunk_info_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata);
 static int H5D__get_chunk_info_by_coord_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata);
+static int H5D__chunk_iter_cb(const H5D_chunk_rec_t *chunk_rec, void *udata);
 
 /* "Nonexistent" layout operation callback */
 static ssize_t H5D__nonexistent_readvv(const H5D_io_info_t *io_info, size_t chunk_max_nseq,
@@ -7649,3 +7656,106 @@ H5D__get_chunk_info_by_coord(const H5D_t *dset, const hsize_t *offset, unsigned
 done:
     FUNC_LEAVE_NOAPI_TAG(ret_value)
 } /* end H5D__get_chunk_info_by_coord() */
+
+/*-------------------------------------------------------------------------
+ * Function:    H5D__chunk_iter_cb
+ *
+ * Purpose:     Call the user-defined function with the chunk data. The iterator continues if
+ *              the user-defined function returns H5_ITER_CONT, and stops if H5_ITER_STOP is
+ *              returned.
+ *
+ * Return:      Success:    H5_ITER_CONT or H5_ITER_STOP
+ *              Failure:    Negative (H5_ITER_ERROR)
+ *
+ * Programmer:  Gaute Hope
+ *              August 2020
+ *
+ *-------------------------------------------------------------------------
+ */
+static int
+H5D__chunk_iter_cb(const H5D_chunk_rec_t *chunk_rec, void *udata)
+{
+    const H5D_chunk_iter_ud_t *data      = (H5D_chunk_iter_ud_t *)udata;
+    const H5O_layout_chunk_t  *chunk     = data->chunk;
+    int                        ret_value = H5_ITER_CONT;
+    hsize_t                    offset[H5O_LAYOUT_NDIMS];
+    unsigned                   ii; /* Match H5O_layout_chunk_t.ndims */
+
+    /* Similar to H5D__get_chunk_info */
+    for (ii = 0; ii < chunk->ndims; ii++)
+        offset[ii] = chunk_rec->scaled[ii] * chunk->dim[ii];
+
+    FUNC_ENTER_PACKAGE_NOERR
+
+    /* Check for callback failure and pass along return value */
+    if ((ret_value = (data->op)(offset, chunk_rec->filter_mask, chunk_rec->chunk_addr, chunk_rec->nbytes,
+                                data->op_data)) < 0)
+        HERROR(H5E_DATASET, H5E_CANTNEXT, "iteration operator failed");
+
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__chunk_iter_cb */
+
+/*-------------------------------------------------------------------------
+ * Function:    H5D__chunk_iter
+ *
+ * Purpose:     Iterate over all the chunks in the dataset with given callback.
+ *
+ * Return:      Success:        Non-negative
+ *              Failure:        Negative
+ *
+ * Programmer:  Gaute Hope
+ *              August 2020
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5D__chunk_iter(H5D_t *dset, H5D_chunk_iter_op_t op, void *op_data)
+{
+    const H5D_rdcc_t  *rdcc   = NULL;       /* Raw data chunk cache */
+    H5O_layout_t      *layout = NULL;       /* Dataset layout */
+    H5D_rdcc_ent_t    *ent;                 /* Cache entry index */
+    H5D_chk_idx_info_t idx_info;            /* Chunked index info */
+    herr_t             ret_value = SUCCEED; /* Return value */
+
+    FUNC_ENTER_PACKAGE_TAG(dset->oloc.addr)
+
+    /* Check args */
+    HDassert(dset);
+    HDassert(dset->shared);
+
+    /* Get dataset layout and raw data chunk cache */
+    layout = &(dset->shared->layout);
+    rdcc   = &(dset->shared->cache.chunk);
+    HDassert(layout);
+    HDassert(rdcc);
+    HDassert(H5D_CHUNKED == layout->type);
+
+    /* Search for cached chunks that haven't been written out */
+    for (ent = rdcc->head; ent; ent = ent->next)
+        /* Flush the chunk out to disk, to make certain the size is correct later */
+        if (H5D__chunk_flush_entry(dset, ent, FALSE) < 0)
+            HGOTO_ERROR(H5E_DATASET, H5E_CANTFLUSH, FAIL, "cannot flush indexed storage buffer")
+
+    /* Compose chunked index info struct */
+    idx_info.f       = dset->oloc.file;
+    idx_info.pline   = &dset->shared->dcpl_cache.pline;
+    idx_info.layout  = &layout->u.chunk;
+    idx_info.storage = &layout->storage.u.chunk;
+
+    /* If the dataset is not written, return without errors */
+    if (H5F_addr_defined(idx_info.storage->idx_addr)) {
+        H5D_chunk_iter_ud_t ud;
+
+        /* Set up info for iteration callback */
+        ud.op      = op;
+        ud.op_data = op_data;
+        ud.chunk   = &dset->shared->layout.u.chunk;
+
+        /* Iterate over the allocated chunks calling the iterator callback */
+        if ((ret_value = (layout->storage.u.chunk.ops->iterate)(&idx_info, H5D__chunk_iter_cb, &ud)) < 0)
+            HERROR(H5E_DATASET, H5E_CANTNEXT, "chunk iteration failed");
+    } /* end if H5F_addr_defined */
+
+done:
+    FUNC_LEAVE_NOAPI_TAG(ret_value)
+} /* end H5D__chunk_iter() */
diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h
index 99f2719..9150e4f 100644
--- a/src/H5Dpkg.h
+++ b/src/H5Dpkg.h
@@ -565,6 +565,7 @@ H5_DLL herr_t  H5D__get_chunk_info(const H5D_t *dset, const H5S_t *space, hsize_
                                    unsigned *filter_mask, haddr_t *offset, hsize_t *size);
 H5_DLL herr_t  H5D__get_chunk_info_by_coord(const H5D_t *dset, const hsize_t *coord, unsigned *filter_mask,
                                             haddr_t *addr, hsize_t *size);
+H5_DLL herr_t  H5D__chunk_iter(H5D_t *dset, H5D_chunk_iter_op_t cb, void *op_data);
 H5_DLL haddr_t H5D__get_offset(const H5D_t *dset);
 H5_DLL herr_t  H5D__vlen_get_buf_size(H5D_t *dset, hid_t type_id, hid_t space_id, hsize_t *size);
 H5_DLL herr_t  H5D__vlen_get_buf_size_gen(H5VL_object_t *vol_obj, hid_t type_id, hid_t space_id,
diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h
index d368120..d39e2c6 100644
--- a/src/H5Dpublic.h
+++ b/src/H5Dpublic.h
@@ -222,6 +222,27 @@ typedef herr_t (*H5D_scatter_func_t)(const void **src_buf /*out*/, size_t *src_b
 typedef herr_t (*H5D_gather_func_t)(const void *dst_buf, size_t dst_buf_bytes_used, void *op_data);
 //! <!-- [H5D_gather_func_t_snip] -->
 
+//! <!-- [H5D_chunk_iter_op_t_snip] -->
+/**
+ * \brief Callback for H5Dchunk_iter()
+ *
+ * \param[in]     offset      Logical position of the chunk’s first element in units of dataset elements
+ * \param[in]     filter_mask Bitmask indicating the filters used when the chunk was written
+ * \param[in]     addr        Chunk address in the file
+ * \param[in]     size        Chunk size in bytes, 0 if the chunk does not exist
+ * \param[in,out] op_data     Pointer to any user-defined data associated with
+ *                            the operation.
+ * \returns \li Zero (#H5_ITER_CONT) causes the iterator to continue, returning
+ *              zero when all elements have been processed.
+ *          \li A positive value (#H5_ITER_STOP) causes the iterator to
+ *              immediately return that value, indicating short-circuit success.
+ *          \li A negative (#H5_ITER_ERROR) causes the iterator to immediately
+ *              return that value, indicating failure.
+ */
+typedef int (*H5D_chunk_iter_op_t)(const hsize_t *offset, uint32_t filter_mask, haddr_t addr, uint32_t size,
+                                   void *op_data);
+//! <!-- [H5D_chunk_iter_op_t_snip] -->
+
 /********************/
 /* Public Variables */
 /********************/
@@ -588,10 +609,10 @@ H5_DLL herr_t H5Dget_num_chunks(hid_t dset_id, hid_t fspace_id, hsize_t *nchunks
  * \brief Retrieves information about a chunk specified by its coordinates
  *
  * \dset_id
- * \param[in]  offset      Logical position of the chunk’s first element
- * \param[out] filter_mask Indicating filters used with the chunk when written
+ * \param[in]  offset      Logical position of the chunk’s first element in units of dataset elements
+ * \param[out] filter_mask Bitmask indicating the filters used when the chunk was written
  * \param[out] addr        Chunk address in the file
- * \param[out] size        Chunk size in bytes, 0 if chunk doesn’t exist
+ * \param[out] size        Chunk size in bytes, 0 if the chunk does not exist
  *
  * \return \herr_t
  *
@@ -612,6 +633,33 @@ H5_DLL herr_t H5Dget_num_chunks(hid_t dset_id, hid_t fspace_id, hsize_t *nchunks
  */
 H5_DLL herr_t H5Dget_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, unsigned *filter_mask,
                                          haddr_t *addr, hsize_t *size);
+/**
+ * --------------------------------------------------------------------------
+ * \ingroup H5D
+ *
+ * \brief Iterate over all chunks of a chunked dataset
+ *
+ * \dset_id
+ * \param[in]  dxpl_id  Identifier of a transfer property list
+ * \param[in]  cb       User callback function, called for every chunk.
+ * \param[in]  op_data  User-defined pointer to data required by op
+ *
+ * \return \herr_t
+ *
+ * \details H5Dchunk_iter iterates over all chunks in the dataset, calling the
+ *          user supplied callback with the details of the chunk and the supplied
+ *          context \p op_data.
+ *
+ * \par Example
+ * For each chunk, print the allocated chunk size (0 for un-allocated chunks).
+ * \snippet H5D_examples.c H5Dchunk_iter_cb
+ * Iterate over all chunked datasets and chunks in a file.
+ * \snippet H5D_examples.c H5Ovisit_cb
+ *
+ * \since 1.12.3, 1.13.0
+ *
+ */
+H5_DLL herr_t H5Dchunk_iter(hid_t dset_id, hid_t dxpl_id, H5D_chunk_iter_op_t cb, void *op_data);
 
 /**
  * --------------------------------------------------------------------------
@@ -622,10 +670,10 @@ H5_DLL herr_t H5Dget_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, u
  * \dset_id
  * \param[in]  fspace_id File dataspace selection identifier (See Note below)
  * \param[in]  chk_idx   Index of the chunk
- * \param[out] offset    Logical position of the chunk’s first element
- * \param[out] filter_mask Indicating filters used with the chunk when written
+ * \param[out] offset    Logical position of the chunk’s first element in units of dataset elements
+ * \param[out] filter_mask Bitmask indicating the filters used when the chunk was written
  * \param[out] addr      Chunk address in the file
- * \param[out] size      Chunk size in bytes, 0 if chunk doesn’t exist
+ * \param[out] size      Chunk size in bytes, 0 if the chunk does not exist
  *
  * \return \herr_t
  *
diff --git a/src/H5VLnative.h b/src/H5VLnative.h
index cb7f0e03..8aaa6c0 100644
--- a/src/H5VLnative.h
+++ b/src/H5VLnative.h
@@ -38,16 +38,17 @@
 #endif                                 /* H5_NO_DEPRECATED_SYMBOLS */
 
 /* Values for native VOL connector dataset optional VOL operations */
-#define H5VL_NATIVE_DATASET_FORMAT_CONVERT          0 /* H5Dformat_convert (internal) */
-#define H5VL_NATIVE_DATASET_GET_CHUNK_INDEX_TYPE    1 /* H5Dget_chunk_index_type      */
-#define H5VL_NATIVE_DATASET_GET_CHUNK_STORAGE_SIZE  2 /* H5Dget_chunk_storage_size    */
-#define H5VL_NATIVE_DATASET_GET_NUM_CHUNKS          3 /* H5Dget_num_chunks            */
-#define H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_IDX   4 /* H5Dget_chunk_info            */
-#define H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_COORD 5 /* H5Dget_chunk_info_by_coord   */
-#define H5VL_NATIVE_DATASET_CHUNK_READ              6 /* H5Dchunk_read                */
-#define H5VL_NATIVE_DATASET_CHUNK_WRITE             7 /* H5Dchunk_write               */
-#define H5VL_NATIVE_DATASET_GET_VLEN_BUF_SIZE       8 /* H5Dvlen_get_buf_size         */
-#define H5VL_NATIVE_DATASET_GET_OFFSET              9 /* H5Dget_offset                */
+#define H5VL_NATIVE_DATASET_FORMAT_CONVERT          0  /* H5Dformat_convert (internal) */
+#define H5VL_NATIVE_DATASET_GET_CHUNK_INDEX_TYPE    1  /* H5Dget_chunk_index_type      */
+#define H5VL_NATIVE_DATASET_GET_CHUNK_STORAGE_SIZE  2  /* H5Dget_chunk_storage_size    */
+#define H5VL_NATIVE_DATASET_GET_NUM_CHUNKS          3  /* H5Dget_num_chunks            */
+#define H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_IDX   4  /* H5Dget_chunk_info            */
+#define H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_COORD 5  /* H5Dget_chunk_info_by_coord   */
+#define H5VL_NATIVE_DATASET_CHUNK_READ              6  /* H5Dchunk_read                */
+#define H5VL_NATIVE_DATASET_CHUNK_WRITE             7  /* H5Dchunk_write               */
+#define H5VL_NATIVE_DATASET_GET_VLEN_BUF_SIZE       8  /* H5Dvlen_get_buf_size         */
+#define H5VL_NATIVE_DATASET_GET_OFFSET              9  /* H5Dget_offset                */
+#define H5VL_NATIVE_DATASET_CHUNK_ITER              10 /* H5Dchunk_iter                */
 
 /* Values for native VOL connector file optional VOL operations */
 #define H5VL_NATIVE_FILE_CLEAR_ELINK_CACHE            0  /* H5Fclear_elink_file_cache            */
diff --git a/src/H5VLnative_dataset.c b/src/H5VLnative_dataset.c
index ad468b5..939f192 100644
--- a/src/H5VLnative_dataset.c
+++ b/src/H5VLnative_dataset.c
@@ -587,6 +587,25 @@ H5VL__native_dataset_optional(void *obj, H5VL_dataset_optional_t optional_type,
             break;
         }
 
+        /* H5Dchunk_iter */
+        case H5VL_NATIVE_DATASET_CHUNK_ITER: {
+            H5D_chunk_iter_op_t op      = HDva_arg(arguments, H5D_chunk_iter_op_t);
+            void               *op_data = HDva_arg(arguments, void *);
+
+            /* Sanity check */
+            HDassert(dset->shared);
+
+            /* Make sure the dataset is chunked */
+            if (H5D_CHUNKED != dset->shared->layout.type)
+                HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset")
+
+            /* Call private function */
+            if ((ret_value = H5D__chunk_iter(dset, op, op_data)) < 0)
+                HERROR(H5E_DATASET, H5E_BADITER, "chunk iteration failed");
+
+            break;
+        }
+
         default:
             HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "invalid optional operation")
     } /* end switch */