Refactor H5Dvlen_get_buf_size to use optional dataset operation, with generic fallback for VOL connectors that don't implement operation

1 files changed, 219 insertions, 26 deletions

diff --git a/src/H5Dint.c b/src/H5Dint.c
index 8815819..96b1dbd 100644
--- a/src/H5Dint.c
+++ b/src/H5Dint.c
@@ -43,16 +43,34 @@
 /* Local Typedefs */
 /******************/
 
-/* Internal data structure for computing variable-length dataset's total size */
+/* Shared data structure for computing variable-length dataset's total size */
+/* (Used for both native and generic 'get vlen buf size' operation) */
 typedef struct {
-    H5D_t *dset;        /* Dataset for operation */
-    H5S_t *fspace;      /* Dataset's dataspace for operation */
-    H5S_t *mspace;      /* Memory dataspace for operation */
     void *fl_tbuf;      /* Ptr to the temporary buffer we are using for fixed-length data */
     void *vl_tbuf;      /* Ptr to the temporary buffer we are using for VL data */
     size_t vl_tbuf_size; /* Current size of the temp. buffer for VL data */
     hsize_t size;       /* Accumulated number of bytes for the selection */
-} H5D_vlen_bufsize_t;
+} H5D_vlen_bufsize_common_t;
+
+/* Internal data structure for computing variable-length dataset's total size */
+/* (Used for native 'get vlen buf size' operation) */
+typedef struct {
+    H5D_t *dset;        /* Dataset for operation */
+    H5S_t *fspace;      /* Dataset's dataspace for operation */
+    H5S_t *mspace;      /* Memory dataspace for operation */
+    H5D_vlen_bufsize_common_t common;   /* VL data buffers & accumulatd size */
+} H5D_vlen_bufsize_native_t;
+
+/* Internal data structure for computing variable-length dataset's total size */
+/* (Used for generic 'get vlen buf size' operation) */
+typedef struct {
+    H5VL_object_t *dset_vol_obj;    /* VOL object for the dataset */
+    hid_t fspace_id;                /* Dataset dataspace ID of the dataset we are working on */
+    H5S_t *fspace;                  /* Dataset's dataspace for operation */
+    hid_t mspace_id;                /* Memory dataspace ID of the dataset we are working on */
+    hid_t dxpl_id;                  /* Dataset transfer property list to pass to dataset read */
+    H5D_vlen_bufsize_common_t common;   /* VL data buffers & accumulatd size */
+} H5D_vlen_bufsize_generic_t;
 
 
 /********************/
@@ -77,6 +95,8 @@ static size_t H5D__calculate_minimum_header_size(H5F_t *file, H5D_t *dset, H5O_t
 static void *H5D__vlen_get_buf_size_alloc(size_t size, void *info);
 static herr_t H5D__vlen_get_buf_size_cb(void *elem, hid_t type_id, unsigned ndim,
     const hsize_t *point, void *op_data);
+static herr_t H5D__vlen_get_buf_size_gen_cb(void *elem, hid_t type_id, unsigned ndim,
+    const hsize_t *point, void *op_data);
 
 
 /*********************/
@@ -2591,23 +2611,23 @@ done:
 static void *
 H5D__vlen_get_buf_size_alloc(size_t size, void *info)
 {
-    H5D_vlen_bufsize_t *vlen_bufsize = (H5D_vlen_bufsize_t *)info;
+    H5D_vlen_bufsize_common_t *vlen_bufsize_com = (H5D_vlen_bufsize_common_t *)info;
     void *ret_value = NULL;     /* Return value */
 
     FUNC_ENTER_STATIC
 
     /* Check for increasing the size of the temporary space for VL data */
-    if(size > vlen_bufsize->vl_tbuf_size) {
-        if(NULL == (vlen_bufsize->vl_tbuf = H5FL_BLK_REALLOC(vlen_vl_buf, vlen_bufsize->vl_tbuf, size)))
+    if(size > vlen_bufsize_com->vl_tbuf_size) {
+        if(NULL == (vlen_bufsize_com->vl_tbuf = H5FL_BLK_REALLOC(vlen_vl_buf, vlen_bufsize_com->vl_tbuf, size)))
             HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, NULL, "can't reallocate temporary VL data buffer")
-        vlen_bufsize->vl_tbuf_size = size;
+        vlen_bufsize_com->vl_tbuf_size = size;
     } /* end if */
 
     /* Increment size of VL data buffer needed */
-    vlen_bufsize->size += size;
+    vlen_bufsize_com->size += size;
 
     /* Set return value */
-    ret_value = vlen_bufsize->vl_tbuf;
+    ret_value = vlen_bufsize_com->vl_tbuf;
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
@@ -2627,7 +2647,7 @@ static herr_t
 H5D__vlen_get_buf_size_cb(void H5_ATTR_UNUSED *elem, hid_t type_id,
     unsigned H5_ATTR_UNUSED ndim, const hsize_t *point, void *op_data)
 {
-    H5D_vlen_bufsize_t *vlen_bufsize = (H5D_vlen_bufsize_t *)op_data;
+    H5D_vlen_bufsize_native_t *vlen_bufsize = (H5D_vlen_bufsize_native_t *)op_data;
     herr_t ret_value = H5_ITER_CONT;    /* Return value */
 
     FUNC_ENTER_STATIC
@@ -2642,7 +2662,7 @@ H5D__vlen_get_buf_size_cb(void H5_ATTR_UNUSED *elem, hid_t type_id,
         HGOTO_ERROR(H5E_DATASET, H5E_CANTCREATE, H5_ITER_ERROR, "can't select point")
 
     /* Read in the point (with the custom VL memory allocator) */
-    if(H5D__read(vlen_bufsize->dset, type_id, vlen_bufsize->mspace, vlen_bufsize->fspace, vlen_bufsize->fl_tbuf) < 0)
+    if(H5D__read(vlen_bufsize->dset, type_id, vlen_bufsize->mspace, vlen_bufsize->fspace, vlen_bufsize->common.fl_tbuf) < 0)
         HGOTO_ERROR(H5E_DATASET, H5E_READERROR, H5_ITER_ERROR, "can't read point")
 
 done:
@@ -2678,7 +2698,7 @@ herr_t
 H5D__vlen_get_buf_size(H5D_t *dset, hid_t type_id, hid_t space_id,
         hsize_t *size)
 {
-    H5D_vlen_bufsize_t vlen_bufsize = {NULL, NULL, NULL, NULL, NULL, 0, 0};
+    H5D_vlen_bufsize_native_t vlen_bufsize = {NULL, NULL, NULL, {NULL, NULL, 0, 0}};
     H5S_t *fspace = NULL;       /* Dataset's dataspace */
     H5S_t *mspace = NULL;       /* Memory dataspace */
     char bogus;                 /* bogus value to pass to H5Diterate() */
@@ -2690,9 +2710,9 @@ H5D__vlen_get_buf_size(H5D_t *dset, hid_t type_id, hid_t space_id,
     FUNC_ENTER_PACKAGE
 
     /* Check args */
-    if(NULL == (type = (H5T_t *)H5I_object_verify(type_id, H5I_DATATYPE)))
+    if(NULL == (type = (H5T_t *)H5I_object(type_id)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not an valid base datatype")
-    if(NULL == (space = (H5S_t *)H5I_object_verify(space_id, H5I_DATASPACE)))
+    if(NULL == (space = (H5S_t *)H5I_object(space_id)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid dataspace")
     if(!(H5S_has_extent(space)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "dataspace does not have extent set")
@@ -2711,18 +2731,18 @@ H5D__vlen_get_buf_size(H5D_t *dset, hid_t type_id, hid_t space_id,
     vlen_bufsize.mspace = mspace;
 
     /* Grab the temporary buffers required */
-    if(NULL == (vlen_bufsize.fl_tbuf = H5FL_BLK_MALLOC(vlen_fl_buf, H5T_get_size(type))))
+    if(NULL == (vlen_bufsize.common.fl_tbuf = H5FL_BLK_MALLOC(vlen_fl_buf, H5T_get_size(type))))
         HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "no temporary buffers available")
-    if(NULL == (vlen_bufsize.vl_tbuf = H5FL_BLK_MALLOC(vlen_vl_buf, (size_t)1)))
+    if(NULL == (vlen_bufsize.common.vl_tbuf = H5FL_BLK_MALLOC(vlen_vl_buf, (size_t)1)))
         HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "no temporary buffers available")
-    vlen_bufsize.vl_tbuf_size = 1;
+    vlen_bufsize.common.vl_tbuf_size = 1;
 
     /* Set the memory manager to the special allocation routine */
-    if(H5CX_set_vlen_alloc_info(H5D__vlen_get_buf_size_alloc, &vlen_bufsize, NULL, NULL) < 0)
+    if(H5CX_set_vlen_alloc_info(H5D__vlen_get_buf_size_alloc, &vlen_bufsize.common, NULL, NULL) < 0)
         HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set VL data allocation routine")
 
     /* Set the initial number of bytes required */
-    vlen_bufsize.size = 0;
+    vlen_bufsize.common.size = 0;
 
     /* Call H5S_select_iterate with args, etc. */
     dset_op.op_type             = H5S_SEL_ITER_OP_APP;
@@ -2733,23 +2753,196 @@ H5D__vlen_get_buf_size(H5D_t *dset, hid_t type_id, hid_t space_id,
 
     /* Get the size if we succeeded */
     if(ret_value >= 0)
-        *size = vlen_bufsize.size;
+        *size = vlen_bufsize.common.size;
 
 done:
     if(fspace && H5S_close(fspace) < 0)
         HDONE_ERROR(H5E_DATASPACE, H5E_CANTRELEASE, FAIL, "unable to release dataspace")
     if(mspace && H5S_close(mspace) < 0)
         HDONE_ERROR(H5E_DATASPACE, H5E_CANTRELEASE, FAIL, "unable to release dataspace")
-    if(vlen_bufsize.fl_tbuf != NULL)
-        vlen_bufsize.fl_tbuf = H5FL_BLK_FREE(vlen_fl_buf, vlen_bufsize.fl_tbuf);
-    if(vlen_bufsize.vl_tbuf != NULL)
-        vlen_bufsize.vl_tbuf = H5FL_BLK_FREE(vlen_vl_buf, vlen_bufsize.vl_tbuf);
+    if(vlen_bufsize.common.fl_tbuf != NULL)
+        vlen_bufsize.common.fl_tbuf = H5FL_BLK_FREE(vlen_fl_buf, vlen_bufsize.common.fl_tbuf);
+    if(vlen_bufsize.common.vl_tbuf != NULL)
+        vlen_bufsize.common.vl_tbuf = H5FL_BLK_FREE(vlen_vl_buf, vlen_bufsize.common.vl_tbuf);
 
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5D__vlen_get_buf_size() */
 
 
 /*-------------------------------------------------------------------------
+ * Function: H5D__vlen_get_buf_size_gen_cb
+ *
+ * Purpose:  This routine checks the number of bytes required to store a single
+ *           element from a dataset in memory, creating a selection with just the
+ *           single element selected to read in the element and using a custom memory
+ *           allocator for any VL data encountered.
+ *           The *size value is modified according to how many bytes are
+ *           required to store the element in memory.
+ *
+ * Implementation: This routine actually performs the read with a custom
+ *      memory manager which basically just counts the bytes requested and
+ *      uses a temporary memory buffer (through the H5FL API) to make certain
+ *      enough space is available to perform the read.  Then the temporary
+ *      buffer is released and the number of bytes allocated is returned.
+ *      Kinda kludgy, but easier than the other method of trying to figure out
+ *      the sizes without actually reading the data in... - QAK
+ *
+ * Return:   Non-negative on success, negative on failure
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__vlen_get_buf_size_gen_cb(void H5_ATTR_UNUSED *elem, hid_t type_id,
+    unsigned H5_ATTR_UNUSED ndim, const hsize_t *point, void *op_data)
+{
+    H5D_vlen_bufsize_generic_t *vlen_bufsize = (H5D_vlen_bufsize_generic_t *)op_data;
+    H5T_t *dt;                          /* Datatype for operation */
+    herr_t ret_value = SUCCEED;         /* Return value */
+
+    FUNC_ENTER_STATIC
+
+    /* Sanity check */
+    HDassert(point);
+    HDassert(op_data);
+
+    /* Check args */
+    if(NULL == (dt = (H5T_t *)H5I_object(type_id)))
+        HGOTO_ERROR(H5E_DATASET, H5E_BADTYPE, FAIL, "not a datatype")
+
+    /* Make certain there is enough fixed-length buffer available */
+    if(NULL == (vlen_bufsize->common.fl_tbuf = H5FL_BLK_REALLOC(vlen_fl_buf, vlen_bufsize->common.fl_tbuf, H5T_get_size(dt))))
+        HGOTO_ERROR(H5E_DATASET, H5E_NOSPACE, FAIL, "can't resize tbuf")
+
+    /* Select point to read in */
+    if(H5S_select_elements(vlen_bufsize->fspace, H5S_SELECT_SET, (size_t)1, point) < 0)
+        HGOTO_ERROR(H5E_DATASET, H5E_CANTCREATE, FAIL, "can't select point")
+
+    /* Read in the point (with the custom VL memory allocator) */
+    if(H5VL_dataset_read(vlen_bufsize->dset_vol_obj, type_id, vlen_bufsize->mspace_id, vlen_bufsize->fspace_id, vlen_bufsize->dxpl_id, vlen_bufsize->common.fl_tbuf, H5_REQUEST_NULL) < 0)
+        HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read point")
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__vlen_get_buf_size_gen_cb() */
+
+
+/*-------------------------------------------------------------------------
+ * Function:    H5D__vlen_get_buf_size_gen
+ *
+ * Purpose: Generic routine to checks the number of bytes required to store the
+ *      VL data from the dataset.
+ *
+ * Return:  Non-negative on success, negative on failure
+ *
+ * Programmer:  Quincey Koziol
+ *              Friday, December 20, 2019
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5D__vlen_get_buf_size_gen(H5VL_object_t *vol_obj, hid_t type_id, hid_t space_id,
+    hsize_t *size)
+{
+    H5D_vlen_bufsize_generic_t vlen_bufsize = {NULL, H5I_INVALID_HID, NULL, H5I_INVALID_HID, H5I_INVALID_HID, {NULL, NULL, 0, 0}};
+    H5P_genplist_t *dxpl = NULL; /* DXPL for operation */
+    H5S_t *mspace = NULL;       /* Memory dataspace */
+    char bogus;                 /* Bogus value to pass to H5Diterate() */
+    H5S_t *space;               /* Dataspace for iteration */
+    H5T_t *type;                /* Datatype */
+    H5S_sel_iter_op_t dset_op;  /* Operator for iteration */
+    herr_t ret_value = SUCCEED; /* Return value */
+
+    FUNC_ENTER_PACKAGE
+
+    /* Check args */
+    if(NULL == (type = (H5T_t *)H5I_object(type_id)))
+        HGOTO_ERROR(H5E_DATASET, H5E_BADTYPE, FAIL, "not an valid datatype")
+    if(NULL == (space = (H5S_t *)H5I_object(space_id)))
+        HGOTO_ERROR(H5E_DATASET, H5E_BADTYPE, FAIL, "invalid dataspace")
+    if(!(H5S_has_extent(space)))
+        HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "dataspace does not have extent set")
+
+    /* Save the dataset */
+    vlen_bufsize.dset_vol_obj = vol_obj;
+
+    /* Get a copy of the dataset's dataspace */
+    if(H5VL_dataset_get(vol_obj, H5VL_DATASET_GET_SPACE, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, &vlen_bufsize.fspace_id) < 0)
+        HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get dataspace")
+    if(NULL == (vlen_bufsize.fspace = (H5S_t *)H5I_object(vlen_bufsize.fspace_id)))
+        HGOTO_ERROR(H5E_DATASET, H5E_BADTYPE, FAIL, "not a dataspace")
+
+    /* Create a scalar for the memory dataspace */
+    if(NULL == (mspace = H5S_create(H5S_SCALAR)))
+        HGOTO_ERROR(H5E_DATASET, H5E_CANTCREATE, FAIL, "can't create dataspace")
+    if((vlen_bufsize.mspace_id = H5I_register(H5I_DATASPACE, mspace, TRUE)) < 0)
+        HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL, "unable to register dataspace ID")
+
+    /* Grab the temporary buffers required */
+    if(NULL == (vlen_bufsize.common.fl_tbuf = H5FL_BLK_MALLOC(vlen_fl_buf, H5T_get_size(type))))
+        HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "no temporary buffers available")
+    if(NULL == (vlen_bufsize.common.vl_tbuf = H5FL_BLK_MALLOC(vlen_vl_buf, (size_t)1)))
+        HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "no temporary buffers available")
+    vlen_bufsize.common.vl_tbuf_size = 1;
+
+    /* Set the memory manager to the special allocation routine */
+    if(H5CX_set_vlen_alloc_info(H5D__vlen_get_buf_size_alloc, &vlen_bufsize.common, NULL, NULL) < 0)
+        HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set VL data allocation routine")
+
+    /* Set the VL allocation callbacks on a DXPL */
+    if(NULL == (dxpl = (H5P_genplist_t *)H5I_object(H5P_DATASET_XFER_DEFAULT)))
+        HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get default DXPL")
+    if((vlen_bufsize.dxpl_id = H5P_copy_plist(dxpl, TRUE)) < 0)
+        HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "can't copy property list");
+    if(NULL == (dxpl = (H5P_genplist_t *)H5I_object(vlen_bufsize.dxpl_id)))
+        HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get copied DXPL")
+    if(H5P_set_vlen_mem_manager(dxpl, H5D__vlen_get_buf_size_alloc, &vlen_bufsize.common, NULL, NULL) < 0)
+        HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set VL data allocation routine on DXPL")
+
+    /* Set the initial number of bytes required */
+    vlen_bufsize.common.size = 0;
+
+    /* Call H5S_select_iterate with args, etc. */
+    dset_op.op_type             = H5S_SEL_ITER_OP_APP;
+    dset_op.u.app_op.op         = H5D__vlen_get_buf_size_gen_cb;
+    dset_op.u.app_op.type_id    = type_id;
+
+    ret_value = H5S_select_iterate(&bogus, type, space, &dset_op, &vlen_bufsize);
+
+    /* Get the size if we succeeded */
+    if(ret_value >= 0)
+        *size = vlen_bufsize.common.size;
+
+done:
+    if(vlen_bufsize.fspace_id >= 0) {
+        if(H5I_dec_app_ref(vlen_bufsize.fspace_id) < 0)
+            HDONE_ERROR(H5E_DATASET, H5E_CANTDEC, FAIL, "problem freeing id")
+        vlen_bufsize.fspace = NULL;
+    } /* end if */
+    if(vlen_bufsize.fspace && H5S_close(vlen_bufsize.fspace) < 0)
+        HDONE_ERROR(H5E_DATASET, H5E_CANTRELEASE, FAIL, "unable to release dataspace")
+    if(vlen_bufsize.mspace_id >= 0) {
+        if(H5I_dec_app_ref(vlen_bufsize.mspace_id) < 0)
+            HDONE_ERROR(H5E_DATASET, H5E_CANTDEC, FAIL, "problem freeing id")
+        mspace = NULL;
+    } /* end if */
+    if(mspace && H5S_close(mspace) < 0)
+        HDONE_ERROR(H5E_DATASET, H5E_CANTRELEASE, FAIL, "unable to release dataspace")
+    if(vlen_bufsize.common.fl_tbuf != NULL)
+        vlen_bufsize.common.fl_tbuf = H5FL_BLK_FREE(vlen_fl_buf, vlen_bufsize.common.fl_tbuf);
+    if(vlen_bufsize.common.vl_tbuf != NULL)
+        vlen_bufsize.common.vl_tbuf = H5FL_BLK_FREE(vlen_vl_buf, vlen_bufsize.common.vl_tbuf);
+    if(vlen_bufsize.dxpl_id != H5I_INVALID_HID) {
+        if(H5I_dec_app_ref(vlen_bufsize.dxpl_id) < 0)
+            HDONE_ERROR(H5E_DATASET, H5E_CANTDEC, FAIL, "can't close property list")
+        dxpl = NULL;
+    } /* end if */
+    if(dxpl && H5P_close(dxpl) < 0)
+        HDONE_ERROR(H5E_DATASET, H5E_CANTRELEASE, FAIL, "unable to release DXPL")
+
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__vlen_get_buf_size_gen() */
+
+
+/*-------------------------------------------------------------------------
  * Function: H5D__check_filters
  *
  * Purpose:  Check if the filters have be initialized for the dataset