Fixed HDFFV-10933

Description:
    Updated the original fix by Kent Y. in commit
        200a77d8c3e51663c375aafffff607ae9b438f4e
    - used internal functions instead of public API
    - moved some code into the subroutine for a cleaner look.
    - added test to dsets.c
Platforms tested:
    Linux/64 (jelly)

3 files changed, 91 insertions, 14 deletions

diff --git a/src/H5Dint.c b/src/H5Dint.c
index 079fef7..f99aabb 100644
--- a/src/H5Dint.c
+++ b/src/H5Dint.c
@@ -1299,18 +1299,24 @@ H5D__create(H5F_t *file, hid_t type_id, const H5S_t *space, hid_t dcpl_id,
 
     /* Check if the dataset has a non-default DCPL & get important values, if so */
     if(new_dset->shared->dcpl_id != H5P_DATASET_CREATE_DEFAULT) {
-        H5O_layout_t    *layout;        /* Dataset's layout information */
-        H5O_pline_t     *pline;         /* Dataset's I/O pipeline information */
-        H5O_fill_t      *fill;          /* Dataset's fill value info */
-        H5O_efl_t       *efl;           /* Dataset's external file list info */
+        H5O_layout_t *layout;        /* Dataset's layout information */
+        H5O_pline_t  *pline;         /* Dataset's I/O pipeline information */
+        H5O_fill_t   *fill;          /* Dataset's fill value info */
+        H5O_efl_t    *efl;           /* Dataset's external file list info */
+        htri_t       ignore_filters = FALSE; /* Ignore optional filters or not */
 
-        /* Check if the filters in the DCPL can be applied to this dataset */
-        if(H5Z_can_apply(new_dset->shared->dcpl_id, new_dset->shared->type_id) < 0)
-            HGOTO_ERROR(H5E_ARGS, H5E_CANTINIT, NULL, "I/O filters can't operate on this dataset")
+        if((ignore_filters = H5Z_ignore_filters(new_dset->shared->dcpl_id, dt, space))<0)
+            HGOTO_ERROR(H5E_ARGS, H5E_CANTINIT, NULL, "H5Z_has_optional_filter() failed")
 
-        /* Make the "set local" filter callbacks for this dataset */
-        if(H5Z_set_local(new_dset->shared->dcpl_id, new_dset->shared->type_id) < 0)
-            HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to set local filter parameters")
+        if(FALSE == ignore_filters) {
+            /* Check if the filters in the DCPL can be applied to this dataset */
+            if(H5Z_can_apply(new_dset->shared->dcpl_id, new_dset->shared->type_id) < 0)
+                HGOTO_ERROR(H5E_ARGS, H5E_CANTINIT, NULL, "I/O filters can't operate on this dataset")
+
+            /* Make the "set local" filter callbacks for this dataset */
+            if(H5Z_set_local(new_dset->shared->dcpl_id, new_dset->shared->type_id) < 0)
+                HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to set local filter parameters")
+        } /* ignore_filters */
 
         /* Get new dataset's property list object */
         if(NULL == (dc_plist = (H5P_genplist_t *)H5I_object(new_dset->shared->dcpl_id)))
@@ -1334,9 +1340,11 @@ H5D__create(H5F_t *file, hid_t type_id, const H5S_t *space, hid_t dcpl_id,
             HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, NULL, "can't retrieve external file list")
         efl_copied = TRUE;
 
-        /* Check that chunked layout is used if filters are enabled */
-        if(pline->nused > 0 && H5D_CHUNKED != layout->type)
-            HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, NULL, "filters can only be used with chunked layout")
+        if(FALSE == ignore_filters) {
+            /* Check that chunked layout is used if filters are enabled */
+            if(pline->nused > 0 && H5D_CHUNKED != layout->type)
+                HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, NULL, "filters can only be used with chunked layout")
+        }
 
         /* Check if the alloc_time is the default and error out */
         if(fill->alloc_time == H5D_ALLOC_TIME_DEFAULT)
diff --git a/src/H5Z.c b/src/H5Z.c
index 616db1f..66acfff 100644
--- a/src/H5Z.c
+++ b/src/H5Z.c
@@ -1004,6 +1004,73 @@ done:
 
 
 /*-------------------------------------------------------------------------
+ * Function: H5Z_ignore_filters
+ *
+ * Purpose:  Determine whether filters can be ignored.
+ *
+ * Description:
+ *      When the filters are optional (i.e., H5Z_FLAG_OPTIONAL is provided,)
+ *      if any of the following conditions is met, the filters will be ignored:
+ *          - dataspace is either H5S_NULL or H5S_SCALAR
+ *          - datatype is variable-length (string or non-string)
+ *      However, if any of these conditions exists and a filter is not
+ *      optional, the function will produce an error.
+ *
+ * Return:   Non-negative(TRUE/FALSE) on success
+ *           Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+htri_t
+H5Z_ignore_filters(hid_t dcpl_id, const H5T_t *type, const H5S_t *space)
+{
+    H5P_genplist_t  *dc_plist;   /* Dataset creation property list object */
+    H5O_pline_t     pline;       /* Object's I/O pipeline information */
+    H5S_class_t     space_class;            /* To check class of space */
+    H5T_class_t     type_class;             /* To check if type is VL */
+    bool            bad_for_filters = FALSE;/* Suitable to have filters */
+    htri_t          ret_value = FALSE;      /* TRUE for ignoring filters */
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    if (NULL == (dc_plist = (H5P_genplist_t *)H5I_object(dcpl_id)))
+        HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get dataset creation property list")
+
+    /* Get pipeline information */
+    if (H5P_peek(dc_plist, H5O_CRT_PIPELINE_NAME, &pline) < 0)
+        HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "can't retrieve pipeline filter")
+
+    /* Get datatype and dataspace classes for quick access */
+    space_class = H5S_GET_EXTENT_TYPE(space);
+    type_class = H5T_get_class(type, FALSE);
+
+    /* These conditions are not suitable for filters */
+    bad_for_filters = (H5S_NULL == space_class || H5S_SCALAR == space_class
+        || H5T_VLEN == type_class
+        || (H5T_STRING == type_class && TRUE == H5T_is_variable_str(type)));
+
+    /* When these conditions occur, if there are required filters in pline,
+       then report a failure, otherwise, set flag that they can be ignored */
+    if (bad_for_filters) {
+        size_t ii;
+        if (pline.nused > 0) {
+            for (ii = 0; ii < pline.nused; ii++)
+            {
+                if (!(pline.filter[ii].flags & H5Z_FLAG_OPTIONAL))
+                    HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "not suitable for filters")
+            }
+
+            /* All filters are optional, we can ignore them */
+            ret_value = TRUE;
+        }
+    } /* bad for filters */
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5Z_ignore_filters() */
+
+
+/*-------------------------------------------------------------------------
  * Function: H5Z_modify
  *
  * Purpose:  Modify filter parameters for specified pipeline.
diff --git a/src/H5Zprivate.h b/src/H5Zprivate.h
index 6868da6..186c168 100644
--- a/src/H5Zprivate.h
+++ b/src/H5Zprivate.h
@@ -25,7 +25,8 @@ typedef struct H5Z_filter_info_t H5Z_filter_info_t;
 #include "H5Zpublic.h"
 
 /* Private headers needed by this file */
-#include "H5Tprivate.h"		/* Datatypes				*/
+#include "H5Tprivate.h"     /* Datatypes				*/
+#include "H5Sprivate.h"     /* Dataspace                */
 
 /**************************/
 /* Library Private Macros */
@@ -89,6 +90,7 @@ H5_DLL herr_t H5Z_can_apply(hid_t dcpl_id, hid_t type_id);
 H5_DLL herr_t H5Z_set_local(hid_t dcpl_id, hid_t type_id);
 H5_DLL herr_t H5Z_can_apply_direct(const struct H5O_pline_t *pline);
 H5_DLL herr_t H5Z_set_local_direct(const struct H5O_pline_t *pline);
+H5_DLL htri_t H5Z_ignore_filters(hid_t dcpl_id, const H5T_t *type, const H5S_t *space);
 H5_DLL H5Z_filter_info_t *H5Z_filter_info(const struct H5O_pline_t *pline,
         H5Z_filter_t filter);
 H5_DLL htri_t H5Z_filter_in_pline(const struct H5O_pline_t *pline, H5Z_filter_t filter);