[svn-r19174] Bug fix for #1239 - The filter's public function CAN_APPLY should return htri_t not

herr_t.  To minimize the change of the library's behavior, in the function 
H5Z_prelude_callback of H5Z.c, if the return value of can_apply is FALSE and
the filter is MANDATE, this function returns a FAILURE.  If the return value is FALSE 
but the filter is OPTIONAL, this function returns a SUCCEED.  During the IO, the filter 
will fail and return a size of zero.  But the pipeline will skip this filter. 

Tested on jam, lnew, and amani.  Tested on jam with szip.

5 files changed, 22 insertions, 23 deletions

diff --git a/src/H5Z.c b/src/H5Z.c
index c295df3..ee46941 100644
--- a/src/H5Z.c
+++ b/src/H5Z.c
@@ -498,7 +498,7 @@ H5Z_prelude_callback(const H5O_pline_t *pline, hid_t dcpl_id, hid_t type_id,
 {
     H5Z_class2_t    *fclass;                /* Individual filter information */
     size_t          u;                      /* Local index variable */
-    herr_t          ret_value = SUCCEED;    /* Return value */
+    htri_t          ret_value = TRUE;    /* Return value */
 
     FUNC_ENTER_NOAPI_NOINIT(H5Z_prelude_callback)
 
@@ -526,17 +526,16 @@ H5Z_prelude_callback(const H5O_pline_t *pline, hid_t dcpl_id, hid_t type_id,
                     /* Check if there is a "can apply" callback */
                     if(fclass->can_apply) {
                         /* Make callback to filter's "can apply" function */
-                        herr_t status = (fclass->can_apply)(dcpl_id, type_id, space_id);
+                        htri_t status = (fclass->can_apply)(dcpl_id, type_id, space_id);
 
-                        /* Check return value */
-                        if(status <= 0) {
-                            /* Indicate filter can't apply to this combination of parameters */
-                            if(status == 0)
-                                HGOTO_ERROR(H5E_PLINE, H5E_CANAPPLY, FAIL, "filter parameters not appropriate")
-                            /* Indicate error during filter callback */
-                            else
-                                HGOTO_ERROR(H5E_PLINE, H5E_CANAPPLY, FAIL, "error during user callback")
-                        } /* end if */
+                        /* Indicate error during filter callback */
+                        if(status < 0)
+                            HGOTO_ERROR(H5E_PLINE, H5E_CANAPPLY, FAIL, "error during user callback")
+
+                        /* Indicate filter can't apply to this combination of parameters.  
+                         * If the filter is NOT optional, returns failure. */
+                        if(status == FALSE && !(pline->filter[u].flags & H5Z_FLAG_OPTIONAL))
+                            HGOTO_ERROR(H5E_PLINE, H5E_CANAPPLY, FAIL, "filter parameters not appropriate")
                     } /* end if */
                     break;
 
diff --git a/src/H5Znbit.c b/src/H5Znbit.c
index 8f785a2..bcdd549 100644
--- a/src/H5Znbit.c
+++ b/src/H5Znbit.c
@@ -38,7 +38,7 @@ typedef struct {
 } parms_atomic;
 
 /* Local function prototypes */
-static herr_t H5Z_can_apply_nbit(hid_t dcpl_id, hid_t type_id, hid_t space_id);
+static htri_t H5Z_can_apply_nbit(hid_t dcpl_id, hid_t type_id, hid_t space_id);
 static herr_t H5Z_set_local_nbit(hid_t dcpl_id, hid_t type_id, hid_t space_id);
 static size_t H5Z_filter_nbit(unsigned flags, size_t cd_nelmts, const unsigned cd_values[],
                               size_t nbytes, size_t *buf_size, void **buf);
@@ -129,11 +129,11 @@ static unsigned parms_index = 0;
  *
  *-------------------------------------------------------------------------
  */
-static herr_t
+static htri_t
 H5Z_can_apply_nbit(hid_t UNUSED dcpl_id, hid_t type_id, hid_t UNUSED space_id)
 {
     const H5T_t	*type;                  /* Datatype */
-    herr_t ret_value = TRUE;            /* Return value */
+    htri_t ret_value = TRUE;            /* Return value */
 
     FUNC_ENTER_NOAPI(H5Z_can_apply_nbit, FAIL)
 
diff --git a/src/H5Zpublic.h b/src/H5Zpublic.h
index 44d2bbb..5d9b5ed 100644
--- a/src/H5Zpublic.h
+++ b/src/H5Zpublic.h
@@ -158,7 +158,7 @@ extern "C" {
  * The "can_apply" callback returns positive a valid combination, zero for an
  * invalid combination and negative for an error.
  */
-typedef herr_t (*H5Z_can_apply_func_t)(hid_t dcpl_id, hid_t type_id, hid_t space_id);
+typedef htri_t (*H5Z_can_apply_func_t)(hid_t dcpl_id, hid_t type_id, hid_t space_id);
 
 /*
  * After the "can_apply" callbacks are checked for new datasets, the "set_local"
diff --git a/src/H5Zscaleoffset.c b/src/H5Zscaleoffset.c
index 5737702..4bfc8b1 100644
--- a/src/H5Zscaleoffset.c
+++ b/src/H5Zscaleoffset.c
@@ -41,7 +41,7 @@ enum H5Z_scaleoffset_t {t_bad=0, t_uchar=1, t_ushort, t_uint, t_ulong, t_ulong_l
 
 /* Local function prototypes */
 static double H5Z_scaleoffset_rnd(double val);
-static herr_t H5Z_can_apply_scaleoffset(hid_t dcpl_id, hid_t type_id, hid_t space_id);
+static htri_t H5Z_can_apply_scaleoffset(hid_t dcpl_id, hid_t type_id, hid_t space_id);
 static enum H5Z_scaleoffset_t H5Z_scaleoffset_get_type(unsigned dtype_class,
     unsigned dtype_size, unsigned dtype_sign);
 static herr_t H5Z_scaleoffset_set_parms_fillval(H5P_genplist_t *dcpl_plist,
@@ -611,13 +611,13 @@ H5Z_class2_t H5Z_SCALEOFFSET[1] = {{
  *
  *-------------------------------------------------------------------------
  */
-static herr_t
+static htri_t
 H5Z_can_apply_scaleoffset(hid_t UNUSED dcpl_id, hid_t type_id, hid_t UNUSED space_id)
 {
     const H5T_t	*type;                  /* Datatype */
     H5T_class_t dtype_class;            /* Datatype's class */
     H5T_order_t dtype_order;            /* Datatype's endianness order */
-    herr_t ret_value = TRUE;            /* Return value */
+    htri_t ret_value = TRUE;            /* Return value */
 
     FUNC_ENTER_NOAPI(H5Z_can_apply_scaleoffset, FAIL)
 
@@ -640,9 +640,9 @@ H5Z_can_apply_scaleoffset(hid_t UNUSED dcpl_id, hid_t type_id, hid_t UNUSED spac
 
         /* Range check datatype's endianness order */
         if(dtype_order != H5T_ORDER_LE && dtype_order != H5T_ORDER_BE)
-            HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype endianness order")
+            HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FALSE, "bad datatype endianness order")
     } else
-        HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "datatype class not supported by scaleoffset")
+        HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FALSE, "datatype class not supported by scaleoffset")
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
diff --git a/src/H5Zszip.c b/src/H5Zszip.c
index 5da92ac..52f5e11 100644
--- a/src/H5Zszip.c
+++ b/src/H5Zszip.c
@@ -34,7 +34,7 @@
 #endif
 
 /* Local function prototypes */
-static herr_t H5Z_can_apply_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id);
+static htri_t H5Z_can_apply_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id);
 static herr_t H5Z_set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id);
 static size_t H5Z_filter_szip (unsigned flags, size_t cd_nelmts,
     const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf);
@@ -76,13 +76,13 @@ H5Z_class2_t H5Z_SZIP[1] = {{
  *
  *-------------------------------------------------------------------------
  */
-static herr_t
+static htri_t
 H5Z_can_apply_szip(hid_t UNUSED dcpl_id, hid_t type_id, hid_t UNUSED space_id)
 {
     const H5T_t	*type;                  /* Datatype */
     unsigned dtype_size;                /* Datatype's size (in bits) */
     H5T_order_t dtype_order;            /* Datatype's endianness order */
-    herr_t ret_value = TRUE;            /* Return value */
+    htri_t ret_value = TRUE;            /* Return value */
 
     FUNC_ENTER_NOAPI(H5Z_can_apply_szip, FAIL)