[svn-r19174] Bug fix for #1239 - The filter's public function CAN_APPLY should return htri_t not

herr_t.  To minimize the change of the library's behavior, in the function 
H5Z_prelude_callback of H5Z.c, if the return value of can_apply is FALSE and
the filter is MANDATE, this function returns a FAILURE.  If the return value is FALSE 
but the filter is OPTIONAL, this function returns a SUCCEED.  During the IO, the filter 
will fail and return a size of zero.  But the pipeline will skip this filter. 

Tested on jam, lnew, and amani.  Tested on jam with szip.

1 files changed, 10 insertions, 11 deletions

diff --git a/src/H5Z.c b/src/H5Z.c
index c295df3..ee46941 100644
--- a/src/H5Z.c
+++ b/src/H5Z.c
@@ -498,7 +498,7 @@ H5Z_prelude_callback(const H5O_pline_t *pline, hid_t dcpl_id, hid_t type_id,
 {
     H5Z_class2_t    *fclass;                /* Individual filter information */
     size_t          u;                      /* Local index variable */
-    herr_t          ret_value = SUCCEED;    /* Return value */
+    htri_t          ret_value = TRUE;    /* Return value */
 
     FUNC_ENTER_NOAPI_NOINIT(H5Z_prelude_callback)
 
@@ -526,17 +526,16 @@ H5Z_prelude_callback(const H5O_pline_t *pline, hid_t dcpl_id, hid_t type_id,
                     /* Check if there is a "can apply" callback */
                     if(fclass->can_apply) {
                         /* Make callback to filter's "can apply" function */
-                        herr_t status = (fclass->can_apply)(dcpl_id, type_id, space_id);
+                        htri_t status = (fclass->can_apply)(dcpl_id, type_id, space_id);
 
-                        /* Check return value */
-                        if(status <= 0) {
-                            /* Indicate filter can't apply to this combination of parameters */
-                            if(status == 0)
-                                HGOTO_ERROR(H5E_PLINE, H5E_CANAPPLY, FAIL, "filter parameters not appropriate")
-                            /* Indicate error during filter callback */
-                            else
-                                HGOTO_ERROR(H5E_PLINE, H5E_CANAPPLY, FAIL, "error during user callback")
-                        } /* end if */
+                        /* Indicate error during filter callback */
+                        if(status < 0)
+                            HGOTO_ERROR(H5E_PLINE, H5E_CANAPPLY, FAIL, "error during user callback")
+
+                        /* Indicate filter can't apply to this combination of parameters.  
+                         * If the filter is NOT optional, returns failure. */
+                        if(status == FALSE && !(pline->filter[u].flags & H5Z_FLAG_OPTIONAL))
+                            HGOTO_ERROR(H5E_PLINE, H5E_CANAPPLY, FAIL, "filter parameters not appropriate")
                     } /* end if */
                     break;