[svn-r23685] Issue 8380 H5Zunregister caused seg fault. I simplied the iteration process over datasets, groups, and files as Quincey suggested. I also removed H5I_search that I introduced in last commit but didn't actually used it.

Tested on jam and koala - simple change. 

4 files changed, 58 insertions, 90 deletions

diff --git a/src/H5I.c b/src/H5I.c
index 6a8c269..581356e 100644
--- a/src/H5I.c
+++ b/src/H5I.c
@@ -2002,55 +2002,21 @@ H5Isearch(H5I_type_t type, H5I_search_func_t func, void *key)
     if(H5I_IS_LIB_TYPE(type))
         HGOTO_ERROR(H5E_ATOM, H5E_BADGROUP, NULL, "cannot call public function on library type")
 
-    if((ret_value = H5I_search(type, func, key)) == NULL)
-        HGOTO_ERROR(H5E_ATOM, H5E_BADGROUP, NULL, "cannot call public function on library type")
-
-done:
-    FUNC_LEAVE_API(ret_value)
-} /* end H5Isearch() */
-
-
-/*-------------------------------------------------------------------------
- * Function:	H5I_search
- *
- * Purpose:	Private function for H5Isearch
- *
- * Return:	Success:	The first object in the type for which FUNC
- *				returns non-zero. NULL if FUNC returned zero
- *				for every object in the type.
- *
- *		Failure:	NULL
- *
- * Programmer:	James Laird
- *		Nathaniel Furrer
- *		Friday, April 23, 2004
- *
- *-------------------------------------------------------------------------
- */
-void *
-H5I_search(H5I_type_t _type, H5I_search_func_t _func, void *_key)
-{
-    H5I_search_ud_t udata;      /* Context for iteration */
-    void *ret_value;            /* Return value */
-
-    FUNC_ENTER_NOAPI(NULL)
-
     /* Set up udata struct */
-    udata.app_cb = _func;
-    udata.app_key = _key;
+    udata.app_cb = func;
+    udata.app_key = key;
     udata.ret_obj = NULL;
 
     /* Note that H5I_iterate returns an error code.  We ignore it 
      * here, as we can't do anything with it without revising the API.
      */
-    if(H5I_iterate(_type, H5I_search_cb, &udata, TRUE) < 0)
-        HGOTO_ERROR(H5E_ATOM, H5E_BADGROUP, NULL, "cannot call public function on library type")
+    H5I_iterate(type, H5I_search_cb, &udata, TRUE);
 
     /* Set return value */
     ret_value = udata.ret_obj;
 
 done:
-    FUNC_LEAVE_NOAPI(ret_value)
+    FUNC_LEAVE_API(ret_value)
 } /* end H5Isearch() */
 
 
diff --git a/src/H5Iprivate.h b/src/H5Iprivate.h
index 28b6327..ff1463d 100644
--- a/src/H5Iprivate.h
+++ b/src/H5Iprivate.h
@@ -78,7 +78,6 @@ H5_DLL H5I_type_t H5I_get_type(hid_t id);
 H5_DLL hid_t H5I_get_file_id(hid_t obj_id, hbool_t app_ref);
 H5_DLL void *H5I_remove(hid_t id);
 H5_DLL void *H5I_remove_verify(hid_t id, H5I_type_t id_type);
-H5_DLL void *H5I_search(H5I_type_t _type, H5I_search_func_t _func, void *_key);
 H5_DLL herr_t H5I_iterate(H5I_type_t type, H5I_search_func_t func, void *udata, hbool_t app_ref);
 H5_DLL int H5I_get_ref(hid_t id, hbool_t app_ref);
 H5_DLL int H5I_inc_ref(hid_t id, hbool_t app_ref);
diff --git a/src/H5Z.c b/src/H5Z.c
index 1652a7d..5861e0e 100644
--- a/src/H5Z.c
+++ b/src/H5Z.c
@@ -71,6 +71,7 @@ static H5Z_stats_t	*H5Z_stat_table_g = NULL;
 /* Local functions */
 static int H5Z_find_idx(H5Z_filter_t id);
 static int H5Z_get_object_cb(void *obj_ptr, hid_t obj_id, void *key);
+static int H5Z_get_file_cb(void *obj_ptr, hid_t obj_id, void *key);
 
 
 /*-------------------------------------------------------------------------
@@ -415,8 +416,6 @@ done:
 herr_t
 H5Z_unregister (H5Z_filter_t filter_id)
 {
-    size_t       num_obj_id = 0;
-    size_t       num_file_id = 0;
     size_t       filter_index;        /* Local index variable for filter */
     H5Z_object_t object;
     herr_t       ret_value=SUCCEED;   /* Return value */
@@ -434,39 +433,27 @@ H5Z_unregister (H5Z_filter_t filter_id)
     if (filter_index>=H5Z_table_used_g)
         HGOTO_ERROR(H5E_PLINE, H5E_NOTFOUND, FAIL, "filter is not registered")
 
-    /* Count the number of opened datasets and groups among all opened files */
-    if(H5F_get_obj_count(NULL, H5F_OBJ_DATASET | H5F_OBJ_GROUP, FALSE, &num_obj_id) < 0)
-        HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get object number")
+    /* Initialize the structure object for iteration */
+    object.filter_id = filter_id;
+    object.found = FALSE;
 
-    if(num_obj_id) {
-        /* Initialize it here because it isn't needed when the object is file */
-        object.filter_id = filter_id;
-        object.found = FALSE;
+    /* Iterate through all opened datasets, returns a failure if any of them uses the filter */
+    if(H5I_iterate(H5I_DATASET, H5Z_get_object_cb, &object, FALSE) < 0)
+        HGOTO_ERROR(H5E_FILE, H5E_BADITER, FAIL, "iteration failed")
 
-        /* Iterate through all opened datasets, returns a failure if any of them uses the filter */
-        if(H5I_iterate(H5I_DATASET, H5Z_get_object_cb, &object, FALSE) < 0)
-            HGOTO_ERROR(H5E_FILE, H5E_BADITER, FAIL, "iteration failed")
+    if(object.found)
+        HGOTO_ERROR(H5E_PLINE, H5E_CANTRELEASE, FAIL, "can't unregister filter because a dataset is still using it")
 
-        if(object.found)
-            HGOTO_ERROR(H5E_PLINE, H5E_CANTRELEASE, FAIL, "can't unregister filter because a dataset is still using it")
+    /* Iterate through all opened groups, returns a failure if any of them uses the filter */
+    if(H5I_iterate(H5I_GROUP, H5Z_get_object_cb, &object, FALSE) < 0)
+        HGOTO_ERROR(H5E_FILE, H5E_BADITER, FAIL, "iteration failed")
 
-        /* Iterate through all opened groups, returns a failure if any of them uses the filter */
-        if(H5I_iterate(H5I_GROUP, H5Z_get_object_cb, &object, FALSE) < 0)
-            HGOTO_ERROR(H5E_FILE, H5E_BADITER, FAIL, "iteration failed")
+    if(object.found)
+        HGOTO_ERROR(H5E_PLINE, H5E_CANTRELEASE, FAIL, "can't unregister filter because a group is still using it")
 
-        if(object.found)
-            HGOTO_ERROR(H5E_PLINE, H5E_CANTRELEASE, FAIL, "can't unregister filter because a group is still using it")
-    }
-
-    /* Count the number of opened files */
-    if(H5F_get_obj_count(NULL, H5F_OBJ_FILE, FALSE, &num_file_id) < 0)
-        HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get file number")
-   
     /* Iterate through all opened files and flush them */
-    if(num_file_id) {
-        if(H5I_iterate(H5I_FILE, H5Z_get_object_cb, &object, FALSE) < 0)
-            HGOTO_ERROR(H5E_FILE, H5E_BADITER, FAIL, "iteration failed")
-    }
+    if(H5I_iterate(H5I_FILE, H5Z_get_file_cb, NULL, FALSE) < 0)
+        HGOTO_ERROR(H5E_FILE, H5E_BADITER, FAIL, "iteration failed")
 
     /* Remove filter from table */
     /* Don't worry about shrinking table size (for now) */
@@ -487,8 +474,7 @@ done:
  * Purpose:	The callback function for H5Z_unregister. It iterates 
  *              through all opened objects.  If the object is a dataset
  *              or a group and it uses the filter to be unregistered, the 
- *              function returns TRUE. If the object is a file, the function
- *              flushes it to the disk. 
+ *              function returns TRUE. 
  *
  * Return:      TRUE if the object uses the filter.
  *              FALSE otherwise.
@@ -516,23 +502,6 @@ H5Z_get_object_cb(void *obj_ptr, hid_t obj_id, void *key)
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "bad object id");
 
     switch(id_type) {
-        case H5I_FILE:
-        {
-            H5F_t    *f = NULL;         /* File to query */
-
-            if(NULL == (f = (H5F_t *)H5I_object_verify(obj_id, H5I_FILE)))
-                HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file")
-
-            /* Call the flush routine for mounted file hierarchies. Do a global flush 
-             * if the file is opened for write */
-            if(H5F_ACC_RDWR & H5F_INTENT(f)) {
-                if(H5F_flush_mounts(f, H5AC_dxpl_id) < 0)
-                    HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush file hierarchy")
-            } /* end if */
-
-            /* H5I_iterate expects FALSE to continue to other files */
-            HGOTO_DONE(FALSE) 
-        } 
         case H5I_GROUP:
             if((ocpl_id = H5G_get_create_plist(obj_ptr)) < 0) 
                 HGOTO_ERROR(H5E_ARGS, H5E_CANTGET, FAIL, "can't get group creation property list")
@@ -566,8 +535,42 @@ H5Z_get_object_cb(void *obj_ptr, hid_t obj_id, void *key)
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5F_get_objects_cb() */
+} /* end H5F_get_object_cb() */
+
+
+/*-------------------------------------------------------------------------
+ * Function:	H5Z_get_file_cb
+ *
+ * Purpose:	The callback function for H5Z_unregister. It iterates 
+ *              through all opened files and flush them. 
+ *
+ * Return:      FALSE if finishes flushing and moves on
+ *              FAIL if there is an error
+ *
+ * Programmer:  Raymond Lu
+ *              6 May 2013
+ *
+ *-------------------------------------------------------------------------
+ */
+static int
+H5Z_get_file_cb(void *obj_ptr, hid_t obj_id, void *key)
+{
+    int             ret_value = FALSE;    /* Return value */
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    HDassert(obj_ptr);
 
+    /* Call the flush routine for mounted file hierarchies. Do a global flush 
+     * if the file is opened for write */
+    if(H5F_ACC_RDWR & H5F_INTENT((H5F_t *)obj_ptr)) {
+        if(H5F_flush_mounts((H5F_t *)obj_ptr, H5AC_dxpl_id) < 0)
+            HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush file hierarchy")
+    } /* end if */
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5F_get_file_cb() */
 
 
 /*-------------------------------------------------------------------------
diff --git a/test/unregister.c b/test/unregister.c
index 297035d..383958d 100644
--- a/test/unregister.c
+++ b/test/unregister.c
@@ -242,14 +242,14 @@ main(void)
 
     if(nerrors)
         goto error;
-    printf("All dataset tests passed.\n");
+    printf("All filter unregistration tests passed.\n");
     h5_cleanup(FILENAME, fapl);
 
     return 0;
 
 error:
     nerrors = MAX(1, nerrors);
-    printf("***** %d DATASET TEST%s FAILED! *****\n",
+    printf("***** %d FILTER UNREGISTRATION TEST%s FAILED! *****\n",
             nerrors, 1 == nerrors ? "" : "S");
     return 1;
 }