[svn-r23656] Issue 8380 H5Zunregister caused seg fault: I added some protection measures:

  1. If any opened object uses the filter, let it fail.
  2. Flush all opened files to push any cached data to files.

Tested with h5committest.

3 files changed, 275 insertions, 7 deletions

diff --git a/test/Makefile.am b/test/Makefile.am
index ac5d169..5e476d3 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -46,7 +46,7 @@ TEST_PROG= testhdf5 lheap ohdr stab gheap cache cache_api cache_tagging \
            big mtime fillval mount flush1 flush2 app_ref enum \
            set_extent ttsafe enc_dec_plist enc_dec_plist_with_endianess\
            getname vfd ntypes dangle dtransform reserved cross_read \
-           freespace mf farray earray btree2 fheap file_image
+           freespace mf farray earray btree2 fheap file_image unregister
 
 # List programs to be built when testing here. error_test and err_compat are
 # built at the same time as the other tests, but executed by testerror.sh.
@@ -150,7 +150,7 @@ CHECK_CLEANFILES+=accum.h5 cmpd_dset.h5 compact_dataset.h5 dataset.h5 dset_offse
     earray.h5 efc[0-5].h5 log_vfd_out.log                            \
     new_multi_file_v16-r.h5 new_multi_file_v16-s.h5                  \
     split_get_file_image_test-m.h5 split_get_file_image_test-r.h5    \
-    file_image_core_test.h5.copy
+    file_image_core_test.h5.copy unregister_filter_1.h5 unregister_filter_2.h5
 
 # Sources for testhdf5 executable
 testhdf5_SOURCES=testhdf5.c tarray.c tattr.c tchecksum.c tconfig.c tfile.c \
diff --git a/test/Makefile.in b/test/Makefile.in
index 65c0fca..e9e2faa 100644
--- a/test/Makefile.in
+++ b/test/Makefile.in
@@ -165,7 +165,8 @@ am__EXEEXT_1 = testhdf5$(EXEEXT) lheap$(EXEEXT) ohdr$(EXEEXT) \
 	vfd$(EXEEXT) ntypes$(EXEEXT) dangle$(EXEEXT) \
 	dtransform$(EXEEXT) reserved$(EXEEXT) cross_read$(EXEEXT) \
 	freespace$(EXEEXT) mf$(EXEEXT) farray$(EXEEXT) earray$(EXEEXT) \
-	btree2$(EXEEXT) fheap$(EXEEXT) file_image$(EXEEXT)
+	btree2$(EXEEXT) fheap$(EXEEXT) file_image$(EXEEXT) \
+	unregister$(EXEEXT)
 @HAVE_SHARED_CONDITIONAL_TRUE@am__EXEEXT_2 = plugin$(EXEEXT)
 am__EXEEXT_3 = gen_bad_ohdr$(EXEEXT) gen_bogus$(EXEEXT) \
 	gen_cross$(EXEEXT) gen_deflate$(EXEEXT) gen_filters$(EXEEXT) \
@@ -484,6 +485,10 @@ unlink_SOURCES = unlink.c
 unlink_OBJECTS = unlink.$(OBJEXT)
 unlink_LDADD = $(LDADD)
 unlink_DEPENDENCIES = libh5test.la $(LIBHDF5)
+unregister_SOURCES = unregister.c
+unregister_OBJECTS = unregister.$(OBJEXT)
+unregister_LDADD = $(LDADD)
+unregister_DEPENDENCIES = libh5test.la $(LIBHDF5)
 vfd_SOURCES = vfd.c
 vfd_OBJECTS = vfd.$(OBJEXT)
 vfd_LDADD = $(LDADD)
@@ -539,7 +544,7 @@ SOURCES = $(libdynlib1_la_SOURCES) $(libdynlib2_la_SOURCES) \
 	mount.c mtime.c ntypes.c objcopy.c ohdr.c plugin.c pool.c \
 	reserved.c set_extent.c space_overflow.c stab.c \
 	tcheck_version.c $(testhdf5_SOURCES) testmeta.c \
-	$(ttsafe_SOURCES) unlink.c vfd.c
+	$(ttsafe_SOURCES) unlink.c unregister.c vfd.c
 DIST_SOURCES = $(am__libdynlib1_la_SOURCES_DIST) \
 	$(am__libdynlib2_la_SOURCES_DIST) \
 	$(am__libdynlib3_la_SOURCES_DIST) $(libh5test_la_SOURCES) \
@@ -558,7 +563,7 @@ DIST_SOURCES = $(am__libdynlib1_la_SOURCES_DIST) \
 	mount.c mtime.c ntypes.c objcopy.c ohdr.c plugin.c pool.c \
 	reserved.c set_extent.c space_overflow.c stab.c \
 	tcheck_version.c $(testhdf5_SOURCES) testmeta.c \
-	$(ttsafe_SOURCES) unlink.c vfd.c
+	$(ttsafe_SOURCES) unlink.c unregister.c vfd.c
 am__can_run_installinfo = \
   case $$AM_UPDATE_INFO_DIR in \
     n|no|NO) false;; \
@@ -879,7 +884,8 @@ CHECK_CLEANFILES = *.chkexe *.chklog *.clog accum.h5 cmpd_dset.h5 \
 	objcopy_ext.dat trefer1.h5 trefer2.h5 app_ref.h5 farray.h5 \
 	earray.h5 efc[0-5].h5 log_vfd_out.log new_multi_file_v16-r.h5 \
 	new_multi_file_v16-s.h5 split_get_file_image_test-m.h5 \
-	split_get_file_image_test-r.h5 file_image_core_test.h5.copy
+	split_get_file_image_test-r.h5 file_image_core_test.h5.copy \
+	unregister_filter_1.h5 unregister_filter_2.h5
 INCLUDES = -I$(top_srcdir)/src -I$(top_builddir)/src
 
 # Test script for error_test and err_compat
@@ -902,7 +908,7 @@ TEST_PROG = testhdf5 lheap ohdr stab gheap cache cache_api cache_tagging \
            big mtime fillval mount flush1 flush2 app_ref enum \
            set_extent ttsafe enc_dec_plist enc_dec_plist_with_endianess\
            getname vfd ntypes dangle dtransform reserved cross_read \
-           freespace mf farray earray btree2 fheap file_image
+           freespace mf farray earray btree2 fheap file_image unregister
 
 
 # These programs generate test files for the tests.  They don't need to be
@@ -1303,6 +1309,9 @@ ttsafe$(EXEEXT): $(ttsafe_OBJECTS) $(ttsafe_DEPENDENCIES) $(EXTRA_ttsafe_DEPENDE
 unlink$(EXEEXT): $(unlink_OBJECTS) $(unlink_DEPENDENCIES) $(EXTRA_unlink_DEPENDENCIES) 
 	@rm -f unlink$(EXEEXT)
 	$(AM_V_CCLD)$(LINK) $(unlink_OBJECTS) $(unlink_LDADD) $(LIBS)
+unregister$(EXEEXT): $(unregister_OBJECTS) $(unregister_DEPENDENCIES) $(EXTRA_unregister_DEPENDENCIES) 
+	@rm -f unregister$(EXEEXT)
+	$(AM_V_CCLD)$(LINK) $(unregister_OBJECTS) $(unregister_LDADD) $(LIBS)
 vfd$(EXEEXT): $(vfd_OBJECTS) $(vfd_DEPENDENCIES) $(EXTRA_vfd_DEPENDENCIES) 
 	@rm -f vfd$(EXEEXT)
 	$(AM_V_CCLD)$(LINK) $(vfd_OBJECTS) $(vfd_LDADD) $(LIBS)
@@ -1421,6 +1430,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tvlstr.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tvltypes.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/unlink.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/unregister.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/vfd.Po@am__quote@
 
 .c.o:
diff --git a/test/unregister.c b/test/unregister.c
new file mode 100644
index 0000000..88f4d1b
--- /dev/null
+++ b/test/unregister.c
@@ -0,0 +1,258 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group.                                               *
+ * All rights reserved.                                                      *
+ *                                                                           *
+ * This file is part of HDF5. The full HDF5 copyright notice, including      *
+ * terms governing use, modification, and redistribution, is contained in    *
+ * the files COPYING and Copyright.html.  COPYING can be found at the root   *
+ * of the source code distribution tree; Copyright.html can be found at the  *
+ * root level of an installed copy of the electronic document set and is     *
+ * linked from the top-level documents page.  It can also be found at        *
+ * http://hdfgroup.org/HDF5/doc/Copyright.html.  If you do not have access   *
+ * to either file, you may request a copy from help@hdfgroup.org.            *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+/*
+ * Programmer:	Raymond Lu
+ *              24 April 2013
+ *
+ * Purpose:	Tests H5Zunregister function
+ */
+#include "h5test.h"
+
+const char *FILENAME[] = {
+    "unregister_filter_1",
+    "unregister_filter_2",
+    NULL
+};
+
+#define GROUP_NAME              "group"
+#define DSET_NAME               "dataset"
+#define FILENAME_BUF_SIZE       1024
+#define DSET_DIM1               100
+#define DSET_DIM2               200
+#define FILTER_CHUNK_DIM1       2
+#define FILTER_CHUNK_DIM2       25
+#define GROUP_ITERATION         1000
+
+#define H5Z_FILTER_DUMMY        312
+
+static size_t filter_dummy(unsigned int flags, size_t cd_nelmts,
+    const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf);
+
+/* Dummy filter for test_unregister_filters only */
+const H5Z_class2_t H5Z_DUMMY[1] = {{
+    H5Z_CLASS_T_VERS,           /* H5Z_class_t version */
+    H5Z_FILTER_DUMMY,		/* Filter id number		*/
+    1, 1,                       /* Encoding and decoding enabled */
+    "dummy",			/* Filter name for debugging	*/
+    NULL,                       /* The "can apply" callback     */
+    NULL,                       /* The "set local" callback     */
+    filter_dummy,		/* The actual filter function	*/
+}};
+
+
+/*-------------------------------------------------------------------------
+ * Function:	filter_dummy
+ *
+ * Purpose:	A dummy compression method that doesn't do anything.  This
+ *              filter is only for test_unregister_filters.  Please don't 
+ *              use it for other tests because it may mess up this test.
+ *
+ * Return:	Success:	Data chunk size
+ *
+ *		Failure:	0
+ *
+ * Programmer:	Raymond Lu
+ *              April 24, 2013
+ *
+ *-------------------------------------------------------------------------
+ */
+static size_t
+filter_dummy(unsigned int UNUSED flags, size_t UNUSED cd_nelmts,
+      const unsigned int UNUSED *cd_values, size_t nbytes,
+      size_t UNUSED *buf_size, void UNUSED **buf)
+{
+    return nbytes;
+}
+
+/*-------------------------------------------------------------------------
+ * Function:	test_unregister_filters
+ *
+ * Purpose:	Tests unregistering filter before closing the file
+ *
+ * Return:	Success:	0
+ *		Failure:	-1
+ *
+ * Programmer:	Raymond Lu
+ *              11 April 2013
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+test_unregister_filters(hid_t my_fapl)
+{
+    hid_t	file1, file2;
+    hid_t	dc;
+    hid_t	gcpl, gid, group;
+    hid_t       dataset, space;
+    int		i, j, n;
+    char        gname[256];
+    char        filename[FILENAME_BUF_SIZE];
+    const hsize_t chunk_size[2] = {FILTER_CHUNK_DIM1, FILTER_CHUNK_DIM2};  /* Chunk dimensions */
+    hsize_t	dims[2];
+    int	        points[DSET_DIM1][DSET_DIM2];
+    herr_t      ret;
+
+    TESTING("Unregistering filter");
+
+    /* Create first file */
+    h5_fixname(FILENAME[0], my_fapl, filename, sizeof filename);
+    if((file1 = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, my_fapl)) < 0) goto error;
+
+    /* Create second file */
+    h5_fixname(FILENAME[1], my_fapl, filename, sizeof filename);
+    if((file2 = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, my_fapl)) < 0) goto error;
+
+    /* Register DUMMY filter */
+    if(H5Zregister(H5Z_DUMMY) < 0) goto error;
+
+    if(H5Zfilter_avail(H5Z_FILTER_DUMMY)!=TRUE) goto error;
+
+    if((gcpl = H5Pcreate(H5P_GROUP_CREATE)) < 0) goto error;
+  
+    /* Use DUMMY filter for creating groups */ 
+    if(H5Pset_filter (gcpl, H5Z_FILTER_DUMMY, H5Z_FLAG_MANDATORY, (size_t)0, NULL) < 0) goto error;
+
+    /* Create a group using this filter */
+    if((gid = H5Gcreate2(file1, GROUP_NAME, H5P_DEFAULT, gcpl, H5P_DEFAULT)) < 0) goto error;
+
+    /* Create multiple groups under the main group */
+    for (i=0; i < GROUP_ITERATION; i++) {
+        sprintf(gname, "group_%d", i);
+        if((group = H5Gcreate2(gid, gname, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) goto error;
+        if(H5Gclose(group) < 0) goto error; 
+    }
+
+    if(H5Fflush(file1, H5F_SCOPE_GLOBAL) < 0) goto error;
+
+    /* Unregister the filter before closing the group.  It should fail */
+    H5E_BEGIN_TRY {
+        ret = H5Zunregister(H5Z_FILTER_DUMMY);
+    } H5E_END_TRY;
+    if(ret>=0) {
+        H5_FAILED();
+        printf("    Line %d: Should not be able to unregister filter\n", __LINE__);
+        goto error;
+    } /* end if */
+
+
+    /* Close the group */
+    if(H5Gclose(gid) < 0) goto error;
+
+    /* Clean up objects used for this test */
+    if(H5Pclose (gcpl) < 0) goto error;
+
+    if((dc = H5Pcreate(H5P_DATASET_CREATE)) < 0) goto error;
+    if(H5Pset_chunk (dc, 2, chunk_size) < 0) goto error;
+
+    if(H5Pset_filter(dc, H5Z_FILTER_DUMMY, 0, (size_t)0, NULL) < 0) goto error;
+
+    /* Initialize the dataset */
+    for(i = n = 0; i < DSET_DIM1; i++)
+        for(j = 0; j < DSET_DIM2; j++)
+            points[i][j] = n++;
+
+    /* Create the data space */
+    dims[0] = DSET_DIM1;
+    dims[1] = DSET_DIM2;
+    if((space = H5Screate_simple(2, dims, NULL)) < 0) goto error;
+
+    /* Create a dataset in the first file */
+    if((dataset = H5Dcreate2(file1, DSET_NAME, H5T_NATIVE_INT, space,
+                             H5P_DEFAULT, dc, H5P_DEFAULT)) < 0) goto error;
+
+    /* Write the data to the dataset */
+    if(H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, points) < 0)
+        goto error;
+
+    /* Unregister the filter before closing the dataset.  It should fail */
+    /*if(H5Zunregister(H5Z_FILTER_DUMMY) < 0) goto error;*/
+    H5E_BEGIN_TRY {
+        ret = H5Zunregister(H5Z_FILTER_DUMMY);
+    } H5E_END_TRY;
+    if(ret>=0) {
+        H5_FAILED();
+        printf("    Line %d: Should not be able to unregister filter\n", __LINE__);
+        goto error;
+    } /* end if */
+
+    if(H5Dclose(dataset) < 0) goto error;
+
+    /* Create a dataset in the second file */
+    if((dataset = H5Dcreate2(file2, DSET_NAME, H5T_NATIVE_INT, space,
+                             H5P_DEFAULT, dc, H5P_DEFAULT)) < 0) goto error;
+
+    /* Write the data to the dataset */
+    if(H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, points) < 0)
+        goto error;
+
+    if(H5Dclose(dataset) < 0) goto error;
+
+    /* Unregister the filter after closing all objects but before closing files. 
+     * It should flush all files. */
+    if(H5Zunregister(H5Z_FILTER_DUMMY) < 0) goto error;
+
+    /* Clean up objects used for this test */
+    if(H5Pclose (dc) < 0) goto error;
+    if(H5Fclose(file1) < 0) goto error;
+    if(H5Fclose(file2) < 0) goto error;
+
+    PASSED();
+    return 0;
+
+error:
+    return -1;
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function:	main
+ *
+ * Purpose:	Tests unregistering filter with H5Zunregister
+ *
+ * Return:	Success:	exit(0)
+ *
+ *		Failure:	exit(1)
+ *
+ * Programmer:	Raymond Lu
+ *              11 April 2013
+ *
+ *-------------------------------------------------------------------------
+ */
+int
+main(void)
+{
+    hid_t		fapl;
+    int	nerrors = 0;
+
+    /* Testing setup */
+    h5_reset();
+    fapl = h5_fileaccess();
+
+    /* Test unregistering filter in its own file */
+    nerrors += (test_unregister_filters(fapl) < 0           ? 1 : 0);
+
+    if(nerrors)
+        goto error;
+    printf("All dataset tests passed.\n");
+    h5_cleanup(FILENAME, fapl);
+
+    return 0;
+
+error:
+    nerrors = MAX(1, nerrors);
+    printf("***** %d DATASET TEST%s FAILED! *****\n",
+            nerrors, 1 == nerrors ? "" : "S");
+    return 1;
+}
+