From 6d682a5a28f1ad599db5ad77e135b25a2efe7ae7 Mon Sep 17 00:00:00 2001 From: Nat Furrer Date: Mon, 28 Jun 2004 14:46:03 -0500 Subject: [svn-r8753] Purpose: Handled SZIP without the encoder present. Description: It is now an error for a user to try to create, extend, or write to a dataset without the encoder present in their SZIP library. Added H5Zget_filter_info to provide users with a way to query HDF5 about the presence (or lack thereof) of the SZIP encoder. Platforms tested: Windows Verbena Arabica Copper Misc. update: --- MANIFEST | 2 +- configure | 70 +++++++- configure.in | 28 ++++ fortran/src/H5Zf.c | 26 +++ fortran/src/H5Zff.f90 | 62 ++++++++ fortran/src/H5_f.c | 2 + fortran/src/H5f90global.f90 | 6 +- fortran/src/H5f90proto.h | 4 +- fortran/test/tH5Z.f90 | 31 +++- src/H5D.c | 53 +++++++ src/H5Dio.c | 24 +++ src/H5Dpkg.h | 1 + src/H5E.c | 3 +- src/H5Epublic.h | 4 +- src/H5MPprivate.h | 1 + src/H5Pdcpl.c | 8 + src/H5Z.c | 53 ++++++- src/H5Zpublic.h | 6 +- src/H5Zszip.c | 1 + src/H5config.h.in | 3 + test/dsets.c | 258 +++++++++++++++++++++++++++++- tools/h5dump/h5dumpgentest.c | 4 +- tools/h5repack/h5repack.sh | 178 --------------------- tools/h5repack/h5repack.sh.in | 291 ++++++++++++++++++++++++++++++++++ tools/h5repack/testh5repack_filters.c | 6 + tools/h5repack/testh5repack_main.c | 29 ++-- windows/all.zip | Bin 321438 -> 321478 bytes 27 files changed, 944 insertions(+), 210 deletions(-) delete mode 100644 tools/h5repack/h5repack.sh create mode 100755 tools/h5repack/h5repack.sh.in diff --git a/MANIFEST b/MANIFEST index ffe0b30..1d19cb1 100644 --- a/MANIFEST +++ b/MANIFEST @@ -1557,7 +1557,7 @@ ./tools/h5repack/testh5repack_layout.c ./tools/h5repack/h5repack_filters.c ./tools/h5repack/h5repack_main.c -./tools/h5repack/h5repack.sh +./tools/h5repack/h5repack.sh.in ./tools/h5repack/testh5repack_attr.c ./tools/h5repack/testh5repack_main.c ./tools/h5repack/h5repack.h diff --git a/configure b/configure index 9f40efe..da7b5b3 100755 --- a/configure +++ b/configure @@ -23598,6 +23598,7 @@ fi USE_FILTER_SZIP="no" + USE_FILTER_SZIP_ENCODER="no" # Check whether --with-szlib or --without-szlib was given. if test "${with_szlib+set}" = set; then @@ -24039,6 +24040,71 @@ _ACEOF USE_FILTER_SZIP="yes" + echo "$as_me:$LINENO: checking for szlib encoder" >&5 +echo $ECHO_N "checking for szlib encoder... $ECHO_C" >&6 + + if test "${hdf5_cv_szlib_can_encode+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + if test "$cross_compiling" = yes; then + { { echo "$as_me:$LINENO: error: cannot run test program while cross compiling" >&5 +echo "$as_me: error: cannot run test program while cross compiling" >&2;} + { (exit 1); exit 1; }; } +else + cat >conftest.$ac_ext <<_ACEOF +#line $LINENO "configure" +#include "confdefs.h" + + #include + + int main(void) + { + /* SZ_encoder_enabled returns 1 if encoder is present */ + if(SZ_encoder_enabled() == 1) + exit(0); + else + exit(1); + } + +_ACEOF +rm -f conftest$ac_exeext +if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 + (eval $ac_link) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { ac_try='./conftest$ac_exeext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + hdf5_cv_szlib_can_encode=yes +else + echo "$as_me: program exited with status $ac_status" >&5 +echo "$as_me: failed program was:" >&5 +cat conftest.$ac_ext >&5 +( exit $ac_status ) +hdf5_cv_szlib_can_encode=no +fi +rm -f core core.* *.core conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext +fi +fi + + + if test ${hdf5_cv_szlib_can_encode} = "yes"; then + +cat >>confdefs.h <<\_ACEOF +#define SZIP_CAN_ENCODE 1 +_ACEOF + + echo "$as_me:$LINENO: result: yes" >&5 +echo "${ECHO_T}yes" >&6 + USE_FILTER_SZIP_ENCODER="yes" + else + echo "$as_me:$LINENO: result: no" >&5 +echo "${ECHO_T}no" >&6 + fi + if test "X$EXTERNAL_FILTERS" != "X"; then EXTERNAL_FILTERS="${EXTERNAL_FILTERS}," fi @@ -31963,7 +32029,7 @@ if test "$HAVE_PABLO" = "yes"; then fi fi -ac_config_files="$ac_config_files src/libhdf5.settings config/depend1 config/depend2 config/depend3 config/depend4 config/dependN config/commence config/conclude Makefile src/Makefile $PABLO_MAKE test/Makefile $PARALLEL_MAKE perform/Makefile tools/Makefile tools/h5dump/Makefile tools/h5dump/testh5dump.sh tools/h5import/Makefile tools/h5diff/Makefile tools/h5repack/Makefile tools/h5ls/Makefile tools/lib/Makefile tools/misc/Makefile tools/misc/h5cc tools/gifconv/Makefile examples/Makefile doc/Makefile doc/html/Makefile doc/html/ed_libs/Makefile doc/html/ed_styles/Makefile doc/html/ADGuide/Makefile doc/html/Graphics/Makefile doc/html/Intro/Makefile doc/html/PSandPDF/Makefile doc/html/TechNotes/Makefile doc/html/Tutor/Makefile doc/html/Tutor/Graphics/Makefile doc/html/Tutor/examples/Makefile doc/html/cpplus/Makefile doc/html/fortran/Makefile" +ac_config_files="$ac_config_files src/libhdf5.settings config/depend1 config/depend2 config/depend3 config/depend4 config/dependN config/commence config/conclude Makefile src/Makefile $PABLO_MAKE test/Makefile $PARALLEL_MAKE perform/Makefile tools/Makefile tools/h5dump/Makefile tools/h5dump/testh5dump.sh tools/h5import/Makefile tools/h5diff/Makefile tools/h5repack/Makefile tools/h5repack/h5repack.sh tools/h5ls/Makefile tools/lib/Makefile tools/misc/Makefile tools/misc/h5cc tools/gifconv/Makefile examples/Makefile doc/Makefile doc/html/Makefile doc/html/ed_libs/Makefile doc/html/ed_styles/Makefile doc/html/ADGuide/Makefile doc/html/Graphics/Makefile doc/html/Intro/Makefile doc/html/PSandPDF/Makefile doc/html/TechNotes/Makefile doc/html/Tutor/Makefile doc/html/Tutor/Graphics/Makefile doc/html/Tutor/examples/Makefile doc/html/cpplus/Makefile doc/html/fortran/Makefile" cat >confcache <<\_ACEOF @@ -32467,6 +32533,7 @@ do "tools/h5import/Makefile" ) CONFIG_FILES="$CONFIG_FILES tools/h5import/Makefile" ;; "tools/h5diff/Makefile" ) CONFIG_FILES="$CONFIG_FILES tools/h5diff/Makefile" ;; "tools/h5repack/Makefile" ) CONFIG_FILES="$CONFIG_FILES tools/h5repack/Makefile" ;; + "tools/h5repack/h5repack.sh" ) CONFIG_FILES="$CONFIG_FILES tools/h5repack/h5repack.sh" ;; "tools/h5ls/Makefile" ) CONFIG_FILES="$CONFIG_FILES tools/h5ls/Makefile" ;; "tools/lib/Makefile" ) CONFIG_FILES="$CONFIG_FILES tools/lib/Makefile" ;; "tools/misc/Makefile" ) CONFIG_FILES="$CONFIG_FILES tools/misc/Makefile" ;; @@ -32608,6 +32675,7 @@ s,@LIBTOOL@,$LIBTOOL,;t t s,@DEPEND@,,;t t s,@USE_FILTER_DEFLATE@,$USE_FILTER_DEFLATE,;t t s,@USE_FILTER_SZIP@,$USE_FILTER_SZIP,;t t +s,@USE_FILTER_SZIP_ENCODER@,$USE_FILTER_SZIP_ENCODER,;t t s,@PABLO@,$PABLO,;t t s,@HAVE_PABLO@,$HAVE_PABLO,;t t s,@SSL@,$SSL,;t t diff --git a/configure.in b/configure.in index 493cb04..10c4bda 100644 --- a/configure.in +++ b/configure.in @@ -893,6 +893,7 @@ dnl command-line switch. The value is an include path and/or a library path. dnl If the library path is specified then it must be preceded by a comma. dnl AC_SUBST(USE_FILTER_SZIP) USE_FILTER_SZIP="no" +AC_SUBST(USE_FILTER_SZIP_ENCODER) USE_FILTER_SZIP_ENCODER="no" AC_ARG_WITH([szlib], [AC_HELP_STRING([--with-szlib=DIR], [Use szlib library for external szlib I/O @@ -967,6 +968,32 @@ if test "x$HAVE_SZLIB" = "xyes" -a "x$HAVE_SZLIB_H" = "xyes"; then [Define if support for szip filter is enabled]) USE_FILTER_SZIP="yes" + dnl SZLIB library is available. Check if it can encode + AC_MSG_CHECKING([for szlib encoder]) + + AC_CACHE_VAL([hdf5_cv_szlib_can_encode], + [AC_TRY_RUN([ + #include + + int main(void) + { + /* SZ_encoder_enabled returns 1 if encoder is present */ + if(SZ_encoder_enabled() == 1) + exit(0); + else + exit(1); + } + ], [hdf5_cv_szlib_can_encode=yes], [hdf5_cv_szlib_can_encode=no],)]) + + if test ${hdf5_cv_szlib_can_encode} = "yes"; then + AC_DEFINE([SZIP_CAN_ENCODE], [1], + [Define if szip encoder is present]) + AC_MSG_RESULT([yes]) + USE_FILTER_SZIP_ENCODER="yes" + else + AC_MSG_RESULT([no]) + fi + dnl Add "szip" to external filter list if test "X$EXTERNAL_FILTERS" != "X"; then EXTERNAL_FILTERS="${EXTERNAL_FILTERS}," @@ -2344,6 +2371,7 @@ AC_CONFIG_FILES([src/libhdf5.settings tools/h5import/Makefile tools/h5diff/Makefile tools/h5repack/Makefile + tools/h5repack/h5repack.sh tools/h5ls/Makefile tools/lib/Makefile tools/misc/Makefile diff --git a/fortran/src/H5Zf.c b/fortran/src/H5Zf.c index 70e0dee..8c2b6b6 100644 --- a/fortran/src/H5Zf.c +++ b/fortran/src/H5Zf.c @@ -67,3 +67,29 @@ nh5zfilter_avail_c ( int_f *filter , int_f *flag ) if ( status < 0 ) ret_value = -1; return ret_value; } + +/*---------------------------------------------------------------------------- + * Name: h5zget_filter_info_c + * Purpose: Call H5Zget_filter_info to find if filter has its encoder + * and/or its decoder available + * Inputs: filter - filter identifier + * Outputs: flag - status flag + * Returns: 0 on success, -1 on failure + * Programmer: Nat Furrer and James Laird + * Wednesday, June 16, 2004 + * Modifications: + *---------------------------------------------------------------------------*/ + +int_f +nh5zget_filter_info_c ( int_f *filter , int_f *flag ) +{ + int ret_value = 0; + H5Z_filter_t c_filter; + unsigned int c_flag; + + c_filter = (H5Z_filter_t)*filter; + ret_value = H5Zget_filter_info(c_filter, &c_flag); + *flag = (int_f)c_flag; + + return ret_value; +} diff --git a/fortran/src/H5Zff.f90 b/fortran/src/H5Zff.f90 index 839ab6f..2d0afde 100644 --- a/fortran/src/H5Zff.f90 +++ b/fortran/src/H5Zff.f90 @@ -130,6 +130,68 @@ END SUBROUTINE h5zfilter_avail_f +!---------------------------------------------------------------------- +! Name: h5zget_filter_info_f +! +! Purpose: Queries if filter has its encoder and/or decoder +! available +! +! Inputs: +! filter - filter +! Outputs: +! config_flags - Bit vector possibly containing the +! following values: +! H5Z_FILTER_ENCODE_ENABLED_F +! H5Z_FILTER_DECODE_ENABLED_F +! hdferr: - error code +! Success: 0 +! Failure: -1 +! Optional parameters: +! NONE +! +! Programmer: Nat Furrer and James Laird +! June 16, 2004 +! +! Modifications: +! +!---------------------------------------------------------------------- + SUBROUTINE h5zget_filter_info_f(filter, config_flags, hdferr) +! +!This definition is needed for Windows DLLs +!DEC$if defined(BUILD_HDF5_DLL) +!DEC$attributes dllexport :: h5zget_filter_info_f +!DEC$endif +! + + IMPLICIT NONE + INTEGER, INTENT(IN) :: filter ! Filter; may be one of the following: + ! H5Z_FILTER_DEFLATE_F + ! H5Z_FILTER_SHUFFLE_F + ! H5Z_FILTER_FLETCHER32_F + ! H5Z_FILTER_SZIP_F + INTEGER, INTENT(OUT) :: config_flags! Flag, indicates if filter + ! has its encoder and/or decoder + ! available + INTEGER, INTENT(OUT) :: hdferr ! Error code + +! INTEGER, EXTERNAL :: h5zget_filter_info_c +! MS FORTRAN needs explicit interface for C functions called here. +! + INTERFACE + INTEGER FUNCTION h5zget_filter_info_c(filter, config_flags) + USE H5GLOBAL + !DEC$ IF DEFINED(HDF5F90_WINDOWS) + !MS$ATTRIBUTES C,reference,alias:'_H5ZGET_FILTER_INFO_C'::h5zget_filter_info_c + !DEC$ ENDIF + INTEGER, INTENT(IN) :: filter + INTEGER, INTENT(OUT) :: config_flags + END FUNCTION h5zget_filter_info_c + END INTERFACE + + hdferr = h5zget_filter_info_c(filter, config_flags) + + END SUBROUTINE h5zget_filter_info_f + END MODULE H5Z diff --git a/fortran/src/H5_f.c b/fortran/src/H5_f.c index db35319..ad8d6aa 100644 --- a/fortran/src/H5_f.c +++ b/fortran/src/H5_f.c @@ -396,6 +396,8 @@ nh5init_flags_c( int_f *h5d_flags, int_f *h5e_flags, int_f *h5f_flags, h5z_flags[8] = H5Z_NO_EDC; h5z_flags[9] = H5Z_FILTER_SZIP; h5z_flags[10] = H5Z_FLAG_OPTIONAL; + h5z_flags[11] = H5Z_FILTER_CONFIG_ENCODE_ENABLED; + h5z_flags[12] = H5Z_FILTER_CONFIG_DECODE_ENABLED; ret_value = 0; return ret_value; diff --git a/fortran/src/H5f90global.f90 b/fortran/src/H5f90global.f90 index 54fac81..5f6f91e 100644 --- a/fortran/src/H5f90global.f90 +++ b/fortran/src/H5f90global.f90 @@ -572,7 +572,7 @@ ! ! H5Z flags declaration ! - INTEGER, PARAMETER :: H5Z_FLAGS_LEN = 11 + INTEGER, PARAMETER :: H5Z_FLAGS_LEN = 13 INTEGER H5Z_flags(H5Z_FLAGS_LEN) !DEC$if defined(BUILD_HDF5_DLL) !DEC$ ATTRIBUTES DLLEXPORT :: /H5Z_FLAGS/ @@ -590,6 +590,8 @@ INTEGER :: H5Z_ENABLE_EDC_F INTEGER :: H5Z_NO_EDC_F INTEGER :: H5Z_FLAG_OPTIONAL_F + INTEGER :: H5Z_FILTER_ENCODE_ENABLED_F + INTEGER :: H5Z_FILTER_DECODE_ENABLED_F EQUIVALENCE(H5Z_flags(1), H5Z_FILTER_ERROR_F) EQUIVALENCE(H5Z_flags(2), H5Z_FILTER_NONE_F) @@ -602,6 +604,8 @@ EQUIVALENCE(H5Z_flags(9), H5Z_NO_EDC_F) EQUIVALENCE(H5Z_flags(10), H5Z_FILTER_SZIP_F) EQUIVALENCE(H5Z_flags(11), H5Z_FLAG_OPTIONAL_F) + EQUIVALENCE(H5Z_flags(12), H5Z_FILTER_ENCODE_ENABLED_F) + EQUIVALENCE(H5Z_flags(13), H5Z_FILTER_DECODE_ENABLED_F) ! diff --git a/fortran/src/H5f90proto.h b/fortran/src/H5f90proto.h index d4d9be2..243a73d 100644 --- a/fortran/src/H5f90proto.h +++ b/fortran/src/H5f90proto.h @@ -1245,14 +1245,16 @@ H5_DLL int_f nh5dont_atexit_c(void); #ifdef DF_CAPFNAMES # define nh5zunregister_c FNAME(H5ZUNREGISTER_C) # define nh5zfilter_avail_c FNAME(H5ZFILTER_AVAIL_C) +# define nh5zget_filter_info_c FNAME(H5ZGET_FILTER_INFO_C) #else # define nh5zunregister_c FNAME(h5zunregister_c) # define nh5zfilter_avail_c FNAME(h5zfilter_avail_c) +# define nh5zget_filter_info_c FNAME(h5zget_filter_info_c) #endif #endif H5_DLL int_f nh5zunregister_c (int_f *filter); H5_DLL int_f nh5zfilter_avail_c ( int_f *filter , int_f *flag ); - +H5_DLL int_f nh5zget_filter_info_c (int_f *filter, int_f *flag ); #endif /* _H5f90proto_H */ diff --git a/fortran/test/tH5Z.f90 b/fortran/test/tH5Z.f90 index 91a3bc7..4216d1b 100644 --- a/fortran/test/tH5Z.f90 +++ b/fortran/test/tH5Z.f90 @@ -137,7 +137,36 @@ INTEGER :: filter_flag = -1 INTEGER(SIZE_T) :: cd_nelemnts = 4 INTEGER(SIZE_T) :: filter_name_len = 4 - INTEGER, DIMENSION(4) :: cd_values + INTEGER, DIMENSION(4) :: cd_values + INTEGER :: config_flag = 0 + + ! + ! Make sure that Szip has an encoder available + ! + CALL h5zget_filter_info_f(H5Z_FILTER_SZIP_F, config_flag, error) + CALL check("h5zget_filter_info", error, total_error) + if ( IAND(config_flag, H5Z_FILTER_ENCODE_ENABLED_F) .EQ. 0 ) then + szip_flag = .FALSE. + total_error = -1 + return + endif + CALL h5zfilter_avail_f(H5Z_FILTER_SZIP_F, flag, error) + CALL check("h5zfilter_avail", error, total_error) + + ! + ! Make sure h5zget_filter_info_f returns the right flag + ! + if( flag ) then + if ( config_flag .NE. IOR( H5Z_FILTER_ENCODE_ENABLED_F, H5Z_FILTER_DECODE_ENABLED_F) ) then + error = -1 + CALL check("h5zget_filter_info config_flag", error, total_error) + endif + else + if ( config_flag .NE. 0 ) then + error = -1 + CALL check("h5zget_filter_info config_flag", error, total_error) + endif + endif options_mask = H5_SZIP_NN_OM_F + H5_SZIP_CHIP_OM_F pix_per_block = 32 diff --git a/src/H5D.c b/src/H5D.c index e6b3ef9..cd67fee 100644 --- a/src/H5D.c +++ b/src/H5D.c @@ -1962,6 +1962,12 @@ H5D_create(H5G_entry_t *loc, const char *name, hid_t type_id, const H5S_t *space if(NULL == (new_dset = H5D_new(dcpl_id,TRUE,has_vl_type))) HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") + /* + * Set the dataset's checked_filters flag to enable writing. + * Make sure that H5Z_can_apply is called at the beginning of this function! + */ + new_dset->checked_filters = TRUE; + /* Make the "set local" filter callbacks for this dataset */ if(H5Z_set_local(new_dset->dcpl_id,type_id)<0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to set local filter parameters") @@ -2718,6 +2724,9 @@ done: * Changed the way to retrieve property for generic property * list. * + * Nat Furrer and James Laird + * June 17, 2004 + * Added check for filter encode capability *------------------------------------------------------------------------- */ static herr_t @@ -2725,6 +2734,8 @@ H5D_extend (H5D_t *dataset, const hsize_t *size, hid_t dxpl_id) { int changed; /* Flag to indicate that the dataspace was successfully extended */ H5S_t *space = NULL; /* Dataset's dataspace */ + H5D_fill_value_t fill_status; + H5D_fill_time_t fill_time; herr_t ret_value=SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(H5D_extend, FAIL) @@ -2739,6 +2750,48 @@ H5D_extend (H5D_t *dataset, const hsize_t *size, hid_t dxpl_id) * able to muck things up. */ + if(! dataset->checked_filters) + { + /* Check if the filters in the DCPL will need to encode, and if so, can they? + * Filters need encoding if fill value is defined and a fill policy is set that requires + * writing on an extend. + */ + if(H5P_is_fill_value_defined(&(dataset->fill), &fill_status) < 0) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Couldn't retrieve fill value from dataset."); + } + + if(fill_status == H5D_FILL_VALUE_DEFAULT || fill_status == H5D_FILL_VALUE_USER_DEFINED) + { + if( H5Pget_fill_time(dataset->dcpl_id, &fill_time) < 0) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Couldn't retrieve fill time from dataset."); + } + + if(fill_time == H5D_FILL_TIME_ALLOC || + (fill_time == H5D_FILL_TIME_IFSET && fill_status == H5D_FILL_VALUE_USER_DEFINED) ) + { + /* Filters must have encoding enabled. Ensure that all filters can be applied */ + hid_t type_id; + + type_id = H5I_register(H5I_DATATYPE, dataset->type); + if(type_id < 0) + HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to register data type") + + if(H5Z_can_apply(dataset->dcpl_id, type_id) <0) + { + H5I_remove(type_id); + HGOTO_ERROR(H5E_PLINE, H5E_CANAPPLY, FAIL, "can't apply filters") + } + + if(H5I_remove(type_id) == NULL) + HGOTO_ERROR(H5E_PLINE, H5E_CANTRELEASE, FAIL, "unable to release data type id") + + dataset->checked_filters = TRUE; + } + } + } + /* Increase the size of the data space */ space=dataset->space; if ((changed=H5S_extend (space, size))<0) diff --git a/src/H5Dio.c b/src/H5Dio.c index 5220884..1e6edf7 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -837,6 +837,9 @@ done: * Removed the must_convert parameter and move preconditions to * H5S__opt_possible() routine * + * Nat Furrer and James Laird + * June 17, 2004 + * Added check for filter encode capability *------------------------------------------------------------------------- */ static herr_t @@ -864,6 +867,27 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, assert(mem_type); assert(buf); + /* All filters in the DCPL must have encoding enabled. */ + if(! dataset->checked_filters) + { + hid_t type_id; + + type_id = H5I_register(H5I_DATATYPE, dataset->type); + if(type_id < 0) + HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to register data type") + + if(H5Z_can_apply(dataset->dcpl_id, type_id) <0) + { + H5I_remove(type_id); + HGOTO_ERROR(H5E_PLINE, H5E_CANAPPLY, FAIL, "can't apply filters") + } + + if(H5I_remove(type_id) == NULL) + HGOTO_ERROR(H5E_PLINE, H5E_CANTRELEASE, FAIL, "unable to release data type id") + + dataset->checked_filters = TRUE; + } + /* If MPI based VFD is used, no VL datatype support yet. */ /* This is because they use the global heap in the file and we don't */ /* support parallel access of that yet */ diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index b72a13d..18af011 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -83,6 +83,7 @@ struct H5D_t { hid_t dcpl_id; /* dataset creation property id */ H5D_dcpl_cache_t dcpl_cache; /* Cached DCPL values */ H5O_layout_t layout; /* data layout */ + hbool_t checked_filters;/* TRUE if dataset passes can_apply check */ /* Cache some frequently accessed values from the DCPL */ H5O_efl_t efl; /* External file list information */ diff --git a/src/H5E.c b/src/H5E.c index 511f3f2..16f8b03 100644 --- a/src/H5E.c +++ b/src/H5E.c @@ -187,7 +187,8 @@ static const H5E_minor_mesg_t H5E_minor_mesg_g[] = { {H5E_NOFILTER, "Requested filter is not available"}, {H5E_CALLBACK, "Callback failed"}, {H5E_CANAPPLY, "Error from filter \"can apply\" callback"}, - {H5E_SETLOCAL, "Error from filter \"set local\" callback"} + {H5E_SETLOCAL, "Error from filter \"set local\" callback"}, + {H5E_NOENCODER, "Filter present, but encoder not enabled"} }; diff --git a/src/H5Epublic.h b/src/H5Epublic.h index 6840dd5..3ac6fd0 100644 --- a/src/H5Epublic.h +++ b/src/H5Epublic.h @@ -218,8 +218,8 @@ typedef enum H5E_minor_t { H5E_NOFILTER, /*requested filter is not available */ H5E_CALLBACK, /*callback failed */ H5E_CANAPPLY, /*error from filter "can apply" callback */ - H5E_SETLOCAL /*error from filter "set local" callback */ - + H5E_SETLOCAL, /*error from filter "set local" callback */ + H5E_NOENCODER /* Filter present, but encoding disabled */ } H5E_minor_t; /* Information about an error */ diff --git a/src/H5MPprivate.h b/src/H5MPprivate.h index 2eeffe4..634d408 100644 --- a/src/H5MPprivate.h +++ b/src/H5MPprivate.h @@ -390,6 +390,7 @@ #define color_H5Zregister "red" #define color_H5Zfilter_avail "red" #define color_H5Zunregister "red" +#define color_H5Zget_filter_info "red" #else #define MPE_LOG_VARS(func_name) /* void */ diff --git a/src/H5Pdcpl.c b/src/H5Pdcpl.c index 6466c3e..95bc649 100644 --- a/src/H5Pdcpl.c +++ b/src/H5Pdcpl.c @@ -1003,6 +1003,10 @@ done: * * Modifications: * + * Nat Furrer and James Laird + * June 17, 2004 + * Now ensures that SZIP encoding is enabled + * *------------------------------------------------------------------------- */ herr_t @@ -1017,6 +1021,9 @@ H5Pset_szip(hid_t plist_id, unsigned options_mask, unsigned pixels_per_block) H5TRACE3("e","iIuIu",plist_id,options_mask,pixels_per_block); /* Check arguments */ +#if !defined( H5_SZIP_CAN_ENCODE) && defined(H5_HAVE_FILTER_SZIP) + HGOTO_ERROR (H5E_PLINE, H5E_NOENCODER, FAIL, "Szip filter present but encoding disabled"); +#endif if ((pixels_per_block%2)==1) HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, "pixels_per_block is not even"); if (pixels_per_block>H5_SZIP_MAX_PIXELS_PER_BLOCK) @@ -1642,3 +1649,4 @@ H5Premove_filter(hid_t plist_id, H5Z_filter_t filter) done: FUNC_LEAVE_API(ret_value); } + diff --git a/src/H5Z.c b/src/H5Z.c index 0d0151a..849f671 100644 --- a/src/H5Z.c +++ b/src/H5Z.c @@ -574,8 +574,15 @@ H5Z_prelude_callback(hid_t dcpl_id, hid_t type_id, H5Z_prelude_type_t prelude_ty case H5Z_PRELUDE_CAN_APPLY: /* Check if there is a "can apply" callback */ if(fclass->can_apply) { + herr_t status; +#ifndef H5_SZIP_CAN_ENCODE + /* If this is the Szip filter, make sure it can encode */ + if (dcpl_pline.filter[u].id == H5Z_FILTER_SZIP) + HGOTO_ERROR(H5E_PLINE, H5E_NOENCODER, FAIL, "Filter present but encoding is disabled"); +#endif + /* Make callback to filter's "can apply" function */ - herr_t status=(fclass->can_apply)(dcpl_id, type_id, space_id); + status=(fclass->can_apply)(dcpl_id, type_id, space_id); /* Check return value */ if(status<=0) { @@ -1210,3 +1217,47 @@ H5Z_delete(H5O_pline_t *pline, H5Z_filter_t filter) done: FUNC_LEAVE_NOAPI(ret_value) } + +/*------------------------------------------------------------------------- + * Function: H5Zget_filter_info + * + * Purpose: Gets information about a pipeline data filter and stores it + * in filter_config_flags. + * + * Return: zero on success / negative on failure + * + * Programmer: James Laird and Nat Furrer + * Monday, June 7, 2004 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t H5Zget_filter_info(H5Z_filter_t filter, unsigned int *filter_config_flags) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(H5Zget_filter_info, FAIL) + + if (filter_config_flags != NULL) + { + if (filter == H5Z_FILTER_SZIP) + { + *filter_config_flags = 0; +#ifdef H5_SZIP_CAN_ENCODE + *filter_config_flags |= H5Z_FILTER_CONFIG_ENCODE_ENABLED; +#endif + *filter_config_flags |= H5Z_FILTER_CONFIG_DECODE_ENABLED; + } + else + *filter_config_flags = H5Z_FILTER_CONFIG_DECODE_ENABLED | H5Z_FILTER_CONFIG_ENCODE_ENABLED; + + /* Make sure the filter exists */ + if (H5Z_find(filter) == NULL) + *filter_config_flags = 0; + } + +done: + FUNC_LEAVE_API(ret_value) +} + diff --git a/src/H5Zpublic.h b/src/H5Zpublic.h index 8fd775d..e4db1b0 100644 --- a/src/H5Zpublic.h +++ b/src/H5Zpublic.h @@ -66,6 +66,10 @@ typedef enum H5Z_EDC_t { H5Z_NO_EDC = 2 /* must be the last */ } H5Z_EDC_t; +/* Bit flags for H5Zget_filter_info */ +#define H5Z_FILTER_CONFIG_ENCODE_ENABLED (0x0001) +#define H5Z_FILTER_CONFIG_DECODE_ENABLED (0x0002) + /* Return values for filter callback function */ typedef enum H5Z_cb_return_t { H5Z_CB_ERROR = -1, @@ -172,7 +176,7 @@ H5_DLL herr_t H5Zregister(const H5Z_class_t *cls); #endif /* H5_WANT_H5_V1_4_COMPAT */ H5_DLL herr_t H5Zunregister(H5Z_filter_t id); H5_DLL htri_t H5Zfilter_avail(H5Z_filter_t id); - +H5_DLL herr_t H5Zget_filter_info(H5Z_filter_t filter, unsigned int *filter_config_flags); #ifdef __cplusplus } diff --git a/src/H5Zszip.c b/src/H5Zszip.c index 76cf017..7df0be2 100644 --- a/src/H5Zszip.c +++ b/src/H5Zszip.c @@ -327,5 +327,6 @@ done: H5MM_xfree(outbuf); FUNC_LEAVE_NOAPI(ret_value); } + #endif /* H5_HAVE_FILTER_SZIP */ diff --git a/src/H5config.h.in b/src/H5config.h.in index 6a0c8dd..e6c92d7 100644 --- a/src/H5config.h.in +++ b/src/H5config.h.in @@ -496,6 +496,9 @@ PTHREAD_SCOPE_SYSTEM) call. */ #undef SYSTEM_SCOPE_THREADS +/* Define if szip encoder is present */ +#undef SZIP_CAN_ENCODE + /* Define to 1 if you can safely include both and . */ #undef TIME_WITH_SYS_TIME diff --git a/test/dsets.c b/test/dsets.c index 707f858..ee9194b 100644 --- a/test/dsets.c +++ b/test/dsets.c @@ -107,6 +107,12 @@ const char *FILENAME[] = { #define FILTER_HS_SIZE1 4 #define FILTER_HS_SIZE2 50 +/* Names for noencoder test */ +#define NOENCODER_FILENAME "noencoder.h5" +#define NOENCODER_TEST_DATASET "noencoder_tdset.h5" +#define NOENCODER_SZIP_DATASET "noencoder_szip_dset.h5" +#define NOENCODER_SZIP_SHUFF_FLETCH_DATASET "noencoder_szip_shuffle_fletcher_dset.h5" + /* Shared global arrays */ #define DSET_DIM1 100 #define DSET_DIM2 200 @@ -1612,7 +1618,201 @@ error: return -1; } - +/*------------------------------------------------------------------------- + * Function: test_filter_noencoder + * + * Purpose: Tests filters with no encoder present. Ensures that data + * can still be decoded correctly and that errors are thrown + * when the application tries to write. + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Nat Furrer and James Laird + * Monday, June 7, 2004 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +test_filter_noencoder(const char *dset_name) +{ + hid_t file_id = -1; + hid_t dset_id = -1; + hid_t test_dset_id = -1; + hid_t dcpl_id = -1; + hid_t space_id = -1; + hsize_t dims = 1; + hsize_t read_dims = 10; + herr_t err; + int test_ints[10] = { 12 }; + int read_buf[10]; + int i; + + file_id = H5Fopen(NOENCODER_FILENAME, H5F_ACC_RDWR, H5P_DEFAULT); + if (file_id < 0) goto error; + + dset_id = H5Dopen(file_id, dset_name); + if (dset_id < 0) goto error; + + TESTING(" decoding without encoder"); + + space_id = H5Screate_simple(1, &read_dims, NULL); + if (space_id < 0) goto error; + + /* Read the dataset and make sure the decoder is working correctly */ + err = H5Dread(dset_id, H5T_NATIVE_INT, space_id, space_id, H5P_DEFAULT, read_buf); + if (err < 0) goto error; + + for(i = 0; i < 10; i++) + if ( read_buf[i] != i ) goto error; + + H5Sclose(space_id); + + PASSED(); + + /* Attempt to copy the DCPL and use it to create a new dataset. + * Since the filter does not have an encoder, the creation + * should fail. + */ + TESTING(" trying to write without encoder"); + + dcpl_id = H5Dget_create_plist(dset_id); + if (dcpl_id < 0) goto error; + + space_id = H5Screate_simple(1, &dims, NULL); + if (space_id < 0) goto error; + + H5E_BEGIN_TRY{ + test_dset_id = H5Dcreate(file_id, NOENCODER_TEST_DATASET, H5T_NATIVE_INT, space_id , dcpl_id); + }H5E_END_TRY + + if (test_dset_id >= 0) goto error; + + /* Attempt to extend the dataset. This should fail because + * the dataset has a fill value and is instructed to fill on + * allocation. + */ + dims = 20; /* Dataset is originally of size 10 */ + H5E_BEGIN_TRY{ + err = H5Dextend(dset_id, &dims); + }H5E_END_TRY + + if (err >= 0) goto error; + + /* Attempt to write to the dataset. This should fail because + * the filter does not have an encoder. + */ + H5E_BEGIN_TRY{ + err = H5Dwrite(dset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, test_ints); + }H5E_END_TRY + + if (err >= 0) goto error; + + H5Fclose(file_id); + H5Dclose(dset_id); + H5Sclose(space_id); + H5Pclose(dcpl_id); + + PASSED(); + + return 0; + +error: + H5_FAILED(); + if (dset_id != -1) + H5Dclose(dset_id); + if (test_dset_id != -1) + H5Dclose(test_dset_id); + if (space_id != -1) + H5Sclose(space_id); + if (dcpl_id != -1) + H5Pclose(dcpl_id); + if (file_id != -1) + H5Fclose(file_id); + return -1; +} + +/*------------------------------------------------------------------------- + * Function: test_get_filter_info + * + * Purpose: Tests the H5Zget_filter_info function. + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Nat Furrer and James Laird + * Thursday, June 10, 2004 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +test_get_filter_info() +{ + unsigned int flags; /* flags returned from H5Zget_filter_info */ + herr_t ret_value = -1; + herr_t err; + + TESTING("H5Zget_filter_info"); + + /* Verify that each filter is reported as having the right combination + * of encoder and decoder. + */ +#ifdef H5_HAVE_FILTER_FLETCHER32 + if(H5Zget_filter_info(H5Z_FILTER_FLETCHER32, &flags) < 0) goto error; + + if(((flags & H5Z_FILTER_CONFIG_ENCODE_ENABLED) == 0) || + ((flags & H5Z_FILTER_CONFIG_DECODE_ENABLED) == 0)) + goto error; +#endif + +#ifdef H5_HAVE_FILTER_SHUFFLE + if(H5Zget_filter_info(H5Z_FILTER_SHUFFLE, &flags) < 0) goto error; + + if(((flags & H5Z_FILTER_CONFIG_ENCODE_ENABLED) == 0) || + ((flags & H5Z_FILTER_CONFIG_DECODE_ENABLED) == 0)) + goto error; +#endif + +#ifdef H5_HAVE_FILTER_DEFLATE + if(H5Zget_filter_info(H5Z_FILTER_DEFLATE, &flags) < 0) goto error; + + if(((flags & H5Z_FILTER_CONFIG_ENCODE_ENABLED) == 0) || + ((flags & H5Z_FILTER_CONFIG_DECODE_ENABLED) == 0)) + goto error; +#endif + +#ifdef H5_HAVE_FILTER_SZIP + if(H5Zget_filter_info(H5Z_FILTER_SZIP, &flags) < 0) goto error; + +#ifdef H5_SZIP_CAN_ENCODE + if(((flags & H5Z_FILTER_CONFIG_ENCODE_ENABLED) == 0) || + ((flags & H5Z_FILTER_CONFIG_DECODE_ENABLED) == 0)) + goto error; +#else + if(((flags & H5Z_FILTER_CONFIG_ENCODE_ENABLED) != 0) || + ((flags & H5Z_FILTER_CONFIG_DECODE_ENABLED) == 0)) + goto error; +#endif /* H5_SZIP_CAN_ENCODE */ +#endif /* H5_HAVE_FILTER_SZIP */ + + /* Verify that get_filter_info throws an error when given a bad filter */ + err = H5Zget_filter_info(-1, &flags); + + if (err < 0) goto error; + if (flags != 0) goto error; + + PASSED(); + return 0; + +error: + H5_FAILED(); + return -1; +} + /*------------------------------------------------------------------------- * Function: test_filters * @@ -1663,7 +1863,10 @@ test_filters(hid_t file) #if (defined H5_HAVE_FILTER_DEFLATE | defined H5_HAVE_FILTER_SZIP) && defined H5_HAVE_FILTER_SHUFFLE && defined H5_HAVE_FILTER_FLETCHER32 hsize_t combo_size; /* Size of dataset with shuffle+deflate filter */ #endif /* H5_HAVE_FILTER_DEFLATE && H5_HAVE_FILTER_SHUFFLE && H5_HAVE_FILTER_FLETCHER32 */ - + + /* test the H5Zget_filter_info function */ + if(test_get_filter_info() < 0) goto error; + /*---------------------------------------------------------- * STEP 0: Test null I/O filter by itself. *---------------------------------------------------------- @@ -1761,13 +1964,27 @@ test_filters(hid_t file) *---------------------------------------------------------- */ #ifdef H5_HAVE_FILTER_SZIP - puts("Testing szip filter"); + TESTING("szip filter (with encoder)"); if((dc = H5Pcreate(H5P_DATASET_CREATE))<0) goto error; if (H5Pset_chunk (dc, 2, chunk_size)<0) goto error; - + +#ifdef H5_SZIP_CAN_ENCODE + puts(""); if (H5Pset_szip(dc, szip_options_mask, szip_pixels_per_block)<0) goto error; if(test_filter_internal(file,DSET_SZIP_NAME,dc,DISABLE_FLETCHER32,DATA_NOT_CORRUPTED,&szip_size)<0) goto error; +#else + SKIPPED(); +#endif /* H5_SZIP_CAN_ENCODE */ + + TESTING("szip filter (without encoder)"); +#ifndef H5_SZIP_CAN_ENCODE + puts(""); + if(test_filter_noencoder(NOENCODER_SZIP_DATASET) < 0) goto error; +#else + SKIPPED(); +#endif /* H5_SZIP_CAN_ENCODE */ + if (H5Pclose (dc)<0) goto error; #else /* H5_HAVE_FILTER_SZIP */ TESTING("szip filter"); @@ -1840,19 +2057,37 @@ test_filters(hid_t file) */ #if defined H5_HAVE_FILTER_SZIP && defined H5_HAVE_FILTER_SHUFFLE && defined H5_HAVE_FILTER_FLETCHER32 - puts("Testing shuffle+szip+checksum filters(checksum first)"); + TESTING("shuffle+szip+checksum filters(checksum first, with encoder)"); if((dc = H5Pcreate(H5P_DATASET_CREATE))<0) goto error; if (H5Pset_chunk (dc, 2, chunk_size)<0) goto error; if (H5Pset_fletcher32 (dc)<0) goto error; if (H5Pset_shuffle (dc)<0) goto error; + + /* Make sure encoding is enabled */ +#ifdef H5_SZIP_CAN_ENCODE + puts(""); if (H5Pset_szip(dc, szip_options_mask, szip_pixels_per_block)<0) goto error; if(test_filter_internal(file,DSET_SHUF_SZIP_FLET_NAME,dc,ENABLE_FLETCHER32,DATA_NOT_CORRUPTED,&combo_size)<0) goto error; +#else + SKIPPED(); +#endif /* H5_SZIP_CAN_ENCODE */ + + TESTING("shuffle+szip+checksum filters(checksum first, without encoder)"); +#ifndef H5_SZIP_CAN_ENCODE + puts(""); + if (test_filter_noencoder(NOENCODER_SZIP_SHUFF_FLETCH_DATASET) < 0) goto error; +#else + SKIPPED(); +#endif /* H5_SZIP_CAN_ENCODE */ /* Clean up objects used for this test */ if (H5Pclose (dc)<0) goto error; - puts("Testing shuffle+szip+checksum filters(checksum last)"); + TESTING("shuffle+szip+checksum filters(checksum last, with encoder)"); + /* Make sure encoding is enabled */ +#ifdef H5_SZIP_CAN_ENCODE + puts(""); if((dc = H5Pcreate(H5P_DATASET_CREATE))<0) goto error; if (H5Pset_chunk (dc, 2, chunk_size)<0) goto error; if (H5Pset_shuffle (dc)<0) goto error; @@ -1863,6 +2098,10 @@ test_filters(hid_t file) /* Clean up objects used for this test */ if (H5Pclose (dc)<0) goto error; +#else + SKIPPED(); +#endif /* H5_SZIP_CAN_ENCODE */ + #else /* H5_HAVE_FILTER_SZIP && H5_HAVE_FILTER_SHUFFLE && H5_HAVE_FILTER_FLETCHER32 */ TESTING("shuffle+szip+fletcher32 filters"); SKIPPED(); @@ -2616,6 +2855,13 @@ file) TESTING("dataset szip filter 'can apply' callback"); #ifdef H5_HAVE_FILTER_SZIP + + /* skip this test if the SZIP encoder is not enabled */ +#ifndef H5_SZIP_CAN_ENCODE + SKIPPED(); + return 0; +#endif /* H5_SZIP_CAN_ENCODE */ + /* Create the data space */ if ((sid = H5Screate_simple(2, dims, NULL))<0) { H5_FAILED(); diff --git a/tools/h5dump/h5dumpgentest.c b/tools/h5dump/h5dumpgentest.c index b06dbd7..f4a59ee 100644 --- a/tools/h5dump/h5dumpgentest.c +++ b/tools/h5dump/h5dumpgentest.c @@ -4548,7 +4548,7 @@ static void gent_filters(void) * SZIP *------------------------------------------------------------------------- */ -#if defined (H5_HAVE_FILTER_SZIP) +#if defined (H5_HAVE_FILTER_SZIP) && defined (H5_SZIP_CAN_ENCODE) /* remove the filters from the dcpl */ ret=H5Premove_filter(dcpl,H5Z_FILTER_ALL); assert(ret>=0); @@ -4628,7 +4628,7 @@ static void gent_filters(void) assert(ret>=0); #endif -#if defined (H5_HAVE_FILTER_SZIP) +#if defined (H5_HAVE_FILTER_SZIP) && defined (H5_SZIP_CAN_ENCODE) szip_options_mask=H5_SZIP_CHIP_OPTION_MASK | H5_SZIP_EC_OPTION_MASK; /* set szip data */ ret=H5Pset_szip (dcpl,szip_options_mask,szip_pixels_per_block); diff --git a/tools/h5repack/h5repack.sh b/tools/h5repack/h5repack.sh deleted file mode 100644 index f1a6281..0000000 --- a/tools/h5repack/h5repack.sh +++ /dev/null @@ -1,178 +0,0 @@ -#! /bin/sh -# -# Copyright by the Board of Trustees of the University of Illinois. -# All rights reserved. -# -# This file is part of HDF5. The full HDF5 copyright notice, including -# terms governing use, modification, and redistribution, is contained in -# the files COPYING and Copyright.html. COPYING can be found at the root -# of the source code distribution tree; Copyright.html can be found at the -# root level of an installed copy of the electronic HDF5 document set and -# is linked from the top-level documents page. It can also be found at -# http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have -# access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. -# -# Tests for the h5repack tool - -H5REPACK=h5repack # The tool name -H5REPACK_BIN=`pwd`/$H5REPACK # The path of the tool binary - -H5DIFF=../h5diff/h5diff # The h5diff tool name -H5DIFF_BIN=`pwd`/$H5DIFF # The path of the h5diff tool binary - -nerrors=0 -verbose=yes - -# The build (current) directory might be different than the source directory. -# -if test -z "$srcdir"; then - srcdir=. -fi - -test -d ../testfiles || mkdir ../testfiles - -# Print a line-line message left justified in a field of 70 characters -# beginning with the word "Testing". -# -TESTING() { - SPACES=" " - echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' -} - -# Print a line-line message left justified in a field of 70 characters -# beginning with the word "Verifying". -# -VERIFY() { - SPACES=" " - echo "Testing h5diff output $* $SPACES" | cut -c1-70 | tr -d '\012' -} - -# Call the h5diff tool -# -DIFFTEST() -{ - VERIFY $@ - if [ "`uname -s`" = "TFLOPS O/S" ]; then - $RUNSERIAL $H5DIFF_BIN $@ - else - $RUNSERIAL $H5DIFF_BIN "$@" - fi - RET=$? - if [ $RET != 0 ] ; then - echo "*FAILED*" - nerrors="`expr $nerrors + 1`" - else - echo " PASSED" - fi - -} - -# Call h5repack -# -TOOLTEST() -{ - # Run test. - # Tflops interprets "$@" as "" when no parameter is given (e.g., the - # case of missing file name). Changed it to use $@ till Tflops fixes it. - TESTING $H5REPACK $@ - - infile=$srcdir/../testfiles/$1 - outfile=out.$1 - shift - if [ "`uname -s`" = "TFLOPS O/S" ]; then - $RUNSERIAL $H5REPACK_BIN -i $infile -o $outfile $@ - else - $RUNSERIAL $H5REPACK_BIN -i $infile -o $outfile "$@" - fi - - RET=$? - if [ $RET != 0 ] ; then - echo "*FAILED*" - nerrors="`expr $nerrors + 1`" - else - echo " PASSED" - DIFFTEST $infile $outfile - fi - rm -f $outfile -} - - -# -# The tests -# We use the files generated by h5repacktst -# Each run generates "file4.out.h5" and the tool h5diff is used to -# compare the input and output files -# - -# copy files -TOOLTEST test1.h5 -TOOLTEST test3.h5 -TOOLTEST test4.h5 - -#TOOLTEST test5.h5 - -# remove all filters -TOOLTEST test4.h5 -f NONE - -# remove one filter -TOOLTEST test4.h5 -f dset_gzip:NONE - -# gzip -TOOLTEST test4.h5 -f dset1:GZIP=9 -TOOLTEST test4.h5 -f GZIP=1 - -# szip -TOOLTEST test4.h5 -f dset1:SZIP=8 -TOOLTEST test4.h5 -f SZIP=8 - -# shuffle -TOOLTEST test4.h5 -f dset1:SHUF -TOOLTEST test4.h5 -f SHUF - -# fletcher -TOOLTEST test4.h5 -f dset1:FLET -TOOLTEST test4.h5 -f FLET - -#layout chunk -TOOLTEST test4.h5 -l dset1:CHUNK=20x10 -TOOLTEST test4.h5 -l CHUNK=20x10 - -#layout compact -TOOLTEST test4.h5 -l dset1:COMPA -TOOLTEST test4.h5 -l COMPA - -#layout contiguous -TOOLTEST test4.h5 -l dset1:CONTI -TOOLTEST test4.h5 -l CONTI - -#conversions -TOOLTEST test4.h5 -l dset_compact:CONTI -TOOLTEST test4.h5 -l dset_compact:CHUNK=2x5 -TOOLTEST test4.h5 -l dset_compact:COMPA -TOOLTEST test4.h5 -l dset_contiguous:COMPA -TOOLTEST test4.h5 -l dset_contiguous:CHUNK=3x6 -TOOLTEST test4.h5 -l dset_contiguous:CONTI -TOOLTEST test4.h5 -l dset_chunk:COMPA -TOOLTEST test4.h5 -l dset_chunk:CONTI -TOOLTEST test4.h5 -l dset_chunk:CHUNK=18x13 - -#filters -TOOLTEST test4.h5 -f dset1:SHUF -f dset1,dset2:GZIP=6 -TOOLTEST test4.h5 -l dset1:CHUNK=20x10 -f dset1,dset2:SZIP=8 - -#filter conversions -TOOLTEST test4.h5 -f dset_gzip:SZIP=8 -TOOLTEST test4.h5 -f dset_szip:GZIP=1 -TOOLTEST test4.h5 -f dset_all:GZIP=1 - -#limit -TOOLTEST test4.h5 -f GZIP=1 -m 1024 - -#file -TOOLTEST test4.h5 -e ../testfiles/h5repack_info.txt - -if test $nerrors -eq 0 ; then - echo "All $H5REPACK tests passed." -fi - -exit $nerrors diff --git a/tools/h5repack/h5repack.sh.in b/tools/h5repack/h5repack.sh.in new file mode 100755 index 0000000..a679df5 --- /dev/null +++ b/tools/h5repack/h5repack.sh.in @@ -0,0 +1,291 @@ +#! /bin/sh +# +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the files COPYING and Copyright.html. COPYING can be found at the root +# of the source code distribution tree; Copyright.html can be found at the +# root level of an installed copy of the electronic HDF5 document set and +# is linked from the top-level documents page. It can also be found at +# http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have +# access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. +# +# Tests for the h5repack tool + +# Determine if SZIP has the encoder available +USE_FILTER_SZIP_ENCODER="@USE_FILTER_SZIP_ENCODER@" +USE_FILTER_SZIP="@USE_FILTER_SZIP@" + +H5REPACK=h5repack # The tool name +H5REPACK_BIN=`pwd`/$H5REPACK # The path of the tool binary + +H5DIFF=../h5diff/h5diff # The h5diff tool name +H5DIFF_BIN=`pwd`/$H5DIFF # The path of the h5diff tool binary + +nerrors=0 +verbose=yes + +# The build (current) directory might be different than the source directory. +# +if test -z "$srcdir"; then + srcdir=. +fi + +test -d ../testfiles || mkdir ../testfiles + +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Testing". +# +TESTING() { + SPACES=" " + echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' +} + +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Verifying". +# +VERIFY() { + SPACES=" " + echo "Testing h5diff output $* $SPACES" | cut -c1-70 | tr -d '\012' +} + +# Print a message that a test has been skipped (because a required filter +# was unavailable) +SKIP() { + TESTING $H5REPACK $@ + echo " -SKIP-" +} + +# Call the h5diff tool +# +DIFFTEST() +{ + VERIFY $@ + if [ "`uname -s`" = "TFLOPS O/S" ]; then + $RUNSERIAL $H5DIFF_BIN $@ + else + $RUNSERIAL $H5DIFF_BIN "$@" + fi + RET=$? + if [ $RET != 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + fi + +} + +# Call h5repack +# +TOOLTEST() +{ + # Run test. + # Tflops interprets "$@" as "" when no parameter is given (e.g., the + # case of missing file name). Changed it to use $@ till Tflops fixes it. + TESTING $H5REPACK $@ + + infile=$srcdir/../testfiles/$1 + outfile=out.$1 + shift + if [ "`uname -s`" = "TFLOPS O/S" ]; then + $RUNSERIAL $H5REPACK_BIN -i $infile -o $outfile $@ + else + $RUNSERIAL $H5REPACK_BIN -i $infile -o $outfile "$@" + fi + + RET=$? + if [ $RET != 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + DIFFTEST $infile $outfile + fi + rm -f $outfile +} + +# +# The tests +# We use the files generated by h5repacktst +# Each run generates "file4.out.h5" and the tool h5diff is used to +# compare the input and output files +# + +# See which filters are usable (and skip tests for filters we +# don't have). Do this by searching H5pubconf.h to see which +# filters are defined. +# A filter value of 3 means the filter can read and write, 2 +# is read-only, 1 is write-only, and 0 is not present. + +# copy files +TOOLTEST test1.h5 +TOOLTEST test3.h5 + + +arg="test4.h5" +if test $USE_FILTER_SZIP_ENCODER = "yes" -o $USE_FILTER_SZIP = "no" ; then + TOOLTEST $arg +else + SKIP $arg +fi + +#TOOLTEST test5.h5 + +# remove all filters +TOOLTEST test4.h5 -f NONE + +# remove one filter +arg="test4.h5 -f dset_gzip:NONE" +if test $USE_FILTER_SZIP_ENCODER = "yes" -o $USE_FILTER_SZIP = "no" ; then + TOOLTEST $arg +else + SKIP $arg +fi + +# gzip +arg="test4.h5 -f dset1:GZIP=9" +if test $USE_FILTER_SZIP_ENCODER = "yes" -o $USE_FILTER_SZIP = "no" ; then + TOOLTEST $arg +else + SKIP $arg +fi +TOOLTEST test4.h5 -f GZIP=1 + +# szip +arg1="test4.h5 -f dset1:SZIP=8" +arg2="test4.h5 -f SZIP=8" +if test $USE_FILTER_SZIP_ENCODER = "yes" -o $USE_FILTER_SZIP = "no" ; then + TOOLTEST $arg1 + TOOLTEST $arg2 +else + SKIP $arg1 + SKIP $arg2 +fi + +# shuffle +arg="test4.h5 -f dset1:SHUF" +if test $USE_FILTER_SZIP_ENCODER = "yes" -o $USE_FILTER_SZIP = "no" ; then + TOOLTEST $arg +else + SKIP $arg +fi +TOOLTEST test4.h5 -f SHUF + +# fletcher +arg="test4.h5 -f dset1:FLET" +if test $USE_FILTER_SZIP_ENCODER = "yes" -o $USE_FILTER_SZIP = "no" ; then + TOOLTEST $arg +else + SKIP $arg +fi +TOOLTEST test4.h5 -f FLET + +#layout chunk +arg1="test4.h5 -l dset1:CHUNK=20x10" +arg2="test4.h5 -l CHUNK=20x10" +if test $USE_FILTER_SZIP_ENCODER = "yes" -o $USE_FILTER_SZIP = "no" ; then + TOOLTEST $arg1 + TOOLTEST $arg2 +else + SKIP $arg1 + SKIP $arg2 +fi + +#layout compact +arg1="test4.h5 -l dset1:COMPA" +arg2="test4.h5 -l COMPA" +if test $USE_FILTER_SZIP_ENCODER = "yes" -o $USE_FILTER_SZIP = "no" ; then + TOOLTEST $arg1 + TOOLTEST $arg2 +else + SKIP $arg1 + SKIP $arg2 +fi + +#layout contiguous +arg1="test4.h5 -l dset1:CONTI" +arg2="test4.h5 -l CONTI" +if test $USE_FILTER_SZIP_ENCODER = "yes" -o $USE_FILTER_SZIP = "no" ; then + TOOLTEST $arg1 + TOOLTEST $arg2 +else + SKIP $arg1 + SKIP $arg2 +fi + +#conversions +arg1="test4.h5 -l dset_compact:CONTI" +arg2="test4.h5 -l dset_compact:CHUNK=2x5" +arg3="test4.h5 -l dset_compact:COMPA" +arg4="test4.h5 -l dset_contiguous:COMPA" +arg5="test4.h5 -l dset_contiguous:CHUNK=3x6" +arg6="test4.h5 -l dset_contiguous:CONTI" +arg7="test4.h5 -l dset_chunk:COMPA" +arg8="test4.h5 -l dset_chunk:CONTI" +arg9="test4.h5 -l dset_chunk:CHUNK=18x13" +if test $USE_FILTER_SZIP_ENCODER = "yes" -o $USE_FILTER_SZIP = "no" ; then + TOOLTEST $arg1 + TOOLTEST $arg2 + TOOLTEST $arg3 + TOOLTEST $arg4 + TOOLTEST $arg5 + TOOLTEST $arg6 + TOOLTEST $arg7 + TOOLTEST $arg8 + TOOLTEST $arg9 +else + SKIP $arg1 + SKIP $arg2 + SKIP $arg3 + SKIP $arg4 + SKIP $arg5 + SKIP $arg6 + SKIP $arg7 + SKIP $arg8 + SKIP $arg9 +fi + +#filters +arg1="test4.h5 -f dset1:SHUF -f dset1,dset2:GZIP=6" +arg2="test4.h5 -l dset1:CHUNK=20x10 -f dset1,dset2:SZIP=8" +if test $USE_FILTER_SZIP_ENCODER = "yes" -o $USE_FILTER_SZIP = "no" ; then + TOOLTEST $arg1 + TOOLTEST $arg2 +else + SKIP $arg1 + SKIP $arg2 +fi + +#filter conversions +arg1="test4.h5 -f dset_gzip:SZIP=8" +arg2="test4.h5 -f dset_szip:GZIP=1" +arg3="test4.h5 -f dset_all:GZIP=1" +if test $USE_FILTER_SZIP_ENCODER = "yes" -o $USE_FILTER_SZIP = "no" ; then + TOOLTEST $arg1 + TOOLTEST $arg2 + TOOLTEST $arg3 +else + SKIP $arg1 + SKIP $arg2 + SKIP $arg3 +fi + +#limit +TOOLTEST test4.h5 -f GZIP=1 -m 1024 + +#file +arg="test4.h5 -e ../testfiles/h5repack_info.txt" +if test $USE_FILTER_SZIP_ENCODER = "yes" -o $USE_FILTER_SZIP = "no" ; then + TOOLTEST $arg +else + SKIP $arg +fi + +if test $nerrors -eq 0 ; then + echo "All $H5REPACK tests passed." +fi + +exit $nerrors diff --git a/tools/h5repack/testh5repack_filters.c b/tools/h5repack/testh5repack_filters.c index b285b28..19f60b6 100644 --- a/tools/h5repack/testh5repack_filters.c +++ b/tools/h5repack/testh5repack_filters.c @@ -76,11 +76,14 @@ int make_filters(hid_t loc_id) * SZIP *------------------------------------------------------------------------- */ + /* Make sure encoding is enabled */ +#if defined (H5_SZIP_CAN_ENCODE) || !defined (H5_HAVE_FILTER_SZIP) /* set szip data */ if(H5Pset_szip (dcpl,szip_options_mask,szip_pixels_per_block)<0) goto out; if (make_dset(loc_id,"dset_szip",sid,dcpl,buf)<0) goto out; +#endif /*------------------------------------------------------------------------- * GZIP @@ -128,6 +131,8 @@ int make_filters(hid_t loc_id) * shuffle + SZIP *------------------------------------------------------------------------- */ + /* Make sure SZIP has the encoder */ +#if defined (H5_SZIP_CAN_ENCODE) || !defined (H5_HAVE_FILTER_SZIP) /* remove the filters from the dcpl */ if (H5Premove_filter(dcpl,H5Z_FILTER_ALL)<0) goto out; @@ -139,6 +144,7 @@ int make_filters(hid_t loc_id) goto out; if (make_dset(loc_id,"dset_all",sid,dcpl,buf)<0) goto out; +#endif /*------------------------------------------------------------------------- diff --git a/tools/h5repack/testh5repack_main.c b/tools/h5repack/testh5repack_main.c index ddecd98..7ba1c81 100644 --- a/tools/h5repack/testh5repack_main.c +++ b/tools/h5repack/testh5repack_main.c @@ -203,7 +203,7 @@ int main (void) TESTING(" removing szip filter"); -#ifdef H5_HAVE_FILTER_SZIP +#if defined (H5_HAVE_FILTER_SZIP) && defined (H5_SZIP_CAN_ENCODE) if (h5repack_init (&pack_options, 0)<0) TEST_ERROR; if (h5repack_addfilter("dset_szip:NONE",&pack_options)<0) @@ -217,7 +217,7 @@ int main (void) if (h5repack_end (&pack_options)<0) TEST_ERROR; - PASSED(); + PASSED(); #else SKIPPED(); #endif @@ -330,13 +330,13 @@ int main (void) TESTING(" adding szip filter"); -#ifdef H5_HAVE_FILTER_SZIP +#if defined (H5_HAVE_FILTER_SZIP) && defined (H5_SZIP_CAN_ENCODE) /*------------------------------------------------------------------------- * test an individual object option *------------------------------------------------------------------------- */ - + if (h5repack_init (&pack_options, 0)<0) TEST_ERROR; if (h5repack_addfilter("dset2:SZIP=8",&pack_options)<0) @@ -352,7 +352,7 @@ int main (void) if (h5repack_end (&pack_options)<0) TEST_ERROR; - PASSED(); + PASSED(); #else SKIPPED(); #endif @@ -364,8 +364,7 @@ int main (void) */ TESTING(" adding szip filter to all"); -#ifdef H5_HAVE_FILTER_SZIP - +#if defined (H5_HAVE_FILTER_SZIP) && defined (H5_SZIP_CAN_ENCODE) if (h5repack_init (&pack_options, 0)<0) TEST_ERROR; if (h5repack_addfilter("SZIP=8",&pack_options)<0) @@ -379,7 +378,7 @@ TESTING(" adding szip filter to all"); if (h5repack_end (&pack_options)<0) TEST_ERROR; - PASSED(); + PASSED(); #else SKIPPED(); #endif @@ -522,8 +521,10 @@ TESTING(" addding shuffle filter to all"); #endif if (h5repack_addfilter("dset1:SHUF",&pack_options)<0) TEST_ERROR; +#if defined (H5_SZIP_CAN_ENCODE) || !defined (H5_HAVE_FILTER_SZIP) if (h5repack_addfilter("dset1:SZIP=8",&pack_options)<0) TEST_ERROR; +#endif if (h5repack_addfilter("dset1:GZIP=1",&pack_options)<0) TEST_ERROR; if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) @@ -542,7 +543,7 @@ TESTING(" addding shuffle filter to all"); * filter conversion from deflate to szip *------------------------------------------------------------------------- */ -#ifdef H5_HAVE_FILTER_DEFLATE +#if (defined (H5_SZIP_CAN_ENCODE) || !defined (H5_HAVE_FILTER_SZIP)) && defined(H5_HAVE_FILTER_DEFLATE) if (h5repack_init (&pack_options, 0)<0) TEST_ERROR; if (h5repack_addfilter("dset_gzip:SZIP=8",&pack_options)<0) @@ -555,7 +556,7 @@ TESTING(" addding shuffle filter to all"); TEST_ERROR; if (h5repack_end (&pack_options)<0) TEST_ERROR; - PASSED(); + PASSED(); #else SKIPPED(); #endif @@ -565,7 +566,7 @@ TESTING(" addding shuffle filter to all"); * filter conversion from szip to deflate *------------------------------------------------------------------------- */ -#ifdef H5_HAVE_FILTER_SZIP +#if defined (H5_HAVE_FILTER_SZIP) && defined (H5_SZIP_CAN_ENCODE) if (h5repack_init (&pack_options, 0)<0) TEST_ERROR; if (h5repack_addfilter("dset_szip:GZIP=1",&pack_options)<0) @@ -578,7 +579,7 @@ TESTING(" addding shuffle filter to all"); TEST_ERROR; if (h5repack_end (&pack_options)<0) TEST_ERROR; - PASSED(); + PASSED(); #else SKIPPED(); #endif @@ -588,7 +589,7 @@ TESTING(" addding shuffle filter to all"); * filter conversion from szip to deflate *------------------------------------------------------------------------- */ -#ifdef H5_HAVE_FILTER_SZIP +#if defined (H5_HAVE_FILTER_SZIP) && defined (H5_SZIP_CAN_ENCODE) if (h5repack_init (&pack_options, 0)<0) TEST_ERROR; if (h5repack_addfilter("dset_all:GZIP=1",&pack_options)<0) @@ -601,7 +602,7 @@ TESTING(" addding shuffle filter to all"); TEST_ERROR; if (h5repack_end (&pack_options)<0) TEST_ERROR; - PASSED(); + PASSED(); #else SKIPPED(); #endif diff --git a/windows/all.zip b/windows/all.zip index 2000d0e..728278d 100755 Binary files a/windows/all.zip and b/windows/all.zip differ -- cgit v0.12