From 8c14af82c61f50146a6c5646c2320a08d09dc32d Mon Sep 17 00:00:00 2001 From: Allen Byrne Date: Wed, 21 Aug 2013 15:59:49 -0500 Subject: [svn-r24050] HDFFV-8498: --minimum option was disabled by HDFFV-8214. Reverted code change and changed default to 0 from 1024. Changed limit test to use h5dump to compare repacked file instead of h5diff. Tested local linux with CMake --- MANIFEST | 1 + tools/h5repack/CMakeTests.cmake | 40 +++++++- tools/h5repack/h5repack.c | 2 +- tools/h5repack/h5repack_copy.c | 10 ++ tools/h5repack/h5repack_main.c | 2 +- tools/h5repack/testfiles/h5repack_layout.h5.ddl | 130 ++++++++++++++++++++++++ 6 files changed, 181 insertions(+), 4 deletions(-) create mode 100644 tools/h5repack/testfiles/h5repack_layout.h5.ddl diff --git a/MANIFEST b/MANIFEST index 9911380..a747476 100644 --- a/MANIFEST +++ b/MANIFEST @@ -2115,6 +2115,7 @@ ./tools/h5repack/testfiles/h5repack_attr.h5 ./tools/h5repack/testfiles/h5repack_hlink.h5 ./tools/h5repack/testfiles/h5repack_layout.h5 +./tools/h5repack/testfiles/h5repack_layout.h5.ddl ./tools/h5repack/testfiles/h5repack_layouto.h5 ./tools/h5repack/testfiles/h5repack_layout2.h5 ./tools/h5repack/testfiles/h5repack_layout3.h5 diff --git a/tools/h5repack/CMakeTests.cmake b/tools/h5repack/CMakeTests.cmake index 79d75a7..0734220 100644 --- a/tools/h5repack/CMakeTests.cmake +++ b/tools/h5repack/CMakeTests.cmake @@ -82,6 +82,7 @@ ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_ext.bin ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/ublock.bin ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack.info + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_layout.h5.ddl # tools/testfiles ${HDF5_TOOLS_SRC_DIR}/testfiles/h5repack_filters.h5.ddl ) @@ -163,7 +164,9 @@ ELSE (${testtype} STREQUAL "SKIP") # If using memchecker add tests without using scripts IF (HDF5_ENABLE_USING_MEMCHECKER) - ADD_TEST (NAME H5REPACK_CMP-${testname} COMMAND $ ${ARGN} testfiles/${resultfile} testfiles/out-${testname}.${resultfile}) + ADD_TEST ( + NAME H5REPACK_CMP-${testname} + COMMAND $ ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile}) ELSE (HDF5_ENABLE_USING_MEMCHECKER) ADD_TEST ( NAME H5REPACK_CMP-${testname} @@ -184,6 +187,39 @@ ENDIF (${testtype} STREQUAL "SKIP") ENDMACRO (ADD_H5_CMP_TEST) + MACRO (ADD_H5_DMP_TEST testname testtype resultcode resultfile) + IF (${testtype} STREQUAL "SKIP") + IF (NOT HDF5_ENABLE_USING_MEMCHECKER) + ADD_TEST ( + NAME H5REPACK_DMP-${testname}-SKIPPED + COMMAND ${CMAKE_COMMAND} -E echo "SKIP ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile}" + ) + ENDIF (NOT HDF5_ENABLE_USING_MEMCHECKER) + ELSE (${testtype} STREQUAL "SKIP") + # If using memchecker add tests without using scripts + ADD_TEST ( + NAME H5REPACK_DMP-${testname} + COMMAND $ ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile}) + IF (NOT "${last_test}" STREQUAL "") + SET_TESTS_PROPERTIES (H5REPACK_DMP-${testname} PROPERTIES DEPENDS ${last_test}) + ENDIF (NOT "${last_test}" STREQUAL "") + IF (NOT HDF5_ENABLE_USING_MEMCHECKER) + ADD_TEST ( + NAME H5REPACK_DMP-h5dump-${testname} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=$" + -D "TEST_ARGS:STRING=-pH;out-${testname}.${resultfile}" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles" + -D "TEST_OUTPUT=${resultfile}-${testname}.out" + -D "TEST_EXPECT=${resultcode}" + -D "TEST_REFERENCE=${resultfile}.ddl" + -P "${HDF5_RESOURCES_DIR}/runTest.cmake" + ) + SET_TESTS_PROPERTIES (H5REPACK_DMP-h5dump-${testname} PROPERTIES DEPENDS "H5REPACK_DMP-${testname}") + ENDIF (NOT HDF5_ENABLE_USING_MEMCHECKER) + ENDIF (${testtype} STREQUAL "SKIP") + ENDMACRO (ADD_H5_DMP_TEST) + MACRO (ADD_H5_VERIFY_TEST testname testtype resultcode testfile testdset testfilter) IF (${testtype} STREQUAL "SKIP") IF (NOT HDF5_ENABLE_USING_MEMCHECKER) @@ -768,7 +804,7 @@ IF (NOT USE_FILTER_DEFLATE) SET (TESTTYPE "SKIP") ENDIF (NOT USE_FILTER_DEFLATE) - ADD_H5_TEST (deflate_limit ${TESTTYPE} ${arg}) + ADD_H5_DMP_TEST (deflate_limit ${TESTTYPE} 0 ${arg}) #file SET (arg ${FILE4} -e ${INFO_FILE}) diff --git a/tools/h5repack/h5repack.c b/tools/h5repack/h5repack.c index 38a000a..d6437cf 100644 --- a/tools/h5repack/h5repack.c +++ b/tools/h5repack/h5repack.c @@ -93,7 +93,7 @@ h5repack_init(pack_opt_t *options, int verbose, H5F_file_space_type_t strategy, int k, n; HDmemset(options, 0, sizeof(pack_opt_t)); - options->min_comp = 1024; + options->min_comp = 0; options->verbose = verbose; options->layout_g = H5D_LAYOUT_ERROR; diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c index f9dd334..02337fd 100644 --- a/tools/h5repack/h5repack_copy.c +++ b/tools/h5repack/h5repack_copy.c @@ -1017,6 +1017,16 @@ int do_copy_objects(hid_t fidin, /* get the storage size of the input dataset */ dsize_in=H5Dget_storage_size(dset_in); + /* check for small size datasets (less than 1k) except + * changing to COMPACT. For the reference, COMPACT is limited + * by size 64K by library. + */ + if (options->layout_g != H5D_COMPACT) + { + if ( size_dset < options->min_comp ) + apply_s=0; + } + /* apply the filter */ if (apply_s) { diff --git a/tools/h5repack/h5repack_main.c b/tools/h5repack/h5repack_main.c index 921b390..ff38068 100644 --- a/tools/h5repack/h5repack_main.c +++ b/tools/h5repack/h5repack_main.c @@ -180,7 +180,7 @@ static void usage(const char *prog) printf("\n"); - printf(" M - is an integer greater than 1, size of dataset in bytes (default is 1024) \n"); + printf(" M - is an integer greater than 1, size of dataset in bytes (default is 0) \n"); printf(" E - is a filename.\n"); printf(" S - is an integer\n"); printf(" U - is a filename.\n"); diff --git a/tools/h5repack/testfiles/h5repack_layout.h5.ddl b/tools/h5repack/testfiles/h5repack_layout.h5.ddl new file mode 100644 index 0000000..b8e7e68 --- /dev/null +++ b/tools/h5repack/testfiles/h5repack_layout.h5.ddl @@ -0,0 +1,130 @@ +HDF5 "out-deflate_limit.h5repack_layout.h5" { +GROUP "/" { + DATASET "dset1" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } + STORAGE_LAYOUT { + CHUNKED ( 40, 20 ) + SIZE 1150 (2.783:1 COMPRESSION) + } + FILTERS { + COMPRESSION DEFLATE { LEVEL 1 } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE 0 + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } + DATASET "dset2" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } + STORAGE_LAYOUT { + CHUNKED ( 40, 20 ) + SIZE 1150 (2.783:1 COMPRESSION) + } + FILTERS { + COMPRESSION DEFLATE { LEVEL 1 } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE 0 + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } + DATASET "dset3" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } + STORAGE_LAYOUT { + CHUNKED ( 40, 20 ) + SIZE 1150 (2.783:1 COMPRESSION) + } + FILTERS { + COMPRESSION DEFLATE { LEVEL 1 } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE 0 + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } + DATASET "dset4" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } + STORAGE_LAYOUT { + CHUNKED ( 40, 20 ) + SIZE 1150 (2.783:1 COMPRESSION) + } + FILTERS { + COMPRESSION DEFLATE { LEVEL 1 } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE 0 + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } + DATASET "dset_chunk" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 40, 20 ) / ( 40, H5S_UNLIMITED ) } + STORAGE_LAYOUT { + CHUNKED ( 20, 10 ) + SIZE 1283 (2.494:1 COMPRESSION) + } + FILTERS { + COMPRESSION DEFLATE { LEVEL 1 } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE 0 + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } + DATASET "dset_compact" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } + STORAGE_LAYOUT { + CHUNKED ( 40, 20 ) + SIZE 1150 (2.783:1 COMPRESSION) + } + FILTERS { + COMPRESSION DEFLATE { LEVEL 1 } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE 0 + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } + DATASET "dset_contiguous" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } + STORAGE_LAYOUT { + CHUNKED ( 40, 20 ) + SIZE 1150 (2.783:1 COMPRESSION) + } + FILTERS { + COMPRESSION DEFLATE { LEVEL 1 } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE 0 + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } +} +} -- cgit v0.12