diff options
Diffstat (limited to 'tools/h5repack')
-rw-r--r-- | tools/h5repack/CMakeLists.txt | 873 | ||||
-rw-r--r-- | tools/h5repack/Makefile.in | 3 | ||||
-rw-r--r-- | tools/h5repack/h5repack.c | 86 | ||||
-rwxr-xr-x | tools/h5repack/h5repack.sh.in | 353 | ||||
-rw-r--r-- | tools/h5repack/h5repack_copy.c | 456 | ||||
-rw-r--r-- | tools/h5repack/h5repack_filters.c | 7 | ||||
-rw-r--r-- | tools/h5repack/h5repack_main.c | 68 | ||||
-rw-r--r-- | tools/h5repack/h5repack_opttable.c | 15 | ||||
-rw-r--r-- | tools/h5repack/h5repack_parse.c | 107 | ||||
-rw-r--r-- | tools/h5repack/h5repack_refs.c | 442 | ||||
-rw-r--r-- | tools/h5repack/h5repack_verify.c | 15 | ||||
-rw-r--r-- | tools/h5repack/h5repacktst.c | 149 | ||||
-rw-r--r-- | tools/h5repack/testfiles/h5repack_layout.h5 | bin | 29384 -> 27280 bytes | |||
-rw-r--r-- | tools/h5repack/testh5repack_detect_szip.c | 4 |
14 files changed, 1558 insertions, 1020 deletions
diff --git a/tools/h5repack/CMakeLists.txt b/tools/h5repack/CMakeLists.txt index 1593d35..6460375 100644 --- a/tools/h5repack/CMakeLists.txt +++ b/tools/h5repack/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required (VERSION 2.8) +cmake_minimum_required (VERSION 2.8.6) PROJECT (HDF5_TOOLS_H5REPACK) #----------------------------------------------------------------------------- @@ -23,6 +23,7 @@ SET (REPACK_COMMON_SRCS ADD_EXECUTABLE (h5repack ${REPACK_COMMON_SRCS} ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/h5repack_main.c) TARGET_NAMING (h5repack ${LIB_TYPE}) TARGET_LINK_LIBRARIES (h5repack ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) +SET_TARGET_PROPERTIES (h5repack PROPERTIES FOLDER tools) SET (H5_DEP_EXECUTABLES h5repack) @@ -39,30 +40,14 @@ IF (BUILD_TESTING) ADD_EXECUTABLE (testh5repack_detect_szip ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testh5repack_detect_szip.c) TARGET_NAMING (testh5repack_detect_szip ${LIB_TYPE}) TARGET_LINK_LIBRARIES (testh5repack_detect_szip ${HDF5_LIB_TARGET} ${HDF5_TOOLS_LIB_TARGET} ${HDF5_TEST_LIB_TARGET}) - - ADD_TEST (NAME testh5repack_detect_szip COMMAND $<TARGET_FILE:testh5repack_detect_szip>) - IF (HDF5_ENABLE_SZIP_SUPPORT) - IF (HDF5_ENABLE_SZIP_ENCODING) - SET (passRegex "yes") - SET_TESTS_PROPERTIES (testh5repack_detect_szip PROPERTIES PASS_REGULAR_EXPRESSION "yes") - ELSE (HDF5_ENABLE_SZIP_ENCODING) - SET (passRegex "no") - SET_TESTS_PROPERTIES (testh5repack_detect_szip PROPERTIES PASS_REGULAR_EXPRESSION "no") - ENDIF (HDF5_ENABLE_SZIP_ENCODING) - ELSE (HDF5_ENABLE_SZIP_SUPPORT) - SET (passRegex "no") - SET_TESTS_PROPERTIES (testh5repack_detect_szip PROPERTIES PASS_REGULAR_EXPRESSION "no") - ENDIF (HDF5_ENABLE_SZIP_SUPPORT) + SET_TARGET_PROPERTIES (testh5repack_detect_szip PROPERTIES FOLDER tools) ADD_EXECUTABLE (h5repacktest ${REPACK_COMMON_SRCS} ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/h5repacktst.c) TARGET_NAMING (h5repacktest ${LIB_TYPE}) TARGET_LINK_LIBRARIES (h5repacktest ${HDF5_TOOLS_LIB_TARGET} ${HDF5_TEST_LIB_TARGET}) - - ADD_TEST (NAME h5repacktest COMMAND $<TARGET_FILE:h5repacktest>) - SET_TESTS_PROPERTIES (h5repacktest PROPERTIES DEPENDS testh5repack_detect_szip) + SET_TARGET_PROPERTIES (h5repacktest PROPERTIES FOLDER tools) IF (HDF5_TEST_VFD) - SET (VFD_LIST sec2 stdio @@ -78,7 +63,7 @@ IF (BUILD_TESTING) MACRO (ADD_VFD_TEST vfdname resultcode) ADD_TEST ( - NAME VFD-${vfdname}-h5repacktest + NAME H5REPACK-VFD-${vfdname}-h5repacktest COMMAND "${CMAKE_COMMAND}" -D "TEST_PROGRAM=$<TARGET_FILE:h5repacktest>" -D "TEST_ARGS:STRING=" @@ -88,79 +73,72 @@ IF (BUILD_TESTING) -D "TEST_FOLDER=${PROJECT_BINARY_DIR}" -P "${HDF5_RESOURCES_DIR}/vfdTest.cmake" ) + IF (NOT "${last_test}" STREQUAL "") + SET_TESTS_PROPERTIES (H5REPACK-VFD-${vfdname}-h5repacktest PROPERTIES DEPENDS ${last_test}) + ENDIF (NOT "${last_test}" STREQUAL "") + SET (last_test "H5REPACK-VFD-${vfdname}-h5repacktest") ENDMACRO (ADD_VFD_TEST) - - # Run test with different Virtual File Driver - FOREACH (vfd ${VFD_LIST}) - ADD_VFD_TEST (${vfd} 0) - ENDFOREACH (vfd ${VFD_LIST}) - ENDIF (HDF5_TEST_VFD) # -------------------------------------------------------------------- - # Copy all the HDF5 files from the test directory into the source directory + # Copy all the HDF5 files from the source directory into the test directory # -------------------------------------------------------------------- - SET (HDF5_REFERENCE_TEST_FILES - h5repack_ext.bin - ublock.bin - h5repack.info - h5repack_attr.h5 - h5repack_attr_refs.h5 - h5repack_deflate.h5 - h5repack_early.h5 - h5repack_ext.h5 - h5repack_fill.h5 - h5repack_filters.h5 - h5repack_fletcher.h5 - h5repack_hlink.h5 - h5repack_layout.h5 - h5repack_layouto.h5 - h5repack_layout2.h5 - h5repack_named_dtypes.h5 - h5repack_nbit.h5 - h5repack_objs.h5 - h5repack_refs.h5 - h5repack_shuffle.h5 - h5repack_soffset.h5 - h5repack_szip.h5 + SET (LIST_HDF5_TEST_FILES + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_attr.h5 + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_attr_refs.h5 + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_deflate.h5 + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_early.h5 + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_ext.h5 + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_fill.h5 + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_filters.h5 + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_fletcher.h5 + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_hlink.h5 + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_layout.h5 + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_layouto.h5 + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_layout2.h5 + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_named_dtypes.h5 + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_nbit.h5 + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_objs.h5 + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_refs.h5 + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_shuffle.h5 + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_soffset.h5 + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_szip.h5 + # h5diff/testfile + ${HDF5_TOOLS_H5DIFF_SOURCE_DIR}/testfiles/h5diff_attr1.h5 + # tools/testfiles + ${HDF5_TOOLS_SRC_DIR}/testfiles/tfamily00000.h5 + ${HDF5_TOOLS_SRC_DIR}/testfiles/tfamily00001.h5 + ${HDF5_TOOLS_SRC_DIR}/testfiles/tfamily00002.h5 + ${HDF5_TOOLS_SRC_DIR}/testfiles/tfamily00003.h5 + ${HDF5_TOOLS_SRC_DIR}/testfiles/tfamily00004.h5 + ${HDF5_TOOLS_SRC_DIR}/testfiles/tfamily00005.h5 + ${HDF5_TOOLS_SRC_DIR}/testfiles/tfamily00006.h5 + ${HDF5_TOOLS_SRC_DIR}/testfiles/tfamily00007.h5 + ${HDF5_TOOLS_SRC_DIR}/testfiles/tfamily00008.h5 + ${HDF5_TOOLS_SRC_DIR}/testfiles/tfamily00009.h5 + ${HDF5_TOOLS_SRC_DIR}/testfiles/tfamily00010.h5 ) - SET (HDF5_COMMON_TEST_FILES - tfamily00000.h5 - tfamily00001.h5 - tfamily00002.h5 - tfamily00003.h5 - tfamily00004.h5 - tfamily00005.h5 - tfamily00006.h5 - tfamily00007.h5 - tfamily00008.h5 - tfamily00009.h5 - tfamily00010.h5 - h5repack_filters.h5.ddl + + SET (LIST_OTHER_TEST_FILES + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_ext.bin + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/ublock.bin + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack.info + # tools/testfiles + ${HDF5_TOOLS_SRC_DIR}/testfiles/h5repack_filters.h5.ddl ) - FOREACH (h5_file ${HDF5_REFERENCE_TEST_FILES}) - SET (dest "${PROJECT_BINARY_DIR}/testfiles/${h5_file}") + FOREACH (h5_file ${LIST_HDF5_TEST_FILES} ${LIST_OTHER_TEST_FILES}) + GET_FILENAME_COMPONENT(fname "${h5_file}" NAME) + SET (dest "${PROJECT_BINARY_DIR}/testfiles/${fname}") #MESSAGE (STATUS " Copying ${h5_file}") ADD_CUSTOM_COMMAND ( TARGET h5repack POST_BUILD COMMAND ${CMAKE_COMMAND} - ARGS -E copy_if_different ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/${h5_file} ${dest} + ARGS -E copy_if_different ${h5_file} ${dest} ) - ENDFOREACH (h5_file ${HDF5_REFERENCE_TEST_FILES}) + ENDFOREACH (h5_file ${LIST_HDF5_TEST_FILES} ${LIST_OTHER_TEST_FILES}) - FOREACH (h5c_file ${HDF5_COMMON_TEST_FILES}) - SET (dest "${PROJECT_BINARY_DIR}/testfiles/${h5c_file}") - #MESSAGE (STATUS " Copying ${h5_file}") - ADD_CUSTOM_COMMAND ( - TARGET h5repack - POST_BUILD - COMMAND ${CMAKE_COMMAND} - ARGS -E copy_if_different ${HDF5_TOOLS_SRC_DIR}/testfiles/${h5c_file} ${dest} - ) - ENDFOREACH (h5c_file ${HDF5_COMMON_TEST_FILES}) - ############################################################################## ############################################################################## ### T H E T E S T S M A C R O S ### @@ -211,7 +189,7 @@ IF (BUILD_TESTING) -D "TEST_PROGRAM=$<TARGET_FILE:h5repack>" -D "TEST_ARGS:STRING=${ARGN};testfiles/${resultfile};testfiles/out.${resultfile}" -D "TEST_FOLDER=${PROJECT_BINARY_DIR}" - -D "TEST_OUTPUT=${resultfile}.out" + -D "TEST_OUTPUT=./testfiles/${resultfile}.out" -D "TEST_EXPECT=${resultcode}" -D "TEST_FILTER:STRING=${testfilter}" -D "TEST_REFERENCE=testfiles/${resultfile}.ddl" @@ -242,7 +220,7 @@ IF (BUILD_TESTING) -D "TEST_PROGRAM=$<TARGET_FILE:h5dump>" -D "TEST_ARGS:STRING=-d;${testdset};-pH;testfiles/out.${testfile}" -D "TEST_FOLDER=${PROJECT_BINARY_DIR}" - -D "TEST_OUTPUT=${testfile}-v.out" + -D "TEST_OUTPUT=./testfiles/${testfile}-v.out" -D "TEST_EXPECT=${resultcode}" -D "TEST_FILTER:STRING=${testfilter}" -D "TEST_REFERENCE=${testfilter}" @@ -268,7 +246,7 @@ IF (BUILD_TESTING) -D "TEST_PROGRAM=$<TARGET_FILE:h5dump>" -D "TEST_ARGS:STRING=-pH;testfiles/out.${testfile}" -D "TEST_FOLDER=${PROJECT_BINARY_DIR}" - -D "TEST_OUTPUT=${testfile}-v.out" + -D "TEST_OUTPUT=./testfiles/${testfile}-v.out" -D "TEST_EXPECT=${resultcode}" -D "TEST_FILTER:STRING=${nottestfilter}" -D "TEST_REFERENCE=${testfilter}" @@ -294,65 +272,69 @@ IF (BUILD_TESTING) NAME H5REPACK-clearall-objects COMMAND ${CMAKE_COMMAND} -E remove - h5repack_attr.h5.out - h5repack_attr.h5.out.err - h5repack_deflate.h5.out - h5repack_deflate.h5.out.err - h5repack_early.h5.out - h5repack_early.h5.out.err - h5repack_ext.h5.out - h5repack_ext.h5.out.err - h5repack_fill.h5.out - h5repack_fill.h5.out.err - h5repack_filters.h5.out - h5repack_filters.h5.out.err - h5repack_fletcher.h5.out - h5repack_fletcher.h5.out.err - h5repack_hlink.h5.out - h5repack_hlink.h5.out.err - h5repack_layout.h5.out - h5repack_layout.h5.out.err - h5repack_layout.h5-v.out - h5repack_layout.h5-v.out.err - h5repack_layouto.h5.out - h5repack_layouto.h5.out.err - h5repack_layout2.h5-v.out - h5repack_layout2.h5-v.out.err - h5repack_named_dtypes.h5.out - h5repack_named_dtypes.h5.out.err - h5repack_nbit.h5.out - h5repack_nbit.h5.out.err - h5repack_objs.h5.out - h5repack_objs.h5.out.err - h5repack_refs.h5.out - h5repack_refs.h5.out.err - h5repack_shuffle.h5.out - h5repack_shuffle.h5.out.err - h5repack_soffset.h5.out - h5repack_soffset.h5.out.err - h5repack_szip.h5.out - h5repack_szip.h5.out.err - h5repack_attr_out.h5 - h5repack_deflate_out.h5 - h5repack_early_out.h5 - h5repack_ext_out.h5 - h5repack_fill_out.h5 - h5repack_filters_out.h5 - h5repack_fletcher_out.h5 - h5repack_hlink_out.h5 - h5repack_layout_out.h5 - h5repack_layouto_out.h5 - h5repack_layout2_out.h5 - h5repack_named_dtypes_out.h5 - h5repack_nbit_out.h5 - h5repack_objs_out.h5 - h5repack_refs_out.h5 - h5repack_shuffle_out.h5 - h5repack_soffset_out.h5 - h5repack_szip_out.h5 - h5repack_ub.h5 - h5repack_ub_out.h5 - h5repack_attr_refs_out.h5 + ./testfiles/h5repack_filters.h5.out + ./testfiles/h5repack_filters.h5.out.err + ./testfiles/h5repack_layout2.h5-v.out + ./testfiles/h5repack_layout2.h5-v.out.err + ./testfiles/h5repack_layout.h5-v.out + ./testfiles/h5repack_layout.h5-v.out.err + ./testfiles/out.h5diff_attr1.h5 + ./testfiles/out.h5repack_attr.h5 + ./testfiles/out.h5repack_deflate.h5 + ./testfiles/out.h5repack_early.h5 + ./testfiles/out.h5repack_fill.h5 + ./testfiles/out.h5repack_filters.h5 + ./testfiles/out.h5repack_fletcher.h5 + ./testfiles/out.h5repack_hlink.h5 + ./testfiles/out.h5repack_layout2.h5 + ./testfiles/out.h5repack_layout.h5 + ./testfiles/out.h5repack_layouto.h5 + ./testfiles/out.h5repack_named_dtypes.h5 + ./testfiles/out.h5repack_nbit.h5 + ./testfiles/out.h5repack_objs.h5 + ./testfiles/out.h5repack_refs.h5 + ./testfiles/out.h5repack_shuffle.h5 + ./testfiles/out.h5repack_soffset.h5 + # from the h5repacktst + h5repack_attr.h5 + h5repack_attr_out.h5 + h5repack_attr_refs.h5 + h5repack_big.h5 + h5repack_deflate.h5 + h5repack_deflate_out.h5 + h5repack_early2.h5 + h5repack_early.h5 + h5repack_early_out.h5 + h5repack_ext.h5 + h5repack_ext_out.h5 + h5repack_fill.h5 + h5repack_fill_out.h5 + h5repack_filters.h5 + h5repack_filters_out.h5 + h5repack_fletcher.h5 + h5repack_fletcher_out.h5 + h5repack_hlink.h5 + h5repack_hlink_out.h5 + h5repack_layout2.h5 + h5repack_layout.h5 + h5repack_layout_out.h5 + h5repack_named_dtypes.h5 + h5repack_named_dtypes_out.h5 + h5repack_nbit.h5 + h5repack_nbit_out.h5 + h5repack_objs.h5 + h5repack_objs_out.h5 + h5repack_refs.h5 + h5repack_shuffle.h5 + h5repack_shuffle_out.h5 + h5repack_soffset.h5 + h5repack_soffset_out.h5 + h5repack_szip.h5 + h5repack_szip_out.h5 + h5repack_ub.h5 + h5repack_ub_out.h5 + h5repack_ext.bin + ublock.bin ) IF (NOT "${last_test}" STREQUAL "") SET_TESTS_PROPERTIES (H5REPACK-clearall-objects PROPERTIES DEPENDS ${last_test}) @@ -384,6 +366,30 @@ IF (BUILD_TESTING) SET (FILE_REF h5repack_refs.h5) SET (FILE_ATTR_REF h5repack_attr_refs.h5) + ADD_TEST (NAME H5REPACK-testh5repack_detect_szip COMMAND $<TARGET_FILE:testh5repack_detect_szip>) + IF (HDF5_ENABLE_SZIP_SUPPORT) + IF (HDF5_ENABLE_SZIP_ENCODING) + SET (passRegex "yes") + SET_TESTS_PROPERTIES (H5REPACK-testh5repack_detect_szip PROPERTIES PASS_REGULAR_EXPRESSION "yes") + ELSE (HDF5_ENABLE_SZIP_ENCODING) + SET (passRegex "no") + SET_TESTS_PROPERTIES (H5REPACK-testh5repack_detect_szip PROPERTIES PASS_REGULAR_EXPRESSION "no") + ENDIF (HDF5_ENABLE_SZIP_ENCODING) + ELSE (HDF5_ENABLE_SZIP_SUPPORT) + SET (passRegex "no") + SET_TESTS_PROPERTIES (H5REPACK-testh5repack_detect_szip PROPERTIES PASS_REGULAR_EXPRESSION "no") + ENDIF (HDF5_ENABLE_SZIP_SUPPORT) + IF (NOT "${last_test}" STREQUAL "") + SET_TESTS_PROPERTIES (H5REPACK-testh5repack_detect_szip PROPERTIES DEPENDS ${last_test}) + ENDIF (NOT "${last_test}" STREQUAL "") + SET (last_test "H5REPACK-testh5repack_detect_szip") + + ADD_TEST (NAME H5REPACK-h5repacktest COMMAND $<TARGET_FILE:h5repacktest>) + IF (NOT "${last_test}" STREQUAL "") + SET_TESTS_PROPERTIES (H5REPACK-h5repacktest PROPERTIES DEPENDS ${last_test}) + ENDIF (NOT "${last_test}" STREQUAL "") + SET (last_test "H5REPACK-h5repacktest") + # # The tests # We use the files generated by h5repacktst @@ -398,398 +404,409 @@ IF (BUILD_TESTING) # filters are defined. # detect whether the encoder is present. -SET (USE_FILTER_SZIP_ENCODER "no") -IF (HDF5_ENABLE_SZIP_ENCODING) - SET (USE_FILTER_SZIP_ENCODER ${testh5repack_detect_szip}) -ENDIF (HDF5_ENABLE_SZIP_ENCODING) - -IF (H5_HAVE_FILTER_DEFLATE) - SET (USE_FILTER_DEFLATE "true") -ENDIF (H5_HAVE_FILTER_DEFLATE) - -IF (H5_HAVE_FILTER_SZIP) - SET (USE_FILTER_SZIP "true") -ENDIF (H5_HAVE_FILTER_SZIP) - -IF (H5_HAVE_FILTER_SHUFFLE) - SET (USE_FILTER_SHUFFLE "true") -ENDIF (H5_HAVE_FILTER_SHUFFLE) - -IF (H5_HAVE_FILTER_FLETCHER32) - SET (USE_FILTER_FLETCHER32 "true") -ENDIF (H5_HAVE_FILTER_FLETCHER32) - -IF (H5_HAVE_FILTER_NBIT) - SET (USE_FILTER_NBIT "true") -ENDIF (H5_HAVE_FILTER_NBIT) - -IF (H5_HAVE_FILTER_SCALEOFFSET) - SET (USE_FILTER_SCALEOFFSET "true") -ENDIF (H5_HAVE_FILTER_SCALEOFFSET) + SET (USE_FILTER_SZIP_ENCODER "no") + IF (HDF5_ENABLE_SZIP_ENCODING) + SET (USE_FILTER_SZIP_ENCODER ${testh5repack_detect_szip}) + ENDIF (HDF5_ENABLE_SZIP_ENCODING) + + IF (H5_HAVE_FILTER_DEFLATE) + SET (USE_FILTER_DEFLATE "true") + ENDIF (H5_HAVE_FILTER_DEFLATE) + + IF (H5_HAVE_FILTER_SZIP) + SET (USE_FILTER_SZIP "true") + ENDIF (H5_HAVE_FILTER_SZIP) + + IF (H5_HAVE_FILTER_SHUFFLE) + SET (USE_FILTER_SHUFFLE "true") + ENDIF (H5_HAVE_FILTER_SHUFFLE) + + IF (H5_HAVE_FILTER_FLETCHER32) + SET (USE_FILTER_FLETCHER32 "true") + ENDIF (H5_HAVE_FILTER_FLETCHER32) + + IF (H5_HAVE_FILTER_NBIT) + SET (USE_FILTER_NBIT "true") + ENDIF (H5_HAVE_FILTER_NBIT) + + IF (H5_HAVE_FILTER_SCALEOFFSET) + SET (USE_FILTER_SCALEOFFSET "true") + ENDIF (H5_HAVE_FILTER_SCALEOFFSET) # copy files (these files have no filters) -ADD_H5_TEST (fill "TEST" ${FILE0}) -ADD_H5_TEST (objs "TEST" ${FILE1}) -ADD_H5_TEST (attr "TEST" ${FILE2}) -ADD_H5_TEST (hlink "TEST" ${FILE3}) -ADD_H5_TEST (layout "TEST" ${FILE4}) -ADD_H5_TEST (early "TEST" ${FILE5}) + ADD_H5_TEST (fill "TEST" ${FILE0}) + ADD_H5_TEST (objs "TEST" ${FILE1}) + ADD_H5_TEST (attr "TEST" ${FILE2}) + ADD_H5_TEST (hlink "TEST" ${FILE3}) + ADD_H5_TEST (layout "TEST" ${FILE4}) + ADD_H5_TEST (early "TEST" ${FILE5}) # use $FILE4 to write some filters (this file has no filters) # gzip with individual object -SET (arg ${FILE4} -f dset1:GZIP=1 -l dset1:CHUNK=20x10) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_DEFLATE) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_DEFLATE) -ADD_H5_TEST (gzip_individual ${TESTTYPE} ${arg}) + SET (arg ${FILE4} -f dset1:GZIP=1 -l dset1:CHUNK=20x10) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_DEFLATE) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_DEFLATE) + ADD_H5_TEST (gzip_individual ${TESTTYPE} ${arg}) # gzip for all -SET (arg ${FILE4} -f GZIP=1) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_DEFLATE) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_DEFLATE) -ADD_H5_TEST (gzip_all ${TESTTYPE} ${arg}) + SET (arg ${FILE4} -f GZIP=1) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_DEFLATE) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_DEFLATE) + ADD_H5_TEST (gzip_all ${TESTTYPE} ${arg}) # szip with individual object -SET (arg ${FILE4} -f dset2:SZIP=8,EC -l dset2:CHUNK=20x10) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP) -ADD_H5_TEST (szip_individual ${TESTTYPE} ${arg}) + SET (arg ${FILE4} -f dset2:SZIP=8,EC -l dset2:CHUNK=20x10) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP) + ADD_H5_TEST (szip_individual ${TESTTYPE} ${arg}) # szip for all -SET (arg ${FILE4} -f SZIP=8,NN) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP) -ADD_H5_TEST (szip_all ${TESTTYPE} ${arg}) + SET (arg ${FILE4} -f SZIP=8,NN) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP) + ADD_H5_TEST (szip_all ${TESTTYPE} ${arg}) # shuffle with individual object -SET (arg ${FILE4} -f dset2:SHUF -l dset2:CHUNK=20x10) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_SHUFFLE) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_SHUFFLE) -ADD_H5_TEST (shuffle_individual ${TESTTYPE} ${arg}) + SET (arg ${FILE4} -f dset2:SHUF -l dset2:CHUNK=20x10) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_SHUFFLE) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_SHUFFLE) + ADD_H5_TEST (shuffle_individual ${TESTTYPE} ${arg}) # shuffle for all -SET (arg ${FILE4} -f SHUF) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_SHUFFLE) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_SHUFFLE) -ADD_H5_TEST (shuffle_all ${TESTTYPE} ${arg}) + SET (arg ${FILE4} -f SHUF) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_SHUFFLE) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_SHUFFLE) + ADD_H5_TEST (shuffle_all ${TESTTYPE} ${arg}) # fletcher32 with individual object -SET (arg ${FILE4} -f dset2:FLET -l dset2:CHUNK=20x10) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_FLETCHER32) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_FLETCHER32) -ADD_H5_TEST (fletcher_individual ${TESTTYPE} ${arg}) + SET (arg ${FILE4} -f dset2:FLET -l dset2:CHUNK=20x10) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_FLETCHER32) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_FLETCHER32) + ADD_H5_TEST (fletcher_individual ${TESTTYPE} ${arg}) # fletcher32 for all -SET (arg ${FILE4} -f FLET) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_FLETCHER32) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_FLETCHER32) -ADD_H5_TEST (fletcher_all ${TESTTYPE} ${arg}) + SET (arg ${FILE4} -f FLET) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_FLETCHER32) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_FLETCHER32) + ADD_H5_TEST (fletcher_all ${TESTTYPE} ${arg}) # all filters -SET (arg ${FILE4} -f dset2:SHUF -f dset2:FLET -f dset2:SZIP=8,NN -f dset2:GZIP=1 -l dset2:CHUNK=20x10) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP OR NOT USE_FILTER_SHUFFLE OR NOT USE_FILTER_FLETCHER32 OR NOT USE_FILTER_DEFLATE) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP OR NOT USE_FILTER_SHUFFLE OR NOT USE_FILTER_FLETCHER32 OR NOT USE_FILTER_DEFLATE) -ADD_H5_TEST (all_filters ${TESTTYPE} ${arg}) + SET (arg ${FILE4} -f dset2:SHUF -f dset2:FLET -f dset2:SZIP=8,NN -f dset2:GZIP=1 -l dset2:CHUNK=20x10) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP OR NOT USE_FILTER_SHUFFLE OR NOT USE_FILTER_FLETCHER32 OR NOT USE_FILTER_DEFLATE) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP OR NOT USE_FILTER_SHUFFLE OR NOT USE_FILTER_FLETCHER32 OR NOT USE_FILTER_DEFLATE) + ADD_H5_TEST (all_filters ${TESTTYPE} ${arg}) # verbose gzip with individual object -SET (arg ${FILE11} -v -f /dset_deflate:GZIP=9) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_DEFLATE) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_DEFLATE) -ADD_H5_CMP_TEST (gzip_verbose_filters "O?...ing file[^\n]+\n" ${TESTTYPE} 0 ${arg}) + SET (arg ${FILE11} -v -f /dset_deflate:GZIP=9) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_DEFLATE) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_DEFLATE) + ADD_H5_CMP_TEST (gzip_verbose_filters "O?...ing file[^\n]+\n" ${TESTTYPE} 0 ${arg}) ########################################################### # the following tests assume the input files have filters ########################################################### # szip copy -SET (arg ${FILE7}) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP) -ADD_H5_TEST (szip_copy ${TESTTYPE} ${arg}) + SET (arg ${FILE7}) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP) + ADD_H5_TEST (szip_copy ${TESTTYPE} ${arg}) # szip remove -SET (arg ${FILE7} --filter=dset_szip:NONE) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP) -ADD_H5_TEST (szip_remove ${TESTTYPE} ${arg}) + SET (arg ${FILE7} --filter=dset_szip:NONE) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP) + ADD_H5_TEST (szip_remove ${TESTTYPE} ${arg}) # deflate copy -SET (arg ${FILE8}) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_DEFLATE) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_DEFLATE) -ADD_H5_TEST (deflate_copy ${TESTTYPE} ${arg}) + SET (arg ${FILE8}) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_DEFLATE) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_DEFLATE) + ADD_H5_TEST (deflate_copy ${TESTTYPE} ${arg}) # deflate remove -SET (arg ${FILE8} -f dset_deflate:NONE) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_DEFLATE) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_DEFLATE) -ADD_H5_TEST (deflate_remove ${TESTTYPE} ${arg}) + SET (arg ${FILE8} -f dset_deflate:NONE) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_DEFLATE) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_DEFLATE) + ADD_H5_TEST (deflate_remove ${TESTTYPE} ${arg}) # shuffle copy -SET (arg ${FILE9}) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_SHUFFLE) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_SHUFFLE) -ADD_H5_TEST (shuffle_copy ${TESTTYPE} ${arg}) + SET (arg ${FILE9}) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_SHUFFLE) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_SHUFFLE) + ADD_H5_TEST (shuffle_copy ${TESTTYPE} ${arg}) # shuffle remove -SET (arg ${FILE9} -f dset_shuffle:NONE) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_SHUFFLE) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_SHUFFLE) -ADD_H5_TEST (shuffle_remove ${TESTTYPE} ${arg}) + SET (arg ${FILE9} -f dset_shuffle:NONE) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_SHUFFLE) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_SHUFFLE) + ADD_H5_TEST (shuffle_remove ${TESTTYPE} ${arg}) # fletcher32 copy -SET (arg ${FILE10}) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_FLETCHER32) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_FLETCHER32) -ADD_H5_TEST (fletcher_copy ${TESTTYPE} ${arg}) + SET (arg ${FILE10}) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_FLETCHER32) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_FLETCHER32) + ADD_H5_TEST (fletcher_copy ${TESTTYPE} ${arg}) # fletcher32 remove -SET (arg ${FILE10} -f dset_fletcher32:NONE) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_FLETCHER32) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_FLETCHER32) -ADD_H5_TEST (fletcher_remove ${TESTTYPE} ${arg}) + SET (arg ${FILE10} -f dset_fletcher32:NONE) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_FLETCHER32) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_FLETCHER32) + ADD_H5_TEST (fletcher_remove ${TESTTYPE} ${arg}) # nbit copy -SET (arg ${FILE12}) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_NBIT) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_NBIT) -ADD_H5_TEST (nbit_copy ${TESTTYPE} ${arg}) + SET (arg ${FILE12}) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_NBIT) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_NBIT) + ADD_H5_TEST (nbit_copy ${TESTTYPE} ${arg}) # nbit remove -SET (arg ${FILE12} -f dset_nbit:NONE) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_NBIT) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_NBIT) -ADD_H5_TEST (nbit_remove ${TESTTYPE} ${arg}) + SET (arg ${FILE12} -f dset_nbit:NONE) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_NBIT) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_NBIT) + ADD_H5_TEST (nbit_remove ${TESTTYPE} ${arg}) # nbit add -SET (arg ${FILE12} -f dset_int31:NBIT) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_NBIT) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_NBIT) -ADD_H5_TEST (nbit_add ${TESTTYPE} ${arg}) + SET (arg ${FILE12} -f dset_int31:NBIT) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_NBIT) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_NBIT) + ADD_H5_TEST (nbit_add ${TESTTYPE} ${arg}) # scaleoffset copy -SET (arg ${FILE13}) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_SCALEOFFSET) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_SCALEOFFSET) -ADD_H5_TEST (scale_copy ${TESTTYPE} ${arg}) + SET (arg ${FILE13}) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_SCALEOFFSET) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_SCALEOFFSET) + ADD_H5_TEST (scale_copy ${TESTTYPE} ${arg}) # scaleoffset add -SET (arg ${FILE13} -f dset_none:SOFF=31,IN) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_SCALEOFFSET) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_SCALEOFFSET) -ADD_H5_TEST (scale_add ${TESTTYPE} ${arg}) + SET (arg ${FILE13} -f dset_none:SOFF=31,IN) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_SCALEOFFSET) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_SCALEOFFSET) + ADD_H5_TEST (scale_add ${TESTTYPE} ${arg}) # scaleoffset remove -SET (arg ${FILE13} -f dset_scaleoffset:NONE) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_SCALEOFFSET) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_SCALEOFFSET) -ADD_H5_TEST (scale_remove ${TESTTYPE} ${arg}) + SET (arg ${FILE13} -f dset_scaleoffset:NONE) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_SCALEOFFSET) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_SCALEOFFSET) + ADD_H5_TEST (scale_remove ${TESTTYPE} ${arg}) # remove all filters -SET (arg ${FILE11} -f NONE) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_FLETCHER32 OR NOT USE_FILTER_DEFLATE OR NOT USE_FILTER_SZIP OR NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SHUFFLE OR NOT USE_FILTER_NBIT OR NOT USE_FILTER_SCALEOFFSET) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_FLETCHER32 OR NOT USE_FILTER_DEFLATE OR NOT USE_FILTER_SZIP OR NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SHUFFLE OR NOT USE_FILTER_NBIT OR NOT USE_FILTER_SCALEOFFSET) -ADD_H5_TEST (remove_all ${TESTTYPE} ${arg}) + SET (arg ${FILE11} -f NONE) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_FLETCHER32 OR NOT USE_FILTER_DEFLATE OR NOT USE_FILTER_SZIP OR NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SHUFFLE OR NOT USE_FILTER_NBIT OR NOT USE_FILTER_SCALEOFFSET) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_FLETCHER32 OR NOT USE_FILTER_DEFLATE OR NOT USE_FILTER_SZIP OR NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SHUFFLE OR NOT USE_FILTER_NBIT OR NOT USE_FILTER_SCALEOFFSET) + ADD_H5_TEST (remove_all ${TESTTYPE} ${arg}) #filter conversions - -SET (arg ${FILE8} -f dset_deflate:SZIP=8,NN) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP OR NOT USE_FILTER_DEFLATE) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP OR NOT USE_FILTER_DEFLATE) -ADD_H5_TEST (deflate_convert ${TESTTYPE} ${arg}) - -SET (arg ${FILE7} -f dset_szip:GZIP=1) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_SZIP OR NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_DEFLATE) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_SZIP OR NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_DEFLATE) -ADD_H5_TEST (szip_convert ${TESTTYPE} ${arg}) + SET (arg ${FILE8} -f dset_deflate:SZIP=8,NN) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP OR NOT USE_FILTER_DEFLATE) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP OR NOT USE_FILTER_DEFLATE) + ADD_H5_TEST (deflate_convert ${TESTTYPE} ${arg}) + + SET (arg ${FILE7} -f dset_szip:GZIP=1) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_SZIP OR NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_DEFLATE) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_SZIP OR NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_DEFLATE) + ADD_H5_TEST (szip_convert ${TESTTYPE} ${arg}) #limit -SET (arg ${FILE4} -f GZIP=1 -m 1024) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_DEFLATE) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_DEFLATE) -ADD_H5_TEST (deflate_limit ${TESTTYPE} ${arg}) + SET (arg ${FILE4} -f GZIP=1 -m 1024) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_DEFLATE) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_DEFLATE) + ADD_H5_TEST (deflate_limit ${TESTTYPE} ${arg}) #file -SET (arg ${FILE4} -e ${INFO_FILE}) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_DEFLATE) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_DEFLATE) -ADD_H5_TEST (deflate_file ${TESTTYPE} ${arg}) + SET (arg ${FILE4} -e ${INFO_FILE}) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_DEFLATE) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_DEFLATE) + ADD_H5_TEST (deflate_file ${TESTTYPE} ${arg}) ######################################################### # layout options (these files have no filters) ######################################################### - -ADD_H5_TEST (dset2_chunk_20x10 "TEST" ${FILE4} --layout=dset2:CHUNK=20x10) -ADD_H5_VERIFY_TEST (dset2_chunk_20x10 "TEST" 0 ${FILE4} dset2 CHUNKED) -ADD_H5_TEST (chunk_20x10 "TEST" ${FILE4} -l CHUNK=20x10) -ADD_H5_VERIFY_TEST (chunk_20x10 "TEST" 1 ${FILE4} null CHUNKED) -ADD_H5_TEST (dset2_conti "TEST" ${FILE4} -l dset2:CONTI) -ADD_H5_VERIFY_TEST (dset2_conti "TEST" 0 ${FILE4} dset2 CONTIGUOUS) -ADD_H5_TEST (conti "TEST" ${FILE4} -l CONTI) -ADD_H5_VERIFY_TEST (conti "TEST" 1 ${FILE4} null CONTIGUOUS) -ADD_H5_TEST (dset2_compa "TEST" ${FILE4} -l dset2:COMPA) -ADD_H5_VERIFY_TEST (dset2_compa "TEST" 0 ${FILE4} dset2 COMPACT) -ADD_H5_TEST (compa "TEST" ${FILE4} -l COMPA) -ADD_H5_VERIFY_TEST (compa "TEST" 1 ${FILE4} null COMPACT) + ADD_H5_TEST (dset2_chunk_20x10 "TEST" ${FILE4} --layout=dset2:CHUNK=20x10) + ADD_H5_VERIFY_TEST (dset2_chunk_20x10 "TEST" 0 ${FILE4} dset2 CHUNKED) + ADD_H5_TEST (chunk_20x10 "TEST" ${FILE4} -l CHUNK=20x10) + ADD_H5_VERIFY_TEST (chunk_20x10 "TEST" 1 ${FILE4} null CHUNKED) + ADD_H5_TEST (dset2_conti "TEST" ${FILE4} -l dset2:CONTI) + ADD_H5_VERIFY_TEST (dset2_conti "TEST" 0 ${FILE4} dset2 CONTIGUOUS) + ADD_H5_TEST (conti "TEST" ${FILE4} -l CONTI) + ADD_H5_VERIFY_TEST (conti "TEST" 1 ${FILE4} null CONTIGUOUS) + ADD_H5_TEST (dset2_compa "TEST" ${FILE4} -l dset2:COMPA) + ADD_H5_VERIFY_TEST (dset2_compa "TEST" 0 ${FILE4} dset2 COMPACT) + ADD_H5_TEST (compa "TEST" ${FILE4} -l COMPA) + ADD_H5_VERIFY_TEST (compa "TEST" 1 ${FILE4} null COMPACT) ################################################################ # layout conversions (file has no filters) ############################################################### - -ADD_H5_TEST (dset_compa_conti "TEST" ${FILE4} -l dset_compact:CONTI) -ADD_H5_VERIFY_TEST (dset_compa_conti "TEST" 0 ${FILE4} dset_compact CONTIGUOUS) -ADD_H5_TEST (dset_compa_chunk "TEST" ${FILE4} -l dset_compact:CHUNK=2x5) -ADD_H5_VERIFY_TEST (dset_compa_chunk "TEST" 0 ${FILE4} dset_compact CHUNKED) -ADD_H5_TEST (dset_compa_compa "TEST" ${FILE4} -l dset_compact:COMPA) -ADD_H5_VERIFY_TEST (dset_compa_compa "TEST" 0 ${FILE4} dset_compact COMPACT) -ADD_H5_TEST (dset_conti_compa "TEST" ${FILE4} -l dset_contiguous:COMPA) -ADD_H5_VERIFY_TEST (dset_conti_compa "TEST" 0 ${FILE4} dset_contiguous COMPACT) -ADD_H5_TEST (dset_conti_chunk "TEST" ${FILE4} -l dset_contiguous:CHUNK=3x6) -ADD_H5_VERIFY_TEST (dset_conti_chunk "TEST" 0 ${FILE4} dset_contiguous CHUNKED) -ADD_H5_TEST (dset_conti_conti "TEST" ${FILE4} -l dset_contiguous:CONTI) -ADD_H5_VERIFY_TEST (dset_conti_conti "TEST" 0 ${FILE4} dset_contiguous CONTIGUOUS) -ADD_H5_TEST (chunk_compa "TEST" ${FILE4} -l dset_chunk:COMPA) -ADD_H5_VERIFY_TEST (chunk_compa "TEST" 0 ${FILE4} dset_chunk COMPACT) -ADD_H5_TEST (chunk_conti "TEST" ${FILE4} -l dset_chunk:CONTI) -ADD_H5_VERIFY_TEST (chunk_conti "TEST" 0 ${FILE4} dset_chunk CONTIGUOUS) -ADD_H5_TEST (chunk_18x13 "TEST" ${FILE4} -l dset_chunk:CHUNK=18x13) -ADD_H5_VERIFY_TEST (chunk_18x13 "TEST" 0 ${FILE4} dset_chunk CHUNKED) + ADD_H5_TEST (dset_compa_conti "TEST" ${FILE4} -l dset_compact:CONTI) + ADD_H5_VERIFY_TEST (dset_compa_conti "TEST" 0 ${FILE4} dset_compact CONTIGUOUS) + ADD_H5_TEST (dset_compa_chunk "TEST" ${FILE4} -l dset_compact:CHUNK=2x5) + ADD_H5_VERIFY_TEST (dset_compa_chunk "TEST" 0 ${FILE4} dset_compact CHUNKED) + ADD_H5_TEST (dset_compa_compa "TEST" ${FILE4} -l dset_compact:COMPA) + ADD_H5_VERIFY_TEST (dset_compa_compa "TEST" 0 ${FILE4} dset_compact COMPACT) + ADD_H5_TEST (dset_conti_compa "TEST" ${FILE4} -l dset_contiguous:COMPA) + ADD_H5_VERIFY_TEST (dset_conti_compa "TEST" 0 ${FILE4} dset_contiguous COMPACT) + ADD_H5_TEST (dset_conti_chunk "TEST" ${FILE4} -l dset_contiguous:CHUNK=3x6) + ADD_H5_VERIFY_TEST (dset_conti_chunk "TEST" 0 ${FILE4} dset_contiguous CHUNKED) + ADD_H5_TEST (dset_conti_conti "TEST" ${FILE4} -l dset_contiguous:CONTI) + ADD_H5_VERIFY_TEST (dset_conti_conti "TEST" 0 ${FILE4} dset_contiguous CONTIGUOUS) + ADD_H5_TEST (chunk_compa "TEST" ${FILE4} -l dset_chunk:COMPA) + ADD_H5_VERIFY_TEST (chunk_compa "TEST" 0 ${FILE4} dset_chunk COMPACT) + ADD_H5_TEST (chunk_conti "TEST" ${FILE4} -l dset_chunk:CONTI) + ADD_H5_VERIFY_TEST (chunk_conti "TEST" 0 ${FILE4} dset_chunk CONTIGUOUS) + ADD_H5_TEST (chunk_18x13 "TEST" ${FILE4} -l dset_chunk:CHUNK=18x13) + ADD_H5_VERIFY_TEST (chunk_18x13 "TEST" 0 ${FILE4} dset_chunk CHUNKED) # test convert small size dataset ( < 1k) to compact layout without -m -ADD_H5_TEST (contig_small_compa "TEST" ${FILE18} -l contig_small:COMPA) -ADD_H5_VERIFY_TEST (contig_small_compa "TEST" 0 ${FILE18} contig_small COMPACT) + ADD_H5_TEST (contig_small_compa "TEST" ${FILE18} -l contig_small:COMPA) + ADD_H5_VERIFY_TEST (contig_small_compa "TEST" 0 ${FILE18} contig_small COMPACT) -ADD_H5_TEST (contig_small_fixed_compa "TEST" ${FILE18} -l chunked_small_fixed:COMPA) -ADD_H5_VERIFY_TEST (contig_small_fixed_compa "TEST" 0 ${FILE18} chunked_small_fixed COMPACT) + ADD_H5_TEST (contig_small_fixed_compa "TEST" ${FILE18} -l chunked_small_fixed:COMPA) + ADD_H5_VERIFY_TEST (contig_small_fixed_compa "TEST" 0 ${FILE18} chunked_small_fixed COMPACT) # Native option # Do not use FILE1, as the named dtype will be converted to native, and h5diff will # report a difference. -ADD_H5_TEST (native_fill "TEST" ${FILE0} -n) -ADD_H5_TEST (native_attr "TEST" ${FILE2} -n) - + ADD_H5_TEST (native_fill "TEST" ${FILE0} -n) + ADD_H5_TEST (native_attr "TEST" ${FILE2} -n) # latest file format with long switches. use FILE4=h5repack_layout.h5 (no filters) -SET (arg ${FILE4} --layout CHUNK=20x10 --filter GZIP=1 --minimum=10 --native --latest --compact=8 --indexed=6 --ssize=8[:dtype]) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_DEFLATE) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_DEFLATE) -ADD_H5_TEST (layout_long_switches ${TESTTYPE} ${arg}) -ADD_H5_VERIFY_TEST (layout_long_switches ${TESTTYPE} 1 ${FILE4} null CHUNKED) + SET (arg ${FILE4} --layout CHUNK=20x10 --filter GZIP=1 --minimum=10 --native --latest --compact=8 --indexed=6 --ssize=8[:dtype]) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_DEFLATE) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_DEFLATE) + ADD_H5_TEST (layout_long_switches ${TESTTYPE} ${arg}) + ADD_H5_VERIFY_TEST (layout_long_switches ${TESTTYPE} 1 ${FILE4} null CHUNKED) # latest file format with short switches. use FILE4=h5repack_layout.h5 (no filters) -SET (arg ${FILE4} -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype]) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_DEFLATE) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_DEFLATE) -ADD_H5_TEST (layout_short_switches ${TESTTYPE} ${arg}) -ADD_H5_VERIFY_TEST (layout_short_switches ${TESTTYPE} 1 ${FILE4} null CHUNKED) + SET (arg ${FILE4} -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype]) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_DEFLATE) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_DEFLATE) + ADD_H5_TEST (layout_short_switches ${TESTTYPE} ${arg}) + ADD_H5_VERIFY_TEST (layout_short_switches ${TESTTYPE} 1 ${FILE4} null CHUNKED) # several global filters -SET (arg ${FILE4} --filter GZIP=1 --filter SHUF) -SET (TESTTYPE "TEST") -IF (NOT USE_FILTER_DEFLATE OR NOT USE_FILTER_SHUFFLE) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_DEFLATE OR NOT USE_FILTER_SHUFFLE) -ADD_H5_TEST (global_filters ${TESTTYPE} ${arg}) + SET (arg ${FILE4} --filter GZIP=1 --filter SHUF) + SET (TESTTYPE "TEST") + IF (NOT USE_FILTER_DEFLATE OR NOT USE_FILTER_SHUFFLE) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_DEFLATE OR NOT USE_FILTER_SHUFFLE) + ADD_H5_TEST (global_filters ${TESTTYPE} ${arg}) # syntax of -i infile -o outfile # latest file format with short switches. use FILE4=h5repack_layout.h5 (no filters) -SET (arg ${FILE4} -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype]) -SET (TESTTYPE "LEGACY") -IF (NOT USE_FILTER_DEFLATE) - SET (TESTTYPE "SKIP") -ENDIF (NOT USE_FILTER_DEFLATE) -#ADD_H5_TEST (old_style_layout_short_switches ${TESTTYPE} ${arg}) + SET (arg ${FILE4} -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype]) + SET (TESTTYPE "LEGACY") + IF (NOT USE_FILTER_DEFLATE) + SET (TESTTYPE "SKIP") + ENDIF (NOT USE_FILTER_DEFLATE) +# ADD_H5_TEST (old_style_layout_short_switches ${TESTTYPE} ${arg}) # add a userblock to file -SET (arg ${FILE1} -u ${PROJECT_BINARY_DIR}/testfiles/ublock.bin -b 2048) -ADD_H5_TEST (add_userblock "TEST" ${arg}) + SET (arg ${FILE1} -u ${PROJECT_BINARY_DIR}/testfiles/ublock.bin -b 2048) + ADD_H5_TEST (add_userblock "TEST" ${arg}) # add alignment -SET (arg ${FILE1} -t 1 -a 1) -ADD_H5_TEST (add_alignment "TEST" ${arg}) + SET (arg ${FILE1} -t 1 -a 1) + ADD_H5_TEST (add_alignment "TEST" ${arg}) # Check repacking file with old version of layout message (should get upgraded -# to new version and be readable, etc.) -ADD_H5_TEST (upgrade_layout "TEST" ${FILE14}) +# to new version and be readable, etc.) + ADD_H5_TEST (upgrade_layout "TEST" ${FILE14}) # test for datum size > H5TOOLS_MALLOCSIZE -ADD_H5_TEST (gt_mallocsize "TEST" ${FILE1} -f GZIP=1) + ADD_H5_TEST (gt_mallocsize "TEST" ${FILE1} -f GZIP=1) # Check repacking file with committed datatypes in odd configurations -ADD_H5_TEST (committed_dt "TEST" ${FILE15}) + ADD_H5_TEST (committed_dt "TEST" ${FILE15}) # tests family driver (file is located in common testfiles folder, uses TOOLTEST1 -#ADD_H5_TEST ( family "FAMILY" ${FILE16}) +# ADD_H5_TEST ( family "FAMILY" ${FILE16}) # test various references (bug 1814 and 1726) -ADD_H5_TEST (bug1814 "TEST" ${FILE_REF}) + ADD_H5_TEST (bug1814 "TEST" ${FILE_REF}) -# test attribute with various references (bug 1797) -# the references in attribute of compund or vlen datatype +# test attribute with various references in attribute of compund +# or vlen datatype +# (HDFFV-2605, 7513) # TODO: include this test when code portion is completed. -ADD_H5_TEST (bug1797 "SKIP" ${FILE_ATTR_REF}) + ADD_H5_TEST (HDFFV-2605 "SKIP" ${FILE_ATTR_REF}) + +# Add test for memory leak in attirbute. This test is verified by CTEST. +# 1. leak from vlen string +# 2. leak from compound type without reference member +# (HDFFV-7840, ) +# Note: this test is experimental for sharing test file among tools + ADD_H5_TEST (HDFFV-7840 "TEST" h5diff_attr1.h5) + + IF (HDF5_TEST_VFD) + # Run test with different Virtual File Driver + FOREACH (vfd ${VFD_LIST}) + ADD_VFD_TEST (${vfd} 0) + ENDFOREACH (vfd ${VFD_LIST}) + ENDIF (HDF5_TEST_VFD) ENDIF (BUILD_TESTING) diff --git a/tools/h5repack/Makefile.in b/tools/h5repack/Makefile.in index 0d067a1..0d749d3 100644 --- a/tools/h5repack/Makefile.in +++ b/tools/h5repack/Makefile.in @@ -195,6 +195,7 @@ F9XMODEXT = @F9XMODEXT@ F9XMODFLAG = @F9XMODFLAG@ F9XSUFFIXFLAG = @F9XSUFFIXFLAG@ FC = @FC@ +FC2003 = @FC2003@ FCFLAGS = @FCFLAGS@ FCFLAGS_f90 = @FCFLAGS_f90@ FCLIBS = @FCLIBS@ @@ -215,10 +216,12 @@ H5_LONE_COLON = @H5_LONE_COLON@ H5_VERSION = @H5_VERSION@ HADDR_T = @HADDR_T@ HAVE_DMALLOC = @HAVE_DMALLOC@ +HAVE_FORTRAN_2003 = @HAVE_FORTRAN_2003@ HDF5_HL = @HDF5_HL@ HDF5_INTERFACES = @HDF5_INTERFACES@ HDF_CXX = @HDF_CXX@ HDF_FORTRAN = @HDF_FORTRAN@ +HDF_FORTRAN2003 = @HDF_FORTRAN2003@ HID_T = @HID_T@ HL = @HL@ HL_FOR = @HL_FOR@ diff --git a/tools/h5repack/h5repack.c b/tools/h5repack/h5repack.c index 1e7e0ed..05160d8 100644 --- a/tools/h5repack/h5repack.c +++ b/tools/h5repack/h5repack.c @@ -91,7 +91,7 @@ h5repack_init(pack_opt_t *options, int verbose, int latest, H5F_file_space_type_t strategy, hsize_t threshold) { int k, n; - memset(options,0,sizeof(pack_opt_t)); + HDmemset(options,0,sizeof(pack_opt_t)); options->min_comp = 1024; options->verbose = verbose; options->latest = latest; @@ -110,7 +110,7 @@ h5repack_init(pack_opt_t *options, int verbose, int latest, return (options_table_init(&(options->op_tbl))); } - + /*------------------------------------------------------------------------- * Function: h5repack_end * @@ -124,7 +124,7 @@ int h5repack_end (pack_opt_t *options) return options_table_free(options->op_tbl); } - + /*------------------------------------------------------------------------- * Function: h5repack_addfilter * @@ -159,7 +159,7 @@ int h5repack_addfilter(const char* str, if(options->n_filter_g > H5_REPACK_MAX_NFILTERS) { error_msg("maximum number of filters exceeded for <%s>\n", str); - free(obj_list); + HDfree(obj_list); return -1; } @@ -168,11 +168,11 @@ int h5repack_addfilter(const char* str, else options_add_filter(obj_list, n_objs, filter, options->op_tbl); - free(obj_list); + HDfree(obj_list); return 0; } - + /*------------------------------------------------------------------------- * Function: h5repack_addlayout * @@ -234,13 +234,13 @@ int h5repack_addlayout(const char* str, &pack, options->op_tbl); - free(obj_list); + HDfree(obj_list); return 0; } -/* Note: The below copy_named_datatype(), named_datatype_free(), copy_attr() - * were located in h5repack_copy.c as static prior to bugfix1726. - * Made shared functions as copy_attr() was needed in h5repack_refs.c. +/* Note: The below copy_named_datatype(), named_datatype_free(), copy_attr() + * were located in h5repack_copy.c as static prior to bugfix1726. + * Made shared functions as copy_attr() was needed in h5repack_refs.c. * However copy_attr() may be obsoleted when H5Acopy is available and put back * others to static in h5repack_copy.c. */ @@ -271,21 +271,21 @@ hid_t copy_named_datatype(hid_t type_in, hid_t fidout, named_dt_t **named_dt_hea if(H5Oget_info(type_in, &oinfo) < 0) goto error; - if(*named_dt_head_p) + if(*named_dt_head_p) { /* Stack already exists, search for the datatype */ while(dt && dt->addr_in != oinfo.addr) dt = dt->next; dt_ret = dt; - } - else + } + else { /* Create the stack */ size_t i; for(i=0; i<travt->nobjs; i++) - if(travt->objs[i].type == H5TRAV_TYPE_NAMED_DATATYPE) + if(travt->objs[i].type == H5TRAV_TYPE_NAMED_DATATYPE) { /* Push onto the stack */ if(NULL == (dt = (named_dt_t *) HDmalloc(sizeof(named_dt_t)))) @@ -298,7 +298,7 @@ hid_t copy_named_datatype(hid_t type_in, hid_t fidout, named_dt_t **named_dt_hea dt->id_out = -1; /* Check if this type is the one requested */ - if(oinfo.addr == dt->addr_in) + if(oinfo.addr == dt->addr_in) { HDassert(!dt_ret); dt_ret = dt; @@ -308,7 +308,7 @@ hid_t copy_named_datatype(hid_t type_in, hid_t fidout, named_dt_t **named_dt_hea /* Handle the case that the requested datatype was not found. This is * possible if the datatype was committed anonymously in the input file. */ - if(!dt_ret) + if(!dt_ret) { /* Push the new datatype onto the stack */ if(NULL == (dt_ret = (named_dt_t *) HDmalloc(sizeof(named_dt_t)))) @@ -323,7 +323,7 @@ hid_t copy_named_datatype(hid_t type_in, hid_t fidout, named_dt_t **named_dt_hea /* If the requested datatype does not yet exist in the output file, copy it * anonymously */ - if(dt_ret->id_out < 0) + if(dt_ret->id_out < 0) { if (options->use_native==1) dt_ret->id_out = h5tools_get_native_type(type_in); @@ -365,7 +365,7 @@ int named_datatype_free(named_dt_t **named_dt_head_p, int ignore_err) { named_dt_t *dt = *named_dt_head_p; - while(dt) + while(dt) { /* Pop the datatype off the stack and free it */ if(H5Tclose(dt->id_out) < 0 && !ignore_err) @@ -418,6 +418,8 @@ int copy_attr(hid_t loc_in, H5O_info_t oinfo; /* object info */ int j; unsigned u; + hbool_t is_ref=0; + H5T_class_t type_class = -1; if(H5Oget_info(loc_in, &oinfo) < 0) goto error; @@ -451,7 +453,7 @@ int copy_attr(hid_t loc_in, /* Copy named dt */ if((wtype_id = copy_named_datatype(ftype_id, fidout, named_dt_head_p, - travt, options)) < 0) + travt, options)) < 0) { H5Fclose(fidout); goto error; @@ -484,15 +486,37 @@ int copy_attr(hid_t loc_in, goto error; /*------------------------------------------------------------------------- - * object references are a special case - * we cannot just copy the buffers, but instead we recreate the reference - * this is done on a second sweep of the file that just copies - * the referenced objects + * object references are a special case. We cannot just copy the buffers, + * but instead we recreate the reference. + * This is done on a second sweep of the file that just copies the referenced + * objects at copy_refs_attr() *------------------------------------------------------------------------- */ + type_class = H5Tget_class(wtype_id); + is_ref = (type_class == H5T_REFERENCE); + if (type_class == H5T_VLEN ||type_class == H5T_ARRAY ) { + hid_t base_type = -1; + base_type = H5Tget_super(ftype_id); + is_ref = (is_ref || (H5Tget_class(base_type)==H5T_REFERENCE)); + H5Tclose(base_type); + } - if(H5T_REFERENCE == H5Tget_class(wtype_id)) { - ; + if (type_class == H5T_COMPOUND) { + int nmembers = H5Tget_nmembers(wtype_id) ; + for (j=0; j<nmembers; j++) { + hid_t mtid = H5Tget_member_type( wtype_id, j ); + H5T_class_t mtclass = H5Tget_class(mtid); + H5Tclose(mtid); + + if (mtclass==H5T_REFERENCE) { + is_ref = 1; + break; + } + } /* for (j=0; i<nmembers; j++) */ + } /* if (type_class == H5T_COMPOUND) */ + + if(is_ref) { + ; /* handled by copy_refs_attr() */ } else { /*------------------------------------------------------------------------- @@ -522,9 +546,9 @@ int copy_attr(hid_t loc_in, if(H5Aclose(attr_out) < 0) goto error; - /* Check if we have VL data in the attribute's datatype that must + /* Check if we have VL data and string in the attribute's datatype that must * be reclaimed */ - if(TRUE == H5Tdetect_class(wtype_id, H5T_VLEN)) + if (TRUE == h5tools_detect_vlen(wtype_id)) H5Dvlen_reclaim(wtype_id, space_id, H5P_DEFAULT, buf); HDfree(buf); buf = NULL; @@ -554,13 +578,13 @@ int copy_attr(hid_t loc_in, error: H5E_BEGIN_TRY { if(buf) { - /* Check if we have VL data in the attribute's datatype that must + /* Check if we have VL data and string in the attribute's datatype that must * be reclaimed */ - if(TRUE == H5Tdetect_class(wtype_id, H5T_VLEN)) + if (TRUE == h5tools_detect_vlen(wtype_id)) H5Dvlen_reclaim(wtype_id, space_id, H5P_DEFAULT, buf); /* Free buf */ - free(buf); + HDfree(buf); } /* end if */ H5Tclose(ftype_id); @@ -980,7 +1004,7 @@ static const char* get_sfilter(H5Z_filter_t filtn) return "SOFF"; else { error_msg("input error in filter type\n"); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } } diff --git a/tools/h5repack/h5repack.sh.in b/tools/h5repack/h5repack.sh.in index 5a60227..c1cbba1 100755 --- a/tools/h5repack/h5repack.sh.in +++ b/tools/h5repack/h5repack.sh.in @@ -41,33 +41,11 @@ H5DUMP=../h5dump/h5dump # The h5dump tool name H5DUMP_BIN=`pwd`/$H5DUMP # The path of the h5dump tool binary GREP='grep' +CP='cp' H5DETECTSZIP=testh5repack_detect_szip H5DETECTSZIP_BIN=`pwd`/$H5DETECTSZIP -INFO_FILE=testfiles/h5repack.info - -FILE0=h5repack_fill.h5 -FILE1=h5repack_objs.h5 -FILE2=h5repack_attr.h5 -FILE3=h5repack_hlink.h5 -FILE4=h5repack_layout.h5 -FILE5=h5repack_early.h5 -FILE7=h5repack_szip.h5 -FILE8=h5repack_deflate.h5 -FILE9=h5repack_shuffle.h5 -FILE10=h5repack_fletcher.h5 -FILE11=h5repack_filters.h5 -FILE12=h5repack_nbit.h5 -FILE13=h5repack_soffset.h5 -FILE14=h5repack_layouto.h5 # A file with an older version of the layout message - # (copy of test/tlayouto.h5) -FILE15=h5repack_named_dtypes.h5 -FILE16=tfamily%05d.h5 # located in common testfiles folder -FILE18=h5repack_layout2.h5 -FILE_REF=h5repack_refs.h5 -FILE_ATTR_REF=h5repack_attr_refs.h5 - nerrors=0 verbose=yes @@ -78,7 +56,96 @@ if test -z "$srcdir"; then srcdir=. fi +# source dirs +SRC_TOOLS="$srcdir/.." +SRC_TOOLS_TESTFILES="$SRC_TOOLS/testfiles" +# testfiles source dirs for tools +SRC_H5LS_TESTFILES="$SRC_TOOLS_TESTFILES" +SRC_H5DUMP_TESTFILES="$SRC_TOOLS_TESTFILES" +SRC_H5DIFF_TESTFILES="$SRC_TOOLS/h5diff/testfiles" +SRC_H5COPY_TESTFILES="$SRC_TOOLS/h5copy/testfiles" +SRC_H5REPACK_TESTFILES="$SRC_TOOLS/h5repack/testfiles" +SRC_H5JAM_TESTFILES="$SRC_TOOLS/h5jam/testfiles" +SRC_H5STAT_TESTFILES="$SRC_TOOLS/h5stat/testfiles" +SRC_H5IMPORT_TESTFILES="$SRC_TOOLS/h5import/testfiles" + +TESTDIR=./testfiles +test -d $TESTDIR || mkdir $TESTDIR + +###################################################################### +# test files +# -------------------------------------------------------------------- +# All the test files copy from source directory to test directory +# NOTE: Keep this framework to add/remove test files. +# Any test files from other tools can be used in this framework. +# This list are also used for checking exist. +# Comment '#' without space can be used. +# -------------------------------------------------------------------- +LIST_HDF5_TEST_FILES=" +$SRC_H5REPACK_TESTFILES/h5repack_attr.h5 +$SRC_H5REPACK_TESTFILES/h5repack_attr_refs.h5 +$SRC_H5REPACK_TESTFILES/h5repack_deflate.h5 +$SRC_H5REPACK_TESTFILES/h5repack_early.h5 +$SRC_H5REPACK_TESTFILES/h5repack_ext.h5 +$SRC_H5REPACK_TESTFILES/h5repack_fill.h5 +$SRC_H5REPACK_TESTFILES/h5repack_filters.h5 +$SRC_H5REPACK_TESTFILES/h5repack_fletcher.h5 +$SRC_H5REPACK_TESTFILES/h5repack_hlink.h5 +$SRC_H5REPACK_TESTFILES/h5repack_layout.h5 +$SRC_H5REPACK_TESTFILES/h5repack_layouto.h5 +$SRC_H5REPACK_TESTFILES/h5repack_layout2.h5 +$SRC_H5REPACK_TESTFILES/h5repack_named_dtypes.h5 +$SRC_H5REPACK_TESTFILES/h5repack_nbit.h5 +$SRC_H5REPACK_TESTFILES/h5repack_objs.h5 +$SRC_H5REPACK_TESTFILES/h5repack_refs.h5 +$SRC_H5REPACK_TESTFILES/h5repack_shuffle.h5 +$SRC_H5REPACK_TESTFILES/h5repack_soffset.h5 +$SRC_H5REPACK_TESTFILES/h5repack_szip.h5 +$SRC_H5DIFF_TESTFILES/h5diff_attr1.h5 +$SRC_TOOLS_TESTFILES/tfamily00000.h5 +$SRC_TOOLS_TESTFILES/tfamily00001.h5 +$SRC_TOOLS_TESTFILES/tfamily00002.h5 +$SRC_TOOLS_TESTFILES/tfamily00003.h5 +$SRC_TOOLS_TESTFILES/tfamily00004.h5 +$SRC_TOOLS_TESTFILES/tfamily00005.h5 +$SRC_TOOLS_TESTFILES/tfamily00006.h5 +$SRC_TOOLS_TESTFILES/tfamily00007.h5 +$SRC_TOOLS_TESTFILES/tfamily00008.h5 +$SRC_TOOLS_TESTFILES/tfamily00009.h5 +$SRC_TOOLS_TESTFILES/tfamily00010.h5 +" + +LIST_OTHER_TEST_FILES=" +$SRC_H5REPACK_TESTFILES/h5repack_ext.bin +$SRC_H5REPACK_TESTFILES/ublock.bin +$SRC_H5REPACK_TESTFILES/h5repack.info +$SRC_TOOLS_TESTFILES/h5repack_filters.h5.ddl +" + +# +# copy test files and expected output files from source dirs to test dir +# +COPY_TESTFILES="$LIST_HDF5_TEST_FILES $LIST_OTHER_TEST_FILES" +COPY_TESTFILES_TO_TESTDIR() +{ + # copy test files. Used -f to make sure get a new copy + for tstfile in $COPY_TESTFILES + do + # ignore '#' comment + echo $tstfile | tr -d ' ' | grep '^#' > /dev/null + RET=$? + if [ $RET -eq 1 ]; then + if [ -a $tstfile ]; then + $CP -f $tstfile $TESTDIR + else + echo "Error: FAILED to copy $tstfile" + echo " $tstfile doesn't exist!" + exit $EXIT_FAILURE + fi + fi + done +} # Print a line-line message left justified in a field of 70 characters # beginning with the word "Testing". @@ -128,8 +195,7 @@ TOOLTEST() { echo $@ TOOLTEST_MAIN $@ - path=`pwd` - outfile=$path/out.$1 + outfile=$TESTDIR/out.$1 rm -f $outfile } @@ -139,9 +205,8 @@ TOOLTEST_MAIN() # Run test. TESTING $H5REPACK $@ - infile=$srcdir/testfiles/$1 - path=`pwd` - outfile=$path/out.$1 + infile=$TESTDIR/$1 + outfile=$TESTDIR/out.$1 shift $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile RET=$? @@ -158,16 +223,15 @@ TOOLTEST_MAIN() # Verifying layouts of a dataset VERIFY_LAYOUT_DSET() { - path=`pwd` - outfile=$path/out.$1 - layoutfile=$path/layout.$1 + outfile=$TESTDIR/out.$1 + layoutfile=$TESTDIR/layout.$1 dset=$2 expectlayout=$3 #--------------------------------- # check the layout from a dataset VERIFY "Layout" - $H5DUMP_BIN -d $dset -pH $outfile > $layoutfile + $RUNSERIAL $H5DUMP_BIN -d $dset -pH $outfile > $layoutfile $GREP $expectlayout $layoutfile > /dev/null if [ $? -eq 0 ]; then echo " PASSED" @@ -184,9 +248,8 @@ VERIFY_LAYOUT_DSET() # Verifying layouts from entire file VERIFY_LAYOUT_ALL() { - path=`pwd` - outfile=$path/out.$1 - layoutfile=$path/layout.$1 + outfile=$TESTDIR/out.$1 + layoutfile=$TESTDIR/layout.$1 expectlayout=$2 #--------------------------------- @@ -195,7 +258,7 @@ VERIFY_LAYOUT_ALL() VERIFY "Layout " # if CONTIGUOUS if [ $expectlayout = "CONTIGUOUS" ]; then - $H5DUMP_BIN -pH $outfile > $layoutfile + $RUNSERIAL $H5DUMP_BIN -pH $outfile > $layoutfile $GREP "COMPACT" $layoutfile > /dev/null if [ $? -eq 0 ]; then echo " FAILED" @@ -210,7 +273,7 @@ VERIFY_LAYOUT_ALL() else # if COMPACT if [ $expectlayout = "COMPACT" ]; then - $H5DUMP_BIN -pH $outfile > $layoutfile + $RUNSERIAL $H5DUMP_BIN -pH $outfile > $layoutfile $GREP "CHUNKED" $layoutfile > /dev/null if [ $? -eq 0 ]; then echo " FAILED" @@ -225,7 +288,7 @@ VERIFY_LAYOUT_ALL() else # if CHUNKED if [ $expectlayout = "CHUNKED" ]; then - $H5DUMP_BIN -pH $outfile > $layoutfile + $RUNSERIAL $H5DUMP_BIN -pH $outfile > $layoutfile $GREP "CONTIGUOUS" $layoutfile > /dev/null if [ $? -eq 0 ]; then echo " FAILED" @@ -253,9 +316,8 @@ TOOLTEST0() # Run test. TESTING $H5REPACK $@ - infile=$srcdir/testfiles/$1 - path=`pwd` - outfile=$path/out.$1 + infile=$TESTDIR/$1 + outfile=$TESTDIR/out.$1 shift $RUNSERIAL $H5REPACK_BIN -i $infile -o $outfile "$@" RET=$? @@ -270,7 +332,7 @@ TOOLTEST0() } -# same as TOOLTEST, but it uses the common testfiles at $srcdir/../testfiles/ +# same as TOOLTEST, but it uses without -i -o options # used to test the family driver, where these files reside # TOOLTEST1() @@ -278,9 +340,8 @@ TOOLTEST1() # Run test. TESTING $H5REPACK $@ - infile=$srcdir/../testfiles/$1 - path=`pwd` - outfile=$path/out.$1 + infile=$TESTDIR/$1 + outfile=$TESTDIR/out.$1 shift $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile RET=$? @@ -300,13 +361,12 @@ TOOLTESTV() { # Run test. TESTING $H5REPACK $@ - expect="$srcdir/../testfiles/$1.ddl" - actual="../testfiles/`basename $1 .ddl`.out" - actual_err="../testfiles/`basename $1 .ddl`.err" + expect="$TESTDIR/$1.ddl" + actual="$TESTDIR/`basename $1 .ddl`.out" + actual_err="$TESTDIR/`basename $1 .ddl`.err" - infile=$srcdir/testfiles/$1 - path=`pwd` - outfile=$path/out.$1 + infile=$TESTDIR/$1 + outfile=$TESTDIR/out.$1 shift $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile >$actual 2>$actual_err cp $actual $actual_sav @@ -325,6 +385,7 @@ TOOLTESTV() rm -f $actual $actual_err $actual_sav } +# This is different from $srcdir/../../bin/output_filter.sh STDOUT_FILTER() { result_file=$1 tmp_file=/tmp/h5test_tmp_$$ @@ -355,19 +416,25 @@ if test $USE_FILTER_SZIP = "yes"; then USE_FILTER_SZIP_ENCODER=`$RUNSERIAL $H5DETECTSZIP_BIN` fi +############################################################################## +### T H E T E S T S +############################################################################## +# prepare for test +COPY_TESTFILES_TO_TESTDIR + # copy files (these files have no filters) -TOOLTEST $FILE0 -TOOLTEST $FILE1 -TOOLTEST $FILE2 -TOOLTEST $FILE3 -TOOLTEST $FILE4 -TOOLTEST $FILE5 +TOOLTEST h5repack_fill.h5 +TOOLTEST h5repack_objs.h5 +TOOLTEST h5repack_attr.h5 +TOOLTEST h5repack_hlink.h5 +TOOLTEST h5repack_layout.h5 +TOOLTEST h5repack_early.h5 -# use $FILE4 to write some filters (this file has no filters) +# use h5repack_layout.h5 to write some filters (this file has no filters) # gzip with individual object -arg="$FILE4 -f dset1:GZIP=1 -l dset1:CHUNK=20x10" +arg="h5repack_layout.h5 -f dset1:GZIP=1 -l dset1:CHUNK=20x10" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else @@ -375,7 +442,7 @@ else fi # gzip for all -arg="$FILE4 -f GZIP=1" +arg="h5repack_layout.h5 -f GZIP=1" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else @@ -383,7 +450,7 @@ else fi # szip with individual object -arg="$FILE4 -f dset2:SZIP=8,EC -l dset2:CHUNK=20x10" +arg="h5repack_layout.h5 -f dset2:SZIP=8,EC -l dset2:CHUNK=20x10" if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then SKIP $arg else @@ -391,7 +458,7 @@ else fi # szip for all -arg="$FILE4 -f SZIP=8,NN" +arg="h5repack_layout.h5 -f SZIP=8,NN" if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then SKIP $arg else @@ -399,7 +466,7 @@ else fi # shuffle with individual object -arg="$FILE4 -f dset2:SHUF -l dset2:CHUNK=20x10" +arg="h5repack_layout.h5 -f dset2:SHUF -l dset2:CHUNK=20x10" if test $USE_FILTER_SHUFFLE != "yes" ; then SKIP $arg else @@ -408,7 +475,7 @@ fi # shuffle for all -arg="$FILE4 -f SHUF" +arg="h5repack_layout.h5 -f SHUF" if test $USE_FILTER_SHUFFLE != "yes" ; then SKIP $arg else @@ -416,7 +483,7 @@ else fi # fletcher32 with individual object -arg="$FILE4 -f dset2:FLET -l dset2:CHUNK=20x10" +arg="h5repack_layout.h5 -f dset2:FLET -l dset2:CHUNK=20x10" if test $USE_FILTER_FLETCHER32 != "yes" ; then SKIP $arg else @@ -424,7 +491,7 @@ else fi # fletcher32 for all -arg="$FILE4 -f FLET" +arg="h5repack_layout.h5 -f FLET" if test $USE_FILTER_FLETCHER32 != "yes" ; then SKIP $arg else @@ -432,7 +499,7 @@ else fi # all filters -arg="$FILE4 -f dset2:SHUF -f dset2:FLET -f dset2:SZIP=8,NN -f dset2:GZIP=1 -l dset2:CHUNK=20x10" +arg="h5repack_layout.h5 -f dset2:SHUF -f dset2:FLET -f dset2:SZIP=8,NN -f dset2:GZIP=1 -l dset2:CHUNK=20x10" if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SHUFFLE != "yes" -o $USE_FILTER_FLETCHER32 != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else @@ -440,7 +507,7 @@ else fi # verbose gzip with individual object -arg="$FILE11 -v -f /dset_deflate:GZIP=9" +arg="h5repack_filters.h5 -v -f /dset_deflate:GZIP=9" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else @@ -452,7 +519,7 @@ fi ########################################################### # szip copy -arg="$FILE7" +arg="h5repack_szip.h5" if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then SKIP $arg else @@ -460,7 +527,7 @@ else fi # szip remove -arg="$FILE7 --filter=dset_szip:NONE" +arg="h5repack_szip.h5 --filter=dset_szip:NONE" if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then SKIP $arg else @@ -468,7 +535,7 @@ else fi # deflate copy -arg="$FILE8" +arg="h5repack_deflate.h5" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else @@ -476,7 +543,7 @@ else fi # deflate remove -arg="$FILE8 -f dset_deflate:NONE" +arg="h5repack_deflate.h5 -f dset_deflate:NONE" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else @@ -484,7 +551,7 @@ else fi # shuffle copy -arg="$FILE9" +arg="h5repack_shuffle.h5" if test $USE_FILTER_SHUFFLE != "yes" ; then SKIP $arg else @@ -492,7 +559,7 @@ else fi # shuffle remove -arg="$FILE9 -f dset_shuffle:NONE" +arg="h5repack_shuffle.h5 -f dset_shuffle:NONE" if test $USE_FILTER_SHUFFLE != "yes" ; then SKIP $arg else @@ -500,7 +567,7 @@ else fi # fletcher32 copy -arg="$FILE10" +arg="h5repack_fletcher.h5" if test $USE_FILTER_FLETCHER32 != "yes" ; then SKIP $arg else @@ -508,7 +575,7 @@ else fi # fletcher32 remove -arg="$FILE10 -f dset_fletcher32:NONE" +arg="h5repack_fletcher.h5 -f dset_fletcher32:NONE" if test $USE_FILTER_FLETCHER32 != "yes" ; then SKIP $arg else @@ -516,7 +583,7 @@ else fi # nbit copy -arg="$FILE12" +arg="h5repack_nbit.h5" if test $USE_FILTER_NBIT != "yes" ; then SKIP $arg else @@ -524,7 +591,7 @@ else fi # nbit remove -arg="$FILE12 -f dset_nbit:NONE" +arg="h5repack_nbit.h5 -f dset_nbit:NONE" if test $USE_FILTER_NBIT != "yes" ; then SKIP $arg else @@ -532,7 +599,7 @@ else fi # nbit add -arg="$FILE12 -f dset_int31:NBIT" +arg="h5repack_nbit.h5 -f dset_int31:NBIT" if test $USE_FILTER_NBIT != "yes" ; then SKIP $arg else @@ -540,7 +607,7 @@ else fi # scaleoffset copy -arg="$FILE13" +arg="h5repack_soffset.h5" if test $USE_FILTER_SCALEOFFSET != "yes" ; then SKIP $arg else @@ -548,7 +615,7 @@ else fi # scaleoffset add -arg="$FILE13 -f dset_none:SOFF=31,IN" +arg="h5repack_soffset.h5 -f dset_none:SOFF=31,IN" if test $USE_FILTER_SCALEOFFSET != "yes" ; then SKIP $arg else @@ -556,7 +623,7 @@ else fi # scaleoffset remove -arg="$FILE13 -f dset_scaleoffset:NONE" +arg="h5repack_soffset.h5 -f dset_scaleoffset:NONE" if test $USE_FILTER_SCALEOFFSET != "yes" ; then SKIP $arg else @@ -564,7 +631,7 @@ else fi # remove all filters -arg="$FILE11 -f NONE" +arg="h5repack_filters.h5 -f NONE" if test $USE_FILTER_FLETCHER32 != "yes" -o $USE_FILTER_DEFLATE != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SHUFFLE != "yes" -o $USE_FILTER_NBIT != "yes" -o $USE_FILTER_SCALEOFFSET != "yes" ; then SKIP $arg else @@ -573,14 +640,14 @@ fi #filter conversions -arg="$FILE8 -f dset_deflate:SZIP=8,NN" +arg="h5repack_deflate.h5 -f dset_deflate:SZIP=8,NN" if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else TOOLTEST $arg fi -arg="$FILE7 -f dset_szip:GZIP=1" +arg="h5repack_szip.h5 -f dset_szip:GZIP=1" if test $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else @@ -589,7 +656,7 @@ fi #limit -arg="$FILE4 -f GZIP=1 -m 1024" +arg="h5repack_layout.h5 -f GZIP=1 -m 1024" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else @@ -597,7 +664,7 @@ else fi #file -arg="$FILE4 -e $srcdir/$INFO_FILE" +arg="h5repack_layout.h5 -e $TESTDIR/h5repack.info" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else @@ -607,90 +674,90 @@ fi ######################################################### # layout options (these files have no filters) ######################################################### -TOOLTEST_MAIN $FILE4 --layout dset2:CHUNK=20x10 -VERIFY_LAYOUT_DSET $FILE4 dset2 CHUNKED +TOOLTEST_MAIN h5repack_layout.h5 --layout dset2:CHUNK=20x10 +VERIFY_LAYOUT_DSET h5repack_layout.h5 dset2 CHUNKED -TOOLTEST_MAIN $FILE4 -l CHUNK=20x10 -VERIFY_LAYOUT_ALL $FILE4 CHUNKED +TOOLTEST_MAIN h5repack_layout.h5 -l CHUNK=20x10 +VERIFY_LAYOUT_ALL h5repack_layout.h5 CHUNKED -TOOLTEST_MAIN $FILE4 -l dset2:CONTI -VERIFY_LAYOUT_DSET $FILE4 dset2 CONTIGUOUS +TOOLTEST_MAIN h5repack_layout.h5 -l dset2:CONTI +VERIFY_LAYOUT_DSET h5repack_layout.h5 dset2 CONTIGUOUS -TOOLTEST_MAIN $FILE4 -l CONTI -VERIFY_LAYOUT_ALL $FILE4 CONTIGUOUS +TOOLTEST_MAIN h5repack_layout.h5 -l CONTI +VERIFY_LAYOUT_ALL h5repack_layout.h5 CONTIGUOUS -TOOLTEST_MAIN $FILE4 -l dset2:COMPA -VERIFY_LAYOUT_DSET $FILE4 dset2 COMPACT +TOOLTEST_MAIN h5repack_layout.h5 -l dset2:COMPA +VERIFY_LAYOUT_DSET h5repack_layout.h5 dset2 COMPACT -TOOLTEST_MAIN $FILE4 -l COMPA -VERIFY_LAYOUT_ALL $FILE4 COMPACT +TOOLTEST_MAIN h5repack_layout.h5 -l COMPA +VERIFY_LAYOUT_ALL h5repack_layout.h5 COMPACT ################################################################ # layout conversions (file has no filters) ############################################################### -TOOLTEST_MAIN $FILE4 -l dset_compact:CONTI -VERIFY_LAYOUT_DSET $FILE4 dset_compact CONTIGUOUS +TOOLTEST_MAIN h5repack_layout.h5 -l dset_compact:CONTI +VERIFY_LAYOUT_DSET h5repack_layout.h5 dset_compact CONTIGUOUS -TOOLTEST_MAIN $FILE4 -l dset_compact:CHUNK=2x5 -VERIFY_LAYOUT_DSET $FILE4 dset_compact CHUNKED +TOOLTEST_MAIN h5repack_layout.h5 -l dset_compact:CHUNK=2x5 +VERIFY_LAYOUT_DSET h5repack_layout.h5 dset_compact CHUNKED -TOOLTEST_MAIN $FILE4 -l dset_compact:COMPA -VERIFY_LAYOUT_DSET $FILE4 dset_compact COMPACT +TOOLTEST_MAIN h5repack_layout.h5 -l dset_compact:COMPA +VERIFY_LAYOUT_DSET h5repack_layout.h5 dset_compact COMPACT -TOOLTEST_MAIN $FILE4 -l dset_contiguous:COMPA -VERIFY_LAYOUT_DSET $FILE4 dset_contiguous COMPACT +TOOLTEST_MAIN h5repack_layout.h5 -l dset_contiguous:COMPA +VERIFY_LAYOUT_DSET h5repack_layout.h5 dset_contiguous COMPACT -TOOLTEST_MAIN $FILE4 -l dset_contiguous:CHUNK=3x6 -VERIFY_LAYOUT_DSET $FILE4 dset_contiguous CHUNKED +TOOLTEST_MAIN h5repack_layout.h5 -l dset_contiguous:CHUNK=3x6 +VERIFY_LAYOUT_DSET h5repack_layout.h5 dset_contiguous CHUNKED -TOOLTEST_MAIN $FILE4 -l dset_contiguous:CONTI -VERIFY_LAYOUT_DSET $FILE4 dset_contiguous CONTIGUOUS +TOOLTEST_MAIN h5repack_layout.h5 -l dset_contiguous:CONTI +VERIFY_LAYOUT_DSET h5repack_layout.h5 dset_contiguous CONTIGUOUS -TOOLTEST_MAIN $FILE4 -l dset_chunk:COMPA -VERIFY_LAYOUT_DSET $FILE4 dset_chunk COMPACT +TOOLTEST_MAIN h5repack_layout.h5 -l dset_chunk:COMPA +VERIFY_LAYOUT_DSET h5repack_layout.h5 dset_chunk COMPACT -TOOLTEST_MAIN $FILE4 -l dset_chunk:CONTI -VERIFY_LAYOUT_DSET $FILE4 dset_chunk CONTIGUOUS +TOOLTEST_MAIN h5repack_layout.h5 -l dset_chunk:CONTI +VERIFY_LAYOUT_DSET h5repack_layout.h5 dset_chunk CONTIGUOUS -TOOLTEST_MAIN $FILE4 -l dset_chunk:CHUNK=18x13 -VERIFY_LAYOUT_DSET $FILE4 dset_chunk CHUNKED +TOOLTEST_MAIN h5repack_layout.h5 -l dset_chunk:CHUNK=18x13 +VERIFY_LAYOUT_DSET h5repack_layout.h5 dset_chunk CHUNKED # test convert small size dataset ( < 1k) to compact layout without -m -TOOLTEST_MAIN $FILE18 -l contig_small:COMPA -VERIFY_LAYOUT_DSET $FILE18 contig_small COMPACT +TOOLTEST_MAIN h5repack_layout2.h5 -l contig_small:COMPA +VERIFY_LAYOUT_DSET h5repack_layout2.h5 contig_small COMPACT -TOOLTEST_MAIN $FILE18 -l chunked_small_fixed:COMPA -VERIFY_LAYOUT_DSET $FILE18 chunked_small_fixed COMPACT +TOOLTEST_MAIN h5repack_layout2.h5 -l chunked_small_fixed:COMPA +VERIFY_LAYOUT_DSET h5repack_layout2.h5 chunked_small_fixed COMPACT # Native option # Do not use FILE1, as the named dtype will be converted to native, and h5diff will # report a difference. -TOOLTEST $FILE0 -n -TOOLTEST $FILE2 -n +TOOLTEST h5repack_fill.h5 -n +TOOLTEST h5repack_attr.h5 -n # latest file format with long switches. use FILE4=h5repack_layout.h5 (no filters) -arg="$FILE4 --layout CHUNK=20x10 --filter GZIP=1 --minimum=10 --native --latest --compact=8 --indexed=6 --ssize=8[:dtype]" +arg="h5repack_layout.h5 --layout CHUNK=20x10 --filter GZIP=1 --minimum=10 --native --latest --compact=8 --indexed=6 --ssize=8[:dtype]" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else TOOLTEST_MAIN $arg - VERIFY_LAYOUT_ALL $FILE4 CHUNKED + VERIFY_LAYOUT_ALL h5repack_layout.h5 CHUNKED fi # latest file format with short switches. use FILE4=h5repack_layout.h5 (no filters) -arg="$FILE4 -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype]" +arg="h5repack_layout.h5 -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype]" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else TOOLTEST_MAIN $arg - VERIFY_LAYOUT_ALL $FILE4 CHUNKED + VERIFY_LAYOUT_ALL h5repack_layout.h5 CHUNKED fi # several global filters -arg="$FILE4 --filter GZIP=1 --filter SHUF" +arg="h5repack_layout.h5 --filter GZIP=1 --filter SHUF" if test $USE_FILTER_DEFLATE != "yes" -o $USE_FILTER_SHUFFLE != "yes" ; then SKIP $arg else @@ -699,7 +766,7 @@ fi # syntax of -i infile -o outfile # latest file format with short switches. use FILE4=h5repack_layout.h5 (no filters) -arg="$FILE4 -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype]" +arg="h5repack_layout.h5 -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype]" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else @@ -707,34 +774,40 @@ else fi # add a userblock to file -arg="$FILE1 -u ublock.bin -b 2048" +arg="h5repack_objs.h5 -u ublock.bin -b 2048" TOOLTEST $arg # add alignment -arg="$FILE1 -t 1 -a 1 " +arg="h5repack_objs.h5 -t 1 -a 1 " TOOLTEST $arg # Check repacking file with old version of layout message (should get upgraded # to new version and be readable, etc.) -TOOLTEST $FILE14 +TOOLTEST h5repack_layouto.h5 # test for datum size > H5TOOLS_MALLOCSIZE -TOOLTEST $FILE1 -f GZIP=1 +TOOLTEST h5repack_objs.h5 -f GZIP=1 # Check repacking file with committed datatypes in odd configurations -TOOLTEST $FILE15 +TOOLTEST h5repack_named_dtypes.h5 # tests family driver (file is located in common testfiles folder, uses TOOLTEST1 -TOOLTEST1 $FILE16 +TOOLTEST1 tfamily%05d.h5 # test various references (bug 1814 and 1726) -TOOLTEST $FILE_REF +TOOLTEST h5repack_refs.h5 -# test attribute with various references (bug 1797) +# test attribute with various references (bug1797 / HDFFV-5932) # the references in attribute of compund or vlen datatype -# TODO: include this test when code portion is completed. -SKIP $FILE_ATTR_REF -#TOOLTEST $FILE_ATTR_REF +TOOLTEST h5repack_attr_refs.h5 + + +# Add test for memory leak in attirbute. This test is verified by CTEST. +# 1. leak from vlen string +# 2. leak from compound type without reference member +# (HDFFV-7840, ) +# Note: this test is experimental for sharing test file among tools +TOOLTEST h5diff_attr1.h5 if test $nerrors -eq 0 ; then echo "All $TESTNAME tests passed." diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c index 5153cc4..55cf932 100644 --- a/tools/h5repack/h5repack_copy.c +++ b/tools/h5repack/h5repack_copy.c @@ -31,18 +31,45 @@ * macros *------------------------------------------------------------------------- */ -#define USERBLOCK_XFER_SIZE 512 /* size of buffer/# of bytes to xfer at a time when copying userblock */ +/* size of buffer/# of bytes to xfer at a time when copying userblock */ +#define USERBLOCK_XFER_SIZE 512 + +/* check H5Dread()/H5Dwrite() error, e.g. memory allocation error inside the library. */ +#define CHECK_H5DRW_ERROR(_fun, _did, _mtid, _msid, _fsid, _pid, _buf) { \ + H5E_BEGIN_TRY { \ + if(_fun(_did, _mtid, _msid, _fsid, _pid, _buf) < 0) { \ + int _err_num = 0; \ + char _msg[80]; \ + H5Ewalk2(H5E_DEFAULT, H5E_WALK_DOWNWARD, walk_error_callback, &_err_num); \ + H5Eget_msg(_err_num, NULL, _msg, 80); \ + error_msg("%s %s -- %s\n", #_fun, "failed", _msg); \ + goto error; \ + } \ + } H5E_END_TRY; \ +} + /*------------------------------------------------------------------------- * local functions *------------------------------------------------------------------------- */ +static int Get_hyperslab (hid_t dcpl_id, int rank_dset, hsize_t dims_dset[], size_t size_datum, hsize_t dims_hslab[], hsize_t * hslab_nbytes_p); static void print_dataset_info(hid_t dcpl_id,char *objname,double per, int pr); static int do_copy_objects(hid_t fidin,hid_t fidout,trav_table_t *travt,pack_opt_t *options); static int copy_user_block(const char *infile, const char *outfile, hsize_t size); #if defined (H5REPACK_DEBUG_USER_BLOCK) static void print_user_block(const char *filename, hid_t fid); #endif +static herr_t walk_error_callback(unsigned n, const H5E_error2_t *err_desc, void *udata); + +/* get the major number from the error stack. */ +static herr_t walk_error_callback(UNUSED unsigned n, const H5E_error2_t *err_desc, void *udata) +{ + if (err_desc) + *((int *)udata) = err_desc->maj_num; + + return 0; +} /*------------------------------------------------------------------------- * Function: copy_objects @@ -105,23 +132,23 @@ int copy_objects(const char* fnamein, goto out; } - if(!options->fs_strategy) - { - if(H5Pget_file_space(fcpl_in, &options->fs_strategy, NULL) < 0) - { - error_msg("failed to retrieve file space strategy\n"); - goto out; - } - } - - if(!options->fs_threshold) - { - if(H5Pget_file_space(fcpl_in, NULL, &options->fs_threshold) < 0) - { - error_msg("failed to retrieve file space threshold\n"); - goto out; - } - } + if(!options->fs_strategy) + { + if(H5Pget_file_space(fcpl_in, &options->fs_strategy, NULL) < 0) + { + error_msg("failed to retrieve file space strategy\n"); + goto out; + } + } + + if(!options->fs_threshold) + { + if(H5Pget_file_space(fcpl_in, NULL, &options->fs_threshold) < 0) + { + error_msg("failed to retrieve file space threshold\n"); + goto out; + } + } if(H5Pclose(fcpl_in) < 0) { @@ -322,28 +349,28 @@ int copy_objects(const char* fnamein, /* either use the FCPL already created or create a new one */ if(fcpl != H5P_DEFAULT) { - /* set file space strategy and free space threshold */ - if(H5Pset_file_space(fcpl, options->fs_strategy, options->fs_threshold) < 0) - { - error_msg("failed to set file space strategy & threshold\n"); - goto out; - } + /* set file space strategy and free space threshold */ + if(H5Pset_file_space(fcpl, options->fs_strategy, options->fs_threshold) < 0) + { + error_msg("failed to set file space strategy & threshold\n"); + goto out; + } } else { - /* create a file creation property list */ - if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) - { - error_msg("fail to create a file creation property list\n"); - goto out; - } - - /* set file space strategy and free space threshold */ - if(H5Pset_file_space(fcpl, options->fs_strategy, options->fs_threshold) < 0) - { - error_msg("failed to set file space strategy & threshold \n"); - goto out; - } + /* create a file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + { + error_msg("fail to create a file creation property list\n"); + goto out; + } + + /* set file space strategy and free space threshold */ + if(H5Pset_file_space(fcpl, options->fs_strategy, options->fs_threshold) < 0) + { + error_msg("failed to set file space strategy & threshold \n"); + goto out; + } } /*------------------------------------------------------------------------- @@ -465,6 +492,184 @@ out: } /*------------------------------------------------------------------------- +* Function: Get_hyperslab +* +* Purpose: Calulate a hyperslab from a dataset for higher performance. +* The size of hyperslab is limitted by H5TOOLS_BUFSIZE. +* Return the hyperslab dimentions and size in byte. +* +* Return: 0 - SUCCEED, -1 FAILED +* +* Parameters: +* dcpl_id : [IN] dataset creation property. +* rank_dset : [IN] dataset rank +* dims_dset[] : [IN] dataset dimentions +* size_datum : [IN] size of a data element in byte +* dims_hslab[] : [OUT] calculated hyperslab dimentions +* * hslab_nbytes_p : [OUT] total byte of the hyperslab +* +* Programmer: Jonathan Kim +* Date: Feburary, 2012 +* Update: +* The hyperslab calucation would be depend on if the dataset is chunked +* or not. +* +* There care 3 conditions to cover: +* 1. If chunked and a chunk fits in buffer, each chunk would be a unit of +* collection and the boundary would be dataset's dims. +* 2. If chunked but a chunk doesn't fit in buffer, each data element would +* be a unit of collection and the boundary would be the chunk itself. +* 3. If not chunked, each data element would be a unit of collection and +* the boundary would be dataset's dims. +* +* The calulation starts from the last dimention (h5dump dims output). +* +* Note: +* Added for JIRA HDFFV-7862. +*-----------------------------------------*/ + +int Get_hyperslab (hid_t dcpl_id, int rank_dset, hsize_t dims_dset[], + size_t size_datum, + hsize_t dims_hslab[], hsize_t * hslab_nbytes_p) +{ + int status = 0; + int k; + H5D_layout_t dset_layout; + int rank_chunk; + hsize_t dims_chunk[H5S_MAX_RANK]; + hsize_t size_chunk=1; + hsize_t nchunk_fit; /* number of chunks that fits in hyperslab buffer (H5TOOLS_BUFSIZE) */ + hsize_t ndatum_fit; /* number of dataum that fits in hyperslab buffer (H5TOOLS_BUFSIZE) */ + hsize_t chunk_dims_map[H5S_MAX_RANK]; /* mapped chunk dimentions */ + hsize_t hs_dims_map[H5S_MAX_RANK]; /* mapped hyperslab dimentions */ + hsize_t hslab_nbytes; /* size of hyperslab in byte */ + + /* init to set as size of a data element */ + hslab_nbytes = size_datum; + + /* get layout of dataset */ + dset_layout = H5Pget_layout(dcpl_id); + + /* if dataset is chunked */ + if ( dset_layout == H5D_CHUNKED ) + { + /* get chunk dims */ + rank_chunk = H5Pget_chunk(dcpl_id, rank_dset, dims_chunk); + if (rank_chunk < 0) + { + status = -1; + goto out; + } + + for (k = rank_dset; k > 0; --k) + size_chunk *= dims_chunk[k-1]; + + /* figure out how many chunks can fit in the hyperslab buffer */ + nchunk_fit = (H5TOOLS_BUFSIZE / size_datum) / size_chunk; + + + /* 1. if a chunk fit in hyperslab buffer */ + if (nchunk_fit >= 1) + { + /* Calulate a hyperslab that contains as many chunks that can fit + * in hyperslab buffer. Hyperslab will be increased starting from + * the last dimention of the dataset (see h5dump's dims output). + * The calculation boundary is dataset dims. + * In the loop, used mapping from a datum to a chunk to figure out + * chunk based hyperslab. + */ + for (k = rank_dset; k > 0; --k) + { + /* map dataset dimentions with a chunk dims */ + chunk_dims_map[k-1]= dims_dset[k-1] / dims_chunk[k-1]; + + /* if reminder exist, increse by 1 to cover partial edge chunks */ + if (dims_dset[k-1] % dims_chunk[k-1] > 0) + chunk_dims_map[k-1]++; + + /* get mapped hyperslab dims */ + hs_dims_map[k-1] = MIN (nchunk_fit, chunk_dims_map[k-1]); + + /* prepare next round */ + nchunk_fit = nchunk_fit / chunk_dims_map[k-1]; + /* if a chunk is bigger than the rest of buffer */ + if (nchunk_fit == 0) + nchunk_fit=1; + + /* get hyperslab dimentions as unmapping to actual size */ + dims_hslab[k-1] = MIN( (hs_dims_map[k-1] * dims_chunk[k-1]), dims_dset[k-1]); + + /* calculate total size for the hyperslab */ + hslab_nbytes *= dims_hslab[k-1]; + } + } + /* 2. if a chunk is bigger than hyperslab buffer */ + else + { + /* Calulate a hyperslab that contains as many data elements that + * can fit in hyperslab buffer. Hyperslab will be increased + * starting from the last dimention of the chunk (see h5dump's dims + * output). + * The calculation boundary is a chunk dims. + */ + for (k = rank_dset; k > 0; --k) + { + ndatum_fit = H5TOOLS_BUFSIZE / hslab_nbytes; + + /* if a datum is bigger than rest of buffer */ + if ( ndatum_fit == 0) + ndatum_fit = 1; + /* get hyperslab dimentions within a chunk boundary */ + dims_hslab[k - 1] = MIN (dims_chunk[k-1], ndatum_fit); + + /* calculate total size for the hyperslab */ + hslab_nbytes *= dims_hslab[k - 1]; + + if (hslab_nbytes <= 0) + { + status = -1; + goto out; + } + } + } + } + /* 3. if dataset is not chunked */ + else + { + /* Calulate a hyperslab that contains as many data elements that can + * fit in hyperslab buffer. Hyperslab will be increased starting from + * the last dimention of the dataset (see h5dump's dims output). + * The calculation boundary is dataset dims. + */ + for (k = rank_dset; k > 0; --k) + { + ndatum_fit = H5TOOLS_BUFSIZE / hslab_nbytes; + + /* if a datum is bigger than rest of buffer */ + if ( ndatum_fit == 0) + ndatum_fit = 1; + /* get hyperslab dimentions within dataset boundary */ + dims_hslab[k - 1] = MIN(dims_dset[k - 1], ndatum_fit); + + /* calculate total size for the hyperslab */ + hslab_nbytes *= dims_hslab[k - 1]; + + if (hslab_nbytes <= 0) + { + status = -1; + goto out; + } + } + } + + /* pass out the hyperslab size*/ + *hslab_nbytes_p = hslab_nbytes; + +out: + return status; +} + +/*------------------------------------------------------------------------- * Function: do_copy_objects * * Purpose: duplicate all HDF5 objects in the file @@ -531,6 +736,30 @@ out: * * May, 1, 2008: Add a printing of the compression ratio of old size / new size * +* Feburary 2012: improve Read/Write by hyperslabs for big datasets. +* Programmer: Jonathan Kim +* +* A threshold of H5TOOLS_MALLOCSIZE is the limit upon which I/O hyperslab is done +* i.e., if the memory needed to read a dataset is greater than this limit, +* then hyperslab I/O is done instead of one operation I/O +* For each dataset, the memory needed is calculated according to +* +* memory needed = number of elements * size of each element +* +* if the memory needed is lower than H5TOOLS_MALLOCSIZE, then the following operations +* are done +* +* H5Dread( input_dataset ) +* H5Dwrite( output_dataset ) +* +* with all elements in the datasets selected. If the memory needed is greater than +* H5TOOLS_MALLOCSIZE, then the following operations are done instead: +* +* 1. figure out a hyperslab (dimentions) and size (refer to Get_hyperslab()). +* 2. Calculate the hyperslab selections as the selection is moving forward. +* Selection would be same as the hyperslab except for the remaining edge portion +* of the dataset. The code take care of the remaining portion if exist. +* *------------------------------------------------------------------------- */ @@ -547,7 +776,7 @@ int do_copy_objects(hid_t fidin, hid_t gcpl_out = -1; /* group creation property list */ hid_t type_in = -1; /* named type ID */ hid_t type_out = -1; /* named type ID */ - hid_t dcpl_id = -1; /* dataset creation property list ID */ + hid_t dcpl_in = -1; /* dataset creation property list ID */ hid_t dcpl_out = -1; /* dataset creation property list ID */ hid_t f_space_id = -1; /* file space ID */ hid_t ftype_id = -1; /* file type ID */ @@ -563,7 +792,7 @@ int do_copy_objects(hid_t fidin, int apply_s; /* flag for apply filter to small dataset sizes */ int apply_f; /* flag for apply filter to return error on H5Dcreate */ void *buf=NULL; /* buffer for raw data */ - void *sm_buf=NULL; /* buffer for raw data */ + void *hslab_buf=NULL; /* hyperslab buffer for raw data */ int has_filter; /* current object has a filter */ int req_filter; /* there was a request for a filter */ unsigned crt_order_flags; /* group creation order flag */ @@ -572,6 +801,7 @@ int do_copy_objects(hid_t fidin, int is_ref=0; htri_t is_named; + /*------------------------------------------------------------------------- * copy the suppplied object list *------------------------------------------------------------------------- @@ -592,7 +822,7 @@ int do_copy_objects(hid_t fidin, { case H5TRAV_TYPE_UNKNOWN: - assert(0); + HDassert(0); break; /*------------------------------------------------------------------------- * H5TRAV_TYPE_GROUP @@ -734,9 +964,9 @@ int do_copy_objects(hid_t fidin, goto error; if((ftype_id = H5Dget_type(dset_in)) < 0) goto error; - if((dcpl_id = H5Dget_create_plist(dset_in)) < 0) + if((dcpl_in = H5Dget_create_plist(dset_in)) < 0) goto error; - if((dcpl_out = H5Pcopy(dcpl_id)) < 0) + if((dcpl_out = H5Pcopy(dcpl_in)) < 0) goto error; if((rank = H5Sget_simple_extent_ndims(f_space_id)) < 0) goto error; @@ -769,7 +999,7 @@ int do_copy_objects(hid_t fidin, * 2) the internal filters might be turned off *------------------------------------------------------------------------- */ - if (h5tools_canreadf((travt->objs[i].name),dcpl_id)==1) + if (h5tools_canreadf((travt->objs[i].name),dcpl_in)==1) { apply_s=1; apply_f=1; @@ -808,6 +1038,11 @@ int do_copy_objects(hid_t fidin, goto error; } + /* unset the unlimimted dimensions, which cannot be applied to layout other than chunked. */ + if (options->layout_g != H5D_CHUNKED) { + H5Sset_extent_simple( f_space_id, rank, dims, NULL ); + } + /*------------------------------------------------------------------------- * create the output dataset; * disable error checking in case the dataset cannot be created with the @@ -824,7 +1059,7 @@ int do_copy_objects(hid_t fidin, printf(" warning: could not create dataset <%s>. Applying original settings\n", travt->objs[i].name); - if((dset_out = H5Dcreate2(fidout, travt->objs[i].name, wtype_id, f_space_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT)) < 0) + if((dset_out = H5Dcreate2(fidout, travt->objs[i].name, wtype_id, f_space_id, H5P_DEFAULT, dcpl_in, H5P_DEFAULT)) < 0) goto error; apply_f = 0; } @@ -842,11 +1077,10 @@ int do_copy_objects(hid_t fidin, buf = HDmalloc(need); if(buf != NULL) { - if(H5Dread(dset_in, wtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) - goto error; - if(H5Dwrite(dset_out, wtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) - goto error; - + /* read/write: use the macro to check error, e.g. memory allocation error inside the library. */ + CHECK_H5DRW_ERROR(H5Dread, dset_in, wtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf); + CHECK_H5DRW_ERROR(H5Dwrite, dset_out, wtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf); + /* Check if we have VL data in the dataset's * datatype that must be reclaimed */ if(TRUE == H5Tdetect_class(wtype_id, H5T_VLEN)) @@ -856,102 +1090,108 @@ int do_copy_objects(hid_t fidin, else /* possibly not enough memory, read/write by hyperslabs */ { size_t p_type_nbytes = msize; /*size of memory type */ - hsize_t p_nelmts = nelmts; /*total selected elmts */ + hsize_t p_nelmts = nelmts; /*total elements */ hsize_t elmtno; /*counter */ int carry; /*counter carry value */ unsigned int vl_data = 0; /*contains VL datatypes */ - /* stripmine info */ - hsize_t sm_size[H5S_MAX_RANK]; /*stripmine size */ - hsize_t sm_nbytes; /*bytes per stripmine */ - hsize_t sm_nelmts; /*elements per stripmine*/ - hid_t sm_space; /*stripmine data space */ - /* hyperslab info */ - hsize_t hs_offset[H5S_MAX_RANK];/*starting offset */ - hsize_t hs_size[H5S_MAX_RANK]; /*size this pass */ - hsize_t hs_nelmts; /*elements in request */ + hsize_t hslab_dims[H5S_MAX_RANK]; /*hyperslab dims */ + hsize_t hslab_nbytes; /*bytes per hyperslab */ + hsize_t hslab_nelmts; /*elements per hyperslab*/ + hid_t hslab_space; /*hyperslab data space */ + + /* hyperslab selection info */ + hsize_t hs_sel_offset[H5S_MAX_RANK];/* selection offset */ + hsize_t hs_sel_count[H5S_MAX_RANK]; /* selection count */ + hsize_t hs_select_nelmts; /* selected elements */ hsize_t zero[8]; /*vector of zeros */ int k; + H5D_layout_t dset_layout; + hid_t dcpl_tmp = -1; /* dataset creation property list ID */ /* check if we have VL data in the dataset's datatype */ if (H5Tdetect_class(wtype_id, H5T_VLEN) == TRUE) vl_data = TRUE; - /* - * determine the strip mine size and allocate a buffer. The strip mine is - * a hyperslab whose size is manageable. - */ - sm_nbytes = p_type_nbytes; - for (k = rank; k > 0; --k) + /* check first if writing dataset is chunked, + * if so use its chunk layout for better performance. */ + dset_layout = H5Pget_layout(dcpl_out); + if (dset_layout == H5D_CHUNKED) + dcpl_tmp = dcpl_out; /* writing dataset */ + else /* if reading dataset is chunked */ { - hsize_t size = H5TOOLS_BUFSIZE / sm_nbytes; - if ( size == 0) /* datum size > H5TOOLS_BUFSIZE */ - size = 1; - sm_size[k - 1] = MIN(dims[k - 1], size); - sm_nbytes *= sm_size[k - 1]; - assert(sm_nbytes > 0); + dset_layout = H5Pget_layout(dcpl_in); + if (dset_layout == H5D_CHUNKED) + dcpl_tmp = dcpl_in; /* reading dataset */ } - sm_buf = HDmalloc((size_t)sm_nbytes); - sm_nelmts = sm_nbytes / p_type_nbytes; - sm_space = H5Screate_simple(1, &sm_nelmts, NULL); + /* get hyperslab dims and size in byte */ + if(Get_hyperslab(dcpl_tmp, rank, dims, p_type_nbytes, hslab_dims, &hslab_nbytes) < 0) + goto error; + + hslab_buf = HDmalloc((size_t)hslab_nbytes); + + hslab_nelmts = hslab_nbytes / p_type_nbytes; + hslab_space = H5Screate_simple(1, &hslab_nelmts, NULL); - /* the stripmine loop */ - memset(hs_offset, 0, sizeof hs_offset); - memset(zero, 0, sizeof zero); + /* the hyperslab selection loop */ + HDmemset(hs_sel_offset, 0, sizeof hs_sel_offset); + HDmemset(zero, 0, sizeof zero); - for (elmtno = 0; elmtno < p_nelmts; elmtno += hs_nelmts) + for (elmtno = 0; elmtno < p_nelmts; elmtno += hs_select_nelmts) { - /* calculate the hyperslab size */ if (rank > 0) { - for (k = 0, hs_nelmts = 1; k < rank; k++) + /* calculate the hyperslab selections. The selection would be same as the hyperslab except for remaining edge portion of the dataset which is smaller then the hyperslab. + */ + for (k = 0, hs_select_nelmts = 1; k < rank; k++) { - hs_size[k] = MIN(dims[k] - hs_offset[k], sm_size[k]); - hs_nelmts *= hs_size[k]; + /* MIN() is used to get the remaining edge portion if exist. + * "dims[k] - hs_sel_offset[k]" is remaining edge portion that is smaller then the hyperslab.*/ + hs_sel_count[k] = MIN(dims[k] - hs_sel_offset[k], hslab_dims[k]); + hs_select_nelmts *= hs_sel_count[k]; } - if (H5Sselect_hyperslab(f_space_id, H5S_SELECT_SET, hs_offset, NULL, hs_size, NULL) < 0) + if (H5Sselect_hyperslab(f_space_id, H5S_SELECT_SET, hs_sel_offset, NULL, hs_sel_count, NULL) < 0) goto error; - if (H5Sselect_hyperslab(sm_space, H5S_SELECT_SET, zero, NULL, &hs_nelmts, NULL) < 0) + if (H5Sselect_hyperslab(hslab_space, H5S_SELECT_SET, zero, NULL, &hs_select_nelmts, NULL) < 0) goto error; } else { H5Sselect_all(f_space_id); - H5Sselect_all(sm_space); - hs_nelmts = 1; + H5Sselect_all(hslab_space); + hs_select_nelmts = 1; } /* rank */ - /* read/write */ - if (H5Dread(dset_in, wtype_id, sm_space, f_space_id, H5P_DEFAULT, sm_buf) < 0) - goto error; - if (H5Dwrite(dset_out, wtype_id, sm_space, f_space_id, H5P_DEFAULT, sm_buf) < 0) - goto error; + /* read/write: use the macro to check error, e.g. memory allocation error inside the library. */ + CHECK_H5DRW_ERROR(H5Dread, dset_in, wtype_id, hslab_space, f_space_id, H5P_DEFAULT, hslab_buf); + CHECK_H5DRW_ERROR(H5Dwrite, dset_out, wtype_id, hslab_space, f_space_id, H5P_DEFAULT, hslab_buf); /* reclaim any VL memory, if necessary */ if(vl_data) - H5Dvlen_reclaim(wtype_id, sm_space, H5P_DEFAULT, sm_buf); + H5Dvlen_reclaim(wtype_id, hslab_space, H5P_DEFAULT, hslab_buf); /* calculate the next hyperslab offset */ for (k = rank, carry = 1; k > 0 && carry; --k) { - hs_offset[k - 1] += hs_size[k - 1]; - if (hs_offset[k - 1] == dims[k - 1]) - hs_offset[k - 1] = 0; + hs_sel_offset[k - 1] += hs_sel_count[k - 1]; + /* if reached the end of a dim */ + if (hs_sel_offset[k - 1] == dims[k - 1]) + hs_sel_offset[k - 1] = 0; else carry = 0; } /* k */ } /* elmtno */ - H5Sclose(sm_space); + H5Sclose(hslab_space); /* free */ - if (sm_buf!=NULL) + if (hslab_buf!=NULL) { - HDfree(sm_buf); - sm_buf=NULL; + HDfree(hslab_buf); + hslab_buf=NULL; } } /* hyperslab read */ } /* if (nelmts>0 && space_status==H5D_SPACE_STATUS_NOT_ALLOCATED) */ @@ -977,7 +1217,7 @@ int do_copy_objects(hid_t fidin, print_dataset_info(dcpl_out,travt->objs[i].name,ratio,1); } else - print_dataset_info(dcpl_id,travt->objs[i].name,ratio,0); + print_dataset_info(dcpl_in,travt->objs[i].name,ratio,0); /* print a message that the filter was not applied (in case there was a filter) @@ -1016,7 +1256,7 @@ int do_copy_objects(hid_t fidin, goto error; if (H5Tclose(wtype_id) < 0) goto error; - if (H5Pclose(dcpl_id) < 0) + if (H5Pclose(dcpl_in) < 0) goto error; if (H5Pclose(dcpl_out) < 0) goto error; @@ -1171,7 +1411,7 @@ error: H5E_BEGIN_TRY { H5Gclose(grp_in); H5Gclose(grp_out); - H5Pclose(dcpl_id); + H5Pclose(dcpl_in); H5Pclose(gcpl_in); H5Pclose(gcpl_out); H5Sclose(f_space_id); @@ -1186,8 +1426,8 @@ error: /* free */ if (buf!=NULL) HDfree(buf); - if (sm_buf!=NULL) - HDfree(sm_buf); + if (hslab_buf!=NULL) + HDfree(hslab_buf); return -1; } @@ -1317,7 +1557,7 @@ copy_user_block(const char *infile, const char *outfile, hsize_t size) int status = 0; /* Return value */ /* User block must be any power of 2 equal to 512 or greater (512, 1024, 2048, etc.) */ - assert(size > 0); + HDassert(size > 0); /* Open files */ if((infid = HDopen(infile, O_RDONLY, 0)) < 0) { @@ -1359,13 +1599,13 @@ copy_user_block(const char *infile, const char *outfile, hsize_t size) status = -1; goto done; } /* end if */ - assert(nwritten > 0); - assert(nwritten <= nbytes); + HDassert(nwritten > 0); + HDassert(nwritten <= nbytes); /* Update # of bytes left & offset in buffer */ nbytes -= nwritten; wbuf += nwritten; - assert(nbytes == 0 || wbuf < (rbuf + USERBLOCK_XFER_SIZE)); + HDassert(nbytes == 0 || wbuf < (rbuf + USERBLOCK_XFER_SIZE)); } /* end while */ /* Update size of userblock left to transfer */ diff --git a/tools/h5repack/h5repack_filters.c b/tools/h5repack/h5repack_filters.c index 213ff65..6c7bc65 100644 --- a/tools/h5repack/h5repack_filters.c +++ b/tools/h5repack/h5repack_filters.c @@ -15,15 +15,16 @@ #include "h5repack.h" #include "h5tools.h" +#include "h5tools_utils.h" /* number of members in an array */ #ifndef NELMTS -# define NELMTS(X) (sizeof(X)/sizeof(X[0])) +# define NELMTS(X) (sizeof(X)/sizeof(X[0])) #endif /* minimum of two values */ #undef MIN -#define MIN(a,b) (((a)<(b)) ? (a) : (b)) +#define MIN(a,b) (((a)<(b)) ? (a) : (b)) /*------------------------------------------------------------------------- * Function: aux_find_obj @@ -329,7 +330,7 @@ int apply_filters(const char* name, /* object name from traverse list */ size = 1; sm_size[i - 1] = MIN(dims[i - 1], size); sm_nbytes *= sm_size[i - 1]; - assert(sm_nbytes > 0); + HDassert(sm_nbytes > 0); } diff --git a/tools/h5repack/h5repack_main.c b/tools/h5repack/h5repack_main.c index 6ec1647..a630559 100644 --- a/tools/h5repack/h5repack_main.c +++ b/tools/h5repack/h5repack_main.c @@ -13,6 +13,7 @@ * access to either file, you may request a copy from help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +#include "h5tools.h" #include "h5tools_utils.h" #include "h5repack.h" @@ -106,6 +107,13 @@ int main(int argc, const char **argv) h5tools_setprogname(PROGRAMNAME); h5tools_setstatus(EXIT_SUCCESS); + /* Initialize h5tools lib */ + h5tools_init(); + + /* update hyperslab buffer size from H5TOOLS_BUFSIZE env if exist */ + if ( h5tools_getenv_update_hyperslab_bufsize() < 0) + HDexit(EXIT_FAILURE); + /* initialize options */ h5repack_init(&options, 0, 0, 0, (hsize_t)0); @@ -124,7 +132,7 @@ int main(int argc, const char **argv) { error_msg("file names cannot be the same\n"); usage(h5tools_getprogname()); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } } @@ -133,7 +141,7 @@ int main(int argc, const char **argv) { error_msg("file names missing\n"); usage(h5tools_getprogname()); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } } @@ -316,10 +324,10 @@ void parse_command_line(int argc, const char **argv, pack_opt_t* options) case 'h': usage(h5tools_getprogname()); - exit(EXIT_SUCCESS); + HDexit(EXIT_SUCCESS); case 'V': print_version(h5tools_getprogname()); - exit(EXIT_SUCCESS); + HDexit(EXIT_SUCCESS); case 'v': options->verbose = 1; break; @@ -329,7 +337,7 @@ void parse_command_line(int argc, const char **argv, pack_opt_t* options) if (h5repack_addfilter( opt_arg, options)<0) { error_msg("in parsing filter\n"); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } break; case 'l': @@ -338,18 +346,18 @@ void parse_command_line(int argc, const char **argv, pack_opt_t* options) if (h5repack_addlayout( opt_arg, options)<0) { error_msg("in parsing layout\n"); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } break; case 'm': - options->min_comp = atoi( opt_arg ); + options->min_comp = HDatoi( opt_arg ); if ((int)options->min_comp<=0) { error_msg("invalid minimum compress size <%s>\n", opt_arg ); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } break; @@ -367,7 +375,7 @@ void parse_command_line(int argc, const char **argv, pack_opt_t* options) case 'c': - options->grp_compact = atoi( opt_arg ); + options->grp_compact = HDatoi( opt_arg ); if (options->grp_compact>0) options->latest = 1; /* must use latest format */ break; @@ -375,7 +383,7 @@ void parse_command_line(int argc, const char **argv, pack_opt_t* options) case 'd': - options->grp_indexed = atoi( opt_arg ); + options->grp_indexed = HDatoi( opt_arg ); if (options->grp_indexed>0) options->latest = 1; /* must use latest format */ break; @@ -390,7 +398,7 @@ void parse_command_line(int argc, const char **argv, pack_opt_t* options) options->latest = 1; /* must use latest format */ if (msgPtr == NULL) { - ssize = atoi( opt_arg ); + ssize = HDatoi( opt_arg ); for (idx=0; idx<5; idx++) options->msg_size[idx] = ssize; } @@ -399,7 +407,7 @@ void parse_command_line(int argc, const char **argv, pack_opt_t* options) char msgType[10]; HDstrcpy(msgType, msgPtr+1); msgPtr[0] = '\0'; - ssize = atoi( opt_arg ); + ssize = HDatoi( opt_arg ); if (HDstrncmp(msgType, "dspace",6) == 0) { options->msg_size[0] = ssize; } @@ -428,26 +436,26 @@ void parse_command_line(int argc, const char **argv, pack_opt_t* options) case 'b': - options->ublock_size = (hsize_t)atol( opt_arg ); + options->ublock_size = (hsize_t)HDatol( opt_arg ); break; case 't': - options->threshold = (hsize_t)atol( opt_arg ); + options->threshold = (hsize_t)HDatol( opt_arg ); break; case 'a': - options->alignment = atol( opt_arg ); + options->alignment = HDatol( opt_arg ); if ( options->alignment < 1 ) { error_msg("invalid alignment size\n", opt_arg ); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } break; case 'S': - { + { char strategy[MAX_NC_NAME]; HDstrcpy(strategy, opt_arg); @@ -461,14 +469,14 @@ void parse_command_line(int argc, const char **argv, pack_opt_t* options) options->fs_strategy = H5F_FILE_SPACE_VFD; else { error_msg("invalid file space management strategy\n", opt_arg ); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } break; } case 'T': - options->fs_threshold = (hsize_t)atol( opt_arg ); + options->fs_threshold = (hsize_t)HDatol( opt_arg ); break; } /* switch */ @@ -482,7 +490,7 @@ void parse_command_line(int argc, const char **argv, pack_opt_t* options) { error_msg("missing file names\n"); usage(h5tools_getprogname()); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } } @@ -513,9 +521,9 @@ void read_info(const char *filename, char c; int i, rc=1; - if ((fp = fopen(filename, "r")) == (FILE *)NULL) { + if ((fp = HDfopen(filename, "r")) == (FILE *)NULL) { error_msg("cannot open options file %s\n", filename); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } /* cycle until end of file reached */ @@ -536,7 +544,7 @@ void read_info(const char *filename, while( c!=' ' ) { fscanf(fp, "%c", &c); - if (feof(fp)) break; + if (HDfeof(fp)) break; } c='0'; /* go until end */ @@ -545,14 +553,14 @@ void read_info(const char *filename, fscanf(fp, "%c", &c); comp_info[i]=c; i++; - if (feof(fp)) break; + if (HDfeof(fp)) break; if (c==10 /*eol*/) break; } comp_info[i-1]='\0'; /*cut the last " */ if (h5repack_addfilter(comp_info,options)==-1){ error_msg("could not add compression option\n"); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } } /*------------------------------------------------------------------------- @@ -566,7 +574,7 @@ void read_info(const char *filename, while( c!=' ' ) { fscanf(fp, "%c", &c); - if (feof(fp)) break; + if (HDfeof(fp)) break; } c='0'; /* go until end */ @@ -575,14 +583,14 @@ void read_info(const char *filename, fscanf(fp, "%c", &c); comp_info[i]=c; i++; - if (feof(fp)) break; + if (HDfeof(fp)) break; if (c==10 /*eol*/) break; } comp_info[i-1]='\0'; /*cut the last " */ if (h5repack_addlayout(comp_info,options)==-1){ error_msg("could not add chunck option\n"); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } } /*------------------------------------------------------------------------- @@ -591,10 +599,10 @@ void read_info(const char *filename, */ else { error_msg("bad file format for %s", filename); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } } - fclose(fp); + HDfclose(fp); return; } diff --git a/tools/h5repack/h5repack_opttable.c b/tools/h5repack/h5repack_opttable.c index 57b5fa7..d0fcff3 100644 --- a/tools/h5repack/h5repack_opttable.c +++ b/tools/h5repack/h5repack_opttable.c @@ -14,6 +14,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include "h5repack.h" +#include "h5tools.h" #include "h5tools_utils.h" /*------------------------------------------------------------------------- @@ -120,7 +121,7 @@ static int aux_inctable(pack_opttbl_t *table, int n_objs ) unsigned int i; table->size += n_objs; - table->objs = (pack_info_t*)realloc(table->objs, table->size * sizeof(pack_info_t)); + table->objs = (pack_info_t*)HDrealloc(table->objs, table->size * sizeof(pack_info_t)); if (table->objs==NULL) { error_msg("not enough memory for options table\n"); return -1; @@ -147,7 +148,7 @@ int options_table_init( pack_opttbl_t **tbl ) unsigned int i; pack_opttbl_t *table; - if(NULL == (table = (pack_opttbl_t *)malloc(sizeof(pack_opttbl_t)))) + if(NULL == (table = (pack_opttbl_t *)HDmalloc(sizeof(pack_opttbl_t)))) { error_msg("not enough memory for options table\n"); return -1; @@ -155,10 +156,10 @@ int options_table_init( pack_opttbl_t **tbl ) table->size = 30; table->nelems = 0; - if(NULL == (table->objs = (pack_info_t*)malloc(table->size * sizeof(pack_info_t)))) + if(NULL == (table->objs = (pack_info_t*)HDmalloc(table->size * sizeof(pack_info_t)))) { error_msg("not enough memory for options table\n"); - free(table); + HDfree(table); return -1; } @@ -182,8 +183,8 @@ int options_table_init( pack_opttbl_t **tbl ) int options_table_free( pack_opttbl_t *table ) { - free(table->objs); - free(table); + HDfree(table->objs); + HDfree(table); return 0; } @@ -229,7 +230,7 @@ int options_add_layout( obj_list_t *obj_list, if (table->objs[i].chunk.rank>0) { error_msg("chunk information already inserted for <%s>\n",obj_list[j].obj); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } /* insert the layout info */ else diff --git a/tools/h5repack/h5repack_parse.c b/tools/h5repack/h5repack_parse.c index d3c6014..70eadd2 100644 --- a/tools/h5repack/h5repack_parse.c +++ b/tools/h5repack/h5repack_parse.c @@ -14,6 +14,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include "h5repack.h" +#include "h5tools.h" #include "h5tools_utils.h" /*------------------------------------------------------------------------- @@ -52,7 +53,7 @@ obj_list_t* parse_filter(const char *str, { unsigned i, u; char c; - size_t len=strlen(str); + size_t len=HDstrlen(str); int j, m, n, k, l, end_obj=-1, no_param=0; char sobj[MAX_NC_NAME]; char scomp[10]; @@ -63,7 +64,7 @@ obj_list_t* parse_filter(const char *str, /* initialize compression info */ - memset(filt,0,sizeof(filter_info_t)); + HDmemset(filt,0,sizeof(filter_info_t)); *is_glb = 0; /* check for the end of object list and number of objects */ @@ -88,7 +89,7 @@ obj_list_t* parse_filter(const char *str, } n++; - obj_list = (obj_list_t*) malloc(n*sizeof(obj_list_t)); + obj_list = (obj_list_t*) HDmalloc(n*sizeof(obj_list_t)); if (obj_list==NULL) { error_msg("could not allocate object list\n"); @@ -105,7 +106,7 @@ obj_list_t* parse_filter(const char *str, { if ( c==',') sobj[k]='\0'; else sobj[k+1]='\0'; HDstrcpy(obj_list[n].obj,sobj); - memset(sobj,0,sizeof(sobj)); + HDmemset(sobj,0,sizeof(sobj)); n++; k=-1; } @@ -113,9 +114,9 @@ obj_list_t* parse_filter(const char *str, /* nothing after : */ if (end_obj+1==(int)len) { - if (obj_list) free(obj_list); + if (obj_list) HDfree(obj_list); error_msg("input Error: Invalid compression type in <%s>\n",str); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } @@ -151,9 +152,9 @@ obj_list_t* parse_filter(const char *str, } c = str[u]; if (!isdigit(c) && l==-1){ - if (obj_list) free(obj_list); + if (obj_list) HDfree(obj_list); error_msg("compression parameter not digit in <%s>\n",str); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } if (l==-1) stype[m]=c; @@ -173,7 +174,7 @@ obj_list_t* parse_filter(const char *str, else { error_msg("szip mask must be 'NN' or 'EC' \n"); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } @@ -211,9 +212,9 @@ obj_list_t* parse_filter(const char *str, } c = str[u]; if (!isdigit(c) && l==-1){ - if (obj_list) free(obj_list); + if (obj_list) HDfree(obj_list); error_msg("compression parameter is not a digit in <%s>\n",str); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } if (l==-1) stype[m]=c; @@ -233,7 +234,7 @@ obj_list_t* parse_filter(const char *str, else { error_msg("scale type must be 'IN' or 'DS' \n"); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } } @@ -255,9 +256,9 @@ obj_list_t* parse_filter(const char *str, { c = str[u]; if (!isdigit(c)){ - if (obj_list) free(obj_list); + if (obj_list) HDfree(obj_list); error_msg("compression parameter is not a digit in <%s>\n",str); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } stype[m]=c; } /* u */ @@ -301,9 +302,9 @@ obj_list_t* parse_filter(const char *str, filt->cd_nelmts = 1; if (no_param) { /*no more parameters, GZIP must have parameter */ - if (obj_list) free(obj_list); + if (obj_list) HDfree(obj_list); error_msg("missing compression parameter in <%s>\n",str); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } } @@ -317,9 +318,9 @@ obj_list_t* parse_filter(const char *str, filt->cd_nelmts = 2; if (no_param) { /*no more parameters, SZIP must have parameter */ - if (obj_list) free(obj_list); + if (obj_list) HDfree(obj_list); error_msg("missing compression parameter in <%s>\n",str); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } } @@ -333,9 +334,9 @@ obj_list_t* parse_filter(const char *str, filt->cd_nelmts = 0; if (m>0) { /*shuffle does not have parameter */ - if (obj_list) free(obj_list); + if (obj_list) HDfree(obj_list); error_msg("extra parameter in SHUF <%s>\n",str); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } } /*------------------------------------------------------------------------- @@ -348,9 +349,9 @@ obj_list_t* parse_filter(const char *str, filt->cd_nelmts = 0; if (m>0) { /*shuffle does not have parameter */ - if (obj_list) free(obj_list); + if (obj_list) HDfree(obj_list); error_msg("extra parameter in FLET <%s>\n",str); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } } /*------------------------------------------------------------------------- @@ -363,9 +364,9 @@ obj_list_t* parse_filter(const char *str, filt->cd_nelmts = 0; if (m>0) { /*nbit does not have parameter */ - if (obj_list) free(obj_list); + if (obj_list) HDfree(obj_list); error_msg("extra parameter in NBIT <%s>\n",str); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } } /*------------------------------------------------------------------------- @@ -378,15 +379,15 @@ obj_list_t* parse_filter(const char *str, filt->cd_nelmts = 2; if (no_param) { /*no more parameters, SOFF must have parameter */ - if (obj_list) free(obj_list); + if (obj_list) HDfree(obj_list); error_msg("missing compression parameter in <%s>\n",str); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } } else { - if (obj_list) free(obj_list); + if (obj_list) HDfree(obj_list); error_msg("invalid filter type in <%s>\n",str); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } } } /*i*/ @@ -407,9 +408,9 @@ obj_list_t* parse_filter(const char *str, case H5Z_FILTER_DEFLATE: if (filt->cd_values[0]>9 ) { - if (obj_list) free(obj_list); + if (obj_list) HDfree(obj_list); error_msg("invalid compression parameter in <%s>\n",str); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } break; @@ -422,21 +423,21 @@ obj_list_t* parse_filter(const char *str, pixels_per_block=filt->cd_values[0]; if ((pixels_per_block%2)==1) { - if (obj_list) free(obj_list); + if (obj_list) HDfree(obj_list); error_msg("pixels_per_block is not even in <%s>\n",str); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } if (pixels_per_block>H5_SZIP_MAX_PIXELS_PER_BLOCK) { - if (obj_list) free(obj_list); + if (obj_list) HDfree(obj_list); error_msg("pixels_per_block is too large in <%s>\n",str); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } if ( (HDstrcmp(smask,"NN")!=0) && (HDstrcmp(smask,"EC")!=0) ) { - if (obj_list) free(obj_list); + if (obj_list) HDfree(obj_list); error_msg("szip mask must be 'NN' or 'EC' \n"); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } break; default: @@ -479,16 +480,16 @@ obj_list_t* parse_layout(const char *str, obj_list_t* obj_list=NULL; unsigned i; char c; - size_t len=strlen(str); + size_t len=HDstrlen(str); int j, n, k, end_obj=-1, c_index; char sobj[MAX_NC_NAME]; char sdim[10]; char slayout[10]; - memset(sdim, '\0', sizeof(sdim)); - memset(sobj, '\0', sizeof(sobj)); - memset(slayout, '\0', sizeof(slayout)); + HDmemset(sdim, '\0', sizeof(sdim)); + HDmemset(sobj, '\0', sizeof(sobj)); + HDmemset(slayout, '\0', sizeof(slayout)); /* check for the end of object list and number of objects */ for ( i=0, n=0; i<len; i++) @@ -509,7 +510,7 @@ obj_list_t* parse_layout(const char *str, } n++; - obj_list = (obj_list_t*) malloc(n*sizeof(obj_list_t)); + obj_list = (obj_list_t*) HDmalloc(n*sizeof(obj_list_t)); if (obj_list==NULL) { error_msg("could not allocate object list\n"); @@ -526,7 +527,7 @@ obj_list_t* parse_layout(const char *str, { if ( c==',') sobj[k]='\0'; else sobj[k+1]='\0'; HDstrcpy(obj_list[n].obj,sobj); - memset(sobj,0,sizeof(sobj)); + HDmemset(sobj,0,sizeof(sobj)); n++; k=-1; } @@ -535,9 +536,9 @@ obj_list_t* parse_layout(const char *str, /* nothing after : */ if (end_obj+1==(int)len) { - if (obj_list) free(obj_list); + if (obj_list) HDfree(obj_list); error_msg("in parse layout, no characters after : in <%s>\n",str); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } /* get layout info */ @@ -554,7 +555,7 @@ obj_list_t* parse_layout(const char *str, pack->layout=H5D_CHUNKED; else { error_msg("in parse layout, not a valid layout in <%s>\n",str); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } } else @@ -576,9 +577,9 @@ obj_list_t* parse_layout(const char *str, if (j>(int)len) { - if (obj_list) free(obj_list); + if (obj_list) HDfree(obj_list); error_msg("in parse layout, <%s> Chunk dimensions missing\n",str); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } for ( i=j, c_index=0; i<len; i++) @@ -590,10 +591,10 @@ obj_list_t* parse_layout(const char *str, if (!isdigit(c) && c!='x' && c!='N' && c!='O' && c!='N' && c!='E' ){ - if (obj_list) free(obj_list); + if (obj_list) HDfree(obj_list); error_msg("in parse layout, <%s> Not a valid character in <%s>\n", sdim,str); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } if ( c=='x' || i==len-1) @@ -603,10 +604,10 @@ obj_list_t* parse_layout(const char *str, k=0; pack->chunk.chunk_lengths[c_index]=atoi(sdim); if (pack->chunk.chunk_lengths[c_index]==0) { - if (obj_list) free(obj_list); + if (obj_list) HDfree(obj_list); error_msg("in parse layout, <%s> conversion to number in <%s>\n", sdim,str); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } c_index++; } @@ -621,10 +622,10 @@ obj_list_t* parse_layout(const char *str, { pack->chunk.chunk_lengths[c_index]=atoi(sdim); if (pack->chunk.chunk_lengths[c_index]==0){ - if (obj_list) free(obj_list); + if (obj_list) HDfree(obj_list); error_msg("in parse layout, <%s> conversion to number in <%s>\n", sdim,str); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } pack->chunk.rank=c_index+1; } diff --git a/tools/h5repack/h5repack_refs.c b/tools/h5repack/h5repack_refs.c index 091133e..7108491 100644 --- a/tools/h5repack/h5repack_refs.c +++ b/tools/h5repack/h5repack_refs.c @@ -27,6 +27,8 @@ static const char* MapIdToName(hid_t refobj_id,trav_table_t *travt); static int copy_refs_attr(hid_t loc_in, hid_t loc_out, pack_opt_t *options, trav_table_t *travt, hid_t fidout); +static herr_t update_ref_value(hid_t obj_id, H5R_type_t ref_type, void *ref_in, + hid_t fid_out, void *ref_out, trav_table_t *travt); /*------------------------------------------------------------------------- * Function: do_copy_refobjs @@ -178,7 +180,7 @@ int do_copy_refobjs(hid_t fidin, } /* end if */ for(u = 0; u < nelmts; u++) { H5E_BEGIN_TRY { - if((refobj_id = H5Rdereference(dset_in, H5R_OBJECT, &buf[u])) < 0) + if((refobj_id = H5Rdereference2(dset_in, H5P_DEFAULT, H5R_OBJECT, &buf[u])) < 0) continue; } H5E_END_TRY; @@ -229,7 +231,7 @@ int do_copy_refobjs(hid_t fidin, * dataset region references *------------------------------------------------------------------------- */ - else if(H5Tequal(mtype_id, H5T_STD_REF_DSETREG)) + else if(H5Tequal(mtype_id, H5T_STD_REF_DSETREG)) { hid_t refobj_id; hdset_reg_ref_t *refbuf = NULL; /* input buffer for region references */ @@ -262,7 +264,7 @@ int do_copy_refobjs(hid_t fidin, for(u = 0; u < nelmts; u++) { H5E_BEGIN_TRY { - if((refobj_id = H5Rdereference(dset_in, H5R_DATASET_REGION, &buf[u])) < 0) + if((refobj_id = H5Rdereference2(dset_in, H5P_DEFAULT, H5R_DATASET_REGION, &buf[u])) < 0) continue; } H5E_END_TRY; @@ -325,7 +327,7 @@ int do_copy_refobjs(hid_t fidin, goto error; } /* end else */ - assert(dset_out != FAIL); + HDassert(dset_out != FAIL); /*------------------------------------------------------------------------- * copy referenced objects in attributes @@ -390,7 +392,7 @@ int do_copy_refobjs(hid_t fidin, } /* end switch */ } /* end for */ - /* Finalize (link) the stack of named datatypes (if any) + /* Finalize (link) the stack of named datatypes (if any) * This function is paired with copy_named_datatype() which is called * in copy_attr(), so need to free. */ @@ -429,6 +431,15 @@ error: * * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu * + * Modifier: xcao@hdfgroup.org, 9/12/2011 + * Update values of references(object and region) for the following types: + * 1) References, + * 2) ARRAY of reference, + * 3) VLEN of references. + * 4) COMPOUND of references. + * This function does not handle references in other complicated structures, + * such as references in nested compound datatypes. + * * Date: October, 28, 2003 * *------------------------------------------------------------------------- @@ -448,34 +459,113 @@ static int copy_refs_attr(hid_t loc_in, hid_t mtype_id = -1; /* memory data type ID */ size_t msize; /* memory size of type */ hsize_t nelmts; /* number of elements in dataset */ - int rank; /* rank of dataset */ hsize_t dims[H5S_MAX_RANK];/* dimensions of dataset */ char name[255]; H5O_info_t oinfo; /* Object info */ - int j; - unsigned u; + unsigned u, i, j; + int rank; + H5T_class_t type_class = -1; + hbool_t is_ref=0, is_ref_vlen=0, is_ref_array=0, is_ref_comp=0; + void *refbuf = NULL; + void *buf = NULL; + const char* refname; + int *ref_comp_index = NULL; + size_t *ref_comp_size = NULL; + int ref_comp_field_n = 0; + if(H5Oget_info(loc_in, &oinfo) < 0) goto error; for(u = 0; u < (unsigned)oinfo.num_attrs; u++) { - /*------------------------------------------------------------------------- - * open - *------------------------------------------------------------------------- - */ + is_ref = is_ref_vlen = is_ref_array = is_ref_comp = 0; + /* open attribute */ if((attr_id = H5Aopen_by_idx(loc_in, ".", H5_INDEX_CRT_ORDER, H5_ITER_INC, (hsize_t)u, H5P_DEFAULT, H5P_DEFAULT)) < 0) goto error; - /* get name */ - if(H5Aget_name(attr_id, 255, name) < 0) - goto error; - /* get the file datatype */ if((ftype_id = H5Aget_type(attr_id)) < 0) goto error; + type_class = H5Tget_class(ftype_id); + + if((mtype_id = h5tools_get_native_type(ftype_id)) < 0) + goto error; + + if((msize = H5Tget_size(mtype_id)) == 0) + goto error; + + is_ref = (type_class == H5T_REFERENCE); + + if (type_class == H5T_VLEN ) { + hid_t base_type = -1; + base_type = H5Tget_super(ftype_id); + is_ref_vlen = (H5Tget_class(base_type)==H5T_REFERENCE); + msize = H5Tget_size(base_type); + H5Tclose(base_type); + } + + if (type_class == H5T_ARRAY ) { + hid_t base_type = -1; + base_type = H5Tget_super(ftype_id); + is_ref_array = (H5Tget_class(base_type)==H5T_REFERENCE); + msize = H5Tget_size(base_type); + H5Tclose(base_type); + } + + if (type_class == H5T_COMPOUND) { + int nmembers = H5Tget_nmembers(ftype_id) ; + if (nmembers < 1) + goto error; + + ref_comp_index = (int *)HDmalloc(nmembers*sizeof (int)); + ref_comp_size = (size_t *)HDmalloc(nmembers*sizeof(ref_comp_size)); + ref_comp_field_n = 0; + + for (i=0; i<(unsigned)nmembers; i++) { + hid_t mtid = H5Tget_member_type( ftype_id, i ); + if ((H5Tget_class(mtid)==H5T_REFERENCE)) { + ref_comp_index[ref_comp_field_n] = i; + ref_comp_size[ref_comp_field_n] = H5Tget_size(mtid); + ref_comp_field_n++; + } + H5Tclose(mtid); + } + + /* if compound don't contain reference type member, free the above + * mallocs. Otherwise there can be memory leaks by the 'continue' + * statement below. */ + if (!ref_comp_field_n) + { + if (ref_comp_index) + { + HDfree(ref_comp_index); + ref_comp_index = NULL; + } + + if (ref_comp_size) + { + HDfree(ref_comp_size); + ref_comp_size = NULL; + } + } + } + + is_ref_comp = (ref_comp_field_n > 0); + + if (!(is_ref || is_ref_vlen || is_ref_array || is_ref_comp)) { + H5Tclose(mtype_id); + H5Tclose(ftype_id); + H5Aclose(attr_id); + continue; + } + + /* get name */ + if(H5Aget_name(attr_id, 255, name) < 0) + goto error; + /* get the dataspace handle */ if((space_id = H5Aget_space(attr_id)) < 0) goto error; @@ -490,35 +580,32 @@ static int copy_refs_attr(hid_t loc_in, *------------------------------------------------------------------------- */ nelmts = 1; - for(j = 0; j < rank; j++) + for(j = 0; j < (unsigned)rank; j++) nelmts *= dims[j]; - if((mtype_id = h5tools_get_native_type(ftype_id)) < 0) - goto error; - - if((msize = H5Tget_size(mtype_id)) == 0) + if (is_ref_array) { + unsigned array_rank = 0; + hsize_t array_size = 1; + hsize_t array_dims[H5S_MAX_RANK]; + hid_t base_type = -1; + base_type = H5Tget_super(ftype_id); + msize = H5Tget_size(base_type); + H5Tclose(base_type); + + array_rank = H5Tget_array_ndims(mtype_id); + H5Tget_array_dims2(mtype_id, array_dims); + for(j = 0; j <array_rank; j++) + array_size *= array_dims[j]; + nelmts *= array_size; + } + + if((attr_out = H5Acreate2(loc_out, name, ftype_id, space_id, H5P_DEFAULT, H5P_DEFAULT)) < 0) goto error; - - /*------------------------------------------------------------------------- - * object references are a special case - * we cannot just copy the buffers, but instead we recreate the reference - *------------------------------------------------------------------------- - */ - if(H5Tequal(mtype_id, H5T_STD_REF_OBJ)) + if (nelmts>0) { - hid_t refobj_id; - hobj_ref_t *refbuf = NULL; - unsigned k; - const char* refname; - hobj_ref_t *buf = NULL; - - /*------------------------------------------------------------------------- - * read input to memory - *------------------------------------------------------------------------- - */ - - if (nelmts) + /* handle object references */ + if((is_ref || is_ref_array) && (H5R_OBJ_REF_BUF_SIZE==msize)) { buf = (hobj_ref_t *)HDmalloc((unsigned)(nelmts * msize)); if(buf == NULL) @@ -536,65 +623,16 @@ static int copy_refs_attr(hid_t loc_in, goto error; } /* end if */ - for(k = 0; k < nelmts; k++) + for(i = 0; i < (unsigned)nelmts; i++) { - H5E_BEGIN_TRY - { - if((refobj_id = H5Rdereference(attr_id, H5R_OBJECT, &buf[k])) < 0) - goto error; - } H5E_END_TRY; - - /* get the name. a valid name could only occur in the - * second traversal of the file - */ - if((refname = MapIdToName(refobj_id, travt)) != NULL) - { - /* create the reference */ - if(H5Rcreate(&refbuf[k], fidout, refname, H5R_OBJECT, -1) < 0) - goto error; - if(options->verbose) - printf("object <%s> reference created to <%s>\n", name, refname); - } - H5Oclose(refobj_id); + if (update_ref_value(attr_id, H5R_OBJECT, &((hobj_ref_t *)buf)[i], fidout, &((hobj_ref_t *)refbuf)[i], travt)<0) + continue; + if(options->verbose) + printf("object <%s> reference created to <%s>\n", name, refname); } /* k */ - } /*nelmts*/ - - /*------------------------------------------------------------------------- - * copy - *------------------------------------------------------------------------- - */ - if((attr_out = H5Acreate2(loc_out, name, ftype_id, space_id, H5P_DEFAULT, H5P_DEFAULT)) < 0) - goto error; - if(nelmts) - if(H5Awrite(attr_out, mtype_id, refbuf) < 0) - goto error; - - if(H5Aclose(attr_out) < 0) - goto error; - - if(refbuf) - HDfree(refbuf); - if(buf) - HDfree(buf); - }/*H5T_STD_REF_OBJ*/ - - /*------------------------------------------------------------------------- - * dataset region references - *------------------------------------------------------------------------- - */ - else if(H5Tequal(mtype_id, H5T_STD_REF_DSETREG)) - { - hid_t refobj_id; - hdset_reg_ref_t *refbuf = NULL; /* input buffer for region references */ - hdset_reg_ref_t *buf = NULL; /* output buffer */ - const char* refname; - unsigned k; - - /*------------------------------------------------------------------------- - * read input to memory - *------------------------------------------------------------------------- - */ - if(nelmts) + }/*H5T_STD_REF_OBJ*/ + /* handle region references */ + else if((is_ref || is_ref_array) && (H5R_DSET_REG_REF_BUF_SIZE == msize)) { buf = (hdset_reg_ref_t *)HDmalloc((unsigned)(nelmts * msize)); if(buf == NULL) @@ -616,56 +654,119 @@ static int copy_refs_attr(hid_t loc_in, goto error; } /* end if */ - for(k = 0; k < nelmts; k++) + for(i = 0; i < (unsigned)nelmts; i++) { - H5E_BEGIN_TRY - { - if((refobj_id = H5Rdereference(attr_id, H5R_DATASET_REGION, &buf[k])) < 0) - continue; - } H5E_END_TRY; - - /* get the name. a valid name could only occur in the - * second traversal of the file - */ - if((refname = MapIdToName(refobj_id, travt)) != NULL) - { - hid_t region_id; /* region id of the referenced dataset */ + if (update_ref_value(attr_id, H5R_DATASET_REGION, &((hdset_reg_ref_t *)buf)[i], fidout, &((hdset_reg_ref_t *)refbuf)[i], travt)<0) + continue; + if(options->verbose) + printf("object <%s> region reference created to <%s>\n", name, refname); + } + } /* H5T_STD_REF_DSETREG */ + else if (is_ref_vlen) { + /* handle VLEN of references */ + + buf = (hvl_t *)HDmalloc((unsigned)(nelmts * sizeof(hvl_t))); + refbuf = buf; /* reuse the read buffer for write */ - if((region_id = H5Rget_region(attr_id, H5R_DATASET_REGION, &buf[k])) < 0) - goto error; + if(buf == NULL) + { + printf( "cannot read into memory\n" ); + goto error; + } /* end if */ - /* create the reference, we need the space_id */ - if(H5Rcreate(&refbuf[k], fidout, refname, H5R_DATASET_REGION, region_id) < 0) - goto error; - if(H5Sclose(region_id) < 0) - goto error; - if(options->verbose) - printf("object <%s> region reference created to <%s>\n", name, refname); - } /* end if */ - H5Oclose(refobj_id); - } /* k */ - } /*nelmts */ + if(H5Aread(attr_id, mtype_id, buf) < 0) + goto error; - /*------------------------------------------------------------------------- - * copy - *------------------------------------------------------------------------- - */ - if((attr_out = H5Acreate2(loc_out, name, ftype_id, space_id, H5P_DEFAULT, H5P_DEFAULT)) < 0) - goto error; - if(nelmts) - { - if(H5Awrite(attr_out, mtype_id, refbuf) < 0) + if (H5R_OBJ_REF_BUF_SIZE==msize) { + hobj_ref_t ref_out; + for (i=0; i<(unsigned)nelmts; i++) { + hobj_ref_t *ptr = (hobj_ref_t *)((hvl_t *)buf)[i].p; + for (j=0; j<((hvl_t *)buf)[i].len; j++ ) { + if (update_ref_value(attr_id, H5R_OBJECT, &(ptr[j]), fidout, &ref_out, travt)<0) + continue; + HDmemcpy(&(ptr[j]), &ref_out, msize); + } + } /* for (i=0; i<nelems; i++) */ + } else if (H5R_DSET_REG_REF_BUF_SIZE == msize) { + + hdset_reg_ref_t ref_out; + for (i=0; i<(unsigned)nelmts; i++) { + hdset_reg_ref_t *ptr = (hdset_reg_ref_t *)((hvl_t *)buf)[i].p; + for (j=0; j<((hvl_t *)buf)[i].len; j++ ) { + if (update_ref_value(attr_id, H5R_DATASET_REGION, &(ptr[j]), fidout, &ref_out, travt)<0) + continue; + HDmemcpy(&(ptr[j]), &ref_out, msize); + } + } /* for (i=0; i<nelems; i++) */ + } + } /* else if (is_ref_vlen) */ + else if (is_ref_comp) { + /* handle ref fields in a compound */ + + buf = HDmalloc((unsigned)(nelmts * msize)); + refbuf = buf; /* reuse the read buffer for write */ + + if(buf == NULL) + { + printf( "cannot read into memory\n" ); goto error; - } + } /* end if */ - if(H5Aclose(attr_out) < 0) - goto error; + if(H5Aread(attr_id, mtype_id, buf) < 0) + goto error; - if(refbuf) - HDfree(refbuf); - if(buf) - HDfree(buf); - } /* H5T_STD_REF_DSETREG */ + for (i=0; i<(unsigned)nelmts; i++) { + for (j=0; j<(unsigned)ref_comp_field_n; j++) { + if (ref_comp_size[j] == H5R_OBJ_REF_BUF_SIZE) { + int idx = i*msize+H5Tget_member_offset( mtype_id, (unsigned)ref_comp_index[j]); + hobj_ref_t ref_out; + if (update_ref_value(attr_id, H5R_OBJECT, (hobj_ref_t *)(((char *)buf)+idx), fidout, &ref_out, travt)<0) + continue; + HDmemcpy(((char *)buf)+idx, &ref_out, ref_comp_size[j]); + } /* if */ + else if (ref_comp_size[j] == H5R_DSET_REG_REF_BUF_SIZE) { + int idx = i*msize+H5Tget_member_offset( mtype_id, (unsigned)ref_comp_index[j]); + hdset_reg_ref_t ref_out; + if (update_ref_value(attr_id, H5R_DATASET_REGION, (hdset_reg_ref_t *)(((char *)buf)+idx), fidout, &ref_out, travt)<0) + continue; + HDmemcpy(((char *)buf)+idx, &ref_out, ref_comp_size[j]); + } /* else if */ + } /* j */ + } /* i */ + } /* else if (is_ref_comp) */ + + if(H5Awrite(attr_out, mtype_id, refbuf) < 0) + goto error; + + if (is_ref_vlen && buf) + H5Dvlen_reclaim (mtype_id, space_id, H5P_DEFAULT, buf); + } /* if (nelmts) */ + + if (refbuf == buf) + refbuf = NULL; /* set it to NULL to avoid double free since buf and refbuf are the same. */ + + if(buf) { + HDfree(buf); + buf = NULL; + } + + if(refbuf) { + HDfree(refbuf); + refbuf = NULL; + } + + if (ref_comp_index) { + HDfree(ref_comp_index); + ref_comp_index = NULL; + } + + if (ref_comp_size) { + HDfree(ref_comp_size); + ref_comp_size = NULL; + } + + if(H5Aclose(attr_out) < 0) + goto error; /*------------------------------------------------------------------------- * close @@ -679,11 +780,22 @@ static int copy_refs_attr(hid_t loc_in, goto error; if(H5Aclose(attr_id) < 0) goto error; - } /* u */ + } /* for(u = 0; u < (unsigned)oinfo.num_attrs; u++) */ return 0; error: + if(refbuf) + HDfree(refbuf); + if(buf) + HDfree(buf); + + if (ref_comp_index) + HDfree(ref_comp_index); + + if (ref_comp_size) + HDfree(ref_comp_size); + H5E_BEGIN_TRY { H5Tclose(ftype_id); H5Tclose(mtype_id); @@ -710,7 +822,7 @@ MapIdToName(hid_t refobj_id, trav_table_t *travt) /* linear search */ for(u = 0; u < travt->nobjs; u++) { - if(travt->objs[u].type == H5O_TYPE_DATASET || + if(travt->objs[u].type == H5O_TYPE_DATASET || travt->objs[u].type == H5O_TYPE_GROUP || travt->objs[u].type == H5O_TYPE_NAMED_DATATYPE) { H5O_info_t ref_oinfo; /* Stat for the refobj id */ @@ -730,3 +842,49 @@ out: return ret; } +/*------------------------------------------------------------------------- + * Function: Update_Ref_value + * + * Purpose: Update a reference value + * + * Programmer: xcao@hdfgroup.org 9/12/2011 + * + *------------------------------------------------------------------------- + */ +static herr_t update_ref_value(hid_t obj_id, H5R_type_t ref_type, void *ref_in, + hid_t fid_out, void *ref_out, trav_table_t *travt) +{ + herr_t ret = -1; + const char* ref_obj_name; + hid_t space_id=-1, ref_obj_id=-1; + + ref_obj_id = H5Rdereference2(obj_id, H5P_DEFAULT, ref_type, ref_in); + if (ref_obj_id<0) + goto done; + + ref_obj_name = MapIdToName(ref_obj_id, travt); + if (ref_obj_name == NULL) + goto done; + + if (ref_type == H5R_DATASET_REGION) { + space_id = H5Rget_region(obj_id, H5R_DATASET_REGION, ref_in); + if (space_id < 0) + goto done; + } + + ret = H5Rcreate(ref_out, fid_out, ref_obj_name, ref_type, space_id); + + if (ret < 0) + goto done; + + ret = 0; + +done: + H5E_BEGIN_TRY { + H5Sclose(space_id); + H5Oclose(ref_obj_id); + } H5E_END_TRY; + + return ret; +} + diff --git a/tools/h5repack/h5repack_verify.c b/tools/h5repack/h5repack_verify.c index b0ab1d1..47cba5b 100644 --- a/tools/h5repack/h5repack_verify.c +++ b/tools/h5repack/h5repack_verify.c @@ -14,16 +14,17 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include "h5repack.h" +#include "h5tools.h" #include "h5tools_utils.h" -static int verify_layout(hid_t pid, pack_info_t *obj); -static int verify_filters(hid_t pid, hid_t tid, int nfilters, filter_info_t *filter); - /* number of members in an array */ #ifndef NELMTS -# define NELMTS(X) (sizeof(X)/sizeof(X[0])) +# define NELMTS(X) (sizeof(X)/sizeof(X[0])) #endif +static int verify_layout(hid_t pid, pack_info_t *obj); +static int verify_filters(hid_t pid, hid_t tid, int nfilters, filter_info_t *filter); + /*------------------------------------------------------------------------- * Function: h5repack_verify @@ -301,9 +302,9 @@ error: * * Purpose: verify which layout is present in the property list DCPL_ID * - * H5D_COMPACT = 0 - * H5D_CONTIGUOUS = 1 - * H5D_CHUNKED = 2 + * H5D_COMPACT = 0 + * H5D_CONTIGUOUS = 1 + * H5D_CHUNKED = 2 * * Return: 1 has, 0 does not, -1 error * diff --git a/tools/h5repack/h5repacktst.c b/tools/h5repack/h5repacktst.c index 0567615..05eac2f 100644 --- a/tools/h5repack/h5repacktst.c +++ b/tools/h5repack/h5repacktst.c @@ -17,6 +17,7 @@ #include "h5test.h" #include "h5diff.h" #include "h5tools.h" +#include "h5tools_utils.h" #define GOERROR {H5_FAILED(); goto error;} @@ -169,15 +170,21 @@ int main (void) { pack_opt_t pack_options; diff_opt_t diff_options; - hsize_t fs_size = 0; /* free space section threshold */ - H5F_file_space_type_t fs_type = H5F_FILE_SPACE_DEFAULT; /* file space handling strategy */ + hsize_t fs_size = 0; /* free space section threshold */ + H5F_file_space_type_t fs_type = H5F_FILE_SPACE_DEFAULT; /* file space handling strategy */ #if defined (H5_HAVE_FILTER_SZIP) int szip_can_encode = 0; #endif + h5tools_setprogname(PROGRAMNAME); + h5tools_setstatus(EXIT_SUCCESS); + + /* Initialize h5tools lib */ + h5tools_init(); + /* initialize */ - memset(&diff_options, 0, sizeof (diff_opt_t)); - memset(&pack_options, 0, sizeof (pack_opt_t)); + HDmemset(&diff_options, 0, sizeof (diff_opt_t)); + HDmemset(&pack_options, 0, sizeof (pack_opt_t)); /* run tests */ puts("Testing h5repack:"); @@ -986,7 +993,7 @@ int main (void) #if defined (H5_HAVE_FILTER_SZIP) if (szip_can_encode) { - /* fs_type = H5F_FILE_SPACE_AGGR_VFD; fs_size = 3 */ + /* fs_type = H5F_FILE_SPACE_AGGR_VFD; fs_size = 3 */ if (h5repack_init (&pack_options, 0, 0, H5_INC_ENUM(H5F_file_space_type_t, fs_type), ++fs_size) < 0) GOERROR; if (h5repack(FNAME7,FNAME7OUT,&pack_options) < 0) @@ -1303,7 +1310,7 @@ int main (void) && defined (H5_HAVE_FILTER_FLETCHER32) && defined (H5_HAVE_FILTER_SHUFFLE) if (szip_can_encode) { - /* fs_type = H5F_FILE_SPACE_VFD; fs_size = 4 */ + /* fs_type = H5F_FILE_SPACE_VFD; fs_size = 4 */ if (h5repack_init (&pack_options, 0, 0, H5_INC_ENUM(H5F_file_space_type_t, fs_type), ++fs_size) < 0) GOERROR; if (h5repack_addfilter("dset_deflate:SZIP=8,NN",&pack_options) < 0) @@ -3004,31 +3011,31 @@ out: * * Purpose: create datasets with contiguous and chunked layouts: * - * contig_small: < 1k, fixed dims datspace - * chunked_small_fixed: < 1k, fixed dims dataspace + * contig_small: < 1k, fixed dims datspace + * chunked_small_fixed: < 1k, fixed dims dataspace * *------------------------------------------------------------------------- */ -#define S_DIM1 4 -#define S_DIM2 10 -#define CONTIG_S "contig_small" -#define CHUNKED_S_FIX "chunked_small_fixed" +#define S_DIM1 4 +#define S_DIM2 10 +#define CONTIG_S "contig_small" +#define CHUNKED_S_FIX "chunked_small_fixed" static int make_layout2(hid_t loc_id) { - hid_t contig_dcpl = -1; /* dataset creation property list */ + hid_t contig_dcpl = -1; /* dataset creation property list */ hid_t chunked_dcpl = -1; /* dataset creation property list */ - int i, j, n; /* Local index variables */ - int ret_value = -1; /* Return value */ - hid_t s_sid = -1; /* dataspace ID */ + int i, j, n; /* Local index variables */ + int ret_value = -1; /* Return value */ + hid_t s_sid = -1; /* dataspace ID */ - hsize_t s_dims[RANK] = {S_DIM1,S_DIM2}; /* Dataspace (< 1 k) */ - hsize_t chunk_dims[RANK] = {S_DIM1/2, S_DIM2/2}; /* Dimension sizes for chunks */ + hsize_t s_dims[RANK] = {S_DIM1,S_DIM2}; /* Dataspace (< 1 k) */ + hsize_t chunk_dims[RANK] = {S_DIM1/2, S_DIM2/2}; /* Dimension sizes for chunks */ - int s_buf[S_DIM1][S_DIM2]; /* Temporary buffer */ + int s_buf[S_DIM1][S_DIM2]; /* Temporary buffer */ for(i = n = 0; i < S_DIM1; i++) { for (j = 0; j < S_DIM2; j++) { @@ -3178,7 +3185,7 @@ int make_big(hid_t loc_id) if (H5Dwrite (did,H5T_NATIVE_SCHAR,m_sid,f_sid,H5P_DEFAULT,buf) < 0) goto out; - free(buf); + HDfree(buf); buf=NULL; /* close */ @@ -3219,8 +3226,8 @@ int make_external(hid_t loc_id) hid_t sid=-1; hid_t dcpl; int buf[2]={1,2}; - hsize_t cur_size[1]; /* data space current size */ - hsize_t max_size[1]; /* data space maximum size */ + hsize_t cur_size[1]; /* data space current size */ + hsize_t max_size[1]; /* data space maximum size */ hsize_t size; cur_size[0] = max_size[0] = 2; @@ -3303,7 +3310,7 @@ make_userblock(void) /* Write userblock data */ nwritten = HDwrite(fd, ub, (size_t)USERBLOCK_SIZE); - assert(nwritten == USERBLOCK_SIZE); + HDassert(nwritten == USERBLOCK_SIZE); /* Close file */ HDclose(fd); @@ -3367,7 +3374,7 @@ verify_userblock( const char* filename) /* Read userblock data */ nread = HDread(fd, ub, (size_t)USERBLOCK_SIZE); - assert(nread == USERBLOCK_SIZE); + HDassert(nread == USERBLOCK_SIZE); /* Verify userblock data */ for(u = 0; u < USERBLOCK_SIZE; u++) @@ -3416,7 +3423,7 @@ make_userblock_file(void) /* write userblock data */ nwritten = HDwrite(fd, ub, (size_t)USERBLOCK_SIZE); - assert(nwritten == USERBLOCK_SIZE); + HDassert(nwritten == USERBLOCK_SIZE); /* close file */ HDclose(fd); @@ -3472,7 +3479,7 @@ int write_dset_in(hid_t loc_id, /* create 1D attributes with dimension [2], 2 elements */ hsize_t dims[1]={2}; hsize_t dims1r[1]={2}; - char buf1[2][2]= {"ab","de"}; /* string */ + char buf1[2][3]= {"ab","de"}; /* string */ char buf2[2]= {1,2}; /* bitfield, opaque */ s_t buf3[2]= {{1,2},{3,4}}; /* compound */ hobj_ref_t buf4[2]; /* reference */ @@ -3486,7 +3493,7 @@ int write_dset_in(hid_t loc_id, /* create 2D attributes with dimension [3][2], 6 elements */ hsize_t dims2[2]={3,2}; hsize_t dims2r[2]={1,1}; - char buf12[6][2]= {"ab","cd","ef","gh","ij","kl"}; /* string */ + char buf12[6][3]= {"ab","cd","ef","gh","ij","kl"}; /* string */ char buf22[3][2]= {{1,2},{3,4},{5,6}}; /* bitfield, opaque */ s_t buf32[6]= {{1,2},{3,4},{5,6},{7,8},{9,10},{11,12}}; /* compound */ hobj_ref_t buf42[1][1]; /* reference */ @@ -3498,7 +3505,7 @@ int write_dset_in(hid_t loc_id, /* create 3D attributes with dimension [4][3][2], 24 elements */ hsize_t dims3[3]={4,3,2}; hsize_t dims3r[3]={1,1,1}; - char buf13[24][2]= {"ab","cd","ef","gh","ij","kl","mn","pq", + char buf13[24][3]= {"ab","cd","ef","gh","ij","kl","mn","pq", "rs","tu","vw","xz","AB","CD","EF","GH", "IJ","KL","MN","PQ","RS","TU","VW","XZ"}; /* string */ char buf23[4][3][2]; /* bitfield, opaque */ @@ -3741,7 +3748,7 @@ int write_dset_in(hid_t loc_id, H5Dclose(did); H5Tclose(tid); H5Sclose(sid); - free( dbuf ); + HDfree( dbuf ); } /*------------------------------------------------------------------------- @@ -3776,7 +3783,7 @@ int write_dset_in(hid_t loc_id, if (make_diffs) { - memset(buf12, 'z', sizeof buf12); + HDmemset(buf12, 'z', sizeof buf12); } @@ -3797,7 +3804,7 @@ int write_dset_in(hid_t loc_id, if (make_diffs) { - memset(buf22,0,sizeof buf22); + HDmemset(buf22,0,sizeof buf22); } if ((tid = H5Tcopy(H5T_STD_B8LE)) < 0) @@ -3827,7 +3834,7 @@ int write_dset_in(hid_t loc_id, if (make_diffs) { - memset(buf32,0,sizeof buf32); + HDmemset(buf32,0,sizeof buf32); } if ((tid = H5Tcreate (H5T_COMPOUND, sizeof(s_t))) < 0) @@ -3919,7 +3926,7 @@ int write_dset_in(hid_t loc_id, if (make_diffs) { - memset(buf62,0,sizeof buf62); + HDmemset(buf62,0,sizeof buf62); } @@ -3937,8 +3944,8 @@ int write_dset_in(hid_t loc_id, if(make_diffs) { - memset(buf72, 0, sizeof buf72); - memset(buf82, 0, sizeof buf82); + HDmemset(buf72, 0, sizeof buf72); + HDmemset(buf82, 0, sizeof buf82); } @@ -3978,7 +3985,7 @@ int write_dset_in(hid_t loc_id, if (make_diffs) { - memset(buf13,'z',sizeof buf13); + HDmemset(buf13,'z',sizeof buf13); } if ((tid = H5Tcopy(H5T_C_S1)) < 0) @@ -4291,9 +4298,9 @@ int make_dset_reg_ref(hid_t loc_id) out: if(wbuf) - free(wbuf); + HDfree(wbuf); if(dwbuf) - free(dwbuf); + HDfree(dwbuf); H5E_BEGIN_TRY { @@ -4345,7 +4352,7 @@ int write_attr_in(hid_t loc_id, /* create 1D attributes with dimension [2], 2 elements */ hsize_t dims[1]={2}; - char buf1[2][2]= {"ab","de"}; /* string */ + char buf1[2][3]= {"ab","de"}; /* string */ char buf2[2]= {1,2}; /* bitfield, opaque */ s_t buf3[2]= {{1,2},{3,4}}; /* compound */ hobj_ref_t buf4[2]; /* reference */ @@ -4358,7 +4365,7 @@ int write_attr_in(hid_t loc_id, /* create 2D attributes with dimension [3][2], 6 elements */ hsize_t dims2[2]={3,2}; - char buf12[6][2]= {"ab","cd","ef","gh","ij","kl"}; /* string */ + char buf12[6][3]= {"ab","cd","ef","gh","ij","kl"}; /* string */ char buf22[3][2]= {{1,2},{3,4},{5,6}}; /* bitfield, opaque */ s_t buf32[6]= {{1,2},{3,4},{5,6},{7,8},{9,10},{11,12}}; /* compound */ hobj_ref_t buf42[3][2]; /* reference */ @@ -4370,7 +4377,7 @@ int write_attr_in(hid_t loc_id, /* create 3D attributes with dimension [4][3][2], 24 elements */ hsize_t dims3[3]={4,3,2}; - char buf13[24][2]= {"ab","cd","ef","gh","ij","kl","mn","pq", + char buf13[24][3]= {"ab","cd","ef","gh","ij","kl","mn","pq", "rs","tu","vw","xz","AB","CD","EF","GH", "IJ","KL","MN","PQ","RS","TU","VW","XZ"}; /* string */ char buf23[4][3][2]; /* bitfield, opaque */ @@ -4698,7 +4705,7 @@ int write_attr_in(hid_t loc_id, */ if (make_diffs) { - memset(buf12, 'z', sizeof buf12); + HDmemset(buf12, 'z', sizeof buf12); } /* @@ -4738,7 +4745,7 @@ int write_attr_in(hid_t loc_id, if (make_diffs) { - memset(buf22,0,sizeof buf22); + HDmemset(buf22,0,sizeof buf22); } /* @@ -4798,7 +4805,7 @@ int write_attr_in(hid_t loc_id, */ if (make_diffs) { - memset(buf32,0,sizeof buf32); + HDmemset(buf32,0,sizeof buf32); } /* @@ -4946,7 +4953,7 @@ int write_attr_in(hid_t loc_id, if (make_diffs) { - memset(buf62,0,sizeof buf62); + HDmemset(buf62,0,sizeof buf62); } /* buf62[6][3]= {{1,2,3},{4,5,6},{7,8,9},{10,11,12},{13,14,15},{16,17,18}}; @@ -5032,7 +5039,7 @@ int write_attr_in(hid_t loc_id, if (make_diffs) { - memset(buf13,'z',sizeof buf13); + HDmemset(buf13,'z',sizeof buf13); } /* @@ -6705,49 +6712,49 @@ static herr_t make_complex_attr_references(hid_t loc_id) out: /* release resources */ - if (objgid < 0) + if (objgid > 0) H5Gclose(objgid); - if (objsid < 0) + if (objsid > 0) H5Sclose(objsid); - if (objdid < 0) + if (objdid > 0) H5Dclose(objdid); - if (objtid < 0) + if (objtid > 0) H5Tclose(objtid); - if (main_gid < 0) + if (main_gid > 0) H5Gclose(main_gid); - if (main_sid < 0) + if (main_sid > 0) H5Sclose(main_sid); - if (main_did < 0) + if (main_did > 0) H5Dclose(main_did); /* comp obj ref */ - if (comp_objref_tid < 0) + if (comp_objref_tid > 0) H5Tclose(comp_objref_tid); - if (comp_objref_aid < 0) + if (comp_objref_aid > 0) H5Aclose(comp_objref_aid); - if (comp_objref_attr_sid < 0) + if (comp_objref_attr_sid > 0) H5Sclose(comp_objref_attr_sid); /* comp region ref */ - if (comp_regref_tid < 0) + if (comp_regref_tid > 0) H5Tclose(comp_regref_tid); - if (comp_regref_aid < 0) + if (comp_regref_aid > 0) H5Aclose(comp_regref_aid); - if (comp_regref_attr_sid < 0) + if (comp_regref_attr_sid > 0) H5Sclose(comp_regref_attr_sid); /* vlen obj ref */ - if (vlen_objref_attr_id < 0); - H5Aclose (vlen_objref_attr_id); - if (vlen_objref_attr_sid < 0); - H5Sclose (vlen_objref_attr_sid); - if (vlen_objref_attr_tid < 0); - H5Tclose (vlen_objref_attr_tid); + if (vlen_objref_attr_id > 0) + H5Aclose(vlen_objref_attr_id); + if (vlen_objref_attr_sid > 0) + H5Sclose(vlen_objref_attr_sid); + if (vlen_objref_attr_tid > 0) + H5Tclose(vlen_objref_attr_tid); /* vlen region ref */ - if (vlen_regref_attr_id < 0); - H5Aclose (vlen_regref_attr_id); - if (vlen_regref_attr_sid < 0); - H5Sclose (vlen_regref_attr_sid); - if (vlen_regref_attr_tid < 0); - H5Tclose (vlen_regref_attr_tid); + if (vlen_regref_attr_id > 0) + H5Aclose(vlen_regref_attr_id); + if (vlen_regref_attr_sid > 0) + H5Sclose(vlen_regref_attr_sid); + if (vlen_regref_attr_tid > 0) + H5Tclose(vlen_regref_attr_tid); return ret; } diff --git a/tools/h5repack/testfiles/h5repack_layout.h5 b/tools/h5repack/testfiles/h5repack_layout.h5 Binary files differindex 5772a31..2cc44b1 100644 --- a/tools/h5repack/testfiles/h5repack_layout.h5 +++ b/tools/h5repack/testfiles/h5repack_layout.h5 diff --git a/tools/h5repack/testh5repack_detect_szip.c b/tools/h5repack/testh5repack_detect_szip.c index 332fcc5..e91b2f7 100644 --- a/tools/h5repack/testh5repack_detect_szip.c +++ b/tools/h5repack/testh5repack_detect_szip.c @@ -16,6 +16,7 @@ #include <stdio.h> #include "h5repack.h" #include "h5tools.h" +#include "h5tools_utils.h" #include "h5test.h" @@ -47,6 +48,9 @@ int main(void) h5tools_setprogname(PROGRAMNAME); h5tools_setstatus(EXIT_SUCCESS); + /* Initialize h5tools lib */ + h5tools_init(); + #ifdef H5_HAVE_FILTER_SZIP if (h5tools_can_encode(H5Z_FILTER_SZIP) == 1) { printf("yes\n"); |