From 7fe0324fc43bbe94e48e2cb412bd02870036e0fc Mon Sep 17 00:00:00 2001 From: Allen Byrne Date: Mon, 11 Apr 2016 08:45:33 -0500 Subject: [svn-r29680] HDFFV-1956: h5diff will warn if a dataset is virtual instead of failing. h5diff does not consider the layout type when comparing datasets. SKIP h5repack VDS tests until h5repack is fixed. --- tools/h5diff/testfiles/h5diff_v1.txt | 2 +- tools/h5diff/testfiles/h5diff_v3.txt | 2 +- tools/h5repack/CMakeTests.cmake | 67 ++++++++++++++--- tools/h5repack/h5repack.sh.in | 135 ++++++++++++++++++++++++++++------- tools/lib/h5diff_dset.c | 41 ++++++++--- 5 files changed, 203 insertions(+), 44 deletions(-) diff --git a/tools/h5diff/testfiles/h5diff_v1.txt b/tools/h5diff/testfiles/h5diff_v1.txt index 31a3eae..8210cf9 100644 --- a/tools/h5diff/testfiles/h5diff_v1.txt +++ b/tools/h5diff/testfiles/h5diff_v1.txt @@ -7,7 +7,7 @@ file1 file2 group : and 0 differences found dataset: and -Not comparable: or is an empty dataset +Warning: or is a virtual dataset Not comparable: has rank 3, dimensions [5x18x8], max dimensions [18446744073709551615x18x8] and has rank 3, dimensions [6x8x14], max dimensions [18446744073709551615x8x14] 0 differences found diff --git a/tools/h5diff/testfiles/h5diff_v3.txt b/tools/h5diff/testfiles/h5diff_v3.txt index 8c5d2a2..57af303 100644 --- a/tools/h5diff/testfiles/h5diff_v3.txt +++ b/tools/h5diff/testfiles/h5diff_v3.txt @@ -1,4 +1,4 @@ -Not comparable: or is an empty dataset +Warning: or is a virtual dataset Not comparable: has rank 3, dimensions [5x18x8], max dimensions [18446744073709551615x18x8] and has rank 3, dimensions [6x8x14], max dimensions [18446744073709551615x8x14] EXIT CODE: 0 diff --git a/tools/h5repack/CMakeTests.cmake b/tools/h5repack/CMakeTests.cmake index 07c3f9c..158e5d2 100644 --- a/tools/h5repack/CMakeTests.cmake +++ b/tools/h5repack/CMakeTests.cmake @@ -115,6 +115,16 @@ ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/plugin_test.h5repack_layout.h5.tst ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_layout.UD.h5-plugin_none.ddl ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/plugin_none.h5repack_layout.UD.h5.tst + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/1_vds.h5-vds_dset_compa-v.ddl + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/1_vds.h5-vds_dset_conti-v.ddl + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/2_vds.h5-vds_null_compa-v.ddl + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/2_vds.h5-vds_null_conti-v.ddl + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/4_vds.h5-vds_compa_compa-v.ddl + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/4_vds.h5-vds_compa_conti-v.ddl + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/4_vds.h5-vds_compa-v.ddl + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/4_vds.h5-vds_conti_compa-v.ddl + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/4_vds.h5-vds_conti_conti-v.ddl + ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/4_vds.h5-vds_conti-v.ddl ) foreach (h5_file ${LIST_HDF5_TEST_FILES} ${LIST_OTHER_TEST_FILES}) @@ -345,6 +355,41 @@ endif ("${testtype}" STREQUAL "SKIP") ENDMACRO (ADD_H5_VERIFY_TEST) + MACRO (ADD_H5_VERIFY_VDS testname testtype resultcode testfile testdset testfilter) + if ("${testtype}" STREQUAL "SKIP") + if (NOT HDF5_ENABLE_USING_MEMCHECKER) + add_test ( + NAME H5REPACK_VERIFY_LAYOUT-${testname}-SKIPPED + COMMAND ${CMAKE_COMMAND} -E echo "SKIP -d ${testdset} -pH ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile}" + ) + endif (NOT HDF5_ENABLE_USING_MEMCHECKER) + else ("${testtype}" STREQUAL "SKIP") + if (NOT HDF5_ENABLE_USING_MEMCHECKER) + add_test ( + NAME H5REPACK_VERIFY_LAYOUT-${testname} + COMMAND $ ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${testfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${testfile} + ) + set_tests_properties (H5REPACK_VERIFY_LAYOUT-${testname} PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + if (NOT "${last_test}" STREQUAL "") + set_tests_properties (H5REPACK_VERIFY_LAYOUT-${testname} PROPERTIES DEPENDS ${last_test}) + endif (NOT "${last_test}" STREQUAL "") + add_test ( + NAME H5REPACK_VERIFY_LAYOUT-${testname}_DMP + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=$" + -D "TEST_ARGS:STRING=-d;${testdset};-p;out-${testname}.${testfile}" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles" + -D "TEST_OUTPUT=${testfile}-${testname}-v.out" + -D "TEST_EXPECT=${resultcode}" + -D "TEST_REFERENCE=${testfile}-${testname}-v.ddl" + -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" + ) + set_tests_properties (H5REPACK_VERIFY_LAYOUT-${testname}_DMP PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + set_tests_properties (H5REPACK_VERIFY_LAYOUT-${testname}_DMP PROPERTIES DEPENDS H5REPACK_VERIFY_LAYOUT-${testname}) + endif (NOT HDF5_ENABLE_USING_MEMCHECKER) + endif ("${testtype}" STREQUAL "SKIP") + ENDMACRO (ADD_H5_VERIFY_VDS) + MACRO (ADD_H5_TEST_META testname testfile) add_test ( NAME H5REPACK_META-${testname}_N @@ -1053,20 +1098,22 @@ ######################################################### # layout options ######################################################### - ADD_H5_VERIFY_TEST (vds_dset_conti "TEST" 0 ${FILEV1} vds_dset CONTIGUOUS -l vds_dset:CONTI) - ADD_H5_VERIFY_TEST (vds_null_conti "TEST" 1 ${FILEV2} null CONTIGUOUS -l CONTI) - ADD_H5_VERIFY_TEST (vds_dset_compa "TEST" 0 ${FILEV1} vds_dset COMPACT -l vds_dset:COMPA) - ADD_H5_VERIFY_TEST (vds_null_compa "TEST" 1 ${FILEV2} null COMPACT -l COMPA) +# skip tests because of HDFFV-9756 + ADD_H5_VERIFY_VDS (vds_dset_conti "SKIP" 0 ${FILEV1} vds_dset CONTIGUOUS -l vds_dset:CONTI) + ADD_H5_VERIFY_VDS (vds_null_conti "SKIP" 1 ${FILEV2} null CONTIGUOUS -l CONTI) + ADD_H5_VERIFY_VDS (vds_dset_compa "SKIP" 0 ${FILEV1} vds_dset COMPACT -l vds_dset:COMPA) + ADD_H5_VERIFY_VDS (vds_null_compa "SKIP" 1 ${FILEV2} null COMPACT -l COMPA) ################################################################ # layout conversions ############################################################### - ADD_H5_VERIFY_TEST (vds_compa_conti "TEST" 0 ${FILEV4} vds_dset CONTIGUOUS -l vds_dset:CONTI) - ADD_H5_VERIFY_TEST (vds_compa_compa "TEST" 0 ${FILEV4} vds_dset COMPACT -l vds_dset:COMPA) - ADD_H5_VERIFY_TEST (vds_conti_compa "TEST" 0 ${FILEV4} vds_dset COMPACT -l vds_dset:COMPA) - ADD_H5_VERIFY_TEST (vds_conti_conti "TEST" 0 ${FILEV4} vds_dset CONTIGUOUS -l vds_dset:CONTI) - ADD_H5_VERIFY_TEST (vds_compa "TEST" 0 ${FILEV4} vds_dset COMPACT -l vds_dset:COMPA) - ADD_H5_VERIFY_TEST (vds_conti "TEST" 0 ${FILEV4} vds_dset CONTIGUOUS -l vds_dset:CONTI) +# skip tests because of HDFFV-9756 + ADD_H5_VERIFY_VDS (vds_compa_conti "SKIP" 0 ${FILEV4} vds_dset CONTIGUOUS -l vds_dset:CONTI) + ADD_H5_VERIFY_VDS (vds_compa_compa "SKIP" 0 ${FILEV4} vds_dset COMPACT -l vds_dset:COMPA) + ADD_H5_VERIFY_VDS (vds_conti_compa "SKIP" 0 ${FILEV4} vds_dset COMPACT -l vds_dset:COMPA) + ADD_H5_VERIFY_VDS (vds_conti_conti "SKIP" 0 ${FILEV4} vds_dset CONTIGUOUS -l vds_dset:CONTI) + ADD_H5_VERIFY_VDS (vds_compa "SKIP" 0 ${FILEV4} vds_dset COMPACT -l vds_dset:COMPA) + ADD_H5_VERIFY_VDS (vds_conti "SKIP" 0 ${FILEV4} vds_dset CONTIGUOUS -l vds_dset:CONTI) ############################################################################## ### P L U G I N T E S T S diff --git a/tools/h5repack/h5repack.sh.in b/tools/h5repack/h5repack.sh.in index 24298d0..d3faee7 100644 --- a/tools/h5repack/h5repack.sh.in +++ b/tools/h5repack/h5repack.sh.in @@ -39,6 +39,8 @@ H5DUMP=../h5dump/h5dump # The h5dump tool name H5DUMP_BIN=`pwd`/$H5DUMP # The path of the h5dump tool binary RM='rm -rf' +CMP='cmp' +DIFF='diff -c' GREP='grep' CP='cp' DIRNAME='dirname' @@ -148,6 +150,16 @@ $SRC_H5REPACK_TESTFILES/h5repack_layout.h5.ddl $SRC_H5REPACK_TESTFILES/h5repack_filters.h5.tst $SRC_H5REPACK_TESTFILES/h5repack_layout.h5-plugin_test.ddl $SRC_H5REPACK_TESTFILES/plugin_test.h5repack_layout.h5.tst +$SRC_H5REPACK_TESTFILES/1_vds.h5-vds_dset_compa-v.ddl +$SRC_H5REPACK_TESTFILES/1_vds.h5-vds_dset_conti-v.ddl +$SRC_H5REPACK_TESTFILES/2_vds.h5-vds_null_compa-v.ddl +$SRC_H5REPACK_TESTFILES/2_vds.h5-vds_null_conti-v.ddl +$SRC_H5REPACK_TESTFILES/4_vds.h5-vds_compa_compa-v.ddl +$SRC_H5REPACK_TESTFILES/4_vds.h5-vds_compa_conti-v.ddl +$SRC_H5REPACK_TESTFILES/4_vds.h5-vds_compa-v.ddl +$SRC_H5REPACK_TESTFILES/4_vds.h5-vds_conti_compa-v.ddl +$SRC_H5REPACK_TESTFILES/4_vds.h5-vds_conti_conti-v.ddl +$SRC_H5REPACK_TESTFILES/4_vds.h5-vds_conti-v.ddl " # @@ -171,7 +183,7 @@ COPY_TESTFILES_TO_TESTDIR() INODE_SDIR=`$LS -i -d $SDIR | $AWK -F' ' '{print $1}'` INODE_DDIR=`$LS -i -d $TESTDIR | $AWK -F' ' '{print $1}'` if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then - $CP -f $tstfile $TESTDIR + $CP -f $tstfile $TESTDIR if [ $? -ne 0 ]; then echo "Error: FAILED to copy $tstfile ." @@ -188,7 +200,7 @@ CLEAN_TESTFILES_AND_TESTDIR() # skip rm if srcdir is same as destdir # this occurs when build/test performed in source dir and # make cp fail - SDIR=`$DIRNAME $tstfile` + SDIR=`$DIRNAME $SRC_H5REPACK_TESTFILES/h5repack-help.txt` INODE_SDIR=`$LS -i -d $SDIR | $AWK -F' ' '{print $1}'` INODE_DDIR=`$LS -i -d $TESTDIR | $AWK -F' ' '{print $1}'` if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then @@ -321,6 +333,70 @@ VERIFY_LAYOUT_DSET() rm -f $layoutfile } +#------------------------------------------ +# Verifying layouts of a dataset +VERIFY_LAYOUT_VDS() +{ + layoutfile=layout-$1.$2 + dset=$3 + expectlayout=$4 + infile=$2 + outfile=out-$1.$2 + + expect="$TESTDIR/$2-$1-v.ddl" + actual="$TESTDIR/$2-$1-v.out" + actual_err="$TESTDIR/$2-$1-v.err" + + shift + shift + shift + shift + + TESTING $H5REPACK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile + ) + RET=$? + if [ $RET != 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + fi + + #--------------------------------- + # check the layout from a dataset + VERIFY "a dataset layout" + ( + cd $TESTDIR + $RUNSERIAL $H5DUMP_BIN -d $dset -p $outfile + ) >$actual 2>$actual_err + + cat $actual_err >> $actual + + if [ ! -f $expect ]; then + # Create the expect file if it doesn't yet exist. + echo " CREATED" + cp $actual $expect + elif $CMP $expect $actual > /dev/null 2>&1 ; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected result (*.ddl) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $caseless $expect $actual |sed 's/^/ /' + fi + + # Clean up output file + if test -z "$HDF5_NOCLEANUP"; then + rm -f $actual $actual_err + fi + + # clean up tmp files + rm -f $outfile +} + #---------------------------------------- # Verifying layouts from entire file VERIFY_LAYOUT_ALL() @@ -469,11 +545,11 @@ TOOLTEST1() ) RET=$? if [ $RET != 0 ] ; then - echo "*FAILED*" - nerrors="`expr $nerrors + 1`" + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" else - echo " PASSED" - DIFFTEST $infile $outfile + echo " PASSED" + DIFFTEST $infile $outfile fi rm -f $outfile } @@ -500,11 +576,11 @@ TOOLTESTV() ) >$actual 2>$actual_err RET=$? if [ $RET != 0 ] ; then - echo "*FAILED*" - nerrors="`expr $nerrors + 1`" + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" else - echo " PASSED" - DIFFTEST $infile $outfile + echo " PASSED" + DIFFTEST $infile $outfile fi # display output compare @@ -634,11 +710,11 @@ TOOLTEST_META() # verify sizes. MESSAGE "Verify the sizes of both output files ($size1 vs $size2)" if [ $size1 -lt $size2 ]; then - # pass - echo " PASSED" + # pass + echo " PASSED" else - #fail - echo "*FAILED*" + #fail + echo "*FAILED*" nerrors="`expr $nerrors + 1`" fi @@ -711,7 +787,7 @@ USE_FILTER_SZIP_ENCODER=`$RUNSERIAL $H5DETECTSZIP_BIN` fi ############################################################################## -### T H E T E S T S +### T H E T E S T S ############################################################################## # prepare for test COPY_TESTFILES_TO_TESTDIR @@ -1057,7 +1133,7 @@ TOOLTEST add_alignment $arg # Check repacking file with old version of layout message (should get upgraded # to new version and be readable, etc.) -TOOLTEST pgrade_layout h5repack_layouto.h5 +TOOLTEST upgrade_layout h5repack_layouto.h5 # test for datum size > H5TOOLS_MALLOCSIZE TOOLTEST gt_mallocsize h5repack_objs.h5 -f GZIP=1 @@ -1091,20 +1167,31 @@ TOOLTEST_META meta_long h5repack_layout.h5 --metadata_block_size=8192 ######################################################### # layout options ######################################################### -VERIFY_LAYOUT_DSET vds_dset_conti 1_vds.h5 vds_dset CONTIGUOUS -l vds_dset:CONTI +# skip test because of HDFFV-9756 +#VERIFY_LAYOUT_VDS vds_dset_conti 1_vds.h5 vds_dset CONTIGUOUS -l vds_dset:CONTI +SKIP vds_dset_conti 1_vds.h5 vds_dset CONTIGUOUS -l vds_dset:CONTI VERIFY_LAYOUT_ALL vds_null_conti 2_vds.h5 CONTIGUOUS -l CONTI -VERIFY_LAYOUT_DSET vds_dset_compa 1_vds.h5 vds_dset COMPACT -l vds_dset:COMPA +# skip test because of HDFFV-9756 +#VERIFY_LAYOUT_VDS vds_dset_compa 1_vds.h5 vds_dset COMPACT -l vds_dset:COMPA +SKIP vds_dset_compa 1_vds.h5 vds_dset COMPACT -l vds_dset:COMPA VERIFY_LAYOUT_ALL vds_null_compa 2_vds.h5 COMPACT -l COMPA ################################################################ # layout conversions ############################################################### -VERIFY_LAYOUT_DSET vds_compa_conti 4_vds.h5 vds_dset CONTIGUOUS -l vds_dset:CONTI -VERIFY_LAYOUT_DSET vds_compa_compa 4_vds.h5 vds_dset COMPACT -l vds_dset:COMPA -VERIFY_LAYOUT_DSET vds_conti_compa 4_vds.h5 vds_dset COMPACT -l vds_dset:COMPA -VERIFY_LAYOUT_DSET vds_conti_conti 4_vds.h5 vds_dset CONTIGUOUS -l vds_dset:CONTI -VERIFY_LAYOUT_DSET vds_compa 4_vds.h5 vds_dset COMPACT -l vds_dset:COMPA -VERIFY_LAYOUT_DSET vds_conti 4_vds.h5 vds_dset CONTIGUOUS -l vds_dset:CONTI +# skip tests because of HDFFV-9756 +#VERIFY_LAYOUT_VDS vds_compa_conti 4_vds.h5 vds_dset CONTIGUOUS -l vds_dset:CONTI +#VERIFY_LAYOUT_VDS vds_compa_compa 4_vds.h5 vds_dset COMPACT -l vds_dset:COMPA +#VERIFY_LAYOUT_VDS vds_conti_compa 4_vds.h5 vds_dset COMPACT -l vds_dset:COMPA +#VERIFY_LAYOUT_VDS vds_conti_conti 4_vds.h5 vds_dset CONTIGUOUS -l vds_dset:CONTI +#VERIFY_LAYOUT_VDS vds_compa 4_vds.h5 vds_dset COMPACT -l vds_dset:COMPA +#VERIFY_LAYOUT_VDS vds_conti 4_vds.h5 vds_dset CONTIGUOUS -l vds_dset:CONTI +SKIP vds_compa_conti 4_vds.h5 vds_dset CONTIGUOUS -l vds_dset:CONTI +SKIP vds_compa_compa 4_vds.h5 vds_dset COMPACT -l vds_dset:COMPA +SKIP vds_conti_compa 4_vds.h5 vds_dset COMPACT -l vds_dset:COMPA +SKIP vds_conti_conti 4_vds.h5 vds_dset CONTIGUOUS -l vds_dset:CONTI +SKIP vds_compa 4_vds.h5 vds_dset COMPACT -l vds_dset:COMPA +SKIP vds_conti 4_vds.h5 vds_dset CONTIGUOUS -l vds_dset:CONTI # Clean up temporary files/directories CLEAN_TESTFILES_AND_TESTDIR diff --git a/tools/lib/h5diff_dset.c b/tools/lib/h5diff_dset.c index b8dd0e8..63f1483 100644 --- a/tools/lib/h5diff_dset.c +++ b/tools/lib/h5diff_dset.c @@ -71,9 +71,7 @@ hsize_t diff_dataset( hid_t file1_id, if((dcpl1 = H5Dget_create_plist(did1)) < 0) goto error; if((dcpl2 = H5Dget_create_plist(did2)) < 0) - { goto error; - } /*------------------------------------------------------------------------- * check if the dataset creation property list has filters that @@ -191,6 +189,10 @@ hsize_t diff_datasetid( hid_t did1, hid_t f_tid2=-1; hid_t m_tid1=-1; hid_t m_tid2=-1; + hid_t dcpl1 = -1; + hid_t dcpl2 = -1; + H5D_layout_t stl1 = -1; + H5D_layout_t stl2 = -1; size_t m_size1; size_t m_size2; H5T_sign_t sign1; @@ -260,6 +262,21 @@ hsize_t diff_datasetid( hid_t did1, goto error; } + + /*------------------------------------------------------------------------- + * get the storage layout type + *------------------------------------------------------------------------- + */ + if((dcpl1 = H5Dget_create_plist(did1)) < 0) + goto error; + if((dcpl2 = H5Dget_create_plist(did2)) < 0) + goto error; + + if((stl1 = H5Pget_layout(dcpl1)) < 0) + goto error; + if((stl2 = H5Pget_layout(dcpl2)) < 0) + goto error; + /*------------------------------------------------------------------------- * check for empty datasets *------------------------------------------------------------------------- @@ -271,10 +288,18 @@ hsize_t diff_datasetid( hid_t did1, if (storage_size1==0 || storage_size2==0) { - if ( (options->m_verbose||options->m_list_not_cmp) && obj1_name && obj2_name) - parallel_print("Not comparable: <%s> or <%s> is an empty dataset\n", obj1_name, obj2_name); - can_compare=0; - options->not_cmp=1; + if (stl1==H5D_VIRTUAL || stl2==H5D_VIRTUAL) + { + if ( (options->m_verbose||options->m_list_not_cmp) && obj1_name && obj2_name) + parallel_print("Warning: <%s> or <%s> is a virtual dataset\n", obj1_name, obj2_name); + } + else + { + if ( (options->m_verbose||options->m_list_not_cmp) && obj1_name && obj2_name) + parallel_print("Not comparable: <%s> or <%s> is an empty dataset\n", obj1_name, obj2_name); + can_compare=0; + options->not_cmp=1; + } } /*------------------------------------------------------------------------- @@ -329,7 +354,7 @@ hsize_t diff_datasetid( hid_t did1, parallel_print("Not comparable: <%s> has sign %s ", obj1_name, get_sign(sign1)); parallel_print("and <%s> has sign %s\n", obj2_name, get_sign(sign2)); } - + can_compare=0; options->not_cmp=1; } @@ -369,7 +394,7 @@ hsize_t diff_datasetid( hid_t did1, h5difftrace("upgrade the smaller memory size?\n"); if (FAIL == match_up_memsize (f_tid1, f_tid2, - &m_tid1, &m_tid2, + &m_tid1, &m_tid2, &m_size1, &m_size2)) goto error; -- cgit v0.12