#! /bin/sh # # Copyright by The HDF Group. # Copyright by the Board of Trustees of the University of Illinois. # All rights reserved. # # This file is part of HDF5. The full HDF5 copyright notice, including # terms governing use, modification, and redistribution, is contained in # the files COPYING and Copyright.html. COPYING can be found at the root # of the source code distribution tree; Copyright.html can be found at the # root level of an installed copy of the electronic HDF5 document set and # is linked from the top-level documents page. It can also be found at # http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have # access to either file, you may request a copy from help@hdfgroup.org. # # Tests for the h5repack tool # # Modification: # Pedro Vicente Nunes, 11/15/2006 # Added $FILEN variables for file names # USE_FILTER_SZIP="@USE_FILTER_SZIP@" USE_FILTER_DEFLATE="@USE_FILTER_DEFLATE@" USE_FILTER_SHUFFLE="@USE_FILTER_SHUFFLE@" USE_FILTER_FLETCHER32="@USE_FILTER_FLETCHER32@" USE_FILTER_NBIT="@USE_FILTER_NBIT@" USE_FILTER_SCALEOFFSET="@USE_FILTER_SCALEOFFSET@" TESTNAME=h5repack EXIT_SUCCESS=0 EXIT_FAILURE=1 H5REPACK=h5repack # The tool name H5REPACK_BIN=`pwd`/$H5REPACK # The path of the tool binary H5DIFF=../h5diff/h5diff # The h5diff tool name H5DIFF_BIN=`pwd`/$H5DIFF # The path of the h5diff tool binary H5DUMP=../h5dump/h5dump # The h5dump tool name H5DUMP_BIN=`pwd`/$H5DUMP # The path of the h5dump tool binary GREP='grep' H5DETECTSZIP=testh5repack_detect_szip H5DETECTSZIP_BIN=`pwd`/$H5DETECTSZIP INFO_FILE=testfiles/h5repack.info FILE0=h5repack_fill.h5 FILE1=h5repack_objs.h5 FILE2=h5repack_attr.h5 FILE3=h5repack_hlink.h5 FILE4=h5repack_layout.h5 FILE5=h5repack_early.h5 FILE7=h5repack_szip.h5 FILE8=h5repack_deflate.h5 FILE9=h5repack_shuffle.h5 FILE10=h5repack_fletcher.h5 FILE11=h5repack_filters.h5 FILE12=h5repack_nbit.h5 FILE13=h5repack_soffset.h5 FILE14=h5repack_layouto.h5 # A file with an older version of the layout message # (copy of test/tlayouto.h5) FILE15=h5repack_named_dtypes.h5 FILE16=tfamily%05d.h5 # located in common testfiles folder FILE18=h5repack_layout2.h5 FILE_REF=h5repack_refs.h5 FILE_ATTR_REF=h5repack_attr_refs.h5 nerrors=0 verbose=yes # The build (current) directory might be different than the source directory. # if test -z "$srcdir"; then srcdir=. fi # Print a line-line message left justified in a field of 70 characters # beginning with the word "Testing". # TESTING() { SPACES=" " echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' } # Print a line-line message left justified in a field of 70 characters # beginning with the word "Verifying". # VERIFY() { SPACES=" " echo "Verifying h5diff output $* $SPACES" | cut -c1-70 | tr -d '\012' } # Print a message that a test has been skipped (because a required filter # was unavailable) SKIP() { TESTING $H5REPACK $@ echo " -SKIP-" } # Call the h5diff tool # DIFFTEST() { VERIFY $@ $RUNSERIAL $H5DIFF_BIN -q "$@" RET=$? if [ $RET != 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" else echo " PASSED" fi } # Call h5repack # # call TOOLTEST_MAIN and delete $output file TOOLTEST() { echo $@ TOOLTEST_MAIN $@ path=`pwd` outfile=$path/out.$1 rm -f $outfile } # TOOLTEST main function, doesn't delete $output file TOOLTEST_MAIN() { # Run test. TESTING $H5REPACK $@ infile=$srcdir/testfiles/$1 path=`pwd` outfile=$path/out.$1 shift $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile RET=$? if [ $RET != 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" else echo " PASSED" DIFFTEST $infile $outfile fi } #------------------------------------------ # Verifying layouts of a dataset VERIFY_LAYOUT_DSET() { path=`pwd` outfile=$path/out.$1 layoutfile=$path/layout.$1 dset=$2 expectlayout=$3 #--------------------------------- # check the layout from a dataset VERIFY "Layout" $H5DUMP_BIN -d $dset -pH $outfile > $layoutfile $GREP $expectlayout $layoutfile > /dev/null if [ $? -eq 0 ]; then echo " PASSED" else echo " FAILED" fi # clean up tmp files rm -f $outfile rm -f $layoutfile } #---------------------------------------- # Verifying layouts from entire file VERIFY_LAYOUT_ALL() { path=`pwd` outfile=$path/out.$1 layoutfile=$path/layout.$1 expectlayout=$2 #--------------------------------- # check the layout from a dataset # check if the other layouts still exsit VERIFY "Layout " # if CONTIGUOUS if [ $expectlayout = "CONTIGUOUS" ]; then $H5DUMP_BIN -pH $outfile > $layoutfile $GREP "COMPACT" $layoutfile > /dev/null if [ $? -eq 0 ]; then echo " FAILED" else $GREP "CHUNKED" $layoutfile > /dev/null if [ $? -eq 0 ]; then echo " FAILED" else echo " PASSED" fi fi else # if COMPACT if [ $expectlayout = "COMPACT" ]; then $H5DUMP_BIN -pH $outfile > $layoutfile $GREP "CHUNKED" $layoutfile > /dev/null if [ $? -eq 0 ]; then echo " FAILED" else $GREP "CONTIGUOUS" $layoutfile > /dev/null if [ $? -eq 0 ]; then echo " FAILED" else echo " PASSED" fi fi else # if CHUNKED if [ $expectlayout = "CHUNKED" ]; then $H5DUMP_BIN -pH $outfile > $layoutfile $GREP "CONTIGUOUS" $layoutfile > /dev/null if [ $? -eq 0 ]; then echo " FAILED" else $GREP "COMPACT" $layoutfile > /dev/null if [ $? -eq 0 ]; then echo " FAILED" else echo " PASSED" fi fi fi fi fi # clean up tmp files rm -f $outfile rm -f $layoutfile } # same as TOOLTEST, but it uses the old syntax -i input_file -o output_file # TOOLTEST0() { # Run test. TESTING $H5REPACK $@ infile=$srcdir/testfiles/$1 path=`pwd` outfile=$path/out.$1 shift $RUNSERIAL $H5REPACK_BIN -i $infile -o $outfile "$@" RET=$? if [ $RET != 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" else echo " PASSED" DIFFTEST $infile $outfile fi rm -f $outfile } # same as TOOLTEST, but it uses the common testfiles at $srcdir/../testfiles/ # used to test the family driver, where these files reside # TOOLTEST1() { # Run test. TESTING $H5REPACK $@ infile=$srcdir/../testfiles/$1 path=`pwd` outfile=$path/out.$1 shift $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile RET=$? if [ $RET != 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" else echo " PASSED" DIFFTEST $infile $outfile fi rm -f $outfile } # Call h5repack and compare output to a text file for -v option # TOOLTESTV() { # Run test. TESTING $H5REPACK $@ expect="$srcdir/../testfiles/$1.ddl" actual="../testfiles/`basename $1 .ddl`.out" actual_err="../testfiles/`basename $1 .ddl`.err" infile=$srcdir/testfiles/$1 path=`pwd` outfile=$path/out.$1 shift $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile >$actual 2>$actual_err cp $actual $actual_sav STDOUT_FILTER $actual cat $actual_err >> $actual if cmp -s $expect $actual; then echo " PASSED" else echo "*FAILED*" echo " Expected result (*.ddl) differs from actual result (*.out)" nerrors="`expr $nerrors + 1`" test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /' fi rm -f $actual $actual_err $actual_sav } STDOUT_FILTER() { result_file=$1 tmp_file=/tmp/h5test_tmp_$$ # Filter name of files. cp $result_file $tmp_file sed -e '/^Opening file/d' -e '/^Making file/d' \ < $tmp_file > $result_file # cleanup rm -f $tmp_file } # # The tests # We use the files generated by h5repacktst # Each run generates ".out.h5" and the tool h5diff is used to # compare the input and output files # # the tests are the same as the program h5repacktst, but run from the CLI # # See which filters are usable (and skip tests for filters we # don't have). Do this by searching H5pubconf.h to see which # filters are defined. # detect whether the encoder is present. USE_FILTER_SZIP_ENCODER="no"; if test $USE_FILTER_SZIP = "yes"; then USE_FILTER_SZIP_ENCODER=`$RUNSERIAL $H5DETECTSZIP_BIN` fi # copy files (these files have no filters) TOOLTEST $FILE0 TOOLTEST $FILE1 TOOLTEST $FILE2 TOOLTEST $FILE3 TOOLTEST $FILE4 TOOLTEST $FILE5 # use $FILE4 to write some filters (this file has no filters) # gzip with individual object arg="$FILE4 -f dset1:GZIP=1 -l dset1:CHUNK=20x10" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else TOOLTEST $arg fi # gzip for all arg="$FILE4 -f GZIP=1" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else TOOLTEST $arg fi # szip with individual object arg="$FILE4 -f dset2:SZIP=8,EC -l dset2:CHUNK=20x10" if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then SKIP $arg else TOOLTEST $arg fi # szip for all arg="$FILE4 -f SZIP=8,NN" if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then SKIP $arg else TOOLTEST $arg fi # shuffle with individual object arg="$FILE4 -f dset2:SHUF -l dset2:CHUNK=20x10" if test $USE_FILTER_SHUFFLE != "yes" ; then SKIP $arg else TOOLTEST $arg fi # shuffle for all arg="$FILE4 -f SHUF" if test $USE_FILTER_SHUFFLE != "yes" ; then SKIP $arg else TOOLTEST $arg fi # fletcher32 with individual object arg="$FILE4 -f dset2:FLET -l dset2:CHUNK=20x10" if test $USE_FILTER_FLETCHER32 != "yes" ; then SKIP $arg else TOOLTEST $arg fi # fletcher32 for all arg="$FILE4 -f FLET" if test $USE_FILTER_FLETCHER32 != "yes" ; then SKIP $arg else TOOLTEST $arg fi # all filters arg="$FILE4 -f dset2:SHUF -f dset2:FLET -f dset2:SZIP=8,NN -f dset2:GZIP=1 -l dset2:CHUNK=20x10" if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SHUFFLE != "yes" -o $USE_FILTER_FLETCHER32 != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else TOOLTEST $arg fi # verbose gzip with individual object arg="$FILE11 -v -f /dset_deflate:GZIP=9" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else TOOLTEST $arg fi ########################################################### # the following tests assume the input files have filters ########################################################### # szip copy arg="$FILE7" if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then SKIP $arg else TOOLTEST $arg fi # szip remove arg="$FILE7 --filter=dset_szip:NONE" if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then SKIP $arg else TOOLTEST $arg fi # deflate copy arg="$FILE8" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else TOOLTEST $arg fi # deflate remove arg="$FILE8 -f dset_deflate:NONE" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else TOOLTEST $arg fi # shuffle copy arg="$FILE9" if test $USE_FILTER_SHUFFLE != "yes" ; then SKIP $arg else TOOLTEST $arg fi # shuffle remove arg="$FILE9 -f dset_shuffle:NONE" if test $USE_FILTER_SHUFFLE != "yes" ; then SKIP $arg else TOOLTEST $arg fi # fletcher32 copy arg="$FILE10" if test $USE_FILTER_FLETCHER32 != "yes" ; then SKIP $arg else TOOLTEST $arg fi # fletcher32 remove arg="$FILE10 -f dset_fletcher32:NONE" if test $USE_FILTER_FLETCHER32 != "yes" ; then SKIP $arg else TOOLTEST $arg fi # nbit copy arg="$FILE12" if test $USE_FILTER_NBIT != "yes" ; then SKIP $arg else TOOLTEST $arg fi # nbit remove arg="$FILE12 -f dset_nbit:NONE" if test $USE_FILTER_NBIT != "yes" ; then SKIP $arg else TOOLTEST $arg fi # nbit add arg="$FILE12 -f dset_int31:NBIT" if test $USE_FILTER_NBIT != "yes" ; then SKIP $arg else TOOLTEST $arg fi # scaleoffset copy arg="$FILE13" if test $USE_FILTER_SCALEOFFSET != "yes" ; then SKIP $arg else TOOLTEST $arg fi # scaleoffset add arg="$FILE13 -f dset_none:SOFF=31,IN" if test $USE_FILTER_SCALEOFFSET != "yes" ; then SKIP $arg else TOOLTEST $arg fi # scaleoffset remove arg="$FILE13 -f dset_scaleoffset:NONE" if test $USE_FILTER_SCALEOFFSET != "yes" ; then SKIP $arg else TOOLTEST $arg fi # remove all filters arg="$FILE11 -f NONE" if test $USE_FILTER_FLETCHER32 != "yes" -o $USE_FILTER_DEFLATE != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SHUFFLE != "yes" -o $USE_FILTER_NBIT != "yes" -o $USE_FILTER_SCALEOFFSET != "yes" ; then SKIP $arg else TOOLTEST $arg fi #filter conversions arg="$FILE8 -f dset_deflate:SZIP=8,NN" if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else TOOLTEST $arg fi arg="$FILE7 -f dset_szip:GZIP=1" if test $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else TOOLTEST $arg fi #limit arg="$FILE4 -f GZIP=1 -m 1024" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else TOOLTEST $arg fi #file arg="$FILE4 -e $srcdir/$INFO_FILE" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else TOOLTEST $arg fi ######################################################### # layout options (these files have no filters) ######################################################### TOOLTEST_MAIN $FILE4 --layout dset2:CHUNK=20x10 VERIFY_LAYOUT_DSET $FILE4 dset2 CHUNKED TOOLTEST_MAIN $FILE4 -l CHUNK=20x10 VERIFY_LAYOUT_ALL $FILE4 CHUNKED TOOLTEST_MAIN $FILE4 -l dset2:CONTI VERIFY_LAYOUT_DSET $FILE4 dset2 CONTIGUOUS TOOLTEST_MAIN $FILE4 -l CONTI VERIFY_LAYOUT_ALL $FILE4 CONTIGUOUS TOOLTEST_MAIN $FILE4 -l dset2:COMPA VERIFY_LAYOUT_DSET $FILE4 dset2 COMPACT TOOLTEST_MAIN $FILE4 -l COMPA VERIFY_LAYOUT_ALL $FILE4 COMPACT ################################################################ # layout conversions (file has no filters) ############################################################### TOOLTEST_MAIN $FILE4 -l dset_compact:CONTI VERIFY_LAYOUT_DSET $FILE4 dset_compact CONTIGUOUS TOOLTEST_MAIN $FILE4 -l dset_compact:CHUNK=2x5 VERIFY_LAYOUT_DSET $FILE4 dset_compact CHUNKED TOOLTEST_MAIN $FILE4 -l dset_compact:COMPA VERIFY_LAYOUT_DSET $FILE4 dset_compact COMPACT TOOLTEST_MAIN $FILE4 -l dset_contiguous:COMPA VERIFY_LAYOUT_DSET $FILE4 dset_contiguous COMPACT TOOLTEST_MAIN $FILE4 -l dset_contiguous:CHUNK=3x6 VERIFY_LAYOUT_DSET $FILE4 dset_contiguous CHUNKED TOOLTEST_MAIN $FILE4 -l dset_contiguous:CONTI VERIFY_LAYOUT_DSET $FILE4 dset_contiguous CONTIGUOUS TOOLTEST_MAIN $FILE4 -l dset_chunk:COMPA VERIFY_LAYOUT_DSET $FILE4 dset_chunk COMPACT TOOLTEST_MAIN $FILE4 -l dset_chunk:CONTI VERIFY_LAYOUT_DSET $FILE4 dset_chunk CONTIGUOUS TOOLTEST_MAIN $FILE4 -l dset_chunk:CHUNK=18x13 VERIFY_LAYOUT_DSET $FILE4 dset_chunk CHUNKED # test convert small size dataset ( < 1k) to compact layout without -m TOOLTEST_MAIN $FILE18 -l contig_small:COMPA VERIFY_LAYOUT_DSET $FILE18 contig_small COMPACT TOOLTEST_MAIN $FILE18 -l chunked_small_fixed:COMPA VERIFY_LAYOUT_DSET $FILE18 chunked_small_fixed COMPACT # Native option # Do not use FILE1, as the named dtype will be converted to native, and h5diff will # report a difference. TOOLTEST $FILE0 -n TOOLTEST $FILE2 -n # latest file format with long switches. use FILE4=h5repack_layout.h5 (no filters) arg="$FILE4 --layout CHUNK=20x10 --filter GZIP=1 --minimum=10 --native --latest --compact=8 --indexed=6 --ssize=8[:dtype]" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else TOOLTEST_MAIN $arg VERIFY_LAYOUT_ALL $FILE4 CHUNKED fi # latest file format with short switches. use FILE4=h5repack_layout.h5 (no filters) arg="$FILE4 -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype]" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else TOOLTEST_MAIN $arg VERIFY_LAYOUT_ALL $FILE4 CHUNKED fi # several global filters arg="$FILE4 --filter GZIP=1 --filter SHUF" if test $USE_FILTER_DEFLATE != "yes" -o $USE_FILTER_SHUFFLE != "yes" ; then SKIP $arg else TOOLTEST $arg fi # syntax of -i infile -o outfile # latest file format with short switches. use FILE4=h5repack_layout.h5 (no filters) arg="$FILE4 -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype]" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else TOOLTEST0 $arg fi # add a userblock to file arg="$FILE1 -u ublock.bin -b 2048" TOOLTEST $arg # add alignment arg="$FILE1 -t 1 -a 1 " TOOLTEST $arg # Check repacking file with old version of layout message (should get upgraded # to new version and be readable, etc.) TOOLTEST $FILE14 # test for datum size > H5TOOLS_MALLOCSIZE TOOLTEST $FILE1 -f GZIP=1 # Check repacking file with committed datatypes in odd configurations TOOLTEST $FILE15 # tests family driver (file is located in common testfiles folder, uses TOOLTEST1 TOOLTEST1 $FILE16 # test various references (bug 1814 and 1726) TOOLTEST $FILE_REF # test attribute with various references (bug 1797) # the references in attribute of compund or vlen datatype # TODO: include this test when code portion is completed. SKIP $FILE_ATTR_REF #TOOLTEST $FILE_ATTR_REF if test $nerrors -eq 0 ; then echo "All $TESTNAME tests passed." exit $EXIT_SUCCESS else echo "$TESTNAME tests failed with $nerrors errors." exit $EXIT_FAILURE fi