diff options
Diffstat (limited to 'tools/test/h5repack/h5repack.sh.in')
-rw-r--r-- | tools/test/h5repack/h5repack.sh.in | 1265 |
1 files changed, 1265 insertions, 0 deletions
diff --git a/tools/test/h5repack/h5repack.sh.in b/tools/test/h5repack/h5repack.sh.in new file mode 100644 index 0000000..4d3cae8 --- /dev/null +++ b/tools/test/h5repack/h5repack.sh.in @@ -0,0 +1,1265 @@ +#! /bin/sh +# +# Copyright by The HDF Group. +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the files COPYING and Copyright.html. COPYING can be found at the root +# of the source code distribution tree; Copyright.html can be found at the +# root level of an installed copy of the electronic HDF5 document set and +# is linked from the top-level documents page. It can also be found at +# http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have +# access to either file, you may request a copy from help@hdfgroup.org. +# +# Tests for the h5repack tool +# +# Modification: +# Pedro Vicente Nunes, 11/15/2006 +# Added $FILEN variables for file names +# + +srcdir=@srcdir@ + +USE_FILTER_SZIP="@USE_FILTER_SZIP@" +USE_FILTER_DEFLATE="@USE_FILTER_DEFLATE@" + +TESTNAME=h5repack +EXIT_SUCCESS=0 +EXIT_FAILURE=1 + +H5REPACK=../../src/h5repack/h5repack # The tool name +H5REPACK_BIN=`pwd`/$H5REPACK # The path of the tool binary + +H5DIFF=../../src/h5diff/h5diff # The h5diff tool name +H5DIFF_BIN=`pwd`/$H5DIFF # The path of the h5diff tool binary + +H5DUMP=../../src/h5dump/h5dump # The h5dump tool name +H5DUMP_BIN=`pwd`/$H5DUMP # The path of the h5dump tool binary + +RM='rm -rf' +CMP='cmp' +DIFF='diff -c' +GREP='grep' +CP='cp' +DIRNAME='dirname' +LS='ls' +AWK='awk' + +H5DETECTSZIP=testh5repack_detect_szip +H5DETECTSZIP_BIN=`pwd`/$H5DETECTSZIP + + +nerrors=0 +verbose=yes + +# source dirs +SRC_TOOLS="$srcdir/../.." +SRC_TOOLS_TESTFILES="$SRC_TOOLS/testfiles" + +# testfiles source dirs for tools +SRC_H5LS_TESTFILES="$SRC_TOOLS_TESTFILES" +SRC_H5DUMP_TESTFILES="$SRC_TOOLS_TESTFILES" +SRC_H5DIFF_TESTFILES="$SRC_TOOLS/test/h5diff/testfiles" +SRC_H5COPY_TESTFILES="$SRC_TOOLS/test/h5copy/testfiles" +SRC_H5REPACK_TESTFILES="$SRC_TOOLS/test/h5repack/testfiles" +SRC_H5JAM_TESTFILES="$SRC_TOOLS/test/h5jam/testfiles" +SRC_H5STAT_TESTFILES="$SRC_TOOLS/test/h5stat/testfiles" +SRC_H5IMPORT_TESTFILES="$SRC_TOOLS/test/h5import/testfiles" + +TESTDIR=./testpack +test -d $TESTDIR || mkdir $TESTDIR + +###################################################################### +# test files +# -------------------------------------------------------------------- +# All the test files copy from source directory to test directory +# NOTE: Keep this framework to add/remove test files. +# Any test files from other tools can be used in this framework. +# This list are also used for checking exist. +# Comment '#' without space can be used. +# -------------------------------------------------------------------- +LIST_HDF5_TEST_FILES=" +$SRC_H5REPACK_TESTFILES/h5repack_attr.h5 +$SRC_H5REPACK_TESTFILES/h5repack_attr_refs.h5 +$SRC_H5REPACK_TESTFILES/h5repack_deflate.h5 +$SRC_H5REPACK_TESTFILES/h5repack_early.h5 +$SRC_H5REPACK_TESTFILES/h5repack_ext.h5 +$SRC_H5REPACK_TESTFILES/h5repack_fill.h5 +$SRC_H5REPACK_TESTFILES/h5repack_filters.h5 +$SRC_H5REPACK_TESTFILES/h5repack_fletcher.h5 +$SRC_H5REPACK_TESTFILES/h5repack_hlink.h5 +$SRC_H5REPACK_TESTFILES/h5repack_layout.h5 +$SRC_H5REPACK_TESTFILES/h5repack_layouto.h5 +$SRC_H5REPACK_TESTFILES/h5repack_layout2.h5 +$SRC_H5REPACK_TESTFILES/h5repack_layout3.h5 +$SRC_H5REPACK_TESTFILES/h5repack_named_dtypes.h5 +$SRC_H5REPACK_TESTFILES/h5repack_nested_8bit_enum.h5 +$SRC_H5REPACK_TESTFILES/h5repack_nested_8bit_enum_deflated.h5 +$SRC_H5REPACK_TESTFILES/h5repack_nbit.h5 +$SRC_H5REPACK_TESTFILES/h5repack_objs.h5 +$SRC_H5REPACK_TESTFILES/h5repack_refs.h5 +$SRC_H5REPACK_TESTFILES/h5repack_shuffle.h5 +$SRC_H5REPACK_TESTFILES/h5repack_soffset.h5 +$SRC_H5REPACK_TESTFILES/h5repack_szip.h5 +$SRC_H5DIFF_TESTFILES/h5diff_attr1.h5 +$SRC_TOOLS_TESTFILES/tfamily00000.h5 +$SRC_TOOLS_TESTFILES/tfamily00001.h5 +$SRC_TOOLS_TESTFILES/tfamily00002.h5 +$SRC_TOOLS_TESTFILES/tfamily00003.h5 +$SRC_TOOLS_TESTFILES/tfamily00004.h5 +$SRC_TOOLS_TESTFILES/tfamily00005.h5 +$SRC_TOOLS_TESTFILES/tfamily00006.h5 +$SRC_TOOLS_TESTFILES/tfamily00007.h5 +$SRC_TOOLS_TESTFILES/tfamily00008.h5 +$SRC_TOOLS_TESTFILES/tfamily00009.h5 +$SRC_TOOLS_TESTFILES/tfamily00010.h5 +$SRC_TOOLS_TESTFILES/vds/1_a.h5 +$SRC_TOOLS_TESTFILES/vds/1_b.h5 +$SRC_TOOLS_TESTFILES/vds/1_c.h5 +$SRC_TOOLS_TESTFILES/vds/1_d.h5 +$SRC_TOOLS_TESTFILES/vds/1_e.h5 +$SRC_TOOLS_TESTFILES/vds/1_f.h5 +$SRC_TOOLS_TESTFILES/vds/1_vds.h5 +$SRC_TOOLS_TESTFILES/vds/2_a.h5 +$SRC_TOOLS_TESTFILES/vds/2_b.h5 +$SRC_TOOLS_TESTFILES/vds/2_c.h5 +$SRC_TOOLS_TESTFILES/vds/2_d.h5 +$SRC_TOOLS_TESTFILES/vds/2_e.h5 +$SRC_TOOLS_TESTFILES/vds/2_vds.h5 +$SRC_TOOLS_TESTFILES/vds/3_1_vds.h5 +$SRC_TOOLS_TESTFILES/vds/3_2_vds.h5 +$SRC_TOOLS_TESTFILES/vds/4_0.h5 +$SRC_TOOLS_TESTFILES/vds/4_1.h5 +$SRC_TOOLS_TESTFILES/vds/4_2.h5 +$SRC_TOOLS_TESTFILES/vds/4_vds.h5 +$SRC_TOOLS_TESTFILES/vds/5_a.h5 +$SRC_TOOLS_TESTFILES/vds/5_b.h5 +$SRC_TOOLS_TESTFILES/vds/5_c.h5 +$SRC_TOOLS_TESTFILES/vds/5_vds.h5 +" + +LIST_OTHER_TEST_FILES=" +$SRC_H5REPACK_TESTFILES/h5repack-help.txt +$SRC_H5REPACK_TESTFILES/h5repack_ext.bin +$SRC_H5REPACK_TESTFILES/ublock.bin +$SRC_H5REPACK_TESTFILES/h5repack.info +$SRC_H5REPACK_TESTFILES/deflate_limit.h5repack_layout.h5.ddl +$SRC_H5REPACK_TESTFILES/h5repack_layout.h5.ddl +$SRC_H5REPACK_TESTFILES/h5repack_filters.h5-gzip_verbose_filters.tst +$SRC_H5REPACK_TESTFILES/h5repack_layout.h5-dset2_chunk_20x10-errstk.tst +$SRC_H5REPACK_TESTFILES/h5repack_layout.h5-plugin_test.ddl +$SRC_H5REPACK_TESTFILES/plugin_test.h5repack_layout.h5.tst +$SRC_H5REPACK_TESTFILES/1_vds.h5-vds_dset_chunk20x10x5-v.ddl +$SRC_H5REPACK_TESTFILES/2_vds.h5-vds_chunk3x6x9-v.ddl +$SRC_H5REPACK_TESTFILES/3_1_vds.h5-vds_chunk2x5x8-v.ddl +$SRC_H5REPACK_TESTFILES/4_vds.h5-vds_compa-v.ddl +$SRC_H5REPACK_TESTFILES/4_vds.h5-vds_conti-v.ddl +" + +# +# copy test files and expected output files from source dirs to test dir +# +COPY_TESTFILES="$LIST_HDF5_TEST_FILES $LIST_OTHER_TEST_FILES" + +COPY_TESTFILES_TO_TESTDIR() +{ + # copy test files. Used -f to make sure get a new copy + for tstfile in $COPY_TESTFILES + do + # ignore '#' comment + echo $tstfile | tr -d ' ' | grep '^#' > /dev/null + RET=$? + if [ $RET -eq 1 ]; then + # skip cp if srcdir is same as destdir + # this occurs when build/test performed in source dir and + # make cp fail + SDIR=`$DIRNAME $tstfile` + INODE_SDIR=`$LS -i -d $SDIR | $AWK -F' ' '{print $1}'` + INODE_DDIR=`$LS -i -d $TESTDIR | $AWK -F' ' '{print $1}'` + if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then + $CP -f $tstfile $TESTDIR + if [ $? -ne 0 ]; then + echo "Error: FAILED to copy $tstfile ." + + # Comment out this to CREATE expected file + exit $EXIT_FAILURE + fi + fi + fi + done +} + +CLEAN_TESTFILES_AND_TESTDIR() +{ + # skip rm if srcdir is same as destdir + # this occurs when build/test performed in source dir and + # make cp fail + SDIR=$SRC_H5REPACK_TESTFILES + INODE_SDIR=`$LS -i -d $SDIR | $AWK -F' ' '{print $1}'` + INODE_DDIR=`$LS -i -d $TESTDIR | $AWK -F' ' '{print $1}'` + if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then + $RM $TESTDIR + fi +} + +# Print a $* message left justified in a field of 70 characters +# +MESSAGE() { + SPACES=" " + echo "$* $SPACES" | cut -c1-70 | tr -d '\012' +} + +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Testing". +# +TESTING() { + MESSAGE "Testing $*" +} + +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Verifying". +# +VERIFY() { + MESSAGE "Verifying $*" +} + +# Print a message that a test has been skipped (because a required filter +# was unavailable) +SKIP() { + TESTING $H5REPACK $@ + echo " -SKIP-" +} + +# Call the h5diff tool +# +DIFFTEST() +{ + VERIFY h5diff output $@ + ( + cd $TESTDIR + $RUNSERIAL $H5DIFF_BIN -q "$@" + ) + RET=$? + if [ $RET != 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + fi + +} + +# Call h5repack +# + + +# call TOOLTEST_MAIN and delete $output file +TOOLTEST() +{ + echo $@ + infile=$2 + outfile=out-$1.$2 + shift + shift + + # Run test. + TESTING $H5REPACK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile + ) + RET=$? + if [ $RET != 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + DIFFTEST $infile $outfile + fi + rm -f $outfile +} + +#------------------------------------------ +# Verifying layouts of a dataset +VERIFY_LAYOUT_DSET() +{ + layoutfile=layout-$1.$2 + dset=$3 + expectlayout=$4 + infile=$2 + outfile=out-$1.$2 + shift + shift + shift + shift + + TESTING $H5REPACK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile + ) + RET=$? + if [ $RET != 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + DIFFTEST $infile $outfile + fi + + #--------------------------------- + # check the layout from a dataset + VERIFY "a dataset layout" + ( + cd $TESTDIR + $RUNSERIAL $H5DUMP_BIN -d $dset -pH $outfile > $layoutfile + ) + $GREP $expectlayout $TESTDIR/$layoutfile > /dev/null + if [ $? -eq 0 ]; then + echo " PASSED" + else + echo " FAILED" + nerrors="`expr $nerrors + 1`" + fi + + # clean up tmp files + rm -f $outfile + rm -f $layoutfile +} + +#------------------------------------------ +# Verifying layouts of a dataset +VERIFY_LAYOUT_VDS() +{ + layoutfile=layout-$1.$2 + dset=$3 + expectlayout=$4 + infile=$2 + outfile=out-$1.$2 + + expect="$TESTDIR/$2-$1-v.ddl" + actual="$TESTDIR/$2-$1-v.out" + actual_err="$TESTDIR/$2-$1-v.err" + + shift + shift + shift + shift + + TESTING $H5REPACK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile + ) + RET=$? + if [ $RET != 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + fi + + #--------------------------------- + # check the layout from a dataset + VERIFY "a dataset layout" + ( + cd $TESTDIR + $RUNSERIAL $H5DUMP_BIN -d $dset -p $outfile + ) >$actual 2>$actual_err + + cat $actual_err >> $actual + + if [ ! -f $expect ]; then + # Create the expect file if it doesn't yet exist. + echo " CREATED" + cp $actual $expect + elif $CMP $expect $actual > /dev/null 2>&1 ; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected result (*.ddl) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $caseless $expect $actual |sed 's/^/ /' + fi + + # Clean up output file + if test -z "$HDF5_NOCLEANUP"; then + rm -f $actual $actual_err + fi + + # clean up tmp files + rm -f $outfile +} + +#---------------------------------------- +# Verifying layouts from entire file +VERIFY_LAYOUT_ALL() +{ + infile=$2 + outfile=out-$1.$2 + layoutfile=layout-$1.$2 + expectlayout=$3 + shift + shift + shift + + TESTING $H5REPACK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile + ) + RET=$? + if [ $RET != 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + DIFFTEST $infile $outfile + fi + + + #--------------------------------- + # check the layout from a dataset + # check if the other layouts still exsit + VERIFY "layouts" + ( + cd $TESTDIR + echo + # if CONTIGUOUS + if [ $expectlayout = "CONTIGUOUS" ]; then + TESTING $H5DUMP_BIN -pH $outfile + ( + $RUNSERIAL $H5DUMP_BIN -pH $outfile > $layoutfile + ) + $GREP "COMPACT" $layoutfile > /dev/null + if [ $? -eq 0 ]; then + echo " FAILED" + nerrors="`expr $nerrors + 1`" + else + $GREP "CHUNKED" $layoutfile > /dev/null + if [ $? -eq 0 ]; then + echo " FAILED" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + fi + fi + else + # if COMPACT + if [ $expectlayout = "COMPACT" ]; then + TESTING $H5DUMP_BIN -pH $outfile + ( + $RUNSERIAL $H5DUMP_BIN -pH $outfile > $layoutfile + ) + $GREP "CHUNKED" $layoutfile > /dev/null + if [ $? -eq 0 ]; then + echo " FAILED" + nerrors="`expr $nerrors + 1`" + else + $GREP "CONTIGUOUS" $layoutfile > /dev/null + if [ $? -eq 0 ]; then + echo " FAILED" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + fi + fi + else + # if CHUNKED + if [ $expectlayout = "CHUNKED" ]; then + TESTING $H5DUMP_BIN -pH $outfile + ( + $RUNSERIAL $H5DUMP_BIN -pH $outfile > $layoutfile + ) + $GREP "CONTIGUOUS" $layoutfile > /dev/null + if [ $? -eq 0 ]; then + echo " FAILED" + nerrors="`expr $nerrors + 1`" + else + $GREP "COMPACT" $layoutfile > /dev/null + if [ $? -eq 0 ]; then + echo " FAILED" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + fi + fi + fi + fi + fi + ) + + # clean up tmp files + rm -f $outfile + rm -f $layoutfile +} + +# same as TOOLTEST, but it uses the old syntax -i input_file -o output_file +# +TOOLTEST0() +{ + infile=$2 + outfile=out-$1.$2 + shift + shift + + # Run test. + TESTING $H5REPACK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5REPACK_BIN -i $infile -o $outfile "$@" + ) + RET=$? + if [ $RET != 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + DIFFTEST $infile $outfile + fi + rm -f $outfile +} + + +# same as TOOLTEST, but it uses without -i -o options +# used to test the family driver, where these files reside +# +TOOLTEST1() +{ + infile=$2 + outfile=out-$1.$2 + shift + shift + + # Run test. + TESTING $H5REPACK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile + ) + RET=$? + if [ $RET != 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + DIFFTEST $infile $outfile + fi + rm -f $outfile +} + +# This is same as TOOLTEST() with comparing display output +# from -v option +# +TOOLTESTV() +{ + expect="$TESTDIR/$2-$1.tst" + actual="$TESTDIR/`basename $2 .ddl`.out" + actual_err="$TESTDIR/`basename $2 .ddl`.err" + + infile=$2 + outfile=out-$1.$2 + shift + shift + + # Run test. + TESTING $H5REPACK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile + ) >$actual 2>$actual_err + RET=$? + if [ $RET != 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + DIFFTEST $infile $outfile + fi + + # display output compare + STDOUT_FILTER $actual + cat $actual_err >> $actual + + VERIFY output from $H5REPACK $@ + if cmp -s $expect $actual; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected result (*.tst) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /' + fi + + rm -f $actual $actual_err + rm -f $outfile +} + +# same as TOOLTESTV but filters error stack output and compares to an error file +# Extract file name, line number, version and thread IDs because they may be different +# ADD_H5ERR_MASK_TEST +TOOLTESTM() { + + expect="$TESTDIR/$2-$1.tst" + actual="$TESTDIR/`basename $2 .tst`.out" + actual_err="$TESTDIR/`basename $2 .tst`.oerr" + actual_sav=${actual}-sav + actual_err_sav=${actual_err}-sav + + infile=$2 + outfile=out-$1.$2 + shift + shift + + # Run test. + TESTING $H5REPACK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile + ) >$actual 2>$actual_err + + # save actual and actual_err in case they are needed later. + cp $actual $actual_sav + cp $actual_err $actual_err_sav + + # Extract file name, line number, version and thread IDs because they may be different + sed -e 's/thread [0-9]*/thread (IDs)/' -e 's/: .*\.c /: (file name) /' \ + -e 's/line [0-9]*/line (number)/' \ + -e 's/v[1-9]*\.[0-9]*\./version (number)\./' \ + -e 's/[1-9]*\.[0-9]*\.[0-9]*[^)]*/version (number)/' \ + -e 's/H5Eget_auto[1-2]*/H5Eget_auto(1 or 2)/' \ + -e 's/H5Eset_auto[1-2]*/H5Eset_auto(1 or 2)/' \ + $actual_err > $actual + + if [ ! -f $expect ]; then + # Create the expect file if it doesn't yet exist. + echo " CREATED" + cp $actual $expect + elif $CMP $expect $actual; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected result (*.tst) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + fi + + # Clean up output file + if test -z "$HDF5_NOCLEANUP"; then + rm -f $actual $actual_err $actual_sav $actual_err_sav + fi + +} + +# This is same as TOOLTESTV() with comparing h5dump output +# from -pH option +# +TOOLTEST_DUMP() +{ + infile=$2 + outfile=out-$1.$2 + expect="$TESTDIR/$1.$2.ddl" + actual="$TESTDIR/out-$1.$2.out" + actual_err="$TESTDIR/out-$1.$2.err" + + shift + shift + + # Run test. + TESTING $H5REPACK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile + ) >$actual 2>$actual_err + RET=$? + if [ $RET != 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + VERIFY h5dump output $@ + ( + cd $TESTDIR + $RUNSERIAL $H5DUMP_BIN -pH $outfile + ) >$actual 2>$actual_err + cat $actual_err >> $actual + + RET=$? + + fi + + if cmp -s $expect $actual; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected result (*.ddl) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /' + fi + + rm -f $actual $actual_err + rm -f $outfile +} + +# TOOLTEST_META: +# Test metadata block size option. +# Reason to create a function here is to localize all special steps related to +# metadata block size option in one place. This is a quick solution. More +# thought out solution needed when there is more time. +# +# $1: test input file +# $2:$: metadata options (either "-M size" or "--metadata_block_size=size") +# +# Algorithm: +# Run it once without the metadata option ($2-$); +# Save the result output file; +# Run it second time with the metadata option; +# Verify the output file of second run is larger than the one of 1st run. +TOOLTEST_META() +{ + infile=$2 + outfile=out-$1.$2 + shift + shift + + # Run test. + TESTING $H5REPACK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5REPACK_BIN $infile $outfile + ) + RET=$? + if [ $RET != 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + DIFFTEST $infile $outfile + fi + # get the size of the first output file + size1=`wc -c $TESTDIR/$outfile | cut -d' ' -f1` + + # 2nd run with metadata option + # Run test. + TESTING $H5REPACK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile + ) + RET=$? + if [ $RET != 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + DIFFTEST $infile $outfile + fi + # get the size of the second output file + size2=`wc -c $TESTDIR/$outfile | cut -d' ' -f1` + + # verify sizes. + MESSAGE "Verify the sizes of both output files ($size1 vs $size2)" + if [ $size1 -lt $size2 ]; then + # pass + echo " PASSED" + else + #fail + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + fi + + rm -f $outfile +} + +# ADD_HELP_TEST +TOOLTEST_HELP() { + + expect="$TESTDIR/$1" + actual="$TESTDIR/`basename $1 .txt`.out" + actual_err="$TESTDIR/`basename $1 .txt`.err" + shift + + # Run test. + TESTING $H5REPACK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5REPACK_BIN "$@" + ) >$actual 2>$actual_err + cat $actual_err >> $actual + + if [ ! -f $expect ]; then + # Create the expect data file if it doesn't yet exist. + echo " CREATED" + cp $actual $expect-CREATED + elif cmp -s $expect $actual; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected output (*.txt) differs from actual output (*.out)" + nerrors="`expr $nerrors + 1`" + fi + + # Clean up output file + if test -z "$HDF5_NOCLEANUP"; then + rm -f $actual $actual_err + fi + +} + +# This is different from $srcdir/../../../bin/output_filter.sh +STDOUT_FILTER() { + result_file=$1 + tmp_file=/tmp/h5test_tmp_$$ + # Filter name of files. + cp $result_file $tmp_file + sed -e '/^Opening file/d' -e '/^Making file/d' \ + < $tmp_file > $result_file + # cleanup + rm -f $tmp_file +} + +# +# The tests +# We use the files generated by h5repacktst +# Each run generates "<file>.out.h5" and the tool h5diff is used to +# compare the input and output files +# +# the tests are the same as the program h5repacktst, but run from the CLI +# + +# See which filters are usable (and skip tests for filters we +# don't have). Do this by searching H5pubconf.h to see which +# filters are defined. + +# detect whether the encoder is present. +USE_FILTER_SZIP_ENCODER="no"; +if test $USE_FILTER_SZIP = "yes"; then +USE_FILTER_SZIP_ENCODER=`$RUNSERIAL $H5DETECTSZIP_BIN` +fi + +############################################################################## +### T H E T E S T S +############################################################################## +# prepare for test +COPY_TESTFILES_TO_TESTDIR + +TOOLTEST_HELP h5repack-help.txt -h + +# copy files (these files have no filters) +TOOLTEST fill h5repack_fill.h5 +TOOLTEST objs h5repack_objs.h5 +TOOLTEST attr h5repack_attr.h5 +TOOLTEST hlink h5repack_hlink.h5 +TOOLTEST layout h5repack_layout.h5 +TOOLTEST early h5repack_early.h5 + +# nested 8bit enum in both deflated and non-deflated datafiles +if [ $USE_FILTER_DEFLATE != "yes" ]; then +TOOLTEST nested_8bit_enum h5repack_nested_8bit_enum.h5 +else +TOOLTEST nested_8bit_enum h5repack_nested_8bit_enum_deflated.h5 +fi + +# use h5repack_layout.h5 to write some filters (this file has no filters) + +# gzip with individual object +arg="h5repack_layout.h5 -f dset1:GZIP=1 -l dset1:CHUNK=20x10" +if test $USE_FILTER_DEFLATE != "yes" ; then + SKIP $arg +else + TOOLTEST gzip_individual $arg +fi + +# gzip for all +arg="h5repack_layout.h5 -f GZIP=1" +if test $USE_FILTER_DEFLATE != "yes" ; then + SKIP $arg +else + TOOLTEST gzip_all $arg +fi + +# szip with individual object +arg="h5repack_layout.h5 -f dset2:SZIP=8,EC -l dset2:CHUNK=20x10" +if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then + SKIP $arg +else + TOOLTEST szip_individual $arg +fi + +# szip for all +arg="h5repack_layout.h5 -f SZIP=8,NN" +if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then + SKIP $arg +else + TOOLTEST szip_all $arg +fi + +# shuffle with individual object +arg="h5repack_layout.h5 -f dset2:SHUF -l dset2:CHUNK=20x10" +TOOLTEST shuffle_individual $arg + + +# shuffle for all +arg="h5repack_layout.h5 -f SHUF" +TOOLTEST shuffle_all $arg + +# fletcher32 with individual object +arg="h5repack_layout.h5 -f dset2:FLET -l dset2:CHUNK=20x10" +TOOLTEST fletcher_individual $arg + +# fletcher32 for all +arg="h5repack_layout.h5 -f FLET" +TOOLTEST fletcher_all $arg + +# all filters +arg="h5repack_layout.h5 -f dset2:SHUF -f dset2:FLET -f dset2:SZIP=8,NN -f dset2:GZIP=1 -l dset2:CHUNK=20x10" +if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then + SKIP $arg +else + TOOLTEST all_filters $arg +fi + +# verbose gzip with individual object +arg="h5repack_filters.h5 -v -f /dset_deflate:GZIP=9" +if test $USE_FILTER_DEFLATE != "yes" ; then + SKIP $arg +else + # compare output + TOOLTESTV gzip_verbose_filters $arg +fi + +########################################################### +# the following tests assume the input files have filters +########################################################### + +# szip copy +arg="h5repack_szip.h5" +if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then + SKIP $arg +else + TOOLTEST szip_copy $arg +fi + +# szip remove +arg="h5repack_szip.h5 --filter=dset_szip:NONE" +if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then + SKIP $arg +else + TOOLTEST szip_remove $arg +fi + +# deflate copy +arg="h5repack_deflate.h5" +if test $USE_FILTER_DEFLATE != "yes" ; then + SKIP $arg +else + TOOLTEST deflate_copy $arg +fi + +# deflate remove +arg="h5repack_deflate.h5 -f dset_deflate:NONE" +if test $USE_FILTER_DEFLATE != "yes" ; then + SKIP $arg +else + TOOLTEST deflate_remove $arg +fi + +# shuffle copy +arg="h5repack_shuffle.h5" +TOOLTEST shuffle_copy $arg + +# shuffle remove +arg="h5repack_shuffle.h5 -f dset_shuffle:NONE" +TOOLTEST shuffle_remove $arg + +# fletcher32 copy +arg="h5repack_fletcher.h5" +TOOLTEST fletcher_copy $arg + +# fletcher32 remove +arg="h5repack_fletcher.h5 -f dset_fletcher32:NONE" +TOOLTEST fletcher_remove $arg + +# nbit copy +arg="h5repack_nbit.h5" +TOOLTEST nbit_copy $arg + +# nbit remove +arg="h5repack_nbit.h5 -f dset_nbit:NONE" +TOOLTEST nbit_remove $arg + +# nbit add +arg="h5repack_nbit.h5 -f dset_int31:NBIT" +TOOLTEST nbit_add $arg + +# scaleoffset copy +arg="h5repack_soffset.h5" +TOOLTEST scale_copy $arg + +# scaleoffset add +arg="h5repack_soffset.h5 -f dset_none:SOFF=31,IN" +TOOLTEST scale_add $arg + +# scaleoffset remove +arg="h5repack_soffset.h5 -f dset_scaleoffset:NONE" +TOOLTEST scale_remove $arg + +# remove all filters +arg="h5repack_filters.h5 -f NONE" +if test $USE_FILTER_DEFLATE != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SZIP_ENCODER != "yes" ; then + SKIP $arg +else + TOOLTEST remove_all $arg +fi + +#filter conversions + +arg="h5repack_deflate.h5 -f dset_deflate:SZIP=8,NN" +if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then + SKIP $arg +else + TOOLTEST deflate_convert $arg +fi + +arg="h5repack_szip.h5 -f dset_szip:GZIP=1" +if test $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then + SKIP $arg +else + TOOLTEST szip_convert $arg +fi + + +#limit +arg="h5repack_layout.h5 -f GZIP=1 -m 1024" +if test $USE_FILTER_DEFLATE != "yes" ; then + SKIP $arg +else + TOOLTEST_DUMP deflate_limit $arg +fi + +#file +arg="h5repack_layout.h5 -e h5repack.info" +if test $USE_FILTER_DEFLATE != "yes" ; then + SKIP $arg +else + TOOLTEST deflate_file $arg +fi + +######################################################### +# layout options (these files have no filters) +######################################################### +VERIFY_LAYOUT_DSET dset2_chunk_20x10 h5repack_layout.h5 dset2 CHUNKED --layout dset2:CHUNK=20x10 + +VERIFY_LAYOUT_ALL chunk_20x10 h5repack_layout.h5 CHUNKED -l CHUNK=20x10 + +VERIFY_LAYOUT_DSET dset2_conti h5repack_layout.h5 dset2 CONTIGUOUS -l dset2:CONTI + +VERIFY_LAYOUT_ALL conti h5repack_layout.h5 CONTIGUOUS -l CONTI + +VERIFY_LAYOUT_DSET dset2_compa h5repack_layout.h5 dset2 COMPACT -l dset2:COMPA + +VERIFY_LAYOUT_ALL compa h5repack_layout.h5 COMPACT -l COMPA + +TOOLTESTM dset2_chunk_20x10-errstk h5repack_layout.h5 --layout=dset2:CHUNK=20x10x5 --enable-error-stack + +################################################################ +# layout conversions (file has no filters) +############################################################### + +VERIFY_LAYOUT_DSET dset_compa_conti h5repack_layout.h5 dset_compact CONTIGUOUS -l dset_compact:CONTI + +VERIFY_LAYOUT_DSET dset_compa_chunk h5repack_layout.h5 dset_compact CHUNKED -l dset_compact:CHUNK=2x5 + +VERIFY_LAYOUT_DSET dset_compa_compa h5repack_layout.h5 dset_compact COMPACT -l dset_compact:COMPA + +VERIFY_LAYOUT_DSET dset_conti_compa h5repack_layout.h5 dset_contiguous COMPACT -l dset_contiguous:COMPA + +VERIFY_LAYOUT_DSET dset_conti_chunk h5repack_layout.h5 dset_contiguous CHUNKED -l dset_contiguous:CHUNK=3x6 + +VERIFY_LAYOUT_DSET dset_conti_conti h5repack_layout.h5 dset_contiguous CONTIGUOUS -l dset_contiguous:CONTI + +VERIFY_LAYOUT_DSET chunk_compa h5repack_layout.h5 dset_chunk COMPACT -l dset_chunk:COMPA + +VERIFY_LAYOUT_DSET chunk_conti h5repack_layout.h5 dset_chunk CONTIGUOUS -l dset_chunk:CONTI + +VERIFY_LAYOUT_DSET chunk_18x13 h5repack_layout.h5 dset_chunk CHUNKED -l dset_chunk:CHUNK=18x13 + +# test convert small size dataset ( < 1k) to compact layout without -m +VERIFY_LAYOUT_DSET contig_small_compa h5repack_layout2.h5 contig_small COMPACT -l contig_small:COMPA + +VERIFY_LAYOUT_DSET contig_small_fixed_compa h5repack_layout2.h5 chunked_small_fixed COMPACT -l chunked_small_fixed:COMPA + +#--------------------------------------------------------------------------- +# Test file contains chunked datasets (need multiple dsets) with +# unlimited max dims. (HDFFV-7933) +# Use first dset to test. +#--------------------------------------------------------------------------- +# chunk to chunk - specify chunk dim bigger than any current dim +VERIFY_LAYOUT_DSET chunk2chunk h5repack_layout3.h5 chunk_unlimit1 CHUNK -l chunk_unlimit1:CHUNK=100x300 + +# chunk to contiguous +VERIFY_LAYOUT_DSET chunk2conti h5repack_layout3.h5 chunk_unlimit1 CONTI -l chunk_unlimit1:CONTI + +# chunk to compact - convert big dataset (should be > 64k) for this purpose, +# should remain as original layout (chunk) +VERIFY_LAYOUT_DSET chunk2compa h5repack_layout3.h5 chunk_unlimit1 CHUNK -l chunk_unlimit1:COMPA + +#-------------------------------------------------------------------------- +# Test -f for some specific cases. Chunked dataset with unlimited max dims. +# (HDFFV-8012) +#-------------------------------------------------------------------------- +# - should not fail +# - should not change max dims from unlimit + +# chunk dim is bigger than dataset dim. ( dset size < 64k ) +VERIFY_LAYOUT_DSET error1 h5repack_layout3.h5 chunk_unlimit1 H5S_UNLIMITED -f chunk_unlimit1:NONE +# chunk dim is bigger than dataset dim. ( dset size > 64k ) +VERIFY_LAYOUT_DSET error2 h5repack_layout3.h5 chunk_unlimit2 H5S_UNLIMITED -f chunk_unlimit2:NONE + +# chunk dims are smaller than dataset dims. ( dset size < 64k ) +#TOOLTEST_MAIN h5repack_layout3.h5 -f chunk_unlimit3:NONE +VERIFY_LAYOUT_DSET error3 h5repack_layout3.h5 chunk_unlimit3 H5S_UNLIMITED -f chunk_unlimit3:NONE + +# file input - should not fail +TOOLTEST error4 h5repack_layout3.h5 -f NONE + +#-------------------------------------------------------------------------- +# Test base: Convert CHUNK to CONTI for a chunked dataset with small dataset +# (dset size < 64K) and with unlimited max dims on a condition as follow. +# (HDFFV-8214) +#-------------------------------------------------------------------------- + +# chunk dim is bigger than dataset dim. should succeed. +VERIFY_LAYOUT_DSET ckdim_biger h5repack_layout3.h5 chunk_unlimit2 CONTI -l chunk_unlimit2:CONTI +# chunk dim is smaller than dataset dim. should succeed. +VERIFY_LAYOUT_DSET ckdim_smaller h5repack_layout3.h5 chunk_unlimit3 CONTI -l chunk_unlimit3:CONTI + + +# Native option +# Do not use FILE1, as the named dtype will be converted to native, and h5diff will +# report a difference. +TOOLTEST native_fill h5repack_fill.h5 -n +TOOLTEST native_attr h5repack_attr.h5 -n + + +# latest file format with long switches. use FILE4=h5repack_layout.h5 (no filters) +arg="h5repack_layout.h5 --layout CHUNK=20x10 --filter GZIP=1 --minimum=10 --native --latest --compact=8 --indexed=6 --ssize=8[:dtype]" +if test $USE_FILTER_DEFLATE != "yes" ; then + SKIP $arg +else + VERIFY_LAYOUT_ALL layout_long_switches h5repack_layout.h5 CHUNKED --layout CHUNK=20x10 --filter GZIP=1 --minimum=10 --native --latest --compact=8 --indexed=6 --ssize=8[:dtype] +fi + +# latest file format with short switches. use FILE4=h5repack_layout.h5 (no filters) +arg="h5repack_layout.h5 -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype]" +if test $USE_FILTER_DEFLATE != "yes" ; then + SKIP $arg +else + VERIFY_LAYOUT_ALL layout_short_switches h5repack_layout.h5 CHUNKED -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype] +fi + +# several global filters + +arg="h5repack_layout.h5 --filter GZIP=1 --filter SHUF" +if test $USE_FILTER_DEFLATE != "yes" ; then + SKIP $arg +else + TOOLTEST global_filters $arg +fi + +# syntax of -i infile -o outfile +# latest file format with short switches. use FILE4=h5repack_layout.h5 (no filters) +arg="h5repack_layout.h5 -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype]" +if test $USE_FILTER_DEFLATE != "yes" ; then + SKIP $arg +else + TOOLTEST0 old_style_layout_short_switches $arg +fi + +# add a userblock to file +arg="h5repack_objs.h5 -u ublock.bin -b 2048" +TOOLTEST add_userblock $arg + +# add alignment +arg="h5repack_objs.h5 -t 1 -a 1 " +TOOLTEST add_alignment $arg + +# Check repacking file with old version of layout message (should get upgraded +# to new version and be readable, etc.) +TOOLTEST upgrade_layout h5repack_layouto.h5 + +# test for datum size > H5TOOLS_MALLOCSIZE +TOOLTEST gt_mallocsize h5repack_objs.h5 -f GZIP=1 + +# Check repacking file with committed datatypes in odd configurations +TOOLTEST committed_dt h5repack_named_dtypes.h5 + +# tests family driver (file is located in common testfiles folder, uses TOOLTEST1 +TOOLTEST1 family tfamily%05d.h5 + +# test various references (bug 1814 and 1726) +TOOLTEST bug1814 h5repack_refs.h5 + +# test attribute with various references (bug1797 / HDFFV-5932) +# the references in attribute of compund or vlen datatype +TOOLTEST HDFFV-5932 h5repack_attr_refs.h5 + +# Add test for memory leak in attirbute. This test is verified by CTEST. +# 1. leak from vlen string +# 2. leak from compound type without reference member +# (HDFFV-7840, ) +# Note: this test is experimental for sharing test file among tools +TOOLTEST HDFFV-7840 h5diff_attr1.h5 + +# tests for metadata block size option +TOOLTEST_META meta_short h5repack_layout.h5 -M 8192 +TOOLTEST_META meta_long h5repack_layout.h5 --metadata_block_size=8192 + +# VDS tests + +################################################################ +# layout conversions +############################################################### +if test $USE_FILTER_DEFLATE != "yes" ; then + SKIP vds_dset_chunk_20x10x5 +else + VERIFY_LAYOUT_VDS vds_dset_chunk_20x10x5 1_vds.h5 vds_dset CHUNKED --layout vds_dset:CHUNK=20x10x5 +fi + +if test $USE_FILTER_DEFLATE != "yes" ; then + SKIP vds_chunk2x5x8 +else + VERIFY_LAYOUT_VDS vds_chunk2x5x8 3_1_vds.h5 vds_dset CHUNKED -l vds_dset:CHUNK=2x5x8 +fi + +if test $USE_FILTER_DEFLATE != "yes" ; then + SKIP vds_chunk3x6x9 +else + VERIFY_LAYOUT_VDS vds_chunk3x6x9 2_vds.h5 vds_dset CHUNKED -l vds_dset:CHUNK=3x6x9 +fi + +if test $USE_FILTER_DEFLATE != "yes" ; then + SKIP vds_compa 4_vds.h5 +else + VERIFY_LAYOUT_VDS vds_compa 4_vds.h5 vds_dset COMPACT -l vds_dset:COMPA +fi + +if test $USE_FILTER_DEFLATE != "yes" ; then + SKIP vds_conti 4_vds.h5 +else + VERIFY_LAYOUT_VDS vds_conti 4_vds.h5 vds_dset CONTIGUOUS -l vds_dset:CONTI +fi + +# Clean up temporary files/directories +CLEAN_TESTFILES_AND_TESTDIR + +if test $nerrors -eq 0 ; then + echo "All $TESTNAME tests passed." + exit $EXIT_SUCCESS +else + echo "$TESTNAME tests failed with $nerrors errors." + exit $EXIT_FAILURE +fi + |