path: root/tools/test/h5repack/h5repack.sh.in

#! /bin/sh
#
# Copyright by The HDF Group.
# All rights reserved.
#
# This file is part of HDF5.  The full HDF5 copyright notice, including
# terms governing use, modification, and redistribution, is contained in
# the COPYING file, which can be found at the root of the source code
# distribution tree, or in https://www.hdfgroup.org/licenses.
# If you do not have access to either file, you may request a copy from
# help@hdfgroup.org.
#
# Tests for the h5repack tool
#

srcdir=@srcdir@

USE_FILTER_SZIP="@USE_FILTER_SZIP@"
USE_FILTER_DEFLATE="@USE_FILTER_DEFLATE@"

TESTNAME=h5repack
EXIT_SUCCESS=0
EXIT_FAILURE=1

H5REPACK=../../src/h5repack/h5repack    # The tool name
H5REPACK_BIN=`pwd`/$H5REPACK            # The path of the tool binary

H5DIFF=../../src/h5diff/h5diff          # The h5diff tool name
H5DIFF_BIN=`pwd`/$H5DIFF                # The path of the h5diff tool binary

H5DUMP=../../src/h5dump/h5dump          # The h5dump tool name
H5DUMP_BIN=`pwd`/$H5DUMP                # The path of the h5dump tool binary

H5STAT=../../src/h5stat/h5stat          # The h5stat tool name
H5STAT_BIN=`pwd`/$H5STAT                # The path of the h5stat tool binary

RM='rm -rf'
CMP='cmp'
DIFF='diff -c'
GREP='grep'
CP='cp'
DIRNAME='dirname'
LS='ls'
AWK='awk'

# Reset the VOL connector to only use the native connector, because of the
#       error stack checking.   QAK - 2019/03/09
ENVCMD="env HDF5_VOL_CONNECTOR=native"

H5DETECTSZIP=testh5repack_detect_szip
H5DETECTSZIP_BIN=`pwd`/$H5DETECTSZIP


nerrors=0
verbose=yes

# source dirs
SRC_TOOLS="$srcdir/../.."
SRC_TOOLS_TESTFILES="$SRC_TOOLS/testfiles"

# testfiles source dirs for tools
SRC_H5LS_TESTFILES="$SRC_TOOLS_TESTFILES"
SRC_H5DUMP_TESTFILES="$SRC_TOOLS_TESTFILES"
SRC_H5DIFF_TESTFILES="$SRC_TOOLS/test/h5diff/testfiles"
SRC_H5COPY_TESTFILES="$SRC_TOOLS/test/h5copy/testfiles"
SRC_H5REPACK_TESTFILES="$SRC_TOOLS/test/h5repack/testfiles"
SRC_H5JAM_TESTFILES="$SRC_TOOLS/test/h5jam/testfiles"
SRC_H5STAT_TESTFILES="$SRC_TOOLS/test/h5stat/testfiles"
SRC_H5IMPORT_TESTFILES="$SRC_TOOLS/test/h5import/testfiles"

TESTDIR=./testpack
test -d $TESTDIR || mkdir $TESTDIR

######################################################################
# test files
# --------------------------------------------------------------------
# All the test files copy from source directory to test directory
# NOTE: Keep this framework to add/remove test files.
#       Any test files from other tools can be used in this framework.
#       This list are also used for checking exist.
#       Comment '#' without space can be used.
# --------------------------------------------------------------------
LIST_HDF5_TEST_FILES="
$SRC_H5REPACK_TESTFILES/bounds_latest_latest.h5
$SRC_H5REPACK_TESTFILES/h5repack_attr.h5
$SRC_H5REPACK_TESTFILES/h5repack_attr_refs.h5
$SRC_H5REPACK_TESTFILES/h5repack_deflate.h5
$SRC_H5REPACK_TESTFILES/h5repack_early.h5
$SRC_H5REPACK_TESTFILES/h5repack_ext.h5
$SRC_H5REPACK_TESTFILES/h5repack_f32le.h5
$SRC_H5REPACK_TESTFILES/h5repack_f32le_ex.h5
$SRC_H5REPACK_TESTFILES/h5repack_fill.h5
$SRC_H5REPACK_TESTFILES/h5repack_filters.h5
$SRC_H5REPACK_TESTFILES/h5repack_fletcher.h5
$SRC_H5REPACK_TESTFILES/h5repack_hlink.h5
$SRC_H5REPACK_TESTFILES/h5repack_int32le_1d.h5
$SRC_H5REPACK_TESTFILES/h5repack_int32le_1d_ex.h5
$SRC_H5REPACK_TESTFILES/h5repack_int32le_2d.h5
$SRC_H5REPACK_TESTFILES/h5repack_int32le_2d_ex.h5
$SRC_H5REPACK_TESTFILES/h5repack_int32le_3d.h5
$SRC_H5REPACK_TESTFILES/h5repack_int32le_3d_ex.h5
$SRC_H5REPACK_TESTFILES/h5repack_layout.h5
$SRC_H5REPACK_TESTFILES/h5repack_layouto.h5
$SRC_H5REPACK_TESTFILES/h5repack_layout2.h5
$SRC_H5REPACK_TESTFILES/h5repack_layout3.h5
$SRC_H5REPACK_TESTFILES/h5repack_named_dtypes.h5
$SRC_H5REPACK_TESTFILES/h5repack_nested_8bit_enum.h5
$SRC_H5REPACK_TESTFILES/h5repack_nested_8bit_enum_deflated.h5
$SRC_H5REPACK_TESTFILES/h5repack_nbit.h5
$SRC_H5REPACK_TESTFILES/h5repack_objs.h5
$SRC_H5REPACK_TESTFILES/h5repack_refs.h5
$SRC_H5REPACK_TESTFILES/h5repack_shuffle.h5
$SRC_H5REPACK_TESTFILES/h5repack_soffset.h5
$SRC_H5REPACK_TESTFILES/h5repack_szip.h5
$SRC_H5REPACK_TESTFILES/h5repack_uint8be.h5
$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex.h5
########fsm#files########
$SRC_H5REPACK_TESTFILES/h5repack_aggr.h5
$SRC_H5REPACK_TESTFILES/h5repack_fsm_aggr_nopersist.h5
$SRC_H5REPACK_TESTFILES/h5repack_fsm_aggr_persist.h5
$SRC_H5REPACK_TESTFILES/h5repack_none.h5
$SRC_H5REPACK_TESTFILES/h5repack_paged_nopersist.h5
$SRC_H5REPACK_TESTFILES/h5repack_paged_persist.h5
########h5diff/testfile########
$SRC_H5DIFF_TESTFILES/h5diff_attr1.h5
########test#HDFFV-10590########
$SRC_H5REPACK_TESTFILES/h5repack_CVE-2018-17432.h5
$SRC_H5REPACK_TESTFILES/h5repack_CVE-2018-14460.h5
########tools/testfiles#for#external#links########
$SRC_TOOLS_TESTFILES/tsoftlinks.h5
$SRC_TOOLS_TESTFILES/textlinkfar.h5
$SRC_TOOLS_TESTFILES/textlinksrc.h5
$SRC_TOOLS_TESTFILES/textlinktar.h5
$SRC_TOOLS_TESTFILES/textlink.h5
$SRC_H5REPACK_TESTFILES/h5copy_extlinks_src.h5
$SRC_H5REPACK_TESTFILES/h5copy_extlinks_trg.h5
########tools/testfiles########
$SRC_TOOLS_TESTFILES/tfamily00000.h5
$SRC_TOOLS_TESTFILES/tfamily00001.h5
$SRC_TOOLS_TESTFILES/tfamily00002.h5
$SRC_TOOLS_TESTFILES/tfamily00003.h5
$SRC_TOOLS_TESTFILES/tfamily00004.h5
$SRC_TOOLS_TESTFILES/tfamily00005.h5
$SRC_TOOLS_TESTFILES/tfamily00006.h5
$SRC_TOOLS_TESTFILES/tfamily00007.h5
$SRC_TOOLS_TESTFILES/tfamily00008.h5
$SRC_TOOLS_TESTFILES/tfamily00009.h5
$SRC_TOOLS_TESTFILES/tfamily00010.h5
$SRC_TOOLS_TESTFILES/tordergr.h5
########reference#conversion#files########
########tools/testfiles/vds########
$SRC_TOOLS_TESTFILES/vds/1_a.h5
$SRC_TOOLS_TESTFILES/vds/1_b.h5
$SRC_TOOLS_TESTFILES/vds/1_c.h5
$SRC_TOOLS_TESTFILES/vds/1_d.h5
$SRC_TOOLS_TESTFILES/vds/1_e.h5
$SRC_TOOLS_TESTFILES/vds/1_f.h5
$SRC_TOOLS_TESTFILES/vds/1_vds.h5
$SRC_TOOLS_TESTFILES/vds/2_a.h5
$SRC_TOOLS_TESTFILES/vds/2_b.h5
$SRC_TOOLS_TESTFILES/vds/2_c.h5
$SRC_TOOLS_TESTFILES/vds/2_d.h5
$SRC_TOOLS_TESTFILES/vds/2_e.h5
$SRC_TOOLS_TESTFILES/vds/2_vds.h5
$SRC_TOOLS_TESTFILES/vds/3_1_vds.h5
$SRC_TOOLS_TESTFILES/vds/3_2_vds.h5
$SRC_TOOLS_TESTFILES/vds/4_0.h5
$SRC_TOOLS_TESTFILES/vds/4_1.h5
$SRC_TOOLS_TESTFILES/vds/4_2.h5
$SRC_TOOLS_TESTFILES/vds/4_vds.h5
$SRC_TOOLS_TESTFILES/vds/5_a.h5
$SRC_TOOLS_TESTFILES/vds/5_b.h5
$SRC_TOOLS_TESTFILES/vds/5_c.h5
$SRC_TOOLS_TESTFILES/vds/5_vds.h5
"

LIST_OTHER_TEST_FILES="
$SRC_H5REPACK_TESTFILES/h5repack-help.txt
$SRC_H5REPACK_TESTFILES/h5repack_ext.bin
$SRC_H5REPACK_TESTFILES/ublock.bin
$SRC_H5REPACK_TESTFILES/h5repack.info
########dat#files########
$SRC_H5REPACK_TESTFILES/h5repack_f32le_ex-0.dat
$SRC_H5REPACK_TESTFILES/h5repack_int32le_1d_ex-0.dat
$SRC_H5REPACK_TESTFILES/h5repack_int32le_1d_ex-1.dat
$SRC_H5REPACK_TESTFILES/h5repack_int32le_2d_ex-0.dat
$SRC_H5REPACK_TESTFILES/h5repack_int32le_3d_ex-0.dat
$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-0.dat
$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-1.dat
$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-2.dat
$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-3.dat
###############
$SRC_H5REPACK_TESTFILES/crtorder.tordergr.h5.ddl
$SRC_H5REPACK_TESTFILES/deflate_limit.h5repack_layout.h5.ddl
$SRC_H5REPACK_TESTFILES/h5repack_layout.h5.ddl
$SRC_H5REPACK_TESTFILES/h5repack_layout.h5-plugin_test.ddl
########fsm#files########
$SRC_H5REPACK_TESTFILES/SP.h5repack_fsm_aggr_nopersist.h5.ddl
$SRC_H5REPACK_TESTFILES/S.h5repack_fsm_aggr_persist.h5.ddl
$SRC_H5REPACK_TESTFILES/STG.h5repack_none.h5.ddl
$SRC_H5REPACK_TESTFILES/GS.h5repack_paged_nopersist.h5.ddl
$SRC_H5REPACK_TESTFILES/SP.h5repack_paged_persist.h5.ddl
$SRC_H5REPACK_TESTFILES/SPT.h5repack_aggr.h5.ddl
########vds#files########
$SRC_H5REPACK_TESTFILES/1_vds.h5-vds_dset_chunk20x10x5-v.ddl
$SRC_H5REPACK_TESTFILES/2_vds.h5-vds_chunk3x6x9-v.ddl
$SRC_H5REPACK_TESTFILES/3_1_vds.h5-vds_chunk2x5x8-v.ddl
$SRC_H5REPACK_TESTFILES/4_vds.h5-vds_compa-v.ddl
$SRC_H5REPACK_TESTFILES/4_vds.h5-vds_conti-v.ddl
########refs#files########
$SRC_H5REPACK_TESTFILES/attrregion.tattrreg.h5.ddl
$SRC_H5REPACK_TESTFILES/dataregion.tdatareg.h5.ddl
########external#links#files########
$SRC_H5REPACK_TESTFILES/textlinkfar-base.textlinkfar.h5.ddl
$SRC_H5REPACK_TESTFILES/textlinksrc-base.textlinksrc.h5.ddl
$SRC_H5REPACK_TESTFILES/textlinktar-base.textlinktar.h5.ddl
$SRC_H5REPACK_TESTFILES/textlink-base.textlink.h5.ddl
$SRC_H5REPACK_TESTFILES/tsoftlinks-base.tsoftlinks.h5.ddl
$SRC_H5REPACK_TESTFILES/h5copy_extlinks_src-base.h5copy_extlinks_src.h5.ddl
$SRC_H5REPACK_TESTFILES/textlinkfar-merge.textlinkfar.h5.tst
$SRC_H5REPACK_TESTFILES/textlinksrc-merge.textlinksrc.h5.tst
$SRC_H5REPACK_TESTFILES/textlinktar-merge.textlinktar.h5.tst
$SRC_H5REPACK_TESTFILES/textlink-merge.textlink.h5.tst
$SRC_H5REPACK_TESTFILES/tsoftlinks-merge.tsoftlinks.h5.tst
$SRC_H5REPACK_TESTFILES/h5copy_extlinks_src-merge.h5copy_extlinks_src.h5.tst
$SRC_H5REPACK_TESTFILES/textlinkfar-prune.textlinkfar.h5.ddl
$SRC_H5REPACK_TESTFILES/textlinksrc-prune.textlinksrc.h5.ddl
$SRC_H5REPACK_TESTFILES/textlinktar-prune.textlinktar.h5.ddl
$SRC_H5REPACK_TESTFILES/textlink-prune.textlink.h5.ddl
$SRC_H5REPACK_TESTFILES/tsoftlinks-prune.tsoftlinks.h5.ddl
$SRC_H5REPACK_TESTFILES/h5copy_extlinks_src-prune.h5copy_extlinks_src.h5.ddl
$SRC_H5REPACK_TESTFILES/textlinkfar-mergeprune.textlinkfar.h5.ddl
$SRC_H5REPACK_TESTFILES/textlinksrc-mergeprune.textlinksrc.h5.ddl
$SRC_H5REPACK_TESTFILES/textlinktar-mergeprune.textlinktar.h5.ddl
$SRC_H5REPACK_TESTFILES/textlink-mergeprune.textlink.h5.ddl
$SRC_H5REPACK_TESTFILES/tsoftlinks-mergeprune.tsoftlinks.h5.ddl
$SRC_H5REPACK_TESTFILES/h5copy_extlinks_src-mergeprune.h5copy_extlinks_src.h5.ddl
########tst#files########
$SRC_H5REPACK_TESTFILES/h5repack_filters.h5-gzip_verbose_filters.tst
$SRC_H5REPACK_TESTFILES/h5repack_layout.h5-dset2_chunk_20x10-errstk.tst
$SRC_H5REPACK_TESTFILES/plugin_test.h5repack_layout.h5.tst
########external#links#tst#files########
$SRC_H5REPACK_TESTFILES/tsoftlinks-merge.tsoftlinks.h5.tst
$SRC_H5REPACK_TESTFILES/textlinkfar-merge.textlinkfar.h5.tst
$SRC_H5REPACK_TESTFILES/textlinksrc-merge.textlinksrc.h5.tst
$SRC_H5REPACK_TESTFILES/textlinktar-merge.textlinktar.h5.tst
$SRC_H5REPACK_TESTFILES/textlink-merge.textlink.h5.tst
$SRC_H5REPACK_TESTFILES/h5copy_extlinks_src-merge.h5copy_extlinks_src.h5.tst
"

#
# copy test files and expected output files from source dirs to test dir
#
COPY_TESTFILES="$LIST_HDF5_TEST_FILES $LIST_OTHER_TEST_FILES"

COPY_TESTFILES_TO_TESTDIR()
{
    # copy test files. Used -f to make sure get a new copy
    for tstfile in $COPY_TESTFILES
    do
        # ignore '#' comment
        echo $tstfile | tr -d ' ' | grep '^#' > /dev/null
        RET=$?
        if [ $RET -eq 1 ]; then
            # skip cp if srcdir is same as destdir
            # this occurs when build/test performed in source dir and
            # make cp fail
            SDIR=`$DIRNAME $tstfile`
            INODE_SDIR=`$LS -i -d $SDIR | $AWK -F' ' '{print $1}'`
            INODE_DDIR=`$LS -i -d $TESTDIR | $AWK -F' ' '{print $1}'`
            if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then
                $CP -f $tstfile $TESTDIR
                if [ $? -ne 0 ]; then
                    echo "Error: FAILED to copy $tstfile ."

                    # Comment out this to CREATE expected file
                    exit $EXIT_FAILURE
                fi
            fi
        fi
    done
}

CLEAN_TESTFILES_AND_TESTDIR()
{
    # skip rm if srcdir is same as destdir
    # this occurs when build/test performed in source dir and
    # make cp fail
    SDIR=$SRC_H5REPACK_TESTFILES
    INODE_SDIR=`$LS -i -d $SDIR | $AWK -F' ' '{print $1}'`
    INODE_DDIR=`$LS -i -d $TESTDIR | $AWK -F' ' '{print $1}'`
    if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then
        $RM $TESTDIR
    fi
}

# Print a $* message left justified in a field of 70 characters
#
MESSAGE() {
   SPACES="                                                               "
   echo "$* $SPACES" | cut -c1-70 | tr -d '\012'
}

# Print a line-line message left justified in a field of 70 characters
# beginning with the word "Testing".
#
TESTING() {
    MESSAGE "Testing $*"
}

# Print a line-line message left justified in a field of 70 characters
# beginning with the word "Verifying".
#
VERIFY() {
    MESSAGE "Verifying $*"
}

# Print a message that a test has been skipped (because a required filter
# was unavailable)
SKIP() {
    TESTING $H5REPACK $@
    echo " -SKIP-"
}

##############################################################################
##############################################################################
###           T H E   T E S T S  M A C R O S                               ###
##############################################################################
##############################################################################

# Call the h5diff tool
#
DIFFTEST()
{
    VERIFY h5diff output $@
    (
        cd $TESTDIR
        $RUNSERIAL $H5DIFF_BIN -q  "$@"
    )
    RET=$?
    if [ $RET != 0 ] ; then
         echo "*FAILED*"
         nerrors="`expr $nerrors + 1`"
    else
         echo " PASSED"
    fi

}

# Call h5repack
#


# call TOOLTEST_MAIN and delete $output file
TOOLTEST()
{
    echo $@
    infile=$2
    outfile=out-$1.$2
    shift
    shift

    # Run test.
    TESTING $H5REPACK $@
    (
        cd $TESTDIR
        $ENVCMD $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
    )
    RET=$?
    if [ $RET != 0 ] ; then
        echo "*FAILED*"
        nerrors="`expr $nerrors + 1`"
    else
        echo " PASSED"
        DIFFTEST $infile $outfile
    fi
    rm -f $outfile
}

#------------------------------------------
# Verifying layouts of a dataset
VERIFY_LAYOUT_DSET()
{
    layoutfile=layout-$1.$2
    dset=$3
    expectlayout=$4
    infile=$2
    outfile=out-$1.$2
    shift
    shift
    shift
    shift

    TESTING  $H5REPACK $@
    (
        cd $TESTDIR
        $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
    )
    RET=$?
    if [ $RET != 0 ] ; then
        echo "*FAILED*"
        nerrors="`expr $nerrors + 1`"
    else
        echo " PASSED"
        DIFFTEST $infile $outfile
    fi

    #---------------------------------
    # check the layout from a dataset
    VERIFY  "a dataset layout"
    (
        cd $TESTDIR
        $RUNSERIAL $H5DUMP_BIN -d $dset -pH $outfile > $layoutfile
    )
    $GREP $expectlayout $TESTDIR/$layoutfile > /dev/null
    if [ $? -eq 0 ]; then
        echo " PASSED"
    else
        echo " FAILED"
        nerrors="`expr $nerrors + 1`"
    fi

    # clean up tmp files
#    rm -f $outfile
#    rm -f $layoutfile
}

#------------------------------------------
# Verifying layouts of a dataset
VERIFY_LAYOUT_VDS()
{
    layoutfile=layout-$1.$2
    dset=$3
    expectlayout=$4
    infile=$2
    outfile=out-$1.$2

    expect="$TESTDIR/$2-$1-v.ddl"
    actual="$TESTDIR/$2-$1-v.out"
    actual_err="$TESTDIR/$2-$1-v.err"

    shift
    shift
    shift
    shift

    TESTING  $H5REPACK $@
    (
        cd $TESTDIR
        $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
    )
    RET=$?
    if [ $RET != 0 ] ; then
        echo "*FAILED*"
        nerrors="`expr $nerrors + 1`"
    else
        echo " PASSED"
    fi

    #---------------------------------
    # check the layout from a dataset
    VERIFY  "a dataset layout"
    (
        cd $TESTDIR
        $RUNSERIAL $H5DUMP_BIN -d $dset -p $outfile
    ) >$actual 2>$actual_err

    cat $actual_err >> $actual

    if [ ! -f $expect ]; then
    # Create the expect file if it doesn't yet exist.
     echo " CREATED"
     cp $actual $expect
     echo "    Expected result (*.ddl) missing"
     nerrors="`expr $nerrors + 1`"
    elif $CMP $expect $actual > /dev/null 2>&1 ; then
     echo " PASSED"
    else
     echo "*FAILED*"
     echo "    Expected result (*.ddl) differs from actual result (*.out)"
     nerrors="`expr $nerrors + 1`"
     test yes = "$verbose" && $DIFF $caseless $expect $actual |sed 's/^/    /'
    fi

    # Clean up output file
    if test -z "$HDF5_NOCLEANUP"; then
    rm -f $actual $actual_err
    fi

    # clean up tmp files
    rm -f $outfile
}

#----------------------------------------
# Verifying layouts from entire file
VERIFY_LAYOUT_ALL()
{
    infile=$2
    outfile=out-$1.$2
    layoutfile=layout-$1.$2
    expectlayout=$3
    shift
    shift
    shift

    TESTING  $H5REPACK $@
    (
        cd $TESTDIR
        $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
    )
    RET=$?
    if [ $RET != 0 ] ; then
        echo "*FAILED*"
        nerrors="`expr $nerrors + 1`"
    else
        echo " PASSED"
        DIFFTEST $infile $outfile
    fi


    #---------------------------------
    # check the layout from a dataset
    # check if the other layouts still exist
    VERIFY  "layouts"
    (
        cd $TESTDIR
        echo
        # if CONTIGUOUS
        if [ $expectlayout = "CONTIGUOUS" ]; then
            TESTING $H5DUMP_BIN -pH $outfile
            (
                $RUNSERIAL $H5DUMP_BIN -pH $outfile > $layoutfile
            )
            $GREP "COMPACT" $layoutfile  > /dev/null
            if [ $? -eq 0 ]; then
                echo " FAILED"
                nerrors="`expr $nerrors + 1`"
            else
                $GREP "CHUNKED" $layoutfile  > /dev/null
                if [ $? -eq 0 ]; then
                    echo " FAILED"
                    nerrors="`expr $nerrors + 1`"
                else
                    echo " PASSED"
                fi
            fi
        else
            # if COMPACT
            if [ $expectlayout = "COMPACT" ]; then
                TESTING $H5DUMP_BIN -pH $outfile
                (
                    $RUNSERIAL $H5DUMP_BIN -pH $outfile > $layoutfile
                )
                $GREP "CHUNKED" $layoutfile  > /dev/null
                if [ $? -eq 0 ]; then
                    echo " FAILED"
                    nerrors="`expr $nerrors + 1`"
                else
                    $GREP "CONTIGUOUS" $layoutfile  > /dev/null
                    if [ $? -eq 0 ]; then
                        echo " FAILED"
                        nerrors="`expr $nerrors + 1`"
                    else
                        echo " PASSED"
                    fi
                fi
            else
                # if CHUNKED
                if [ $expectlayout = "CHUNKED" ]; then
                    TESTING $H5DUMP_BIN -pH $outfile
                    (
                        $RUNSERIAL $H5DUMP_BIN -pH $outfile > $layoutfile
                    )
                    $GREP "CONTIGUOUS" $layoutfile  > /dev/null
                    if [ $? -eq 0 ]; then
                        echo " FAILED"
                        nerrors="`expr $nerrors + 1`"
                    else
                        $GREP "COMPACT" $layoutfile  > /dev/null
                        if [ $? -eq 0 ]; then
                            echo " FAILED"
                            nerrors="`expr $nerrors + 1`"
                        else
                            echo " PASSED"
                        fi
                    fi
                fi
           fi
        fi
    )

    # clean up tmp files
    rm -f $outfile
    rm -f $layoutfile
}

#--------------------------------------------------------------
# Verifying superblock of a file with different version bounds
VERIFY_SUPERBLOCK()
{
    lowbound=$1
    highbound=$2
    superblock=$3
    infile=$4
    layoutfile=verbounds-low$lowbound-high$highbound.$4
    outfile=out-verbounds-low$lowbound-high$highbound.$infile
    expectedsuperblock="SUPERBLOCK_VERSION $superblock"

    shift
    shift
    shift
    shift

    TESTING  $H5REPACK $@
    (
        cd $TESTDIR
        $RUNSERIAL $H5REPACK_BIN -j $lowbound -k $highbound $infile $outfile
    )
    RET=$?
    if [ $RET != 0 ] ; then
        echo "*FAILED*"
        nerrors="`expr $nerrors + 1`"
    else
        echo " PASSED"
        DIFFTEST $infile $outfile
    fi

    #---------------------------------
    # check the layout from a dataset
    VERIFY  "superblock"
    (
        cd $TESTDIR
        $RUNSERIAL $H5DUMP_BIN -H -B $outfile > $layoutfile
    )
    $GREP "$expectedsuperblock" $TESTDIR/$layoutfile > /dev/null
    if [ $? -eq 0 ]; then
        echo " PASSED"
    else
        echo " FAILED"
        nerrors="`expr $nerrors + 1`"
    fi

    # clean up tmp files
    rm -f $outfile
    rm -f $layoutfile

} # end of VERIFY_SUPERBLOCK

#------------------------------------------------------------------
# Verifying that a latest file cannot be opened with earlier bounds
VERIFY_INVALIDBOUNDS()
{
    lowbound=$1
    highbound=$2
    infile=$3
    outfile=out-verbounds-low$lowbound-high$highbound.$infile

    #--------------------------------------
    # check for failure with invalid bounds
    TESTING  $H5REPACK $@
    (
        cd $TESTDIR
        $RUNSERIAL $H5REPACK_BIN -j $lowbound -k $highbound $infile $outfile
    )
    RET=$?
    if [ $RET != 0 ] ; then
        echo " PASSED"
    else
        echo "*FAILED*"
        nerrors="`expr $nerrors + 1`"
    fi

    # clean up tmp files
    rm -f $outfile

} # end of VERIFY_INVALIDBOUNDS

# -----------------------------------------------------------------------------
# Expect h5diff to fail
# Use only by VERIFY_EXTERNAL_CONSOLIDATION
# -----------------------------------------------------------------------------
DIFFFAIL()
{
    VERIFY h5diff unequal $@
    (
        cd $TESTDIR
        $RUNSERIAL $H5DIFF_BIN -q  "$@"
    )
    RET=$?
    if [ $RET -eq 0 ] ; then
         echo "*FAILED*"
         nerrors="`expr $nerrors + 1`"
    else
         echo " PASSED"
    fi
}

# -----------------------------------------------------------------------------
# Catchall test for repacking with external files
# Loops over all (internally-listed) cases and applies the given arguments
# to h5repack.
# Each test file family must be listed in the for loop at the start of the
# 'function'.
# Expects case files to be of format:
# + h5repack_<NAME>.h5
# + h5repack_<NAME>_ex.h5
# + h5repack_<NAME>_ex-<N>.dat
# ...where NAME is part of the internal list, and N is a positive decimal
# number; multiple external files (*.dat) are allowed per file, but they must
# follow the pattern and be in contiguous numerical sequence starting at 0.
# -----------------------------------------------------------------------------
VERIFY_EXTERNAL_CONSOLIDATION()
{
    for name in f32le int32le_1d int32le_2d int32le_3d uint8be
    do
        basename="h5repack_${name}"
        base_ext="${basename}_ex"
        base_rpkg="${basename}_rp"
        canonical="${basename}.h5"
        external="${base_ext}.h5"
        ex_dat="${base_ext}-0.dat"
        ex_dat_mv="__${ext_dat}"
        repacked="${base_rpkg}.h5"
        prev_nerrors=$nerrors

        TESTING external consolidation on $name with h5repack $@
        echo " . . ."

        # baseline comparison of known files
        DIFFTEST $canonical $external

        # repack the external file, verify integrity
        (   cd $TESTDIR
            $H5REPACK_BIN $@ $external $repacked
        )
        DIFFTEST $repacked $canonical
        DIFFTEST $repacked $external

        # invalidate external file by moving its first data file; verify
        (   cd $TESTDIR
            mv $ex_dat $ex_dat_mv
        )
        DIFFFAIL $canonical $external
        DIFFFAIL $repacked  $external
        DIFFTEST $canonical $repacked

        # reset test state for re-runs with different h5repack arguments
        (   cd $TESTDIR
            mv $ex_dat_mv $ex_dat
            rm $repacked
        )

        # flag if any of the above checks failed
        if [ $prev_nerrors != $nerrors ]
        then
             echo ". . . *FAILED*"
             nerrors="`expr $prev_nerrors + 1`"
        else
             echo ". . . PASSED"
        fi
    done
} # end VERIFY_EXTERNAL_CONSOLIDATION

# -----------------------------------------------------------------------------
# same as TOOLTEST, but it uses the old syntax -i input_file -o output_file
# -----------------------------------------------------------------------------
TOOLTEST0()
{
    infile=$2
    outfile=out-$1.$2
    shift
    shift

    # Run test.
    TESTING $H5REPACK $@
    (
        cd $TESTDIR
        $RUNSERIAL $H5REPACK_BIN -i $infile -o $outfile "$@"
    )
    RET=$?
    if [ $RET != 0 ] ; then
        echo "*FAILED*"
        nerrors="`expr $nerrors + 1`"
    else
        echo " PASSED"
        DIFFTEST $infile $outfile
    fi
    rm -f $outfile
}


# same as TOOLTEST, but it uses without -i -o options
# used to test the family driver, where these files reside
#
TOOLTEST1()
{
    infile=$2
    outfile=out-$1.$2
    shift
    shift

    # Run test.
    TESTING $H5REPACK $@
    (
        cd $TESTDIR
        $ENVCMD $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
    )
    RET=$?
    if [ $RET != 0 ] ; then
    echo "*FAILED*"
    nerrors="`expr $nerrors + 1`"
    else
    echo " PASSED"
    DIFFTEST $infile $outfile
    fi
    rm -f $outfile
}

# This is same as TOOLTEST() with comparing display output
# from -v option
#
TOOLTESTV()
{
    expect="$TESTDIR/$2-$1.tst"
    actual="$TESTDIR/`basename $2 .ddl`.out"
    actual_err="$TESTDIR/`basename $2 .ddl`.err"

    infile=$2
    outfile=out-$1.$2
    shift
    shift

    # Run test.
    TESTING $H5REPACK $@
    (
        cd $TESTDIR
        $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
    ) >$actual 2>$actual_err
    RET=$?
    if [ $RET != 0 ] ; then
    echo "*FAILED*"
    nerrors="`expr $nerrors + 1`"
    else
    echo " PASSED"
    DIFFTEST $infile $outfile
    fi

    # display output compare
    STDOUT_FILTER $actual
    cat $actual_err >> $actual

    VERIFY output from $H5REPACK $@
    if cmp -s $expect $actual; then
     echo " PASSED"
    else
     echo "*FAILED*"
     echo "    Expected result (*.tst) differs from actual result (*.out)"
     nerrors="`expr $nerrors + 1`"
     test yes = "$verbose" && diff -c $expect $actual |sed 's/^/    /'
    fi

   rm -f $actual $actual_err
   rm -f $outfile
}

# Same as TOOLTEST, but expects h5repack fails
#
TOOLTEST_FAIL()
{
    infile=$1
    outfile=$2
    expect="$TESTDIR/$2-$1.ddl"
    actual="$TESTDIR/$2-$1.out"
    actual_err="$TESTDIR/$2-$1.err"
    shift
    shift

    # Run test.
    TESTING $H5REPACK $@
    (
        cd $TESTDIR
        $ENVCMD $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
    ) >&$actual
    RET=$?

    # Normally h5repack of files tested with this function are expected
    # to return not 0, but if the command results in "Segmentation fault"
    # or "core dumped" it is a failure regardless of the return value.
    failure=`grep -e 'Segmentation fault' -e 'core dumped' $actual`
    if [ "$failure" != "" ]; then
        nerrors="`expr $nerrors + 1`"
        echo " FAILED"
        echo "    $failure"
    else
        if [ $RET == 0 ] ; then
            nerrors="`expr $nerrors + 1`"
            echo " FAILED"
        else
            echo " PASSED"
        fi
    fi
    rm -f $outfile
}

# This is same as TOOLTESTV() with comparing display output
# with actual filename swapped
#
TOOLTESTSV()
{
    expect="$TESTDIR/$1.$2.tst"
    actual="$TESTDIR/`basename $2 .ddl`.out"
    actual_err="$TESTDIR/`basename $2 .ddl`.err"

    infile=$2
    outfile=out-$1.$2
    shift
    shift

    # Run test.
    TESTING $H5REPACK $@
    (
        cd $TESTDIR
        $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
    ) >$actual 2>$actual_err
    RET=$?
    if [ $RET != 0 ] ; then
    echo "*FAILED*"
    nerrors="`expr $nerrors + 1`"
    else
    echo " PASSED"
    VERIFY h5diff equal $@
    (
        cd $TESTDIR
        $RUNSERIAL $H5DIFF_BIN -v $infile $outfile
    ) >$actual 2>$actual_err
    RET=$?
    if [ $RET != 0 ] ; then
         echo "*FAILED*"
         nerrors="`expr $nerrors + 1`"
    else
         echo " PASSED"
    fi
    fi

    # display output compare
    STDOUT_FILTER $actual
    cat $actual_err >> $actual

    VERIFY output from $H5REPACK $@
    if cmp -s $expect $actual; then
     echo " PASSED"
    else
     echo "*FAILED*"
     echo "    Expected result (*.tst) differs from actual result (*.out)"
     nerrors="`expr $nerrors + 1`"
     test yes = "$verbose" && diff -c $expect $actual |sed 's/^/    /'
    fi

   rm -f $actual $actual_err
   rm -f $outfile
}

# This is same as TOOLTESTSV() but expects a diff fail
#
TOOLFAILSV()
{
    expect="$TESTDIR/$1.$2.tst"
    actual="$TESTDIR/`basename $2 .ddl`.out"
    actual_err="$TESTDIR/`basename $2 .ddl`.err"

    infile=$2
    outfile=out-$1.$2
    shift
    shift

    # Run test.
    TESTING $H5REPACK $@
    (
        cd $TESTDIR
        $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
    ) >$actual 2>$actual_err
    RET=$?
    if [ $RET != 0 ] ; then
    echo "*FAILED*"
    nerrors="`expr $nerrors + 1`"
    else
    echo " PASSED"
    VERIFY h5diff unequal $@
    (
        cd $TESTDIR
        $RUNSERIAL $H5DIFF_BIN -v $infile $outfile
    ) >$actual 2>$actual_err
    RET=$?
    if [ $RET == 0 ] ; then
         echo "*FAILED*"
         nerrors="`expr $nerrors + 1`"
    else
         echo " PASSED"
    fi
    fi

    # display output compare
    STDOUT_FILTER $actual
    cat $actual_err >> $actual

    VERIFY output from $H5REPACK $@
    if cmp -s $expect $actual; then
     echo " PASSED"
    else
     echo "*FAILED*"
     echo "    Expected result (*.tst) differs from actual result (*.out)"
     nerrors="`expr $nerrors + 1`"
     test yes = "$verbose" && diff -c $expect $actual |sed 's/^/    /'
    fi

   rm -f $actual $actual_err
   rm -f $outfile
}

# same as TOOLTESTV but filters error stack output and compares to an error file
# Extract file name, line number, version and thread IDs because they may be different
# ADD_H5ERR_MASK_TEST
TOOLTESTM() {

    expect="$TESTDIR/$2-$1.tst"
    actual="$TESTDIR/`basename $2 .tst`.out"
    actual_err="$TESTDIR/`basename $2 .tst`.oerr"
    actual_sav=${actual}-sav
    actual_err_sav=${actual_err}-sav

    infile=$2
    outfile=out-$1.$2
    shift
    shift

    # Run test.
    TESTING $H5REPACK $@
    (
        cd $TESTDIR
        $ENVCMD $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
    ) >$actual 2>$actual_err

    # save actual and actual_err in case they are needed later.
    cp $actual $actual_sav
    cp $actual_err $actual_err_sav

    # Extract file name, line number, version and thread IDs because they may be different
    sed -e 's/thread [0-9]*/thread (IDs)/' -e 's/: .*\.c /: (file name) /' \
        -e 's/line [0-9]*/line (number)/' \
        -e 's/v[1-9]*\.[0-9]*\./version (number)\./' \
        -e 's/[1-9]*\.[0-9]*\.[0-9]*[^)]*/version (number)/' \
        -e 's/H5Eget_auto[1-2]*/H5Eget_auto(1 or 2)/' \
        -e 's/H5Eset_auto[1-2]*/H5Eset_auto(1 or 2)/' \
     $actual_err > $actual

    if [ ! -f $expect ]; then
    # Create the expect file if it doesn't yet exist.
     echo " CREATED"
     cp $actual $expect
     echo "    Expected result (*.tst) missing"
     nerrors="`expr $nerrors + 1`"
    elif $CMP $expect $actual; then
      echo " PASSED"
    else
     echo "*FAILED*"
     echo "    Expected result (*.tst) differs from actual result (*.out)"
     nerrors="`expr $nerrors + 1`"
     test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/    /'
    fi

    # Clean up output file
    if test -z "$HDF5_NOCLEANUP"; then
        rm -f $actual $actual_err $actual_sav $actual_err_sav
    fi

}

# This is same as TOOLTESTV() with comparing h5dump output
# from -pH option
#
TOOLTEST_DUMP()
{
    infile=$2
    outfile=out-$1.$2
    expect="$TESTDIR/$1.$2.ddl"
    actual="$TESTDIR/out-$1.$2.out"
    actual_err="$TESTDIR/out-$1.$2.err"

    shift
    shift

    # Run test.
    TESTING $H5REPACK $@
    (
        cd $TESTDIR
        $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
    ) >$actual 2>$actual_err
    RET=$?
    if [ $RET != 0 ] ; then
    echo "*FAILED*"
    nerrors="`expr $nerrors + 1`"
    else
     echo " PASSED"
     VERIFY h5dump output $@
     (
        cd $TESTDIR
        $RUNSERIAL $H5DUMP_BIN -q creation_order -pH $outfile
     ) >$actual 2>$actual_err
     cat $actual_err >> $actual

     RET=$?

    fi

    if cmp -s $expect $actual; then
     echo " PASSED"
    else
     echo "*FAILED*"
     echo "    Expected result (*.ddl) differs from actual result (*.out)"
     nerrors="`expr $nerrors + 1`"
     test yes = "$verbose" && diff -c $expect $actual |sed 's/^/    /'
    fi

   rm -f $actual $actual_err
   rm -f $outfile
}

# This is same as TOOLTEST_DUMP() with comparing h5dump output
# without any option
#
TOOLTEST_DUMP_NO_OPT()
{
    infile=$2
    outfile=out-$1.$2
    expect="$TESTDIR/$1.$2.ddl"
    actual="$TESTDIR/out-$1.$2.out"
    actual_err="$TESTDIR/out-$1.$2.err"

    shift
    shift

    # Run test.
    TESTING $H5REPACK $@
    (
        cd $TESTDIR
        $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
    ) >$actual 2>$actual_err
    RET=$?
    if [ $RET != 0 ] ; then
    echo "*FAILED*"
    nerrors="`expr $nerrors + 1`"
    else
     echo " PASSED"
     VERIFY h5dump output $@
     (
        cd $TESTDIR
        $RUNSERIAL $H5DUMP_BIN $outfile
     ) >$actual 2>$actual_err
     cat $actual_err >> $actual

     RET=$?

    fi

    if cmp -s $expect $actual; then
     echo " PASSED"
    else
     echo "*FAILED*"
     echo "    Expected result (*.ddl) differs from actual result (*.out)"
     nerrors="`expr $nerrors + 1`"
     test yes = "$verbose" && diff -c $expect $actual |sed 's/^/    /'
    fi

   rm -f $actual $actual_err
   rm -f $outfile
}

# This is similar to TOOLTEST_DUMP().
# Test h5repack with options added for paged aggregation.
# h5stat is used on the repacked file and the expected output
# is compared for correctness.
#
TOOLTEST_STAT()
{
    infile=$2
    outfile=out-$1.$2
    expect="$TESTDIR/$1.$2.ddl"
    actual="$TESTDIR/out-$1.$2.out"
    actual_err="$TESTDIR/out-$1.$2.err"

    shift
    shift

    # Run test.
    TESTING $H5REPACK $@
    (
        cd $TESTDIR
        $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
    ) >$actual 2>$actual_err
    RET=$?
    if [ $RET != 0 ] ; then
    echo "*FAILED*"
    nerrors="`expr $nerrors + 1`"
    else
     echo " PASSED"
     VERIFY h5stat output $@
     (
        cd $TESTDIR
        $RUNSERIAL $H5STAT_BIN -S -s $outfile
     ) >$actual 2>$actual_err
     cat $actual_err >> $actual

     RET=$?

    fi

    if cmp -s $expect $actual; then
     echo " PASSED"
    else
     echo "*FAILED*"
     echo "    Expected result (*.ddl) differs from actual result (*.out)"
     nerrors="`expr $nerrors + 1`"
     test yes = "$verbose" && diff -c $expect $actual |sed 's/^/    /'
    fi

   rm -f $actual $actual_err
   rm -f $outfile
}

# TOOLTEST_META:
# Test metadata block size option.
# Reason to create a function here is to localize all special steps related to
# metadata block size option in one place.  This is a quick solution.  More
# thought out solution needed when there is more time.
#
# $1: test input file
# $2:$: metadata options (either "-M size" or "--metadata_block_size=size")
#
# Algorithm:
# Run it once without the metadata option ($2-$);
# Save the result output file;
# Run it second time with the metadata option;
# Verify the output file of second run is larger than the one of 1st run.
TOOLTEST_META()
{
    infile=$2
    outfile=out-$1.$2
    shift
    shift

    # Run test.
    TESTING $H5REPACK $@
    (
        cd $TESTDIR
        $RUNSERIAL $H5REPACK_BIN $infile $outfile
    )
    RET=$?
    if [ $RET != 0 ] ; then
        echo "*FAILED*"
        nerrors="`expr $nerrors + 1`"
    else
        echo " PASSED"
        DIFFTEST $infile $outfile
    fi
    # get the size of the first output file
    size1=`wc -c $TESTDIR/$outfile | cut -d' ' -f1`

    # 2nd run with metadata option
    # Run test.
    TESTING $H5REPACK $@
    (
        cd $TESTDIR
        $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
    )
    RET=$?
    if [ $RET != 0 ] ; then
        echo "*FAILED*"
        nerrors="`expr $nerrors + 1`"
    else
        echo " PASSED"
        DIFFTEST $infile $outfile
    fi
    # get the size of the second output file
    size2=`wc -c $TESTDIR/$outfile | cut -d' ' -f1`

    # verify sizes.
    MESSAGE "Verify the sizes of both output files ($size1 vs $size2)"
    if [ $size1 -lt $size2 ]; then
        # pass
    echo " PASSED"
    else
    #fail
    echo "*FAILED*"
        nerrors="`expr $nerrors + 1`"
    fi

    rm -f $outfile
}

# ADD_HELP_TEST
TOOLTEST_HELP() {

    expect="$TESTDIR/$1"
    actual="$TESTDIR/`basename $1 .txt`.out"
    actual_err="$TESTDIR/`basename $1 .txt`.err"
    shift

    # Run test.
    TESTING $H5REPACK $@
    (
      cd $TESTDIR
      $RUNSERIAL $H5REPACK_BIN "$@"
    ) >$actual 2>$actual_err
    cat $actual_err >> $actual

    if [ ! -f $expect ]; then
    # Create the expect data file if it doesn't yet exist.
      echo " CREATED"
      cp $actual $expect-CREATED
      echo "    Expected result (*.txt) missing"
      nerrors="`expr $nerrors + 1`"
    elif cmp -s $expect $actual; then
      echo " PASSED"
    else
      echo "*FAILED*"
      echo "    Expected output (*.txt) differs from actual output (*.out)"
      nerrors="`expr $nerrors + 1`"
    fi

    # Clean up output file
    if test -z "$HDF5_NOCLEANUP"; then
     rm -f $actual $actual_err
    fi

}

# This is different from $srcdir/../../../bin/output_filter.sh
STDOUT_FILTER() {
    result_file=$1
    tmp_file=/tmp/h5test_tmp_$$
    # Filter name of files.
    cp $result_file $tmp_file
    sed -e '/^Opening file/d' -e '/^Making file/d' \
    < $tmp_file > $result_file
    # cleanup
    rm -f $tmp_file
}

#
# The tests
# We use the files generated by h5repacktst
# Each run generates "<file>.out.h5" and the tool h5diff is used to
# compare the input and output files
#
# the tests are the same as the program h5repacktst, but run from the CLI
#

# See which filters are usable (and skip tests for filters we
# don't have).  Do this by searching H5pubconf.h to see which
# filters are defined.

# detect whether the encoder is present.
USE_FILTER_SZIP_ENCODER="no";
if test $USE_FILTER_SZIP = "yes"; then
USE_FILTER_SZIP_ENCODER=`$RUNSERIAL $H5DETECTSZIP_BIN`
fi

##############################################################################
###              T H E   T E S T S
##############################################################################

# prepare for test
COPY_TESTFILES_TO_TESTDIR

TOOLTEST_HELP h5repack-help.txt -h

# copy files (these files have no filters)
TOOLTEST fill h5repack_fill.h5
TOOLTEST objs h5repack_objs.h5
TOOLTEST attr h5repack_attr.h5
TOOLTEST hlink h5repack_hlink.h5
TOOLTEST layout h5repack_layout.h5
TOOLTEST early h5repack_early.h5

# nested 8bit enum in both deflated and non-deflated datafiles
if [ $USE_FILTER_DEFLATE != "yes" ]; then
TOOLTEST nested_8bit_enum h5repack_nested_8bit_enum.h5
else
TOOLTEST nested_8bit_enum h5repack_nested_8bit_enum_deflated.h5
fi

# use h5repack_layout.h5 to write some filters  (this file has  no filters)

# gzip with individual object
arg="h5repack_layout.h5 -f dset1:GZIP=1  -l dset1:CHUNK=20x10"
if test  $USE_FILTER_DEFLATE != "yes" ; then
 SKIP $arg
else
 TOOLTEST gzip_individual $arg
fi

# gzip for all
arg="h5repack_layout.h5 -f GZIP=1"
if test  $USE_FILTER_DEFLATE != "yes" ; then
 SKIP $arg
else
 TOOLTEST gzip_all $arg
fi

# szip with individual object
arg="h5repack_layout.h5 -f dset2:SZIP=8,EC  -l dset2:CHUNK=20x10"
if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then
 SKIP $arg
else
 TOOLTEST szip_individual $arg
fi

# szip for all
arg="h5repack_layout.h5 -f SZIP=8,NN"
if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then
 SKIP $arg
else
 TOOLTEST szip_all $arg
fi

# shuffle with individual object
arg="h5repack_layout.h5 -f dset2:SHUF  -l dset2:CHUNK=20x10"
TOOLTEST shuffle_individual $arg


# shuffle for all
arg="h5repack_layout.h5 -f SHUF"
TOOLTEST shuffle_all $arg

# fletcher32  with individual object
arg="h5repack_layout.h5 -f dset2:FLET  -l dset2:CHUNK=20x10"
TOOLTEST fletcher_individual $arg

# fletcher32 for all
arg="h5repack_layout.h5 -f FLET"
TOOLTEST fletcher_all $arg

# all filters
arg="h5repack_layout.h5 -f dset2:SHUF -f dset2:FLET -f dset2:SZIP=8,NN -f dset2:GZIP=1 -l dset2:CHUNK=20x10"
if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then
 SKIP $arg
else
 TOOLTEST all_filters $arg
fi

# verbose gzip with individual object
arg="h5repack_filters.h5 -v -f /dset_deflate:GZIP=9"
if test  $USE_FILTER_DEFLATE != "yes" ; then
 SKIP $arg
else
 # compare output
 TOOLTESTV gzip_verbose_filters $arg
fi

###########################################################
# the following tests assume the input files have filters
###########################################################

# szip copy
arg="h5repack_szip.h5"
if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then
 SKIP $arg
else
 TOOLTEST szip_copy $arg
fi

# szip remove
arg="h5repack_szip.h5 --filter=dset_szip:NONE"
if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then
 SKIP $arg
else
 TOOLTEST szip_remove $arg
fi

# deflate copy
arg="h5repack_deflate.h5"
if test $USE_FILTER_DEFLATE != "yes" ; then
 SKIP $arg
else
 TOOLTEST deflate_copy $arg
fi

# deflate remove
arg="h5repack_deflate.h5 -f dset_deflate:NONE"
if test $USE_FILTER_DEFLATE != "yes" ; then
 SKIP $arg
else
 TOOLTEST deflate_remove $arg
fi

# shuffle copy
arg="h5repack_shuffle.h5"
TOOLTEST shuffle_copy $arg

# shuffle remove
arg="h5repack_shuffle.h5 -f dset_shuffle:NONE"
TOOLTEST shuffle_remove $arg

# fletcher32 copy
arg="h5repack_fletcher.h5"
TOOLTEST fletcher_copy $arg

# fletcher32 remove
arg="h5repack_fletcher.h5 -f dset_fletcher32:NONE"
TOOLTEST fletcher_remove $arg

# nbit copy
arg="h5repack_nbit.h5"
TOOLTEST nbit_copy $arg

# nbit remove
arg="h5repack_nbit.h5 -f dset_nbit:NONE"
TOOLTEST nbit_remove $arg

# nbit add
arg="h5repack_nbit.h5 -f dset_int31:NBIT"
TOOLTEST nbit_add $arg

# scaleoffset copy
arg="h5repack_soffset.h5"
TOOLTEST scale_copy $arg

# scaleoffset add
arg="h5repack_soffset.h5 -f dset_none:SOFF=31,IN"
TOOLTEST scale_add $arg

# scaleoffset remove
arg="h5repack_soffset.h5 -f dset_scaleoffset:NONE"
TOOLTEST scale_remove $arg

# remove all  filters
arg="h5repack_filters.h5 -f NONE"
if test $USE_FILTER_DEFLATE != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SZIP_ENCODER != "yes" ; then
 SKIP $arg
else
 TOOLTEST remove_all $arg
fi

#filter conversions

arg="h5repack_deflate.h5 -f dset_deflate:SZIP=8,NN"
if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then
 SKIP $arg
else
 TOOLTEST deflate_convert $arg
fi

arg="h5repack_szip.h5 -f dset_szip:GZIP=1"
if test $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SZIP_ENCODER != "yes"  -o $USE_FILTER_DEFLATE != "yes" ; then
 SKIP $arg
else
 TOOLTEST szip_convert $arg
fi


#limit
arg="h5repack_layout.h5 -f GZIP=1 -m 1024"
if test $USE_FILTER_DEFLATE != "yes"  ; then
 SKIP $arg
else
 TOOLTEST_DUMP deflate_limit $arg
fi

#file
arg="h5repack_layout.h5 -e h5repack.info"
if test $USE_FILTER_DEFLATE != "yes" ; then
 SKIP $arg
else
 TOOLTEST deflate_file $arg
fi

#crtorder
arg="tordergr.h5 -L"
TOOLTEST_DUMP crtorder $arg

###################################################################################################
# Testing paged aggregation related options:
#   -G pagesize
#   -P 1 or 0
#   -S strategy
#   -T threshold
#
# The testfiles used are generated by test/gen_filespace.c and the file names are prepended with "h5repack_":
#   (1) "fsm_aggr_nopersist.h5"  /* H5F_FSPACE_STRATEGY_FSM_AGGR + not persisting free-space */
#   (2) "fsm_aggr_persist.h5"    /* H5F_FSPACE_STRATEGY_FSM_AGGR + persisting free-space */
#   (3) "paged_nopersist.h5"     /* H5F_FSPACE_STRATEGY_PAGE + not persisting free-space */
#   (4) "paged_persist.h5"       /* H5F_FSPACE_STRATEGY_PAGE + persisting free-space */
#   (5) "aggr.h5"                /* H5F_FSPACE_STRATEGY_AGGR */
#   (6) "none.h5"                /* H5F_FSPACE_STRATEGY_NONE */
#
#####################################################################################################
#
arg="h5repack_fsm_aggr_nopersist.h5 -S PAGE -P 1"
TOOLTEST_STAT SP $arg
#
arg="h5repack_fsm_aggr_persist.h5 -S AGGR"
TOOLTEST_STAT S $arg
#
arg="h5repack_none.h5 -S PAGE -T 10 -G 2048"
TOOLTEST_STAT STG $arg
#
arg="h5repack_paged_nopersist.h5 -G 512 -S AGGR"
TOOLTEST_STAT GS $arg
#
arg="h5repack_paged_persist.h5 -S NONE -P 1"
TOOLTEST_STAT SP $arg
#
arg="h5repack_aggr.h5 -S FSM_AGGR -P 1 -T 5"
TOOLTEST_STAT SPT $arg


#########################################################
# layout options (these files have no filters)
#########################################################
VERIFY_LAYOUT_DSET dset2_chunk_20x10 h5repack_layout.h5 dset2 CHUNKED --layout dset2:CHUNK=20x10
VERIFY_LAYOUT_ALL chunk_20x10 h5repack_layout.h5 CHUNKED -l CHUNK=20x10
VERIFY_LAYOUT_DSET dset2_conti h5repack_layout.h5 dset2 CONTIGUOUS -l dset2:CONTI
VERIFY_LAYOUT_ALL conti h5repack_layout.h5 CONTIGUOUS -l CONTI
VERIFY_LAYOUT_DSET dset2_compa h5repack_layout.h5 dset2 COMPACT -l dset2:COMPA
VERIFY_LAYOUT_ALL compa h5repack_layout.h5 COMPACT -l COMPA
TOOLTESTM dset2_chunk_20x10-errstk h5repack_layout.h5 --layout=dset2:CHUNK=20x10x5 --enable-error-stack

################################################################
# layout conversions (file has no filters)
###############################################################
VERIFY_LAYOUT_DSET dset_compa_conti h5repack_layout.h5 dset_compact CONTIGUOUS -l dset_compact:CONTI
VERIFY_LAYOUT_DSET dset_compa_chunk h5repack_layout.h5 dset_compact CHUNKED -l dset_compact:CHUNK=2x5
VERIFY_LAYOUT_DSET dset_compa_compa h5repack_layout.h5 dset_compact COMPACT -l dset_compact:COMPA
VERIFY_LAYOUT_DSET dset_conti_compa h5repack_layout.h5 dset_contiguous COMPACT -l dset_contiguous:COMPA
VERIFY_LAYOUT_DSET dset_conti_chunk h5repack_layout.h5 dset_contiguous CHUNKED -l dset_contiguous:CHUNK=3x6
VERIFY_LAYOUT_DSET dset_conti_conti h5repack_layout.h5 dset_contiguous CONTIGUOUS -l dset_contiguous:CONTI
VERIFY_LAYOUT_DSET chunk_compa h5repack_layout.h5 dset_chunk COMPACT -l dset_chunk:COMPA
VERIFY_LAYOUT_DSET chunk_conti h5repack_layout.h5 dset_chunk CONTIGUOUS -l dset_chunk:CONTI
VERIFY_LAYOUT_DSET chunk_18x13 h5repack_layout.h5 dset_chunk CHUNKED -l dset_chunk:CHUNK=18x13

# test convert small size dataset ( < 1k) to compact layout without -m
VERIFY_LAYOUT_DSET contig_small_compa h5repack_layout2.h5 contig_small COMPACT -l contig_small:COMPA
VERIFY_LAYOUT_DSET contig_small_fixed_compa h5repack_layout2.h5 chunked_small_fixed COMPACT -l chunked_small_fixed:COMPA

#---------------------------------------------------------------------------
# Test file contains chunked datasets (need multiple dsets) with
# unlimited max dims.   (HDFFV-7933)
# Use first dset to test.
#---------------------------------------------------------------------------
# chunk to chunk - specify chunk dim bigger than any current dim
VERIFY_LAYOUT_DSET chunk2chunk h5repack_layout3.h5 chunk_unlimit1 CHUNK -l chunk_unlimit1:CHUNK=100x300

# chunk to contiguous
VERIFY_LAYOUT_DSET chunk2conti h5repack_layout3.h5 chunk_unlimit1 CONTI -l chunk_unlimit1:CONTI

# chunk to compact - convert big dataset (should be > 64k) for this purpose,
# should remain as original layout (chunk)
VERIFY_LAYOUT_DSET chunk2compa h5repack_layout3.h5 chunk_unlimit1 CHUNK -l chunk_unlimit1:COMPA

#--------------------------------------------------------------------------
# Test -f for some specific cases. Chunked dataset with unlimited max dims.
# (HDFFV-8012)
#--------------------------------------------------------------------------
# - should not fail
# - should not change max dims from unlimit

# chunk dim is bigger than dataset dim. ( dset size < 64k )
VERIFY_LAYOUT_DSET error1 h5repack_layout3.h5 chunk_unlimit1 H5S_UNLIMITED -f chunk_unlimit1:NONE
# chunk dim is bigger than dataset dim. ( dset size > 64k )

VERIFY_LAYOUT_DSET error2 h5repack_layout3.h5 chunk_unlimit2 H5S_UNLIMITED -f chunk_unlimit2:NONE

# chunk dims are smaller than dataset dims. ( dset size < 64k )
#TOOLTEST_MAIN h5repack_layout3.h5  -f chunk_unlimit3:NONE
VERIFY_LAYOUT_DSET error3 h5repack_layout3.h5 chunk_unlimit3 H5S_UNLIMITED -f chunk_unlimit3:NONE

# file input - should not fail
TOOLTEST error4 h5repack_layout3.h5 -f NONE

#--------------------------------------------------------------------------
# Test base: Convert CHUNK to CONTI for a chunked dataset with small dataset
# (dset size < 64K) and with unlimited max dims on a condition as follow.
# (HDFFV-8214)
#--------------------------------------------------------------------------
# chunk dim is bigger than dataset dim. should succeed.
VERIFY_LAYOUT_DSET ckdim_biger h5repack_layout3.h5 chunk_unlimit2 CONTI -l chunk_unlimit2:CONTI
# chunk dim is smaller than dataset dim. should succeed.
VERIFY_LAYOUT_DSET ckdim_smaller h5repack_layout3.h5 chunk_unlimit3 CONTI -l chunk_unlimit3:CONTI


# Native option
# Do not use FILE1, as the named dtype will be converted to native, and h5diff will
# report a difference.
TOOLTEST native_fill h5repack_fill.h5 -n
TOOLTEST native_attr h5repack_attr.h5 -n


# latest file format with long switches. use FILE4=h5repack_layout.h5 (no filters)
arg="h5repack_layout.h5 --layout CHUNK=20x10 --filter GZIP=1 --minimum=10 --native --latest --compact=8 --indexed=6 --ssize=8[:dtype]"
if test $USE_FILTER_DEFLATE != "yes" ; then
 SKIP $arg
else
 VERIFY_LAYOUT_ALL layout_long_switches h5repack_layout.h5 CHUNKED --layout CHUNK=20x10 --filter GZIP=1 --minimum=10 --native --latest --compact=8 --indexed=6 --ssize=8[:dtype]
fi

# latest file format with short switches. use FILE4=h5repack_layout.h5 (no filters)
arg="h5repack_layout.h5 -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype]"
if test $USE_FILTER_DEFLATE != "yes" ; then
 SKIP $arg
else
 VERIFY_LAYOUT_ALL layout_short_switches h5repack_layout.h5 CHUNKED -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype]
fi

# several global filters
arg="h5repack_layout.h5 --filter GZIP=1 --filter SHUF"
if test $USE_FILTER_DEFLATE != "yes" ; then
 SKIP $arg
else
 TOOLTEST global_filters $arg
fi

# syntax of -i infile -o outfile
# latest file format with short switches. use FILE4=h5repack_layout.h5 (no filters)
arg="h5repack_layout.h5 -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype]"
if test $USE_FILTER_DEFLATE != "yes" ; then
 SKIP $arg
else
 TOOLTEST0 old_style_layout_short_switches $arg
fi

# add a userblock to file
arg="h5repack_objs.h5 -u ublock.bin -b 2048"
TOOLTEST add_userblock $arg

# add alignment
arg="h5repack_objs.h5 -t 1 -a 1 "
TOOLTEST add_alignment $arg

# Check repacking file with old version of layout message (should get upgraded
#       to new version and be readable, etc.)
TOOLTEST upgrade_layout h5repack_layouto.h5

# test for datum size > H5TOOLS_MALLOCSIZE
arg="h5repack_objs.h5 -f GZIP=1"
if test $USE_FILTER_DEFLATE != "yes" ; then
 SKIP $arg
else
 TOOLTEST gt_mallocsize $arg
fi

# Check repacking file with committed datatypes in odd configurations
TOOLTEST committed_dt h5repack_named_dtypes.h5

# tests family driver (file is located in common testfiles folder, uses TOOLTEST1
TOOLTEST1 family tfamily%05d.h5

# test various references (bug 1814 and 1726)
TOOLTEST bug1814 h5repack_refs.h5

# test attribute with various references (bug1797 / HDFFV-5932)
# the references in attribute of compound or vlen datatype
TOOLTEST HDFFV-5932 h5repack_attr_refs.h5

# Add test for memory leak in attribute. This test is verified by CTEST.
# 1. leak from vlen string
# 2. leak from compound type without reference member
# (HDFFV-7840, )
# Note: this test is experimental for sharing test file among tools
TOOLTEST HDFFV-7840 h5diff_attr1.h5

# test HDFFV-10590
arg="h5repack_CVE-2018-17432.h5 h5repack_CVE-2018-17432_out.h5 --low=1 --high=2 -f GZIP=8 -l dset1:CHUNK=5x6"
TOOLTEST_FAIL $arg

# test HDFFV-11223
arg="h5repack_CVE-2018-14460.h5 h5repack_CVE-2018-14460_out.h5"
TOOLTEST_FAIL $arg

# tests for metadata block size option
TOOLTEST_META meta_short h5repack_layout.h5 -M 8192
TOOLTEST_META meta_long h5repack_layout.h5 --metadata_block_size=8192

# VDS tests

################################################################
# layout conversions
###############################################################
if test  $USE_FILTER_DEFLATE != "yes" ; then
    SKIP vds_dset_chunk_20x10x5
else
    VERIFY_LAYOUT_VDS vds_dset_chunk20x10x5 1_vds.h5 vds_dset CHUNKED --layout vds_dset:CHUNK=20x10x5
fi

if test  $USE_FILTER_DEFLATE != "yes" ; then
    SKIP vds_chunk2x5x8
else
    VERIFY_LAYOUT_VDS vds_chunk2x5x8 3_1_vds.h5 vds_dset CHUNKED -l vds_dset:CHUNK=2x5x8
fi

if test  $USE_FILTER_DEFLATE != "yes" ; then
    SKIP vds_chunk3x6x9
else
    VERIFY_LAYOUT_VDS vds_chunk3x6x9 2_vds.h5 vds_dset CHUNKED -l vds_dset:CHUNK=3x6x9
fi

if test  $USE_FILTER_DEFLATE != "yes" ; then
    SKIP vds_compa 4_vds.h5
else
    VERIFY_LAYOUT_VDS vds_compa 4_vds.h5 vds_dset COMPACT -l vds_dset:COMPA
fi

if test  $USE_FILTER_DEFLATE != "yes" ; then
 SKIP vds_conti 4_vds.h5
else
 VERIFY_LAYOUT_VDS vds_conti 4_vds.h5 vds_dset CONTIGUOUS -l vds_dset:CONTI
fi

################################################################
# reference new api conversions
###############################################################
#TOOLTEST_DUMP attrregion tattrreg.h5
#TOOLTEST_DUMP dataregion tdatareg.h5

##############################################################################
###    V E R S I O N  B O U N D S  T E S T S
##############################################################################
# -j 0 -k 2, superblock will be 0
VERIFY_SUPERBLOCK 0 2 0 h5repack_layout.h5 -j 0 -k 2 h5repack_layout.h5
# -j 1 -k 2, superblock will be 2
VERIFY_SUPERBLOCK 1 2 2 h5repack_layout.h5 -j 1 -k 2 h5repack_layout.h5
# -j 2 -k 2, superblock will be 3
VERIFY_SUPERBLOCK 2 2 3 h5repack_layout.h5 -j 2 -k 2 h5repack_layout.h5
# -j 0 -k 1, file cannot be opened
VERIFY_INVALIDBOUNDS 0 1 bounds_latest_latest.h5

##############################################################################
###    E X T E R N A L  S T O R A G E  T E S T S
##############################################################################
VERIFY_EXTERNAL_CONSOLIDATION -l CONTI

##############################################################################
###    E X T E R N A L  L I N K  T E S T S
##############################################################################
### HDFFV-11128 needs fixed to enable the following test
#TOOLTEST_DUMP h5copy_extlinks_src-base h5copy_extlinks_src.h5 --enable-error-stack
TOOLTEST_DUMP tsoftlinks-base tsoftlinks.h5 --enable-error-stack
TOOLTEST_DUMP textlink-base textlink.h5 --enable-error-stack
TOOLTEST_DUMP textlinkfar-base textlinkfar.h5 --enable-error-stack
TOOLTEST_DUMP textlinksrc-base textlinksrc.h5 --enable-error-stack
TOOLTEST_DUMP textlinktar-base textlinktar.h5 --enable-error-stack

TOOLTESTSV h5copy_extlinks_src-merge h5copy_extlinks_src.h5 --merge
TOOLFAILSV tsoftlinks-merge tsoftlinks.h5 --merge
TOOLTESTSV textlink-merge textlink.h5 --merge
### HDFFV-11128 needs fixed to enable the following test
#TOOLFAILSV textlinkfar-merge textlinkfar.h5 --merge
### HDFFV-11128 needs fixed to enable the following test
#TOOLFAILSV textlinksrc-merge textlinksrc.h5 --merge
### HDFFV-11128 needs fixed to enable the following test
#TOOLFAILSV textlinktar-merge textlinktar.h5 --merge

TOOLTEST_DUMP h5copy_extlinks_src-prune h5copy_extlinks_src.h5 --prune --enable-error-stack
TOOLTEST_DUMP tsoftlinks-prune tsoftlinks.h5 --prune --enable-error-stack
TOOLTEST_DUMP textlink-prune textlink.h5 --prune --enable-error-stack
TOOLTEST_DUMP textlinkfar-prune textlinkfar.h5 --prune --enable-error-stack
TOOLTEST_DUMP textlinksrc-prune textlinksrc.h5 --prune --enable-error-stack
TOOLTEST_DUMP textlinktar-prune textlinktar.h5 --prune --enable-error-stack

TOOLTEST_DUMP h5copy_extlinks_src-mergeprune h5copy_extlinks_src.h5 --merge --prune --enable-error-stack
TOOLTEST_DUMP tsoftlinks-mergeprune tsoftlinks.h5 --merge --prune --enable-error-stack
TOOLTEST_DUMP textlink-mergeprune textlink.h5 --merge --prune --enable-error-stack
### HDFFV-11128 needs fixed to enable the following test
#TOOLTEST_DUMP textlinkfar-mergeprune textlinkfar.h5 --merge --prune --enable-error-stack
### HDFFV-11128 needs fixed to enable the following test
#TOOLTEST_DUMP textlinksrc-mergeprune textlinksrc.h5 --merge --prune --enable-error-stack
### HDFFV-11128 needs fixed to enable the following test
#TOOLTEST_DUMP textlinktar-mergeprune textlinktar.h5 --merge --prune --enable-error-stack

# Clean up temporary files/directories
CLEAN_TESTFILES_AND_TESTDIR

if test $nerrors -eq 0 ; then
    echo "All $TESTNAME tests passed."
    exit $EXIT_SUCCESS
else
    echo "$TESTNAME tests failed with $nerrors errors."
    exit $EXIT_FAILURE
fi