path: root/test/API/tools/testh5copy.sh

#! /bin/sh
#
# Copyright by The HDF Group.
# Copyright by the Board of Trustees of the University of Illinois.
# All rights reserved.
#
# This file is part of HDF5.  The full HDF5 copyright notice, including
# terms governing use, modification, and redistribution, is contained in
# the COPYING file, which can be found at the root of the source code
# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases.
# If you do not have access to either file, you may request a copy from
# help@hdfgroup.org.
#
# Tests for the h5copy tool

TESTNAME=h5copy
EXIT_SUCCESS=0
EXIT_FAILURE=1

H5COPY=h5copy               # The tool name
H5DIFF=h5diff               # The h5diff tool name
H5DELETE='h5delete -f'      # The h5delete tool name
H5REPACK=h5repack           # The h5repack tool name

RM='rm -rf'
CMP='cmp'
DIFF='diff -c'
GREP='grep'
CP='cp'
DIRNAME='dirname'
LS='ls'
AWK='awk'

nerrors=0
verbose=yes
h5haveexitcode=yes        # default is yes

######################################################################
# Input files
# --------------------------------------------------------------------

# Where the tool's HDF5 input files are located
TESTFILES_HDF5_DIR="./testfiles/hdf5"

# Where the tool's expected output files are located
H5COPY_TESTFILES_OUT_DIR="./testfiles/expected/h5copy"

######################################################################
# Output files
# --------------------------------------------------------------------

# Where the text output goes
TEXT_OUTPUT_DIR=./h5copy_test_output

# Where the repacked HDF5 input files go
REPACK_OUTPUT_DIR=./h5copy_repack_output

# Where the HDF5 output generated by the tool goes.
HDF5_OUTPUT_DIR=./h5copy_test_hdf5

######################################################################
# test files
# --------------------------------------------------------------------

# HDF5 test files.
#
# Kept in       $H5REPACK_TESTFILES_HDF5_DIR
# Repacked to   $REPACK_OUTPUT_DIR
#
HDF5_FILES="
h5copytst.h5
"

# List of expect files that will be copied over to local test dir
# Expected output files.
#
# Kept in       $H5COPY_TESTFILES_OUT_DIR
# Copied to     $TEXT_OUTPUT_DIR
#
# NOTE: This is ALL the files - they have not been culled based on the HDF5
#       files in the above list.
#
EXPECTED_OUTPUT_FILES="
h5copy_misc1.out
h5copy_misc1.err
"

# Generated HDF5 files.
#
# Generated in  $HDF5_OUTPUT_DIR
#
# This list is needed for cleanup since we can't simply delete a directory
# with VOL connectors that use non-file storage.
#
HDF5_OUTPUT_FILES="
simple.out.h5
chunk.out.h5
compact.out.h5
compound.out.h5
named_vl.out.h5
nested_vl.out.h5
dset_attr.out.h5
simple_top.out.h5
dsrename.out.h5
grp_empty.out.h5
grp_attr.out.h5
simple_group.out.h5
A_B1_simple.out.h5
A_B2_simple2.out.h5
C_D_simple.out.h5
h5copy_misc1.out.h5
samefile1.out.h5
"


######################################################################
# Utility functions
# --------------------------------------------------------------------

# Copy the expected text output files to the text output directory
# to make it easier to diff the expected and actual output.
#
COPY_EXPECTED_OUTPUT_FILES()
{
    for outfile in $EXPECTED_OUTPUT_FILES
    do
        filepath="$H5COPY_TESTFILES_OUT_DIR/$outfile"

        # Use -f to make sure get a new copy
        $CP -f $filepath $TEXT_OUTPUT_DIR
        if [ $? -ne 0 ]; then
            echo "Error: FAILED to copy expected output file: $filepath ."
        fi
    done
}

# Repack the HDF5 files to the repack directory.
#
REPACK_HDF5_FILES()
{
    for repackfile in $HDF5_FILES
    do
        inpath="$TESTFILES_HDF5_DIR/$repackfile"
        outpath="$REPACK_OUTPUT_DIR/$repackfile"

        # Repack the file
        $H5REPACK --src-vol-name=native --enable-error-stack $inpath $outpath
        if [ $? -ne 0 ]; then
            echo "Error: FAILED to repack HDF5 file: $inpath ."
        fi
    done
}

# Cleans up HDF5 and text output from the tests. Only cleans if the
# HDF5_NOCLEANUP variable is unset.
#
CLEAN_OUTPUT()
{
    # Remove output if the "no cleanup" environment variable is not defined
    if test -z "$HDF5_NOCLEANUP"; then
        # Text output
        $RM $TEXT_OUTPUT_DIR

        # HDF5 output
        #
        # NOTE: Both repacked and generated HDF5 files exist for the
        #       h5copy tests.
        #
        # Can't just rm -rf the directory if the VOL storage doesn't map to
        # a normal file, so we'll use h5delete to delete the file.
        for hdf5file in $HDF5_FILES
        do
            filepath="$REPACK_OUTPUT_DIR/$hdf5file"
            $H5DELETE $filepath
        done

        for hdf5file in $HDF5_OUTPUT_FILES
        do
            filepath="$HDF5_OUTPUT_DIR/$hdf5file"
            $H5DELETE $filepath
        done

        # The HDF5 output directory is always created, even if the VOL
        # storage won't use it. Delete it here.
        $RM $REPACK_OUTPUT_DIR
        $RM $HDF5_OUTPUT_DIR
   fi
}


# RUNSERIAL is used. Check if it can return exit code from executalbe correctly.
if [ -n "$RUNSERIAL_NOEXITCODE" ]; then
    echo "***Warning*** Serial Exit Code is not passed back to shell corretly."
    echo "***Warning*** Exit code checking is skipped."
    h5haveexitcode=no
fi


# Print a "SKIP" message
SKIP() {
    TESTING $H5COPY $@
    echo  " -SKIP-"
}

# Print a line-line message left justified in a field of 70 characters
# beginning with the word "Testing".
TESTING()
{
    SPACES="                                                               "
    echo "Testing $* $SPACES" |cut -c1-70 |tr -d '\012'
}

# Print a line-line message left justified in a field of 70 characters
# beginning with the word "Verifying".
#
VERIFY()
{
    SPACES="                                                               "
    echo "Verifying h5diff output $* $SPACES" | cut -c1-70 | tr -d '\012'
}

# Print a line-line message left justified in a field of 70 characters
# beginning with the word "Verifying".
#
VERIFY_OUTPUT()
{
    SPACES="                                                               "
    echo "Verifying output files $* $SPACES" | cut -c1-70 | tr -d '\012'
}

# Source in the output filter function definitions.
. ./output_filter.sh

######################################################################
# Main testing functions
# --------------------------------------------------------------------

# Run a test and print PASS or *FAIL*. If h5copy can complete
# with exit status 0, consider it pass. If a test fails then increment
# the `nerrors' global variable.
# Assumed arguments:
# $1 is -i
# $2 is input file
# $3 is -o
# $4 is output file
# $* everything else arguments for h5copy.

TOOLTEST()
{
    actualout="$TEXT_OUTPUT_DIR/tooltest.actualout"
    actualerr="$TEXT_OUTPUT_DIR/tooltest.actualerr"
    runh5diff=yes
    if [ "$1" = -i ]; then
        inputfile=$2
    else
        if [ "$1" = -f ]; then
            inputfile=$4
        else
            inputfile=$3
        fi
        runh5diff=no
    fi
    if [ "$3" = -o ]; then
        outputfile=$4
    else
        if [ "$1" = -f ]; then
            outputfile=$6
        else
            outputfile=$5
        fi
        runh5diff=no
    fi

    TESTING $H5COPY $@
    (
        echo "#############################"
        echo " output for '$H5COPY $@'"
        echo "#############################"
        $RUNSERIAL $H5COPY $@
    ) > $actualout 2> $actualerr
    RET=$?
    if [ $RET != 0 ]; then
        echo "*FAILED*"
        echo "failed result is:"
        cat $actualout
        nerrors="`expr $nerrors + 1`"
    else
        echo " PASSED"

        if [ $runh5diff != no ]; then
            H5DIFFTEST $inputfile $outputfile $7 $9
        fi
    fi
}

# TOOLTEST back-to-back
TOOLTEST_PREFILL()
{
    actualout="$TEXT_OUTPUT_DIR/tooltest.actualout"
    actualerr="$TEXT_OUTPUT_DIR/tooltest.actualerr"
    runh5diff=yes
    if [ "$1" = -i ]; then
        inputfile=$2
    else
        runh5diff=no
    fi
    if [ "$3" = -o ]; then
        outputfile=$4
    else
        runh5diff=no
    fi

    grp_name=$5
    grp_name2=$6
    obj_name=$7
    obj_name2=$8

    TESTING $H5COPY $@
    (
        echo "#############################"
        echo " output for '$H5COPY $@'"
        echo "#############################"
        $RUNSERIAL $H5COPY  -i $inputfile -o $outputfile -v -s $grp_name -d $grp_name2
    ) > $actualout 2> $actualerr
    RET=$?
    if [ $RET != 0 ]; then
        echo "*FAILED*"
        echo "failed result is:"
        cat $actualout
        nerrors="`expr $nerrors + 1`"
    else
        TESTING $H5COPY $@
        (
            echo "#############################"
            echo " output for '$H5COPY $@'"
            echo "#############################"
            $RUNSERIAL $H5COPY  -i $inputfile -o $outputfile -v -s $obj_name -d $obj_name2
        ) > $actualout 2> $actualerr
        RET=$?
        if [ $RET != 0 ]; then
            echo "*FAILED*"
            echo "failed result is:"
            cat $actualout
            nerrors="`expr $nerrors + 1`"
        else
            echo " PASSED"

            if [ $runh5diff != no ]; then
                H5DIFFTEST $inputfile $outputfile $obj_name $obj_name2
            fi
        fi
    fi
}

# TOOLTEST back-to-back
TOOLTEST_SAME()
{
    actualout="$TEXT_OUTPUT_DIR/tooltest.actualout"
    actualerr="$TEXT_OUTPUT_DIR/tooltest.actualerr"
    runh5diff=yes
    if [ "$1" = -i ]; then
        inputfile=$2
    else
        runh5diff=no
    fi
    if [ "$3" = -o ]; then
        outputfile=$4
    else
        runh5diff=no
    fi

    grp_name=$5
    grp_name2=$6

    TESTING $H5COPY $@
    (
        echo "#############################"
        echo " output for '$H5COPY $@'"
        echo "#############################"
        $RUNSERIAL $H5COPY  -i $inputfile -o $outputfile -v -s $grp_name -d $grp_name
    ) > $actualout 2> $actualerr
    RET=$?
    if [ $RET != 0 ]; then
        echo "*FAILED*"
        echo "failed result is:"
        cat $actualout
        nerrors="`expr $nerrors + 1`"
    else
        TESTING $H5COPY $@
        (
            echo "#############################"
            echo " output for '$H5COPY $@'"
            echo "#############################"
            $RUNSERIAL $H5COPY  -i $outputfile -o $outputfile -v -s $grp_name -d $grp_name2
        ) > $actualout 2> $actualerr
        RET=$?
        if [ $RET != 0 ]; then
            echo "*FAILED*"
            echo "failed result is:"
            cat $actualout
            nerrors="`expr $nerrors + 1`"
        else
            echo " PASSED"

            if [ $runh5diff != no ]; then
                H5DIFFTEST $outputfile $outputfile $grp_name $grp_name2
            fi
        fi
    fi
}


# Compare the two text files
# PASS if same
# FAIL if different, and show the diff
#
# Assumed arguments:
# $1 is text file1 (expected output)
# $2 is text file2 (actual output)
CMP_OUTPUT()
{
    expect=$1
    actual=$2

    VERIFY_OUTPUT $@
    if [ ! -f $expect ]; then
        # Create the expect file if it doesn't yet exist.
        echo " CREATED"
        cp $actual $expect
        echo "    Expected result (*.ddl) missing"
        nerrors="`expr $nerrors + 1`"
    elif $CMP $expect $actual; then
        echo " PASSED"
    else
        echo "*FAILED*"
        echo "    Expected output differs from actual output"
        nerrors="`expr $nerrors + 1`"
        test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/    /'
    fi
}

TOOLTEST_FAIL()
{
    expectout="$TEXT_OUTPUT_DIR/$1"
    expecterr="$TEXT_OUTPUT_DIR/`basename $1 .out`.err"
    actualout="$TEXT_OUTPUT_DIR/$1.actualout"
    actualerr="$TEXT_OUTPUT_DIR/$1.actualerr"
    actualout_sav=${actualout}-sav
    actualerr_sav=${actualerr}-sav
    shift
    if [ "$1" = -i ]; then
        inputfile=$2
    fi
    if [ "$3" = -o ]; then
        outputfile=$4
    fi

    TESTING $H5COPY $@
    (
        #echo "#############################"
        #echo " output for '$H5COPY $@'"
        #echo "#############################"
        $RUNSERIAL $H5COPY $@
    ) > $actualout 2> $actualerr

    RET=$?
    # save actualout and actualerr in case they are needed later.
    cp $actualout $actualout_sav
    STDOUT_FILTER $actualout
    cp $actualerr $actualerr_sav
    STDERR_FILTER $actualerr
    if [ $RET != 0 ]; then
        echo " PASSED"
        # Verifying output text from h5copy
        if [ "$expectout" != "SKIP" ]; then
            CMP_OUTPUT $expecterr $actualerr
        fi
    else
        echo "*FAILED*"
        echo "failed result is:"
        cat $actualout
        nerrors="`expr $nerrors + 1`"
    fi
}


# Call the h5diff tool
#
H5DIFFTEST()
{
    VERIFY  $@
    $RUNSERIAL $H5DIFF -q "$@"
    RET=$?
    if [ $RET != 0 ] ; then
        echo "*FAILED*"
        nerrors="`expr $nerrors + 1`"
    else
        echo " PASSED"
    fi
}

# Call the h5diff tool with a call that is expected to fail
#
H5DIFFTEST_FAIL()
{
    VERIFY  $@
    $RUNSERIAL $H5DIFF -q "$@"
    RET=$?

    if [ $h5haveexitcode = 'yes' -a $RET != 1 ] ; then
        echo "*FAILED*"
        nerrors="`expr $nerrors + 1`"
    else
        echo " PASSED"
    fi
}

##############################################################################
##############################################################################
###        T H E   T E S T S                                               ###
##############################################################################
##############################################################################
# Prepare for test
#
# We create the repack output dir in case it's needed. If a VOL connector doesn't
# use normal files, it'll just stay empty and get deleted later.
CLEAN_OUTPUT
test -d $TEXT_OUTPUT_DIR || mkdir -p $TEXT_OUTPUT_DIR
test -d $REPACK_OUTPUT_DIR || mkdir -p $REPACK_OUTPUT_DIR
test -d $HDF5_OUTPUT_DIR || mkdir -p $HDF5_OUTPUT_DIR
COPY_EXPECTED_OUTPUT_FILES
REPACK_HDF5_FILES

TESTFILE="$REPACK_OUTPUT_DIR/h5copytst.h5"

# Copy single datasets of various forms from one group to another,
#       adding object copied to the destination file each time

echo "Test copying various forms of datasets"
TOOLTEST -i $TESTFILE -o $HDF5_OUTPUT_DIR/simple.out.h5 -v -s simple     -d simple
TOOLTEST -i $TESTFILE -o $HDF5_OUTPUT_DIR/chunk.out.h5 -v -s chunk      -d chunk
TOOLTEST -i $TESTFILE -o $HDF5_OUTPUT_DIR/compact.out.h5 -v -s compact    -d compact
TOOLTEST -i $TESTFILE -o $HDF5_OUTPUT_DIR/compound.out.h5 -v -s compound   -d compound
TOOLTEST -i $TESTFILE -o $HDF5_OUTPUT_DIR/named_vl.out.h5 -v -s named_vl   -d named_vl
TOOLTEST -i $TESTFILE -o $HDF5_OUTPUT_DIR/nested_vl.out.h5 -v -s nested_vl  -d nested_vl
#TOOLTEST -i $TESTFILE -o $HDF5_OUTPUT_DIR/dset_attr.out.h5 -v -s /dset_attr  -d /dset_attr

echo "Test copying dataset within group in source file to root of destination"
TOOLTEST -i $TESTFILE -o $HDF5_OUTPUT_DIR/simple_top.out.h5 -v -s grp_dsets/simple  -d simple_top

echo "Test copying & renaming dataset"
TOOLTEST -i $TESTFILE -o $HDF5_OUTPUT_DIR/dsrename.out.h5 -v -s compound   -d rename

echo "Test copying empty, 'full' & 'nested' groups"
TOOLTEST -i $TESTFILE -o $HDF5_OUTPUT_DIR/grp_empty.out.h5 -v -s grp_empty  -d grp_empty
#TOOLTEST -i $TESTFILE -o $HDF5_OUTPUT_DIR/grp_attr.out.h5 -v -s grp_attr -d grp_attr

#echo "Test copying dataset within group in source file to group in destination"
#TOOLTEST_PREFILL -i $TESTFILE -o $HDF5_OUTPUT_DIR/simple_group.out.h5 grp_dsets grp_dsets /grp_dsets/simple /grp_dsets/simple_group

echo "Test copying objects into group hier. that doesn't exist yet in destination file"
TOOLTEST -i $TESTFILE -o $HDF5_OUTPUT_DIR/A_B1_simple.out.h5 -vp -s simple    -d /A/B1/simple
TOOLTEST -i $TESTFILE -o $HDF5_OUTPUT_DIR/A_B2_simple2.out.h5 -vp -s simple    -d /A/B2/simple2
TOOLTEST -i $TESTFILE -o $HDF5_OUTPUT_DIR/C_D_simple.out.h5 -vp -s /grp_dsets/simple    -d /C/D/simple

# Misc tests
echo "Test copying object into group which doesn't exist, without -p"
TOOLTEST_FAIL h5copy_misc1.out -i $TESTFILE -o $HDF5_OUTPUT_DIR/h5copy_misc1.out.h5 -v -s /simple  -d /g1/g2/simple

echo "Test copying objects to the same file "
TOOLTEST_SAME -i $TESTFILE -o $HDF5_OUTPUT_DIR/samefile1.out.h5 /simple /simple_cp

# Clean up generated files/directories
CLEAN_OUTPUT

if test $nerrors -eq 0 ; then
    echo "All $TESTNAME tests passed."
    exit $EXIT_SUCCESS
else
    echo "$TESTNAME tests failed with $nerrors errors."
    exit $EXIT_FAILURE
fi