path: root/tools/test/h5import/h5importtestutil.sh.in

#!/bin/sh
#
# Copyright by The HDF Group.
# Copyright by the Board of Trustees of the University of Illinois.
# All rights reserved.
#
# This file is part of HDF5.  The full HDF5 copyright notice, including
# terms governing use, modification, and redistribution, is contained in
# the COPYING file, which can be found at the root of the source code
# distribution tree, or in https://www.hdfgroup.org/licenses.
# If you do not have access to either file, you may request a copy from
# help@hdfgroup.org.
#
# Tests for the  h5import tool

srcdir=@srcdir@

# Determine which filters are available
USE_FILTER_DEFLATE="@USE_FILTER_DEFLATE@"

TESTNAME=h5import
EXIT_SUCCESS=0
EXIT_FAILURE=1

DUMPER=../../src/h5dump/h5dump                     # The tool name
DUMPER_BIN=`pwd`/$DUMPER          # The path of the tool binary

H5DIFF=../../src/h5diff/h5diff           # The h5diff tool name
H5DIFF_BIN=`pwd`/$H5DIFF          # The path of the h5diff  tool binary

H5IMPORT=../../src/h5import/h5import     # The h5import tool name
H5IMPORT_BIN=`pwd`/$H5IMPORT      # The path of the h5import  tool binary

RM='rm -rf'
CP='cp'
DIRNAME='dirname'
LS='ls'
AWK='awk'

# initialize errors variable
nerrors=0

# source dirs
SRC_TOOLS="$srcdir/../.."
SRC_TOOLS_TESTFILES="$SRC_TOOLS/testfiles"

# testfiles source dirs for tools
SRC_H5LS_TESTFILES="$SRC_TOOLS_TESTFILES"
SRC_H5DUMP_TESTFILES="$SRC_TOOLS_TESTFILES"
SRC_H5DIFF_TESTFILES="$SRC_TOOLS/test/h5diff/testfiles"
SRC_H5COPY_TESTFILES="$SRC_TOOLS/test/h5copy/testfiles"
SRC_H5REPACK_TESTFILES="$SRC_TOOLS/test/h5repack/testfiles"
SRC_H5JAM_TESTFILES="$SRC_TOOLS/test/h5jam/testfiles"
SRC_H5STAT_TESTFILES="$SRC_TOOLS/test/h5stat/testfiles"
SRC_H5IMPORT_TESTFILES="$SRC_TOOLS/test/h5import/testfiles"

TESTDIR=./testfiles
test -d $TESTDIR || mkdir $TESTDIR

######################################################################
# test files
# --------------------------------------------------------------------
# All the test files copy from source directory to test directory
# NOTE: Keep this framework to add/remove test files.
#       Any test files from other tools can be used in this framework.
#       This list are also used for checking exist.
#       Comment '#' without space can be used.
# --------------------------------------------------------------------
LIST_HDF5_TEST_FILES="
$SRC_H5IMPORT_TESTFILES/binfp64.h5
$SRC_H5IMPORT_TESTFILES/binin8.h5
$SRC_H5IMPORT_TESTFILES/binin8w.h5
$SRC_H5IMPORT_TESTFILES/binin16.h5
$SRC_H5IMPORT_TESTFILES/binin32.h5
$SRC_H5IMPORT_TESTFILES/binuin16.h5
$SRC_H5IMPORT_TESTFILES/binuin32.h5
$SRC_H5IMPORT_TESTFILES/txtfp32.h5
$SRC_H5IMPORT_TESTFILES/txtfp64.h5
$SRC_H5IMPORT_TESTFILES/txtin8.h5
$SRC_H5IMPORT_TESTFILES/txtin16.h5
$SRC_H5IMPORT_TESTFILES/txtin32.h5
$SRC_H5IMPORT_TESTFILES/txtuin16.h5
$SRC_H5IMPORT_TESTFILES/txtuin32.h5
$SRC_H5IMPORT_TESTFILES/txtstr.h5
$SRC_H5IMPORT_TESTFILES/textpfe.h5
$SRC_TOOLS_TESTFILES/tall.h5
$SRC_TOOLS_TESTFILES/tintsattrs.h5
"

LIST_OTHER_TEST_FILES="
$SRC_H5IMPORT_TESTFILES/txtfp32.conf
$SRC_H5IMPORT_TESTFILES/txtfp64.conf
$SRC_H5IMPORT_TESTFILES/txtin8.conf
$SRC_H5IMPORT_TESTFILES/txtin16.conf
$SRC_H5IMPORT_TESTFILES/txtin32.conf
$SRC_H5IMPORT_TESTFILES/txtuin16.conf
$SRC_H5IMPORT_TESTFILES/txtuin32.conf
$SRC_H5IMPORT_TESTFILES/textpfe.conf
$SRC_H5IMPORT_TESTFILES/txtstr.conf
$SRC_H5IMPORT_TESTFILES/txtfp32.txt
$SRC_H5IMPORT_TESTFILES/txtfp64.txt
$SRC_H5IMPORT_TESTFILES/txtuin16.txt
$SRC_H5IMPORT_TESTFILES/txtuin32.txt
$SRC_H5IMPORT_TESTFILES/txtin8.txt
$SRC_H5IMPORT_TESTFILES/txtin16.txt
$SRC_H5IMPORT_TESTFILES/txtin32.txt
$SRC_H5IMPORT_TESTFILES/textpfe64.txt
$SRC_H5IMPORT_TESTFILES/txtstr.txt
$SRC_H5IMPORT_TESTFILES/dbinfp64.h5.txt
$SRC_H5IMPORT_TESTFILES/dbinin8.h5.txt
$SRC_H5IMPORT_TESTFILES/dbinin8w.h5.txt
$SRC_H5IMPORT_TESTFILES/dbinin16.h5.txt
$SRC_H5IMPORT_TESTFILES/dbinin32.h5.txt
$SRC_H5IMPORT_TESTFILES/dbinuin16.h5.txt
$SRC_H5IMPORT_TESTFILES/dbinuin32.h5.txt
$SRC_H5IMPORT_TESTFILES/dtxtstr.h5.txt
$SRC_H5IMPORT_TESTFILES/tall_fp32.ddl
$SRC_H5IMPORT_TESTFILES/tall_i32.ddl
$SRC_H5IMPORT_TESTFILES/tintsattrs_u32.ddl
"

#
# copy test files and expected output files from source dirs to test dir
#
COPY_TESTFILES="$LIST_HDF5_TEST_FILES $LIST_OTHER_TEST_FILES"

COPY_TESTFILES_TO_TESTDIR()
{
    # copy test files. Used -f to make sure get a new copy
    for tstfile in $COPY_TESTFILES
    do
        # ignore '#' comment
        echo $tstfile | tr -d ' ' | grep '^#' > /dev/null
        RET=$?
        if [ $RET -eq 1 ]; then
            # skip cp if srcdir is same as destdir
            # this occurs when build/test performed in source dir and
            # make cp fail
            SDIR=`$DIRNAME $tstfile`
            INODE_SDIR=`$LS -i -d $SDIR | $AWK -F' ' '{print $1}'`
            INODE_DDIR=`$LS -i -d $TESTDIR | $AWK -F' ' '{print $1}'`
            if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then
                $CP -f $tstfile $TESTDIR
                if [ $? -ne 0 ]; then
                    echo "Error: FAILED to copy $tstfile ."

                    # Comment out this to CREATE expected file
                    exit $EXIT_FAILURE
                fi
            fi
        fi
    done
}

CLEAN_TESTFILES_AND_TESTDIR()
{
    # skip rm if srcdir is same as destdir
    # this occurs when build/test performed in source dir and
    # make cp fail
    SDIR=$SRC_H5IMPORT_TESTFILES
    INODE_SDIR=`$LS -i -d $SDIR | $AWK -F' ' '{print $1}'`
    INODE_DDIR=`$LS -i -d $TESTDIR | $AWK -F' ' '{print $1}'`
    if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then
        $RM $TESTDIR
    fi
}

# Print a line-line message left justified in a field of 70 characters
# beginning with the word "Testing".
#
TESTING() {
   SPACES="                                                               "
   echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012'
}

# Print a "SKIP" message
SKIP() {
   TESTING $TESTNAME $@
    echo  " -SKIP-"
}

TOOLTEST()
{
err=0
$RUNSERIAL $H5IMPORT_BIN $*
$RUNSERIAL $DUMPER_BIN $5 >log2

cd tmp_testfiles
$RUNSERIAL $DUMPER_BIN $5 >log1
cd ..

cmp -s tmp_testfiles/log1 log2 || err=1
rm -f log2 tmp_testfiles/log1
if [ $err -eq 1 ]; then
nerrors="` expr $nerrors + 1 `";
  echo "*FAILED*"
else
  echo " PASSED"
fi
}

# Use h5dump output as input to h5import for binary numbers
# Use h5diff to verify results
TOOLTEST2()
{
err=0
$RUNSERIAL $DUMPER_BIN -p -d $1 -o d$2.bin -b NATIVE tmp_testfiles/$2 > d$2.dmp
$RUNSERIAL $H5IMPORT_BIN d$2.bin -c d$2.dmp -o d$2 > d$2.imp
$RUNSERIAL $H5DIFF_BIN -v d$2 tmp_testfiles/$2 $1 $1 > log2
$CP -f $SRC_H5IMPORT_TESTFILES/d$2.txt log1

cmp -s log1 log2 || err=1
rm -f log1 log2
if [ $err -eq 1 ]; then
nerrors="` expr $nerrors + 1 `";
  echo "*FAILED*"
else
  echo " PASSED"
fi
}

# Same as TOOLTEST2 except for strings
# Use h5dump output as input to h5import for strings
# Use h5diff to verify results
TOOLTEST3()
{
err=0
$RUNSERIAL $DUMPER_BIN -p -d $1 -o d$2.bin -y --width=1 tmp_testfiles/$2 > d$2.dmp
$RUNSERIAL $H5IMPORT_BIN d$2.bin -c d$2.dmp -o d$2 > d$2.imp
$RUNSERIAL $H5DIFF_BIN -v d$2 tmp_testfiles/$2 $1 $1 > log2
$CP -f $SRC_H5IMPORT_TESTFILES/d$2.txt log1

cmp -s log1 log2 || err=1
rm -f log1 log2
if [ $err -eq 1 ]; then
nerrors="` expr $nerrors + 1 `";
  echo "*FAILED*"
else
  echo " PASSED"
fi
}

# Same as TOOLTEST3 except for h5diff uses report mode without warnings
# Use h5dump output as input to h5import for strings
# Use h5diff to verify results
TOOLTEST4()
{
err=0
$RUNSERIAL $DUMPER_BIN -p -d $1 -o d$2.bin -y --width=1 tmp_testfiles/$2 > d$2.dmp
$RUNSERIAL $H5IMPORT_BIN d$2.bin -c d$2.dmp -o d$2 > d$2.imp
$RUNSERIAL $H5DIFF_BIN -r d$2 tmp_testfiles/$2 $1 $1 > log2
$CP -f $SRC_H5IMPORT_TESTFILES/d$2.txt log1


cmp -s log1 log2 || err=1
rm -f log1 log2
if [ $err -eq 1 ]; then
nerrors="` expr $nerrors + 1 `";
  echo "*FAILED*"
else
  echo " PASSED"
fi
}

# Same as TOOLTEST2 except for subsets
# Use h5dump output as input to h5import for binary numbers
# Use h5dump to verify results
TOOLTEST5()
{
err=0
$RUNSERIAL $DUMPER_BIN -p -d $3 $4 -o ds$1.bin -b NATIVE tmp_testfiles/$2 > ds$1.dmp
$RUNSERIAL $H5IMPORT_BIN ds$1.bin -c ds$1.dmp -o ds$1.h5 > ds$1.imp
$RUNSERIAL $DUMPER_BIN -p ds$1.h5 > log2
$CP -f $SRC_H5IMPORT_TESTFILES/$1.ddl log1

cmp -s log1 log2 || err=1
rm -f log1 log2
if [ $err -eq 1 ]; then
nerrors="` expr $nerrors + 1 `";
  echo "*FAILED*"
else
  echo " PASSED"
fi
}

echo ""
echo "=============================="
echo "H5IMPORT tests started"
echo "=============================="

#echo "** Testing h5import  ***"

rm -f  output.h5 log1 tx* b* *.dat

# prepare for test
COPY_TESTFILES_TO_TESTDIR

mkdir tmp_testfiles
$CP $TESTDIR/*.h5 ./tmp_testfiles/

$RUNSERIAL ./h5importtest

################################################
###        T H E   T E S T S
################################################

TESTING "ASCII I32 rank 3 - Output BE " ;
TOOLTEST $TESTDIR/txtin32.txt -c $TESTDIR/txtin32.conf -o txtin32.h5

TESTING "ASCII I16 rank 3 - Output LE - CHUNKED - extended"
TOOLTEST $TESTDIR/txtin16.txt -c $TESTDIR/txtin16.conf -o txtin16.h5

TESTING "ASCII I8 - rank 3 - Output I8 LE-Chunked+Extended+Compressed "
TOOLTEST $TESTDIR/txtin8.txt -c $TESTDIR/txtin8.conf  -o txtin8.h5


TESTING "ASCII UI16 - rank 2 - Output LE+Chunked+Compressed "
TOOLTEST $TESTDIR/txtuin16.txt -c $TESTDIR/txtuin16.conf -o txtuin16.h5

TESTING "ASCII UI32 - rank 3 - Output BE"
TOOLTEST $TESTDIR/txtuin32.txt -c $TESTDIR/txtuin32.conf -o txtuin32.h5


TESTING "ASCII F32 - rank 3 - Output LE "
TOOLTEST $TESTDIR/txtfp32.txt -c $TESTDIR/txtfp32.conf -o txtfp32.h5

TESTING "ASCII F64 - rank 3 - Output BE + CHUNKED+Extended+Compressed "
TOOLTEST $TESTDIR/txtfp64.txt -c $TESTDIR/txtfp64.conf -o txtfp64.h5


TESTING "BINARY F64 - rank 3 - Output LE+CHUNKED+Extended+Compressed "
TOOLTEST binfp64.bin -c binfp64.conf -o binfp64.h5
TESTING "H5DUMP-BINARY F64 - rank 3 - Output LE+CHUNKED+Extended+Compressed "
if test $USE_FILTER_DEFLATE != "yes"; then
 SKIP "/fp/bin/64-bit" binfp64.h5
else
 TOOLTEST2 "/fp/bin/64-bit" binfp64.h5
fi


TESTING "BINARY I8 - rank 3 - Output I16LE + Chunked+Extended+Compressed "
TOOLTEST binin8.bin -c binin8.conf -o binin8.h5
TESTING "H5DUMP-BINARY I8 - rank 3 - Output I16LE + Chunked+Extended+Compressed "
if test $USE_FILTER_DEFLATE != "yes"; then
 SKIP "/int/bin/8-bit" binin8.h5
else
 TOOLTEST2 "/int/bin/8-bit" binin8.h5
fi

TESTING "BINARY I16 - rank 3 - Output order LE + CHUNKED + extended "
TOOLTEST binin16.bin -c binin16.conf -o binin16.h5
TESTING "H5DUMP-BINARY I16 - rank 3 - Output order LE + CHUNKED + extended "
TOOLTEST2 "/int/bin/16-bit" binin16.h5

TESTING "BINARY I32 - rank 3 - Output BE + CHUNKED "
TOOLTEST binin32.bin -c binin32.conf -o binin32.h5
TESTING "H5DUMP-BINARY I32 - rank 3 - Output BE + CHUNKED "
TOOLTEST2 "/int/bin/32-bit" binin32.h5


TESTING "BINARY UI16 - rank 3 - Output byte BE + CHUNKED "
TOOLTEST binuin16.bin -c binuin16.conf -o binuin16.h5
TESTING "H5DUMP-BINARY UI16 - rank 3 - Output byte BE + CHUNKED "
TOOLTEST2 "/int/buin/16-bit" binuin16.h5

TESTING "BINARY UI32 - rank 3 - Output LE + CHUNKED "
TOOLTEST binuin32.bin -c binuin32.conf -o binuin32.h5
TESTING "H5DUMP-BINARY UI32 - rank 3 - Output LE + CHUNKED "
TOOLTEST2 "/int/buin/32-bit" binuin32.h5


TESTING "STR"
TOOLTEST $TESTDIR/txtstr.txt -c $TESTDIR/txtstr.conf -o txtstr.h5
TESTING "H5DUMP-STR"
TOOLTEST4 "/mytext/data" txtstr.h5


TESTING "BINARY I8 CR LF EOF"
TOOLTEST binin8w.bin -c binin8w.conf -o binin8w.h5
TESTING "H5DUMP-BINARY I8 CR LF EOF"
TOOLTEST2 "/dataset0" binin8w.h5

TESTING "ASCII F64 - rank 1 - INPUT-CLASS TEXTFPE "
TOOLTEST $TESTDIR/textpfe64.txt -c $TESTDIR/textpfe.conf -o textpfe.h5

TESTING "Binary Subset FP"
TOOLTEST5 tall_fp32 tall.h5 "/g2/dset2.2" "--start=1,1 --stride=2,3 --count=1,2 --block=1,1"
TESTING "Binary Subset INT"
TOOLTEST5 tall_i32 tall.h5 "/g1/g1.1/dset1.1.1" "--start=1,1 --stride=2,3 --count=3,2 --block=1,1"
TESTING "Binary Subset UINT"
TOOLTEST5 tintsattrs_u32 tintsattrs.h5 "/DU32BITS" "--start=1,1 --stride=2,3 --count=3,2 --block=1,1"


rm -f  txtin32.txt txtin16.txt txtin8.txt txtuin32.txt txtuin16.txt *.bin *.dmp *.conf *.imp *.h5
rm -rf tmp_testfiles

# Clean up temporary files/directories
CLEAN_TESTFILES_AND_TESTDIR

#
# Check errors result
if test $nerrors -eq 0 ; then
    echo "All $TESTNAME tests passed."
    exit $EXIT_SUCCESS
else
    echo "$TESTNAME tests failed with $nerrors errors."
    exit $EXIT_FAILURE
fi