From 59e94f5009a1f738d9924be942448f8cf6355b00 Mon Sep 17 00:00:00 2001 From: Vailin Choi Date: Sat, 29 Jul 2017 01:10:25 -0500 Subject: Add h5repack tests for paged aggregation Add tests to h5repack.sh.in to verify options added for paged aggregation work as expected. --- MANIFEST | 12 +++ tools/test/h5repack/h5repack.sh.in | 103 +++++++++++++++++++++ .../testfiles/GS.h5repack_paged_nopersist.h5.ddl | 15 +++ .../testfiles/S.h5repack_fsm_aggr_persist.h5.ddl | 15 +++ .../SP.h5repack_fsm_aggr_nopersist.h5.ddl | 16 ++++ .../testfiles/SP.h5repack_paged_persist.h5.ddl | 15 +++ .../h5repack/testfiles/SPT.h5repack_aggr.h5.ddl | 16 ++++ .../h5repack/testfiles/STG.h5repack_none.h5.ddl | 15 +++ tools/test/h5repack/testfiles/h5repack_aggr.h5 | Bin 0 -> 2448 bytes .../testfiles/h5repack_fsm_aggr_nopersist.h5 | Bin 0 -> 2448 bytes .../testfiles/h5repack_fsm_aggr_persist.h5 | Bin 0 -> 2565 bytes tools/test/h5repack/testfiles/h5repack_none.h5 | Bin 0 -> 1808 bytes .../h5repack/testfiles/h5repack_paged_nopersist.h5 | Bin 0 -> 8192 bytes .../h5repack/testfiles/h5repack_paged_persist.h5 | Bin 0 -> 16384 bytes 14 files changed, 207 insertions(+) create mode 100644 tools/test/h5repack/testfiles/GS.h5repack_paged_nopersist.h5.ddl create mode 100644 tools/test/h5repack/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl create mode 100644 tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl create mode 100644 tools/test/h5repack/testfiles/SP.h5repack_paged_persist.h5.ddl create mode 100644 tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl create mode 100644 tools/test/h5repack/testfiles/STG.h5repack_none.h5.ddl create mode 100644 tools/test/h5repack/testfiles/h5repack_aggr.h5 create mode 100644 tools/test/h5repack/testfiles/h5repack_fsm_aggr_nopersist.h5 create mode 100644 tools/test/h5repack/testfiles/h5repack_fsm_aggr_persist.h5 create mode 100644 tools/test/h5repack/testfiles/h5repack_none.h5 create mode 100644 tools/test/h5repack/testfiles/h5repack_paged_nopersist.h5 create mode 100644 tools/test/h5repack/testfiles/h5repack_paged_persist.h5 diff --git a/MANIFEST b/MANIFEST index 27f38be..af4dc7e 100644 --- a/MANIFEST +++ b/MANIFEST @@ -2405,6 +2405,7 @@ #test files for h5repack ./tools/test/h5repack/testfiles/README +./tools/test/h5repack/testfiles/h5repack_aggr.h5 ./tools/test/h5repack/testfiles/h5repack_attr.h5 ./tools/test/h5repack/testfiles/h5repack_attr_refs.h5 ./tools/test/h5repack/testfiles/h5repack_deflate.h5 @@ -2414,6 +2415,8 @@ ./tools/test/h5repack/testfiles/h5repack_fill.h5 ./tools/test/h5repack/testfiles/h5repack_filters.h5 ./tools/test/h5repack/testfiles/h5repack_fletcher.h5 +./tools/test/h5repack/testfiles/h5repack_fsm_aggr_nopersist.h5 +./tools/test/h5repack/testfiles/h5repack_fsm_aggr_persist.h5 ./tools/test/h5repack/testfiles/h5repack_hlink.h5 ./tools/test/h5repack/testfiles/h5repack.info ./tools/test/h5repack/testfiles/h5repack_layout.h5 @@ -2425,7 +2428,10 @@ ./tools/test/h5repack/testfiles/h5repack_nested_8bit_enum_deflated.h5 ./tools/test/h5repack/testfiles/h5repack_nested_8bit_enum.h5 ./tools/test/h5repack/testfiles/h5repack_nbit.h5 +./tools/test/h5repack/testfiles/h5repack_none.h5 ./tools/test/h5repack/testfiles/h5repack_objs.h5 +./tools/test/h5repack/testfiles/h5repack_paged_nopersist.h5 +./tools/test/h5repack/testfiles/h5repack_paged_persist.h5 ./tools/test/h5repack/testfiles/h5repack_refs.h5 ./tools/test/h5repack/testfiles/h5repack_shuffle.h5 ./tools/test/h5repack/testfiles/h5repack_soffset.h5 @@ -2444,6 +2450,12 @@ ./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_test.ddl ./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_version_test.ddl ./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_zero.tst +./tools/test/h5repack/testfiles/GS.h5repack_paged_nopersist.h5.ddl +./tools/test/h5repack/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl +./tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl +./tools/test/h5repack/testfiles/SP.h5repack_paged_persist.h5.ddl +./tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl +./tools/test/h5repack/testfiles/STG.h5repack_none.h5.ddl ./tools/test/h5repack/testfiles/1_vds.h5-vds_dset_chunk20x10x5-v.ddl ./tools/test/h5repack/testfiles/2_vds.h5-vds_chunk3x6x9-v.ddl ./tools/test/h5repack/testfiles/3_1_vds.h5-vds_chunk2x5x8-v.ddl diff --git a/tools/test/h5repack/h5repack.sh.in b/tools/test/h5repack/h5repack.sh.in index bc6b527..e474bc3 100644 --- a/tools/test/h5repack/h5repack.sh.in +++ b/tools/test/h5repack/h5repack.sh.in @@ -36,6 +36,9 @@ H5DIFF_BIN=`pwd`/$H5DIFF # The path of the h5diff tool binary H5DUMP=../../src/h5dump/h5dump # The h5dump tool name H5DUMP_BIN=`pwd`/$H5DUMP # The path of the h5dump tool binary +H5STAT=../../src/h5stat/h5stat # The h5stat tool name +H5STAT_BIN=`pwd`/$H5STAT # The path of the h5stat tool binary + RM='rm -rf' CMP='cmp' DIFF='diff -c' @@ -101,6 +104,12 @@ $SRC_H5REPACK_TESTFILES/h5repack_refs.h5 $SRC_H5REPACK_TESTFILES/h5repack_shuffle.h5 $SRC_H5REPACK_TESTFILES/h5repack_soffset.h5 $SRC_H5REPACK_TESTFILES/h5repack_szip.h5 +$SRC_H5REPACK_TESTFILES/h5repack_aggr.h5 +$SRC_H5REPACK_TESTFILES/h5repack_fsm_aggr_nopersist.h5 +$SRC_H5REPACK_TESTFILES/h5repack_fsm_aggr_persist.h5 +$SRC_H5REPACK_TESTFILES/h5repack_none.h5 +$SRC_H5REPACK_TESTFILES/h5repack_paged_nopersist.h5 +$SRC_H5REPACK_TESTFILES/h5repack_paged_persist.h5 $SRC_H5DIFF_TESTFILES/h5diff_attr1.h5 $SRC_TOOLS_TESTFILES/tfamily00000.h5 $SRC_TOOLS_TESTFILES/tfamily00001.h5 @@ -156,6 +165,12 @@ $SRC_H5REPACK_TESTFILES/2_vds.h5-vds_chunk3x6x9-v.ddl $SRC_H5REPACK_TESTFILES/3_1_vds.h5-vds_chunk2x5x8-v.ddl $SRC_H5REPACK_TESTFILES/4_vds.h5-vds_compa-v.ddl $SRC_H5REPACK_TESTFILES/4_vds.h5-vds_conti-v.ddl +$SRC_H5REPACK_TESTFILES/SP.h5repack_fsm_aggr_nopersist.h5.ddl +$SRC_H5REPACK_TESTFILES/S.h5repack_fsm_aggr_persist.h5.ddl +$SRC_H5REPACK_TESTFILES/STG.h5repack_none.h5.ddl +$SRC_H5REPACK_TESTFILES/GS.h5repack_paged_nopersist.h5.ddl +$SRC_H5REPACK_TESTFILES/SP.h5repack_paged_persist.h5.ddl +$SRC_H5REPACK_TESTFILES/SPT.h5repack_aggr.h5.ddl " # @@ -707,6 +722,58 @@ TOOLTEST_DUMP() rm -f $outfile } +# This is similar to TOOLTEST_DUMP(). +# Test h5repack with options added for paged aggregation. +# h5stat is used on the repacked file and the expected output +# is compared for correctness. +# +TOOLTEST_STAT() +{ + infile=$2 + outfile=out-$1.$2 + expect="$TESTDIR/$1.$2.ddl" + actual="$TESTDIR/out-$1.$2.out" + actual_err="$TESTDIR/out-$1.$2.err" + + shift + shift + + # Run test. + TESTING $H5REPACK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile + ) >$actual 2>$actual_err + RET=$? + if [ $RET != 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + VERIFY h5stat output $@ + ( + cd $TESTDIR + $RUNSERIAL $H5STAT_BIN -S -s $outfile + ) >$actual 2>$actual_err + cat $actual_err >> $actual + + RET=$? + + fi + + if cmp -s $expect $actual; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected result (*.ddl) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /' + fi + + rm -f $actual $actual_err + rm -f $outfile +} + # TOOLTEST_META: # Test metadata block size option. # Reason to create a function here is to localize all special steps related to @@ -1057,6 +1124,42 @@ fi arg="tordergr.h5 -L" TOOLTEST_DUMP crtorder $arg +################################################################################################### +# Testing paged aggregation related options: +# -G pagesize +# -P 1 or 0 +# -S strategy +# -T threshold +# +# The testfiles used are generated by test/gen_filespace.c and the file names are prepended with "h5repack_": +# (1) "fsm_aggr_nopersist.h5" /* H5F_FSPACE_STRATEGY_FSM_AGGR + not persisting free-space */ +# (2) "fsm_aggr_persist.h5" /* H5F_FSPACE_STRATEGY_FSM_AGGR + persisting free-space */ +# (3) "paged_nopersist.h5" /* H5F_FSPACE_STRATEGY_PAGE + not persisting free-space */ +# (4) "paged_persist.h5" /* H5F_FSPACE_STRATEGY_PAGE + persisting free-space */ +# (5) "aggr.h5" /* H5F_FSPACE_STRATEGY_AGGR */ +# (6) "none.h5" /* H5F_FSPACE_STRATEGY_NONE */ +# +##################################################################################################### +# +arg="h5repack_fsm_aggr_nopersist.h5 -S PAGE -P 1" +TOOLTEST_STAT SP $arg +# +arg="h5repack_fsm_aggr_persist.h5 -S AGGR" +TOOLTEST_STAT S $arg +# +arg="h5repack_none.h5 -S PAGE -T 10 -G 2048" +TOOLTEST_STAT STG $arg +# +arg="h5repack_paged_nopersist.h5 -G 512 -S AGGR" +TOOLTEST_STAT GS $arg +# +arg="h5repack_paged_persist.h5 -S NONE -P 1" +TOOLTEST_STAT SP $arg +# +arg="h5repack_aggr.h5 -S FSM_AGGR -P 1 -T 5" +TOOLTEST_STAT SPT $arg + + ######################################################### # layout options (these files have no filters) ######################################################### diff --git a/tools/test/h5repack/testfiles/GS.h5repack_paged_nopersist.h5.ddl b/tools/test/h5repack/testfiles/GS.h5repack_paged_nopersist.h5.ddl new file mode 100644 index 0000000..d8ca992 --- /dev/null +++ b/tools/test/h5repack/testfiles/GS.h5repack_paged_nopersist.h5.ddl @@ -0,0 +1,15 @@ +Filename: out-GS.h5repack_paged_nopersist.h5 +Free-space persist: FALSE +Free-space section threshold: 1 bytes +Small size free-space sections (< 10 bytes): + Total # of small size sections: 0 +Free-space section bins: + Total # of sections: 0 +File space management strategy: H5F_FSPACE_STRATEGY_AGGR +File space page size: 512 bytes +Summary of file space information: + File metadata: 1272 bytes + Raw data: 400 bytes + Amount/Percent of tracked free space: 0 bytes/0.0% + Unaccounted space: 776 bytes +Total space: 2448 bytes diff --git a/tools/test/h5repack/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl b/tools/test/h5repack/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl new file mode 100644 index 0000000..87ccbc4 --- /dev/null +++ b/tools/test/h5repack/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl @@ -0,0 +1,15 @@ +Filename: out-S.h5repack_fsm_aggr_persist.h5 +Free-space persist: FALSE +Free-space section threshold: 1 bytes +Small size free-space sections (< 10 bytes): + Total # of small size sections: 0 +Free-space section bins: + Total # of sections: 0 +File space management strategy: H5F_FSPACE_STRATEGY_AGGR +File space page size: 4096 bytes +Summary of file space information: + File metadata: 1272 bytes + Raw data: 400 bytes + Amount/Percent of tracked free space: 0 bytes/0.0% + Unaccounted space: 776 bytes +Total space: 2448 bytes diff --git a/tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl b/tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl new file mode 100644 index 0000000..affa93b --- /dev/null +++ b/tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl @@ -0,0 +1,16 @@ +Filename: out-SP.h5repack_fsm_aggr_nopersist.h5 +Free-space persist: TRUE +Free-space section threshold: 1 bytes +Small size free-space sections (< 10 bytes): + Total # of small size sections: 0 +Free-space section bins: + # of sections of size 1000 - 9999: 2 + Total # of sections: 2 +File space management strategy: H5F_FSPACE_STRATEGY_PAGE +File space page size: 4096 bytes +Summary of file space information: + File metadata: 1602 bytes + Raw data: 400 bytes + Amount/Percent of tracked free space: 6307 bytes/38.5% + Unaccounted space: 8075 bytes +Total space: 16384 bytes diff --git a/tools/test/h5repack/testfiles/SP.h5repack_paged_persist.h5.ddl b/tools/test/h5repack/testfiles/SP.h5repack_paged_persist.h5.ddl new file mode 100644 index 0000000..1084090 --- /dev/null +++ b/tools/test/h5repack/testfiles/SP.h5repack_paged_persist.h5.ddl @@ -0,0 +1,15 @@ +Filename: out-SP.h5repack_paged_persist.h5 +Free-space persist: FALSE +Free-space section threshold: 1 bytes +Small size free-space sections (< 10 bytes): + Total # of small size sections: 0 +Free-space section bins: + Total # of sections: 0 +File space management strategy: H5F_FSPACE_STRATEGY_NONE +File space page size: 4096 bytes +Summary of file space information: + File metadata: 1272 bytes + Raw data: 400 bytes + Amount/Percent of tracked free space: 0 bytes/0.0% + Unaccounted space: 0 bytes +Total space: 1672 bytes diff --git a/tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl b/tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl new file mode 100644 index 0000000..1ce06b2 --- /dev/null +++ b/tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl @@ -0,0 +1,16 @@ +Filename: out-SPT.h5repack_aggr.h5 +Free-space persist: TRUE +Free-space section threshold: 5 bytes +Small size free-space sections (< 10 bytes): + Total # of small size sections: 0 +Free-space section bins: + # of sections of size 100 - 999: 1 + Total # of sections: 1 +File space management strategy: H5F_FSPACE_STRATEGY_FSM_AGGR +File space page size: 4096 bytes +Summary of file space information: + File metadata: 1485 bytes + Raw data: 400 bytes + Amount/Percent of tracked free space: 680 bytes/26.5% + Unaccounted space: 0 bytes +Total space: 2565 bytes diff --git a/tools/test/h5repack/testfiles/STG.h5repack_none.h5.ddl b/tools/test/h5repack/testfiles/STG.h5repack_none.h5.ddl new file mode 100644 index 0000000..51cd7e5 --- /dev/null +++ b/tools/test/h5repack/testfiles/STG.h5repack_none.h5.ddl @@ -0,0 +1,15 @@ +Filename: out-STG.h5repack_none.h5 +Free-space persist: FALSE +Free-space section threshold: 10 bytes +Small size free-space sections (< 10 bytes): + Total # of small size sections: 0 +Free-space section bins: + Total # of sections: 0 +File space management strategy: H5F_FSPACE_STRATEGY_PAGE +File space page size: 2048 bytes +Summary of file space information: + File metadata: 1272 bytes + Raw data: 400 bytes + Amount/Percent of tracked free space: 0 bytes/0.0% + Unaccounted space: 2424 bytes +Total space: 4096 bytes diff --git a/tools/test/h5repack/testfiles/h5repack_aggr.h5 b/tools/test/h5repack/testfiles/h5repack_aggr.h5 new file mode 100644 index 0000000..8da0751 Binary files /dev/null and b/tools/test/h5repack/testfiles/h5repack_aggr.h5 differ diff --git a/tools/test/h5repack/testfiles/h5repack_fsm_aggr_nopersist.h5 b/tools/test/h5repack/testfiles/h5repack_fsm_aggr_nopersist.h5 new file mode 100644 index 0000000..1cd5f3f Binary files /dev/null and b/tools/test/h5repack/testfiles/h5repack_fsm_aggr_nopersist.h5 differ diff --git a/tools/test/h5repack/testfiles/h5repack_fsm_aggr_persist.h5 b/tools/test/h5repack/testfiles/h5repack_fsm_aggr_persist.h5 new file mode 100644 index 0000000..469dbce Binary files /dev/null and b/tools/test/h5repack/testfiles/h5repack_fsm_aggr_persist.h5 differ diff --git a/tools/test/h5repack/testfiles/h5repack_none.h5 b/tools/test/h5repack/testfiles/h5repack_none.h5 new file mode 100644 index 0000000..2e4d789 Binary files /dev/null and b/tools/test/h5repack/testfiles/h5repack_none.h5 differ diff --git a/tools/test/h5repack/testfiles/h5repack_paged_nopersist.h5 b/tools/test/h5repack/testfiles/h5repack_paged_nopersist.h5 new file mode 100644 index 0000000..352e586 Binary files /dev/null and b/tools/test/h5repack/testfiles/h5repack_paged_nopersist.h5 differ diff --git a/tools/test/h5repack/testfiles/h5repack_paged_persist.h5 b/tools/test/h5repack/testfiles/h5repack_paged_persist.h5 new file mode 100644 index 0000000..64c3b23 Binary files /dev/null and b/tools/test/h5repack/testfiles/h5repack_paged_persist.h5 differ -- cgit v0.12