[svn-r8869] Purpose:

h5repack changes

Description:
there were some requests to change some minor h5repack features
h5repack only made a warning about a non available filter in verbose mode ( -v )
without -v it kept silent, and users sometimes missed this warning

the request was that it should print this warning always. so, the new format, is e.g

./h5repack -i test_szip.h5 -o out.h5
Warning: dataset </dset_szip> cannot be read, SZIP filter is not available


due to this, and to avoid a lot of these messages in the shell test script, I modified
the script h5repack.sh so that it detects the presence of all filters in the environment
(previously it only detected SZIP)
the test files were also divided in more files , to make the script code easier to
follow


Solution:

Platforms tested:
linux
AIX (no szip)
solaris (no szip, no gzip )

Misc. update:

1 files changed, 198 insertions, 136 deletions

diff --git a/tools/h5repack/h5repack.sh.in b/tools/h5repack/h5repack.sh.in
index 17c6869..e4d65de 100755
--- a/tools/h5repack/h5repack.sh.in
+++ b/tools/h5repack/h5repack.sh.in
@@ -17,6 +17,10 @@
 # Determine if SZIP has the encoder available
 USE_FILTER_SZIP_ENCODER="@USE_FILTER_SZIP_ENCODER@"
 USE_FILTER_SZIP="@USE_FILTER_SZIP@"
+USE_FILTER_DEFLATE="@USE_FILTER_DEFLATE@"
+USE_FILTER_SHUFFLE="@USE_FILTER_SHUFFLE@"
+USE_FILTER_FLETCHER32="@USE_FILTER_FLETCHER32@"
+
 
 H5REPACK=h5repack               # The tool name
 H5REPACK_BIN=`pwd`/$H5REPACK    # The path of the tool binary
@@ -64,20 +68,20 @@ DIFFTEST()
 {
     VERIFY  $@
     if [ "`uname -s`" = "TFLOPS O/S" ]; then
-	$RUNSERIAL $H5DIFF_BIN $@
+     $RUNSERIAL $H5DIFF_BIN $@
     else
-	$RUNSERIAL $H5DIFF_BIN "$@"
+     $RUNSERIAL $H5DIFF_BIN "$@"
     fi
     RET=$?
     if [ $RET != 0 ] ; then
- 	echo "*FAILED*"
+         echo "*FAILED*"
          nerrors="`expr $nerrors + 1`"
     else
          echo " PASSED"
     fi
-				    
+        
 }
-										
+          
 # Call h5repack
 #
 TOOLTEST() 
@@ -91,18 +95,18 @@ TOOLTEST()
     outfile=out.$1
     shift
     if [ "`uname -s`" = "TFLOPS O/S" ]; then
-	$RUNSERIAL $H5REPACK_BIN -i $infile -o $outfile $@
+     $RUNSERIAL $H5REPACK_BIN -i $infile -o $outfile $@
     else
-	$RUNSERIAL $H5REPACK_BIN -i $infile -o $outfile "$@"
+     $RUNSERIAL $H5REPACK_BIN -i $infile -o $outfile "$@"
     fi
 
     RET=$?
     if [ $RET != 0 ] ; then
-	echo "*FAILED*"
-	nerrors="`expr $nerrors + 1`"
+     echo "*FAILED*"
+     nerrors="`expr $nerrors + 1`"
     else
-	echo " PASSED"
-	DIFFTEST $infile $outfile
+     echo " PASSED"
+     DIFFTEST $infile $outfile
     fi
     rm -f $outfile
 }
@@ -110,9 +114,11 @@ TOOLTEST()
 #
 # The tests
 # We use the files generated by h5repacktst
-# Each run generates "file4.out.h5" and the tool h5diff is used to
+# Each run generates "<file>.out.h5" and the tool h5diff is used to
 # compare the input and output files
 #
+# the tests are the same as the program h5repacktst, but run from the CLI 
+#
 
 # See which filters are usable (and skip tests for filters we
 # don't have).  Do this by searching H5pubconf.h to see which
@@ -120,177 +126,233 @@ TOOLTEST()
 # A filter value of 3 means the filter can read and write, 2
 # is read-only, 1 is write-only, and 0 is not present.
 
-# copy files
+# copy files (these files have no filters; test2.h5 has copied references that h5diff detects)
 TOOLTEST test1.h5
 TOOLTEST test3.h5
+TOOLTEST test4.h5
+TOOLTEST test5.h5
 
+# use test4.h5 to write some filters  (this file has  no filters)
 
-arg="test4.h5"
-if test $USE_FILTER_SZIP_ENCODER = "yes" -o $USE_FILTER_SZIP = "no" ; then
+# gzip with individual object
+arg="test4.h5 -f dset1:GZIP=1  -l dset1:CHUNK=20x10"
+if test  $USE_FILTER_DEFLATE != "yes" ; then
+ SKIP $arg
+else
  TOOLTEST $arg
+fi
+  
+# gzip for all 
+arg="test4.h5 -f GZIP=1"
+if test  $USE_FILTER_DEFLATE != "yes" ; then
+ SKIP $arg
 else
- SKIP $arg 
+ TOOLTEST $arg
 fi
 
-#TOOLTEST test5.h5
-
-# remove all  filters
-TOOLTEST test4.h5 -f NONE
-
-# remove one filter
-arg="test4.h5 -f dset_gzip:NONE"
-if test $USE_FILTER_SZIP_ENCODER = "yes" -o $USE_FILTER_SZIP = "no" ; then
+# szip with individual object
+arg="test4.h5 -f dset2:SZIP=8  -l dset2:CHUNK=20x10"
+if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then
+ SKIP $arg
+else
  TOOLTEST $arg 
+fi
+
+# szip for all
+arg="test4.h5 -f SZIP=8"
+if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then
+ SKIP $arg
 else
- SKIP $arg 
+ TOOLTEST $arg 
 fi
 
-# gzip
-arg="test4.h5 -f dset1:GZIP=9"
-if test $USE_FILTER_SZIP_ENCODER = "yes" -o $USE_FILTER_SZIP = "no" ; then 
+# shuffle with individual object
+arg="test4.h5 -f dset2:SHUF  -l dset2:CHUNK=20x10"
+if test $USE_FILTER_SHUFFLE != "yes"  ; then
+ SKIP $arg
+else
  TOOLTEST $arg 
+fi
+  
+
+# shuffle for all
+arg="test4.h5 -f SHUF"
+if test $USE_FILTER_SHUFFLE != "yes"  ; then
+ SKIP $arg
 else
- SKIP $arg 
+ TOOLTEST $arg
 fi
-TOOLTEST test4.h5 -f GZIP=1
-
-# szip
-arg1="test4.h5 -f dset1:SZIP=8"
-arg2="test4.h5 -f SZIP=8"
-if test $USE_FILTER_SZIP_ENCODER = "yes" -a $USE_FILTER_SZIP = "yes" ; then
- TOOLTEST $arg1 
- TOOLTEST $arg2 
+  
+# fletcher32  with individual object
+arg="test4.h5 -f dset2:FLET  -l dset2:CHUNK=20x10"
+if test $USE_FILTER_FLETCHER32 != "yes"  ; then
+ SKIP $arg
 else
- SKIP $arg1 
- SKIP $arg2 
+ TOOLTEST $arg
 fi
 
-# shuffle
-arg="test4.h5 -f dset1:SHUF"
-if test $USE_FILTER_SZIP_ENCODER = "yes" -o $USE_FILTER_SZIP = "no" ; then
- TOOLTEST $arg 
+# fletcher32 for all
+arg="test4.h5 -f FLET"
+if test $USE_FILTER_FLETCHER32 != "yes"  ; then
+ SKIP $arg
 else
- SKIP $arg 
+ TOOLTEST $arg
 fi
-TOOLTEST test4.h5 -f SHUF
 
-# fletcher
-arg="test4.h5 -f dset1:FLET"
-if test $USE_FILTER_SZIP_ENCODER = "yes" -o $USE_FILTER_SZIP = "no" ; then
- TOOLTEST $arg 
+# all filters
+arg="test4.h5 -f dset2:SHUF -f dset2:FLET -f dset2:SZIP=8 -f dset2:GZIP=1 -l dset2:CHUNK=20x10"
+if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SHUFFLE != "yes" -o $USE_FILTER_FLETCHER32 != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then
+ SKIP $arg
 else
- SKIP $arg 
+ TOOLTEST $arg
 fi
-TOOLTEST test4.h5 -f FLET
-
-#layout chunk
-arg1="test4.h5 -l dset1:CHUNK=20x10"
-arg2="test4.h5 -l CHUNK=20x10"
-if test $USE_FILTER_SZIP_ENCODER = "yes" -o $USE_FILTER_SZIP = "no" ; then
- TOOLTEST $arg1 
- TOOLTEST $arg2 
+  
+###########################################################
+# the following tests assume the input files have filters
+###########################################################
+
+# szip copy
+arg="test_szip.h5"
+if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then
+ SKIP $arg
 else
- SKIP $arg1 
- SKIP $arg2 
+ TOOLTEST $arg
+fi
+  
+# szip remove
+arg="test_szip.h5 -f dset_szip:NONE"
+if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then
+ SKIP $arg
+else
+ TOOLTEST $arg
+fi
+  
+# deflate copy
+arg="test_deflate.h5"
+if test $USE_FILTER_DEFLATE != "yes" ; then
+ SKIP $arg
+else
+ TOOLTEST $arg
 fi
 
-#layout compact
-arg1="test4.h5 -l dset1:COMPA"
-arg2="test4.h5 -l COMPA"
-if test $USE_FILTER_SZIP_ENCODER = "yes" -o $USE_FILTER_SZIP = "no" ; then
- TOOLTEST $arg1 
- TOOLTEST $arg2 
+# deflate remove
+arg="test_deflate.h5 -f dset_deflate:NONE"
+if test $USE_FILTER_DEFLATE != "yes" ; then
+ SKIP $arg
 else
- SKIP $arg1 
- SKIP $arg2 
+ TOOLTEST $arg
+fi
+    
+# shuffle copy
+arg="test_shuffle.h5"
+if test $USE_FILTER_SHUFFLE != "yes" ; then
+ SKIP $arg
+else
+ TOOLTEST $arg
 fi
 
-#layout contiguous
-arg1="test4.h5 -l dset1:CONTI"
-arg2="test4.h5 -l CONTI"
-if test $USE_FILTER_SZIP_ENCODER = "yes" -o $USE_FILTER_SZIP = "no" ; then
- TOOLTEST $arg1 
- TOOLTEST $arg2 
+# shuffle remove
+arg="test_shuffle.h5 -f dset_shuffle:NONE"
+if test $USE_FILTER_SHUFFLE != "yes" ; then
+ SKIP $arg
 else
- SKIP $arg1 
- SKIP $arg2 
+ TOOLTEST $arg
 fi
 
-#conversions
-arg1="test4.h5 -l dset_compact:CONTI"
-arg2="test4.h5 -l dset_compact:CHUNK=2x5"
-arg3="test4.h5 -l dset_compact:COMPA"
-arg4="test4.h5 -l dset_contiguous:COMPA"
-arg5="test4.h5 -l dset_contiguous:CHUNK=3x6"
-arg6="test4.h5 -l dset_contiguous:CONTI"
-arg7="test4.h5 -l dset_chunk:COMPA"
-arg8="test4.h5 -l dset_chunk:CONTI"
-arg9="test4.h5 -l dset_chunk:CHUNK=18x13"
-if test $USE_FILTER_SZIP_ENCODER = "yes" -o $USE_FILTER_SZIP = "no" ; then
- TOOLTEST $arg1 
- TOOLTEST $arg2 
- TOOLTEST $arg3 
- TOOLTEST $arg4 
- TOOLTEST $arg5 
- TOOLTEST $arg6 
- TOOLTEST $arg7 
- TOOLTEST $arg8 
- TOOLTEST $arg9 
+# fletcher32 copy
+arg="test_fletcher32.h5"
+if test $USE_FILTER_FLETCHER32 != "yes" ; then
+ SKIP $arg
 else
- SKIP $arg1 
- SKIP $arg2 
- SKIP $arg3 
- SKIP $arg4 
- SKIP $arg5 
- SKIP $arg6 
- SKIP $arg7 
- SKIP $arg8 
- SKIP $arg9 
+ TOOLTEST $arg
 fi
 
-#filters
-arg1="test4.h5 -f dset1:SHUF -f dset1,dset2:GZIP=6"
-arg2="test4.h5 -l dset1:CHUNK=20x10 -f dset1,dset2:SZIP=8"
-if test $USE_FILTER_SZIP_ENCODER = "yes" -a $USE_FILTER_SZIP = "yes"; then
- TOOLTEST $arg1 
- TOOLTEST $arg2 
-elif test $USE_FILTER_SZIP = "yes" ; then    # SZIP has no encoder
- SKIP $arg1
- SKIP $arg2
-else        # SZIP is absent
- TOOLTEST $arg1 
- SKIP $arg2 
+# fletcher32 remove
+arg="test_fletcher32.h5 -f dset_fletcher32:NONE"
+if test $USE_FILTER_FLETCHER32 != "yes" ; then
+ SKIP $arg
+else
+ TOOLTEST $arg
+fi        
+
+# remove all  filters
+arg="test_all.h5 -f NONE"
+if test $USE_FILTER_FLETCHER32 != "yes" -o $USE_FILTER_DEFLATE != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SHUFFLE != "yes"  ; then
+ SKIP $arg
+else
+ TOOLTEST $arg
 fi
 
 #filter conversions
-arg1="test4.h5 -f dset_gzip:SZIP=8"
-arg2="test4.h5 -f dset_szip:GZIP=1"
-arg3="test4.h5 -f dset_all:GZIP=1"
-if test $USE_FILTER_SZIP_ENCODER = "yes" -a $USE_FILTER_SZIP = "yes" ; then
- TOOLTEST $arg1 
- TOOLTEST $arg2 
- TOOLTEST $arg3 
-elif test $USE_FILTER_SZIP = "yes" ; then    # SZIP has no encoder
- SKIP $arg1 
- SKIP $arg2 
- SKIP $arg3 
-else        # SZIP is absent
- SKIP $arg1
- TOOLTEST $arg2
- TOOLTEST $arg3
+
+arg="test_deflate.h5 -f dset_deflate:SZIP=8"
+if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then
+ SKIP $arg
+else
+ TOOLTEST $arg 
+fi
+
+arg="test_szip.h5 -f dset_szip:GZIP=1"
+if test $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SZIP_ENCODER != "yes"  -o $USE_FILTER_DEFLATE != "yes" ; then
+ SKIP $arg
+else
+ TOOLTEST $arg 
 fi
 
+
 #limit
-TOOLTEST test4.h5 -f GZIP=1 -m 1024
+arg="test4.h5 -f GZIP=1 -m 1024"
+if test $USE_FILTER_DEFLATE != "yes"  ; then
+ SKIP $arg
+else
+ TOOLTEST $arg
+fi
 
 #file
 arg="test4.h5 -e ../testfiles/h5repack_info.txt"
-if test $USE_FILTER_SZIP_ENCODER = "yes" -a $USE_FILTER_SZIP = "yes" ; then
- TOOLTEST $arg 
-else
+if test $USE_FILTER_DEFLATE != "yes" ; then
  SKIP $arg 
+else
+ TOOLTEST $arg 
 fi
 
+
+#########################################################
+# layout options (these files have no filters)
+#########################################################
+
+TOOLTEST test4.h5 -l dset2:CHUNK=20x10
+TOOLTEST test4.h5 -l CHUNK=20x10
+TOOLTEST test4.h5 -l dset2:CONTI
+TOOLTEST test4.h5 -l CONTI
+TOOLTEST test4.h5 -l dset2:COMPA
+TOOLTEST test4.h5 -l COMPA
+
+
+################################################################
+# layout conversions (file has no filters)
+###############################################################
+
+arg1="test4.h5 -l dset_compact:CONTI"
+arg2="test4.h5 -l dset_compact:CHUNK=2x5"
+arg3="test4.h5 -l dset_compact:COMPA"
+arg4="test4.h5 -l dset_contiguous:COMPA"
+arg5="test4.h5 -l dset_contiguous:CHUNK=3x6"
+arg6="test4.h5 -l dset_contiguous:CONTI"
+arg7="test4.h5 -l dset_chunk:COMPA"
+arg8="test4.h5 -l dset_chunk:CONTI"
+arg9="test4.h5 -l dset_chunk:CHUNK=18x13"
+TOOLTEST $arg1
+TOOLTEST $arg2
+TOOLTEST $arg3
+TOOLTEST $arg4
+TOOLTEST $arg5
+TOOLTEST $arg6
+TOOLTEST $arg7
+TOOLTEST $arg8
+TOOLTEST $arg9
+
+
 if test $nerrors -eq 0 ; then
    echo "All $H5REPACK tests passed."
 fi