[svn-r8229] Purpose:

1) new function  for tools library
2) new test script for h5repack


Description:
1) currently all the tools (h5dump, h5diff, etc)  do not check if a filter is available
for reading some dataset that might have a filter not available on the current configuration (the behaviour
of the tools until now was to trigger a library error, saying that the dataset cannot be read
due to the lack of the filter)


Solution:
1) added a new function h5tools_canreadf that checks if a dataset can be read
depending on the availability of filters.
this function was added in calls for h5diff and h5repack.
instead of triggering the library error, a message is printed, saying that the dataset
cannot be read (the print is optional, it is on on verbose mode)

2) added a shell script that tests the commannd line tool behaviour of h5repack
the script does a series of runs of h5repack  with several options on the same file (this file test4.h5
was added to the testfiles dir).
then, it runs the h5diff tool, with the input and output files , in each run.
the goal of the test is also to check item 1) . the binary file was saved with filters
that might not be available on other configurations


Platforms tested:
linux (all filters enabled)
linux (some filters disabled)
solaris (some filters disabled)
AIX (some filters disabled)
windows (all filters on and off )


Misc. update:

11 files changed, 163 insertions, 34 deletions

diff --git a/tools/h5repack/Makefile.in b/tools/h5repack/Makefile.in
index 757771d..d9e2f92 100644
--- a/tools/h5repack/Makefile.in
+++ b/tools/h5repack/Makefile.in
@@ -28,7 +28,8 @@ CPPFLAGS=-I. -I$(srcdir) -I$(top_builddir)/src -I$(top_srcdir)/src  \
 ## Test programs and scripts.
 ##
 TEST_PROGS=h5repacktst
-TEST_SCRIPTS=
+TEST_SCRIPTS=$(srcdir)/h5repack.sh
+
 
 ## These are our main targets: library and tools.
 ##
diff --git a/tools/h5repack/h5repack.c b/tools/h5repack/h5repack.c
index 8cca76d..3863846 100644
--- a/tools/h5repack/h5repack.c
+++ b/tools/h5repack/h5repack.c
@@ -408,10 +408,10 @@ void read_info(const char *filename,
    break;
    
  /*-------------------------------------------------------------------------
-  * comp
+  * filter
   *-------------------------------------------------------------------------
   */
-  if (strcmp(stype,"-t") == 0) { 
+  if (strcmp(stype,"-f") == 0) { 
 
    /* find begining of info */
    i=0; c='0';
@@ -435,10 +435,10 @@ void read_info(const char *filename,
    }
   }
  /*-------------------------------------------------------------------------
-  * chunk
+  * layout
   *-------------------------------------------------------------------------
   */
-  else if (strcmp(stype,"-c") == 0) { 
+  else if (strcmp(stype,"-l") == 0) { 
    
    /* find begining of info */
    i=0; c='0';
diff --git a/tools/h5repack/h5repack.h b/tools/h5repack/h5repack.h
index fa1c4b4..5abaf13 100644
--- a/tools/h5repack/h5repack.h
+++ b/tools/h5repack/h5repack.h
@@ -18,6 +18,9 @@
 
 #include "hdf5.h"
 #include "h5trav.h"
+#include "h5diff.h"
+#include "h5tools.h"
+
 
 #if 0
 #define H5_REPACK_DEBUG
@@ -203,6 +206,10 @@ int check_szip(hid_t type_id,   /* dataset datatype */
                unsigned *szip_pixels_per_block /*IN,OUT*/,
                pack_opt_t *options);
 
+int can_read(const char* name,    /* object name from traverse list */
+             hid_t dcpl_id,       /* dataset creation property list */
+             pack_opt_t *options); /* repack options */
+
 
 /*-------------------------------------------------------------------------
  * layout functions
diff --git a/tools/h5repack/h5repack.sh b/tools/h5repack/h5repack.sh
index e55e8aa..8b8f44a 100644
--- a/tools/h5repack/h5repack.sh
+++ b/tools/h5repack/h5repack.sh
@@ -14,6 +14,110 @@
 #
 # Tests for the h5repack tool
 
+H5REPACK=h5repack               # The tool name
+H5REPACK_BIN=`pwd`/$H5REPACK    # The path of the tool binary
 
-#run the h5repack test
-./h5repacktst
+H5DIFF=../h5diff/h5diff         # The h5diff tool name
+H5DIFF_BIN=`pwd`/$H5DIFF        # The path of the h5diff  tool binary
+
+nerrors=0
+verbose=yes
+
+# The build (current) directory might be different than the source directory.
+#
+if test -z "$srcdir"; then
+   srcdir=.
+fi
+
+test -d ../testfiles || mkdir ../testfiles
+
+# Print a line-line message left justified in a field of 70 characters
+# beginning with the word "Testing".
+#
+TESTING() {
+   SPACES="                                                               "
+   echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012'
+}
+
+# Print a line-line message left justified in a field of 70 characters
+# beginning with the word "Verifying".
+#
+VERIFY() {
+ SPACES="                                                               "
+ echo "Testing h5diff output $* $SPACES" | cut -c1-70 | tr -d '\012'
+}
+      
+
+# Call h5repack
+#
+TOOLTEST() 
+{
+   # Run test.
+   # Tflops interprets "$@" as "" when no parameter is given (e.g., the
+   # case of missing file name).  Changed it to use $@ till Tflops fixes it.
+   TESTING $H5REPACK $@
+   (
+      cd $srcdir/../testfiles
+      if [ "`uname -s`" = "TFLOPS O/S" ]; then
+        $RUNSERIAL $H5REPACK_BIN $@
+      else
+        $RUNSERIAL $H5REPACK_BIN "$@"
+      fi
+   ) 
+echo " PASSED"
+}
+
+# Call the h5diff tool
+#
+DIFFTEST() 
+{
+   VERIFY  $@
+      (
+	cd $srcdir/../testfiles
+	if [ "`uname -s`" = "TFLOPS O/S" ]; then
+	$RUNSERIAL $H5DIFF_BIN $@
+	else
+	$RUNSERIAL $H5DIFF_BIN "$@"
+	fi
+      )
+  echo " PASSED"
+}
+										
+
+#
+#The tests
+#We use the file "test4.h5" generated by h5repacktst
+#Each run generates "file4.out.h5" and the tool h5diff is used to
+# compare the input and output files
+#
+TOOLTEST -i test4.h5 -o test4.out.h5
+DIFFTEST test4.h5 test4.out.h5  
+TOOLTEST -i test4.h5 -o test4.out.h5 -f "GZIP 1"
+DIFFTEST test4.h5 test4.out.h5
+TOOLTEST -i test4.h5 -o test4.out.h5 -f "SZIP 8"
+DIFFTEST test4.h5 test4.out.h5
+TOOLTEST -i test4.h5 -o test4.out.h5 -f "SHUF"
+DIFFTEST test4.h5 test4.out.h5
+TOOLTEST -i test4.h5 -o test4.out.h5 -f "FLET"
+DIFFTEST test4.h5 test4.out.h5
+TOOLTEST -i test4.h5 -o test4.out.h5 -f "dset1:SHUF" -f "dset1,dset2:GZIP 6"
+DIFFTEST test4.h5 test4.out.h5
+TOOLTEST -i test4.h5 -o test4.out.h5 -l "dset1:CHUNK 20x20" -f "dset1,dset2:SZIP 8"
+DIFFTEST test4.h5 test4.out.h5
+TOOLTEST -i test4.h5 -o test4.out.h5 -l "CHUNK 20x20" 
+DIFFTEST test4.h5 test4.out.h5
+TOOLTEST -i test4.h5 -o test4.out.h5 -l "COMPA" 
+DIFFTEST test4.h5 test4.out.h5
+TOOLTEST -i test4.h5 -o test4.out.h5 -l "CONTI" 
+DIFFTEST test4.h5 test4.out.h5
+TOOLTEST -i test4.h5 -o test4.out.h5 -f "GZIP 1" -m 1024
+DIFFTEST test4.h5 test4.out.h5
+TOOLTEST -i test4.h5 -o test4.out.h5 -f "NONE" 
+DIFFTEST test4.h5 test4.out.h5
+TOOLTEST -i test4.h5 -o test4.out.h5 -e "h5repack_info.txt" 
+DIFFTEST test4.h5 test4.out.h5
+
+
+
+
+exit $nerrors
diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c
index 5134891..c74f5a4 100644
--- a/tools/h5repack/h5repack_copy.c
+++ b/tools/h5repack/h5repack_copy.c
@@ -232,6 +232,15 @@ int do_copy_objects(hid_t fidin,
    if (options->verbose)
     print_filters(dcpl_id);
 
+/*-------------------------------------------------------------------------
+ * check if the dataset creation property list has filters that 
+ * are not registered in the current configuration 
+ * 1) the external filters GZIP and SZIP might not be available
+ * 2) the internal filters might be turned off
+ *-------------------------------------------------------------------------
+ */
+   if (h5tools_canreadf((options->verbose?travt->objs[i].name:NULL),dcpl_id)==1)
+   {
 
 /*-------------------------------------------------------------------------
  * references are a special case
@@ -314,6 +323,8 @@ int do_copy_objects(hid_t fidin,
      free(buf);
     
    }/*H5T_STD_REF_OBJ*/
+   }/*can_read*/
+
 
 /*-------------------------------------------------------------------------
  * close
diff --git a/tools/h5repack/h5repack_filters.c b/tools/h5repack/h5repack_filters.c
index ab592a0..9aacdd3 100644
--- a/tools/h5repack/h5repack_filters.c
+++ b/tools/h5repack/h5repack_filters.c
@@ -110,9 +110,6 @@ int filter_this(const char* name,    /* object name from traverse list */
  *
  *-------------------------------------------------------------------------
  */
-
-
-
 int apply_filters(const char* name,    /* object name from traverse list */
                   int rank,            /* rank of dataset */
                   hsize_t *dims,       /* dimensions of dataset */
@@ -260,7 +257,6 @@ out:
  if (options->verbose)
   printf("Warning: Filter could not be applied to <%s>\n",name);
  return 0;
-
 }
 
 
diff --git a/tools/h5repack/h5repack_main.c b/tools/h5repack/h5repack_main.c
index ef18dfa..71308d9 100644
--- a/tools/h5repack/h5repack_main.c
+++ b/tools/h5repack/h5repack_main.c
@@ -66,7 +66,6 @@ int main(int argc, char **argv)
   }
 
   else if (strcmp(argv[i], "-m") == 0) {       
-   
    options.threshold = parse_number(argv[i+1]);
    if (options.threshold==-1) {
     printf("Error: Invalid treshold size <%s>\n",argv[i+1]);
@@ -115,7 +114,7 @@ int main(int argc, char **argv)
 static 
 void usage(void)
 {
- printf("h5repack -i input -o output [-h] [-v] [-t 'comp_info'] [-c 'chunk_info'][-m number] \n");
+ printf("h5repack -i input -o output [-h] [-v] [-f 'comp_info'] [-l 'chunk_info'][-m number][-e file] \n");
  printf("\n");
  printf("-i input          Input HDF5 File\n");
  printf("-o output         Output HDF5 File\n");
@@ -147,7 +146,7 @@ void usage(void)
  printf("       it is the chunk size of each dimension:\n");
  printf("       <dim_1 x dim_2 x ... dim_n>\n");
  printf("\n");
- printf("-e file           File with the above informatuion info in it (instead of the two above options)\n");
+ printf("-e file           File with the -f and -l options (only filter and layout flags)\n");
  printf("-m number         Do not apply the filter to objects which size in bytes is smaller than number.\n");
  printf("                  If no size is specified a minimum of 1024 bytes is assumed.\n");
  printf("\n");
diff --git a/tools/h5repack/h5repack_refs.c b/tools/h5repack/h5repack_refs.c
index a39a5ae..5955a08 100644
--- a/tools/h5repack/h5repack_refs.c
+++ b/tools/h5repack/h5repack_refs.c
@@ -127,6 +127,16 @@ int do_copy_refobjs(hid_t fidin,
     goto error;
 
 /*-------------------------------------------------------------------------
+ * check if the dataset creation property list has filters that 
+ * are not registered in the current configuration 
+ * 1) the external filters GZIP and SZIP might not be available
+ * 2) the internal filters might be turned off
+ *-------------------------------------------------------------------------
+ */
+   if (h5tools_canreadf((options->verbose?travt->objs[i].name:NULL),dcpl_id)==1)
+   {
+
+/*-------------------------------------------------------------------------
  * test for a valid output dataset
  *-------------------------------------------------------------------------
  */
@@ -311,7 +321,7 @@ int do_copy_refobjs(hid_t fidin,
  */
    if (copy_refs_attr(dset_in,dset_out,options,travt,fidout)<0) 
     goto error;
-
+   
 
 /*-------------------------------------------------------------------------
  * check for hard links
@@ -319,14 +329,18 @@ int do_copy_refobjs(hid_t fidin,
  */
    if (travt->objs[i].nlinks)
    {
-    for ( j=0; j<travt->objs[i].nlinks; j++)
-    {
-      H5Glink(fidout, 
+    for ( j=0; j<travt->objs[i].nlinks; j++){
+     H5Glink(fidout, 
       H5G_LINK_HARD,
       travt->objs[i].name,
       travt->objs[i].links[j].new_name);
     }
    }
+
+   if (H5Dclose(dset_out)<0) 
+    goto error;
+
+   }/*can_read*/
    
    /*-------------------------------------------------------------------------
     * close
@@ -343,9 +357,7 @@ int do_copy_refobjs(hid_t fidin,
     goto error;
    if (H5Dclose(dset_in)<0) 
     goto error;
-   if (H5Dclose(dset_out)<0) 
-    goto error;
-   
+      
    break;
    
   /*-------------------------------------------------------------------------
@@ -359,11 +371,9 @@ int do_copy_refobjs(hid_t fidin,
   
    if (H5Tclose(type_in)<0) 
     goto error;
- 
    
    break;
    
-   
   /*-------------------------------------------------------------------------
    * H5G_LINK
    *-------------------------------------------------------------------------
diff --git a/tools/h5repack/h5repack_verify.c b/tools/h5repack/h5repack_verify.c
index 61529e3..cee2ab7 100644
--- a/tools/h5repack/h5repack_verify.c
+++ b/tools/h5repack/h5repack_verify.c
@@ -321,7 +321,8 @@ error:
   H5Sclose(space_id);
   H5Dclose(dset_id);
   H5Fclose(fid);
-  trav_table_free(travt);
+  if (travt)
+   trav_table_free(travt);
  } H5E_END_TRY;
  return -1;
 }
diff --git a/tools/h5repack/testh5repack_filters.c b/tools/h5repack/testh5repack_filters.c
index 6c58764..7a2265c 100644
--- a/tools/h5repack/testh5repack_filters.c
+++ b/tools/h5repack/testh5repack_filters.c
@@ -75,7 +75,6 @@ int make_filters(hid_t loc_id)
    buf[i][j]=n++;
   }
  }
-
  
 /*-------------------------------------------------------------------------
  * make several dataset with no filters
@@ -87,7 +86,6 @@ int make_filters(hid_t loc_id)
   if (write_dset(loc_id,RANK,dims,name,H5T_NATIVE_INT,buf)<0)
    return -1;
  }
-
  
 /*-------------------------------------------------------------------------
  * make several dataset with filters
@@ -104,6 +102,8 @@ int make_filters(hid_t loc_id)
  if(H5Pset_chunk(dcpl, RANK, chunk_dims)<0)
   goto out;
 
+
+
 /*-------------------------------------------------------------------------
  * SZIP
  *-------------------------------------------------------------------------
@@ -145,6 +145,7 @@ int make_filters(hid_t loc_id)
  * checksum
  *-------------------------------------------------------------------------
  */
+#if 0
  /* remove the filters from the dcpl */
  if (H5Premove_filter(dcpl,H5Z_FILTER_ALL)<0) 
   goto out;
@@ -153,9 +154,10 @@ int make_filters(hid_t loc_id)
   goto out;
  if (make_dset(loc_id,"dset_fletcher32",sid,dcpl,buf)<0)
   goto out;
+#endif
 
 /*-------------------------------------------------------------------------
- * shuffle + checksum + SZIP
+ * shuffle + SZIP
  *-------------------------------------------------------------------------
  */
  /* remove the filters from the dcpl */
@@ -164,9 +166,6 @@ int make_filters(hid_t loc_id)
  /* set the shuffle filter */
  if (H5Pset_shuffle(dcpl)<0) 
   goto out;
- /* set the checksum filter */
- if (H5Pset_fletcher32(dcpl)<0) 
-  goto out;
  /* set szip data */
  if(H5Pset_szip (dcpl,szip_options_mask,szip_pixels_per_block)<0)
   goto out;
diff --git a/tools/h5repack/testh5repack_main.c b/tools/h5repack/testh5repack_main.c
index 89f2a97..2bcee27 100644
--- a/tools/h5repack/testh5repack_main.c
+++ b/tools/h5repack/testh5repack_main.c
@@ -97,7 +97,8 @@ test_copy(void)
   TEST_ERROR;
 
 /*-------------------------------------------------------------------------
- * file with filters
+ * file with filters. we cannot compare with cmpdcpl, because the current
+ * configuration might not have saved datasets with deflate and SZIP filters
  *-------------------------------------------------------------------------
  */
  
@@ -109,9 +110,7 @@ test_copy(void)
   TEST_ERROR;
  if (h5repack_verify(FNAME4OUT,&pack_options)<=0)
   TEST_ERROR;
- if (h5repack_cmpdcpl(FNAME4,FNAME4OUT)<=0)
-  TEST_ERROR;
-  if (h5repack_end (&pack_options)<0)
+ if (h5repack_end (&pack_options)<0)
   TEST_ERROR;
 /*-------------------------------------------------------------------------
  * end
@@ -177,6 +176,7 @@ test_filter_none(void)
  *-------------------------------------------------------------------------
  */
 
+#ifdef H5_HAVE_FILTER_DEFLATE
  if (h5repack_init (&pack_options, 0)<0)
   TEST_ERROR;
  if (h5repack_addfilter("dset_gzip:NONE",&pack_options)<0)
@@ -191,6 +191,7 @@ test_filter_none(void)
   TEST_ERROR;
  if (h5repack_end (&pack_options)<0)
   TEST_ERROR;
+#endif
  
  PASSED();  
  return 0;