[svn-r7996] Purpose:

h5repack new features

Description:
added support for shuffle and checksum filters

Solution:

Platforms tested:
linux
solaris
AIX

Misc. update:

3 files changed, 267 insertions, 22 deletions

diff --git a/tools/h5repack/h5repack_filters.c b/tools/h5repack/h5repack_filters.c
index 6c88b51..09e2e75 100644
--- a/tools/h5repack/h5repack_filters.c
+++ b/tools/h5repack/h5repack_filters.c
@@ -50,8 +50,6 @@ int apply_filter(hid_t dcpl_id,
  unsigned     aggression;     /* the deflate level */
  unsigned     szip_options_mask=H5_SZIP_NN_OPTION_MASK;
  unsigned     szip_pixels_per_block;
-
-
  
  /* get information about filters */
  if ((nfilters = H5Pget_nfilters(dcpl_id))<0) 
@@ -139,7 +137,26 @@ int apply_filter(hid_t dcpl_id,
   }
   
   break;
+
   
+ case H5Z_FILTER_SHUFFLE:
+  
+  if(H5Pset_chunk(dcpl_id, obj->chunk.rank, obj->chunk.chunk_lengths)<0)
+   return -1;
+  if (H5Pset_shuffle(dcpl_id)<0) 
+   return -1;
+  
+  break;
+  
+ case H5Z_FILTER_FLETCHER32:
+
+  if(H5Pset_chunk(dcpl_id, obj->chunk.rank, obj->chunk.chunk_lengths)<0)
+   return -1;
+  if (H5Pset_fletcher32(dcpl_id)<0) 
+   return -1;
+
+  break;
+
   
  default:
   break;
diff --git a/tools/h5repack/testh5repack_main.c b/tools/h5repack/testh5repack_main.c
index e46dd53..4385247 100644
--- a/tools/h5repack/testh5repack_main.c
+++ b/tools/h5repack/testh5repack_main.c
@@ -46,10 +46,12 @@ test_copy(void)
   TEST_ERROR;
  if (h5repack(FNAME1,FNAME1OUT,&pack_options)<0)
   TEST_ERROR;
- if (h5repack_end (&pack_options)<0)
-  TEST_ERROR;
  if (h5diff(FNAME1,FNAME1OUT,NULL,NULL,&diff_options) == 1)
   TEST_ERROR;
+ if (h5repack_verify(FNAME1OUT,&pack_options)<=0)
+  TEST_ERROR;
+ if (h5repack_end (&pack_options)<0)
+  TEST_ERROR;
  PASSED(); 
 
  TESTING("    copy of attributes");
@@ -57,10 +59,12 @@ test_copy(void)
   TEST_ERROR;
  if (h5repack(FNAME2,FNAME2OUT,&pack_options)<0)
   TEST_ERROR;
- if (h5repack_end (&pack_options)<0)
-  TEST_ERROR;
  if (h5diff(FNAME2,FNAME2OUT,NULL,NULL,&diff_options) == 1)
   TEST_ERROR;
+ if (h5repack_verify(FNAME2OUT,&pack_options)<=0)
+  TEST_ERROR;
+ if (h5repack_end (&pack_options)<0)
+  TEST_ERROR;
  PASSED();      
 
  TESTING("    copy of hardlinks");
@@ -68,10 +72,12 @@ test_copy(void)
   TEST_ERROR;
  if (h5repack(FNAME3,FNAME3OUT,&pack_options)<0)
   TEST_ERROR;
- if (h5repack_end (&pack_options)<0)
-  TEST_ERROR;
  if (h5diff(FNAME3,FNAME3OUT,NULL,NULL,&diff_options) == 1)
   TEST_ERROR;
+ if (h5repack_verify(FNAME3OUT,&pack_options)<=0)
+  TEST_ERROR;
+ if (h5repack_end (&pack_options)<0)
+  TEST_ERROR;
  PASSED();          
  
  return 0; 
@@ -113,6 +119,28 @@ test_filter_deflate(void)
 
 #ifdef H5_HAVE_FILTER_DEFLATE
 
+/*-------------------------------------------------------------------------
+ * test no object option (preserve old filters)
+ *-------------------------------------------------------------------------
+ */
+
+ if (h5repack_init (&pack_options, 0)<0)
+  TEST_ERROR;
+ if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0)
+  TEST_ERROR;
+ if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1)
+  TEST_ERROR;
+ if (h5repack_verify(FNAME4OUT,&pack_options)<=0)
+  TEST_ERROR;
+ if (h5repack_end (&pack_options)<0)
+  TEST_ERROR;
+
+
+/*-------------------------------------------------------------------------
+ * test an individual object option
+ *-------------------------------------------------------------------------
+ */
+
  if (h5repack_init (&pack_options, 0)<0)
   TEST_ERROR;
  if (h5repack_addfilter("dset1:GZIP 9",&pack_options)<0)
@@ -128,6 +156,11 @@ test_filter_deflate(void)
  if (h5repack_end (&pack_options)<0)
   TEST_ERROR;
 
+/*-------------------------------------------------------------------------
+ * test all objects option
+ *-------------------------------------------------------------------------
+ */
+
  if (h5repack_init (&pack_options, 0)<0)
   TEST_ERROR;
  if (h5repack_addfilter("GZIP 9",&pack_options)<0)
@@ -190,6 +223,11 @@ test_filter_szip(void)
 
 #ifdef H5_HAVE_FILTER_SZIP
 
+/*-------------------------------------------------------------------------
+ * test an individual object option
+ *-------------------------------------------------------------------------
+ */
+
  if (h5repack_init (&pack_options, 0)<0)
   TEST_ERROR;
  if (h5repack_addfilter("dset2:SZIP 8",&pack_options)<0)
@@ -204,6 +242,30 @@ test_filter_szip(void)
   TEST_ERROR;
  if (h5repack_end (&pack_options)<0)
   TEST_ERROR;
+
+/*-------------------------------------------------------------------------
+ * test all objects option
+ *-------------------------------------------------------------------------
+ */
+
+
+#if 0
+ if (h5repack_init (&pack_options, 0)<0)
+  TEST_ERROR;
+ if (h5repack_addfilter("SZIP 8",&pack_options)<0)
+  TEST_ERROR;
+ if (h5repack_addchunk("20x10",&pack_options)<0)
+  TEST_ERROR;
+ if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0)
+  TEST_ERROR;
+ if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1)
+  TEST_ERROR;
+ if (h5repack_verify(FNAME4OUT,&pack_options)<=0)
+  TEST_ERROR;
+ if (h5repack_end (&pack_options)<0)
+  TEST_ERROR;
+#endif
+ 
  
  PASSED();  
 #else
@@ -216,10 +278,184 @@ error:
  return 1;
 #endif
 
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function: test_filter_shuffle
+ *
+ * Purpose: 
+ *
+ * 1) compress/chunk FILENAME with the shuffle filter
+ * 2) use the h5diff utility to compare the input and output file; 
+ *     it returns RET==0 if the objects have the same data
+ * 3) use API functions to verify the compression/chunking input on the output file
+ *
+ * Return: Success: zero
+ *  Failure: 1
+ *
+ * Programmer: Pedro Vicente <pvn@ncsa.uiuc.edu>
+ *             September, 19, 2003
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+static int
+test_filter_shuffle(void)
+{
+ pack_opt_t  pack_options;
+ diff_opt_t  diff_options;
+ memset(&diff_options, 0, sizeof (diff_opt_t));
+ memset(&pack_options, 0, sizeof (pack_opt_t));
+
+ TESTING("    shuffle filter");
+
+#ifdef H5_HAVE_FILTER_SHUFFLE
+
+/*-------------------------------------------------------------------------
+ * test an individual object option
+ *-------------------------------------------------------------------------
+ */
+
+ if (h5repack_init (&pack_options, 0)<0)
+  TEST_ERROR;
+ if (h5repack_addfilter("dset1:SHUF",&pack_options)<0)
+  TEST_ERROR;
+ if (h5repack_addchunk("dset1:20x10",&pack_options)<0)
+  TEST_ERROR;
+ if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0)
+  TEST_ERROR;
+ if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1)
+  TEST_ERROR;
+ if (h5repack_verify(FNAME4OUT,&pack_options)<=0)
+  TEST_ERROR;
+ if (h5repack_end (&pack_options)<0)
+  TEST_ERROR;
+
+/*-------------------------------------------------------------------------
+ * test all objects option
+ *-------------------------------------------------------------------------
+ */
+
+ if (h5repack_init (&pack_options, 0)<0)
+  TEST_ERROR;
+ if (h5repack_addfilter("SHUF",&pack_options)<0)
+  TEST_ERROR;
+ if (h5repack_addchunk("20x10",&pack_options)<0)
+  TEST_ERROR;
+ if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0)
+  TEST_ERROR;
+ if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1)
+  TEST_ERROR;
+ if (h5repack_verify(FNAME4OUT,&pack_options)<=0)
+  TEST_ERROR;
+ if (h5repack_end (&pack_options)<0)
+  TEST_ERROR;
+ 
+ PASSED();  
+#else
+ SKIPPED();
+#endif
+ return 0; 
+ 
+#ifdef H5_HAVE_FILTER_SHUFFLE
+error:                                                       
+ return 1;
+#endif
+
+
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function: test_filter_checksum 
+ *
+ * Purpose: 
+ *
+ * 1) compress/chunk FILENAME with the checksum filter
+ * 2) use the h5diff utility to compare the input and output file; 
+ *     it returns RET==0 if the objects have the same data
+ * 3) use API functions to verify the compression/chunking input on the output file
+ *
+ * Return: Success: zero
+ *  Failure: 1
+ *
+ * Programmer: Pedro Vicente <pvn@ncsa.uiuc.edu>
+ *             September, 19, 2003
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+static int
+test_filter_checksum(void)
+{
+ pack_opt_t  pack_options;
+ diff_opt_t  diff_options;
+ memset(&diff_options, 0, sizeof (diff_opt_t));
+ memset(&pack_options, 0, sizeof (pack_opt_t));
+
+ TESTING("    checksum filter");
+
+#ifdef H5_HAVE_FILTER_FLETCHER32
+
+/*-------------------------------------------------------------------------
+ * test an individual object option
+ *-------------------------------------------------------------------------
+ */
+
+ if (h5repack_init (&pack_options, 0)<0)
+  TEST_ERROR;
+ if (h5repack_addfilter("dset1:FLET",&pack_options)<0)
+  TEST_ERROR;
+ if (h5repack_addchunk("dset1:20x10",&pack_options)<0)
+  TEST_ERROR;
+ if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0)
+  TEST_ERROR;
+ if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1)
+  TEST_ERROR;
+ if (h5repack_verify(FNAME4OUT,&pack_options)<=0)
+  TEST_ERROR;
+ if (h5repack_end (&pack_options)<0)
+  TEST_ERROR;
+
+/*-------------------------------------------------------------------------
+ * test all objects option
+ *-------------------------------------------------------------------------
+ */
+
+ if (h5repack_init (&pack_options, 0)<0)
+  TEST_ERROR;
+ if (h5repack_addfilter("FLET",&pack_options)<0)
+  TEST_ERROR;
+ if (h5repack_addchunk("20x10",&pack_options)<0)
+  TEST_ERROR;
+ if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0)
+  TEST_ERROR;
+ if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1)
+  TEST_ERROR;
+ if (h5repack_verify(FNAME4OUT,&pack_options)<=0)
+  TEST_ERROR;
+ if (h5repack_end (&pack_options)<0)
+  TEST_ERROR;
+ 
+ PASSED();  
+#else
+ SKIPPED();
+#endif
+ return 0; 
+ 
+#ifdef H5_HAVE_FILTER_FLETCHER32
+error:                                                       
+ return 1;
+#endif
+
 
 }
 
 
+
 /*-------------------------------------------------------------------------
  * Function: main
  *
@@ -256,6 +492,12 @@ int main (void)
  /* test a copy with the szip filter */
  nerrors += test_filter_szip();
 
+ /* test a copy with the shuffle filter */
+ nerrors += test_filter_shuffle();
+
+ /* test a copy with the checksum filter */
+ nerrors += test_filter_checksum();
+
  /* check for errors */
  if (nerrors)
   goto error;
diff --git a/tools/h5repack/testh5repack_make.c b/tools/h5repack/testh5repack_make.c
index 34d3029..9d3fb3f 100644
--- a/tools/h5repack/testh5repack_make.c
+++ b/tools/h5repack/testh5repack_make.c
@@ -274,20 +274,6 @@ int make_special_objects(hid_t loc_id)
  if ((group3_id = H5Gcreate(group2_id,"g3",0))<0)
   return -1;
 
- /*
-  H5Glink2(curr_loc_id, current_name, link_type, new_loc_id, new_name ) 
-  hid_t curr_loc_id 
-    IN: The file or group identifier for the original object. 
-  const char * current_name 
-    IN: Name of the existing object if link is a hard link. 
-  H5G_link_t link_type 
-    IN: Link type. Possible values are H5G_LINK_HARD and H5G_LINK_SOFT. 
-  hid_t new_loc_id 
-    IN: The file or group identifier for the new link. 
-  const char * new_name 
-    IN: New name for the object.
-  */
-
  if (H5Glink2(loc_id, "g1", H5G_LINK_HARD, group2_id, "link1 to g1")<0)
   return -1;
  if (H5Glink2(group1_id, "g2", H5G_LINK_HARD, group3_id, "link1 to g2")<0)