[svn-r8218] Purpose:

bug fix in H5Zshuffle.c
add more tests to h5repack that exposed the bug

Description:
when creating a dataset with the shuffle filter and duplicating it in a new dataset (file)
the call to H5Z_set_local_shuffle failed. this is because the value of cd_nelmts of the filter
structure is set to 1 (H5Z_SHUFFLE_TOTAL_NPARMS) when the original dataset is created, but when
the new dataset is created there is a checking instruction that fails if the value of
cd_nelmts is not 0 (its original value, H5Z_SHUFFLE_USER_NPARMS)

Solution:
just remove that check condition, since the value of cd_nelmts is not used anyway.
if we decide that the value of cd_nelmts is necessary, then the H5O_pline_copy function
must be changed to update this value (a different update for each filter)

Platforms tested:
linux
solaris
AIX

Misc. update:

5 files changed, 165 insertions, 176 deletions

diff --git a/tools/h5repack/h5repack.h b/tools/h5repack/h5repack.h
index 5bda25c..fa1c4b4 100644
--- a/tools/h5repack/h5repack.h
+++ b/tools/h5repack/h5repack.h
@@ -294,9 +294,7 @@ void write_dset_in(hid_t loc_id,
                    hid_t file_id,
                    int make_diffs /* flag to modify data buffers */);
 
-int make_deflate(hid_t loc_id);
-int make_szip(hid_t loc_id);
-int make_nofilters(hid_t loc_id);
+int make_filters(hid_t loc_id);
 
 
 
diff --git a/tools/h5repack/h5repack_filters.c b/tools/h5repack/h5repack_filters.c
index 9a91679..ab592a0 100644
--- a/tools/h5repack/h5repack_filters.c
+++ b/tools/h5repack/h5repack_filters.c
@@ -149,44 +149,23 @@ int apply_filters(const char* name,    /* object name from traverse list */
     name,options->threshold);
    return 0;
   }
-  
 
  /* get information about input filters */
  if ((nfilters = H5Pget_nfilters(dcpl_id))<0) 
   return -1;
- 
-/*-------------------------------------------------------------------------
- * check if we have the H5Z_FILTER_NONE filter
- * if so, just delete all filters from the DCPL and exit
- *-------------------------------------------------------------------------
- */
-
- for ( i=0; i<obj->nfilters; i++)
- {
-  if (obj->filter[i].filtn==H5Z_FILTER_NONE)
-  {
-   if (nfilters && H5Premove_filter(dcpl_id,H5Z_FILTER_NONE)<0) 
-    return -1;
-   return 0;
-  }
- }
-
 /*-------------------------------------------------------------------------
  * check if we have filters in the pipeline
  * we want to replace them with the input filters
  *-------------------------------------------------------------------------
  */
-
  if (nfilters) {
   if (H5Premove_filter(dcpl_id,H5Z_FILTER_ALL)<0) 
    return -1;
  }
-
 /*-------------------------------------------------------------------------
  * filters require CHUNK layout; if we do not have one define a default
  *-------------------------------------------------------------------------
  */
- 
  if (obj->chunk.rank<=0)
  {
   obj->chunk.rank=rank;
@@ -204,7 +183,6 @@ int apply_filters(const char* name,    /* object name from traverse list */
  * H5Z_FILTER_SZIP       4 , szip compression 
  *-------------------------------------------------------------------------
  */
- 
  for ( i=0; i<obj->nfilters; i++)
  {
   switch (obj->filter[i].filtn)
diff --git a/tools/h5repack/testh5repack_filters.c b/tools/h5repack/testh5repack_filters.c
index 7b7837b..5a26f5d 100644
--- a/tools/h5repack/testh5repack_filters.c
+++ b/tools/h5repack/testh5repack_filters.c
@@ -22,144 +22,162 @@
 #define CDIM2 DIM2/2
 #define RANK  2
 
+/*-------------------------------------------------------------------------
+ * Function: make_dset
+ *
+ * Purpose: utility function to create and write a dataset in LOC_ID
+ *-------------------------------------------------------------------------
+ */
 
+static 
+int make_dset(hid_t loc_id,const char *name,hid_t sid, hid_t dcpl,void *buf)
+{
+ hid_t   dsid;
+ /* create the dataset */
+ if((dsid = H5Dcreate (loc_id,name,H5T_NATIVE_INT,sid,dcpl))<0)
+  return -1;
+ /* write */
+ if(H5Dwrite(dsid,H5T_NATIVE_INT,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0)
+  goto out;
+ /* close */
+ if(H5Dclose(dsid)<0)
+  return -1;
+ return 0;
+ out:
+ H5E_BEGIN_TRY {
+  H5Dclose(dsid);
+ } H5E_END_TRY;
+ return -1;
+  
+}
 
 /*-------------------------------------------------------------------------
- * Function: make_deflate
+ * Function: make_filters
  *
- * Purpose: make a dataset using DEFLATE (GZIP) compression in LOC_ID
+ * Purpose: make several datasets with filters in location LOC_ID
  *
  *-------------------------------------------------------------------------
  */
-int make_deflate(hid_t loc_id)
+int make_filters(hid_t loc_id)
 {
  hid_t    dcpl; /* dataset creation property list */
- hid_t    dsid; /* dataset ID */
  hid_t    sid;  /* dataspace ID */
+ unsigned szip_options_mask=H5_SZIP_ALLOW_K13_OPTION_MASK|H5_SZIP_NN_OPTION_MASK;
+ unsigned szip_pixels_per_block=8;
  hsize_t  dims[RANK]={DIM1,DIM2};
  hsize_t  chunk_dims[RANK]={CDIM1,CDIM2};
  int      buf[DIM1][DIM2];
- int      i, j, n=0;
+ char     name[5];
+ int      i, j, n;
 
- for (i=0; i<DIM1; i++){
+ for (i=n=0; i<DIM1; i++){
   for (j=0; j<DIM2; j++){
    buf[i][j]=n++;
   }
  }
- 
+  
  /* create a space */
  if((sid = H5Screate_simple(RANK, dims, NULL))<0)
   return -1;
-
- /* create the dataset creation property list */
+ /* create a dataset creation property list; the same DCPL is used for all dsets */
  if ((dcpl = H5Pcreate(H5P_DATASET_CREATE))<0)
   goto out;
- 
- /* set up for deflated data */
+ /* set up chunk */
  if(H5Pset_chunk(dcpl, RANK, chunk_dims)<0)
   goto out;
- if(H5Pset_deflate(dcpl, 9)<0)
-  goto out;
 
- /* create the dataset */
- if((dsid = H5Dcreate (loc_id, "dset_gzip", H5T_NATIVE_INT, sid, dcpl))<0)
+/*-------------------------------------------------------------------------
+ * make several dataset with no filters
+ *-------------------------------------------------------------------------
+ */
+ for (i=0; i<4; i++)
+ {
+  sprintf(name,"dset%d",i+1);
+  if (write_dset(loc_id,RANK,dims,name,H5T_NATIVE_INT,buf)<0)
+   return -1;
+ }
+/*-------------------------------------------------------------------------
+ * SZIP
+ *-------------------------------------------------------------------------
+ */
+ /* set szip data */
+ if(H5Pset_szip (dcpl,szip_options_mask,szip_pixels_per_block)<0)
+  goto out;
+ if (make_dset(loc_id,"dset_szip",sid,dcpl,buf)<0)
   goto out;
 
- /* write the data to the dataset */
- if(H5Dwrite(dsid, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf)<0)
+/*-------------------------------------------------------------------------
+ * GZIP
+ *-------------------------------------------------------------------------
+ */
+ /* remove the filters from the dcpl */
+ if (H5Premove_filter(dcpl,H5Z_FILTER_ALL)<0) 
   goto out;
- 
- /* close */
- if(H5Dclose(dsid)<0)
+ /* set deflate data */
+ if(H5Pset_deflate(dcpl, 9)<0)
+  goto out;
+ if (make_dset(loc_id,"dset_gzip",sid,dcpl,buf)<0)
   goto out;
 
 /*-------------------------------------------------------------------------
- * close
+ * shuffle
  *-------------------------------------------------------------------------
  */
-
- if(H5Sclose(sid)<0)
+ /* remove the filters from the dcpl */
+ if (H5Premove_filter(dcpl,H5Z_FILTER_ALL)<0) 
   goto out;
- if(H5Pclose(dcpl)<0)
+ /* set the shuffle filter */
+ if (H5Pset_shuffle(dcpl)<0) 
+  goto out;
+ if (make_dset(loc_id,"dset_shuffle",sid,dcpl,buf)<0)
   goto out;
- 
- return 0;                                                 
- 
-out:
- H5E_BEGIN_TRY {
-  H5Dclose(dsid);
-  H5Pclose(dcpl);
-  H5Sclose(sid);
- } H5E_END_TRY;
- return -1;
-}
 
 /*-------------------------------------------------------------------------
- * Function: make_szip
- *
- * Purpose: make a dataset using SZIP compression in LOC_ID
- *
+ * checksum
  *-------------------------------------------------------------------------
  */
-int make_szip(hid_t loc_id)
-{
- hid_t    dcpl; /* dataset creation property list */
- hid_t    dsid; /* dataset ID */
- hid_t    sid;  /* dataspace ID */
- unsigned szip_options_mask=H5_SZIP_ALLOW_K13_OPTION_MASK|H5_SZIP_NN_OPTION_MASK;
- unsigned szip_pixels_per_block;
- hsize_t  dims[RANK]={DIM1,DIM2};
- hsize_t  chunk_dims[RANK]={CDIM1,CDIM2};
- int      buf[DIM1][DIM2];
- int      i, j, n=0;
-
- for (i=0; i<DIM1; i++){
-  for (j=0; j<DIM2; j++){
-   buf[i][j]=n++;
-  }
- }
- 
-  /* 
-  pixels_per_block must be an even number, and <= pixels_per_scanline 
-  and <= MAX_PIXELS_PER_BLOCK
-  */
- szip_pixels_per_block=8;
- 
- /* create a space */
- if((sid = H5Screate_simple(RANK, dims, NULL))<0)
-  return -1;
-
- /* create the dataset creation property list */
- if ((dcpl = H5Pcreate(H5P_DATASET_CREATE))<0)
+ /* remove the filters from the dcpl */
+ if (H5Premove_filter(dcpl,H5Z_FILTER_ALL)<0) 
   goto out;
- 
- /* set up for sziped data */
- if(H5Pset_chunk(dcpl, RANK, chunk_dims)<0)
+ /* set the checksum filter */
+ if (H5Pset_fletcher32(dcpl)<0) 
   goto out;
- if(H5Pset_szip (dcpl, szip_options_mask, szip_pixels_per_block)<0)
+ if (make_dset(loc_id,"dset_fletcher32",sid,dcpl,buf)<0)
   goto out;
 
- /* create the dataset */
- if((dsid = H5Dcreate (loc_id, "dset_szip", H5T_NATIVE_INT, sid, dcpl))<0)
+/*-------------------------------------------------------------------------
+ * shuffle + checksum + SZIP
+ *-------------------------------------------------------------------------
+ */
+ /* remove the filters from the dcpl */
+ if (H5Premove_filter(dcpl,H5Z_FILTER_ALL)<0) 
   goto out;
-
- /* write the data to the dataset */
- if(H5Dwrite(dsid, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf)<0)
+ /* set the shuffle filter */
+ if (H5Pset_shuffle(dcpl)<0) 
   goto out;
- 
- /* close */
- if(H5Dclose(dsid)<0)
+ /* set the checksum filter */
+ if (H5Pset_fletcher32(dcpl)<0) 
   goto out;
- if(H5Pclose(dcpl)<0)
+ /* set szip data */
+ if(H5Pset_szip (dcpl,szip_options_mask,szip_pixels_per_block)<0)
   goto out;
+ if (make_dset(loc_id,"dset_all",sid,dcpl,buf)<0)
+  goto out;
+
+
+/*-------------------------------------------------------------------------
+ * close space and dcpl
+ *-------------------------------------------------------------------------
+ */
  if(H5Sclose(sid)<0)
   goto out;
+ if(H5Pclose(dcpl)<0)
+  goto out;
  
  return 0;                                                 
  
 out:
  H5E_BEGIN_TRY {
-  H5Dclose(dsid);
   H5Pclose(dcpl);
   H5Sclose(sid);
  } H5E_END_TRY;
@@ -167,35 +185,6 @@ out:
 }
 
 
-/*-------------------------------------------------------------------------
- * Function: make_nofilters
- *
- * Purpose: make several dataset with no filters
- *
- *-------------------------------------------------------------------------
- */
-int make_nofilters(hid_t loc_id)
-{
- char     name[5];
- hsize_t  dims[RANK]={DIM1,DIM2};
- int      buf[DIM1][DIM2];
- int      i, j, n=0;
 
- for (i=0; i<DIM1; i++){
-  for (j=0; j<DIM2; j++){
-   buf[i][j]=n++;
-  }
- }
- 
- for (i=0; i<4; i++)
- {
-  sprintf(name,"dset%d",i+1);
-  if (write_dset(loc_id,RANK,dims,name,H5T_NATIVE_INT,buf)<0)
-   return -1;
- }
-
-
- return 0;
-}
 
 
diff --git a/tools/h5repack/testh5repack_main.c b/tools/h5repack/testh5repack_main.c
index 6b829ee..89f2a97 100644
--- a/tools/h5repack/testh5repack_main.c
+++ b/tools/h5repack/testh5repack_main.c
@@ -42,27 +42,29 @@ test_copy(void)
  memset(&diff_options, 0, sizeof (diff_opt_t));
 
 
- TESTING("    copy with no filters");
+ TESTING("    copy with no input filters");
+
+/*-------------------------------------------------------------------------
+ * file with all kinds of dataset datatypes
+ *-------------------------------------------------------------------------
+ */
  if (h5repack_init (&pack_options, 0)<0)
   TEST_ERROR;
-
-#if defined(H5_REPACK_DEBUG)
- diff_options.verbose=1;
- pack_options.verbose=1;
-#endif
-
  if (h5repack(FNAME1,FNAME1OUT,&pack_options)<0)
   TEST_ERROR;
  if (h5diff(FNAME1,FNAME1OUT,NULL,NULL,&diff_options) == 1)
   TEST_ERROR;
  if (h5repack_verify(FNAME1OUT,&pack_options)<=0)
   TEST_ERROR;
+ if (h5repack_cmpdcpl(FNAME1,FNAME1OUT)<=0)
+  TEST_ERROR;
  if (h5repack_end (&pack_options)<0)
   TEST_ERROR;
 
- PASSED(); 
-
- TESTING("    copy of attributes");
+/*-------------------------------------------------------------------------
+ * file with attributes
+ *-------------------------------------------------------------------------
+ */
  if (h5repack_init (&pack_options, 0)<0)
   TEST_ERROR;
  if (h5repack(FNAME2,FNAME2OUT,&pack_options)<0)
@@ -71,11 +73,16 @@ test_copy(void)
   TEST_ERROR;
  if (h5repack_verify(FNAME2OUT,&pack_options)<=0)
   TEST_ERROR;
+ if (h5repack_cmpdcpl(FNAME2,FNAME2OUT)<=0)
+  TEST_ERROR;
  if (h5repack_end (&pack_options)<0)
   TEST_ERROR;
- PASSED();      
 
- TESTING("    copy of hardlinks");
+/*-------------------------------------------------------------------------
+ * file with hardlinks
+ *-------------------------------------------------------------------------
+ */
+
  if (h5repack_init (&pack_options, 0)<0)
   TEST_ERROR;
  if (h5repack(FNAME3,FNAME3OUT,&pack_options)<0)
@@ -84,10 +91,33 @@ test_copy(void)
   TEST_ERROR;
  if (h5repack_verify(FNAME3OUT,&pack_options)<=0)
   TEST_ERROR;
+ if (h5repack_cmpdcpl(FNAME3,FNAME3OUT)<=0)
+  TEST_ERROR;
   if (h5repack_end (&pack_options)<0)
   TEST_ERROR;
- PASSED();          
+
+/*-------------------------------------------------------------------------
+ * file with filters
+ *-------------------------------------------------------------------------
+ */
  
+ if (h5repack_init (&pack_options, 0)<0)
+  TEST_ERROR;
+ if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0)
+  TEST_ERROR;
+ if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1)
+  TEST_ERROR;
+ if (h5repack_verify(FNAME4OUT,&pack_options)<=0)
+  TEST_ERROR;
+ if (h5repack_cmpdcpl(FNAME4,FNAME4OUT)<=0)
+  TEST_ERROR;
+  if (h5repack_end (&pack_options)<0)
+  TEST_ERROR;
+/*-------------------------------------------------------------------------
+ * end
+ *-------------------------------------------------------------------------
+ */
+ PASSED();          
  return 0; 
  
 error:                                                       
@@ -100,7 +130,7 @@ error:
  *
  * Purpose: 
  *
- * 1) delete filters form the filter pipeline
+ * 1) remove filters form the filter pipeline
  * 2) use the h5diff utility to compare the input and output file; 
  *     it returns RET==0 if the objects have the same data
  * 3) use API functions to verify the compression/chunking input on the output file
@@ -122,7 +152,7 @@ test_filter_none(void)
  memset(&diff_options, 0, sizeof (diff_opt_t));
  memset(&pack_options, 0, sizeof (pack_opt_t));
 
- TESTING("    delete filters");
+ TESTING("    removing filters");
 
 /*-------------------------------------------------------------------------
  * test the NONE global option
@@ -131,7 +161,7 @@ test_filter_none(void)
 
  if (h5repack_init (&pack_options, 0)<0)
   TEST_ERROR;
- if (h5repack_addfilter("dset_gzip:NONE",&pack_options)<0)
+ if (h5repack_addfilter("NONE",&pack_options)<0)
   TEST_ERROR;
  if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0)
   TEST_ERROR;
@@ -167,8 +197,6 @@ test_filter_none(void)
  
 error:                                                       
  return 1;
-
-
 }
 
 /*-------------------------------------------------------------------------
diff --git a/tools/h5repack/testh5repack_make.c b/tools/h5repack/testh5repack_make.c
index e5aa5ee..f625d2a 100644
--- a/tools/h5repack/testh5repack_make.c
+++ b/tools/h5repack/testh5repack_make.c
@@ -36,52 +36,48 @@ int make_testfiles(void)
 
  TESTING("    generating datasets");
 
- /* create a file for general copy test */
+/*-------------------------------------------------------------------------
+ * create a file for general copy test 
+ *-------------------------------------------------------------------------
+ */
  if((loc_id = H5Fcreate(FNAME1,H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT))<0)
   return -1;
- 
  if (make_all_objects(loc_id)<0)
   goto out;
-
   /* close */
  if(H5Fclose(loc_id)<0)
   return -1;
 
- /* create a file for attributes copy test */
+/*-------------------------------------------------------------------------
+ * create a file for attributes copy test
+ *-------------------------------------------------------------------------
+ */
  if((loc_id = H5Fcreate(FNAME2,H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT))<0)
   return -1;
-
  if (make_attributes(loc_id)<0)
   goto out;
-
   /* close */
  if(H5Fclose(loc_id)<0)
   return -1;
-
- /* create a file for special items test */
+/*-------------------------------------------------------------------------
+ * create a file for special items test
+ *-------------------------------------------------------------------------
+ */
  if((loc_id = H5Fcreate(FNAME3,H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT))<0)
   return -1;
-
  if (make_special_objects(loc_id)<0)
   goto out;
-
  /* close */
  if(H5Fclose(loc_id)<0)
   return -1;
-
- /* create a file for the filters test */
+/*-------------------------------------------------------------------------
+ * create a file for the filters test
+ *-------------------------------------------------------------------------
+ */
  if((loc_id = H5Fcreate(FNAME4,H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT))<0)
   return -1;
-
- if (make_nofilters(loc_id)<0)
-  goto out;
-
- if (make_deflate(loc_id)<0)
+ if (make_filters(loc_id)<0)
   goto out;
-
- if (make_szip(loc_id)<0)
-  goto out;
-
  /* close */
  if(H5Fclose(loc_id)<0)
   return -1;