[svn-r12205] Purpose:

new feature

Description:
added the printout of the compression ratio for filters in h5repack, after the filter name, obtained with H5Dget_storage_size, before and after applying the filter

Solution:

Platforms tested:
linux
solaris
AIX

Misc. update:

10 files changed, 119 insertions, 99 deletions

diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c
index 436012a..a385fa2 100644
--- a/tools/h5repack/h5repack_copy.c
+++ b/tools/h5repack/h5repack_copy.c
@@ -19,20 +19,25 @@
 #include "H5private.h"
 #include "h5repack.h"
 
-#if 0
-#define PRINT_DEBUG
-#endif
+#define PER(A,B) { per = 0;                                      \
+                   if (A!=0)                                      \
+                    per = (float)fabs(1-( (float)B / (float)A )); \
+                 }
+
+#define FORMAT_OBJ " %-21s %s\n"  /* obj type, name */
 
 /*-------------------------------------------------------------------------
- * Function: print_obj
+ * Function: print_dataset_info
  *
- * Purpose: print name and filters of an object
+ * Purpose: print name, filters, percentage compression of a dataset
  *
  *-------------------------------------------------------------------------
  */
-static void print_obj(hid_t dcpl_id, char *name)
+static void print_dataset_info(hid_t dcpl_id, 
+                               char *objname,
+                               float per)
 {
- char         str[255];
+ char         strfilter[255];
 #if defined (PRINT_DEBUG )
  char         temp[255];
 #endif
@@ -41,10 +46,10 @@ static void print_obj(hid_t dcpl_id, char *name)
  H5Z_filter_t filtn;          /* filter identification number */
  unsigned     cd_values[20];  /* filter client data values */
  size_t       cd_nelmts;      /* filter client number of values */
- char         f_name[256];    /* filter name */
+ char         f_objname[256];    /* filter objname */
  int          i;
 
- strcpy(str,"\0");
+ strcpy(strfilter,"\0");
 
  /* get information about input filters */
  if ((nfilters = H5Pget_nfilters(dcpl_id))<0)
@@ -60,16 +65,16 @@ static void print_obj(hid_t dcpl_id, char *name)
    &filt_flags,
    &cd_nelmts,
    cd_values,
-   sizeof(f_name),
-   f_name);
+   sizeof(f_objname),
+   f_objname);
 #else
   filtn = H5Pget_filter(dcpl_id,
    (unsigned)i,
    &filt_flags,
    &cd_nelmts,
    cd_values,
-   sizeof(f_name),
-   f_name,
+   sizeof(f_objname),
+   f_objname,
    NULL);
 #endif /* H5_WANT_H5_V1_6_COMPAT */
 
@@ -79,52 +84,64 @@ static void print_obj(hid_t dcpl_id, char *name)
   default:
    break;
   case H5Z_FILTER_DEFLATE:
-   strcat(str,"GZIP ");
+   strcat(strfilter,"GZIP ");
 
 #if defined (PRINT_DEBUG)
    {
     unsigned level=cd_values[0];
     sprintf(temp,"(%d)",level);
-    strcat(str,temp);
+    strcat(strfilter,temp);
    }
 #endif
 
    break;
   case H5Z_FILTER_SZIP:
-   strcat(str,"SZIP ");
+   strcat(strfilter,"SZIP ");
 
 #if defined (PRINT_DEBUG)
    {
     unsigned options_mask=cd_values[0]; /* from dcpl, not filt*/
     unsigned ppb=cd_values[1];
     sprintf(temp,"(%d,",ppb);
-    strcat(str,temp);
+    strcat(strfilter,temp);
     if (options_mask & H5_SZIP_EC_OPTION_MASK)
      strcpy(temp,"EC) ");
     else if (options_mask & H5_SZIP_NN_OPTION_MASK)
      strcpy(temp,"NN) ");
    }
-   strcat(str,temp);
+   strcat(strfilter,temp);
 
 #endif
 
    break;
   case H5Z_FILTER_SHUFFLE:
-   strcat(str,"SHUF ");
+   strcat(strfilter,"SHUF ");
    break;
   case H5Z_FILTER_FLETCHER32:
-   strcat(str,"FLET ");
+   strcat(strfilter,"FLET ");
    break;
   case H5Z_FILTER_NBIT:
-   strcat(str,"NBIT ");
+   strcat(strfilter,"NBIT ");
    break;
   case H5Z_FILTER_SCALEOFFSET:
-   strcat(str,"SCALEOFFSET ");
+   strcat(strfilter,"SCALEOFFSET ");
    break;
   } /* switch */
  }/*i*/
 
- printf(" %-10s %s %s\n", "dataset",str,name );
+ if (strcmp(strfilter,"\0")==0)
+  printf(FORMAT_OBJ,"dset",objname );
+ else
+ {
+  char str[255], temp[20];
+  strcpy(str,"dset  ");
+  strcat(str,strfilter);
+  sprintf(temp,"(%.2f%%)",per);
+  strcat(str,temp);
+  printf(FORMAT_OBJ,str,objname);
+ }
+  
+
 
 }
 
@@ -150,7 +167,7 @@ int copy_objects(const char* fnamein,
                  const char *argv[])
 {
  hid_t         fidin;
- hid_t         fidout=(-1);
+ hid_t         fidout;
  trav_table_t  *travt=NULL;
 
 /*-------------------------------------------------------------------------
@@ -244,31 +261,39 @@ int do_copy_objects(hid_t fidin,
                     trav_table_t *travt,
                     pack_opt_t *options) /* repack options */
 {
- hid_t     grp_in=(-1);       /* group ID */
- hid_t     grp_out=(-1);      /* group ID */
- hid_t     dset_in=(-1);      /* read dataset ID */
- hid_t     dset_out=(-1);     /* write dataset ID */
- hid_t     type_in=(-1);      /* named type ID */
- hid_t     type_out=(-1);     /* named type ID */
- hid_t     dcpl_id=(-1);      /* dataset creation property list ID */
- hid_t     dcpl_out=(-1);     /* dataset creation property list ID */
- hid_t     space_id=(-1);     /* space ID */
- hid_t     ftype_id=(-1);     /* file data type ID */
- hid_t     mtype_id=(-1);     /* memory data type ID */
+ hid_t     grp_in;       /* group ID */
+ hid_t     grp_out;      /* group ID */
+ hid_t     dset_in;      /* read dataset ID */
+ hid_t     dset_out;     /* write dataset ID */
+ hid_t     type_in;      /* named type ID */
+ hid_t     type_out;     /* named type ID */
+ hid_t     dcpl_id;      /* dataset creation property list ID */
+ hid_t     dcpl_out;     /* dataset creation property list ID */
+ hid_t     space_id;     /* space ID */
+ hid_t     ftype_id;     /* file data type ID */
+ hid_t     mtype_id;     /* memory data type ID */
  size_t    msize;        /* memory size of memory type */
  void      *buf=NULL;    /* data buffer */
  hsize_t   nelmts;       /* number of elements in dataset */
  int       rank;         /* rank of dataset */
  hsize_t   dims[H5S_MAX_RANK];/* dimensions of dataset */
  hsize_t   dsize_in;     /* input dataset size before filter */
+ hsize_t   dsize_out;    /* output dataset size after filter */
  int       next;         /* external files */
  int       i, j;
+ int       apply;
 
 /*-------------------------------------------------------------------------
  * copy the suppplied object list
  *-------------------------------------------------------------------------
  */
 
+ if (options->verbose) {
+  printf("-----------------------------------------\n");
+  printf(" Type  Filter (Ratio)  Name\n");
+  printf("-----------------------------------------\n");
+ }
+
  for ( i = 0; i < travt->nobjs; i++)
  {
 
@@ -281,7 +306,7 @@ int do_copy_objects(hid_t fidin,
  */
   case H5G_GROUP:
    if (options->verbose)
-    printf(" %-10s %s\n", "group",travt->objs[i].name );
+    printf(FORMAT_OBJ,"group",travt->objs[i].name );
 
    if ((grp_out=H5Gcreate(fidout,travt->objs[i].name, 0))<0)
     goto error;
@@ -329,10 +354,7 @@ int do_copy_objects(hid_t fidin,
    for (j=0; j<rank; j++)
     nelmts*=dims[j];
 
-			if (options->verbose)
-				print_obj(dcpl_id,travt->objs[i].name );
-
-   if ((mtype_id=h5tools_get_native_type(ftype_id))<0)
+		 if ((mtype_id=h5tools_get_native_type(ftype_id))<0)
     goto error;
 
    if ((msize=H5Tget_size(mtype_id))==0)
@@ -345,9 +367,9 @@ int do_copy_objects(hid_t fidin,
    if ((next=H5Pget_external_count (dcpl_id))<0)
     goto error;
 
-   if (next) {
-    fprintf(stderr,"Warning: <%s> has external files, ignoring read...\n",travt->objs[i].name );
-   }
+   if (next) 
+    fprintf(stderr,"warning: <%s> has external files, ignoring read...\n",
+    travt->objs[i].name );
 
 /*-------------------------------------------------------------------------
  * check if the dataset creation property list has filters that
@@ -367,12 +389,11 @@ int do_copy_objects(hid_t fidin,
  */
    if ( (H5T_REFERENCE!=H5Tget_class(mtype_id)))
    {
-
     /* get the storage size of the input dataset */
     dsize_in=H5Dget_storage_size(dset_in);
 
   /*-------------------------------------------------------------------------
-   * read to memory
+   * read 
    *-------------------------------------------------------------------------
    */
     if (nelmts)
@@ -385,12 +406,22 @@ int do_copy_objects(hid_t fidin,
      if (H5Dread(dset_in,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0)
       goto error;
 
+     /*-------------------------------------------------------------------------
+      * check for datasets too small 
+      *-------------------------------------------------------------------------
+      */
+     apply=1;
+     if (nelmts*msize < options->threshold )
+      apply=0;
+     
     /*-------------------------------------------------------------------------
      * apply the filter
      *-------------------------------------------------------------------------
      */
-     if (apply_filters(travt->objs[i].name,rank,dims,dcpl_out,mtype_id,options)<0)
-      goto error;
+     if (apply){
+      if (apply_filters(travt->objs[i].name,rank,dims,dcpl_out,mtype_id,options)<0)
+       goto error;
+     }
 
     }/*nelmts*/
 
@@ -411,7 +442,7 @@ int do_copy_objects(hid_t fidin,
 						goto error;
 
 					if (options->verbose)
-						printf("Warning: Could not apply the filter to <%s>\n", travt->objs[i].name);
+						printf("warning: could not apply the filter to <%s>\n", travt->objs[i].name);
 				}
 
 				/*-------------------------------------------------------------------------
@@ -423,6 +454,23 @@ int do_copy_objects(hid_t fidin,
      if (H5Dwrite(dset_out,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0)
       goto error;
     }
+
+   /*-------------------------------------------------------------------------
+    * amount of compression used
+    *-------------------------------------------------------------------------
+    */
+    if (options->verbose) {
+     float per; /* percent utilization of storage */
+     /* get the storage size of the input dataset */
+     dsize_out=H5Dget_storage_size(dset_out);
+     PER((hssize_t)dsize_in,(hssize_t)dsize_out);
+     print_dataset_info(dcpl_out,travt->objs[i].name,per*(float)100.0);
+    }
+   
+    if (apply==0 && options->verbose)
+     printf("warning: filter not applied to <%s>. dataset smaller than <%d> bytes\n",
+       travt->objs[i].name,(int)options->threshold);
+
     /*-------------------------------------------------------------------------
      * copy attrs
      *-------------------------------------------------------------------------
@@ -488,7 +536,7 @@ int do_copy_objects(hid_t fidin,
     goto error;
 
    if (options->verbose)
-    printf(" %-10s %s\n","datatype",travt->objs[i].name );
+    printf(FORMAT_OBJ,"type",travt->objs[i].name );
 
    break;
 
@@ -521,14 +569,12 @@ int do_copy_objects(hid_t fidin,
     free(targbuf);
 
     if (options->verbose)
-     printf(" %-10s %s\n","link",travt->objs[i].name );
+     printf(FORMAT_OBJ,"link",travt->objs[i].name );
 
    }
    break;
 
   default:
-   if (options->verbose)
-    printf(" %-10s %s\n","User defined object",travt->objs[i].name);
    goto error;
   }
  }
diff --git a/tools/h5repack/h5repack_filters.c b/tools/h5repack/h5repack_filters.c
index a96d5be..1cd6955 100644
--- a/tools/h5repack/h5repack_filters.c
+++ b/tools/h5repack/h5repack_filters.c
@@ -194,16 +194,11 @@ int apply_filters(const char* name,    /* object name from traverse list */
                   pack_opt_t *options) /* repack options */
 {
 	int          nfilters;       /* number of filters in DCPL */
- hsize_t      nelmts;         /* number of elements in dataset */
- size_t       size;           /* size of datatype in bytes */
  hsize_t      chsize[64];     /* chunk size in elements */
  H5D_layout_t layout;
  int          i;
 	pack_info_t  obj;
 
- if (rank==0)
-  goto out;
-
 /*-------------------------------------------------------------------------
 	* initialize the assigment object
 	*-------------------------------------------------------------------------
@@ -220,20 +215,6 @@ int apply_filters(const char* name,    /* object name from traverse list */
  /* get information about input filters */
  if ((nfilters = H5Pget_nfilters(dcpl_id))<0)
   return -1;
-
- /* check for datasets too small */
-	if ((size=H5Tget_size(type_id))==0)
-		return 0;
-	nelmts=1;
-	for (i=0; i<rank; i++)
-		nelmts*=dims[i];
-	if (nelmts*size < options->threshold )
-	{
-		if (nfilters && options->verbose)
-			printf("Warning: Filter not applied to <%s>. Dataset smaller than <%d> bytes\n",
-			name,(int)options->threshold);
-		return 0;
-	}
 	
  /*-------------------------------------------------------------------------
 		* check if we have filters in the pipeline
@@ -420,10 +401,6 @@ int apply_filters(const char* name,    /* object name from traverse list */
 
  return 0;
 
-out:
- if (options->verbose)
-  printf("Warning: Filter could not be applied to <%s>\n",name);
- return 0;
 }
 
 
diff --git a/tools/h5repack/h5repack_list.c b/tools/h5repack/h5repack_list.c
index b98af5c..7831380 100644
--- a/tools/h5repack/h5repack_list.c
+++ b/tools/h5repack/h5repack_list.c
@@ -133,7 +133,7 @@ int check_objects(const char* fname,
 
     if (csize < ppb )
     {
-     printf("Warning: SZIP settins, chunk size is smaller than pixels per block...Exiting\n");
+     printf("warning: SZIP settins, chunk size is smaller than pixels per block...Exiting\n");
      goto out;
     }
 
diff --git a/tools/h5repack/h5repack_refs.c b/tools/h5repack/h5repack_refs.c
index 4b075d5..83b1c13 100644
--- a/tools/h5repack/h5repack_refs.c
+++ b/tools/h5repack/h5repack_refs.c
@@ -52,15 +52,15 @@ int do_copy_refobjs(hid_t fidin,
                     trav_table_t *travt,
                     pack_opt_t *options) /* repack options */
 {
- hid_t     grp_in=(-1);            /* read group ID */
- hid_t     grp_out=(-1);           /* write group ID */
- hid_t     dset_in=(-1);           /* read dataset ID */
- hid_t     dset_out=(-1);          /* write dataset ID */
- hid_t     type_in=(-1);           /* named type ID */
- hid_t     dcpl_id=(-1);           /* dataset creation property list ID */
- hid_t     space_id=(-1);          /* space ID */
- hid_t     ftype_id=(-1);          /* file data type ID */
- hid_t     mtype_id=(-1);          /* memory data type ID */
+ hid_t     grp_in;            /* read group ID */
+ hid_t     grp_out;           /* write group ID */
+ hid_t     dset_in;           /* read dataset ID */
+ hid_t     dset_out;          /* write dataset ID */
+ hid_t     type_in;           /* named type ID */
+ hid_t     dcpl_id;           /* dataset creation property list ID */
+ hid_t     space_id;          /* space ID */
+ hid_t     ftype_id;          /* file data type ID */
+ hid_t     mtype_id;          /* memory data type ID */
  size_t    msize;             /* memory size of memory type */
  hsize_t   nelmts;            /* number of elements in dataset */
  int       rank;              /* rank of dataset */
@@ -466,11 +466,11 @@ static int copy_refs_attr(hid_t loc_in,
                           hid_t fidout         /* for saving references */
                           )
 {
- hid_t      attr_id=-1;      /* attr ID */
- hid_t      attr_out=-1;     /* attr ID */
- hid_t      space_id=-1;     /* space ID */
- hid_t      ftype_id=-1;     /* file data type ID */
- hid_t      mtype_id=-1;     /* memory data type ID */
+ hid_t      attr_id;      /* attr ID */
+ hid_t      attr_out;     /* attr ID */
+ hid_t      space_id;     /* space ID */
+ hid_t      ftype_id;     /* file data type ID */
+ hid_t      mtype_id;     /* memory data type ID */
  size_t     msize;        /* memory size of type */
  hsize_t    nelmts;       /* number of elements in dataset */
  int        rank;         /* rank of dataset */
diff --git a/tools/h5repack/testh5repack_main.c b/tools/h5repack/testh5repack_main.c
index 19dc956..58a7c2a 100644
--- a/tools/h5repack/testh5repack_main.c
+++ b/tools/h5repack/testh5repack_main.c
@@ -18,9 +18,6 @@
 #include "testh5repack.h"
 #include "h5diff.h"
 
-#if 0
-#define PACK_DEBUG
-#endif
 
 /*-------------------------------------------------------------------------
  * Function: main
diff --git a/tools/lib/h5diff_dset.c b/tools/lib/h5diff_dset.c
index 8802067..3fd280c 100644
--- a/tools/lib/h5diff_dset.c
+++ b/tools/lib/h5diff_dset.c
@@ -566,7 +566,7 @@ int diff_can_type( hid_t       f_type1, /* file data type */
 
  if ( (H5Tequal(f_type1, f_type2)==0) && options->m_verbose && obj1_name)
  {
-  printf("Warning: Different storage datatype\n");
+  printf("warning: different storage datatype\n");
   printf("<%s> has file datatype ", obj1_name);
   print_type(f_type1);
   printf("\n");
@@ -643,7 +643,7 @@ int diff_can_type( hid_t       f_type1, /* file data type */
  if (maxdim1 && maxdim2 && maxdim_diff==1 && obj1_name )
  {
   if (options->m_verbose) {
-   printf( "Warning: Different maximum dimensions\n");
+   printf( "warning: different maximum dimensions\n");
    printf("<%s> has max dimensions ", obj1_name);
    print_dims(rank1,maxdim1);
    printf("\n");
diff --git a/tools/lib/h5diff_util.c b/tools/lib/h5diff_util.c
index 6334d8d..76a3774 100644
--- a/tools/lib/h5diff_util.c
+++ b/tools/lib/h5diff_util.c
@@ -75,7 +75,7 @@ void parallel_print(const char* format, ...)
     
     overflow_file = HDtmpfile();
     if(overflow_file == NULL)
-     fprintf(stderr, "Warning: Could not create overflow file.  Output may be truncated.\n");
+     fprintf(stderr, "warning: could not create overflow file.  Output may be truncated.\n");
     else
      bytes_written = HDvfprintf(overflow_file, format, ap);
    }
diff --git a/tools/lib/h5tools_filters.c b/tools/lib/h5tools_filters.c
index 099f132..e765ddf 100644
--- a/tools/lib/h5tools_filters.c
+++ b/tools/lib/h5tools_filters.c
@@ -21,7 +21,7 @@
  */
 static void print_warning(const char *dname, const char *fname)
 {
- fprintf(stderr,"Warning: dataset <%s> cannot be read, %s filter is not available\n",
+ fprintf(stderr,"warning: dataset <%s> cannot be read, %s filter is not available\n",
   dname,fname);
 }
 
diff --git a/tools/testfiles/h5diff_50.txt b/tools/testfiles/h5diff_50.txt
index e76231e..045f795 100644
--- a/tools/testfiles/h5diff_50.txt
+++ b/tools/testfiles/h5diff_50.txt
@@ -2,7 +2,7 @@
 Expected output for 'h5diff file4.h5 file4.h5 -v dset0a dset0b'
 #############################
 Dataset:     </dset0a> and </dset0b>
-Warning: Different storage datatype
+warning: different storage datatype
 </dset0a> has file datatype H5T_STD_I16LE
 </dset0b> has file datatype H5T_STD_I32LE
 position        dset0a          dset0b          difference          
diff --git a/tools/testfiles/h5diff_57.txt b/tools/testfiles/h5diff_57.txt
index fb5137b..0fd7843 100644
--- a/tools/testfiles/h5diff_57.txt
+++ b/tools/testfiles/h5diff_57.txt
@@ -2,7 +2,7 @@
 Expected output for 'h5diff file4.h5 file4.h5 -v dset7a dset7b'
 #############################
 Dataset:     </dset7a> and </dset7b>
-Warning: Different storage datatype
+warning: different storage datatype
 </dset7a> has file datatype H5T_STD_I8LE
 </dset7b> has file datatype H5T_STD_U8LE
 Comparison not supported: </dset7a> has sign H5T_SGN_2 and </dset7b> has sign H5T_SGN_NONE