[svn-r8176] Purpose:

new feature, bug fix, changed function

Description:
1) implemented the option that says if the dataset is too small , do not compress it
2) bug fix in the SZIP checking . only apply szip to atomic datatypes
3) made the apply_filters function more compact


Solution:

Platforms tested:
linux
AIX
solaris

Misc. update:

4 files changed, 150 insertions, 99 deletions

diff --git a/tools/h5repack/h5repack.c b/tools/h5repack/h5repack.c
index 2cbcf86..8cca76d 100644
--- a/tools/h5repack/h5repack.c
+++ b/tools/h5repack/h5repack.c
@@ -338,9 +338,9 @@ static int check_options(pack_opt_t *options)
     szip_pixels_per_block=pack.filter[j].cd_values[0];
     
     /* check szip parameters */
-    if (check_szip(pack.chunk.rank,
+    if (check_szip(0, /* do not test size */
+     pack.chunk.rank,
      pack.chunk.chunk_lengths,
-     0, /* do not test size */
      szip_options_mask,
      &szip_pixels_per_block,
      options)==0)
diff --git a/tools/h5repack/h5repack.h b/tools/h5repack/h5repack.h
index eb71fec..5bda25c 100644
--- a/tools/h5repack/h5repack.h
+++ b/tools/h5repack/h5repack.h
@@ -180,10 +180,13 @@ int filter_this(const char* name,
                 pack_opt_t *options,
                 pack_info_t *pack); /* info about object to filter */
 
-int apply_filters(hid_t dcpl_id,
-                  size_t size,         /* size of datatype in bytes */
+int apply_filters(const char* name,    /* object name from traverse list */
+                  int rank,            /* rank of dataset */
+                  hsize_t *dims,       /* dimensions of dataset */
+                  hid_t dcpl_id,       /* dataset creation property list */
+                  hid_t type_id,       /* datatype */
                   pack_opt_t *options, /* repack options */
-                  pack_info_t *pack);  /* info about object to filter */
+                  pack_info_t *obj);   /* info about object to filter */
 
 int has_filter(hid_t dcpl_id,
                H5Z_filter_t filtnin);
@@ -193,10 +196,10 @@ int check_szip_params( unsigned bits_per_pixel,
                        unsigned pixels_per_scanline, 
                        hsize_t image_pixels);
 
-int check_szip(int rank,        /* chunk rank */
+int check_szip(hid_t type_id,   /* dataset datatype */
+               int rank,        /* chunk rank */
                hsize_t *dims,   /* chunk dims */
-               size_t size,     /* size of datatype in bytes */
-               unsigned szip_options_mask,
+               unsigned szip_options_mask /*IN*/,
                unsigned *szip_pixels_per_block /*IN,OUT*/,
                pack_opt_t *options);
 
diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c
index f9859aa..5134891 100644
--- a/tools/h5repack/h5repack_copy.c
+++ b/tools/h5repack/h5repack_copy.c
@@ -268,34 +268,18 @@ int do_copy_objects(hid_t fidin,
       */
      if (layout_this(dcpl_id,travt->objs[i].name,options,&obj))
      {
+      obj.chunk.rank=rank;
       if (apply_layout(dcpl_id,&obj)<0)
        goto error;
      }
+
     /*-------------------------------------------------------------------------
-     * apply the filter; check first if the object is to be filtered.
+     * apply the filter; check if the object is to be filtered.
      *-------------------------------------------------------------------------
      */
-     if (filter_this(travt->objs[i].name,options,&obj))
-     {
-      if (rank)
-      {
-       /* filters require CHUNK layout; if we do not have one define a default */
-       if (obj.chunk.rank<=0)
-       {
-        obj.chunk.rank=rank;
-        for (j=0; j<rank; j++) 
-         obj.chunk.chunk_lengths[j] = dims[j];
-       }
-       if (apply_filters(dcpl_id,msize,options,&obj)<0)
-        goto error;
-      }
-      else
-      {
-       if (options->verbose)
-        printf("Warning: Filter could not be applied to <%s>\n",
-        travt->objs[i].name);
-      }/*rank*/
-     }/*filter_this*/
+     if (apply_filters(travt->objs[i].name,rank,dims,dcpl_id,mtype_id,options,&obj)<0)
+      goto error;
+
     }/*nelmts*/
     
     /*-------------------------------------------------------------------------
diff --git a/tools/h5repack/h5repack_filters.c b/tools/h5repack/h5repack_filters.c
index 81a99ce..9a91679 100644
--- a/tools/h5repack/h5repack_filters.c
+++ b/tools/h5repack/h5repack_filters.c
@@ -94,62 +94,6 @@ int filter_this(const char* name,    /* object name from traverse list */
 }
 
 
-/*-------------------------------------------------------------------------
- * Function: print_filters
- *
- * Purpose: print the filters in DCPL
- *
- * Return: 0, ok, -1 no
- *
- * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu
- *
- * Date: December 19, 2003
- *
- *-------------------------------------------------------------------------
- */
-
-int print_filters(hid_t dcpl_id)   
-{
- int          nfilters;       /* number of filters */
- unsigned     filt_flags;     /* filter flags */
- H5Z_filter_t filtn;          /* filter identification number */
- unsigned     cd_values[20];  /* filter client data values */
- size_t       cd_nelmts;      /* filter client number of values */
- size_t       cd_num;         /* filter client data counter */
- char         f_name[256];    /* filter/file name */
- char         s[64];          /* temporary string buffer */
- int          i;
-
- /* get information about filters */
- if ((nfilters = H5Pget_nfilters(dcpl_id))<0) 
-  return -1;
- 
- for (i=0; i<nfilters; i++) 
- {
-  cd_nelmts = NELMTS(cd_values);
-  filtn = H5Pget_filter(dcpl_id, 
-   (unsigned)i, 
-   &filt_flags, 
-   &cd_nelmts,
-   cd_values, 
-   sizeof(f_name), 
-   f_name);
-  
-  f_name[sizeof(f_name)-1] = '\0';
-  sprintf(s, "Filter-%d:", i);
-  printf("    %-10s %s-%u %s {", s,
-   f_name[0]?f_name:"method",
-   (unsigned)filtn,
-   filt_flags & H5Z_FLAG_OPTIONAL?"OPT":"");
-  for (cd_num=0; cd_num<cd_nelmts; cd_num++) {
-   printf("%s%u", cd_num?", ":"", cd_values[cd_num]);
-  }
-  printf("}\n");
- } 
- 
- return 0;
-}
-
 
 /*-------------------------------------------------------------------------
  * Function: apply_filters
@@ -167,17 +111,46 @@ int print_filters(hid_t dcpl_id)
  *-------------------------------------------------------------------------
  */
 
-int apply_filters(hid_t dcpl_id,
-                  size_t size,         /* size of datatype in bytes */
+
+
+int apply_filters(const char* name,    /* object name from traverse list */
+                  int rank,            /* rank of dataset */
+                  hsize_t *dims,       /* dimensions of dataset */
+                  hid_t dcpl_id,       /* dataset creation property list */
+                  hid_t type_id,       /* dataset datatype */
                   pack_opt_t *options, /* repack options */
                   pack_info_t *obj)    /* info about object to filter */
 {
  int          nfilters;       /* number of filters in DCPL */
  unsigned     aggression;     /* the deflate level */
+ hsize_t      nelmts;         /* number of elements in dataset */
+ size_t       size;           /* size of datatype in bytes */
  unsigned     szip_options_mask=H5_SZIP_NN_OPTION_MASK;
  unsigned     szip_pixels_per_block;
  int          i;
 
+ /* check first if the object is to be filtered */
+ if (filter_this(name,options,obj)==0)
+  return 0;
+
+ if (rank==0)
+  goto out;
+
+ /* check for datasets too small */
+  if ((size=H5Tget_size(type_id))==0)
+   return 0;
+  nelmts=1;
+  for (i=0; i<rank; i++) 
+   nelmts*=dims[i];
+  if (nelmts*size < options->threshold )
+  {
+   if (options->verbose)
+    printf("Warning: Filter not applied to <%s>. Dataset smaller than <%d> bytes\n",
+    name,options->threshold);
+   return 0;
+  }
+  
+
  /* get information about input filters */
  if ((nfilters = H5Pget_nfilters(dcpl_id))<0) 
   return -1;
@@ -210,6 +183,18 @@ int apply_filters(hid_t dcpl_id,
  }
 
 /*-------------------------------------------------------------------------
+ * filters require CHUNK layout; if we do not have one define a default
+ *-------------------------------------------------------------------------
+ */
+ 
+ if (obj->chunk.rank<=0)
+ {
+  obj->chunk.rank=rank;
+  for (i=0; i<rank; i++) 
+   obj->chunk.chunk_lengths[i] = dims[i];
+ }
+
+/*-------------------------------------------------------------------------
  * the type of filter and additional parameter 
  * type can be one of the filters
  * H5Z_FILTER_NONE       0,  uncompress if compressed
@@ -219,7 +204,7 @@ int apply_filters(hid_t dcpl_id,
  * H5Z_FILTER_SZIP       4 , szip compression 
  *-------------------------------------------------------------------------
  */
-
+ 
  for ( i=0; i<obj->nfilters; i++)
  {
   switch (obj->filter[i].filtn)
@@ -247,9 +232,9 @@ int apply_filters(hid_t dcpl_id,
   case H5Z_FILTER_SZIP:
    szip_pixels_per_block=obj->filter[i].cd_values[0];
    /* check szip parameters */
-   if (check_szip(obj->chunk.rank,
+   if (check_szip(type_id,
+    obj->chunk.rank,
     obj->chunk.chunk_lengths,
-    size,
     szip_options_mask,
     &szip_pixels_per_block,
     options)==1)
@@ -262,7 +247,8 @@ int apply_filters(hid_t dcpl_id,
    }
    else
    {
-    printf("Warning: SZIP filter cannot be applied\n");
+    if (options->verbose)
+     printf("Warning: SZIP filter cannot be applied to <%s>\n",name);
    }
    break;
 
@@ -291,9 +277,71 @@ int apply_filters(hid_t dcpl_id,
  }/*i*/
 
  return 0;
+
+out:
+ if (options->verbose)
+  printf("Warning: Filter could not be applied to <%s>\n",name);
+ return 0;
+
 }
 
 
+/*-------------------------------------------------------------------------
+ * Function: print_filters
+ *
+ * Purpose: print the filters in DCPL
+ *
+ * Return: 0, ok, -1 no
+ *
+ * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu
+ *
+ * Date: December 19, 2003
+ *
+ *-------------------------------------------------------------------------
+ */
+
+int print_filters(hid_t dcpl_id)   
+{
+ int          nfilters;       /* number of filters */
+ unsigned     filt_flags;     /* filter flags */
+ H5Z_filter_t filtn;          /* filter identification number */
+ unsigned     cd_values[20];  /* filter client data values */
+ size_t       cd_nelmts;      /* filter client number of values */
+ size_t       cd_num;         /* filter client data counter */
+ char         f_name[256];    /* filter/file name */
+ char         s[64];          /* temporary string buffer */
+ int          i;
+
+ /* get information about filters */
+ if ((nfilters = H5Pget_nfilters(dcpl_id))<0) 
+  return -1;
+ 
+ for (i=0; i<nfilters; i++) 
+ {
+  cd_nelmts = NELMTS(cd_values);
+  filtn = H5Pget_filter(dcpl_id, 
+   (unsigned)i, 
+   &filt_flags, 
+   &cd_nelmts,
+   cd_values, 
+   sizeof(f_name), 
+   f_name);
+  
+  f_name[sizeof(f_name)-1] = '\0';
+  sprintf(s, "Filter-%d:", i);
+  printf("    %-10s %s-%u %s {", s,
+   f_name[0]?f_name:"method",
+   (unsigned)filtn,
+   filt_flags & H5Z_FLAG_OPTIONAL?"OPT":"");
+  for (cd_num=0; cd_num<cd_nelmts; cd_num++) {
+   printf("%s%u", cd_num?", ":"", cd_values[cd_num]);
+  }
+  printf("}\n");
+ } 
+ 
+ return 0;
+}
+
 
 /*-------------------------------------------------------------------------
  * Function: check_szip
@@ -317,17 +365,35 @@ int apply_filters(hid_t dcpl_id,
  *-------------------------------------------------------------------------
  */
 
-int check_szip(int rank,        /* chunk rank */
+int check_szip(hid_t type_id,   /* dataset datatype */
+               int rank,        /* chunk rank */
                hsize_t *dims,   /* chunk dims */
-               size_t size,     /* size of datatype in bytes */
                unsigned szip_options_mask /*IN*/,
                unsigned *szip_pixels_per_block /*IN,OUT*/,
                pack_opt_t *options)
 {
+ size_t      size;     /* size of datatype in bytes */
  szip_comp_t szip;
  int         i;
  unsigned    ppb=*szip_pixels_per_block;
 
+ if (type_id)
+ {
+  if ((size=H5Tget_size(type_id))==0)
+   return 0;
+  
+  switch (H5Tget_class(type_id)) 
+  {
+  default:
+   return 0;
+   break;
+  case H5T_INTEGER:
+  case H5T_FLOAT:
+   break;
+  }
+ }
+
+
  /*
  pixels_per_scanline = size of the fastest-changing dimension 
  Must be <= MAX_PIXELS_PER_SCANLINE and <= pixels
@@ -380,12 +446,10 @@ int check_szip(int rank,        /* chunk rank */
   bits_per_pixel
   Must be in range 1..24,32,64
   */
+ szip.bits_per_pixel = 0;
+ if (type_id) {
  switch(size) 
  {
- case 0:
-  /* size was not provided for test */
-  szip.bits_per_pixel = 0;
-  break;
  case 1:
   szip.bits_per_pixel = 8;
   break;
@@ -401,7 +465,7 @@ int check_szip(int rank,        /* chunk rank */
  default:
   printf("Warning: Invalid numeric type of size <%d> for SZIP\n",size);
   return 0;
- }
+ }}
 
  return check_szip_params( szip.bits_per_pixel, 
                            szip.pixels_per_block,