summaryrefslogtreecommitdiffstats
path: root/tools/h5repack
diff options
context:
space:
mode:
authorPedro Vicente Nunes <pvn@hdfgroup.org>2004-02-10 18:33:20 (GMT)
committerPedro Vicente Nunes <pvn@hdfgroup.org>2004-02-10 18:33:20 (GMT)
commit192c316dd5de41d9de2b8489406abe94bffccb7b (patch)
treef24ee502aab010bd3e3e373eaeda07a0f4ea9d0c /tools/h5repack
parentf1031738ef0f945a3f1d2b8f90363ee657220e17 (diff)
downloadhdf5-192c316dd5de41d9de2b8489406abe94bffccb7b.zip
hdf5-192c316dd5de41d9de2b8489406abe94bffccb7b.tar.gz
hdf5-192c316dd5de41d9de2b8489406abe94bffccb7b.tar.bz2
[svn-r8176] Purpose:
new feature, bug fix, changed function Description: 1) implemented the option that says if the dataset is too small , do not compress it 2) bug fix in the SZIP checking . only apply szip to atomic datatypes 3) made the apply_filters function more compact Solution: Platforms tested: linux AIX solaris Misc. update:
Diffstat (limited to 'tools/h5repack')
-rw-r--r--tools/h5repack/h5repack.c4
-rw-r--r--tools/h5repack/h5repack.h15
-rw-r--r--tools/h5repack/h5repack_copy.c28
-rw-r--r--tools/h5repack/h5repack_filters.c202
4 files changed, 150 insertions, 99 deletions
diff --git a/tools/h5repack/h5repack.c b/tools/h5repack/h5repack.c
index 2cbcf86..8cca76d 100644
--- a/tools/h5repack/h5repack.c
+++ b/tools/h5repack/h5repack.c
@@ -338,9 +338,9 @@ static int check_options(pack_opt_t *options)
szip_pixels_per_block=pack.filter[j].cd_values[0];
/* check szip parameters */
- if (check_szip(pack.chunk.rank,
+ if (check_szip(0, /* do not test size */
+ pack.chunk.rank,
pack.chunk.chunk_lengths,
- 0, /* do not test size */
szip_options_mask,
&szip_pixels_per_block,
options)==0)
diff --git a/tools/h5repack/h5repack.h b/tools/h5repack/h5repack.h
index eb71fec..5bda25c 100644
--- a/tools/h5repack/h5repack.h
+++ b/tools/h5repack/h5repack.h
@@ -180,10 +180,13 @@ int filter_this(const char* name,
pack_opt_t *options,
pack_info_t *pack); /* info about object to filter */
-int apply_filters(hid_t dcpl_id,
- size_t size, /* size of datatype in bytes */
+int apply_filters(const char* name, /* object name from traverse list */
+ int rank, /* rank of dataset */
+ hsize_t *dims, /* dimensions of dataset */
+ hid_t dcpl_id, /* dataset creation property list */
+ hid_t type_id, /* datatype */
pack_opt_t *options, /* repack options */
- pack_info_t *pack); /* info about object to filter */
+ pack_info_t *obj); /* info about object to filter */
int has_filter(hid_t dcpl_id,
H5Z_filter_t filtnin);
@@ -193,10 +196,10 @@ int check_szip_params( unsigned bits_per_pixel,
unsigned pixels_per_scanline,
hsize_t image_pixels);
-int check_szip(int rank, /* chunk rank */
+int check_szip(hid_t type_id, /* dataset datatype */
+ int rank, /* chunk rank */
hsize_t *dims, /* chunk dims */
- size_t size, /* size of datatype in bytes */
- unsigned szip_options_mask,
+ unsigned szip_options_mask /*IN*/,
unsigned *szip_pixels_per_block /*IN,OUT*/,
pack_opt_t *options);
diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c
index f9859aa..5134891 100644
--- a/tools/h5repack/h5repack_copy.c
+++ b/tools/h5repack/h5repack_copy.c
@@ -268,34 +268,18 @@ int do_copy_objects(hid_t fidin,
*/
if (layout_this(dcpl_id,travt->objs[i].name,options,&obj))
{
+ obj.chunk.rank=rank;
if (apply_layout(dcpl_id,&obj)<0)
goto error;
}
+
/*-------------------------------------------------------------------------
- * apply the filter; check first if the object is to be filtered.
+ * apply the filter; check if the object is to be filtered.
*-------------------------------------------------------------------------
*/
- if (filter_this(travt->objs[i].name,options,&obj))
- {
- if (rank)
- {
- /* filters require CHUNK layout; if we do not have one define a default */
- if (obj.chunk.rank<=0)
- {
- obj.chunk.rank=rank;
- for (j=0; j<rank; j++)
- obj.chunk.chunk_lengths[j] = dims[j];
- }
- if (apply_filters(dcpl_id,msize,options,&obj)<0)
- goto error;
- }
- else
- {
- if (options->verbose)
- printf("Warning: Filter could not be applied to <%s>\n",
- travt->objs[i].name);
- }/*rank*/
- }/*filter_this*/
+ if (apply_filters(travt->objs[i].name,rank,dims,dcpl_id,mtype_id,options,&obj)<0)
+ goto error;
+
}/*nelmts*/
/*-------------------------------------------------------------------------
diff --git a/tools/h5repack/h5repack_filters.c b/tools/h5repack/h5repack_filters.c
index 81a99ce..9a91679 100644
--- a/tools/h5repack/h5repack_filters.c
+++ b/tools/h5repack/h5repack_filters.c
@@ -94,62 +94,6 @@ int filter_this(const char* name, /* object name from traverse list */
}
-/*-------------------------------------------------------------------------
- * Function: print_filters
- *
- * Purpose: print the filters in DCPL
- *
- * Return: 0, ok, -1 no
- *
- * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu
- *
- * Date: December 19, 2003
- *
- *-------------------------------------------------------------------------
- */
-
-int print_filters(hid_t dcpl_id)
-{
- int nfilters; /* number of filters */
- unsigned filt_flags; /* filter flags */
- H5Z_filter_t filtn; /* filter identification number */
- unsigned cd_values[20]; /* filter client data values */
- size_t cd_nelmts; /* filter client number of values */
- size_t cd_num; /* filter client data counter */
- char f_name[256]; /* filter/file name */
- char s[64]; /* temporary string buffer */
- int i;
-
- /* get information about filters */
- if ((nfilters = H5Pget_nfilters(dcpl_id))<0)
- return -1;
-
- for (i=0; i<nfilters; i++)
- {
- cd_nelmts = NELMTS(cd_values);
- filtn = H5Pget_filter(dcpl_id,
- (unsigned)i,
- &filt_flags,
- &cd_nelmts,
- cd_values,
- sizeof(f_name),
- f_name);
-
- f_name[sizeof(f_name)-1] = '\0';
- sprintf(s, "Filter-%d:", i);
- printf(" %-10s %s-%u %s {", s,
- f_name[0]?f_name:"method",
- (unsigned)filtn,
- filt_flags & H5Z_FLAG_OPTIONAL?"OPT":"");
- for (cd_num=0; cd_num<cd_nelmts; cd_num++) {
- printf("%s%u", cd_num?", ":"", cd_values[cd_num]);
- }
- printf("}\n");
- }
-
- return 0;
-}
-
/*-------------------------------------------------------------------------
* Function: apply_filters
@@ -167,17 +111,46 @@ int print_filters(hid_t dcpl_id)
*-------------------------------------------------------------------------
*/
-int apply_filters(hid_t dcpl_id,
- size_t size, /* size of datatype in bytes */
+
+
+int apply_filters(const char* name, /* object name from traverse list */
+ int rank, /* rank of dataset */
+ hsize_t *dims, /* dimensions of dataset */
+ hid_t dcpl_id, /* dataset creation property list */
+ hid_t type_id, /* dataset datatype */
pack_opt_t *options, /* repack options */
pack_info_t *obj) /* info about object to filter */
{
int nfilters; /* number of filters in DCPL */
unsigned aggression; /* the deflate level */
+ hsize_t nelmts; /* number of elements in dataset */
+ size_t size; /* size of datatype in bytes */
unsigned szip_options_mask=H5_SZIP_NN_OPTION_MASK;
unsigned szip_pixels_per_block;
int i;
+ /* check first if the object is to be filtered */
+ if (filter_this(name,options,obj)==0)
+ return 0;
+
+ if (rank==0)
+ goto out;
+
+ /* check for datasets too small */
+ if ((size=H5Tget_size(type_id))==0)
+ return 0;
+ nelmts=1;
+ for (i=0; i<rank; i++)
+ nelmts*=dims[i];
+ if (nelmts*size < options->threshold )
+ {
+ if (options->verbose)
+ printf("Warning: Filter not applied to <%s>. Dataset smaller than <%d> bytes\n",
+ name,options->threshold);
+ return 0;
+ }
+
+
/* get information about input filters */
if ((nfilters = H5Pget_nfilters(dcpl_id))<0)
return -1;
@@ -210,6 +183,18 @@ int apply_filters(hid_t dcpl_id,
}
/*-------------------------------------------------------------------------
+ * filters require CHUNK layout; if we do not have one define a default
+ *-------------------------------------------------------------------------
+ */
+
+ if (obj->chunk.rank<=0)
+ {
+ obj->chunk.rank=rank;
+ for (i=0; i<rank; i++)
+ obj->chunk.chunk_lengths[i] = dims[i];
+ }
+
+/*-------------------------------------------------------------------------
* the type of filter and additional parameter
* type can be one of the filters
* H5Z_FILTER_NONE 0, uncompress if compressed
@@ -219,7 +204,7 @@ int apply_filters(hid_t dcpl_id,
* H5Z_FILTER_SZIP 4 , szip compression
*-------------------------------------------------------------------------
*/
-
+
for ( i=0; i<obj->nfilters; i++)
{
switch (obj->filter[i].filtn)
@@ -247,9 +232,9 @@ int apply_filters(hid_t dcpl_id,
case H5Z_FILTER_SZIP:
szip_pixels_per_block=obj->filter[i].cd_values[0];
/* check szip parameters */
- if (check_szip(obj->chunk.rank,
+ if (check_szip(type_id,
+ obj->chunk.rank,
obj->chunk.chunk_lengths,
- size,
szip_options_mask,
&szip_pixels_per_block,
options)==1)
@@ -262,7 +247,8 @@ int apply_filters(hid_t dcpl_id,
}
else
{
- printf("Warning: SZIP filter cannot be applied\n");
+ if (options->verbose)
+ printf("Warning: SZIP filter cannot be applied to <%s>\n",name);
}
break;
@@ -291,9 +277,71 @@ int apply_filters(hid_t dcpl_id,
}/*i*/
return 0;
+
+out:
+ if (options->verbose)
+ printf("Warning: Filter could not be applied to <%s>\n",name);
+ return 0;
+
}
+/*-------------------------------------------------------------------------
+ * Function: print_filters
+ *
+ * Purpose: print the filters in DCPL
+ *
+ * Return: 0, ok, -1 no
+ *
+ * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu
+ *
+ * Date: December 19, 2003
+ *
+ *-------------------------------------------------------------------------
+ */
+
+int print_filters(hid_t dcpl_id)
+{
+ int nfilters; /* number of filters */
+ unsigned filt_flags; /* filter flags */
+ H5Z_filter_t filtn; /* filter identification number */
+ unsigned cd_values[20]; /* filter client data values */
+ size_t cd_nelmts; /* filter client number of values */
+ size_t cd_num; /* filter client data counter */
+ char f_name[256]; /* filter/file name */
+ char s[64]; /* temporary string buffer */
+ int i;
+
+ /* get information about filters */
+ if ((nfilters = H5Pget_nfilters(dcpl_id))<0)
+ return -1;
+
+ for (i=0; i<nfilters; i++)
+ {
+ cd_nelmts = NELMTS(cd_values);
+ filtn = H5Pget_filter(dcpl_id,
+ (unsigned)i,
+ &filt_flags,
+ &cd_nelmts,
+ cd_values,
+ sizeof(f_name),
+ f_name);
+
+ f_name[sizeof(f_name)-1] = '\0';
+ sprintf(s, "Filter-%d:", i);
+ printf(" %-10s %s-%u %s {", s,
+ f_name[0]?f_name:"method",
+ (unsigned)filtn,
+ filt_flags & H5Z_FLAG_OPTIONAL?"OPT":"");
+ for (cd_num=0; cd_num<cd_nelmts; cd_num++) {
+ printf("%s%u", cd_num?", ":"", cd_values[cd_num]);
+ }
+ printf("}\n");
+ }
+
+ return 0;
+}
+
/*-------------------------------------------------------------------------
* Function: check_szip
@@ -317,17 +365,35 @@ int apply_filters(hid_t dcpl_id,
*-------------------------------------------------------------------------
*/
-int check_szip(int rank, /* chunk rank */
+int check_szip(hid_t type_id, /* dataset datatype */
+ int rank, /* chunk rank */
hsize_t *dims, /* chunk dims */
- size_t size, /* size of datatype in bytes */
unsigned szip_options_mask /*IN*/,
unsigned *szip_pixels_per_block /*IN,OUT*/,
pack_opt_t *options)
{
+ size_t size; /* size of datatype in bytes */
szip_comp_t szip;
int i;
unsigned ppb=*szip_pixels_per_block;
+ if (type_id)
+ {
+ if ((size=H5Tget_size(type_id))==0)
+ return 0;
+
+ switch (H5Tget_class(type_id))
+ {
+ default:
+ return 0;
+ break;
+ case H5T_INTEGER:
+ case H5T_FLOAT:
+ break;
+ }
+ }
+
+
/*
pixels_per_scanline = size of the fastest-changing dimension
Must be <= MAX_PIXELS_PER_SCANLINE and <= pixels
@@ -380,12 +446,10 @@ int check_szip(int rank, /* chunk rank */
bits_per_pixel
Must be in range 1..24,32,64
*/
+ szip.bits_per_pixel = 0;
+ if (type_id) {
switch(size)
{
- case 0:
- /* size was not provided for test */
- szip.bits_per_pixel = 0;
- break;
case 1:
szip.bits_per_pixel = 8;
break;
@@ -401,7 +465,7 @@ int check_szip(int rank, /* chunk rank */
default:
printf("Warning: Invalid numeric type of size <%d> for SZIP\n",size);
return 0;
- }
+ }}
return check_szip_params( szip.bits_per_pixel,
szip.pixels_per_block,