summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPedro Vicente Nunes <pvn@hdfgroup.org>2005-09-19 21:12:35 (GMT)
committerPedro Vicente Nunes <pvn@hdfgroup.org>2005-09-19 21:12:35 (GMT)
commitdca5ad5754071a785c1977c0252e49b7ff3f4721 (patch)
treeade2fafa0d05d2ce2249c4fd9864432ae092d8f9
parent465f9be6e3f818d20c650976a14d7698063cad00 (diff)
downloadhdf5-dca5ad5754071a785c1977c0252e49b7ff3f4721.zip
hdf5-dca5ad5754071a785c1977c0252e49b7ff3f4721.tar.gz
hdf5-dca5ad5754071a785c1977c0252e49b7ff3f4721.tar.bz2
[svn-r11443] Purpose:
new features Description: added support for the scale/offset filter there is a new filter symbol 'SOFF' -f SOFF=<scale_factor,scale_type> scale_factor = integer scale_type = 'IN' or 'DS' Solution: Platforms tested: Linux SunOS Misc. update:
-rw-r--r--tools/h5repack/h5repack.h20
-rw-r--r--tools/h5repack/h5repack_filters.c57
-rw-r--r--tools/h5repack/h5repack_main.c22
-rw-r--r--tools/h5repack/h5repack_opttable.c2
-rw-r--r--tools/h5repack/h5repack_parse.c89
-rw-r--r--tools/h5repack/testh5repack_main.c3
6 files changed, 132 insertions, 61 deletions
diff --git a/tools/h5repack/h5repack.h b/tools/h5repack/h5repack.h
index 0ced373..dab4ece 100644
--- a/tools/h5repack/h5repack.h
+++ b/tools/h5repack/h5repack.h
@@ -30,7 +30,6 @@
#define MAX_NC_NAME 256 /* max length of a name */
#define MAX_VAR_DIMS 32 /* max per variable dimensions */
-
/*-------------------------------------------------------------------------
* data structures for command line options
*-------------------------------------------------------------------------
@@ -44,11 +43,13 @@ typedef struct {
/*
the type of filter and additional parameter
type can be one of the filters
- H5Z_FILTER_NONE 0, uncompress if compressed
- H5Z_FILTER_DEFLATE 1 , deflation like gzip
- H5Z_FILTER_SHUFFLE 2 , shuffle the data
- H5Z_FILTER_FLETCHER32 3 , letcher32 checksum of EDC
- H5Z_FILTER_SZIP 4 , szip compression
+ H5Z_FILTER_NONE 0, uncompress if compressed
+ H5Z_FILTER_DEFLATE 1 , deflation like gzip
+ H5Z_FILTER_SHUFFLE 2 , shuffle the data
+ H5Z_FILTER_FLETCHER32 3 , letcher32 checksum of EDC
+ H5Z_FILTER_SZIP 4 , szip compression
+ H5Z_FILTER_NBIT 5 , nbit compression
+ H5Z_FILTER_SCALEOFFSET 6 , scaleoffset compression
*/
#define CDVALUES 2
@@ -56,11 +57,6 @@ typedef struct {
typedef struct {
H5Z_filter_t filtn; /* filter identification number */
int cd_values[CDVALUES]; /* filter client data values */
- /* extra input for szip, selects the coding method
- entropy coding method: EC=0
- nearest neighbor coding method: NN=1
- */
- int szip_coding;
} filter_info_t;
/* chunk lengths along each dimension and rank */
@@ -71,7 +67,7 @@ typedef struct {
/* we currently define a maximum value for the filters array,
that corresponds to the current library filters */
-#define H5_REPACK_MAX_NFILTERS 4
+#define H5_REPACK_MAX_NFILTERS 6
/* information for one object, contains PATH, CHUNK info and FILTER info */
typedef struct {
diff --git a/tools/h5repack/h5repack_filters.c b/tools/h5repack/h5repack_filters.c
index db3277b..1f4c2d7 100644
--- a/tools/h5repack/h5repack_filters.c
+++ b/tools/h5repack/h5repack_filters.c
@@ -147,8 +147,6 @@ int aux_assign_obj(const char* name, /* object name from traverse lis
tmp.nfilters=1;
tmp.filter[0]=options->filter_g;
}
-
-
if (options->all_layout)
{
/* assign the global layout info to the OBJ info */
@@ -163,10 +161,7 @@ int aux_assign_obj(const char* name, /* object name from traverse lis
default:
break;
}/*switch*/
-
}
-
-
}
*obj = tmp;
@@ -199,7 +194,6 @@ int apply_filters(const char* name, /* object name from traverse list */
pack_opt_t *options) /* repack options */
{
int nfilters; /* number of filters in DCPL */
- unsigned aggression; /* the deflate level */
hsize_t nelmts; /* number of elements in dataset */
size_t size; /* size of datatype in bytes */
hsize_t chsize[64]; /* chunk size in elements */
@@ -276,10 +270,7 @@ int apply_filters(const char* name, /* object name from traverse list */
}
}
-
-
-
- /*-------------------------------------------------------------------------
+/*-------------------------------------------------------------------------
* the type of filter and additional parameter
* type can be one of the filters
* H5Z_FILTER_NONE 0, uncompress if compressed
@@ -318,12 +309,16 @@ int apply_filters(const char* name, /* object name from traverse list */
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_DEFLATE:
- aggression=obj.filter[i].cd_values[0];
- /* set up for deflated data */
- if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0)
- return -1;
- if(H5Pset_deflate(dcpl_id,aggression)<0)
- return -1;
+ {
+ unsigned aggression; /* the deflate level */
+
+ aggression = obj.filter[i].cd_values[0];
+ /* set up for deflated data */
+ if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0)
+ return -1;
+ if(H5Pset_deflate(dcpl_id,aggression)<0)
+ return -1;
+ }
break;
/*-------------------------------------------------------------------------
@@ -335,12 +330,9 @@ int apply_filters(const char* name, /* object name from traverse list */
unsigned options_mask;
unsigned pixels_per_block;
- pixels_per_block=obj.filter[i].cd_values[0];
- if (obj.filter[i].szip_coding==0)
- options_mask=H5_SZIP_NN_OPTION_MASK;
- else
- options_mask=H5_SZIP_EC_OPTION_MASK;
-
+ options_mask = obj.filter[i].cd_values[0];
+ pixels_per_block = obj.filter[i].cd_values[1];
+
/* set up for szip data */
if(H5Pset_chunk(dcpl_id,obj.chunk.rank,obj.chunk.chunk_lengths)<0)
return -1;
@@ -385,15 +377,20 @@ int apply_filters(const char* name, /* object name from traverse list */
* H5Z_FILTER_SCALEOFFSET , scale+offset compression
*-------------------------------------------------------------------------
*/
+
case H5Z_FILTER_SCALEOFFSET:
- aggression=obj.filter[i].cd_values[0];
- if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0)
- return -1;
- /* The following line needs to be changed in the future,
- H5_SO_INT needs to be passed as a parameter, it can
- be either H5_SO_FLOAT_DSCALE or H5_SO_INT. KY, 2005/8/31 */
- if (H5Pset_scaleoffset(dcpl_id,H5Z_SO_INT,aggression)<0)
- return -1;
+ {
+ H5Z_SO_scale_type_t scale_type;
+ int scale_factor;
+
+ scale_type = obj.filter[i].cd_values[0];
+ scale_factor = obj.filter[i].cd_values[1];
+
+ if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0)
+ return -1;
+ if (H5Pset_scaleoffset(dcpl_id,scale_type,scale_factor)<0)
+ return -1;
+ }
break;
} /* switch */
}/*i*/
diff --git a/tools/h5repack/h5repack_main.c b/tools/h5repack/h5repack_main.c
index 665f8e1..96eb787 100644
--- a/tools/h5repack/h5repack_main.c
+++ b/tools/h5repack/h5repack_main.c
@@ -136,9 +136,12 @@ void usage(void)
printf("-i input Input HDF5 File\n");
printf("-o output Output HDF5 File\n");
printf("[-h] Print usage message\n");
- printf("[-v] Verbose mode. Print output (list of objects, filters, warnings)\n");
+ printf("[-v] Verbose mode. Print more output (list of objects,\n");
+ printf(" filters, warnings)\n");
printf("[-f 'filter'] Filter type: 'filter' is a string with the format\n");
+ printf("\n");
printf(" <list of objects> : <name of filter> = <filter parameters>\n");
+ printf("\n");
printf(" <list of objects> is a comma separated list of object names\n");
printf(" meaning apply compression only to those objects.\n");
printf(" if no object names are specified, the filter is applied to all objects\n");
@@ -148,15 +151,24 @@ void usage(void)
printf(" SHUF, to apply the HDF5 shuffle filter\n");
printf(" FLET, to apply the HDF5 checksum filter\n");
printf(" NBIT, to apply the HDF5 NBIT filter (NBIT compression)\n");
+ printf(" SOFF, to apply the HDF5 Scale/Offset filter\n");
printf(" NONE, to remove the filter\n");
printf(" <filter parameters> is optional compression info\n");
printf(" SHUF (no parameter)\n");
printf(" FLET (no parameter)\n");
printf(" NBIT (no parameter)\n");
printf(" GZIP=<deflation level> from 1-9\n");
- printf(" SZIP=<pixels per block,coding> (pixels per block is a even number in 2-32 and coding method is 'EC' or 'NN')\n");
+ printf(" SZIP=<pixels per block,coding>\n");
+ printf(" (pixels per block is a even number in 2-32 and coding method\n");
+ printf(" is 'EC' or 'NN')\n");
+ printf(" SOFF=<scale_factor,scale_type>\n");
+ printf(" (scale_factor is an integer and scale_type is either 'IN'\n");
+ printf(" for integer type, or 'DS', for floating point type\n");
+ printf(" using the D-scaling method)\n");
printf("[-l 'layout'] Layout type. 'layout' is a string with the format\n");
+ printf("\n");
printf(" <list of objects> : <layout type>\n");
+ printf("\n");
printf(" <list of objects> is a comma separated list of object names,\n");
printf(" meaning that layout information is supplied for those objects.\n");
printf(" if no object names are specified, the layout is applied to all objects\n");
@@ -169,8 +181,9 @@ void usage(void)
printf(" <dim_1 x dim_2 x ... dim_n>\n");
printf("\n");
printf("-e file File with the -f and -l options (only filter and layout flags)\n");
- printf("-m number Do not apply the filter to objects which size in bytes is smaller than number.\n");
- printf(" If no size is specified a minimum of 1024 bytes is assumed.\n");
+ printf("-m size Do not apply the filter to objects which size in bytes\n");
+ printf(" is smaller than number. If no size is specified a minimum of\n");
+ printf(" 1024 bytes is assumed.\n");
printf("\n");
printf("Examples of use:\n");
printf("\n");
@@ -188,4 +201,3 @@ void usage(void)
printf("\n");
}
-
diff --git a/tools/h5repack/h5repack_opttable.c b/tools/h5repack/h5repack_opttable.c
index 7bb6d10..d4c2532 100644
--- a/tools/h5repack/h5repack_opttable.c
+++ b/tools/h5repack/h5repack_opttable.c
@@ -17,7 +17,6 @@
#include "h5repack.h"
-
/*-------------------------------------------------------------------------
* Function: init_packobject
*
@@ -36,7 +35,6 @@ void init_packobject(pack_info_t *obj)
for ( j=0; j<H5_REPACK_MAX_NFILTERS; j++)
{
obj->filter[j].filtn = -1;
- obj->filter[j].szip_coding = -1;
for ( k=0; k<CDVALUES; k++)
obj->filter[j].cd_values[k] = -1;
}
diff --git a/tools/h5repack/h5repack_parse.c b/tools/h5repack/h5repack_parse.c
index 7e82fde..1c67216 100644
--- a/tools/h5repack/h5repack_parse.c
+++ b/tools/h5repack/h5repack_parse.c
@@ -35,7 +35,7 @@
* SHUF, to apply the HDF5 shuffle filter
* FLET, to apply the HDF5 checksum filter
* NBIT, to apply the HDF5 NBIT filter (NBIT compression)
- * S+O, to apply the HDF5 scale+offset filter (compression)
+ * SOFF, to apply the HDF5 scale+offset filter (compression)
* NONE, to remove the filter
*
* Examples:
@@ -135,11 +135,13 @@ obj_list_t* parse_filter(const char *str,
if ( c=='=') { /*one more parameter */
scomp[k]='\0'; /*cut space */
- /*SZIP is a special case , it can be
- SZIP=8,EC
- SZIP=8,NN
- */
-
+/*-------------------------------------------------------------------------
+ * H5Z_FILTER_SZIP
+ * szip has the format SZIP=<pixels per block,coding>
+ * pixels per block is a even number in 2-32 and coding method is 'EC' or 'NN'
+ * example SZIP=8,NN
+ *-------------------------------------------------------------------------
+ */
if (strcmp(scomp,"SZIP")==0)
{
l=-1; /* mask index check */
@@ -169,9 +171,9 @@ obj_list_t* parse_filter(const char *str,
i=len-1; /* end */
(*n_objs)--; /* we counted an extra ',' */
if (strcmp(smask,"NN")==0)
- filt->szip_coding=0;
+ filt->cd_values[j++]=H5_SZIP_NN_OPTION_MASK;
else if (strcmp(smask,"EC")==0)
- filt->szip_coding=1;
+ filt->cd_values[j++]=H5_SZIP_EC_OPTION_MASK;
else
{
printf("Input Error: szip mask must be 'NN' or 'EC' \n");
@@ -184,6 +186,71 @@ obj_list_t* parse_filter(const char *str,
} /* u */
} /*if */
+/*-------------------------------------------------------------------------
+ * H5Z_FILTER_SCALEOFFSET
+ * scaleoffset has the format SOFF=<scale_factor,scale_type>
+ * scale_type can be
+ * integer datatype, H5Z_SO_INT (IN)
+ * float datatype using D-scaling method, H5Z_SO_FLOAT_DSCALE (DS)
+ * float datatype using E-scaling method, H5Z_SO_FLOAT_ESCALE (ES) , not yet implemented
+ * for integer datatypes, scale_factor denotes Minimum Bits
+ * for float datatypes, scale_factor denotes decimal scale factor
+ * examples
+ * SOFF=31,IN
+ * SOFF=3,DF
+ *-------------------------------------------------------------------------
+ */
+
+ else if (strcmp(scomp,"SOFF")==0)
+ {
+ l=-1; /* mask index check */
+ for ( m=0,u=i+1; u<len; u++,m++)
+ {
+ if (str[u]==',')
+ {
+ stype[m]='\0'; /* end digit */
+ l=0; /* start 'IN' , 'DS', or 'ES' search */
+ u++; /* skip ',' */
+ }
+ c = str[u];
+ if (!isdigit(c) && l==-1){
+ if (obj_list) free(obj_list);
+ printf("Input Error: Compression parameter not digit in <%s>\n",str);
+ exit(1);
+ }
+ if (l==-1)
+ stype[m]=c;
+ else
+ {
+ smask[l]=c;
+ l++;
+ if (l==2)
+ {
+ smask[l]='\0';
+ i=len-1; /* end */
+ (*n_objs)--; /* we counted an extra ',' */
+ if (strcmp(smask,"IN")==0)
+ filt->cd_values[j++]=H5Z_SO_INT;
+ else if (strcmp(smask,"DS")==H5Z_SO_FLOAT_DSCALE)
+ filt->cd_values[j++]=H5Z_SO_FLOAT_DSCALE;
+ else
+ {
+ printf("Input Error: scale type must be 'IN' or 'DS' \n");
+ exit(1);
+ }
+
+ }
+ }
+
+ } /* u */
+ } /*if */
+
+
+/*-------------------------------------------------------------------------
+ * all other filters
+ *-------------------------------------------------------------------------
+ */
+
else
{
/* here we could have 1 or 2 digits */
@@ -289,10 +356,10 @@ obj_list_t* parse_filter(const char *str,
* H5Z_FILTER_SCALEOFFSET
*-------------------------------------------------------------------------
*/
- else if (strcmp(scomp,"S+O")==0)
+ else if (strcmp(scomp,"SOFF")==0)
{
filt->filtn=H5Z_FILTER_SCALEOFFSET;
- if (no_param) { /*no more parameters, S+O must have parameter */
+ if (no_param) { /*no more parameters, SOFF must have parameter */
if (obj_list) free(obj_list);
printf("Input Error: Missing compression parameter in <%s>\n",str);
exit(1);
@@ -379,7 +446,7 @@ const char* get_sfilter(H5Z_filter_t filtn)
else if (filtn==H5Z_FILTER_NBIT)
return "NBIT";
else if (filtn==H5Z_FILTER_SCALEOFFSET)
- return "S+O";
+ return "SOFF";
else {
printf("Input error in filter type\n");
exit(1);
diff --git a/tools/h5repack/testh5repack_main.c b/tools/h5repack/testh5repack_main.c
index 26dcfc7..ab6c09e 100644
--- a/tools/h5repack/testh5repack_main.c
+++ b/tools/h5repack/testh5repack_main.c
@@ -1052,12 +1052,13 @@ if (szip_can_encode) {
SKIPPED();
#endif
+
TESTING(" adding scaleoffset filter");
#ifdef H5_HAVE_FILTER_SCALEOFFSET
if (h5repack_init (&pack_options, 0)<0)
TEST_ERROR;
- if (h5repack_addfilter("dset_none:S+O=31",&pack_options)<0)
+ if (h5repack_addfilter("dset_none:SOFF=31,IN",&pack_options)<0)
TEST_ERROR;
if (h5repack(FNAME13,FNAME13OUT,&pack_options)<0)
TEST_ERROR;