diff options
| -rw-r--r-- | tools/h5repack/h5repack.h | 20 | ||||
| -rw-r--r-- | tools/h5repack/h5repack_filters.c | 57 | ||||
| -rw-r--r-- | tools/h5repack/h5repack_main.c | 22 | ||||
| -rw-r--r-- | tools/h5repack/h5repack_opttable.c | 2 | ||||
| -rw-r--r-- | tools/h5repack/h5repack_parse.c | 89 | ||||
| -rw-r--r-- | tools/h5repack/testh5repack_main.c | 3 |
6 files changed, 132 insertions, 61 deletions
diff --git a/tools/h5repack/h5repack.h b/tools/h5repack/h5repack.h index 0ced373..dab4ece 100644 --- a/tools/h5repack/h5repack.h +++ b/tools/h5repack/h5repack.h @@ -30,7 +30,6 @@ #define MAX_NC_NAME 256 /* max length of a name */ #define MAX_VAR_DIMS 32 /* max per variable dimensions */ - /*------------------------------------------------------------------------- * data structures for command line options *------------------------------------------------------------------------- @@ -44,11 +43,13 @@ typedef struct { /* the type of filter and additional parameter type can be one of the filters - H5Z_FILTER_NONE 0, uncompress if compressed - H5Z_FILTER_DEFLATE 1 , deflation like gzip - H5Z_FILTER_SHUFFLE 2 , shuffle the data - H5Z_FILTER_FLETCHER32 3 , letcher32 checksum of EDC - H5Z_FILTER_SZIP 4 , szip compression + H5Z_FILTER_NONE 0, uncompress if compressed + H5Z_FILTER_DEFLATE 1 , deflation like gzip + H5Z_FILTER_SHUFFLE 2 , shuffle the data + H5Z_FILTER_FLETCHER32 3 , letcher32 checksum of EDC + H5Z_FILTER_SZIP 4 , szip compression + H5Z_FILTER_NBIT 5 , nbit compression + H5Z_FILTER_SCALEOFFSET 6 , scaleoffset compression */ #define CDVALUES 2 @@ -56,11 +57,6 @@ typedef struct { typedef struct { H5Z_filter_t filtn; /* filter identification number */ int cd_values[CDVALUES]; /* filter client data values */ - /* extra input for szip, selects the coding method - entropy coding method: EC=0 - nearest neighbor coding method: NN=1 - */ - int szip_coding; } filter_info_t; /* chunk lengths along each dimension and rank */ @@ -71,7 +67,7 @@ typedef struct { /* we currently define a maximum value for the filters array, that corresponds to the current library filters */ -#define H5_REPACK_MAX_NFILTERS 4 +#define H5_REPACK_MAX_NFILTERS 6 /* information for one object, contains PATH, CHUNK info and FILTER info */ typedef struct { diff --git a/tools/h5repack/h5repack_filters.c b/tools/h5repack/h5repack_filters.c index db3277b..1f4c2d7 100644 --- a/tools/h5repack/h5repack_filters.c +++ b/tools/h5repack/h5repack_filters.c @@ -147,8 +147,6 @@ int aux_assign_obj(const char* name, /* object name from traverse lis tmp.nfilters=1; tmp.filter[0]=options->filter_g; } - - if (options->all_layout) { /* assign the global layout info to the OBJ info */ @@ -163,10 +161,7 @@ int aux_assign_obj(const char* name, /* object name from traverse lis default: break; }/*switch*/ - } - - } *obj = tmp; @@ -199,7 +194,6 @@ int apply_filters(const char* name, /* object name from traverse list */ pack_opt_t *options) /* repack options */ { int nfilters; /* number of filters in DCPL */ - unsigned aggression; /* the deflate level */ hsize_t nelmts; /* number of elements in dataset */ size_t size; /* size of datatype in bytes */ hsize_t chsize[64]; /* chunk size in elements */ @@ -276,10 +270,7 @@ int apply_filters(const char* name, /* object name from traverse list */ } } - - - - /*------------------------------------------------------------------------- +/*------------------------------------------------------------------------- * the type of filter and additional parameter * type can be one of the filters * H5Z_FILTER_NONE 0, uncompress if compressed @@ -318,12 +309,16 @@ int apply_filters(const char* name, /* object name from traverse list */ *------------------------------------------------------------------------- */ case H5Z_FILTER_DEFLATE: - aggression=obj.filter[i].cd_values[0]; - /* set up for deflated data */ - if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) - return -1; - if(H5Pset_deflate(dcpl_id,aggression)<0) - return -1; + { + unsigned aggression; /* the deflate level */ + + aggression = obj.filter[i].cd_values[0]; + /* set up for deflated data */ + if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) + return -1; + if(H5Pset_deflate(dcpl_id,aggression)<0) + return -1; + } break; /*------------------------------------------------------------------------- @@ -335,12 +330,9 @@ int apply_filters(const char* name, /* object name from traverse list */ unsigned options_mask; unsigned pixels_per_block; - pixels_per_block=obj.filter[i].cd_values[0]; - if (obj.filter[i].szip_coding==0) - options_mask=H5_SZIP_NN_OPTION_MASK; - else - options_mask=H5_SZIP_EC_OPTION_MASK; - + options_mask = obj.filter[i].cd_values[0]; + pixels_per_block = obj.filter[i].cd_values[1]; + /* set up for szip data */ if(H5Pset_chunk(dcpl_id,obj.chunk.rank,obj.chunk.chunk_lengths)<0) return -1; @@ -385,15 +377,20 @@ int apply_filters(const char* name, /* object name from traverse list */ * H5Z_FILTER_SCALEOFFSET , scale+offset compression *------------------------------------------------------------------------- */ + case H5Z_FILTER_SCALEOFFSET: - aggression=obj.filter[i].cd_values[0]; - if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) - return -1; - /* The following line needs to be changed in the future, - H5_SO_INT needs to be passed as a parameter, it can - be either H5_SO_FLOAT_DSCALE or H5_SO_INT. KY, 2005/8/31 */ - if (H5Pset_scaleoffset(dcpl_id,H5Z_SO_INT,aggression)<0) - return -1; + { + H5Z_SO_scale_type_t scale_type; + int scale_factor; + + scale_type = obj.filter[i].cd_values[0]; + scale_factor = obj.filter[i].cd_values[1]; + + if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) + return -1; + if (H5Pset_scaleoffset(dcpl_id,scale_type,scale_factor)<0) + return -1; + } break; } /* switch */ }/*i*/ diff --git a/tools/h5repack/h5repack_main.c b/tools/h5repack/h5repack_main.c index 665f8e1..96eb787 100644 --- a/tools/h5repack/h5repack_main.c +++ b/tools/h5repack/h5repack_main.c @@ -136,9 +136,12 @@ void usage(void) printf("-i input Input HDF5 File\n"); printf("-o output Output HDF5 File\n"); printf("[-h] Print usage message\n"); - printf("[-v] Verbose mode. Print output (list of objects, filters, warnings)\n"); + printf("[-v] Verbose mode. Print more output (list of objects,\n"); + printf(" filters, warnings)\n"); printf("[-f 'filter'] Filter type: 'filter' is a string with the format\n"); + printf("\n"); printf(" <list of objects> : <name of filter> = <filter parameters>\n"); + printf("\n"); printf(" <list of objects> is a comma separated list of object names\n"); printf(" meaning apply compression only to those objects.\n"); printf(" if no object names are specified, the filter is applied to all objects\n"); @@ -148,15 +151,24 @@ void usage(void) printf(" SHUF, to apply the HDF5 shuffle filter\n"); printf(" FLET, to apply the HDF5 checksum filter\n"); printf(" NBIT, to apply the HDF5 NBIT filter (NBIT compression)\n"); + printf(" SOFF, to apply the HDF5 Scale/Offset filter\n"); printf(" NONE, to remove the filter\n"); printf(" <filter parameters> is optional compression info\n"); printf(" SHUF (no parameter)\n"); printf(" FLET (no parameter)\n"); printf(" NBIT (no parameter)\n"); printf(" GZIP=<deflation level> from 1-9\n"); - printf(" SZIP=<pixels per block,coding> (pixels per block is a even number in 2-32 and coding method is 'EC' or 'NN')\n"); + printf(" SZIP=<pixels per block,coding>\n"); + printf(" (pixels per block is a even number in 2-32 and coding method\n"); + printf(" is 'EC' or 'NN')\n"); + printf(" SOFF=<scale_factor,scale_type>\n"); + printf(" (scale_factor is an integer and scale_type is either 'IN'\n"); + printf(" for integer type, or 'DS', for floating point type\n"); + printf(" using the D-scaling method)\n"); printf("[-l 'layout'] Layout type. 'layout' is a string with the format\n"); + printf("\n"); printf(" <list of objects> : <layout type>\n"); + printf("\n"); printf(" <list of objects> is a comma separated list of object names,\n"); printf(" meaning that layout information is supplied for those objects.\n"); printf(" if no object names are specified, the layout is applied to all objects\n"); @@ -169,8 +181,9 @@ void usage(void) printf(" <dim_1 x dim_2 x ... dim_n>\n"); printf("\n"); printf("-e file File with the -f and -l options (only filter and layout flags)\n"); - printf("-m number Do not apply the filter to objects which size in bytes is smaller than number.\n"); - printf(" If no size is specified a minimum of 1024 bytes is assumed.\n"); + printf("-m size Do not apply the filter to objects which size in bytes\n"); + printf(" is smaller than number. If no size is specified a minimum of\n"); + printf(" 1024 bytes is assumed.\n"); printf("\n"); printf("Examples of use:\n"); printf("\n"); @@ -188,4 +201,3 @@ void usage(void) printf("\n"); } - diff --git a/tools/h5repack/h5repack_opttable.c b/tools/h5repack/h5repack_opttable.c index 7bb6d10..d4c2532 100644 --- a/tools/h5repack/h5repack_opttable.c +++ b/tools/h5repack/h5repack_opttable.c @@ -17,7 +17,6 @@ #include "h5repack.h" - /*------------------------------------------------------------------------- * Function: init_packobject * @@ -36,7 +35,6 @@ void init_packobject(pack_info_t *obj) for ( j=0; j<H5_REPACK_MAX_NFILTERS; j++) { obj->filter[j].filtn = -1; - obj->filter[j].szip_coding = -1; for ( k=0; k<CDVALUES; k++) obj->filter[j].cd_values[k] = -1; } diff --git a/tools/h5repack/h5repack_parse.c b/tools/h5repack/h5repack_parse.c index 7e82fde..1c67216 100644 --- a/tools/h5repack/h5repack_parse.c +++ b/tools/h5repack/h5repack_parse.c @@ -35,7 +35,7 @@ * SHUF, to apply the HDF5 shuffle filter * FLET, to apply the HDF5 checksum filter * NBIT, to apply the HDF5 NBIT filter (NBIT compression) - * S+O, to apply the HDF5 scale+offset filter (compression) + * SOFF, to apply the HDF5 scale+offset filter (compression) * NONE, to remove the filter * * Examples: @@ -135,11 +135,13 @@ obj_list_t* parse_filter(const char *str, if ( c=='=') { /*one more parameter */ scomp[k]='\0'; /*cut space */ - /*SZIP is a special case , it can be - SZIP=8,EC - SZIP=8,NN - */ - +/*------------------------------------------------------------------------- + * H5Z_FILTER_SZIP + * szip has the format SZIP=<pixels per block,coding> + * pixels per block is a even number in 2-32 and coding method is 'EC' or 'NN' + * example SZIP=8,NN + *------------------------------------------------------------------------- + */ if (strcmp(scomp,"SZIP")==0) { l=-1; /* mask index check */ @@ -169,9 +171,9 @@ obj_list_t* parse_filter(const char *str, i=len-1; /* end */ (*n_objs)--; /* we counted an extra ',' */ if (strcmp(smask,"NN")==0) - filt->szip_coding=0; + filt->cd_values[j++]=H5_SZIP_NN_OPTION_MASK; else if (strcmp(smask,"EC")==0) - filt->szip_coding=1; + filt->cd_values[j++]=H5_SZIP_EC_OPTION_MASK; else { printf("Input Error: szip mask must be 'NN' or 'EC' \n"); @@ -184,6 +186,71 @@ obj_list_t* parse_filter(const char *str, } /* u */ } /*if */ +/*------------------------------------------------------------------------- + * H5Z_FILTER_SCALEOFFSET + * scaleoffset has the format SOFF=<scale_factor,scale_type> + * scale_type can be + * integer datatype, H5Z_SO_INT (IN) + * float datatype using D-scaling method, H5Z_SO_FLOAT_DSCALE (DS) + * float datatype using E-scaling method, H5Z_SO_FLOAT_ESCALE (ES) , not yet implemented + * for integer datatypes, scale_factor denotes Minimum Bits + * for float datatypes, scale_factor denotes decimal scale factor + * examples + * SOFF=31,IN + * SOFF=3,DF + *------------------------------------------------------------------------- + */ + + else if (strcmp(scomp,"SOFF")==0) + { + l=-1; /* mask index check */ + for ( m=0,u=i+1; u<len; u++,m++) + { + if (str[u]==',') + { + stype[m]='\0'; /* end digit */ + l=0; /* start 'IN' , 'DS', or 'ES' search */ + u++; /* skip ',' */ + } + c = str[u]; + if (!isdigit(c) && l==-1){ + if (obj_list) free(obj_list); + printf("Input Error: Compression parameter not digit in <%s>\n",str); + exit(1); + } + if (l==-1) + stype[m]=c; + else + { + smask[l]=c; + l++; + if (l==2) + { + smask[l]='\0'; + i=len-1; /* end */ + (*n_objs)--; /* we counted an extra ',' */ + if (strcmp(smask,"IN")==0) + filt->cd_values[j++]=H5Z_SO_INT; + else if (strcmp(smask,"DS")==H5Z_SO_FLOAT_DSCALE) + filt->cd_values[j++]=H5Z_SO_FLOAT_DSCALE; + else + { + printf("Input Error: scale type must be 'IN' or 'DS' \n"); + exit(1); + } + + } + } + + } /* u */ + } /*if */ + + +/*------------------------------------------------------------------------- + * all other filters + *------------------------------------------------------------------------- + */ + else { /* here we could have 1 or 2 digits */ @@ -289,10 +356,10 @@ obj_list_t* parse_filter(const char *str, * H5Z_FILTER_SCALEOFFSET *------------------------------------------------------------------------- */ - else if (strcmp(scomp,"S+O")==0) + else if (strcmp(scomp,"SOFF")==0) { filt->filtn=H5Z_FILTER_SCALEOFFSET; - if (no_param) { /*no more parameters, S+O must have parameter */ + if (no_param) { /*no more parameters, SOFF must have parameter */ if (obj_list) free(obj_list); printf("Input Error: Missing compression parameter in <%s>\n",str); exit(1); @@ -379,7 +446,7 @@ const char* get_sfilter(H5Z_filter_t filtn) else if (filtn==H5Z_FILTER_NBIT) return "NBIT"; else if (filtn==H5Z_FILTER_SCALEOFFSET) - return "S+O"; + return "SOFF"; else { printf("Input error in filter type\n"); exit(1); diff --git a/tools/h5repack/testh5repack_main.c b/tools/h5repack/testh5repack_main.c index 26dcfc7..ab6c09e 100644 --- a/tools/h5repack/testh5repack_main.c +++ b/tools/h5repack/testh5repack_main.c @@ -1052,12 +1052,13 @@ if (szip_can_encode) { SKIPPED(); #endif + TESTING(" adding scaleoffset filter"); #ifdef H5_HAVE_FILTER_SCALEOFFSET if (h5repack_init (&pack_options, 0)<0) TEST_ERROR; - if (h5repack_addfilter("dset_none:S+O=31",&pack_options)<0) + if (h5repack_addfilter("dset_none:SOFF=31,IN",&pack_options)<0) TEST_ERROR; if (h5repack(FNAME13,FNAME13OUT,&pack_options)<0) TEST_ERROR; |