From a5ba6ac3977d660d3122387ad470374c0b2bfbc6 Mon Sep 17 00:00:00 2001 From: Pedro Vicente Nunes Date: Tue, 4 Dec 2007 14:59:54 -0500 Subject: [svn-r14320] Added a function that matches the filters on the dataset's DCPL and in the requested list Used to verify if the filters requested are present in the output dataset tested: windows, linux, solaris --- tools/h5repack/h5repack.c | 1 + tools/h5repack/h5repack.h | 20 +--- tools/h5repack/h5repack_filters.c | 16 +-- tools/h5repack/h5repack_parse.c | 92 +++++++++++---- tools/h5repack/h5repack_verify.c | 232 ++++++++++++++++++++++++++++++++++---- 5 files changed, 291 insertions(+), 70 deletions(-) diff --git a/tools/h5repack/h5repack.c b/tools/h5repack/h5repack.c index 076ab18..374a53a 100644 --- a/tools/h5repack/h5repack.c +++ b/tools/h5repack/h5repack.c @@ -101,6 +101,7 @@ int h5repack_init (pack_opt_t *options, for ( n = 0; n < H5_REPACK_MAX_NFILTERS; n++) { options->filter_g[n].filtn = -1; + options->filter_g[n].cd_nelmts = -1; for ( k = 0; k < CDVALUES; k++) options->filter_g[n].cd_values[k] = -1; } diff --git a/tools/h5repack/h5repack.h b/tools/h5repack/h5repack.h index 31f4038..018fd5f 100644 --- a/tools/h5repack/h5repack.h +++ b/tools/h5repack/h5repack.h @@ -51,11 +51,12 @@ typedef struct { H5Z_FILTER_SCALEOFFSET 6 , scaleoffset compression */ -#define CDVALUES 2 +#define CDVALUES 4 /* SZIP returns 4 values */ typedef struct { H5Z_filter_t filtn; /* filter identification number */ int cd_values[CDVALUES]; /* filter client data values */ + size_t cd_nelmts; /* filter client number of values */ } filter_info_t; /* chunk lengths along each dimension and rank */ @@ -96,23 +97,10 @@ typedef struct { pack_opttbl_t *op_tbl; /*table with all -c and -f options */ int all_layout; /*apply the layout to all objects */ int all_filter; /*apply the filter to all objects */ - -#if defined (NOT_ALL) - - filter_info_t filter_g; /*global filter INFO for the ALL case */ - - -#else - - filter_info_t filter_g[H5_REPACK_MAX_NFILTERS]; /*global filter array for the ALL case */ - int n_filter_g; /*number of global filters */ - -#endif - + filter_info_t filter_g[H5_REPACK_MAX_NFILTERS]; /*global filter array for the ALL case */ + int n_filter_g; /*number of global filters */ chunk_info_t chunk_g; /*global chunk INFO for the ALL case */ H5D_layout_t layout_g; /*global layout information for the ALL case */ - - int verbose; /*verbose mode */ hsize_t threshold; /*minimum size to compress, in bytes */ int use_native; /*use a native type in write */ diff --git a/tools/h5repack/h5repack_filters.c b/tools/h5repack/h5repack_filters.c index c1c001c..c7a58dc 100644 --- a/tools/h5repack/h5repack_filters.c +++ b/tools/h5repack/h5repack_filters.c @@ -265,12 +265,12 @@ int apply_filters(const char* name, /* object name from traverse list */ /*------------------------------------------------------------------------- * the type of filter and additional parameter * type can be one of the filters - * H5Z_FILTER_NONE 0, uncompress if compressed - * H5Z_FILTER_DEFLATE 1 , deflation like gzip - * H5Z_FILTER_SHUFFLE 2 , shuffle the data - * H5Z_FILTER_FLETCHER32 3 , fletcher32 checksum of EDC - * H5Z_FILTER_SZIP 4 , szip compression - * H5Z_FILTER_NBIT 5 , nbit compression + * H5Z_FILTER_NONE 0 , uncompress if compressed + * H5Z_FILTER_DEFLATE 1 , deflation like gzip + * H5Z_FILTER_SHUFFLE 2 , shuffle the data + * H5Z_FILTER_FLETCHER32 3 , fletcher32 checksum of EDC + * H5Z_FILTER_SZIP 4 , szip compression + * H5Z_FILTER_NBIT 5 , nbit compression * H5Z_FILTER_SCALEOFFSET 6 , scaleoffset compression *------------------------------------------------------------------------- */ @@ -278,10 +278,10 @@ int apply_filters(const char* name, /* object name from traverse list */ if (obj.nfilters) { - /*------------------------------------------------------------------------- + /*------------------------------------------------------------------------- * filters require CHUNK layout; if we do not have one define a default *------------------------------------------------------------------------- - */ + */ if (obj.layout==-1) { obj.chunk.rank=rank; diff --git a/tools/h5repack/h5repack_parse.c b/tools/h5repack/h5repack_parse.c index afb2271..2475068 100644 --- a/tools/h5repack/h5repack_parse.c +++ b/tools/h5repack/h5repack_parse.c @@ -74,7 +74,7 @@ obj_list_t* parse_filter(const char *str, *is_glb = 0; /* check for the end of object list and number of objects */ - for ( i=0, n=0; iall_filter=1; *is_glb = 1; } n++; - obj_list=malloc(n*sizeof(obj_list_t)); + obj_list = malloc(n*sizeof(obj_list_t)); if (obj_list==NULL) { error_msg(progname, "could not allocate object list\n"); @@ -103,10 +104,10 @@ obj_list_t* parse_filter(const char *str, *n_objs=n; /* get object list */ - for ( j=0, k=0, n=0; jfiltn=H5Z_FILTER_NONE; + filt->cd_nelmts = 0; + } - /*------------------------------------------------------------------------- - * H5Z_FILTER_DEFLATE - *------------------------------------------------------------------------- + /*------------------------------------------------------------------------- + * H5Z_FILTER_DEFLATE + *------------------------------------------------------------------------- */ else if (strcmp(scomp,"GZIP")==0) { filt->filtn=H5Z_FILTER_DEFLATE; - if (no_param) { /*no more parameters, GZIP must have parameter */ + filt->cd_nelmts = 1; + if (no_param) + { /*no more parameters, GZIP must have parameter */ if (obj_list) free(obj_list); error_msg(progname, "missing compression parameter in <%s>\n",str); exit(1); } } - /*------------------------------------------------------------------------- + /*------------------------------------------------------------------------- * H5Z_FILTER_SZIP *------------------------------------------------------------------------- */ else if (strcmp(scomp,"SZIP")==0) { filt->filtn=H5Z_FILTER_SZIP; - if (no_param) { /*no more parameters, SZIP must have parameter */ + filt->cd_nelmts = 2; + if (no_param) + { /*no more parameters, SZIP must have parameter */ if (obj_list) free(obj_list); error_msg(progname, "missing compression parameter in <%s>\n",str); exit(1); @@ -323,46 +337,54 @@ obj_list_t* parse_filter(const char *str, else if (strcmp(scomp,"SHUF")==0) { filt->filtn=H5Z_FILTER_SHUFFLE; - if (m>0){ /*shuffle does not have parameter */ + filt->cd_nelmts = 0; + if (m>0) + { /*shuffle does not have parameter */ if (obj_list) free(obj_list); error_msg(progname, "extra parameter in SHUF <%s>\n",str); exit(1); } } - /*------------------------------------------------------------------------- + /*------------------------------------------------------------------------- * H5Z_FILTER_FLETCHER32 *------------------------------------------------------------------------- */ else if (strcmp(scomp,"FLET")==0) { filt->filtn=H5Z_FILTER_FLETCHER32; - if (m>0){ /*shuffle does not have parameter */ + filt->cd_nelmts = 0; + if (m>0) + { /*shuffle does not have parameter */ if (obj_list) free(obj_list); error_msg(progname, "extra parameter in FLET <%s>\n",str); exit(1); } } - /*------------------------------------------------------------------------- + /*------------------------------------------------------------------------- * H5Z_FILTER_NBIT *------------------------------------------------------------------------- */ else if (strcmp(scomp,"NBIT")==0) { filt->filtn=H5Z_FILTER_NBIT; - if (m>0){ /*nbit does not have parameter */ + filt->cd_nelmts = 0; + if (m>0) + { /*nbit does not have parameter */ if (obj_list) free(obj_list); error_msg(progname, "extra parameter in NBIT <%s>\n",str); exit(1); } } - /*------------------------------------------------------------------------- + /*------------------------------------------------------------------------- * H5Z_FILTER_SCALEOFFSET *------------------------------------------------------------------------- */ else if (strcmp(scomp,"SOFF")==0) { filt->filtn=H5Z_FILTER_SCALEOFFSET; - if (no_param) { /*no more parameters, SOFF must have parameter */ + filt->cd_nelmts = 2; + if (no_param) + { /*no more parameters, SOFF must have parameter */ if (obj_list) free(obj_list); error_msg(progname, "missing compression parameter in <%s>\n",str); exit(1); @@ -376,43 +398,63 @@ obj_list_t* parse_filter(const char *str, } } /*i*/ - /*------------------------------------------------------------------------- + /*------------------------------------------------------------------------- * check valid parameters *------------------------------------------------------------------------- */ switch (filt->filtn) { + + /*------------------------------------------------------------------------- + * H5Z_FILTER_DEFLATE + *------------------------------------------------------------------------- + */ case H5Z_FILTER_DEFLATE: - if (filt->cd_values[0]<0 || filt->cd_values[0]>9 ){ + if (filt->cd_values[0]<0 || filt->cd_values[0]>9 ) + { if (obj_list) free(obj_list); error_msg(progname, "invalid compression parameter in <%s>\n",str); exit(1); } break; + /*------------------------------------------------------------------------- + * H5Z_FILTER_SZIP + *------------------------------------------------------------------------- + */ case H5Z_FILTER_SZIP: pixels_per_block=filt->cd_values[0]; - if ((pixels_per_block%2)==1) { + if ((pixels_per_block%2)==1) + { if (obj_list) free(obj_list); error_msg(progname, "pixels_per_block is not even in <%s>\n",str); exit(1); } - if (pixels_per_block>H5_SZIP_MAX_PIXELS_PER_BLOCK) { + if (pixels_per_block>H5_SZIP_MAX_PIXELS_PER_BLOCK) + { if (obj_list) free(obj_list); error_msg(progname, "pixels_per_block is too large in <%s>\n",str); exit(1); } - if ( (strcmp(smask,"NN")!=0) && (strcmp(smask,"EC")!=0) ) { + if ( (strcmp(smask,"NN")!=0) && (strcmp(smask,"EC")!=0) ) + { if (obj_list) free(obj_list); error_msg(progname, "szip mask must be 'NN' or 'EC' \n"); exit(1); } break; + + /*------------------------------------------------------------------------- + * H5Z_FILTER_SCALEOFFSET + *------------------------------------------------------------------------- + */ + case H5Z_FILTER_SCALEOFFSET: - if (filt->cd_values[0]<0 ){ + if (filt->cd_values[0]<0 ) + { if (obj_list) free(obj_list); error_msg(progname, "invalid compression parameter in <%s>\n",str); exit(1); diff --git a/tools/h5repack/h5repack_verify.c b/tools/h5repack/h5repack_verify.c index f50e018..7f4f636 100644 --- a/tools/h5repack/h5repack_verify.c +++ b/tools/h5repack/h5repack_verify.c @@ -20,6 +20,8 @@ extern char *progname; static int has_filter(hid_t dcpl_id, H5Z_filter_t filtnin); static int has_layout(hid_t dcpl_id,pack_info_t *obj); +static int has_filters(hid_t dcpl_id, pack_opt_t *options); +static int filtcmp( filter_info_t f1, filter_info_t f2); @@ -46,9 +48,9 @@ int h5repack_verify(const char *fname, hid_t dcpl_id = -1; /* dataset creation property list ID */ hid_t space_id = -1; /* space ID */ unsigned int i; - int j; - trav_table_t *travt = NULL; - int have = 1; + int j; + trav_table_t *travt = NULL; + int ok = 1; /* open the file */ if((fid = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0 ) @@ -77,7 +79,7 @@ int h5repack_verify(const char *fname, for(j = 0; j < obj->nfilters; j++) { if(has_filter(dcpl_id, obj->filter[j].filtn) == 0) - have = 0; + ok = 0; } @@ -87,11 +89,11 @@ int h5repack_verify(const char *fname, *------------------------------------------------------------------------- */ if((obj->layout != -1) && (has_layout(dcpl_id, obj) == 0)) - have = 0; + ok = 0; - /*------------------------------------------------------------------------- - * close - *------------------------------------------------------------------------- + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- */ if(H5Pclose(dcpl_id) < 0) goto error; @@ -142,13 +144,8 @@ int h5repack_verify(const char *fname, */ if(options->all_filter == 1) { - int k; - - for (k = 0; k < options->n_filter_g; k++ ) - { - if (has_filter(dcpl_id, options->filter_g[k].filtn) == 0) - have = 0; - } + if (has_filters(dcpl_id, options) == 0) + ok = 0; } /*------------------------------------------------------------------------- @@ -162,7 +159,7 @@ int h5repack_verify(const char *fname, pack.layout = options->layout_g; pack.chunk = options->chunk_g; if(has_layout(dcpl_id, &pack) == 0) - have = 0; + ok = 0; } @@ -192,7 +189,7 @@ int h5repack_verify(const char *fname, if (H5Fclose(fid) < 0) return -1; - return have; + return ok; error: H5E_BEGIN_TRY { @@ -320,10 +317,6 @@ int has_layout(hid_t dcpl_id, return 1; } - - - - /*------------------------------------------------------------------------- * Function: h5repack_cmpdcpl * @@ -471,3 +464,200 @@ error: return -1; } + + +/*------------------------------------------------------------------------- + * Function: has_filters + * + * Purpose: verify if all requested filters are present in the + * property list DCPL_ID + * + * Return: 1 has, 0 does not, -1 error + * + * Programmer: Pedro Vicente, pvn@hdfgroup.org + * + * Date: December 3, 2007 + * + *------------------------------------------------------------------------- + */ + +static int has_filters(hid_t dcpl_id, pack_opt_t *options) +{ + unsigned nfilters_dcpl; /* number of filters in DCPL*/ + unsigned nfilters_opt; /* number of filters in OPTIONS*/ + unsigned filt_flags; /* filter flags */ + H5Z_filter_t filtn; /* filter identification number */ + unsigned cd_values[20]; /* filter client data values */ + size_t cd_nelmts; /* filter client number of values */ + char f_name[256]; /* filter name */ + int have = 0; /* flag, filter is present */ + unsigned i, j; /* index */ + filter_info_t filter_dcpl[H5_REPACK_MAX_NFILTERS]; /* filter array in the DCPL*/ + filter_info_t filter_opt[H5_REPACK_MAX_NFILTERS]; /* filter array in options */ + + /* get information about filters */ + if((nfilters_dcpl = H5Pget_nfilters(dcpl_id)) < 0) + return -1; + + /* if we do not have filters and the requested filter is NONE, return 1 */ + if(!nfilters_dcpl && + options->n_filter_g==1 && + options->filter_g[0].filtn == H5Z_FILTER_NONE ) + return 1; + + /*------------------------------------------------------------------------- + * build a list with DCPL filters + *------------------------------------------------------------------------- + */ + + for( i = 0; i < nfilters_dcpl; i++) + { + cd_nelmts = NELMTS(cd_values); + filtn = H5Pget_filter2(dcpl_id, i, &filt_flags, &cd_nelmts, + cd_values, sizeof(f_name), f_name, NULL); + + filter_dcpl[i].filtn = filtn; + filter_dcpl[i].cd_nelmts = cd_nelmts; + for( j = 0; j < cd_nelmts; j++) + { + filter_dcpl[i].cd_values[j] = cd_values[j]; + + } + + } + + /*------------------------------------------------------------------------- + * build a list with options filters + *------------------------------------------------------------------------- + */ + + nfilters_opt = options->n_filter_g; + + for( i = 0; i < nfilters_opt; i++) + { + + filter_opt[i].filtn = options->filter_g[i].filtn; + filter_opt[i].cd_nelmts = options->filter_g[i].cd_nelmts; + for( j = 0; j < options->filter_g[i].cd_nelmts; j++) + { + filter_opt[i].cd_values[j] = options->filter_g[i].cd_values[j]; + + } + + } + + /*------------------------------------------------------------------------- + * match the 2 lists + *------------------------------------------------------------------------- + */ + + + if (nfilters_dcpl != nfilters_opt) + return 0; + + for( i = 0; i < nfilters_opt; i++) + { + + /* criteria is filter compare, returns same as strcmp */ + if ( filtcmp( filter_dcpl[i], filter_opt[i] ) != 0 ) + return 0; + + + } + + + + return 1; +} + + + +/*------------------------------------------------------------------------- + * Function: filtcmp + * + * Purpose: compare 2 filters + * + * Return: same as strcmp: + * < 0 F1 "less" than F2 + * 0 F1 identical to F2 + * > 0 F1 "greater" than F2 + * + * Programmer: Pedro Vicente, pvn@hdfgroup.org + * + * Date: December 3, 2007 + * + *------------------------------------------------------------------------- + */ +static int filtcmp( filter_info_t f1 /*DCPL*/, filter_info_t f2 /*OPT*/) +{ + unsigned i; + + /*------------------------------------------------------------------------- + * compare cd_nelmts and cd_values + *------------------------------------------------------------------------- + */ + + if ( f1.filtn == 4 ) /* SZIP special case */ + { + + if ( f1.cd_nelmts != 4 && f2.cd_nelmts != 2 ) + return -1; + + if ( f2.cd_values[0] != f1.cd_values[2] && + f2.cd_values[1] != f1.cd_values[1] ) + return -1; + + } + + else if ( f1.filtn == 2 ) /* SHUFFLE returns 1 cd_nelmts */ + { + + if ( f1.cd_nelmts != 1 && f2.cd_nelmts != 0 ) + return -1; + + } + + else + + { + + if ( f1.cd_nelmts != f2.cd_nelmts ) + return -1; + + /* consider different filter values as "less" */ + for( i = 0; i < f1.cd_nelmts; i++) + { + if (f1.cd_values[i] != f2.cd_values[i]) + { + return -1; + } + + } + + } + + /*------------------------------------------------------------------------- + * compare the filter type + *------------------------------------------------------------------------- + */ + + + if (f1.filtn == f2.filtn) + { + return 0; + } + else if (f1.filtn < f2.filtn) + { + return -1; + } + else if (f1.filtn > f2.filtn) + { + return 1; + } + + assert(0); + return -1; + +} + + -- cgit v0.12