From 54a5561c78a2a229075b84219c883a38e28e5690 Mon Sep 17 00:00:00 2001 From: Pedro Vicente Nunes Date: Tue, 12 Feb 2008 15:36:48 -0500 Subject: [svn-r14557] 1st batch of commits regarding making 1.6 h5repack like 1.8 1) have several global filters 2) do a check on the verify code about private filter values Note: some shuffle and szip private symbols were made public tested: windows, linux, solaris --- src/H5Zpublic.h | 14 + src/H5Zshuffle.c | 2 - src/H5Zszip.c | 7 - tools/h5repack/h5repack.c | 569 ++++++++++++++++++++++++++++++++++++- tools/h5repack/h5repack.h | 70 ++++- tools/h5repack/h5repack_filters.c | 11 +- tools/h5repack/h5repack_opttable.c | 2 +- tools/h5repack/h5repack_parse.c | 323 +++++++++++++++++++++ tools/h5repack/h5repack_verify.c | 561 +++++++++++++++++++++++++++++++++++- tools/h5repack/h5repacktst.c | 30 ++ 10 files changed, 1572 insertions(+), 17 deletions(-) diff --git a/src/H5Zpublic.h b/src/H5Zpublic.h index cb03ba7..accd763 100644 --- a/src/H5Zpublic.h +++ b/src/H5Zpublic.h @@ -62,6 +62,20 @@ typedef int H5Z_filter_t; #define H5_SZIP_NN_OPTION_MASK 32 #define H5_SZIP_MAX_PIXELS_PER_BLOCK 32 +/* Macros for the shuffle filter */ +#define H5Z_SHUFFLE_USER_NPARMS 0 /* Number of parameters that users can set */ +#define H5Z_SHUFFLE_TOTAL_NPARMS 1 /* Total number of parameters for filter */ + +/* Macros for the szip filter */ +#define H5Z_SZIP_USER_NPARMS 2 /* Number of parameters that users can set */ +#define H5Z_SZIP_TOTAL_NPARMS 4 /* Total number of parameters for filter */ +#define H5Z_SZIP_PARM_MASK 0 /* "User" parameter for option mask */ +#define H5Z_SZIP_PARM_PPB 1 /* "User" parameter for pixels-per-block */ +#define H5Z_SZIP_PARM_BPP 2 /* "Local" parameter for bits-per-pixel */ +#define H5Z_SZIP_PARM_PPS 3 /* "Local" parameter for pixels-per-scanline */ + + + /* Values to decide if EDC is enabled for reading data */ typedef enum H5Z_EDC_t { H5Z_ERROR_EDC = -1, /* error value */ diff --git a/src/H5Zshuffle.c b/src/H5Zshuffle.c index c714cc9..83e5f40 100644 --- a/src/H5Zshuffle.c +++ b/src/H5Zshuffle.c @@ -40,8 +40,6 @@ const H5Z_class_t H5Z_SHUFFLE[1] = {{ }}; /* Local macros */ -#define H5Z_SHUFFLE_USER_NPARMS 0 /* Number of parameters that users can set */ -#define H5Z_SHUFFLE_TOTAL_NPARMS 1 /* Total number of parameters for filter */ #define H5Z_SHUFFLE_PARM_SIZE 0 /* "Local" parameter for shuffling size */ diff --git a/src/H5Zszip.c b/src/H5Zszip.c index 62cc461..07f784f 100644 --- a/src/H5Zszip.c +++ b/src/H5Zszip.c @@ -46,13 +46,6 @@ const H5Z_class_t H5Z_SZIP[1] = {{ H5Z_filter_szip, /* The actual filter function */ }}; -/* Local macros */ -#define H5Z_SZIP_USER_NPARMS 2 /* Number of parameters that users can set */ -#define H5Z_SZIP_TOTAL_NPARMS 4 /* Total number of parameters for filter */ -#define H5Z_SZIP_PARM_MASK 0 /* "User" parameter for option mask */ -#define H5Z_SZIP_PARM_PPB 1 /* "User" parameter for pixels-per-block */ -#define H5Z_SZIP_PARM_BPP 2 /* "Local" parameter for bits-per-pixel */ -#define H5Z_SZIP_PARM_PPS 3 /* "Local" parameter for pixels-per-scanline */ /*------------------------------------------------------------------------- diff --git a/tools/h5repack/h5repack.c b/tools/h5repack/h5repack.c index c10c608..1318129 100644 --- a/tools/h5repack/h5repack.c +++ b/tools/h5repack/h5repack.c @@ -19,6 +19,569 @@ #include "h5diff.h" #include "h5tools.h" #include "h5tools_utils.h" +#include "H5private.h" + + +extern char *progname; + +/*------------------------------------------------------------------------- + * File: h5repack.c + * Purpose: Public API functions + *------------------------------------------------------------------------- + */ + +static int check_options(pack_opt_t *options); +static int check_objects(const char* fname, pack_opt_t *options); +static const char* get_sfilter (H5Z_filter_t filtn); +static int have_request(pack_opt_t *options); + + + +/*------------------------------------------------------------------------- + * Function: h5repack + * + * Purpose: locate all high-level HDF5 objects in the file + * and compress/chunk them using options + * + * Algorythm: 2 traversals are made to the file; the 1st builds a list of + * the objects, the 2nd makes a copy of them, using the options; + * the reason for the 1st traversal is to check for invalid + * object name requests + * + * Return: 0, ok, -1, fail + * + * Programmer: pvn@ncsa.uiuc.edu + * + * Date: September, 22, 2003 + * + *------------------------------------------------------------------------- + */ +int h5repack(const char* infile, + const char* outfile, + pack_opt_t *options) +{ + /* check input */ + if (check_options(options)<0) + return -1; + + /* check for objects in input that are in the file */ + if (check_objects(infile,options) < 0) + return -1; + + /* copy the objects */ + if (copy_objects(infile,outfile,options) < 0) + return -1; + + + return 0; +} + + + +/*------------------------------------------------------------------------- + * Function: h5repack_init + * + * Purpose: initialize options + * + * Return: 0, ok, -1, fail + * + *------------------------------------------------------------------------- + */ + +int h5repack_init (pack_opt_t *options, + int verbose) +{ + int k, n; + memset(options,0,sizeof(pack_opt_t)); + options->threshold = 1024; + options->verbose = verbose; + + for ( n = 0; n < H5_REPACK_MAX_NFILTERS; n++) + { + options->filter_g[n].filtn = -1; + options->filter_g[n].cd_nelmts = -1; + for ( k = 0; k < CD_VALUES; k++) + options->filter_g[n].cd_values[k] = -1; + } + + return (options_table_init(&(options->op_tbl))); +} + +/*------------------------------------------------------------------------- + * Function: h5repack_end + * + * Purpose: free options table + * + *------------------------------------------------------------------------- + */ + +int h5repack_end (pack_opt_t *options) +{ + return options_table_free(options->op_tbl); +} + +/*------------------------------------------------------------------------- + * Function: h5repack_addfilter + * + * Purpose: add a compression -f option to table + * Example: -f dset:GZIP=6 + * + * Return: 0, ok, -1, fail + * + *------------------------------------------------------------------------- + */ + +int h5repack_addfilter(const char* str, + pack_opt_t *options) +{ + obj_list_t *obj_list=NULL; /* one object list for the -f and -l option entry */ + filter_info_t filter; /* filter info for the current -f option entry */ + int n_objs; /* number of objects in the current -f or -l option entry */ + int is_glb; /* is the filter global */ + + + + /* parse the -f option */ + obj_list=parse_filter(str,&n_objs,&filter,options,&is_glb); + if (obj_list==NULL) + { + return -1; + } + + /* if it applies to all objects */ + if (is_glb) + { + + int n; + + n = options->n_filter_g++; /* increase # of global filters */ + + if (options->n_filter_g > H5_REPACK_MAX_NFILTERS) + { + error_msg(progname, "maximum number of filters exceeded for <%s>\n",str); + return -1; + + } + + options->filter_g[n] = filter; + } + + else + options_add_filter(obj_list,n_objs,filter,options->op_tbl); + + free(obj_list); + return 0; +} + + +/*------------------------------------------------------------------------- + * Function: h5repack_addlayout + * + * Purpose: add a layout option + * + * Return: 0, ok, -1, fail + * + *------------------------------------------------------------------------- + */ + + +int h5repack_addlayout(const char* str, + pack_opt_t *options) +{ + + obj_list_t *obj_list=NULL; /*one object list for the -t and -c option entry */ + int n_objs; /*number of objects in the current -t or -c option entry */ + pack_info_t pack; /*info about layout to extract from parse */ + int j; + + init_packobject(&pack); + + if (options->all_layout==1){ + error_msg(progname, "invalid layout input: 'all' option \ + is present with other objects <%s>\n",str); + return -1; + } + + /* parse the layout option */ + obj_list=parse_layout(str,&n_objs,&pack,options); + if (obj_list==NULL) + return -1; + + /* set global layout option */ + if (options->all_layout==1 ) + { + options->layout_g=pack.layout; + if (pack.layout==H5D_CHUNKED) + { + /* -2 means the NONE option, remove chunking + and set the global layout to contiguous */ + if (pack.chunk.rank==-2) + { + options->layout_g = H5D_CONTIGUOUS; + } + /* otherwise set the global chunking type */ + else + { + options->chunk_g.rank=pack.chunk.rank; + for (j = 0; j < pack.chunk.rank; j++) + options->chunk_g.chunk_lengths[j] = pack.chunk.chunk_lengths[j]; + } + } + } + + if (options->all_layout==0) + options_add_layout(obj_list, + n_objs, + &pack, + options->op_tbl); + + free(obj_list); + return 0; +} + + +/*------------------------------------------------------------------------- + * Function: check_options + * + * Purpose: print options, checks for invalid options + * + * Return: void, return -1 on error + * + * Programmer: pvn@ncsa.uiuc.edu + * + * Date: September, 22, 2003 + * + * Modification: + * Peter Cao, July 9, 2007 + * Add "-L, --latest" and other options to pack a file with the latest file format + * + *------------------------------------------------------------------------- + */ +static int check_options(pack_opt_t *options) +{ + unsigned int i; + int k, j, has_cp=0, has_ck=0; + char slayout[30]; + + /*------------------------------------------------------------------------- + * objects to layout + *------------------------------------------------------------------------- + */ + if (options->verbose && have_request(options) /* only print if requested */) + { + printf("Objects to modify layout are...\n"); + if (options->all_layout==1) { + switch (options->layout_g) + { + case H5D_COMPACT: + strcpy(slayout,"compact"); + break; + case H5D_CONTIGUOUS: + strcpy(slayout,"contiguous"); + break; + case H5D_CHUNKED: + strcpy(slayout,"chunked"); + break; + default: + strcpy(slayout,"unknown"); + break; + } + printf(" Apply %s layout to all\n", slayout); + if (H5D_CHUNKED==options->layout_g) { + printf("with dimension ["); + for ( j = 0; j < options->chunk_g.rank; j++) + printf("%d ",(int)options->chunk_g.chunk_lengths[j]); + printf("]\n"); + } + } + }/* verbose */ + + for ( i = 0; i < options->op_tbl->nelems; i++) + { + char* name=options->op_tbl->objs[i].path; + + if (options->op_tbl->objs[i].chunk.rank>0) + { + if (options->verbose){ + printf(" <%s> with chunk size ",name); + for ( k = 0; k < options->op_tbl->objs[i].chunk.rank; k++) + printf("%d ",(int)options->op_tbl->objs[i].chunk.chunk_lengths[k]); + printf("\n"); + } + has_ck=1; + } + else if (options->op_tbl->objs[i].chunk.rank==-2) + { + if (options->verbose) + printf(" <%s> %s\n",name,"NONE (contigous)"); + has_ck=1; + } + } + + if (options->all_layout==1 && has_ck){ + error_msg(progname, "invalid chunking input: 'all' option\ + is present with other objects\n"); + return -1; + } + + /*------------------------------------------------------------------------- + * objects to filter + *------------------------------------------------------------------------- + */ + + if (options->verbose && have_request(options) /* only print if requested */) + { + printf("Objects to apply filter are...\n"); + if (options->all_filter==1) + { + + for (k = 0; k < options->n_filter_g; k++ ) + { + H5Z_filter_t filtn=options->filter_g[k].filtn; + switch (filtn) + { + case H5Z_FILTER_NONE: + printf(" Uncompress all\n"); + break; + case H5Z_FILTER_SHUFFLE: + case H5Z_FILTER_FLETCHER32: + printf(" All with %s\n",get_sfilter(filtn)); + break; + case H5Z_FILTER_SZIP: + case H5Z_FILTER_DEFLATE: + printf(" All with %s, parameter %d\n", + get_sfilter(filtn), + options->filter_g[k].cd_values[0]); + break; + } /* k */ + }; + } + } /* verbose */ + + for ( i = 0; i < options->op_tbl->nelems; i++) + { + pack_info_t pack = options->op_tbl->objs[i]; + char* name = pack.path; + + for ( j=0; jverbose) + { + printf(" <%s> with %s filter\n", + name, + get_sfilter(pack.filter[j].filtn)); + } + + has_cp=1; + + } /* j */ + } /* i */ + + if (options->all_filter==1 && has_cp){ + error_msg(progname, "invalid compression input: 'all' option\ + is present with other objects\n"); + return -1; + } + + + + return 0; +} + + +/*------------------------------------------------------------------------- + * Function: check_objects + * + * Purpose: locate all HDF5 objects in the file and compare with user + * supplied list + * + * Return: 0, ok, -1 no + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: September, 23, 2003 + * + *------------------------------------------------------------------------- + */ +static int check_objects(const char* fname, + pack_opt_t *options) +{ + hid_t fid; + unsigned int i; + trav_table_t *travt = NULL; + + /* nothing to do */ + if(options->op_tbl->nelems == 0) + return 0; + + /*------------------------------------------------------------------------- + * open the file + *------------------------------------------------------------------------- + */ + if((fid = h5tools_fopen(fname, NULL, NULL, 0)) < 0){ + printf("<%s>: %s\n", fname, H5FOPENERROR ); + return -1; + } + + /*------------------------------------------------------------------------- + * get the list of objects in the file + *------------------------------------------------------------------------- + */ + + /* init table */ + trav_table_init(&travt); + + /* get the list of objects in the file */ + if(h5trav_gettable(fid, travt) < 0) + goto out; + + /*------------------------------------------------------------------------- + * compare with user supplied list + *------------------------------------------------------------------------- + */ + + if(options->verbose) + printf("Opening file <%s>. Searching for objects to modify...\n", fname); + + for(i = 0; i < options->op_tbl->nelems; i++) { + char* name=options->op_tbl->objs[i].path; + if(options->verbose) + printf(" <%s>",name); + + /* the input object names are present in the file and are valid */ + if(h5trav_getindext(name, travt) < 0) { + error_msg(progname, "%s Could not find <%s> in file <%s>. Exiting...\n", + (options->verbose?"\n":""),name,fname); + goto out; + } + if(options->verbose) + printf("...Found\n"); + + /* check for extra filter conditions */ + switch(options->op_tbl->objs[i].filter->filtn) { + /* chunk size must be smaller than pixels per block */ + case H5Z_FILTER_SZIP: + { + int j; + int csize = 1; + int ppb = options->op_tbl->objs[i].filter->cd_values[0]; + hsize_t dims[H5S_MAX_RANK]; + int rank; + hid_t did; + hid_t sid; + + if(options->op_tbl->objs[i].chunk.rank > 0) { + rank = options->op_tbl->objs[i].chunk.rank; + for(j = 0; j < rank; j++) + csize *= (int)options->op_tbl->objs[i].chunk.chunk_lengths[j]; + } + else { + if((did = H5Dopen(fid, name)) < 0) + goto out; + if((sid = H5Dget_space(did)) < 0) + goto out; + if((rank = H5Sget_simple_extent_ndims(sid)) < 0) + goto out; + HDmemset(dims, 0, sizeof dims); + if(H5Sget_simple_extent_dims(sid, dims, NULL) < 0) + goto out; + for(j = 0; j < rank; j++) + csize *= (int)dims[j]; + if(H5Sclose(sid) < 0) + goto out; + if(H5Dclose(did) < 0) + goto out; + } + + if (csize < ppb ) { + printf(" \n"); + goto out; + } + } + break; + } + } /* i */ + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + H5Fclose(fid); + trav_table_free(travt); + return 0; + +out: + H5Fclose(fid); + trav_table_free(travt); + return -1; +} + + + + + +/*------------------------------------------------------------------------- + * Function: have_request + * + * Purpose: check if a filter or layout was requested + * + * Return: 1 yes, 0 no + * + * Date: May, 24, 2007 + * + *------------------------------------------------------------------------- + */ +static int have_request(pack_opt_t *options) +{ + + if (options->all_filter || options->all_layout || options->op_tbl->nelems) + return 1; + + return 0; + +} + + +/*------------------------------------------------------------------------- + * Function: get_sfilter + * + * Purpose: return the filter as a string name + * + * Return: name of filter, exit on error + * + *------------------------------------------------------------------------- + */ + +static const char* get_sfilter(H5Z_filter_t filtn) +{ + if (filtn==H5Z_FILTER_NONE) + return "NONE"; + else if (filtn==H5Z_FILTER_DEFLATE) + return "GZIP"; + else if (filtn==H5Z_FILTER_SZIP) + return "SZIP"; + else if (filtn==H5Z_FILTER_SHUFFLE) + return "SHUFFLE"; + else if (filtn==H5Z_FILTER_FLETCHER32) + return "FLETCHER32"; + else { + error_msg(progname, "input error in filter type\n"); + exit(1); + } + return NULL; +} + + + + + + + + + + + +#if 0 extern char *progname; @@ -395,4 +958,8 @@ int have_request(pack_opt_t *options) return 0; -} \ No newline at end of file +} + +#endif + + diff --git a/tools/h5repack/h5repack.h b/tools/h5repack/h5repack.h index 9ecb839..13af382 100644 --- a/tools/h5repack/h5repack.h +++ b/tools/h5repack/h5repack.h @@ -51,6 +51,9 @@ typedef struct { */ + +#if 0 + #define CDVALUES 2 typedef struct { @@ -58,6 +61,19 @@ typedef struct { int cd_values[CDVALUES]; /* filter client data values */ } filter_info_t; +#else + +#define CD_VALUES 20 + +typedef struct { + H5Z_filter_t filtn; /* filter identification number */ + unsigned cd_values[CD_VALUES]; /* filter client data values */ + size_t cd_nelmts; /* filter client number of values */ +} filter_info_t; + + +#endif + /* chunk lengths along each dimension and rank */ typedef struct { hsize_t chunk_lengths[MAX_VAR_DIMS]; @@ -92,11 +108,22 @@ typedef struct { */ /* all the above, ready to go to the hrepack call */ + + +#if 0 + typedef struct { pack_opttbl_t *op_tbl; /*table with all -c and -f options */ int all_layout; /*apply the layout to all objects */ int all_filter; /*apply the filter to all objects */ + +#ifdef OLD filter_info_t filter_g; /*global filter INFO for the ALL case */ +#else + filter_info_t filter_g[H5_REPACK_MAX_NFILTERS]; /*global filter array for the ALL case */ +#endif + + chunk_info_t chunk_g; /*global chunk INFO for the ALL case */ H5D_layout_t layout_g; /*global layout information for the ALL case */ int verbose; /*verbose mode */ @@ -104,7 +131,25 @@ typedef struct { int use_native; /*use a native type in write */ } pack_opt_t; +#else +typedef struct { + pack_opttbl_t *op_tbl; /*table with all -c and -f options */ + int all_layout; /*apply the layout to all objects */ + int all_filter; /*apply the filter to all objects */ + filter_info_t filter_g[H5_REPACK_MAX_NFILTERS]; /*global filter array for the ALL case */ + int n_filter_g; /*number of global filters */ + chunk_info_t chunk_g; /*global chunk INFO for the ALL case */ + H5D_layout_t layout_g; /*global layout information for the ALL case */ + int verbose; /*verbose mode */ + hsize_t threshold; /*minimum size to compress, in bytes */ + int use_native; /*use a native type in write */ + +} pack_opt_t; + + + +#endif /*------------------------------------------------------------------------- * public functions @@ -136,10 +181,12 @@ int h5repack_cmpdcpl (const char *fname1, *------------------------------------------------------------------------- */ - +#if 0 int check_objects(const char* fname, pack_opt_t *options); +#endif + int copy_objects(const char* fnamein, const char* fnameout, pack_opt_t *options); @@ -152,7 +199,10 @@ int do_copy_refobjs(hid_t fidin, void init_packobject(pack_info_t *obj); int print_filters(hid_t dcpl_id); + +#if 0 int have_request(pack_opt_t *options); +#endif @@ -183,8 +233,10 @@ int can_read(const char* name, /* object name from traverse list */ *------------------------------------------------------------------------- */ +#if 0 int has_layout(hid_t dcpl_id, pack_info_t *obj); +#endif int layout_this(hid_t dcpl_id, /* DCPL from input object */ const char* name, /* object name from traverse list */ @@ -217,17 +269,33 @@ pack_info_t* options_get_object( const char *path, *------------------------------------------------------------------------- */ + +#if 0 obj_list_t* parse_filter(const char *str, int *n_objs, filter_info_t *filt, pack_opt_t *options); +#else + +obj_list_t* parse_filter(const char *str, + int *n_objs, + filter_info_t *filt, + pack_opt_t *options, + int *is_glb); + +#endif + obj_list_t* parse_layout(const char *str, int *n_objs, pack_info_t *pack, /* info about object */ pack_opt_t *options); +#if 0 const char* get_sfilter (H5Z_filter_t filtn); +#endif + + int parse_number(char *str); diff --git a/tools/h5repack/h5repack_filters.c b/tools/h5repack/h5repack_filters.c index 0f5bdd8..7a1ad72 100644 --- a/tools/h5repack/h5repack_filters.c +++ b/tools/h5repack/h5repack_filters.c @@ -122,7 +122,7 @@ int aux_assign_obj(const char* name, /* object name from traverse lis { /* assign the global filter */ tmp.nfilters=1; - tmp.filter[0]=options->filter_g; + tmp.filter[0]=options->filter_g[0]; } /* if all */ else { @@ -144,9 +144,12 @@ int aux_assign_obj(const char* name, /* object name from traverse lis if (options->all_filter) { - /* assign the global filter */ - tmp.nfilters=1; - tmp.filter[0]=options->filter_g; + int k; + + /* assign the global filters */ + tmp.nfilters=options->n_filter_g; + for ( k = 0; k < options->n_filter_g; k++) + tmp.filter[k]=options->filter_g[k]; } if (options->all_layout) { diff --git a/tools/h5repack/h5repack_opttable.c b/tools/h5repack/h5repack_opttable.c index 62c8370..102a38a 100644 --- a/tools/h5repack/h5repack_opttable.c +++ b/tools/h5repack/h5repack_opttable.c @@ -38,7 +38,7 @@ void init_packobject(pack_info_t *obj) for ( j=0; jfilter[j].filtn = -1; - for ( k=0; kfilter[j].cd_values[k] = -1; } obj->chunk.rank = -1; diff --git a/tools/h5repack/h5repack_parse.c b/tools/h5repack/h5repack_parse.c index a067011..7184391 100644 --- a/tools/h5repack/h5repack_parse.c +++ b/tools/h5repack/h5repack_parse.c @@ -47,6 +47,7 @@ extern char *progname; *------------------------------------------------------------------------- */ +#if 0 obj_list_t* parse_filter(const char *str, int *n_objs, @@ -330,6 +331,328 @@ obj_list_t* parse_filter(const char *str, } +#else + +obj_list_t* parse_filter(const char *str, + int *n_objs, + filter_info_t *filt, + pack_opt_t *options, + int *is_glb) +{ + unsigned i, u; + char c; + size_t len=strlen(str); + int j, m, n, k, l, end_obj=-1, no_param=0; + char sobj[MAX_NC_NAME]; + char scomp[10]; + char stype[5]; + char smask[3]; + obj_list_t* obj_list=NULL; + unsigned pixels_per_block; + + + /* initialize compression info */ + memset(filt,0,sizeof(filter_info_t)); + *is_glb = 0; + + /* check for the end of object list and number of objects */ + for ( i = 0, n = 0; i < len; i++) + { + c = str[i]; + if ( c==':' ) + { + end_obj=i; + } + if ( c==',' ) + { + n++; + } + } + + if (end_obj==-1) /* missing : */ + { + /* apply to all objects */ + options->all_filter=1; + *is_glb = 1; + } + + n++; + obj_list = malloc(n*sizeof(obj_list_t)); + if (obj_list==NULL) + { + error_msg(progname, "could not allocate object list\n"); + return NULL; + } + *n_objs=n; + + /* get object list */ + for ( j = 0, k = 0, n = 0; j < end_obj; j++, k++) + { + c = str[j]; + sobj[k] = c; + if ( c==',' || j==end_obj-1) + { + if ( c==',') sobj[k]='\0'; else sobj[k+1]='\0'; + strcpy(obj_list[n].obj,sobj); + memset(sobj,0,sizeof(sobj)); + n++; + k=-1; + } + } + /* nothing after : */ + if (end_obj+1==(int)len) + { + if (obj_list) free(obj_list); + error_msg(progname, "input Error: Invalid compression type in <%s>\n",str); + exit(1); + } + + + /* get filter additional parameters */ + m=0; + for ( i=end_obj+1, k=0, j=0; i + * pixels per block is a even number in 2-32 and coding method is 'EC' or 'NN' + * example SZIP=8,NN + *------------------------------------------------------------------------- + */ + if (strcmp(scomp,"SZIP")==0) + { + l=-1; /* mask index check */ + for ( m=0,u=i+1; u\n",str); + exit(1); + } + if (l==-1) + stype[m]=c; + else + { + smask[l]=c; + l++; + if (l==2) + { + smask[l]='\0'; + i=len-1; /* end */ + (*n_objs)--; /* we counted an extra ',' */ + if (strcmp(smask,"NN")==0) + filt->cd_values[j++]=H5_SZIP_NN_OPTION_MASK; + else if (strcmp(smask,"EC")==0) + filt->cd_values[j++]=H5_SZIP_EC_OPTION_MASK; + else + { + error_msg(progname, "szip mask must be 'NN' or 'EC' \n"); + exit(1); + } + + + } + } + + } /* u */ + } /*if */ + + + + + /*------------------------------------------------------------------------- + * all other filters + *------------------------------------------------------------------------- + */ + + else + { + /* here we could have 1 or 2 digits */ + for ( m=0,u=i+1; u\n",str); + exit(1); + } + stype[m]=c; + } /* u */ + + stype[m]='\0'; + } /*if */ + + + + filt->cd_values[j++]=atoi(stype); + i+=m; /* jump */ + } + else if (i==len-1) + { /*no more parameters */ + scomp[k+1]='\0'; + no_param=1; + } + + /*------------------------------------------------------------------------- + * translate from string to filter symbol + *------------------------------------------------------------------------- + */ + + /*------------------------------------------------------------------------- + * H5Z_FILTER_NONE + *------------------------------------------------------------------------- + */ + if (strcmp(scomp,"NONE")==0) + { + filt->filtn=H5Z_FILTER_NONE; + filt->cd_nelmts = 0; + } + + /*------------------------------------------------------------------------- + * H5Z_FILTER_DEFLATE + *------------------------------------------------------------------------- + */ + else if (strcmp(scomp,"GZIP")==0) + { + filt->filtn=H5Z_FILTER_DEFLATE; + filt->cd_nelmts = 1; + if (no_param) + { /*no more parameters, GZIP must have parameter */ + if (obj_list) free(obj_list); + error_msg(progname, "missing compression parameter in <%s>\n",str); + exit(1); + } + } + + /*------------------------------------------------------------------------- + * H5Z_FILTER_SZIP + *------------------------------------------------------------------------- + */ + else if (strcmp(scomp,"SZIP")==0) + { + filt->filtn=H5Z_FILTER_SZIP; + filt->cd_nelmts = 2; + if (no_param) + { /*no more parameters, SZIP must have parameter */ + if (obj_list) free(obj_list); + error_msg(progname, "missing compression parameter in <%s>\n",str); + exit(1); + } + } + + /*------------------------------------------------------------------------- + * H5Z_FILTER_SHUFFLE + *------------------------------------------------------------------------- + */ + else if (strcmp(scomp,"SHUF")==0) + { + filt->filtn=H5Z_FILTER_SHUFFLE; + filt->cd_nelmts = 0; + if (m>0) + { /*shuffle does not have parameter */ + if (obj_list) free(obj_list); + error_msg(progname, "extra parameter in SHUF <%s>\n",str); + exit(1); + } + } + /*------------------------------------------------------------------------- + * H5Z_FILTER_FLETCHER32 + *------------------------------------------------------------------------- + */ + else if (strcmp(scomp,"FLET")==0) + { + filt->filtn=H5Z_FILTER_FLETCHER32; + filt->cd_nelmts = 0; + if (m>0) + { /*shuffle does not have parameter */ + if (obj_list) free(obj_list); + error_msg(progname, "extra parameter in FLET <%s>\n",str); + exit(1); + } + } + + else { + if (obj_list) free(obj_list); + error_msg(progname, "invalid filter type in <%s>\n",str); + exit(1); + } + } + } /*i*/ + + /*------------------------------------------------------------------------- + * check valid parameters + *------------------------------------------------------------------------- + */ + + switch (filt->filtn) + { + + /*------------------------------------------------------------------------- + * H5Z_FILTER_DEFLATE + *------------------------------------------------------------------------- + */ + + case H5Z_FILTER_DEFLATE: + if (filt->cd_values[0]<0 || filt->cd_values[0]>9 ) + { + if (obj_list) free(obj_list); + error_msg(progname, "invalid compression parameter in <%s>\n",str); + exit(1); + } + break; + + /*------------------------------------------------------------------------- + * H5Z_FILTER_SZIP + *------------------------------------------------------------------------- + */ + + case H5Z_FILTER_SZIP: + pixels_per_block=filt->cd_values[0]; + if ((pixels_per_block%2)==1) + { + if (obj_list) free(obj_list); + error_msg(progname, "pixels_per_block is not even in <%s>\n",str); + exit(1); + } + if (pixels_per_block>H5_SZIP_MAX_PIXELS_PER_BLOCK) + { + if (obj_list) free(obj_list); + error_msg(progname, "pixels_per_block is too large in <%s>\n",str); + exit(1); + } + if ( (strcmp(smask,"NN")!=0) && (strcmp(smask,"EC")!=0) ) + { + if (obj_list) free(obj_list); + error_msg(progname, "szip mask must be 'NN' or 'EC' \n"); + exit(1); + } + break; + + + }; + + return obj_list; +} + + + +#endif + /*------------------------------------------------------------------------- * Function: get_sfilter * diff --git a/tools/h5repack/h5repack_verify.c b/tools/h5repack/h5repack_verify.c index bb910f6..c734405 100644 --- a/tools/h5repack/h5repack_verify.c +++ b/tools/h5repack/h5repack_verify.c @@ -15,10 +15,18 @@ #include "h5repack.h" #include "h5test.h" -#include "h5tools.h" #include "h5tools_utils.h" extern char *progname; +static int has_layout(hid_t pid, pack_info_t *obj); +static int has_filters(hid_t pid, hid_t tid, unsigned nfilters, filter_info_t *filter); + + + + +#if 0 + +extern char *progname; /*------------------------------------------------------------------------- @@ -485,3 +493,554 @@ error: } +#else + + +extern char *progname; +static int has_layout(hid_t pid, pack_info_t *obj); +static int has_filters(hid_t pid, hid_t tid, unsigned nfilters, filter_info_t *filter); + + +/*------------------------------------------------------------------------- + * Function: h5repack_verify + * + * Purpose: verify if filters and layout in the input file match the output file + * + * Return: + * 1 match + * 0 do not match + * -1 error + * + * Programmer: Pedro Vicente, pvn@hdfgroup.org + * + * Date: December 19, 2003 + * Modified: December, 19, 2007 (exactly 4 years later :-) ) + * Separate into 3 cases + * 1) no filter input, get all datasets and compare DCPLs. TO DO + * 2) filter input on selected datasets, get each one trough OBJ and match + * 3) filter input on all datasets, get all objects and match + * + *------------------------------------------------------------------------- + */ + +int h5repack_verify(const char *fname, + pack_opt_t *options) +{ + hid_t fid; /* file ID */ + hid_t did; /* dataset ID */ + hid_t pid; /* dataset creation property list ID */ + hid_t sid; /* space ID */ + hid_t tid; /* type ID */ + unsigned int i; + trav_table_t *travt = NULL; + int ok = 1; + + /* open the file */ + if((fid = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0 ) + return -1; + + for(i = 0; i < options->op_tbl->nelems; i++) + { + char* name = options->op_tbl->objs[i].path; + pack_info_t *obj = &options->op_tbl->objs[i]; + + /*------------------------------------------------------------------------- + * open + *------------------------------------------------------------------------- + */ + if((did = H5Dopen(fid, name)) < 0) + goto error; + if((sid = H5Dget_space(did)) < 0) + goto error; + if((pid = H5Dget_create_plist(did)) < 0) + goto error; + if((tid = H5Dget_type(did)) < 0) + goto error; + + /*------------------------------------------------------------------------- + * filter check + *------------------------------------------------------------------------- + */ + if(has_filters(pid, tid, obj->nfilters, obj->filter) <= 0) + ok = 0; + + + /*------------------------------------------------------------------------- + * layout check + *------------------------------------------------------------------------- + */ + if((obj->layout != -1) && (has_layout(pid, obj) == 0)) + ok = 0; + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + if(H5Pclose(pid) < 0) + goto error; + if (H5Sclose(sid) < 0) + goto error; + if (H5Dclose(did) < 0) + goto error; + if (H5Tclose(tid) < 0) + goto error; + + } + + + /*------------------------------------------------------------------------- + * check for the "all" objects option + *------------------------------------------------------------------------- + */ + + if(options->all_filter == 1 || options->all_layout == 1) + { + + /* init table */ + trav_table_init(&travt); + + /* get the list of objects in the file */ + if(h5trav_gettable(fid, travt) < 0) + goto error; + + for(i = 0; i < travt->nobjs; i++) + { + char *name = travt->objs[i].name; + + if(travt->objs[i].type == H5G_DATASET) + { + + /*------------------------------------------------------------------------- + * open + *------------------------------------------------------------------------- + */ + if((did = H5Dopen(fid, name)) < 0) + goto error; + if((sid = H5Dget_space(did)) < 0) + goto error; + if((pid = H5Dget_create_plist(did)) < 0) + goto error; + if((tid = H5Dget_type(did)) < 0) + goto error; + + /*------------------------------------------------------------------------- + * filter check + *------------------------------------------------------------------------- + */ + if(options->all_filter == 1) + { + + if(has_filters(pid, tid, options->n_filter_g, options->filter_g) <= 0) + ok = 0; + } + + /*------------------------------------------------------------------------- + * layout check + *------------------------------------------------------------------------- + */ + if(options->all_layout == 1) + { + pack_info_t pack; + init_packobject(&pack); + pack.layout = options->layout_g; + pack.chunk = options->chunk_g; + if(has_layout(pid, &pack) == 0) + ok = 0; + } + + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + if (H5Pclose(pid) < 0) + goto error; + if (H5Sclose(sid) < 0) + goto error; + if (H5Dclose(did) < 0) + goto error; + if (H5Tclose(tid) < 0) + goto error; + } /* if */ + + } /* i */ + + /* free table */ + trav_table_free(travt); + } + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + + if (H5Fclose(fid) < 0) + return -1; + + return ok; + +error: + H5E_BEGIN_TRY { + H5Pclose(pid); + H5Sclose(sid); + H5Dclose(did); + H5Fclose(fid); + if (travt) + trav_table_free(travt); + } H5E_END_TRY; + return -1; +} + + + +/*------------------------------------------------------------------------- + * Function: has_layout + * + * Purpose: verify which layout is present in the property list DCPL_ID + * + * H5D_COMPACT = 0 + * H5D_CONTIGUOUS = 1 + * H5D_CHUNKED = 2 + * + * Return: 1 has, 0 does not, -1 error + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: December 30, 2003 + * + *------------------------------------------------------------------------- + */ + +int has_layout(hid_t pid, + pack_info_t *obj) +{ + hsize_t chsize[64]; /* chunk size in elements */ + H5D_layout_t layout; /* layout */ + int nfilters; /* number of filters */ + int rank; /* rank */ + int i; /* index */ + + /* if no information about the input layout is requested return exit */ + if (obj==NULL) + return 1; + + /* check if we have filters in the input object */ + if ((nfilters = H5Pget_nfilters(pid)) < 0) + return -1; + + /* a non chunked layout was requested on a filtered object; avoid the test */ + if (nfilters && obj->layout!=H5D_CHUNKED) + return 1; + + /* get layout */ + if ((layout = H5Pget_layout(pid)) < 0) + return -1; + + if (obj->layout != layout) + return 0; + + if (layout==H5D_CHUNKED) + { + if ((rank = H5Pget_chunk(pid,NELMTS(chsize),chsize/*out*/)) < 0) + return -1; + if (obj->chunk.rank != rank) + return 0; + for ( i=0; ichunk.chunk_lengths[i]) + return 0; + } + + return 1; +} + +/*------------------------------------------------------------------------- + * Function: h5repack_cmpdcpl + * + * Purpose: compare 2 files for identical property lists of all objects + * + * Return: 1=identical, 0=not identical, -1=error + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: December 31, 2003 + * + *------------------------------------------------------------------------- + */ + +int h5repack_cmpdcpl(const char *fname1, + const char *fname2) +{ + hid_t fid1=-1; /* file ID */ + hid_t fid2=-1; /* file ID */ + hid_t dset1=-1; /* dataset ID */ + hid_t dset2=-1; /* dataset ID */ + hid_t dcpl1=-1; /* dataset creation property list ID */ + hid_t dcpl2=-1; /* dataset creation property list ID */ + trav_table_t *travt1=NULL; + trav_table_t *travt2=NULL; + int ret=1; + unsigned int i; + + /*------------------------------------------------------------------------- + * open the files + *------------------------------------------------------------------------- + */ + + /* disable error reporting */ + H5E_BEGIN_TRY + { + + /* Open the files */ + if ((fid1=H5Fopen(fname1,H5F_ACC_RDONLY,H5P_DEFAULT)) < 0 ) + { + error_msg(progname, "<%s>: %s\n", fname1, H5FOPENERROR ); + return -1; + } + if ((fid2=H5Fopen(fname2,H5F_ACC_RDONLY,H5P_DEFAULT)) < 0 ) + { + error_msg(progname, "<%s>: %s\n", fname2, H5FOPENERROR ); + H5Fclose(fid1); + return -1; + } + /* enable error reporting */ + } H5E_END_TRY; + + /*------------------------------------------------------------------------- + * get file table list of objects + *------------------------------------------------------------------------- + */ + trav_table_init(&travt1); + trav_table_init(&travt2); + if(h5trav_gettable(fid1, travt1) < 0) + goto error; + if(h5trav_gettable(fid2, travt2) < 0) + goto error; + + + /*------------------------------------------------------------------------- + * traverse the suppplied object list + *------------------------------------------------------------------------- + */ + + for(i = 0; i < travt1->nobjs; i++) + { + if(travt1->objs[i].type == H5G_DATASET) + { + if((dset1 = H5Dopen(fid1, travt1->objs[i].name)) < 0) + goto error; + if((dset2 = H5Dopen(fid2, travt1->objs[i].name)) < 0) + goto error; + if((dcpl1 = H5Dget_create_plist(dset1)) < 0) + goto error; + if((dcpl2 = H5Dget_create_plist(dset2)) < 0) + goto error; + + /*------------------------------------------------------------------------- + * compare the property lists + *------------------------------------------------------------------------- + */ + if((ret = H5Pequal(dcpl1, dcpl2)) < 0) + goto error; + + if(ret == 0) + { + error_msg(progname, "property lists for <%s> are different\n",travt1->objs[i].name); + goto error; + } + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + if(H5Pclose(dcpl1) < 0) + goto error; + if(H5Pclose(dcpl2) < 0) + goto error; + if(H5Dclose(dset1) < 0) + goto error; + if(H5Dclose(dset2) < 0) + goto error; + } /*if*/ + } /*i*/ + + /*------------------------------------------------------------------------- + * free + *------------------------------------------------------------------------- + */ + + trav_table_free(travt1); + trav_table_free(travt2); + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + + H5Fclose(fid1); + H5Fclose(fid2); + return ret; + + /*------------------------------------------------------------------------- + * error + *------------------------------------------------------------------------- + */ + +error: + H5E_BEGIN_TRY + { + H5Pclose(dcpl1); + H5Pclose(dcpl2); + H5Dclose(dset1); + H5Dclose(dset2); + H5Fclose(fid1); + H5Fclose(fid2); + trav_table_free(travt1); + trav_table_free(travt2); + } H5E_END_TRY; + return -1; + +} + + +/*------------------------------------------------------------------------- + * Function: has_filters + * + * Purpose: verify if all requested filters in the array FILTER obtained + * from user input are present in the property list PID obtained from + * the output file + * + * Return: + * 1 match + * 0 do not match + * -1 error + * + * Programmer: Pedro Vicente, pvn@hdfgroup.org + * + * Date: December 21, 2007 + * + *------------------------------------------------------------------------- + */ + +static int has_filters(hid_t pid, hid_t tid, unsigned nfilters, filter_info_t *filter) +{ + unsigned nfilters_dcpl; /* number of filters in DCPL*/ + unsigned filt_flags; /* filter flags */ + H5Z_filter_t filtn; /* filter identification number */ + unsigned cd_values[20]; /* filter client data values */ + size_t cd_nelmts; /* filter client number of values */ + char f_name[256]; /* filter name */ + size_t size; /* type size */ + unsigned i, j; /* index */ + + /* get information about filters */ + if((nfilters_dcpl = H5Pget_nfilters(pid)) < 0) + return -1; + + /* if we do not have filters and the requested filter is NONE, return 1 */ + if(!nfilters_dcpl && + nfilters == 1 && + filter[0].filtn == H5Z_FILTER_NONE ) + return 1; + + /* else the numbers of filters must match */ + if (nfilters_dcpl != nfilters ) + return 0; + + /*------------------------------------------------------------------------- + * build a list with DCPL filters + *------------------------------------------------------------------------- + */ + + for( i = 0; i < nfilters_dcpl; i++) + { + cd_nelmts = NELMTS(cd_values); + + + filtn = H5Pget_filter(pid, + (unsigned)i, + &filt_flags, + &cd_nelmts, + cd_values, + sizeof(f_name), + f_name); + + + /* filter ID */ + if (filtn != filter[i].filtn) + return 0; + + /* compare client data values. some filters do return local values */ + switch (filtn) + { + + case H5Z_FILTER_SHUFFLE: + + /* 1 private client value is returned by DCPL */ + if ( cd_nelmts != H5Z_SHUFFLE_TOTAL_NPARMS && filter[i].cd_nelmts != H5Z_SHUFFLE_USER_NPARMS ) + return 0; + + /* get dataset's type size */ + if((size = H5Tget_size(tid)) <= 0) + return -1; + + /* the private client value holds the dataset's type size */ + if ( size != cd_values[0] ) + return 0; + + + break; + + case H5Z_FILTER_SZIP: + + /* 4 private client values are returned by DCPL */ + if ( cd_nelmts != H5Z_SZIP_TOTAL_NPARMS && filter[i].cd_nelmts != H5Z_SZIP_USER_NPARMS ) + return 0; + + /* "User" parameter for pixels-per-block (index 1) */ + if ( cd_values[H5Z_SZIP_PARM_PPB] != filter[i].cd_values[H5Z_SZIP_PARM_PPB] ) + return 0; + + + break; + + + + + + /* for these filters values must match, no local values set in DCPL */ + case H5Z_FILTER_FLETCHER32: + case H5Z_FILTER_DEFLATE: + + if ( cd_nelmts != filter[i].cd_nelmts) + return 0; + + for( j = 0; j < cd_nelmts; j++) + { + if (cd_values[j] != filter[i].cd_values[j]) + { + return 0; + } + + } + + + + break; + + + + } /* switch */ + + } + + return 1; +} + + + + + + + + +#endif diff --git a/tools/h5repack/h5repacktst.c b/tools/h5repack/h5repacktst.c index 69bfea4..f5bc835 100644 --- a/tools/h5repack/h5repacktst.c +++ b/tools/h5repack/h5repacktst.c @@ -1175,6 +1175,36 @@ if (szip_can_encode) { GOERROR; PASSED(); + /*------------------------------------------------------------------------- + * test several global filters + *------------------------------------------------------------------------- + */ + + TESTING(" several global filters"); + +#if defined (H5_HAVE_FILTER_DEFLATE) && defined (H5_HAVE_FILTER_SHUFFLE) + + if (h5repack_init (&pack_options, 0) < 0) + GOERROR; + if (h5repack_addfilter("GZIP=1",&pack_options) < 0) + GOERROR; + if (h5repack_addfilter("SHUF",&pack_options) < 0) + GOERROR; + if (h5repack(FNAME11,FNAME11OUT,&pack_options) < 0) + GOERROR; + if (h5diff(FNAME11,FNAME11OUT,NULL,NULL,&diff_options) >0) + GOERROR; + if (h5repack_verify(FNAME11OUT,&pack_options)<=0) + GOERROR; + if (h5repack_end (&pack_options) < 0) + GOERROR; + + PASSED(); +#else + SKIPPED(); +#endif + + /*------------------------------------------------------------------------- * clean temporary test files -- cgit v0.12