diff options
author | Quincey Koziol <koziol@hdfgroup.org> | 2008-06-16 22:17:29 (GMT) |
---|---|---|
committer | Quincey Koziol <koziol@hdfgroup.org> | 2008-06-16 22:17:29 (GMT) |
commit | db2c4c497abab139485dbf6797e56b1340e4577d (patch) | |
tree | f7dfb321a3643658ed24aaf776ec4c1e48bc76a3 /tools/h5repack | |
parent | 55af47c47c3c0c7bc17266968f5e0e782189e819 (diff) | |
download | hdf5-db2c4c497abab139485dbf6797e56b1340e4577d.zip hdf5-db2c4c497abab139485dbf6797e56b1340e4577d.tar.gz hdf5-db2c4c497abab139485dbf6797e56b1340e4577d.tar.bz2 |
[svn-r15221] Description:
Bring changes from trunk from the time the branch was created (r14280)
up to the 1.8.0 release (r14525) back into the metadata journaling branch.
Tested on:
FreeBSD/32 6.2 (duty) in debug mode
FreeBSD/64 6.2 (liberty) w/C++ & FORTRAN, in debug mode
Linux/32 2.6 (kagiso) w/PGI compilers, w/C++ & FORTRAN, w/threadsafe,
in debug mode
Linux/64-amd64 2.6 (smirom) w/default API=1.6.x, w/C++ & FORTRAN,
in production mode
Linux/64-ia64 2.6 (cobalt) w/Intel compilers, w/C++ & FORTRAN,
in production mode
Solaris/32 2.10 (linew) w/deprecated symbols disabled, w/C++ & FORTRAN,
w/szip filter, in production mode
Linux/64-ia64 2.4 (tg-login3) w/parallel, w/FORTRAN, in production mode
Diffstat (limited to 'tools/h5repack')
-rw-r--r-- | tools/h5repack/Makefile.am | 4 | ||||
-rw-r--r-- | tools/h5repack/Makefile.in | 12 | ||||
-rw-r--r-- | tools/h5repack/h5repack.c | 789 | ||||
-rw-r--r-- | tools/h5repack/h5repack.h | 68 | ||||
-rwxr-xr-x | tools/h5repack/h5repack.sh.in | 21 | ||||
-rw-r--r-- | tools/h5repack/h5repack_copy.c | 482 | ||||
-rw-r--r-- | tools/h5repack/h5repack_filters.c | 77 | ||||
-rw-r--r-- | tools/h5repack/h5repack_list.c | 153 | ||||
-rw-r--r-- | tools/h5repack/h5repack_main.c | 699 | ||||
-rw-r--r-- | tools/h5repack/h5repack_opttable.c | 476 | ||||
-rw-r--r-- | tools/h5repack/h5repack_parse.c | 1082 | ||||
-rw-r--r-- | tools/h5repack/h5repack_refs.c | 2 | ||||
-rw-r--r-- | tools/h5repack/h5repack_verify.c | 848 | ||||
-rw-r--r-- | tools/h5repack/h5repacktst.c | 37 |
14 files changed, 2348 insertions, 2402 deletions
diff --git a/tools/h5repack/Makefile.am b/tools/h5repack/Makefile.am index e053e83..f677de0 100644 --- a/tools/h5repack/Makefile.am +++ b/tools/h5repack/Makefile.am @@ -41,8 +41,8 @@ LDADD=$(LIBH5TOOLS) $(LIBH5TEST) $(LIBHDF5) # Source files COMMON_SOURCES=h5repack.c h5repack_copy.c h5repack_filters.c \ - h5repack_list.c h5repack_opttable.c h5repack_parse.c h5repack_refs.c \ - h5repack_verify.c + h5repack_opttable.c h5repack_parse.c h5repack_refs.c \ + h5repack_verify.c h5repack_SOURCES=$(COMMON_SOURCES) h5repack_main.c diff --git a/tools/h5repack/Makefile.in b/tools/h5repack/Makefile.in index 1823f8b..97d3620 100644 --- a/tools/h5repack/Makefile.in +++ b/tools/h5repack/Makefile.in @@ -69,9 +69,9 @@ binPROGRAMS_INSTALL = $(INSTALL_PROGRAM) am__EXEEXT_1 = h5repacktst$(EXEEXT) PROGRAMS = $(bin_PROGRAMS) $(noinst_PROGRAMS) am__objects_1 = h5repack.$(OBJEXT) h5repack_copy.$(OBJEXT) \ - h5repack_filters.$(OBJEXT) h5repack_list.$(OBJEXT) \ - h5repack_opttable.$(OBJEXT) h5repack_parse.$(OBJEXT) \ - h5repack_refs.$(OBJEXT) h5repack_verify.$(OBJEXT) + h5repack_filters.$(OBJEXT) h5repack_opttable.$(OBJEXT) \ + h5repack_parse.$(OBJEXT) h5repack_refs.$(OBJEXT) \ + h5repack_verify.$(OBJEXT) am_h5repack_OBJECTS = $(am__objects_1) h5repack_main.$(OBJEXT) h5repack_OBJECTS = $(am_h5repack_OBJECTS) h5repack_LDADD = $(LDADD) @@ -224,6 +224,7 @@ SET_MAKE = @SET_MAKE@ SHELL = /bin/sh SIZE_T = @SIZE_T@ STATIC_SHARED = @STATIC_SHARED@ +STRICT_FORMAT_CHECKS = @STRICT_FORMAT_CHECKS@ STRIP = @STRIP@ TESTPARALLEL = @TESTPARALLEL@ TIME = @TIME@ @@ -351,8 +352,8 @@ LDADD = $(LIBH5TOOLS) $(LIBH5TEST) $(LIBHDF5) # Source files COMMON_SOURCES = h5repack.c h5repack_copy.c h5repack_filters.c \ - h5repack_list.c h5repack_opttable.c h5repack_parse.c h5repack_refs.c \ - h5repack_verify.c + h5repack_opttable.c h5repack_parse.c h5repack_refs.c \ + h5repack_verify.c h5repack_SOURCES = $(COMMON_SOURCES) h5repack_main.c h5repacktst_SOURCES = $(COMMON_SOURCES) h5repacktst.c @@ -470,7 +471,6 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/h5repack.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/h5repack_copy.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/h5repack_filters.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/h5repack_list.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/h5repack_main.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/h5repack_opttable.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/h5repack_parse.Po@am__quote@ diff --git a/tools/h5repack/h5repack.c b/tools/h5repack/h5repack.c index 45409c4..954f3e7 100644 --- a/tools/h5repack/h5repack.c +++ b/tools/h5repack/h5repack.c @@ -15,9 +15,14 @@ #include <stdlib.h> #include <string.h> +#include <ctype.h> + +#include "H5private.h" #include "h5repack.h" +#include "h5tools.h" #include "h5tools_utils.h" + extern char *progname; @@ -28,25 +33,11 @@ extern char *progname; */ static int check_options(pack_opt_t *options); +static int check_objects(const char* fname, pack_opt_t *options); +static const char* get_sfilter (H5Z_filter_t filtn); +static int have_request(pack_opt_t *options); -/*------------------------------------------------------------------------- - * Function: aux_initglb_filter - * - * Purpose: auxiliary function, initialize the options global filter - * - * Return: void - * - *------------------------------------------------------------------------- - */ -static void aux_initglb_filter(pack_opt_t *options) -{ - int k; - options->filter_g.filtn = -1; - for ( k=0; k<CDVALUES; k++) - options->filter_g.cd_values[k] = -1; -} - /*------------------------------------------------------------------------- * Function: h5repack @@ -71,20 +62,20 @@ int h5repack(const char* infile, const char* outfile, pack_opt_t *options) { - /* check input */ - if (check_options(options)<0) - return -1; - - /* check for objects in input that are in the file */ - if (check_objects(infile,options) < 0) - return -1; - - /* copy the objects */ - if (copy_objects(infile,outfile,options) < 0) - return -1; - - - return 0; + /* check input */ + if (check_options(options)<0) + return -1; + + /* check for objects in input that are in the file */ + if (check_objects(infile,options) < 0) + return -1; + + /* copy the objects */ + if (copy_objects(infile,outfile,options) < 0) + return -1; + + + return 0; } @@ -102,10 +93,20 @@ int h5repack(const char* infile, int h5repack_init (pack_opt_t *options, int verbose) { - memset(options,0,sizeof(pack_opt_t)); - options->threshold = 1024; - options->verbose = verbose; - return (options_table_init(&(options->op_tbl))); + int k, n; + memset(options,0,sizeof(pack_opt_t)); + options->threshold = 1024; + options->verbose = verbose; + + for ( n = 0; n < H5_REPACK_MAX_NFILTERS; n++) + { + options->filter_g[n].filtn = -1; + options->filter_g[n].cd_nelmts = -1; + for ( k = 0; k < CD_VALUES; k++) + options->filter_g[n].cd_values[k] = -1; + } + + return (options_table_init(&(options->op_tbl))); } /*------------------------------------------------------------------------- @@ -118,7 +119,7 @@ int h5repack_init (pack_opt_t *options, int h5repack_end (pack_opt_t *options) { - return options_table_free(options->op_tbl); + return options_table_free(options->op_tbl); } /*------------------------------------------------------------------------- @@ -135,35 +136,43 @@ int h5repack_end (pack_opt_t *options) int h5repack_addfilter(const char* str, pack_opt_t *options) { - obj_list_t *obj_list=NULL; /*one object list for the -f and -c option entry */ - filter_info_t filt; /*filter info for the current -f option entry */ - int n_objs; /*number of objects in the current -f or -c option entry */ - - if (options->all_filter==1){ - error_msg(progname, "invalid compression input: 'all' option is present \ - with other objects <%s>\n",str); - return -1; - } - - /* parse the -f option */ - obj_list=parse_filter(str,&n_objs,&filt,options); - if (obj_list==NULL) - { - return -1; - } - - if (options->all_filter==1) - { - /* if we are compressing all set the global filter type */ - aux_initglb_filter(options); - options->filter_g=filt; - } - - if (options->all_filter==0) - options_add_filter(obj_list,n_objs,filt,options->op_tbl); - - free(obj_list); - return 0; + obj_list_t *obj_list=NULL; /* one object list for the -f and -l option entry */ + filter_info_t filter; /* filter info for the current -f option entry */ + int n_objs; /* number of objects in the current -f or -l option entry */ + int is_glb; /* is the filter global */ + + + + /* parse the -f option */ + obj_list=parse_filter(str,&n_objs,&filter,options,&is_glb); + if (obj_list==NULL) + { + return -1; + } + + /* if it applies to all objects */ + if (is_glb) + { + + int n; + + n = options->n_filter_g++; /* increase # of global filters */ + + if (options->n_filter_g > H5_REPACK_MAX_NFILTERS) + { + error_msg(progname, "maximum number of filters exceeded for <%s>\n",str); + return -1; + + } + + options->filter_g[n] = filter; + } + + else + options_add_filter(obj_list,n_objs,filter,options->op_tbl); + + free(obj_list); + return 0; } @@ -181,55 +190,55 @@ int h5repack_addfilter(const char* str, int h5repack_addlayout(const char* str, pack_opt_t *options) { - - obj_list_t *obj_list=NULL; /*one object list for the -t and -c option entry */ - int n_objs; /*number of objects in the current -t or -c option entry */ - pack_info_t pack; /*info about layout to extract from parse */ - int j; - - init_packobject(&pack); - - if (options->all_layout==1){ - error_msg(progname, "invalid layout input: 'all' option \ - is present with other objects <%s>\n",str); - return -1; - } - - /* parse the layout option */ - obj_list=parse_layout(str,&n_objs,&pack,options); - if (obj_list==NULL) - return -1; - - /* set global layout option */ - if (options->all_layout==1 ) - { - options->layout_g=pack.layout; - if (pack.layout==H5D_CHUNKED) - { - /* -2 means the NONE option, remove chunking - and set the global layout to contiguous */ - if (pack.chunk.rank==-2) - { - options->layout_g = H5D_CONTIGUOUS; - } - /* otherwise set the global chunking type */ - else - { - options->chunk_g.rank=pack.chunk.rank; - for (j = 0; j < pack.chunk.rank; j++) - options->chunk_g.chunk_lengths[j] = pack.chunk.chunk_lengths[j]; - } - } - } - - if (options->all_layout==0) - options_add_layout(obj_list, - n_objs, - &pack, - options->op_tbl); - - free(obj_list); - return 0; + + obj_list_t *obj_list=NULL; /*one object list for the -t and -c option entry */ + int n_objs; /*number of objects in the current -t or -c option entry */ + pack_info_t pack; /*info about layout to extract from parse */ + int j; + + init_packobject(&pack); + + if (options->all_layout==1){ + error_msg(progname, "invalid layout input: 'all' option \ + is present with other objects <%s>\n",str); + return -1; + } + + /* parse the layout option */ + obj_list=parse_layout(str,&n_objs,&pack,options); + if (obj_list==NULL) + return -1; + + /* set global layout option */ + if (options->all_layout==1 ) + { + options->layout_g=pack.layout; + if (pack.layout==H5D_CHUNKED) + { + /* -2 means the NONE option, remove chunking + and set the global layout to contiguous */ + if (pack.chunk.rank==-2) + { + options->layout_g = H5D_CONTIGUOUS; + } + /* otherwise set the global chunking type */ + else + { + options->chunk_g.rank=pack.chunk.rank; + for (j = 0; j < pack.chunk.rank; j++) + options->chunk_g.chunk_lengths[j] = pack.chunk.chunk_lengths[j]; + } + } + } + + if (options->all_layout==0) + options_add_layout(obj_list, + n_objs, + &pack, + options->op_tbl); + + free(obj_list); + return 0; } @@ -252,270 +261,285 @@ int h5repack_addlayout(const char* str, */ static int check_options(pack_opt_t *options) { - unsigned int i; - int k, j, has_cp=0, has_ck=0; - char slayout[30]; - -/*------------------------------------------------------------------------- - * objects to layout - *------------------------------------------------------------------------- - */ - if (options->verbose && have_request(options) /* only print if requested */) - { - printf("Objects to modify layout are...\n"); - if (options->all_layout==1) { - switch (options->layout_g) - { - case H5D_COMPACT: - strcpy(slayout,"compact"); - break; - case H5D_CONTIGUOUS: - strcpy(slayout,"contiguous"); - break; - case H5D_CHUNKED: - strcpy(slayout,"chunked"); - break; - default: - strcpy(slayout,"unknown"); - break; - } - printf(" Apply %s layout to all\n", slayout); - if (H5D_CHUNKED==options->layout_g) { - printf("with dimension ["); - for ( j = 0; j < options->chunk_g.rank; j++) - printf("%d ",(int)options->chunk_g.chunk_lengths[j]); - printf("]\n"); - } - } - }/* verbose */ - - for ( i = 0; i < options->op_tbl->nelems; i++) - { - char* name=options->op_tbl->objs[i].path; - - if (options->op_tbl->objs[i].chunk.rank>0) - { - if (options->verbose){ - printf(" <%s> with chunk size ",name); - for ( k = 0; k < options->op_tbl->objs[i].chunk.rank; k++) - printf("%d ",(int)options->op_tbl->objs[i].chunk.chunk_lengths[k]); - printf("\n"); - } - has_ck=1; - } - else if (options->op_tbl->objs[i].chunk.rank==-2) - { - if (options->verbose) - printf(" <%s> %s\n",name,"NONE (contigous)"); - has_ck=1; - } - } - - if (options->all_layout==1 && has_ck){ - error_msg(progname, "invalid chunking input: 'all' option\ - is present with other objects\n"); - return -1; - } - -/*------------------------------------------------------------------------- - * objects to filter - *------------------------------------------------------------------------- - */ - - if (options->verbose && have_request(options) /* only print if requested */) - { - printf("Objects to apply filter are...\n"); - if (options->all_filter==1) - { - H5Z_filter_t filtn=options->filter_g.filtn; - switch (filtn) - { - case H5Z_FILTER_NONE: - printf(" Uncompress all\n"); - break; - case H5Z_FILTER_SHUFFLE: - case H5Z_FILTER_FLETCHER32: - printf(" All with %s\n",get_sfilter(filtn)); - break; - case H5Z_FILTER_SZIP: - case H5Z_FILTER_DEFLATE: - printf(" All with %s, parameter %d\n", - get_sfilter(filtn), - options->filter_g.cd_values[0]); - break; - }; - } - } /* verbose */ - - for ( i = 0; i < options->op_tbl->nelems; i++) - { - pack_info_t pack = options->op_tbl->objs[i]; - char* name = pack.path; - - for ( j=0; j<pack.nfilters; j++) - { - if (options->verbose) - { - printf(" <%s> with %s filter\n", - name, - get_sfilter(pack.filter[j].filtn)); - } - - has_cp=1; - - } /* j */ - } /* i */ - - if (options->all_filter==1 && has_cp){ - error_msg(progname, "invalid compression input: 'all' option\ - is present with other objects\n"); - return -1; - } - - /* check options for the latest format */ - if (options->grp_compact < 0) { - error_msg(progname, "invalid maximum number of links to store as header messages\n"); - return -1; - } - if (options->grp_indexed < 0) { - error_msg(progname, "invalid minimum number of links to store in the indexed format\n"); - return -1; - } - if (options->grp_indexed > options->grp_compact) { - error_msg(progname, "minimum indexed size is greater than the maximum compact size\n"); - return -1; - } - for (i=0; i<8; i++) { - if (options->msg_size[i]<0) { - error_msg(progname, "invalid shared message size\n"); - return -1; - } - } - - return 0; + unsigned int i; + int k, j, has_cp=0, has_ck=0; + char slayout[30]; + + /*------------------------------------------------------------------------- + * objects to layout + *------------------------------------------------------------------------- + */ + if (options->verbose && have_request(options) /* only print if requested */) + { + printf("Objects to modify layout are...\n"); + if (options->all_layout==1) { + switch (options->layout_g) + { + case H5D_COMPACT: + strcpy(slayout,"compact"); + break; + case H5D_CONTIGUOUS: + strcpy(slayout,"contiguous"); + break; + case H5D_CHUNKED: + strcpy(slayout,"chunked"); + break; + default: + strcpy(slayout,"unknown"); + break; + } + printf(" Apply %s layout to all\n", slayout); + if (H5D_CHUNKED==options->layout_g) { + printf("with dimension ["); + for ( j = 0; j < options->chunk_g.rank; j++) + printf("%d ",(int)options->chunk_g.chunk_lengths[j]); + printf("]\n"); + } + } + }/* verbose */ + + for ( i = 0; i < options->op_tbl->nelems; i++) + { + char* name=options->op_tbl->objs[i].path; + + if (options->op_tbl->objs[i].chunk.rank>0) + { + if (options->verbose){ + printf(" <%s> with chunk size ",name); + for ( k = 0; k < options->op_tbl->objs[i].chunk.rank; k++) + printf("%d ",(int)options->op_tbl->objs[i].chunk.chunk_lengths[k]); + printf("\n"); + } + has_ck=1; + } + else if (options->op_tbl->objs[i].chunk.rank==-2) + { + if (options->verbose) + printf(" <%s> %s\n",name,"NONE (contigous)"); + has_ck=1; + } + } + + if (options->all_layout==1 && has_ck){ + error_msg(progname, "invalid chunking input: 'all' option\ + is present with other objects\n"); + return -1; + } + + /*------------------------------------------------------------------------- + * objects to filter + *------------------------------------------------------------------------- + */ + + if (options->verbose && have_request(options) /* only print if requested */) + { + printf("Objects to apply filter are...\n"); + if (options->all_filter==1) + { + + for (k = 0; k < options->n_filter_g; k++ ) + { + H5Z_filter_t filtn=options->filter_g[k].filtn; + switch (filtn) + { + case H5Z_FILTER_NONE: + printf(" Uncompress all\n"); + break; + case H5Z_FILTER_SHUFFLE: + case H5Z_FILTER_FLETCHER32: + printf(" All with %s\n",get_sfilter(filtn)); + break; + case H5Z_FILTER_SZIP: + case H5Z_FILTER_DEFLATE: + printf(" All with %s, parameter %d\n", + get_sfilter(filtn), + options->filter_g[k].cd_values[0]); + break; + } /* k */ + }; + } + } /* verbose */ + + for ( i = 0; i < options->op_tbl->nelems; i++) + { + pack_info_t pack = options->op_tbl->objs[i]; + char* name = pack.path; + + for ( j=0; j<pack.nfilters; j++) + { + if (options->verbose) + { + printf(" <%s> with %s filter\n", + name, + get_sfilter(pack.filter[j].filtn)); + } + + has_cp=1; + + } /* j */ + } /* i */ + + if (options->all_filter==1 && has_cp){ + error_msg(progname, "invalid compression input: 'all' option\ + is present with other objects\n"); + return -1; + } + + /* check options for the latest format */ + if (options->grp_compact < 0) { + error_msg(progname, "invalid maximum number of links to store as header messages\n"); + return -1; + } + if (options->grp_indexed < 0) { + error_msg(progname, "invalid minimum number of links to store in the indexed format\n"); + return -1; + } + if (options->grp_indexed > options->grp_compact) { + error_msg(progname, "minimum indexed size is greater than the maximum compact size\n"); + return -1; + } + for (i=0; i<8; i++) { + if (options->msg_size[i]<0) { + error_msg(progname, "invalid shared message size\n"); + return -1; + } + } + + return 0; } + /*------------------------------------------------------------------------- - * Function: read_info + * Function: check_objects * - * Purpose: read comp and chunk options from file + * Purpose: locate all HDF5 objects in the file and compare with user + * supplied list * - * Return: void, exit on error + * Return: 0, ok, -1 no * - * Programmer: pvn@ncsa.uiuc.edu + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu * - * Date: September, 22, 2003 + * Date: September, 23, 2003 * *------------------------------------------------------------------------- */ - -void read_info(const char *filename, - pack_opt_t *options) +static int check_objects(const char* fname, + pack_opt_t *options) { - - char stype[10]; - char comp_info[1024]; - FILE *fp; - char c; - int i, rc=1; - char *srcdir = getenv("srcdir"); /* the source directory */ - char data_file[512]=""; /* buffer to hold name of existing file */ - - /* compose the name of the file to open, using the srcdir, if appropriate */ - if (srcdir){ - strcpy(data_file,srcdir); - strcat(data_file,"/"); - } - strcat(data_file,filename); - - - if ((fp = fopen(data_file, "r")) == (FILE *)NULL) { - error_msg(progname, "cannot open options file %s\n", filename); - exit(1); - } - - /* cycle until end of file reached */ - while( 1 ) - { - rc=fscanf(fp, "%s", stype); - if (rc==-1) - break; - - /*------------------------------------------------------------------------- - * filter - *------------------------------------------------------------------------- - */ - if (strcmp(stype,"-f") == 0) { - - /* find begining of info */ - i=0; c='0'; - while( c!=' ' ) - { - fscanf(fp, "%c", &c); - if (feof(fp)) break; - } - c='0'; - /* go until end */ - while( c!=' ' ) - { - fscanf(fp, "%c", &c); - comp_info[i]=c; - i++; - if (feof(fp)) break; - if (c==10 /*eol*/) break; - } - comp_info[i-1]='\0'; /*cut the last " */ - - if (h5repack_addfilter(comp_info,options)==-1){ - error_msg(progname, "could not add compression option\n"); - exit(1); - } - } - /*------------------------------------------------------------------------- - * layout - *------------------------------------------------------------------------- - */ - else if (strcmp(stype,"-l") == 0) { - - /* find begining of info */ - i=0; c='0'; - while( c!=' ' ) - { - fscanf(fp, "%c", &c); - if (feof(fp)) break; - } - c='0'; - /* go until end */ - while( c!=' ' ) - { - fscanf(fp, "%c", &c); - comp_info[i]=c; - i++; - if (feof(fp)) break; - if (c==10 /*eol*/) break; - } - comp_info[i-1]='\0'; /*cut the last " */ - - if (h5repack_addlayout(comp_info,options)==-1){ - error_msg(progname, "could not add chunck option\n"); - exit(1); - } - } - /*------------------------------------------------------------------------- - * not valid - *------------------------------------------------------------------------- - */ - else { - error_msg(progname, "bad file format for %s", filename); - exit(1); - } - } - - fclose(fp); - return; + hid_t fid; + unsigned int i; + trav_table_t *travt = NULL; + + /* nothing to do */ + if(options->op_tbl->nelems == 0) + return 0; + + /*------------------------------------------------------------------------- + * open the file + *------------------------------------------------------------------------- + */ + if((fid = h5tools_fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT, NULL, NULL, 0)) < 0){ + printf("<%s>: %s\n", fname, H5FOPENERROR ); + return -1; + } + + /*------------------------------------------------------------------------- + * get the list of objects in the file + *------------------------------------------------------------------------- + */ + + /* init table */ + trav_table_init(&travt); + + /* get the list of objects in the file */ + if(h5trav_gettable(fid, travt) < 0) + goto out; + + /*------------------------------------------------------------------------- + * compare with user supplied list + *------------------------------------------------------------------------- + */ + + if(options->verbose) + printf("Opening file <%s>. Searching for objects to modify...\n", fname); + + for(i = 0; i < options->op_tbl->nelems; i++) { + char* name=options->op_tbl->objs[i].path; + if(options->verbose) + printf(" <%s>",name); + + /* the input object names are present in the file and are valid */ + if(h5trav_getindext(name, travt) < 0) { + error_msg(progname, "%s Could not find <%s> in file <%s>. Exiting...\n", + (options->verbose?"\n":""),name,fname); + goto out; + } + if(options->verbose) + printf("...Found\n"); + + /* check for extra filter conditions */ + switch(options->op_tbl->objs[i].filter->filtn) { + /* chunk size must be smaller than pixels per block */ + case H5Z_FILTER_SZIP: + { + int j; + int csize = 1; + int ppb = options->op_tbl->objs[i].filter->cd_values[0]; + hsize_t dims[H5S_MAX_RANK]; + int rank; + hid_t did; + hid_t sid; + + if(options->op_tbl->objs[i].chunk.rank > 0) { + rank = options->op_tbl->objs[i].chunk.rank; + for(j = 0; j < rank; j++) + csize *= (int)options->op_tbl->objs[i].chunk.chunk_lengths[j]; + } + else { + if((did = H5Dopen2(fid, name, H5P_DEFAULT)) < 0) + goto out; + if((sid = H5Dget_space(did)) < 0) + goto out; + if((rank = H5Sget_simple_extent_ndims(sid)) < 0) + goto out; + HDmemset(dims, 0, sizeof dims); + if(H5Sget_simple_extent_dims(sid, dims, NULL) < 0) + goto out; + for(j = 0; j < rank; j++) + csize *= (int)dims[j]; + if(H5Sclose(sid) < 0) + goto out; + if(H5Dclose(did) < 0) + goto out; + } + + if (csize < ppb ) { + printf(" <warning: SZIP settins, chunk size is smaller than pixels per block>\n"); + goto out; + } + } + break; + } + } /* i */ + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + H5Fclose(fid); + trav_table_free(travt); + return 0; + +out: + H5Fclose(fid); + trav_table_free(travt); + return -1; } + + + /*------------------------------------------------------------------------- * Function: have_request * @@ -527,7 +551,7 @@ void read_info(const char *filename, * *------------------------------------------------------------------------- */ -int have_request(pack_opt_t *options) +static int have_request(pack_opt_t *options) { if (options->all_filter || options->all_layout || options->op_tbl->nelems) @@ -536,3 +560,38 @@ int have_request(pack_opt_t *options) return 0; } + + +/*------------------------------------------------------------------------- + * Function: get_sfilter + * + * Purpose: return the filter as a string name + * + * Return: name of filter, exit on error + * + *------------------------------------------------------------------------- + */ + +static const char* get_sfilter(H5Z_filter_t filtn) +{ + if (filtn==H5Z_FILTER_NONE) + return "NONE"; + else if (filtn==H5Z_FILTER_DEFLATE) + return "GZIP"; + else if (filtn==H5Z_FILTER_SZIP) + return "SZIP"; + else if (filtn==H5Z_FILTER_SHUFFLE) + return "SHUFFLE"; + else if (filtn==H5Z_FILTER_FLETCHER32) + return "FLETCHER32"; + else if (filtn==H5Z_FILTER_NBIT) + return "NBIT"; + else if (filtn==H5Z_FILTER_SCALEOFFSET) + return "SOFF"; + else { + error_msg(progname, "input error in filter type\n"); + exit(1); + } + return NULL; +} + diff --git a/tools/h5repack/h5repack.h b/tools/h5repack/h5repack.h index d986cc0..baa888e 100644 --- a/tools/h5repack/h5repack.h +++ b/tools/h5repack/h5repack.h @@ -19,6 +19,8 @@ #include "hdf5.h" #include "h5trav.h" +#include "H5Zprivate.h" /* H5Z_COMMON_CD_VALUES */ + #define H5FOPENERROR "unable to open file" @@ -51,11 +53,13 @@ typedef struct { H5Z_FILTER_SCALEOFFSET 6 , scaleoffset compression */ -#define CDVALUES 2 +/* #define CD_VALUES H5Z_COMMON_CD_VALUES */ +#define CD_VALUES 20 typedef struct { - H5Z_filter_t filtn; /* filter identification number */ - int cd_values[CDVALUES]; /* filter client data values */ + H5Z_filter_t filtn; /* filter identification number */ + unsigned cd_values[CD_VALUES]; /* filter client data values */ + size_t cd_nelmts; /* filter client number of values */ } filter_info_t; /* chunk lengths along each dimension and rank */ @@ -96,7 +100,8 @@ typedef struct { pack_opttbl_t *op_tbl; /*table with all -c and -f options */ int all_layout; /*apply the layout to all objects */ int all_filter; /*apply the filter to all objects */ - filter_info_t filter_g; /*global filter INFO for the ALL case */ + filter_info_t filter_g[H5_REPACK_MAX_NFILTERS]; /*global filter array for the ALL case */ + int n_filter_g; /*number of global filters */ chunk_info_t chunk_g; /*global chunk INFO for the ALL case */ H5D_layout_t layout_g; /*global layout information for the ALL case */ int verbose; /*verbose mode */ @@ -142,58 +147,38 @@ int h5repack_cmpdcpl (const char *fname1, */ -int check_objects(const char* fname, - pack_opt_t *options); +/*------------------------------------------------------------------------- + * copy module + *------------------------------------------------------------------------- + */ -int copy_objects(const char* fnamein, - const char* fnameout, - pack_opt_t *options); +int copy_objects (const char* fnamein, + const char* fnameout, + pack_opt_t *options); int do_copy_refobjs(hid_t fidin, hid_t fidout, trav_table_t *travt, - pack_opt_t *options); /* repack options */ - + pack_opt_t *options); -void read_info(const char *filename,pack_opt_t *options); +/*------------------------------------------------------------------------- + * filters and verify module + *------------------------------------------------------------------------- + */ void init_packobject(pack_info_t *obj); -int print_filters(hid_t dcpl_id); -int have_request(pack_opt_t *options); - /*------------------------------------------------------------------------- - * filters + * filters and copy module *------------------------------------------------------------------------- */ int apply_filters(const char* name, /* object name from traverse list */ int rank, /* rank of dataset */ hsize_t *dims, /* dimensions of dataset */ - hid_t dcpl_id, /* (IN,OUT) dataset creation property list */ + hid_t dcpl_id, /* dataset creation property list */ pack_opt_t *options, /* repack options */ - int *has_filter); /* (OUT) object NAME has a filter */ - -int can_read(const char* name, /* object name from traverse list */ - hid_t dcpl_id, /* dataset creation property list */ - pack_opt_t *options); /* repack options */ - - -/*------------------------------------------------------------------------- - * layout functions - *------------------------------------------------------------------------- - */ - -int has_layout(hid_t dcpl_id, - pack_info_t *obj); - -int layout_this(hid_t dcpl_id, /* DCPL from input object */ - const char* name, /* object name from traverse list */ - pack_opt_t *options, /* repack options */ - pack_info_t *pack /*OUT*/) /* object to apply layout */; - -int apply_layout(hid_t dcpl_id, - pack_info_t *pack); /* info about object */ + int *has_filter); /* (OUT) object NAME has a filter */ /*------------------------------------------------------------------------- @@ -221,15 +206,14 @@ pack_info_t* options_get_object( const char *path, obj_list_t* parse_filter(const char *str, int *n_objs, filter_info_t *filt, - pack_opt_t *options); + pack_opt_t *options, + int *is_glb); obj_list_t* parse_layout(const char *str, int *n_objs, pack_info_t *pack, /* info about object */ pack_opt_t *options); -const char* get_sfilter (H5Z_filter_t filtn); -int parse_number(char *str); diff --git a/tools/h5repack/h5repack.sh.in b/tools/h5repack/h5repack.sh.in index 9fce08a..2e411b0 100755 --- a/tools/h5repack/h5repack.sh.in +++ b/tools/h5repack/h5repack.sh.in @@ -36,7 +36,6 @@ H5DIFF_BIN=`pwd`/$H5DIFF # The path of the h5diff tool binary H5DETECTSZIP=testh5repack_detect_szip # The tool name H5DETECTSZIP_BIN=`pwd`/$H5DETECTSZIP # The path of the tool binary - INFO_FILE=../testfiles/info.h5repack FILE0=h5repack_fill.h5 @@ -428,6 +427,26 @@ TOOLTEST $FILE4 -l dset_chunk:CHUNK=18x13 # Native option TOOLTEST $FILE1 -n + +# long swtiches. use FILE4=h5repack_layout.h5 (no filters) + +arg="$FILE4 --layout CHUNK=20x10 --filter GZIP=1 --threshold=10 --native --latest --compact=8 --indexed=6 --ssize=8[:dtype]" +if test $USE_FILTER_DEFLATE != "yes" ; then + SKIP $arg +else + TOOLTEST $arg +fi + +# several global filters + +arg="$FILE4 --filter GZIP=1 --filter SHUF" +if test $USE_FILTER_DEFLATE != "yes" -o $USE_FILTER_SHUFFLE != "yes" ; then + SKIP $arg +else + TOOLTEST $arg +fi + + if test $nerrors -eq 0 ; then echo "All $H5REPACK tests passed." fi diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c index 6ad97d7..01ccc21 100644 --- a/tools/h5repack/h5repack_copy.c +++ b/tools/h5repack/h5repack_copy.c @@ -29,10 +29,6 @@ extern char *progname; */ #define FORMAT_OBJ " %-27s %s\n" /* obj type, name */ #define FORMAT_OBJ_ATTR " %-27s %s\n" /* obj type, name */ -#define PER(A,B) { per = 0; \ - if (A!=0) \ - per = (double) fabs( (double)(B-A) / (double)A ); \ - } #define USERBLOCK_XFER_SIZE 512 /* Size of buffer/# of bytes to xfer at a time when copying userblock */ /*------------------------------------------------------------------------- @@ -134,23 +130,23 @@ int copy_objects(const char* fnamein, if(options->msg_size[i] > 0) { switch(i) { case 0: - mesg_type_flags[nindex] = H5O_MESG_SDSPACE_FLAG; + mesg_type_flags[nindex] = H5O_SHMESG_SDSPACE_FLAG; break; case 1: - mesg_type_flags[nindex] = H5O_MESG_DTYPE_FLAG; + mesg_type_flags[nindex] = H5O_SHMESG_DTYPE_FLAG; break; case 2: - mesg_type_flags[nindex] = H5O_MESG_FILL_FLAG; + mesg_type_flags[nindex] = H5O_SHMESG_FILL_FLAG; break; case 3: - mesg_type_flags[nindex] = H5O_MESG_PLINE_FLAG; + mesg_type_flags[nindex] = H5O_SHMESG_PLINE_FLAG; break; case 4: - mesg_type_flags[nindex] = H5O_MESG_ATTR_FLAG; + mesg_type_flags[nindex] = H5O_SHMESG_ATTR_FLAG; break; } /* end switch */ min_mesg_sizes[nindex] = (unsigned)options->msg_size[i]; @@ -180,7 +176,7 @@ int copy_objects(const char* fnamein, goto out; } /* end if */ - if(H5Pset_latest_format(fapl, TRUE) < 0) { + if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) { error_msg(progname, "Could not set property for using latest version of the format\n"); goto out; } /* end if */ @@ -355,7 +351,6 @@ int do_copy_objects(hid_t fidin, hsize_t dsize_out; /* output dataset size after filter */ int apply_s; /* flag for apply filter to small dataset sizes */ int apply_f; /* flag for apply filter to return error on H5Dcreate */ - double per; /* percent utilization of storage */ void *buf=NULL; /* buffer for raw data */ void *sm_buf=NULL; /* buffer for raw data */ int has_filter; /* current object has a filter */ @@ -369,7 +364,7 @@ int do_copy_objects(hid_t fidin, if (options->verbose) { printf("-----------------------------------------\n"); - printf(" Type Filter (Ratio) Name\n"); + printf(" Type Filter (%%Savings) Name\n"); printf("-----------------------------------------\n"); } @@ -437,9 +432,6 @@ int do_copy_objects(hid_t fidin, */ case H5TRAV_TYPE_DATASET: - if (options->verbose) - printf(FORMAT_OBJ,"dset",travt->objs[i].name ); - has_filter = 0; /* early detection of references */ @@ -534,7 +526,13 @@ int do_copy_objects(hid_t fidin, H5E_BEGIN_TRY { dset_out = H5Dcreate2(fidout, travt->objs[i].name, wtype_id, f_space_id, H5P_DEFAULT, dcpl_out, H5P_DEFAULT); } H5E_END_TRY; - if(dset_out == FAIL) { + + if(dset_out == FAIL) + { + if(options->verbose) + printf(" warning: could not create dataset <%s>. Applying original settings\n", + travt->objs[i].name); + if((dset_out = H5Dcreate2(fidout, travt->objs[i].name, wtype_id, f_space_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT)) < 0) goto error; apply_f = 0; @@ -665,10 +663,20 @@ int do_copy_objects(hid_t fidin, { if (apply_s && apply_f) { + double per=0; + hssize_t a, b; + /* get the storage size of the input dataset */ dsize_out=H5Dget_storage_size(dset_out); - PER((hssize_t)dsize_in,(hssize_t)dsize_out); - print_dataset_info(dcpl_out,travt->objs[i].name,per*100.0); + + a = dsize_in; b = dsize_out; + if (a!=0) + per = (double) (b-a)/a; + + per = -per; + per *=100; + + print_dataset_info(dcpl_out,travt->objs[i].name,per); } else print_dataset_info(dcpl_id,travt->objs[i].name,0.0); @@ -769,6 +777,11 @@ int do_copy_objects(hid_t fidin, if(H5Dclose(dset_out) < 0) goto error; + + if (options->verbose) + printf(FORMAT_OBJ,"dset",travt->objs[i].name ); + + } /* end do we have request for filter/chunking */ @@ -894,143 +907,143 @@ int copy_attr(hid_t loc_in, pack_opt_t *options ) { - hid_t attr_id=-1; /* attr ID */ - hid_t attr_out=-1; /* attr ID */ - hid_t space_id=-1; /* space ID */ - hid_t ftype_id=-1; /* file type ID */ - hid_t wtype_id=-1; /* read/write type ID */ - size_t msize; /* size of type */ - void *buf=NULL; /* data buffer */ - hsize_t nelmts; /* number of elements in dataset */ - int rank; /* rank of dataset */ - hsize_t dims[H5S_MAX_RANK];/* dimensions of dataset */ - char name[255]; - H5O_info_t oinfo; /* Object info */ - int j; - unsigned u; - - if(H5Oget_info(loc_in, &oinfo) < 0) - goto error; - -/*------------------------------------------------------------------------- - * copy all attributes - *------------------------------------------------------------------------- - */ - - for ( u = 0; u < (unsigned)oinfo.num_attrs; u++) - { - - buf=NULL; - - /* open attribute */ - if((attr_id = H5Aopen_by_idx(loc_in, ".", H5_INDEX_CRT_ORDER, H5_ITER_INC, (hsize_t)u, H5P_DEFAULT, H5P_DEFAULT)) < 0) - goto error; - - /* get name */ - if (H5Aget_name( attr_id, (size_t)255, name ) < 0) - goto error; - - /* get the file datatype */ - if ((ftype_id = H5Aget_type( attr_id )) < 0 ) - goto error; - - /* get the dataspace handle */ - if ((space_id = H5Aget_space( attr_id )) < 0 ) - goto error; - - /* get dimensions */ - if ( (rank = H5Sget_simple_extent_dims(space_id, dims, NULL)) < 0 ) - goto error; - - nelmts=1; - for (j=0; j<rank; j++) - nelmts*=dims[j]; - - if (options->use_native==1) - wtype_id = h5tools_get_native_type(ftype_id); - else - wtype_id = H5Tcopy(ftype_id); - - if ((msize=H5Tget_size(wtype_id))==0) - goto error; - -/*------------------------------------------------------------------------- - * object references are a special case - * we cannot just copy the buffers, but instead we recreate the reference - * this is done on a second sweep of the file that just copies - * the referenced objects - *------------------------------------------------------------------------- - */ - - if (H5T_REFERENCE==H5Tget_class(wtype_id)) - { - ; - } - else - { - /*------------------------------------------------------------------------- - * read to memory - *------------------------------------------------------------------------- - */ - - buf = (void *)HDmalloc((size_t)(nelmts * msize)); - if(buf == NULL) { - error_msg(progname, "cannot read into memory\n" ); - goto error; - } - if(H5Aread(attr_id, wtype_id, buf) < 0) - goto error; - + hid_t attr_id=-1; /* attr ID */ + hid_t attr_out=-1; /* attr ID */ + hid_t space_id=-1; /* space ID */ + hid_t ftype_id=-1; /* file type ID */ + hid_t wtype_id=-1; /* read/write type ID */ + size_t msize; /* size of type */ + void *buf=NULL; /* data buffer */ + hsize_t nelmts; /* number of elements in dataset */ + int rank; /* rank of dataset */ + hsize_t dims[H5S_MAX_RANK];/* dimensions of dataset */ + char name[255]; + H5O_info_t oinfo; /* object info */ + int j; + unsigned u; + + if(H5Oget_info(loc_in, &oinfo) < 0) + goto error; + /*------------------------------------------------------------------------- - * copy + * copy all attributes *------------------------------------------------------------------------- */ - - if((attr_out = H5Acreate2(loc_out, name, ftype_id, space_id, H5P_DEFAULT, H5P_DEFAULT)) < 0) - goto error; - if(H5Awrite(attr_out, wtype_id, buf) < 0) - goto error; - - /*close*/ - if(H5Aclose(attr_out) < 0) - goto error; - - - if(buf) - free(buf); - - } /*H5T_REFERENCE*/ - - - if(options->verbose) - printf(FORMAT_OBJ_ATTR, "attr", name); - -/*------------------------------------------------------------------------- - * close - *------------------------------------------------------------------------- - */ - - if (H5Tclose(ftype_id) < 0) goto error; - if (H5Tclose(wtype_id) < 0) goto error; - if (H5Sclose(space_id) < 0) goto error; - if (H5Aclose(attr_id) < 0) goto error; - - } /* u */ - - - return 0; - + + for ( u = 0; u < (unsigned)oinfo.num_attrs; u++) + { + + buf=NULL; + + /* open attribute */ + if((attr_id = H5Aopen_by_idx(loc_in, ".", H5_INDEX_CRT_ORDER, H5_ITER_INC, (hsize_t)u, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + + /* get name */ + if (H5Aget_name( attr_id, (size_t)255, name ) < 0) + goto error; + + /* get the file datatype */ + if ((ftype_id = H5Aget_type( attr_id )) < 0 ) + goto error; + + /* get the dataspace handle */ + if ((space_id = H5Aget_space( attr_id )) < 0 ) + goto error; + + /* get dimensions */ + if ( (rank = H5Sget_simple_extent_dims(space_id, dims, NULL)) < 0 ) + goto error; + + nelmts=1; + for (j=0; j<rank; j++) + nelmts*=dims[j]; + + if (options->use_native==1) + wtype_id = h5tools_get_native_type(ftype_id); + else + wtype_id = H5Tcopy(ftype_id); + + if ((msize=H5Tget_size(wtype_id))==0) + goto error; + + /*------------------------------------------------------------------------- + * object references are a special case + * we cannot just copy the buffers, but instead we recreate the reference + * this is done on a second sweep of the file that just copies + * the referenced objects + *------------------------------------------------------------------------- + */ + + if (H5T_REFERENCE==H5Tget_class(wtype_id)) + { + ; + } + else + { + /*------------------------------------------------------------------------- + * read to memory + *------------------------------------------------------------------------- + */ + + buf = (void *)HDmalloc((size_t)(nelmts * msize)); + if(buf == NULL) { + error_msg(progname, "cannot read into memory\n" ); + goto error; + } + if(H5Aread(attr_id, wtype_id, buf) < 0) + goto error; + + /*------------------------------------------------------------------------- + * copy + *------------------------------------------------------------------------- + */ + + if((attr_out = H5Acreate2(loc_out, name, ftype_id, space_id, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + if(H5Awrite(attr_out, wtype_id, buf) < 0) + goto error; + + /*close*/ + if(H5Aclose(attr_out) < 0) + goto error; + + + if(buf) + free(buf); + + } /*H5T_REFERENCE*/ + + + if(options->verbose) + printf(FORMAT_OBJ_ATTR, "attr", name); + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + + if (H5Tclose(ftype_id) < 0) goto error; + if (H5Tclose(wtype_id) < 0) goto error; + if (H5Sclose(space_id) < 0) goto error; + if (H5Aclose(attr_id) < 0) goto error; + + } /* u */ + + + return 0; + error: - H5E_BEGIN_TRY { - H5Tclose(ftype_id); - H5Tclose(wtype_id); - H5Sclose(space_id); - H5Aclose(attr_id); - H5Aclose(attr_out); - if (buf) - free(buf); - } H5E_END_TRY; - return -1; + H5E_BEGIN_TRY { + H5Tclose(ftype_id); + H5Tclose(wtype_id); + H5Sclose(space_id); + H5Aclose(attr_id); + H5Aclose(attr_out); + if (buf) + free(buf); + } H5E_END_TRY; + return -1; } @@ -1047,95 +1060,96 @@ static void print_dataset_info(hid_t dcpl_id, char *objname, double per) { - char strfilter[255]; + char strfilter[255]; #if defined (PRINT_DEBUG ) - char temp[255]; + char temp[255]; #endif - int nfilters; /* number of filters */ - unsigned filt_flags; /* filter flags */ - H5Z_filter_t filtn; /* filter identification number */ - unsigned cd_values[20]; /* filter client data values */ - size_t cd_nelmts; /* filter client number of values */ - char f_objname[256]; /* filter objname */ - int i; - - strcpy(strfilter,"\0"); - - /* get information about input filters */ - if((nfilters = H5Pget_nfilters(dcpl_id)) < 0) - return; - - for(i = 0; i < nfilters; i++) { - cd_nelmts = NELMTS(cd_values); - - filtn = H5Pget_filter2(dcpl_id, (unsigned)i, &filt_flags, &cd_nelmts, - cd_values, sizeof(f_objname), f_objname, NULL); - - switch(filtn) { - default: - break; - - case H5Z_FILTER_DEFLATE: - strcat(strfilter,"GZIP "); - + int nfilters; /* number of filters */ + unsigned filt_flags; /* filter flags */ + H5Z_filter_t filtn; /* filter identification number */ + unsigned cd_values[20]; /* filter client data values */ + size_t cd_nelmts; /* filter client number of values */ + char f_objname[256]; /* filter objname */ + int i; + + + strcpy(strfilter,"\0"); + + /* get information about input filters */ + if((nfilters = H5Pget_nfilters(dcpl_id)) < 0) + return; + + for(i = 0; i < nfilters; i++) { + cd_nelmts = NELMTS(cd_values); + + filtn = H5Pget_filter2(dcpl_id, (unsigned)i, &filt_flags, &cd_nelmts, + cd_values, sizeof(f_objname), f_objname, NULL); + + switch(filtn) { + default: + break; + + case H5Z_FILTER_DEFLATE: + strcat(strfilter,"GZIP "); + #if defined (PRINT_DEBUG) - { - unsigned level=cd_values[0]; - sprintf(temp,"(%d)",level); - strcat(strfilter,temp); - } + { + unsigned level=cd_values[0]; + sprintf(temp,"(%d)",level); + strcat(strfilter,temp); + } #endif - break; - - case H5Z_FILTER_SZIP: - strcat(strfilter,"SZIP "); - + break; + + case H5Z_FILTER_SZIP: + strcat(strfilter,"SZIP "); + #if defined (PRINT_DEBUG) - { - unsigned options_mask=cd_values[0]; /* from dcpl, not filt*/ - unsigned ppb=cd_values[1]; - sprintf(temp,"(%d,",ppb); - strcat(strfilter,temp); - if (options_mask & H5_SZIP_EC_OPTION_MASK) - strcpy(temp,"EC) "); - else if (options_mask & H5_SZIP_NN_OPTION_MASK) - strcpy(temp,"NN) "); - } - strcat(strfilter,temp); - + { + unsigned options_mask=cd_values[0]; /* from dcpl, not filt*/ + unsigned ppb=cd_values[1]; + sprintf(temp,"(%d,",ppb); + strcat(strfilter,temp); + if (options_mask & H5_SZIP_EC_OPTION_MASK) + strcpy(temp,"EC) "); + else if (options_mask & H5_SZIP_NN_OPTION_MASK) + strcpy(temp,"NN) "); + } + strcat(strfilter,temp); + #endif - - break; - - case H5Z_FILTER_SHUFFLE: - strcat(strfilter,"SHUF "); - break; - - case H5Z_FILTER_FLETCHER32: - strcat(strfilter,"FLET "); - break; - - case H5Z_FILTER_NBIT: - strcat(strfilter,"NBIT "); - break; - - case H5Z_FILTER_SCALEOFFSET: - strcat(strfilter,"SCALEOFFSET "); - break; - } /* switch */ - }/*i*/ - - if(*strfilter == '\0') - printf(FORMAT_OBJ,"dset",objname ); - else - { - char str[255], temp[20]; - strcpy(str,"dset "); - strcat(str,strfilter); - sprintf(temp," (%.1f%%)",per); - strcat(str,temp); - printf(FORMAT_OBJ,str,objname); - } + + break; + + case H5Z_FILTER_SHUFFLE: + strcat(strfilter,"SHUF "); + break; + + case H5Z_FILTER_FLETCHER32: + strcat(strfilter,"FLET "); + break; + + case H5Z_FILTER_NBIT: + strcat(strfilter,"NBIT "); + break; + + case H5Z_FILTER_SCALEOFFSET: + strcat(strfilter,"SCALEOFFSET "); + break; + } /* switch */ + }/*i*/ + + if(*strfilter == '\0') + printf(FORMAT_OBJ,"dset",objname ); + else + { + char str[255], temp[20]; + strcpy(str,"dset "); + strcat(str,strfilter); + sprintf(temp," (%.1f%%)",per); + strcat(str,temp); + printf(FORMAT_OBJ,str,objname); + } } /*------------------------------------------------------------------------- diff --git a/tools/h5repack/h5repack_filters.c b/tools/h5repack/h5repack_filters.c index 0e7a557..98ef07e 100644 --- a/tools/h5repack/h5repack_filters.c +++ b/tools/h5repack/h5repack_filters.c @@ -122,7 +122,7 @@ int aux_assign_obj(const char* name, /* object name from traverse lis { /* assign the global filter */ tmp.nfilters=1; - tmp.filter[0]=options->filter_g; + tmp.filter[0]=options->filter_g[0]; } /* if all */ else { @@ -144,9 +144,12 @@ int aux_assign_obj(const char* name, /* object name from traverse lis if (options->all_filter) { - /* assign the global filter */ - tmp.nfilters=1; - tmp.filter[0]=options->filter_g; + int k; + + /* assign the global filters */ + tmp.nfilters=options->n_filter_g; + for ( k = 0; k < options->n_filter_g; k++) + tmp.filter[k]=options->filter_g[k]; } if (options->all_layout) { @@ -168,8 +171,7 @@ int aux_assign_obj(const char* name, /* object name from traverse lis *obj = tmp; return 1; -} - +} /*------------------------------------------------------------------------- @@ -191,7 +193,7 @@ int aux_assign_obj(const char* name, /* object name from traverse lis int apply_filters(const char* name, /* object name from traverse list */ int rank, /* rank of dataset */ hsize_t *dims, /* dimensions of dataset */ - hid_t dcpl_id, /* (IN,OUT) dataset creation property list */ + hid_t dcpl_id, /* dataset creation property list */ pack_opt_t *options, /* repack options */ int *has_filter) /* (OUT) object NAME has a filter */ @@ -262,12 +264,12 @@ int apply_filters(const char* name, /* object name from traverse list */ /*------------------------------------------------------------------------- * the type of filter and additional parameter * type can be one of the filters - * H5Z_FILTER_NONE 0, uncompress if compressed - * H5Z_FILTER_DEFLATE 1 , deflation like gzip - * H5Z_FILTER_SHUFFLE 2 , shuffle the data - * H5Z_FILTER_FLETCHER32 3 , fletcher32 checksum of EDC - * H5Z_FILTER_SZIP 4 , szip compression - * H5Z_FILTER_NBIT 5 , nbit compression + * H5Z_FILTER_NONE 0 , uncompress if compressed + * H5Z_FILTER_DEFLATE 1 , deflation like gzip + * H5Z_FILTER_SHUFFLE 2 , shuffle the data + * H5Z_FILTER_FLETCHER32 3 , fletcher32 checksum of EDC + * H5Z_FILTER_SZIP 4 , szip compression + * H5Z_FILTER_NBIT 5 , nbit compression * H5Z_FILTER_SCALEOFFSET 6 , scaleoffset compression *------------------------------------------------------------------------- */ @@ -275,10 +277,10 @@ int apply_filters(const char* name, /* object name from traverse list */ if (obj.nfilters) { - /*------------------------------------------------------------------------- + /*------------------------------------------------------------------------- * filters require CHUNK layout; if we do not have one define a default *------------------------------------------------------------------------- - */ + */ if (obj.layout==-1) { obj.chunk.rank=rank; @@ -412,48 +414,3 @@ int apply_filters(const char* name, /* object name from traverse list */ return 0; } -/*------------------------------------------------------------------------- - * Function: print_filters - * - * Purpose: print the filters in DCPL - * - * Return: 0, ok, -1 no - * - *------------------------------------------------------------------------- - */ - -int print_filters(hid_t dcpl_id) -{ - int nfilters; /* number of filters */ - unsigned filt_flags; /* filter flags */ - H5Z_filter_t filtn; /* filter identification number */ - unsigned cd_values[20]; /* filter client data values */ - size_t cd_nelmts; /* filter client number of values */ - size_t cd_num; /* filter client data counter */ - char f_name[256]; /* filter name */ - char s[64]; /* temporary string buffer */ - int i; - - /* get information about filters */ - if((nfilters = H5Pget_nfilters(dcpl_id)) < 0) - return -1; - - for(i = 0; i < nfilters; i++) { - cd_nelmts = NELMTS(cd_values); - filtn = H5Pget_filter2(dcpl_id, (unsigned)i, &filt_flags, &cd_nelmts, - cd_values, sizeof(f_name), f_name, NULL); - - f_name[sizeof(f_name)-1] = '\0'; - sprintf(s, "Filter-%d:", i); - printf(" %-10s %s-%u %s {", s, - f_name[0] ? f_name : "method", - (unsigned)filtn, - filt_flags & H5Z_FLAG_OPTIONAL?"OPT":""); - for(cd_num = 0; cd_num < cd_nelmts; cd_num++) - printf("%s%u", cd_num?", ":"", cd_values[cd_num]); - printf("}\n"); - } - - return 0; -} - diff --git a/tools/h5repack/h5repack_list.c b/tools/h5repack/h5repack_list.c deleted file mode 100644 index 4e1e072..0000000 --- a/tools/h5repack/h5repack_list.c +++ /dev/null @@ -1,153 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Copyright by The HDF Group. * - * Copyright by the Board of Trustees of the University of Illinois. * - * All rights reserved. * - * * - * This file is part of HDF5. The full HDF5 copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the files COPYING and Copyright.html. COPYING can be found at the root * - * of the source code distribution tree; Copyright.html can be found at the * - * root level of an installed copy of the electronic HDF5 document set and * - * is linked from the top-level documents page. It can also be found at * - * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * - * access to either file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include <stdlib.h> -#include <string.h> -#include <stdio.h> -#include "h5repack.h" -#include "H5private.h" -#include "h5tools_utils.h" -#include "h5diff.h" -#include "h5tools.h" - -extern char *progname; - -/*------------------------------------------------------------------------- - * Function: check_objects - * - * Purpose: locate all HDF5 objects in the file and compare with user - * supplied list - * - * Return: 0, ok, -1 no - * - * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu - * - * Date: September, 23, 2003 - * - *------------------------------------------------------------------------- - */ -int check_objects(const char* fname, - pack_opt_t *options) -{ - hid_t fid; - unsigned int i; - trav_table_t *travt = NULL; - - /* nothing to do */ - if(options->op_tbl->nelems == 0) - return 0; - - /*------------------------------------------------------------------------- - * open the file - *------------------------------------------------------------------------- - */ - if((fid = h5tools_fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT, NULL, NULL, 0)) < 0){ - printf("<%s>: %s\n", fname, H5FOPENERROR ); - return -1; - } - - /*------------------------------------------------------------------------- - * get the list of objects in the file - *------------------------------------------------------------------------- - */ - - /* init table */ - trav_table_init(&travt); - - /* get the list of objects in the file */ - if(h5trav_gettable(fid, travt) < 0) - goto out; - - /*------------------------------------------------------------------------- - * compare with user supplied list - *------------------------------------------------------------------------- - */ - - if(options->verbose) - printf("Opening file <%s>. Searching for objects to modify...\n", fname); - - for(i = 0; i < options->op_tbl->nelems; i++) { - char* name=options->op_tbl->objs[i].path; - if(options->verbose) - printf(" <%s>",name); - - /* the input object names are present in the file and are valid */ - if(h5trav_getindext(name, travt) < 0) { - error_msg(progname, "%s Could not find <%s> in file <%s>. Exiting...\n", - (options->verbose?"\n":""),name,fname); - goto out; - } - if(options->verbose) - printf("...Found\n"); - - /* check for extra filter conditions */ - switch(options->op_tbl->objs[i].filter->filtn) { - /* chunk size must be smaller than pixels per block */ - case H5Z_FILTER_SZIP: - { - int j; - int csize = 1; - int ppb = options->op_tbl->objs[i].filter->cd_values[0]; - hsize_t dims[H5S_MAX_RANK]; - int rank; - hid_t did; - hid_t sid; - - if(options->op_tbl->objs[i].chunk.rank > 0) { - rank = options->op_tbl->objs[i].chunk.rank; - for(j = 0; j < rank; j++) - csize *= (int)options->op_tbl->objs[i].chunk.chunk_lengths[j]; - } - else { - if((did = H5Dopen2(fid, name, H5P_DEFAULT)) < 0) - goto out; - if((sid = H5Dget_space(did)) < 0) - goto out; - if((rank = H5Sget_simple_extent_ndims(sid)) < 0) - goto out; - HDmemset(dims, 0, sizeof dims); - if(H5Sget_simple_extent_dims(sid, dims, NULL) < 0) - goto out; - for(j = 0; j < rank; j++) - csize *= (int)dims[j]; - if(H5Sclose(sid) < 0) - goto out; - if(H5Dclose(did) < 0) - goto out; - } - - if (csize < ppb ) { - printf(" <warning: SZIP settins, chunk size is smaller than pixels per block>\n"); - goto out; - } - } - break; - } - } /* i */ - - /*------------------------------------------------------------------------- - * close - *------------------------------------------------------------------------- - */ - H5Fclose(fid); - trav_table_free(travt); - return 0; - -out: - H5Fclose(fid); - trav_table_free(travt); - return -1; -} - diff --git a/tools/h5repack/h5repack_main.c b/tools/h5repack/h5repack_main.c index 6f65bc9..17056c4 100644 --- a/tools/h5repack/h5repack_main.c +++ b/tools/h5repack/h5repack_main.c @@ -15,18 +15,16 @@ #include <stdlib.h> #include <string.h> +#include <ctype.h> + #include "h5tools_utils.h" #include "h5repack.h" -#if 0 -#define OLD -#endif -#if defined (OLD) -static void usage(void); -#else static void usage(const char *prog); -#endif +static int parse_number(char *str); +static void parse_command_line(int argc, const char* argv[], pack_opt_t* options); +static void read_info(const char *filename,pack_opt_t *options); /* module-scoped variables */ @@ -54,161 +52,199 @@ static struct long_options l_opts[] = { { NULL, 0, '\0' } }; + + + /*------------------------------------------------------------------------- - * Function: parse_command_line + * Function: main * - * Purpose: parse command line input + * Purpose: h5repack main program + * + * Return: 1, error, 0, no error + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: May 9, 2003 + * + * Comments: + * + * Modifications: + * July 2004: Introduced the extra EC or NN option for SZIP + * October 2006: Added a new switch -n, that allows to write the dataset + * using a native type. The default to write is the file type. * + * Modification: + * Peter Cao, June 13, 2007 + * Add "-L, --latest" option to pack a file with the latest file format + * PVN, November 19, 2007 + * adopted the syntax h5repack [OPTIONS] file1 file2 + * PVN, November 28, 2007 + * added support for multiple global filters *------------------------------------------------------------------------- */ - -#if 0 -#define OLD -#endif - -#if defined (OLD) - - -void parse_command_line(int argc, - const char* argv[], - char** fname1, - char** fname2, - pack_opt_t* options) +int main(int argc, char **argv) { + char *infile = NULL; + char *outfile = NULL; + pack_opt_t options; /*the global options */ + int ret; - int i; + /* initialize options */ + h5repack_init (&options,0); + + parse_command_line(argc, argv, &options); - if (argc<2) + if ( argv[ opt_ind ] != NULL && argv[ opt_ind + 1 ] != NULL ) { - usage(); - exit(1); + infile = argv[ opt_ind ]; + outfile = argv[ opt_ind + 1 ]; + + if ( strcmp( infile, outfile ) == 0 ) + { + error_msg(progname, "file names cannot be the same\n"); + usage(progname); + exit(EXIT_FAILURE); + + } } - for ( i = 1; i < argc; i++) + else { - if (strcmp(argv[i], "-h") == 0) { - usage(); - exit(0); - } - else if (strcmp(argv[i], "-V") == 0) { - print_version(progname); - exit(0); - } - else if (strcmp(argv[i], "-i") == 0) { - *fname1 = argv[++i]; - } - else if (strcmp(argv[i], "-o") == 0) { - *fname2 = argv[++i]; - } - else if (strcmp(argv[i], "-v") == 0) { - options->verbose = 1; - } - else if (strcmp(argv[i], "-f") == 0) { - - /* add the -f filter option */ - if (h5repack_addfilter(argv[i+1], options)<0) - { - error_msg(progname, "in parsing filter\n"); - exit(1); - } - - /* jump to next */ - ++i; - } - else if (strcmp(argv[i], "-l") == 0) { - - /* parse the -l layout option */ - if (h5repack_addlayout(argv[i+1], options)<0) - { - error_msg(progname, "in parsing layout\n"); - exit(1); - } - - /* jump to next */ - ++i; - } - - else if (strcmp(argv[i], "-m") == 0) { - options->threshold = parse_number(argv[i+1]); - if ((int)options->threshold==-1) { - error_msg(progname, "invalid treshold size <%s>\n",argv[i+1]); - exit(1); - } - ++i; - } - - else if (strcmp(argv[i], "-e") == 0) { - read_info(argv[++i], options); - } - else if (strcmp(argv[i], "-n") == 0) { - options->use_native = 1; - } - - else if ( (strcmp(argv[i], "-L") == 0) || (strcmp(argv[i], "--latest") == 0)) { - options->latest = 1; - } - - else if ( strncmp(argv[i], "-compact=", 9) == 0 ) { - options->grp_compact = atoi(argv[i]+9); - if (options->grp_compact>0) - options->latest = 1; /* must use latest format */ - } - - else if ( strncmp(argv[i], "-indexed=", 9) == 0 ) { - options->grp_indexed = atoi(argv[i]+9); - if (options->grp_indexed>0) - options->latest = 1; /* must use latest format */ - } - - else if ( strncmp(argv[i], "-ssize=", 7) == 0 ) { - int idx = 0; - int ssize = 0; - char *msgPtr = strchr(argv[i]+7, ':'); - options->latest = 1; /* must use latest format */ - if (msgPtr == NULL) { - ssize = atoi(argv[i]+7); - for (idx=0; idx<5; idx++) - options->msg_size[idx] = ssize; - } - else { - char msgType[10]; - strcpy(msgType, msgPtr+1); - msgPtr[0] = '\0'; - ssize = atoi(argv[i]+7); - if (strcmp(msgType, "dspace") == 0) { - options->msg_size[0] = ssize; - } - else if (strcmp(msgType, "dtype") == 0) { - options->msg_size[1] = ssize; - } - else if (strcmp(msgType, "fill") == 0) { - options->msg_size[2] = ssize; - } - else if (strcmp(msgType, "pline") == 0) { - options->msg_size[3] = ssize; - } - else if (strcmp(msgType, "attr") == 0) { - options->msg_size[4] = ssize; - } - } - } /* else if ( strncmp(argv[i], "-ssize=", 7) == 0 ) */ - - else if (argv[i][0] == '-') { - error_msg(progname, " - is not a valid argument\n"); - usage(); - exit(1); - } - - } - + error_msg(progname, "file names missing\n"); + usage(progname); + exit(EXIT_FAILURE); + } + + + /* pack it */ + ret=h5repack(infile,outfile,&options); + + /* free tables */ + h5repack_end(&options); + + if (ret==-1) + return 1; + else + return 0; +} + + + +/*------------------------------------------------------------------------- + * Function: usage + * + * Purpose: print usage + * + * Return: void + * + *------------------------------------------------------------------------- + */ +static void usage(const char *prog) +{ + printf("usage: %s [OPTIONS] file1 file2\n", prog); + printf(" file1 Input HDF5 File\n"); + printf(" file2 Output HDF5 File\n"); + printf(" OPTIONS\n"); + printf(" -h, --help Print a usage message and exit\n"); + printf(" -v, --verbose Verbose mode, print object information\n"); + printf(" -V, --version Print version number and exit\n"); + printf(" -n, --native Use a native HDF5 type when repacking\n"); + printf(" -L, --latest Use latest version of file format\n"); + printf(" -c L1, --compact=L1 Maximum number of links in header messages\n"); + printf(" -i L2, --indexed=L2 Minimum number of links in the indexed format\n"); + printf(" -s S[:F], --ssize=S[:F] Shared object header message minimum size\n"); + printf(" -m T, --threshold=T Do not apply the filter to datasets smaller than T\n"); + printf(" -e M, --file=M Name of file M with the -f and -l options\n"); + printf(" -f FILT, --filter=FILT Filter type\n"); + printf(" -l LAYT, --layout=LAYT Layout type\n"); + printf("\n"); + + printf(" T - is an integer greater than 1, size of dataset in bytes \n"); + printf(" M - is a filename.\n"); + printf(" F - is the shared object header message type, any of <dspace|dtype|fill|\n"); + printf(" pline|attr>. If F is not specified, S applies to all messages\n"); + + printf("\n"); + + printf(" FILT - is a string with the format:\n"); + printf("\n"); + printf(" <list of objects>:<name of filter>=<filter parameters>\n"); + printf("\n"); + printf(" <list of objects> is a comma separated list of object names, meaning apply\n"); + printf(" compression only to those objects. If no names are specified, the filter\n"); + printf(" is applied to all objects\n"); + printf(" <name of filter> can be:\n"); + printf(" GZIP, to apply the HDF5 GZIP filter (GZIP compression)\n"); + printf(" SZIP, to apply the HDF5 SZIP filter (SZIP compression)\n"); + printf(" SHUF, to apply the HDF5 shuffle filter\n"); + printf(" FLET, to apply the HDF5 checksum filter\n"); + printf(" NBIT, to apply the HDF5 NBIT filter (NBIT compression)\n"); + printf(" SOFF, to apply the HDF5 Scale/Offset filter\n"); + printf(" NONE, to remove all filters\n"); + printf(" <filter parameters> is optional filter parameter information\n"); + printf(" GZIP=<deflation level> from 1-9\n"); + printf(" SZIP=<pixels per block,coding> pixels per block is a even number in\n"); + printf(" 2-32 and coding method is either EC or NN\n"); + printf(" SHUF (no parameter)\n"); + printf(" FLET (no parameter)\n"); + printf(" NBIT (no parameter)\n"); + printf(" SOFF=<scale_factor,scale_type> scale_factor is an integer and scale_type\n"); + printf(" is either IN or DS\n"); + printf(" NONE (no parameter)\n"); + printf("\n"); + printf(" LAYT - is a string with the format:\n"); + printf("\n"); + printf(" <list of objects>:<layout type>=<layout parameters>\n"); + printf("\n"); + printf(" <list of objects> is a comma separated list of object names, meaning that\n"); + printf(" layout information is supplied for those objects. If no names are\n"); + printf(" specified, the layout type is applied to all objects\n"); + printf(" <layout type> can be:\n"); + printf(" CHUNK, to apply chunking layout\n"); + printf(" COMPA, to apply compact layout\n"); + printf(" CONTI, to apply continuous layout\n"); + printf(" <layout parameters> is optional layout information\n"); + printf(" CHUNK=DIM[xDIM...xDIM], the chunk size of each dimension\n"); + printf(" COMPA (no parameter)\n"); + printf(" CONTI (no parameter)\n"); + printf("\n"); + printf("Examples of use:\n"); + printf("\n"); + printf("1) h5repack -v -f GZIP=1 file1 file2\n"); + printf("\n"); + printf(" GZIP compression with level 1 to all objects\n"); + printf("\n"); + printf("2) h5repack -v -f A:SZIP=8,NN file1 file2\n"); + printf("\n"); + printf(" SZIP compression with 8 pixels per block and NN coding method to object A\n"); + printf("\n"); + printf("3) h5repack -v -l A,B:CHUNK=20x10 -f C,D,F:NONE file1 file2\n"); + printf("\n"); + printf(" Chunked layout, with a layout size of 20x10, to objects A and B\n"); + printf(" and remove filters to objects C, D, F\n"); + printf("\n"); + printf("4) h5repack -L -c 10 -s 20:dtype file1 file2 \n"); + printf("\n"); + printf(" Using latest file format with maximum compact group size of 10 and\n"); + printf(" and minimum shared datatype size of 20\n"); + printf("\n"); + printf("5) h5repack -f SHUF -f GZIP=1 file1 file2 \n"); + printf("\n"); + printf(" Add both filters SHUF and GZIP in this order to all datasets\n"); + printf("\n"); } -#else -void parse_command_line(int argc, - const char* argv[], - pack_opt_t* options) +/*------------------------------------------------------------------------- + * Function: parse_command_line + * + * Purpose: parse command line input + * + *------------------------------------------------------------------------- + */ + +static void parse_command_line(int argc, const char* argv[], pack_opt_t* options) { int opt; @@ -336,275 +372,164 @@ void parse_command_line(int argc, usage(progname); exit(EXIT_FAILURE); } - - + } - -#endif - /*------------------------------------------------------------------------- - * Function: main + * Function: parse_number * - * Purpose: h5repack main program + * Purpose: read a number from command line argument * - * Return: 1, error, 0, no error + * Return: number, -1 for FAIL * * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu * - * Date: May 9, 2003 + * Date: September, 23, 2003 * - * Comments: - * - * Modifications: - * July 2004: Introduced the extra EC or NN option for SZIP - * October 2006: Added a new switch -n, that allows to write the dataset - * using a native type. The default to write is the file type. - * - * Modification: - * Peter Cao, June 13, 2007 - * Add "-L, --latest" option to pack a file with the latest file format - * PVN, November 19, 2007 - * adopted the syntax h5repack [OPTIONS] file1 file2 *------------------------------------------------------------------------- */ -int main(int argc, char **argv) -{ - char *infile = NULL; - char *outfile = NULL; - pack_opt_t options; /*the global options */ - int ret; - - /* initialize options */ - h5repack_init (&options,0); -#if defined (OLD) - - parse_command_line(argc, argv, &infile, &outfile, &options); - - -#else - parse_command_line(argc, argv, &options); - - if ( argv[ opt_ind ] != NULL && argv[ opt_ind + 1 ] != NULL ) - { - infile = argv[ opt_ind ]; - outfile = argv[ opt_ind + 1 ]; - } +int parse_number(char *str) +{ + unsigned i; + int n; + char c; + size_t len=strlen(str); - else + for ( i=0; i<len; i++) { - error_msg(progname, "file names missing\n"); - usage(progname); - exit(EXIT_FAILURE); + c = str[i]; + if (!isdigit(c)){ + return -1; + } } - -#endif - - /* pack it */ - ret=h5repack(infile,outfile,&options); - - /* free tables */ - h5repack_end(&options); - - if (ret==-1) - return 1; - else - return 0; + str[i]='\0'; + n=atoi(str); + return n; } - /*------------------------------------------------------------------------- - * Function: usage + * Function: read_info * - * Purpose: print usage + * Purpose: read comp and chunk options from a file * - * Return: void + * Return: void, exit on error + * + * Programmer: pvn@ncsa.uiuc.edu + * + * Date: September, 22, 2003 * *------------------------------------------------------------------------- */ -#if defined OLD -static void usage(void) +static +void read_info(const char *filename, + pack_opt_t *options) { - printf("usage: h5repack [-h] [-v] [-V] [-n] [-L] [-c] [-i] [-s] [-m size] [-e file] [-f FILTER] [-l LAYOUT] file1 file2\n"); - printf("\n"); - printf("file1 Input HDF5 File\n"); - printf("file2 Output HDF5 File\n"); - printf("[-h] Print this message\n"); - printf("[-v] Verbose mode, print object information\n"); - printf("[-V] Print HDF5 version number and exit\n"); - printf("[-n] Use a native HDF5 type when repacking. Default is the file type\n"); - printf("[-L, --latest] Use latest version of file format to create groups, datasets and datatypes\n"); - printf("[-c <size>, --compact=<size>] Set the maximum number of links to store as header messages in a group\n"); - printf("[-i <size>, --indexed=<size>] Set the minimum number of links to store in the indexed format\n"); - printf("[-s, --ssize=<size>[:<dspace|dtype|fill|pline|attr>]] Set the shared object header message minimum size\n"); - printf("[-m size] Do not apply the filter to objects smaller than size\n"); - printf("[-e file] Name of file with the -f and -l options\n"); - printf("[-f FILTER] Filter type\n"); - printf("[-l LAYOUT] Layout type\n"); - printf("\n"); - printf("FILTER is a string with the format:\n"); - printf("\n"); - printf(" <list of objects>:<name of filter>=<filter parameters>\n"); - printf("\n"); - printf(" <list of objects> is a comma separated list of object names, meaning apply\n"); - printf(" compression only to those objects. If no names are specified, the filter\n"); - printf(" is applied to all objects\n"); - printf(" <name of filter> can be:\n"); - printf(" GZIP, to apply the HDF5 GZIP filter (GZIP compression)\n"); - printf(" SZIP, to apply the HDF5 SZIP filter (SZIP compression)\n"); - printf(" SHUF, to apply the HDF5 shuffle filter\n"); - printf(" FLET, to apply the HDF5 checksum filter\n"); - printf(" NBIT, to apply the HDF5 NBIT filter (NBIT compression)\n"); - printf(" SOFF, to apply the HDF5 Scale/Offset filter\n"); - printf(" NONE, to remove all filters\n"); - printf(" <filter parameters> is optional filter parameter information\n"); - printf(" GZIP=<deflation level> from 1-9\n"); - printf(" SZIP=<pixels per block,coding> pixels per block is a even number in\n"); - printf(" 2-32 and coding method is either EC or NN\n"); - printf(" SHUF (no parameter)\n"); - printf(" FLET (no parameter)\n"); - printf(" NBIT (no parameter)\n"); - printf(" SOFF=<scale_factor,scale_type> scale_factor is an integer and scale_type\n"); - printf(" is either IN or DS\n"); - printf(" NONE (no parameter)\n"); - printf("\n"); - printf("LAYOUT is a string with the format:\n"); - printf("\n"); - printf(" <list of objects>:<layout type>=<layout parameters>\n"); - printf("\n"); - printf(" <list of objects> is a comma separated list of object names, meaning that\n"); - printf(" layout information is supplied for those objects. If no names are\n"); - printf(" specified, the layout type is applied to all objects\n"); - printf(" <layout type> can be:\n"); - printf(" CHUNK, to apply chunking layout\n"); - printf(" COMPA, to apply compact layout\n"); - printf(" CONTI, to apply continuous layout\n"); - printf(" <layout parameters> is optional layout information\n"); - printf(" CHUNK=DIM[xDIM...xDIM], the chunk size of each dimension\n"); - printf(" COMPA (no parameter)\n"); - printf(" CONTI (no parameter)\n"); - printf("\n"); - printf("Examples of use:\n"); - printf("\n"); - printf("1) h5repack -v -i file1 -o file2 -f GZIP=1\n"); - printf("\n"); - printf(" GZIP compression with level 1 to all objects\n"); - printf("\n"); - printf("2) h5repack -v -i file1 -o file2 -f A:SZIP=8,NN\n"); - printf("\n"); - printf(" SZIP compression with 8 pixels per block and NN coding method to object A\n"); - printf("\n"); - printf("3) h5repack -v -i file1 -o file2 -l A,B:CHUNK=20x10 -f C,D,F:NONE\n"); - printf("\n"); - printf(" Chunked layout, with a layout size of 20x10, to objects A and B\n"); - printf(" and remove filters to objects C, D, F\n"); - printf("\n"); - printf("4) h5repack -L -compact=10 -ssize=20:dtype -i file1 -o file2 \n"); - printf("\n"); - printf(" Using latest file format with maximum compact group size fo 10 and minimum shared datatype size of 20\n"); - printf("\n"); -} - -#else - -static void usage(const char *prog) -{ - printf("usage: %s [OPTIONS] file1 file2\n", prog); - printf(" file1 Input HDF5 File\n"); - printf(" file2 Output HDF5 File\n"); - printf(" OPTIONS\n"); - printf(" -h, --help Print a usage message and exit\n"); - printf(" -v, --verbose Verbose mode, print object information\n"); - printf(" -V, --version Print version number and exit\n"); - printf(" -n, --native Use a native HDF5 type when repacking\n"); - printf(" -L, --latest Use latest version of file format\n"); - printf(" -c L1, --compact=L1 Maximum number of links in header messages\n"); - printf(" -i L2, --indexed=L2 Minimum number of links in the indexed format\n"); - printf(" -s S[:F], --ssize=S[:F] Shared object header message minimum size\n"); - printf(" -m T, --threshold=T Do not apply the filter to objects smaller than T\n"); - printf(" -e M, --file=M Name of file M with the -f and -l options\n"); - printf(" -f FILT, --filter=FILT Filter type\n"); - printf(" -l LAYT, --layout=LAYT Layout type\n"); - - printf("\n"); - - printf(" T - is an integer greater than 1\n"); - printf(" M - is a filename.\n"); - printf(" F - is the shared object header message type, any of\n"); - printf(" <dspace|dtype|fill|pline|attr>\n"); - - printf("\n"); - - printf(" FILT - is a string with the format:\n"); - printf("\n"); - printf(" <list of objects>:<name of filter>=<filter parameters>\n"); - printf("\n"); - printf(" <list of objects> is a comma separated list of object names, meaning apply\n"); - printf(" compression only to those objects. If no names are specified, the filter\n"); - printf(" is applied to all objects\n"); - printf(" <name of filter> can be:\n"); - printf(" GZIP, to apply the HDF5 GZIP filter (GZIP compression)\n"); - printf(" SZIP, to apply the HDF5 SZIP filter (SZIP compression)\n"); - printf(" SHUF, to apply the HDF5 shuffle filter\n"); - printf(" FLET, to apply the HDF5 checksum filter\n"); - printf(" NBIT, to apply the HDF5 NBIT filter (NBIT compression)\n"); - printf(" SOFF, to apply the HDF5 Scale/Offset filter\n"); - printf(" NONE, to remove all filters\n"); - printf(" <filter parameters> is optional filter parameter information\n"); - printf(" GZIP=<deflation level> from 1-9\n"); - printf(" SZIP=<pixels per block,coding> pixels per block is a even number in\n"); - printf(" 2-32 and coding method is either EC or NN\n"); - printf(" SHUF (no parameter)\n"); - printf(" FLET (no parameter)\n"); - printf(" NBIT (no parameter)\n"); - printf(" SOFF=<scale_factor,scale_type> scale_factor is an integer and scale_type\n"); - printf(" is either IN or DS\n"); - printf(" NONE (no parameter)\n"); - printf("\n"); - printf(" LAYT - is a string with the format:\n"); - printf("\n"); - printf(" <list of objects>:<layout type>=<layout parameters>\n"); - printf("\n"); - printf(" <list of objects> is a comma separated list of object names, meaning that\n"); - printf(" layout information is supplied for those objects. If no names are\n"); - printf(" specified, the layout type is applied to all objects\n"); - printf(" <layout type> can be:\n"); - printf(" CHUNK, to apply chunking layout\n"); - printf(" COMPA, to apply compact layout\n"); - printf(" CONTI, to apply continuous layout\n"); - printf(" <layout parameters> is optional layout information\n"); - printf(" CHUNK=DIM[xDIM...xDIM], the chunk size of each dimension\n"); - printf(" COMPA (no parameter)\n"); - printf(" CONTI (no parameter)\n"); - printf("\n"); - printf("Examples of use:\n"); - printf("\n"); - printf("1) h5repack -v -f GZIP=1 file1 file2\n"); - printf("\n"); - printf(" GZIP compression with level 1 to all objects\n"); - printf("\n"); - printf("2) h5repack -v -f A:SZIP=8,NN file1 file2\n"); - printf("\n"); - printf(" SZIP compression with 8 pixels per block and NN coding method to object A\n"); - printf("\n"); - printf("3) h5repack -v -l A,B:CHUNK=20x10 -f C,D,F:NONE file1 file2\n"); - printf("\n"); - printf(" Chunked layout, with a layout size of 20x10, to objects A and B\n"); - printf(" and remove filters to objects C, D, F\n"); - printf("\n"); - printf("4) h5repack -L -c 10 -s 20:dtype file1 file2 \n"); - printf("\n"); - printf(" Using latest file format with maximum compact group size of 10 and\n"); - printf(" and minimum shared datatype size of 20\n"); - printf("\n"); + + char stype[10]; + char comp_info[1024]; + FILE *fp; + char c; + int i, rc=1; + char *srcdir = getenv("srcdir"); /* the source directory */ + char data_file[512]=""; /* buffer to hold name of existing file */ + + /* compose the name of the file to open, using the srcdir, if appropriate */ + if (srcdir){ + strcpy(data_file,srcdir); + strcat(data_file,"/"); + } + strcat(data_file,filename); + + + if ((fp = fopen(data_file, "r")) == (FILE *)NULL) { + error_msg(progname, "cannot open options file %s\n", filename); + exit(1); + } + + /* cycle until end of file reached */ + while( 1 ) + { + rc=fscanf(fp, "%s", stype); + if (rc==-1) + break; + + /*------------------------------------------------------------------------- + * filter + *------------------------------------------------------------------------- + */ + if (strcmp(stype,"-f") == 0) { + + /* find begining of info */ + i=0; c='0'; + while( c!=' ' ) + { + fscanf(fp, "%c", &c); + if (feof(fp)) break; + } + c='0'; + /* go until end */ + while( c!=' ' ) + { + fscanf(fp, "%c", &c); + comp_info[i]=c; + i++; + if (feof(fp)) break; + if (c==10 /*eol*/) break; + } + comp_info[i-1]='\0'; /*cut the last " */ + + if (h5repack_addfilter(comp_info,options)==-1){ + error_msg(progname, "could not add compression option\n"); + exit(1); + } + } + /*------------------------------------------------------------------------- + * layout + *------------------------------------------------------------------------- + */ + else if (strcmp(stype,"-l") == 0) { + + /* find begining of info */ + i=0; c='0'; + while( c!=' ' ) + { + fscanf(fp, "%c", &c); + if (feof(fp)) break; + } + c='0'; + /* go until end */ + while( c!=' ' ) + { + fscanf(fp, "%c", &c); + comp_info[i]=c; + i++; + if (feof(fp)) break; + if (c==10 /*eol*/) break; + } + comp_info[i-1]='\0'; /*cut the last " */ + + if (h5repack_addlayout(comp_info,options)==-1){ + error_msg(progname, "could not add chunck option\n"); + exit(1); + } + } + /*------------------------------------------------------------------------- + * not valid + *------------------------------------------------------------------------- + */ + else { + error_msg(progname, "bad file format for %s", filename); + exit(1); + } + } + + fclose(fp); + return; } - -#endif - diff --git a/tools/h5repack/h5repack_opttable.c b/tools/h5repack/h5repack_opttable.c index 8492885..4fd450d 100644 --- a/tools/h5repack/h5repack_opttable.c +++ b/tools/h5repack/h5repack_opttable.c @@ -32,19 +32,19 @@ extern char *progname; void init_packobject(pack_info_t *obj) { - int j, k; - - strcpy(obj->path,"\0"); - for ( j=0; j<H5_REPACK_MAX_NFILTERS; j++) - { - obj->filter[j].filtn = -1; - for ( k=0; k<CDVALUES; k++) - obj->filter[j].cd_values[k] = -1; - } - obj->chunk.rank = -1; - obj->refobj_id = -1; - obj->layout = H5D_LAYOUT_ERROR; - obj->nfilters = 0; + int j, k; + + strcpy(obj->path,"\0"); + for ( j=0; j<H5_REPACK_MAX_NFILTERS; j++) + { + obj->filter[j].filtn = -1; + for ( k=0; k<CD_VALUES; k++) + obj->filter[j].cd_values[k] = -1; + } + obj->chunk.rank = -1; + obj->refobj_id = -1; + obj->layout = H5D_LAYOUT_ERROR; + obj->nfilters = 0; } /*------------------------------------------------------------------------- @@ -61,15 +61,15 @@ static void aux_tblinsert_filter(pack_opttbl_t *table, unsigned int I, filter_info_t filt) { - if (table->objs[ I ].nfilters<H5_REPACK_MAX_NFILTERS) - { - table->objs[ I ].filter[ table->objs[ I ].nfilters++ ] = filt; - } - else - { - error_msg(progname, "cannot insert the filter in this object.\ - Maximum capacity exceeded\n"); - } + if (table->objs[ I ].nfilters<H5_REPACK_MAX_NFILTERS) + { + table->objs[ I ].filter[ table->objs[ I ].nfilters++ ] = filt; + } + else + { + error_msg(progname, "cannot insert the filter in this object.\ + Maximum capacity exceeded\n"); + } } /*------------------------------------------------------------------------- @@ -86,26 +86,26 @@ static void aux_tblinsert_layout(pack_opttbl_t *table, unsigned int I, pack_info_t *pack) { - int k; - - table->objs[I].layout = pack->layout; - if (H5D_CHUNKED==pack->layout) - { - /* -2 means the NONE option, remove chunking - and set the layout to contiguous */ - if (pack->chunk.rank==-2) - { - table->objs[I].layout = H5D_CONTIGUOUS; - table->objs[I].chunk.rank = -2; - } - /* otherwise set the chunking type */ - else - { - table->objs[I].chunk.rank = pack->chunk.rank; - for (k = 0; k < pack->chunk.rank; k++) - table->objs[I].chunk.chunk_lengths[k] = pack->chunk.chunk_lengths[k]; - } - } + int k; + + table->objs[I].layout = pack->layout; + if (H5D_CHUNKED==pack->layout) + { + /* -2 means the NONE option, remove chunking + and set the layout to contiguous */ + if (pack->chunk.rank==-2) + { + table->objs[I].layout = H5D_CONTIGUOUS; + table->objs[I].chunk.rank = -2; + } + /* otherwise set the chunking type */ + else + { + table->objs[I].chunk.rank = pack->chunk.rank; + for (k = 0; k < pack->chunk.rank; k++) + table->objs[I].chunk.chunk_lengths[k] = pack->chunk.chunk_lengths[k]; + } + } } @@ -121,19 +121,19 @@ static void aux_tblinsert_layout(pack_opttbl_t *table, static int aux_inctable(pack_opttbl_t *table, int n_objs ) { - unsigned int i; - - table->size += n_objs; - table->objs = (pack_info_t*)realloc(table->objs, table->size * sizeof(pack_info_t)); - if (table->objs==NULL) { - error_msg(progname, "not enough memory for options table\n"); - return -1; - } - for (i = table->nelems; i < table->size; i++) - { - init_packobject(&table->objs[i]); - } - return 0; + unsigned int i; + + table->size += n_objs; + table->objs = (pack_info_t*)realloc(table->objs, table->size * sizeof(pack_info_t)); + if (table->objs==NULL) { + error_msg(progname, "not enough memory for options table\n"); + return -1; + } + for (i = table->nelems; i < table->size; i++) + { + init_packobject(&table->objs[i]); + } + return 0; } /*------------------------------------------------------------------------- @@ -148,28 +148,28 @@ static int aux_inctable(pack_opttbl_t *table, int n_objs ) int options_table_init( pack_opttbl_t **tbl ) { - unsigned int i; - pack_opttbl_t* table = (pack_opttbl_t*) malloc(sizeof(pack_opttbl_t)); - if (table==NULL) { - error_msg(progname, "not enough memory for options table\n"); - return -1; - } - - table->size = 30; - table->nelems = 0; - table->objs = (pack_info_t*) malloc(table->size * sizeof(pack_info_t)); - if (table->objs==NULL) { - error_msg(progname, "not enough memory for options table\n"); - return -1; - } - - for ( i=0; i<table->size; i++) - { - init_packobject(&table->objs[i]); - } - - *tbl = table; - return 0; + unsigned int i; + pack_opttbl_t* table = (pack_opttbl_t*) malloc(sizeof(pack_opttbl_t)); + if (table==NULL) { + error_msg(progname, "not enough memory for options table\n"); + return -1; + } + + table->size = 30; + table->nelems = 0; + table->objs = (pack_info_t*) malloc(table->size * sizeof(pack_info_t)); + if (table->objs==NULL) { + error_msg(progname, "not enough memory for options table\n"); + return -1; + } + + for ( i=0; i<table->size; i++) + { + init_packobject(&table->objs[i]); + } + + *tbl = table; + return 0; } /*------------------------------------------------------------------------- @@ -184,9 +184,9 @@ int options_table_init( pack_opttbl_t **tbl ) int options_table_free( pack_opttbl_t *table ) { - free(table->objs); - free(table); - return 0; + free(table->objs); + free(table); + return 0; } /*------------------------------------------------------------------------- @@ -205,85 +205,85 @@ int options_add_layout( obj_list_t *obj_list, pack_info_t *pack, pack_opttbl_t *table ) { - unsigned int i, I; - int j, added=0, found=0; - - /* increase the size of the collection by N_OBJS if necessary */ - if (table->nelems+n_objs >= table->size) - { - if (aux_inctable(table,n_objs)<0) - return -1; - } - - /* search if this object is already in the table; "path" is the key */ - if (table->nelems>0) - { - /* go tru the supplied list of names */ - for (j = 0; j < n_objs; j++) - { - /* linear table search */ - for (i = 0; i < table->nelems; i++) - { - /*already on the table */ - if (strcmp(obj_list[j].obj,table->objs[i].path)==0) + unsigned int i, I; + int j, added=0, found=0; + + /* increase the size of the collection by N_OBJS if necessary */ + if (table->nelems+n_objs >= table->size) { - /* already chunk info inserted for this one; exit */ - if (table->objs[i].chunk.rank>0) - { - error_msg(progname, "chunk information already inserted for <%s>\n",obj_list[j].obj); - exit(1); - } - /* insert the layout info */ - else - { - aux_tblinsert_layout(table,i,pack); - found=1; - break; - } - } /* if */ - } /* i */ - - if (found==0) - { - /* keep the grow in a temp var */ - I = table->nelems + added; - added++; - strcpy(table->objs[I].path,obj_list[j].obj); - aux_tblinsert_layout(table,I,pack); + if (aux_inctable(table,n_objs)<0) + return -1; } - /* cases where we have an already inserted name but there is a new name also - example: - -f dset1:GZIP=1 -l dset1,dset2:CHUNK=20x20 - dset1 is already inserted, but dset2 must also be - */ - else if (found==1 && strcmp(obj_list[j].obj,table->objs[i].path)!=0) - { - /* keep the grow in a temp var */ - I = table->nelems + added; - added++; - strcpy(table->objs[I].path,obj_list[j].obj); - aux_tblinsert_layout(table,I,pack); - } - } /* j */ - } - - /* first time insertion */ - else - { - /* go tru the supplied list of names */ - for (j = 0; j < n_objs; j++) - { - I = table->nelems + added; - added++; - strcpy(table->objs[I].path,obj_list[j].obj); - aux_tblinsert_layout(table,I,pack); - - } - } - - table->nelems+= added; - - return 0; + + /* search if this object is already in the table; "path" is the key */ + if (table->nelems>0) + { + /* go tru the supplied list of names */ + for (j = 0; j < n_objs; j++) + { + /* linear table search */ + for (i = 0; i < table->nelems; i++) + { + /*already on the table */ + if (strcmp(obj_list[j].obj,table->objs[i].path)==0) + { + /* already chunk info inserted for this one; exit */ + if (table->objs[i].chunk.rank>0) + { + error_msg(progname, "chunk information already inserted for <%s>\n",obj_list[j].obj); + exit(1); + } + /* insert the layout info */ + else + { + aux_tblinsert_layout(table,i,pack); + found=1; + break; + } + } /* if */ + } /* i */ + + if (found==0) + { + /* keep the grow in a temp var */ + I = table->nelems + added; + added++; + strcpy(table->objs[I].path,obj_list[j].obj); + aux_tblinsert_layout(table,I,pack); + } + /* cases where we have an already inserted name but there is a new name also + example: + -f dset1:GZIP=1 -l dset1,dset2:CHUNK=20x20 + dset1 is already inserted, but dset2 must also be + */ + else if (found==1 && strcmp(obj_list[j].obj,table->objs[i].path)!=0) + { + /* keep the grow in a temp var */ + I = table->nelems + added; + added++; + strcpy(table->objs[I].path,obj_list[j].obj); + aux_tblinsert_layout(table,I,pack); + } + } /* j */ + } + + /* first time insertion */ + else + { + /* go tru the supplied list of names */ + for (j = 0; j < n_objs; j++) + { + I = table->nelems + added; + added++; + strcpy(table->objs[I].path,obj_list[j].obj); + aux_tblinsert_layout(table,I,pack); + + } + } + + table->nelems+= added; + + return 0; } @@ -303,76 +303,76 @@ int options_add_filter(obj_list_t *obj_list, filter_info_t filt, pack_opttbl_t *table ) { - - unsigned int i, I; - int j, added=0, found=0; - - /* increase the size of the collection by N_OBJS if necessary */ - if (table->nelems+n_objs >= table->size) - { - if (aux_inctable(table,n_objs)<0) - return -1; - } - - /* search if this object is already in the table; "path" is the key */ - if (table->nelems>0) - { - /* go tru the supplied list of names */ - for (j = 0; j < n_objs; j++) - { - /* linear table search */ - for (i = 0; i < table->nelems; i++) - { - /*already on the table */ - if (strcmp(obj_list[j].obj,table->objs[i].path)==0) + + unsigned int i, I; + int j, added=0, found=0; + + /* increase the size of the collection by N_OBJS if necessary */ + if (table->nelems+n_objs >= table->size) { - /* insert */ - aux_tblinsert_filter(table,i,filt); - found=1; - break; - } /* if */ - } /* i */ - - if (found==0) - { - /* keep the grow in a temp var */ - I = table->nelems + added; - added++; - strcpy(table->objs[I].path,obj_list[j].obj); - aux_tblinsert_filter(table,I,filt); - } - /* cases where we have an already inserted name but there is a new name also - example: - -l dset1:CHUNK=20x20 -f dset1,dset2:GZIP=1 - dset1 is already inserted, but dset2 must also be - */ - else if (found==1 && strcmp(obj_list[j].obj,table->objs[i].path)!=0) - { - /* keep the grow in a temp var */ - I = table->nelems + added; - added++; - strcpy(table->objs[I].path,obj_list[j].obj); - aux_tblinsert_filter(table,I,filt); - } - } /* j */ - } - - /* first time insertion */ - else - { - /* go tru the supplied list of names */ - for (j = 0; j < n_objs; j++) - { - I = table->nelems + added; - added++; - strcpy(table->objs[I].path,obj_list[j].obj); - aux_tblinsert_filter(table,I,filt); - } - } - - table->nelems+= added; - - return 0; + if (aux_inctable(table,n_objs)<0) + return -1; + } + + /* search if this object is already in the table; "path" is the key */ + if (table->nelems>0) + { + /* go tru the supplied list of names */ + for (j = 0; j < n_objs; j++) + { + /* linear table search */ + for (i = 0; i < table->nelems; i++) + { + /*already on the table */ + if (strcmp(obj_list[j].obj,table->objs[i].path)==0) + { + /* insert */ + aux_tblinsert_filter(table,i,filt); + found=1; + break; + } /* if */ + } /* i */ + + if (found==0) + { + /* keep the grow in a temp var */ + I = table->nelems + added; + added++; + strcpy(table->objs[I].path,obj_list[j].obj); + aux_tblinsert_filter(table,I,filt); + } + /* cases where we have an already inserted name but there is a new name also + example: + -l dset1:CHUNK=20x20 -f dset1,dset2:GZIP=1 + dset1 is already inserted, but dset2 must also be + */ + else if (found==1 && strcmp(obj_list[j].obj,table->objs[i].path)!=0) + { + /* keep the grow in a temp var */ + I = table->nelems + added; + added++; + strcpy(table->objs[I].path,obj_list[j].obj); + aux_tblinsert_filter(table,I,filt); + } + } /* j */ + } + + /* first time insertion */ + else + { + /* go tru the supplied list of names */ + for (j = 0; j < n_objs; j++) + { + I = table->nelems + added; + added++; + strcpy(table->objs[I].path,obj_list[j].obj); + aux_tblinsert_filter(table,I,filt); + } + } + + table->nelems+= added; + + return 0; } /*------------------------------------------------------------------------- @@ -388,18 +388,18 @@ int options_add_filter(obj_list_t *obj_list, pack_info_t* options_get_object( const char *path, pack_opttbl_t *table ) { - unsigned int i; - - for ( i = 0; i < table->nelems; i++) - { - /* found it */ - if (strcmp(table->objs[i].path,path)==0) - { - return (&table->objs[i]); - } - } - - return NULL; + unsigned int i; + + for ( i = 0; i < table->nelems; i++) + { + /* found it */ + if (strcmp(table->objs[i].path,path)==0) + { + return (&table->objs[i]); + } + } + + return NULL; } diff --git a/tools/h5repack/h5repack_parse.c b/tools/h5repack/h5repack_parse.c index cf78cac..2475068 100644 --- a/tools/h5repack/h5repack_parse.c +++ b/tools/h5repack/h5repack_parse.c @@ -17,6 +17,7 @@ #include <string.h> #include <stdio.h> #include <ctype.h> + #include "h5repack.h" #include "h5tools_utils.h" @@ -53,407 +54,418 @@ extern char *progname; obj_list_t* parse_filter(const char *str, int *n_objs, filter_info_t *filt, - pack_opt_t *options) + pack_opt_t *options, + int *is_glb) { - unsigned i, u; - char c; - size_t len=strlen(str); - int j, m, n, k, l, end_obj=-1, no_param=0; - char sobj[MAX_NC_NAME]; - char scomp[10]; - char stype[5]; - char smask[3]; - obj_list_t* obj_list=NULL; - unsigned pixels_per_block; - -#if defined(PARSE_DEBUG) - fprintf(stderr,"%s\n",str); -#endif - - /* initialize compression info */ - memset(filt,0,sizeof(filter_info_t)); - - /* check for the end of object list and number of objects */ - for ( i=0, n=0; i<len; i++) - { - c = str[i]; - if ( c==':' ) - { - end_obj=i; - } - if ( c==',' ) - { - n++; - } - } - - if (end_obj==-1) { /* missing : */ - /* apply to all objects */ - options->all_filter=1; - } - - n++; - obj_list=malloc(n*sizeof(obj_list_t)); - if (obj_list==NULL) - { - error_msg(progname, "could not allocate object list\n"); - return NULL; - } - *n_objs=n; - - /* get object list */ - for ( j=0, k=0, n=0; j<end_obj; j++,k++) - { - c = str[j]; - sobj[k]=c; - if ( c==',' || j==end_obj-1) - { - if ( c==',') sobj[k]='\0'; else sobj[k+1]='\0'; - strcpy(obj_list[n].obj,sobj); - memset(sobj,0,sizeof(sobj)); - n++; - k=-1; - } - } - /* nothing after : */ - if (end_obj+1==(int)len) - { - if (obj_list) free(obj_list); - error_msg(progname, "input Error: Invalid compression type in <%s>\n",str); - exit(1); - } - - - /* get filter additional parameters */ - m=0; - for ( i=end_obj+1, k=0, j=0; i<len; i++,k++) - { - c = str[i]; - scomp[k]=c; - if ( c=='=' || i==len-1) - { - if ( c=='=') { /*one more parameter */ - scomp[k]='\0'; /*cut space */ - -/*------------------------------------------------------------------------- - * H5Z_FILTER_SZIP - * szip has the format SZIP=<pixels per block,coding> - * pixels per block is a even number in 2-32 and coding method is 'EC' or 'NN' - * example SZIP=8,NN - *------------------------------------------------------------------------- - */ - if (strcmp(scomp,"SZIP")==0) + unsigned i, u; + char c; + size_t len=strlen(str); + int j, m, n, k, l, end_obj=-1, no_param=0; + char sobj[MAX_NC_NAME]; + char scomp[10]; + char stype[5]; + char smask[3]; + obj_list_t* obj_list=NULL; + unsigned pixels_per_block; + + + /* initialize compression info */ + memset(filt,0,sizeof(filter_info_t)); + *is_glb = 0; + + /* check for the end of object list and number of objects */ + for ( i = 0, n = 0; i < len; i++) { - l=-1; /* mask index check */ - for ( m=0,u=i+1; u<len; u++,m++) - { - if (str[u]==',') - { - stype[m]='\0'; /* end digit of szip */ - l=0; /* start EC or NN search */ - u++; /* skip ',' */ - } - c = str[u]; - if (!isdigit(c) && l==-1){ - if (obj_list) free(obj_list); - error_msg(progname, "compression parameter not digit in <%s>\n",str); - exit(1); - } - if (l==-1) - stype[m]=c; - else - { - smask[l]=c; - l++; - if (l==2) - { - smask[l]='\0'; - i=len-1; /* end */ - (*n_objs)--; /* we counted an extra ',' */ - if (strcmp(smask,"NN")==0) - filt->cd_values[j++]=H5_SZIP_NN_OPTION_MASK; - else if (strcmp(smask,"EC")==0) - filt->cd_values[j++]=H5_SZIP_EC_OPTION_MASK; - else + c = str[i]; + if ( c==':' ) { - error_msg(progname, "szip mask must be 'NN' or 'EC' \n"); - exit(1); + end_obj=i; } - - } - } - - } /* u */ - } /*if */ - -/*------------------------------------------------------------------------- - * H5Z_FILTER_SCALEOFFSET - * scaleoffset has the format SOFF=<scale_factor,scale_type> - * scale_type can be - * integer datatype, H5Z_SO_INT (IN) - * float datatype using D-scaling method, H5Z_SO_FLOAT_DSCALE (DS) - * float datatype using E-scaling method, H5Z_SO_FLOAT_ESCALE (ES) , not yet implemented - * for integer datatypes, scale_factor denotes Minimum Bits - * for float datatypes, scale_factor denotes decimal scale factor - * examples - * SOFF=31,IN - * SOFF=3,DF - *------------------------------------------------------------------------- - */ - - else if (strcmp(scomp,"SOFF")==0) + if ( c==',' ) + { + n++; + } + } + + if (end_obj==-1) /* missing : */ + { + /* apply to all objects */ + options->all_filter=1; + *is_glb = 1; + } + + n++; + obj_list = malloc(n*sizeof(obj_list_t)); + if (obj_list==NULL) { - l=-1; /* mask index check */ - for ( m=0,u=i+1; u<len; u++,m++) - { - if (str[u]==',') - { - stype[m]='\0'; /* end digit */ - l=0; /* start 'IN' , 'DS', or 'ES' search */ - u++; /* skip ',' */ - } - c = str[u]; - if (!isdigit(c) && l==-1){ - if (obj_list) free(obj_list); - error_msg(progname, "compression parameter is not a digit in <%s>\n",str); - exit(1); - } - if (l==-1) - stype[m]=c; - else - { - smask[l]=c; - l++; - if (l==2) - { - smask[l]='\0'; - i=len-1; /* end */ - (*n_objs)--; /* we counted an extra ',' */ - if (strcmp(smask,"IN")==0) - filt->cd_values[j++]=H5Z_SO_INT; - else if (strcmp(smask,"DS")==H5Z_SO_FLOAT_DSCALE) - filt->cd_values[j++]=H5Z_SO_FLOAT_DSCALE; - else + error_msg(progname, "could not allocate object list\n"); + return NULL; + } + *n_objs=n; + + /* get object list */ + for ( j = 0, k = 0, n = 0; j < end_obj; j++, k++) + { + c = str[j]; + sobj[k] = c; + if ( c==',' || j==end_obj-1) { - error_msg(progname, "scale type must be 'IN' or 'DS' \n"); - exit(1); + if ( c==',') sobj[k]='\0'; else sobj[k+1]='\0'; + strcpy(obj_list[n].obj,sobj); + memset(sobj,0,sizeof(sobj)); + n++; + k=-1; } - - } - } - - } /* u */ - } /*if */ - - -/*------------------------------------------------------------------------- - * all other filters - *------------------------------------------------------------------------- - */ - - else + } + /* nothing after : */ + if (end_obj+1==(int)len) { - /* here we could have 1 or 2 digits */ - for ( m=0,u=i+1; u<len; u++,m++) - { - c = str[u]; - if (!isdigit(c)){ - if (obj_list) free(obj_list); - error_msg(progname, "compression parameter is not a digit in <%s>\n",str); - exit(1); - } - stype[m]=c; - } /* u */ - - stype[m]='\0'; - } /*if */ - - - - filt->cd_values[j++]=atoi(stype); - i+=m; /* jump */ + if (obj_list) free(obj_list); + error_msg(progname, "input Error: Invalid compression type in <%s>\n",str); + exit(1); + } + + + /* get filter additional parameters */ + m=0; + for ( i=end_obj+1, k=0, j=0; i<len; i++,k++) + { + c = str[i]; + scomp[k]=c; + if ( c=='=' || i==len-1) + { + if ( c=='=') /*one more parameter */ + { + scomp[k]='\0'; /*cut space */ + + /*------------------------------------------------------------------------- + * H5Z_FILTER_SZIP + * szip has the format SZIP=<pixels per block,coding> + * pixels per block is a even number in 2-32 and coding method is 'EC' or 'NN' + * example SZIP=8,NN + *------------------------------------------------------------------------- + */ + if (strcmp(scomp,"SZIP")==0) + { + l=-1; /* mask index check */ + for ( m=0,u=i+1; u<len; u++,m++) + { + if (str[u]==',') + { + stype[m]='\0'; /* end digit of szip */ + l=0; /* start EC or NN search */ + u++; /* skip ',' */ + } + c = str[u]; + if (!isdigit(c) && l==-1){ + if (obj_list) free(obj_list); + error_msg(progname, "compression parameter not digit in <%s>\n",str); + exit(1); + } + if (l==-1) + stype[m]=c; + else + { + smask[l]=c; + l++; + if (l==2) + { + smask[l]='\0'; + i=len-1; /* end */ + (*n_objs)--; /* we counted an extra ',' */ + if (strcmp(smask,"NN")==0) + filt->cd_values[j++]=H5_SZIP_NN_OPTION_MASK; + else if (strcmp(smask,"EC")==0) + filt->cd_values[j++]=H5_SZIP_EC_OPTION_MASK; + else + { + error_msg(progname, "szip mask must be 'NN' or 'EC' \n"); + exit(1); + } + + + } + } + + } /* u */ + } /*if */ + + /*------------------------------------------------------------------------- + * H5Z_FILTER_SCALEOFFSET + * scaleoffset has the format SOFF=<scale_factor,scale_type> + * scale_type can be + * integer datatype, H5Z_SO_INT (IN) + * float datatype using D-scaling method, H5Z_SO_FLOAT_DSCALE (DS) + * float datatype using E-scaling method, H5Z_SO_FLOAT_ESCALE (ES) , not yet implemented + * for integer datatypes, scale_factor denotes Minimum Bits + * for float datatypes, scale_factor denotes decimal scale factor + * examples + * SOFF=31,IN + * SOFF=3,DF + *------------------------------------------------------------------------- + */ + + else if (strcmp(scomp,"SOFF")==0) + { + l=-1; /* mask index check */ + for ( m=0,u=i+1; u<len; u++,m++) + { + if (str[u]==',') + { + stype[m]='\0'; /* end digit */ + l=0; /* start 'IN' , 'DS', or 'ES' search */ + u++; /* skip ',' */ + } + c = str[u]; + if (!isdigit(c) && l==-1){ + if (obj_list) free(obj_list); + error_msg(progname, "compression parameter is not a digit in <%s>\n",str); + exit(1); + } + if (l==-1) + stype[m]=c; + else + { + smask[l]=c; + l++; + if (l==2) + { + smask[l]='\0'; + i=len-1; /* end */ + (*n_objs)--; /* we counted an extra ',' */ + if (strcmp(smask,"IN")==0) + filt->cd_values[j++]=H5Z_SO_INT; + else if (strcmp(smask,"DS")==H5Z_SO_FLOAT_DSCALE) + filt->cd_values[j++]=H5Z_SO_FLOAT_DSCALE; + else + { + error_msg(progname, "scale type must be 'IN' or 'DS' \n"); + exit(1); + } + + } + } + + } /* u */ + } /*if */ + + + /*------------------------------------------------------------------------- + * all other filters + *------------------------------------------------------------------------- + */ + + else + { + /* here we could have 1 or 2 digits */ + for ( m=0,u=i+1; u<len; u++,m++) + { + c = str[u]; + if (!isdigit(c)){ + if (obj_list) free(obj_list); + error_msg(progname, "compression parameter is not a digit in <%s>\n",str); + exit(1); + } + stype[m]=c; + } /* u */ + + stype[m]='\0'; + } /*if */ + + + + filt->cd_values[j++]=atoi(stype); + i+=m; /* jump */ } - else if (i==len-1) { /*no more parameters */ - scomp[k+1]='\0'; - no_param=1; + else if (i==len-1) + { /*no more parameters */ + scomp[k+1]='\0'; + no_param=1; } -/*------------------------------------------------------------------------- - * H5Z_FILTER_NONE - *------------------------------------------------------------------------- - */ + /*------------------------------------------------------------------------- + * translate from string to filter symbol + *------------------------------------------------------------------------- + */ + + /*------------------------------------------------------------------------- + * H5Z_FILTER_NONE + *------------------------------------------------------------------------- + */ if (strcmp(scomp,"NONE")==0) - filt->filtn=H5Z_FILTER_NONE; - -/*------------------------------------------------------------------------- - * H5Z_FILTER_DEFLATE - *------------------------------------------------------------------------- - */ + { + filt->filtn=H5Z_FILTER_NONE; + filt->cd_nelmts = 0; + } + + /*------------------------------------------------------------------------- + * H5Z_FILTER_DEFLATE + *------------------------------------------------------------------------- + */ else if (strcmp(scomp,"GZIP")==0) { - filt->filtn=H5Z_FILTER_DEFLATE; - if (no_param) { /*no more parameters, GZIP must have parameter */ - if (obj_list) free(obj_list); - error_msg(progname, "missing compression parameter in <%s>\n",str); - exit(1); - } + filt->filtn=H5Z_FILTER_DEFLATE; + filt->cd_nelmts = 1; + if (no_param) + { /*no more parameters, GZIP must have parameter */ + if (obj_list) free(obj_list); + error_msg(progname, "missing compression parameter in <%s>\n",str); + exit(1); + } } - -/*------------------------------------------------------------------------- - * H5Z_FILTER_SZIP - *------------------------------------------------------------------------- - */ + + /*------------------------------------------------------------------------- + * H5Z_FILTER_SZIP + *------------------------------------------------------------------------- + */ else if (strcmp(scomp,"SZIP")==0) { - filt->filtn=H5Z_FILTER_SZIP; - if (no_param) { /*no more parameters, SZIP must have parameter */ - if (obj_list) free(obj_list); - error_msg(progname, "missing compression parameter in <%s>\n",str); - exit(1); - } + filt->filtn=H5Z_FILTER_SZIP; + filt->cd_nelmts = 2; + if (no_param) + { /*no more parameters, SZIP must have parameter */ + if (obj_list) free(obj_list); + error_msg(progname, "missing compression parameter in <%s>\n",str); + exit(1); + } } - -/*------------------------------------------------------------------------- - * H5Z_FILTER_SHUFFLE - *------------------------------------------------------------------------- - */ + + /*------------------------------------------------------------------------- + * H5Z_FILTER_SHUFFLE + *------------------------------------------------------------------------- + */ else if (strcmp(scomp,"SHUF")==0) { - filt->filtn=H5Z_FILTER_SHUFFLE; - if (m>0){ /*shuffle does not have parameter */ - if (obj_list) free(obj_list); - error_msg(progname, "extra parameter in SHUF <%s>\n",str); - exit(1); - } + filt->filtn=H5Z_FILTER_SHUFFLE; + filt->cd_nelmts = 0; + if (m>0) + { /*shuffle does not have parameter */ + if (obj_list) free(obj_list); + error_msg(progname, "extra parameter in SHUF <%s>\n",str); + exit(1); + } } -/*------------------------------------------------------------------------- - * H5Z_FILTER_FLETCHER32 - *------------------------------------------------------------------------- - */ + /*------------------------------------------------------------------------- + * H5Z_FILTER_FLETCHER32 + *------------------------------------------------------------------------- + */ else if (strcmp(scomp,"FLET")==0) { - filt->filtn=H5Z_FILTER_FLETCHER32; - if (m>0){ /*shuffle does not have parameter */ - if (obj_list) free(obj_list); - error_msg(progname, "extra parameter in FLET <%s>\n",str); - exit(1); - } + filt->filtn=H5Z_FILTER_FLETCHER32; + filt->cd_nelmts = 0; + if (m>0) + { /*shuffle does not have parameter */ + if (obj_list) free(obj_list); + error_msg(progname, "extra parameter in FLET <%s>\n",str); + exit(1); + } } -/*------------------------------------------------------------------------- - * H5Z_FILTER_NBIT - *------------------------------------------------------------------------- - */ + /*------------------------------------------------------------------------- + * H5Z_FILTER_NBIT + *------------------------------------------------------------------------- + */ else if (strcmp(scomp,"NBIT")==0) { - filt->filtn=H5Z_FILTER_NBIT; - if (m>0){ /*nbit does not have parameter */ - if (obj_list) free(obj_list); - error_msg(progname, "extra parameter in NBIT <%s>\n",str); - exit(1); - } + filt->filtn=H5Z_FILTER_NBIT; + filt->cd_nelmts = 0; + if (m>0) + { /*nbit does not have parameter */ + if (obj_list) free(obj_list); + error_msg(progname, "extra parameter in NBIT <%s>\n",str); + exit(1); + } } -/*------------------------------------------------------------------------- - * H5Z_FILTER_SCALEOFFSET - *------------------------------------------------------------------------- - */ + /*------------------------------------------------------------------------- + * H5Z_FILTER_SCALEOFFSET + *------------------------------------------------------------------------- + */ else if (strcmp(scomp,"SOFF")==0) { - filt->filtn=H5Z_FILTER_SCALEOFFSET; - if (no_param) { /*no more parameters, SOFF must have parameter */ - if (obj_list) free(obj_list); - error_msg(progname, "missing compression parameter in <%s>\n",str); - exit(1); - } + filt->filtn=H5Z_FILTER_SCALEOFFSET; + filt->cd_nelmts = 2; + if (no_param) + { /*no more parameters, SOFF must have parameter */ + if (obj_list) free(obj_list); + error_msg(progname, "missing compression parameter in <%s>\n",str); + exit(1); + } } else { - if (obj_list) free(obj_list); - error_msg(progname, "invalid filter type in <%s>\n",str); - exit(1); + if (obj_list) free(obj_list); + error_msg(progname, "invalid filter type in <%s>\n",str); + exit(1); } } } /*i*/ + + /*------------------------------------------------------------------------- + * check valid parameters + *------------------------------------------------------------------------- + */ + + switch (filt->filtn) + { -/*------------------------------------------------------------------------- - * check valid parameters - *------------------------------------------------------------------------- - */ - - switch (filt->filtn) - { - - case H5Z_FILTER_DEFLATE: - if (filt->cd_values[0]<0 || filt->cd_values[0]>9 ){ - if (obj_list) free(obj_list); - error_msg(progname, "invalid compression parameter in <%s>\n",str); - exit(1); - } - break; + /*------------------------------------------------------------------------- + * H5Z_FILTER_DEFLATE + *------------------------------------------------------------------------- + */ + + case H5Z_FILTER_DEFLATE: + if (filt->cd_values[0]<0 || filt->cd_values[0]>9 ) + { + if (obj_list) free(obj_list); + error_msg(progname, "invalid compression parameter in <%s>\n",str); + exit(1); + } + break; + + /*------------------------------------------------------------------------- + * H5Z_FILTER_SZIP + *------------------------------------------------------------------------- + */ + + case H5Z_FILTER_SZIP: + pixels_per_block=filt->cd_values[0]; + if ((pixels_per_block%2)==1) + { + if (obj_list) free(obj_list); + error_msg(progname, "pixels_per_block is not even in <%s>\n",str); + exit(1); + } + if (pixels_per_block>H5_SZIP_MAX_PIXELS_PER_BLOCK) + { + if (obj_list) free(obj_list); + error_msg(progname, "pixels_per_block is too large in <%s>\n",str); + exit(1); + } + if ( (strcmp(smask,"NN")!=0) && (strcmp(smask,"EC")!=0) ) + { + if (obj_list) free(obj_list); + error_msg(progname, "szip mask must be 'NN' or 'EC' \n"); + exit(1); + } + break; + /*------------------------------------------------------------------------- + * H5Z_FILTER_SCALEOFFSET + *------------------------------------------------------------------------- + */ - case H5Z_FILTER_SZIP: - pixels_per_block=filt->cd_values[0]; - if ((pixels_per_block%2)==1) { - if (obj_list) free(obj_list); - error_msg(progname, "pixels_per_block is not even in <%s>\n",str); - exit(1); - } - if (pixels_per_block>H5_SZIP_MAX_PIXELS_PER_BLOCK) { - if (obj_list) free(obj_list); - error_msg(progname, "pixels_per_block is too large in <%s>\n",str); - exit(1); - } - if ( (strcmp(smask,"NN")!=0) && (strcmp(smask,"EC")!=0) ) { - if (obj_list) free(obj_list); - error_msg(progname, "szip mask must be 'NN' or 'EC' \n"); - exit(1); - } - break; - case H5Z_FILTER_SCALEOFFSET: - if (filt->cd_values[0]<0 ){ - if (obj_list) free(obj_list); - error_msg(progname, "invalid compression parameter in <%s>\n",str); - exit(1); - } - break; - }; - - return obj_list; + case H5Z_FILTER_SCALEOFFSET: + if (filt->cd_values[0]<0 ) + { + if (obj_list) free(obj_list); + error_msg(progname, "invalid compression parameter in <%s>\n",str); + exit(1); + } + break; + }; + + return obj_list; } -/*------------------------------------------------------------------------- - * Function: get_sfilter - * - * Purpose: return the filter as a string name - * - * Return: name of filter, exit on error - * - *------------------------------------------------------------------------- - */ - -const char* get_sfilter(H5Z_filter_t filtn) -{ - if (filtn==H5Z_FILTER_NONE) - return "NONE"; - else if (filtn==H5Z_FILTER_DEFLATE) - return "GZIP"; - else if (filtn==H5Z_FILTER_SZIP) - return "SZIP"; - else if (filtn==H5Z_FILTER_SHUFFLE) - return "SHUFFLE"; - else if (filtn==H5Z_FILTER_FLETCHER32) - return "FLETCHER32"; - else if (filtn==H5Z_FILTER_NBIT) - return "NBIT"; - else if (filtn==H5Z_FILTER_SCALEOFFSET) - return "SOFF"; - else { - error_msg(progname, "input error in filter type\n"); - exit(1); - } - return NULL; -} - /*------------------------------------------------------------------------- * Function: parse_layout @@ -481,205 +493,171 @@ obj_list_t* parse_layout(const char *str, pack_info_t *pack, /* info about layout needed */ pack_opt_t *options) { - obj_list_t* obj_list=NULL; - unsigned i; - char c; - size_t len=strlen(str); - int j, n, k, end_obj=-1, c_index; - char sobj[MAX_NC_NAME]; - char sdim[10]; - char slayout[10]; - - - memset(sdim, '\0', sizeof(sdim)); - memset(sobj, '\0', sizeof(sobj)); - memset(slayout, '\0', sizeof(slayout)); - - /* check for the end of object list and number of objects */ - for ( i=0, n=0; i<len; i++) - { - c = str[i]; - if ( c==':' ) - { - end_obj=i; - } - if ( c==',' ) - { - n++; - } - } - - if (end_obj==-1) { /* missing : chunk all */ - options->all_layout=1; - } - - n++; - obj_list=malloc(n*sizeof(obj_list_t)); - if (obj_list==NULL) - { - error_msg(progname, "could not allocate object list\n"); - return NULL; - } - *n_objs=n; - - /* get object list */ - for ( j=0, k=0, n=0; j<end_obj; j++,k++) - { - c = str[j]; - sobj[k]=c; - if ( c==',' || j==end_obj-1) - { - if ( c==',') sobj[k]='\0'; else sobj[k+1]='\0'; - strcpy(obj_list[n].obj,sobj); - memset(sobj,0,sizeof(sobj)); - n++; - k=-1; - } - } - - /* nothing after : */ - if (end_obj+1==(int)len) - { - if (obj_list) free(obj_list); - error_msg(progname, "in parse layout, no characters after : in <%s>\n",str); - exit(1); - } - - /* get layout info */ - for ( j=end_obj+1, n=0; n<=5; j++,n++) - { - if (n==5) - { - slayout[n]='\0'; /*cut string */ - if (strcmp(slayout,"COMPA")==0) - pack->layout=H5D_COMPACT; - else if (strcmp(slayout,"CONTI")==0) - pack->layout=H5D_CONTIGUOUS; - else if (strcmp(slayout,"CHUNK")==0) - pack->layout=H5D_CHUNKED; - else { - error_msg(progname, "in parse layout, not a valid layout in <%s>\n",str); - exit(1); - } - } - else - { - c = str[j]; - slayout[n]=c; - } - } /* j */ - - - if ( pack->layout==H5D_CHUNKED ) - { - -/*------------------------------------------------------------------------- - * get chunk info - *------------------------------------------------------------------------- - */ - k=0; - - if (j>(int)len) - { - if (obj_list) free(obj_list); - error_msg(progname, "in parse layout, <%s> Chunk dimensions missing\n",str); - exit(1); - } - - for ( i=j, c_index=0; i<len; i++) - { - c = str[i]; - sdim[k]=c; - k++; /*increment sdim index */ - - if (!isdigit(c) && c!='x' - && c!='N' && c!='O' && c!='N' && c!='E' - ){ - if (obj_list) free(obj_list); - error_msg(progname, "in parse layout, <%s> Not a valid character in <%s>\n", - sdim,str); - exit(1); + obj_list_t* obj_list=NULL; + unsigned i; + char c; + size_t len=strlen(str); + int j, n, k, end_obj=-1, c_index; + char sobj[MAX_NC_NAME]; + char sdim[10]; + char slayout[10]; + + + memset(sdim, '\0', sizeof(sdim)); + memset(sobj, '\0', sizeof(sobj)); + memset(slayout, '\0', sizeof(slayout)); + + /* check for the end of object list and number of objects */ + for ( i=0, n=0; i<len; i++) + { + c = str[i]; + if ( c==':' ) + { + end_obj=i; + } + if ( c==',' ) + { + n++; + } } - - if ( c=='x' || i==len-1) + + if (end_obj==-1) { /* missing : chunk all */ + options->all_layout=1; + } + + n++; + obj_list=malloc(n*sizeof(obj_list_t)); + if (obj_list==NULL) + { + error_msg(progname, "could not allocate object list\n"); + return NULL; + } + *n_objs=n; + + /* get object list */ + for ( j=0, k=0, n=0; j<end_obj; j++,k++) + { + c = str[j]; + sobj[k]=c; + if ( c==',' || j==end_obj-1) + { + if ( c==',') sobj[k]='\0'; else sobj[k+1]='\0'; + strcpy(obj_list[n].obj,sobj); + memset(sobj,0,sizeof(sobj)); + n++; + k=-1; + } + } + + /* nothing after : */ + if (end_obj+1==(int)len) { - if ( c=='x') { - sdim[k-1]='\0'; - k=0; - pack->chunk.chunk_lengths[c_index]=atoi(sdim); - if (pack->chunk.chunk_lengths[c_index]==0) { - if (obj_list) free(obj_list); - error_msg(progname, "in parse layout, <%s> conversion to number in <%s>\n", - sdim,str); - exit(1); - } - c_index++; - } - else if (i==len-1) { /*no more parameters */ - sdim[k]='\0'; - k=0; - if (strcmp(sdim,"NONE")==0) - { - pack->chunk.rank=-2; - } - else - { - pack->chunk.chunk_lengths[c_index]=atoi(sdim); - if (pack->chunk.chunk_lengths[c_index]==0){ if (obj_list) free(obj_list); - error_msg(progname, "in parse layout, <%s> conversion to number in <%s>\n", - sdim,str); + error_msg(progname, "in parse layout, no characters after : in <%s>\n",str); exit(1); - } - pack->chunk.rank=c_index+1; - } - } /*if */ - } /*if c=='x' || i==len-1 */ - } /*i*/ - - - } /*H5D_CHUNKED*/ - - - return obj_list; + } + + /* get layout info */ + for ( j=end_obj+1, n=0; n<=5; j++,n++) + { + if (n==5) + { + slayout[n]='\0'; /*cut string */ + if (strcmp(slayout,"COMPA")==0) + pack->layout=H5D_COMPACT; + else if (strcmp(slayout,"CONTI")==0) + pack->layout=H5D_CONTIGUOUS; + else if (strcmp(slayout,"CHUNK")==0) + pack->layout=H5D_CHUNKED; + else { + error_msg(progname, "in parse layout, not a valid layout in <%s>\n",str); + exit(1); + } + } + else + { + c = str[j]; + slayout[n]=c; + } + } /* j */ + + + if ( pack->layout==H5D_CHUNKED ) + { + + /*------------------------------------------------------------------------- + * get chunk info + *------------------------------------------------------------------------- + */ + k=0; + + if (j>(int)len) + { + if (obj_list) free(obj_list); + error_msg(progname, "in parse layout, <%s> Chunk dimensions missing\n",str); + exit(1); + } + + for ( i=j, c_index=0; i<len; i++) + { + c = str[i]; + sdim[k]=c; + k++; /*increment sdim index */ + + if (!isdigit(c) && c!='x' + && c!='N' && c!='O' && c!='N' && c!='E' + ){ + if (obj_list) free(obj_list); + error_msg(progname, "in parse layout, <%s> Not a valid character in <%s>\n", + sdim,str); + exit(1); + } + + if ( c=='x' || i==len-1) + { + if ( c=='x') { + sdim[k-1]='\0'; + k=0; + pack->chunk.chunk_lengths[c_index]=atoi(sdim); + if (pack->chunk.chunk_lengths[c_index]==0) { + if (obj_list) free(obj_list); + error_msg(progname, "in parse layout, <%s> conversion to number in <%s>\n", + sdim,str); + exit(1); + } + c_index++; + } + else if (i==len-1) { /*no more parameters */ + sdim[k]='\0'; + k=0; + if (strcmp(sdim,"NONE")==0) + { + pack->chunk.rank=-2; + } + else + { + pack->chunk.chunk_lengths[c_index]=atoi(sdim); + if (pack->chunk.chunk_lengths[c_index]==0){ + if (obj_list) free(obj_list); + error_msg(progname, "in parse layout, <%s> conversion to number in <%s>\n", + sdim,str); + exit(1); + } + pack->chunk.rank=c_index+1; + } + } /*if */ + } /*if c=='x' || i==len-1 */ + } /*i*/ + + + } /*H5D_CHUNKED*/ + + + return obj_list; } -/*------------------------------------------------------------------------- - * Function: parse_number - * - * Purpose: read a number from command line argument - * - * Return: number, -1 for FAIL - * - * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu - * - * Date: September, 23, 2003 - * - *------------------------------------------------------------------------- - */ - - -int parse_number(char *str) -{ - unsigned i; - int n; - char c; - size_t len=strlen(str); - - for ( i=0; i<len; i++) - { - c = str[i]; - if (!isdigit(c)){ - return -1; - } - } - str[i]='\0'; - n=atoi(str); - return n; -} - diff --git a/tools/h5repack/h5repack_refs.c b/tools/h5repack/h5repack_refs.c index 64d1990..85af317 100644 --- a/tools/h5repack/h5repack_refs.c +++ b/tools/h5repack/h5repack_refs.c @@ -31,7 +31,7 @@ static const char* MapIdToName(hid_t refobj_id,trav_table_t *travt); static int copy_refs_attr(hid_t loc_in, hid_t loc_out, pack_opt_t *options, - trav_table_t *travt, hid_t fidout); + trav_table_t *travt, hid_t fidout); /*------------------------------------------------------------------------- * Function: do_copy_refobjs diff --git a/tools/h5repack/h5repack_verify.c b/tools/h5repack/h5repack_verify.c index 995aca1..465fdbf 100644 --- a/tools/h5repack/h5repack_verify.c +++ b/tools/h5repack/h5repack_verify.c @@ -18,59 +18,202 @@ #include "h5tools_utils.h" extern char *progname; +static int has_layout(hid_t pid, pack_info_t *obj); +static int has_filters(hid_t pid, hid_t tid, unsigned nfilters, filter_info_t *filter); /*------------------------------------------------------------------------- - * Function: has_filter + * Function: h5repack_verify * - * Purpose: verify if a filter is present in the property list DCPL_ID + * Purpose: verify if filters and layout in the input file match the output file * - * Return: 1 has, 0 does not, -1 error + * Return: + * 1 match + * 0 do not match + * -1 error * - * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * Programmer: Pedro Vicente, pvn@hdfgroup.org * * Date: December 19, 2003 + * Modified: December, 19, 2007 (exactly 4 years later :-) ) + * Separate into 3 cases + * 1) no filter input, get all datasets and compare DCPLs. TO DO + * 2) filter input on selected datasets, get each one trough OBJ and match + * 3) filter input on all datasets, get all objects and match * *------------------------------------------------------------------------- */ -static int -has_filter(hid_t dcpl_id, H5Z_filter_t filtnin) +int h5repack_verify(const char *fname, + pack_opt_t *options) { - int nfilters; /* number of filters */ - unsigned filt_flags; /* filter flags */ - H5Z_filter_t filtn; /* filter identification number */ - unsigned cd_values[20]; /* filter client data values */ - size_t cd_nelmts; /* filter client number of values */ - char f_name[256]; /* filter name */ - int have = 0; /* flag, filter is present */ - int i; /* index */ - - /* if no information about the input filter is requested return exit */ - if(filtnin == -1) - return 1; - - /* get information about filters */ - if((nfilters = H5Pget_nfilters(dcpl_id)) < 0) - return -1; - - /* if we do not have filters and the requested filter is NONE, return 1 */ - if(!nfilters && filtnin == H5Z_FILTER_NONE) - return 1; - - for(i = 0; i < nfilters; i++) { - cd_nelmts = NELMTS(cd_values); - filtn = H5Pget_filter2(dcpl_id, (unsigned)i, &filt_flags, &cd_nelmts, - cd_values, sizeof(f_name), f_name, NULL); - - if(filtnin == filtn) - have = 1; - } - - return have; + hid_t fid; /* file ID */ + hid_t did; /* dataset ID */ + hid_t pid; /* dataset creation property list ID */ + hid_t sid; /* space ID */ + hid_t tid; /* type ID */ + unsigned int i; + trav_table_t *travt = NULL; + int ok = 1; + + /* open the file */ + if((fid = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0 ) + return -1; + + for(i = 0; i < options->op_tbl->nelems; i++) + { + char* name = options->op_tbl->objs[i].path; + pack_info_t *obj = &options->op_tbl->objs[i]; + + /*------------------------------------------------------------------------- + * open + *------------------------------------------------------------------------- + */ + if((did = H5Dopen2(fid, name, H5P_DEFAULT)) < 0) + goto error; + if((sid = H5Dget_space(did)) < 0) + goto error; + if((pid = H5Dget_create_plist(did)) < 0) + goto error; + if((tid = H5Dget_type(did)) < 0) + goto error; + + /*------------------------------------------------------------------------- + * filter check + *------------------------------------------------------------------------- + */ + if(has_filters(pid, tid, obj->nfilters, obj->filter) <= 0) + ok = 0; + + + /*------------------------------------------------------------------------- + * layout check + *------------------------------------------------------------------------- + */ + if((obj->layout != -1) && (has_layout(pid, obj) == 0)) + ok = 0; + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + if(H5Pclose(pid) < 0) + goto error; + if (H5Sclose(sid) < 0) + goto error; + if (H5Dclose(did) < 0) + goto error; + if (H5Tclose(tid) < 0) + goto error; + + } + + + /*------------------------------------------------------------------------- + * check for the "all" objects option + *------------------------------------------------------------------------- + */ + + if(options->all_filter == 1 || options->all_layout == 1) + { + + /* init table */ + trav_table_init(&travt); + + /* get the list of objects in the file */ + if(h5trav_gettable(fid, travt) < 0) + goto error; + + for(i = 0; i < travt->nobjs; i++) + { + char *name = travt->objs[i].name; + + if(travt->objs[i].type == H5TRAV_TYPE_DATASET) + { + + /*------------------------------------------------------------------------- + * open + *------------------------------------------------------------------------- + */ + if((did = H5Dopen2(fid, name, H5P_DEFAULT)) < 0) + goto error; + if((sid = H5Dget_space(did)) < 0) + goto error; + if((pid = H5Dget_create_plist(did)) < 0) + goto error; + if((tid = H5Dget_type(did)) < 0) + goto error; + + /*------------------------------------------------------------------------- + * filter check + *------------------------------------------------------------------------- + */ + if(options->all_filter == 1) + { + + if(has_filters(pid, tid, options->n_filter_g, options->filter_g) <= 0) + ok = 0; + } + + /*------------------------------------------------------------------------- + * layout check + *------------------------------------------------------------------------- + */ + if(options->all_layout == 1) + { + pack_info_t pack; + init_packobject(&pack); + pack.layout = options->layout_g; + pack.chunk = options->chunk_g; + if(has_layout(pid, &pack) == 0) + ok = 0; + } + + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + if (H5Pclose(pid) < 0) + goto error; + if (H5Sclose(sid) < 0) + goto error; + if (H5Dclose(did) < 0) + goto error; + if (H5Tclose(tid) < 0) + goto error; + } /* if */ + + } /* i */ + + /* free table */ + trav_table_free(travt); + } + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + + if (H5Fclose(fid) < 0) + return -1; + + return ok; + +error: + H5E_BEGIN_TRY { + H5Pclose(pid); + H5Sclose(sid); + H5Dclose(did); + H5Fclose(fid); + if (travt) + trav_table_free(travt); + } H5E_END_TRY; + return -1; } + /*------------------------------------------------------------------------- * Function: has_layout * @@ -89,358 +232,341 @@ has_filter(hid_t dcpl_id, H5Z_filter_t filtnin) *------------------------------------------------------------------------- */ -int has_layout(hid_t dcpl_id, +int has_layout(hid_t pid, pack_info_t *obj) { - hsize_t chsize[64]; /* chunk size in elements */ - H5D_layout_t layout; /* layout */ - int nfilters; /* number of filters */ - int rank; /* rank */ - int i; /* index */ - - /* if no information about the input layout is requested return exit */ - if (obj==NULL) - return 1; - - /* check if we have filters in the input object */ - if ((nfilters = H5Pget_nfilters(dcpl_id)) < 0) - return -1; - - /* a non chunked layout was requested on a filtered object; avoid the test */ - if (nfilters && obj->layout!=H5D_CHUNKED) - return 1; - - /* get layout */ - if ((layout = H5Pget_layout(dcpl_id)) < 0) - return -1; - - if (obj->layout != layout) - return 0; - - if (layout==H5D_CHUNKED) - { - if ((rank = H5Pget_chunk(dcpl_id,NELMTS(chsize),chsize/*out*/)) < 0) - return -1; - if (obj->chunk.rank != rank) - return 0; - for ( i=0; i<rank; i++) - if (chsize[i] != obj->chunk.chunk_lengths[i]) - return 0; - } - - return 1; + hsize_t chsize[64]; /* chunk size in elements */ + H5D_layout_t layout; /* layout */ + int nfilters; /* number of filters */ + int rank; /* rank */ + int i; /* index */ + + /* if no information about the input layout is requested return exit */ + if (obj==NULL) + return 1; + + /* check if we have filters in the input object */ + if ((nfilters = H5Pget_nfilters(pid)) < 0) + return -1; + + /* a non chunked layout was requested on a filtered object; avoid the test */ + if (nfilters && obj->layout!=H5D_CHUNKED) + return 1; + + /* get layout */ + if ((layout = H5Pget_layout(pid)) < 0) + return -1; + + if (obj->layout != layout) + return 0; + + if (layout==H5D_CHUNKED) + { + if ((rank = H5Pget_chunk(pid,NELMTS(chsize),chsize/*out*/)) < 0) + return -1; + if (obj->chunk.rank != rank) + return 0; + for ( i=0; i<rank; i++) + if (chsize[i] != obj->chunk.chunk_lengths[i]) + return 0; + } + + return 1; } - /*------------------------------------------------------------------------- - * Function: h5repack_verify + * Function: h5repack_cmpdcpl * - * Purpose: verify if the filters specified in the options list are - * present on the OUTPUT file + * Purpose: compare 2 files for identical property lists of all objects * - * Return: 1=filter present, 0=filter not present, -1=error + * Return: 1=identical, 0=not identical, -1=error * * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu * - * Date: December 19, 2003 + * Date: December 31, 2003 * *------------------------------------------------------------------------- */ -int h5repack_verify(const char *fname, - pack_opt_t *options) +int h5repack_cmpdcpl(const char *fname1, + const char *fname2) { - hid_t fid; /* file ID */ - hid_t dset_id = -1; /* dataset ID */ - hid_t dcpl_id = -1; /* dataset creation property list ID */ - hid_t space_id = -1; /* space ID */ - int ret = 1; - unsigned int i; - int j; - trav_table_t *travt = NULL; - - /* open the file */ - if((fid = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0 ) - return -1; - - for(i = 0; i < options->op_tbl->nelems; i++) { - char* name = options->op_tbl->objs[i].path; - pack_info_t *obj = &options->op_tbl->objs[i]; - -/*------------------------------------------------------------------------- - * open - *------------------------------------------------------------------------- - */ - if((dset_id = H5Dopen2(fid, name, H5P_DEFAULT)) < 0) - goto error; - if((space_id = H5Dget_space(dset_id)) < 0) - goto error; - if((dcpl_id = H5Dget_create_plist(dset_id)) < 0) - goto error; - -/*------------------------------------------------------------------------- - * filter check - *------------------------------------------------------------------------- - */ - for(j = 0; j < obj->nfilters; j++) { - if(has_filter(dcpl_id, obj->filter[j].filtn) == 0) - ret = 0; - } - -/*------------------------------------------------------------------------- - * layout check - *------------------------------------------------------------------------- - */ - if((obj->layout != -1) && (has_layout(dcpl_id, obj) == 0)) - ret = 0; - -/*------------------------------------------------------------------------- - * close - *------------------------------------------------------------------------- - */ - if(H5Pclose(dcpl_id) < 0) - goto error; - if (H5Sclose(space_id) < 0) - goto error; - if (H5Dclose(dset_id) < 0) - goto error; - - } - - -/*------------------------------------------------------------------------- - * check for the "all" objects option - *------------------------------------------------------------------------- - */ - - if(options->all_filter == 1 || options->all_layout == 1) { - - /* init table */ - trav_table_init(&travt); - - /* get the list of objects in the file */ - if(h5trav_gettable(fid, travt) < 0) - goto error; - - for(i = 0; i < travt->nobjs; i++) { - char *name = travt->objs[i].name; - - if(travt->objs[i].type == H5TRAV_TYPE_DATASET) { - - /*------------------------------------------------------------------------- - * open - *------------------------------------------------------------------------- - */ - if((dset_id = H5Dopen2(fid, name, H5P_DEFAULT)) < 0) - goto error; - if((space_id = H5Dget_space(dset_id)) < 0) - goto error; - if((dcpl_id = H5Dget_create_plist(dset_id)) < 0) - goto error; - - /*------------------------------------------------------------------------- - * filter check - *------------------------------------------------------------------------- - */ - if(options->all_filter == 1) - if(has_filter(dcpl_id, options->filter_g.filtn) == 0) - ret = 0; - - /*------------------------------------------------------------------------- - * layout check - *------------------------------------------------------------------------- - */ - if(options->all_layout == 1) { - pack_info_t pack; - init_packobject(&pack); - pack.layout = options->layout_g; - pack.chunk = options->chunk_g; - if(has_layout(dcpl_id, &pack) == 0) - ret = 0; - } - - - /*------------------------------------------------------------------------- - * close - *------------------------------------------------------------------------- - */ - if (H5Pclose(dcpl_id) < 0) - goto error; - if (H5Sclose(space_id) < 0) - goto error; - if (H5Dclose(dset_id) < 0) - goto error; - } /* if */ - - } /* i */ - - /* free table */ - trav_table_free(travt); - } - -/*------------------------------------------------------------------------- - * close - *------------------------------------------------------------------------- - */ - - if (H5Fclose(fid) < 0) - return -1; - - return ret; - + hid_t fid1=-1; /* file ID */ + hid_t fid2=-1; /* file ID */ + hid_t dset1=-1; /* dataset ID */ + hid_t dset2=-1; /* dataset ID */ + hid_t dcpl1=-1; /* dataset creation property list ID */ + hid_t dcpl2=-1; /* dataset creation property list ID */ + trav_table_t *travt1=NULL; + trav_table_t *travt2=NULL; + int ret=1; + unsigned int i; + + /*------------------------------------------------------------------------- + * open the files + *------------------------------------------------------------------------- + */ + + /* disable error reporting */ + H5E_BEGIN_TRY + { + + /* Open the files */ + if ((fid1=H5Fopen(fname1,H5F_ACC_RDONLY,H5P_DEFAULT)) < 0 ) + { + error_msg(progname, "<%s>: %s\n", fname1, H5FOPENERROR ); + return -1; + } + if ((fid2=H5Fopen(fname2,H5F_ACC_RDONLY,H5P_DEFAULT)) < 0 ) + { + error_msg(progname, "<%s>: %s\n", fname2, H5FOPENERROR ); + H5Fclose(fid1); + return -1; + } + /* enable error reporting */ + } H5E_END_TRY; + + /*------------------------------------------------------------------------- + * get file table list of objects + *------------------------------------------------------------------------- + */ + trav_table_init(&travt1); + trav_table_init(&travt2); + if(h5trav_gettable(fid1, travt1) < 0) + goto error; + if(h5trav_gettable(fid2, travt2) < 0) + goto error; + + + /*------------------------------------------------------------------------- + * traverse the suppplied object list + *------------------------------------------------------------------------- + */ + + for(i = 0; i < travt1->nobjs; i++) + { + if(travt1->objs[i].type == H5TRAV_TYPE_DATASET) + { + if((dset1 = H5Dopen2(fid1, travt1->objs[i].name, H5P_DEFAULT)) < 0) + goto error; + if((dset2 = H5Dopen2(fid2, travt1->objs[i].name, H5P_DEFAULT)) < 0) + goto error; + if((dcpl1 = H5Dget_create_plist(dset1)) < 0) + goto error; + if((dcpl2 = H5Dget_create_plist(dset2)) < 0) + goto error; + + /*------------------------------------------------------------------------- + * compare the property lists + *------------------------------------------------------------------------- + */ + if((ret = H5Pequal(dcpl1, dcpl2)) < 0) + goto error; + + if(ret == 0) + { + error_msg(progname, "property lists for <%s> are different\n",travt1->objs[i].name); + goto error; + } + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + if(H5Pclose(dcpl1) < 0) + goto error; + if(H5Pclose(dcpl2) < 0) + goto error; + if(H5Dclose(dset1) < 0) + goto error; + if(H5Dclose(dset2) < 0) + goto error; + } /*if*/ + } /*i*/ + + /*------------------------------------------------------------------------- + * free + *------------------------------------------------------------------------- + */ + + trav_table_free(travt1); + trav_table_free(travt2); + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + + H5Fclose(fid1); + H5Fclose(fid2); + return ret; + + /*------------------------------------------------------------------------- + * error + *------------------------------------------------------------------------- + */ + error: - H5E_BEGIN_TRY { - H5Pclose(dcpl_id); - H5Sclose(space_id); - H5Dclose(dset_id); - H5Fclose(fid); - if (travt) - trav_table_free(travt); - } H5E_END_TRY; - return -1; + H5E_BEGIN_TRY + { + H5Pclose(dcpl1); + H5Pclose(dcpl2); + H5Dclose(dset1); + H5Dclose(dset2); + H5Fclose(fid1); + H5Fclose(fid2); + trav_table_free(travt1); + trav_table_free(travt2); + } H5E_END_TRY; + return -1; + } - /*------------------------------------------------------------------------- - * Function: h5repack_cmpdcpl + * Function: has_filters * - * Purpose: compare 2 files for identical property lists of all objects + * Purpose: verify if all requested filters in the array FILTER obtained + * from user input are present in the property list PID obtained from + * the output file * - * Return: 1=identical, 0=not identical, -1=error + * Return: + * 1 match + * 0 do not match + * -1 error * - * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * Programmer: Pedro Vicente, pvn@hdfgroup.org * - * Date: December 31, 2003 + * Date: December 21, 2007 * *------------------------------------------------------------------------- */ -int h5repack_cmpdcpl(const char *fname1, - const char *fname2) +static int has_filters(hid_t pid, hid_t tid, unsigned nfilters, filter_info_t *filter) { - hid_t fid1=-1; /* file ID */ - hid_t fid2=-1; /* file ID */ - hid_t dset1=-1; /* dataset ID */ - hid_t dset2=-1; /* dataset ID */ - hid_t dcpl1=-1; /* dataset creation property list ID */ - hid_t dcpl2=-1; /* dataset creation property list ID */ - trav_table_t *travt1=NULL; - trav_table_t *travt2=NULL; - int ret=1; - unsigned int i; - -/*------------------------------------------------------------------------- - * open the files first; if they are not valid, no point in continuing - *------------------------------------------------------------------------- - */ - - /* disable error reporting */ - H5E_BEGIN_TRY { - - /* Open the files */ - if ((fid1=H5Fopen(fname1,H5F_ACC_RDONLY,H5P_DEFAULT)) < 0 ) - { - error_msg(progname, "<%s>: %s\n", fname1, H5FOPENERROR ); - return -1; - } - if ((fid2=H5Fopen(fname2,H5F_ACC_RDONLY,H5P_DEFAULT)) < 0 ) - { - error_msg(progname, "<%s>: %s\n", fname2, H5FOPENERROR ); - H5Fclose(fid1); - return -1; - } - /* enable error reporting */ - } H5E_END_TRY; - -/*------------------------------------------------------------------------- - * get file table list of objects - *------------------------------------------------------------------------- - */ - trav_table_init(&travt1); - trav_table_init(&travt2); - if(h5trav_gettable(fid1, travt1) < 0) - goto error; - if(h5trav_gettable(fid2, travt2) < 0) - goto error; - - -/*------------------------------------------------------------------------- - * traverse the suppplied object list - *------------------------------------------------------------------------- - */ - - for(i = 0; i < travt1->nobjs; i++) { - if(travt1->objs[i].type == H5TRAV_TYPE_DATASET) { - if((dset1 = H5Dopen2(fid1, travt1->objs[i].name, H5P_DEFAULT)) < 0) - goto error; - if((dset2 = H5Dopen2(fid2, travt1->objs[i].name, H5P_DEFAULT)) < 0) - goto error; - if((dcpl1 = H5Dget_create_plist(dset1)) < 0) - goto error; - if((dcpl2 = H5Dget_create_plist(dset2)) < 0) - goto error; - -/*------------------------------------------------------------------------- - * compare the property lists - *------------------------------------------------------------------------- - */ - if((ret = H5Pequal(dcpl1, dcpl2)) < 0) - goto error; - - if(ret == 0) { - error_msg(progname, "property lists for <%s> are different\n",travt1->objs[i].name); - goto error; - } - -/*------------------------------------------------------------------------- - * close - *------------------------------------------------------------------------- - */ - if(H5Pclose(dcpl1) < 0) - goto error; - if(H5Pclose(dcpl2) < 0) - goto error; - if(H5Dclose(dset1) < 0) - goto error; - if(H5Dclose(dset2) < 0) - goto error; - } /*if*/ - } /*i*/ - -/*------------------------------------------------------------------------- - * free - *------------------------------------------------------------------------- - */ - - trav_table_free(travt1); - trav_table_free(travt2); - -/*------------------------------------------------------------------------- - * close - *------------------------------------------------------------------------- - */ - - H5Fclose(fid1); - H5Fclose(fid2); - return ret; - -/*------------------------------------------------------------------------- - * error - *------------------------------------------------------------------------- - */ - -error: - H5E_BEGIN_TRY { - H5Pclose(dcpl1); - H5Pclose(dcpl2); - H5Dclose(dset1); - H5Dclose(dset2); - H5Fclose(fid1); - H5Fclose(fid2); - trav_table_free(travt1); - trav_table_free(travt2); - } H5E_END_TRY; - return -1; + unsigned nfilters_dcpl; /* number of filters in DCPL*/ + unsigned filt_flags; /* filter flags */ + H5Z_filter_t filtn; /* filter identification number */ + unsigned cd_values[20]; /* filter client data values */ + size_t cd_nelmts; /* filter client number of values */ + char f_name[256]; /* filter name */ + size_t size; /* type size */ + unsigned i, j; /* index */ + + /* get information about filters */ + if((nfilters_dcpl = H5Pget_nfilters(pid)) < 0) + return -1; + + /* if we do not have filters and the requested filter is NONE, return 1 */ + if(!nfilters_dcpl && + nfilters == 1 && + filter[0].filtn == H5Z_FILTER_NONE ) + return 1; + + /* else the numbers of filters must match */ + if (nfilters_dcpl != nfilters ) + return 0; + + /*------------------------------------------------------------------------- + * build a list with DCPL filters + *------------------------------------------------------------------------- + */ + + for( i = 0; i < nfilters_dcpl; i++) + { + cd_nelmts = NELMTS(cd_values); + filtn = H5Pget_filter2(pid, i, &filt_flags, &cd_nelmts, + cd_values, sizeof(f_name), f_name, NULL); + + /* filter ID */ + if (filtn != filter[i].filtn) + return 0; + + /* compare client data values. some filters do return local values */ + switch (filtn) + { + + case H5Z_FILTER_SHUFFLE: + + /* 1 private client value is returned by DCPL */ + if ( cd_nelmts != H5Z_SHUFFLE_TOTAL_NPARMS && filter[i].cd_nelmts != H5Z_SHUFFLE_USER_NPARMS ) + return 0; + + /* get dataset's type size */ + if((size = H5Tget_size(tid)) <= 0) + return -1; + + /* the private client value holds the dataset's type size */ + if ( size != cd_values[0] ) + return 0; + + + break; + + case H5Z_FILTER_SZIP: + + /* 4 private client values are returned by DCPL */ + if ( cd_nelmts != H5Z_SZIP_TOTAL_NPARMS && filter[i].cd_nelmts != H5Z_SZIP_USER_NPARMS ) + return 0; + + /* "User" parameter for pixels-per-block (index 1) */ + if ( cd_values[H5Z_SZIP_PARM_PPB] != filter[i].cd_values[H5Z_SZIP_PARM_PPB] ) + return 0; + + + break; + + case H5Z_FILTER_NBIT: + + /* only client data values number of values checked */ + if ( H5Z_NBIT_USER_NPARMS != filter[i].cd_nelmts) + return 0; + + + + break; + + case H5Z_FILTER_SCALEOFFSET: + + /* only client data values checked */ + for( j = 0; j < H5Z_SCALEOFFSET_USER_NPARMS; j++) + { + if (cd_values[j] != filter[i].cd_values[j]) + { + return 0; + } + + } + + + break; + + /* for these filters values must match, no local values set in DCPL */ + case H5Z_FILTER_FLETCHER32: + case H5Z_FILTER_DEFLATE: + + if ( cd_nelmts != filter[i].cd_nelmts) + return 0; + + for( j = 0; j < cd_nelmts; j++) + { + if (cd_values[j] != filter[i].cd_values[j]) + { + return 0; + } + + } + + + + break; + + + + } /* switch */ + } + + return 1; } + diff --git a/tools/h5repack/h5repacktst.c b/tools/h5repack/h5repacktst.c index 0e99d4f..99448c4 100644 --- a/tools/h5repack/h5repacktst.c +++ b/tools/h5repack/h5repacktst.c @@ -896,6 +896,8 @@ if (szip_can_encode) { GOERROR; if (h5repack_verify(FNAME7OUT,&pack_options)<=0) GOERROR; + if (h5repack_cmpdcpl(FNAME7,FNAME7OUT)<=0) + GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -1097,6 +1099,7 @@ if (szip_can_encode) { SKIPPED(); #endif + TESTING(" adding nbit filter"); #ifdef H5_HAVE_FILTER_NBIT @@ -1117,6 +1120,8 @@ if (szip_can_encode) { #else SKIPPED(); #endif + + TESTING(" copy of scaleoffset filter"); #ifdef H5_HAVE_FILTER_SCALEOFFSET @@ -1179,6 +1184,8 @@ if (szip_can_encode) { SKIPPED(); #endif + + /*------------------------------------------------------------------------- * file with all filters * dset_all @@ -1355,6 +1362,36 @@ if (szip_can_encode) { /*------------------------------------------------------------------------- + * test several global filters + *------------------------------------------------------------------------- + */ + + TESTING(" several global filters"); + +#if defined (H5_HAVE_FILTER_DEFLATE) && defined (H5_HAVE_FILTER_SHUFFLE) + + if (h5repack_init (&pack_options, 0) < 0) + GOERROR; + if (h5repack_addfilter("GZIP=1",&pack_options) < 0) + GOERROR; + if (h5repack_addfilter("SHUF",&pack_options) < 0) + GOERROR; + if (h5repack(FNAME11,FNAME11OUT,&pack_options) < 0) + GOERROR; + if (h5diff(FNAME11,FNAME11OUT,NULL,NULL,&diff_options) >0) + GOERROR; + if (h5repack_verify(FNAME11OUT,&pack_options)<=0) + GOERROR; + if (h5repack_end (&pack_options) < 0) + GOERROR; + + PASSED(); +#else + SKIPPED(); +#endif + + +/*------------------------------------------------------------------------- * clean temporary test files *------------------------------------------------------------------------- */ |