From f503a7249136b967e010bbccc413a866947b3def Mon Sep 17 00:00:00 2001 From: Pedro Vicente Nunes Date: Mon, 29 Dec 2003 15:26:21 -0500 Subject: [svn-r7994] Purpose: h5repack new features Description: added checking routines for the filters that were applied to the output file added tests for szip filter Solution: Platforms tested: linux solaris (IRIX is not available) : Misc. update: --- tools/h5repack/Makefile.in | 6 +- tools/h5repack/h5repack.c | 160 +++++++------- tools/h5repack/h5repack.h | 148 +++++++++---- tools/h5repack/h5repack_copy.c | 111 ++++++++-- tools/h5repack/h5repack_filters.c | 345 ++++++++++++++++++++++++++++++ tools/h5repack/h5repack_main.c | 2 +- tools/h5repack/h5repack_opttable.c | 56 +++-- tools/h5repack/h5repack_parse.c | 134 ++++++++---- tools/h5repack/h5repack_refs.c | 25 +-- tools/h5repack/h5repack_verify.c | 241 +++++++++++++++++++++ tools/h5repack/testh5repack_dset.c | 3 + tools/h5repack/testh5repack_filters.c | 385 +++++++++------------------------- tools/h5repack/testh5repack_main.c | 106 +++++++++- tools/h5repack/testh5repack_make.c | 318 ++++++++++++++++++++++++++++ 14 files changed, 1532 insertions(+), 508 deletions(-) create mode 100644 tools/h5repack/h5repack_filters.c create mode 100644 tools/h5repack/h5repack_verify.c create mode 100644 tools/h5repack/testh5repack_make.c diff --git a/tools/h5repack/Makefile.in b/tools/h5repack/Makefile.in index dc4d686..5b3cbc0 100644 --- a/tools/h5repack/Makefile.in +++ b/tools/h5repack/Makefile.in @@ -50,10 +50,10 @@ MOSTLYCLEAN=*.h5 ## Source and object files for programs... ## -PROG_SRC=h5repack.c h5repack_copy.c h5repack_refs.c h5repack_list.c h5repack_main.c h5repack_opttable.c h5repack_parse.c testh5repack_attr.c testh5repack_dset.c testh5repack_filters.c testh5repack_main.c +PROG_SRC=h5repack.c h5repack_copy.c h5repack_filters.c h5repack_refs.c h5repack_list.c h5repack_main.c h5repack_opttable.c h5repack_parse.c h5repack_verify.c testh5repack_attr.c testh5repack_dset.c testh5repack_filters.c testh5repack_main.c testh5repack_make.c PROG_OBJ=$(PROG_SRC:.c=.lo) -OBJS=h5repack.lo h5repack_copy.lo h5repack_refs.lo h5repack_list.lo h5repack_main.lo h5repack_opttable.lo h5repack_parse.lo -TEST_OBJS=h5repack.lo h5repack_copy.lo h5repack_refs.lo h5repack_list.lo h5repack_opttable.lo h5repack_parse.lo testh5repack_attr.lo testh5repack_dset.lo testh5repack_filters.lo testh5repack_main.lo +OBJS=h5repack.lo h5repack_filters.lo h5repack_copy.lo h5repack_refs.lo h5repack_list.lo h5repack_main.lo h5repack_opttable.lo h5repack_parse.lo h5repack_verify.lo +TEST_OBJS=h5repack.lo h5repack_filters.lo h5repack_copy.lo h5repack_refs.lo h5repack_list.lo h5repack_opttable.lo h5repack_parse.lo h5repack_verify.lo testh5repack_attr.lo testh5repack_dset.lo testh5repack_filters.lo testh5repack_main.lo testh5repack_make.lo PRIVATE_HDR= diff --git a/tools/h5repack/h5repack.c b/tools/h5repack/h5repack.c index 24a90f7..6ba20bd 100644 --- a/tools/h5repack/h5repack.c +++ b/tools/h5repack/h5repack.c @@ -97,7 +97,7 @@ int h5repack_end (pack_opt_t *options) } /*------------------------------------------------------------------------- - * Function: h5repack_addcomp + * Function: h5repack_addfilter * * Purpose: add a compression -t option to table * Example: -t "*:GZIP 6" , STR = "*:GZIP 6" @@ -107,45 +107,34 @@ int h5repack_end (pack_opt_t *options) *------------------------------------------------------------------------- */ -int h5repack_addcomp(const char* str, - pack_opt_t *options) +int h5repack_addfilter(const char* str, + pack_opt_t *options) { - obj_list_t *obj_list=NULL; /*one object list for the -t and -c option entry */ - comp_info_t comp; /*compression info for the current -t option entry */ - int n_objs; /*number of objects in the current -t or -c option entry */ - int i; + obj_list_t *obj_list=NULL; /*one object list for the -f and -c option entry */ + filter_info_t filt; /*filter info for the current -f option entry */ + int n_objs; /*number of objects in the current -f or -c option entry */ - if (options->all_comp==1){ - printf("Error: Invalid compression input: '*' is present with other objects <%s>\n",str); + if (options->all_filter==1){ + printf("Error: Invalid compression input: all option is present \ + with other objects <%s>\n",str); return -1; } - /* parse the -t option */ - obj_list=parse_comp(str,&n_objs,&comp); + /* parse the -f option */ + obj_list=parse_filter(str,&n_objs,&filt,options); if (obj_list==NULL) - return -1; - - /* searh for the "*" all objects character */ - for (i = 0; i < n_objs; i++) { - if (strcmp("*",obj_list[i].obj)==0) - { - /* if we are compressing all set the global comp type */ - options->all_comp=1; - options->comp_g=comp; - } + return -1; } - if (i>1) + if (options->all_filter==1) { - printf("\nError: '*' cannot be with other objects, <%s>. Exiting...\n",str); - free(obj_list); - options_table_free(options->op_tbl); - return -1; + /* if we are compressing all set the global filter type */ + options->filter_g=filt; } - if (options->all_comp==0) - options_add_comp(obj_list,n_objs,comp,options->op_tbl); + if (options->all_filter==0) + options_add_filter(obj_list,n_objs,filt,options->op_tbl); free(obj_list); return 0; @@ -172,7 +161,7 @@ int h5repack_addchunk(const char* str, int n_objs; /*number of objects in the current -t or -c option entry */ hsize_t chunk_lengths[MAX_VAR_DIMS]; /* chunk lengths along each dimension */ int chunk_rank; /*global rank for chunks */ - int i, j; + int j; if (options->all_chunk==1){ printf("Error: Invalid chunking input: '*' is present with other objects <%s>\n",str); @@ -180,29 +169,16 @@ int h5repack_addchunk(const char* str, } /* parse the -c option */ - obj_list=parse_chunk(str,&n_objs,chunk_lengths,&chunk_rank); + obj_list=parse_chunk(str,&n_objs,chunk_lengths,&chunk_rank,options); if (obj_list==NULL) return -1; - /* searh for the "*" all objects character */ - for (i = 0; i < n_objs; i++) - { - if (strcmp("*",obj_list[i].obj)==0) - { - /* if we are chunking all set the global chunking type */ - options->all_chunk=1; - options->chunk_g.rank=chunk_rank; - for (j = 0; j < chunk_rank; j++) - options->chunk_g.chunk_lengths[j] = chunk_lengths[j]; - } - } - - if (i>1) + if (options->all_chunk==1) { - printf("\nError: '*' cannot be with other objects, <%s>. Exiting...\n",str); - free(obj_list); - options_table_free(options->op_tbl); - return -1; + /* if we are chunking all set the global chunking type */ + options->chunk_g.rank=chunk_rank; + for (j = 0; j < chunk_rank; j++) + options->chunk_g.chunk_lengths[j] = chunk_lengths[j]; } if (options->all_chunk==0) @@ -230,6 +206,9 @@ static int check_options(pack_opt_t *options) { int i, k, j, has_cp=0, has_ck=0; + unsigned szip_options_mask=H5_SZIP_NN_OPTION_MASK; + unsigned szip_pixels_per_block; + /*------------------------------------------------------------------------- * objects to chunk *------------------------------------------------------------------------- @@ -248,12 +227,12 @@ static int check_options(pack_opt_t *options) for ( i = 0; i < options->op_tbl->nelems; i++) { - char* obj_name=options->op_tbl->objs[i].path; + char* name=options->op_tbl->objs[i].path; if (options->op_tbl->objs[i].chunk.rank>0) { if (options->verbose){ - printf("\t<%s> with chunk size ",obj_name); + printf("\t<%s> with chunk size ",name); for ( k = 0; k < options->op_tbl->objs[i].chunk.rank; k++) printf("%d ",(int)options->op_tbl->objs[i].chunk.chunk_lengths[k]); printf("\n"); @@ -263,13 +242,14 @@ static int check_options(pack_opt_t *options) else if (options->op_tbl->objs[i].chunk.rank==-2) { if (options->verbose) - printf("\t%s %s\n",obj_name,"NONE"); + printf("\t%s %s\n",name,"NONE"); has_ck=1; } } if (options->all_chunk==1 && has_ck){ - printf("Error: Invalid chunking input: '*' is present with other objects\n"); + printf("Error: Invalid chunking input: all option\ + is present with other objects\n"); return -1; } @@ -280,23 +260,22 @@ static int check_options(pack_opt_t *options) if (options->verbose) { - printf("Objects to compress are...\n"); - if (options->all_comp==1) + printf("Objects to filter are...\n"); + if (options->all_filter==1) { - switch (options->comp_g.type) + H5Z_filter_t filtn=options->filter_g.filtn; + switch (filtn) { case H5Z_FILTER_NONE: - printf("\tUncompress all %s\n", - get_scomp(options->comp_g.type)); + printf("\tUncompress all %s\n",get_sfilter(filtn)); break; case H5Z_FILTER_SZIP: - printf("\tCompress all with %s compression\n", - get_scomp(options->comp_g.type)); + printf("\tCompress all with %s compression\n",get_sfilter(filtn)); break; case H5Z_FILTER_DEFLATE: printf("\tCompress all with %s compression, parameter %d\n", - get_scomp(options->comp_g.type), - options->comp_g.info); + get_sfilter(filtn), + options->filter_g.cd_values[0]); break; }; } @@ -304,22 +283,54 @@ static int check_options(pack_opt_t *options) for ( i = 0; i < options->op_tbl->nelems; i++) { - pack_info_t obj=options->op_tbl->objs[i]; - if (obj.comp.type>0) + pack_info_t obj = options->op_tbl->objs[i]; + char* name = obj.path; + if (obj.filter.filtn>0) { - char* obj_name=obj.path; - if (options->verbose) { - printf("\t<%s> with %s compression, parameter %d\n", - obj_name, - get_scomp(obj.comp.type), - obj.comp.info); + if (options->verbose) + { + printf("\t<%s> with %s filter", + name, + get_sfilter(obj.filter.filtn)); } has_cp=1; - } - } + + /*check for invalid combination of options */ + switch (obj.filter.filtn) + { + default: + break; + case H5Z_FILTER_SZIP: + + szip_pixels_per_block=obj.filter.cd_values[0]; + + /* check szip parameters */ + if (check_szip(obj.chunk.rank, + obj.chunk.chunk_lengths, + 0, /* do not test size */ + szip_options_mask, + szip_pixels_per_block)==0) + { + /* Return: 1=can apply the filter + 0=cannot apply the filter + Reset this object filter info + */ + + options->op_tbl->objs[i].filter.filtn=-1; + options->op_tbl->objs[i].chunk.rank=-1; + printf("\tObject <%s> cannot be filtered\n",name); + + + } + + break; + } /* switch */ + } /* filtn */ + } /* i */ - if (options->all_comp==1 && has_cp){ - printf("Error: Invalid compression input: * is present with other objects\n"); + if (options->all_filter==1 && has_cp){ + printf("Error: Invalid compression input: all option\ + is present with other objects\n"); return -1; } return 0; @@ -342,6 +353,7 @@ static int check_options(pack_opt_t *options) void read_info(const char *filename, pack_opt_t *options) { + char stype[10]; char comp_info[1024]; FILE *fp; @@ -383,7 +395,7 @@ void read_info(const char *filename, } comp_info[i-1]='\0'; /*cut the last " */ - if (h5repack_addcomp(comp_info,options)==-1){ + if (h5repack_addfilter(comp_info,options)==-1){ printf( "Could not add compression option. Exiting\n"); exit(1); } diff --git a/tools/h5repack/h5repack.h b/tools/h5repack/h5repack.h index a372a6f..ae20cc4 100644 --- a/tools/h5repack/h5repack.h +++ b/tools/h5repack/h5repack.h @@ -42,16 +42,21 @@ typedef struct { } obj_list_t; /* - the type of compression and additional parameter + the type of filter and additional parameter type can be one of the filters - H5Z_FILTER_NONE 0, uncompress if compressed - H5Z_FILTER_DEFLATE 1 , deflation like gzip - H5Z_FILTER_SZIP 4 , szip compression + H5Z_FILTER_NONE 0, uncompress if compressed + H5Z_FILTER_DEFLATE 1 , deflation like gzip + H5Z_FILTER_SHUFFLE 2 , shuffle the data + H5Z_FILTER_FLETCHER32 3 , letcher32 checksum of EDC + H5Z_FILTER_SZIP 4 , szip compression */ + +#define CDVALUES 2 + typedef struct { - int type; - int info; -} comp_info_t; + H5Z_filter_t filtn; /* filter identification number */ + int cd_values[CDVALUES]; /* filter client data values */ +} filter_info_t; /* chunk lengths along each dimension and rank */ typedef struct { @@ -59,18 +64,18 @@ typedef struct { int rank; } chunk_info_t; -/* information for one object, contains PATH, CHUNK info and COMP info */ +/* information for one object, contains PATH, CHUNK info and FILTER info */ typedef struct { - char path[MAX_NC_NAME]; /* name of object */ - comp_info_t comp; /* compression information */ - chunk_info_t chunk; /* chunk information */ - hid_t refobj_id; /* object ID */ + char path[MAX_NC_NAME]; /* name of object */ + filter_info_t filter; /* filter information */ + chunk_info_t chunk; /* chunk information */ + hid_t refobj_id; /* object ID, references */ } pack_info_t; /* store a table of all objects */ typedef struct { - int size; - int nelems; + int size; + int nelems; pack_info_t *objs; } pack_opttbl_t; @@ -82,14 +87,13 @@ typedef struct { /* all the above, ready to go to the hrepack call */ typedef struct { - pack_opttbl_t *op_tbl; /*table with all -c and -t options */ - int all_chunk; /*chunk all objects, input of "*" */ - int all_comp; /*comp all objects, input of "*" */ - comp_info_t comp_g; /*global compress INFO for the ALL case */ + pack_opttbl_t *op_tbl; /*table with all -c and -f options */ + int all_chunk; /*chunk all objects */ + int all_filter; /*filter all objects */ + filter_info_t filter_g; /*global filter INFO for the ALL case */ chunk_info_t chunk_g; /*global chunk INFO for the ALL case */ int verbose; /*verbose mode */ int threshold; /*minimum size to compress, in bytes */ - } pack_opt_t; @@ -103,11 +107,12 @@ typedef struct { extern "C" { #endif -int h5repack (const char* infile, const char* outfile, pack_opt_t *options); -int h5repack_addcomp (const char* str, pack_opt_t *options); -int h5repack_addchunk(const char* str, pack_opt_t *options); -int h5repack_init (pack_opt_t *options, int verbose); -int h5repack_end (pack_opt_t *options); +int h5repack (const char* infile, const char* outfile, pack_opt_t *options); +int h5repack_addfilter (const char* str, pack_opt_t *options); +int h5repack_addchunk (const char* str, pack_opt_t *options); +int h5repack_init (pack_opt_t *options, int verbose); +int h5repack_end (pack_opt_t *options); +int h5repack_verify (const char *fname,pack_opt_t *options); #ifdef __cplusplus } @@ -139,14 +144,11 @@ int do_copy_file(hid_t fidin, int copy_attr(hid_t loc_in, hid_t loc_out, - pack_opt_t *options, - trav_table_t *travt, - hid_t fidout /* for saving references */ + pack_opt_t *options ); const char* MapIdToName(hid_t refobj_id, - trav_table_t *travt, - pack_opt_t *options) /* repack options */; + trav_table_t *travt); int do_copy_refobjs(hid_t fidin, hid_t fidout, @@ -163,6 +165,34 @@ int do_copy_refobjs_inattr(hid_t loc_in, void read_info(const char *filename,pack_opt_t *options); void close_obj(H5G_obj_t obj_type, hid_t obj_id); + +/*------------------------------------------------------------------------- + * filters + *------------------------------------------------------------------------- + */ + +int apply_filter(hid_t dcpl_id, + size_t size, /* size of datatype in bytes */ + pack_opt_t *options, /* repack options */ + pack_info_t *obj); /* info about object to filter */ + +int has_filter(hid_t dcpl_id, + H5Z_filter_t filtnin); + +int check_szip_params( unsigned bits_per_pixel, + unsigned pixels_per_block, + unsigned pixels_per_scanline, + hsize_t image_pixels); + +int check_szip(int rank, /* chunk rank */ + hsize_t *dims, /* chunk dims */ + size_t size, /* size of datatype in bytes */ + unsigned szip_options_mask, + unsigned szip_pixels_per_block); + + + + /*------------------------------------------------------------------------- * options table *------------------------------------------------------------------------- @@ -174,9 +204,9 @@ int options_add_chunk ( obj_list_t *obj_list, hsize_t *chunk_lengths, int chunk_rank, pack_opttbl_t *table ); -int options_add_comp ( obj_list_t *obj_list, +int options_add_filter ( obj_list_t *obj_list, int n_objs, - comp_info_t comp, + filter_info_t filt, pack_opttbl_t *table ); pack_info_t* options_get_object( const char *path, pack_opttbl_t *table); @@ -186,14 +216,16 @@ pack_info_t* options_get_object( const char *path, *------------------------------------------------------------------------- */ -obj_list_t* parse_comp (const char *str, +obj_list_t* parse_filter(const char *str, int *n_objs, - comp_info_t *comp); + filter_info_t *filt, + pack_opt_t *options); obj_list_t* parse_chunk (const char *str, int *n_objs, hsize_t *chunk_lengths, - int *chunk_rank); -const char* get_scomp (int code); + int *chunk_rank, + pack_opt_t *options); +const char* get_sfilter (H5Z_filter_t filtn); int parse_number(char *str); /*------------------------------------------------------------------------- @@ -211,9 +243,7 @@ int parse_number(char *str); #define FNAME4OUT "test4out.h5" int make_testfiles(void); -int make_all_objects(hid_t fid); -int make_attributes(hid_t fid); -int make_special_objects(hid_t loc_id); + int make_attr(hid_t fid); int write_dset( hid_t loc_id, int rank, @@ -236,6 +266,48 @@ void write_dset_in(hid_t loc_id, hid_t file_id, int make_diffs /* flag to modify data buffers */); +int make_deflate(hid_t loc_id); +int make_szip(hid_t loc_id); +int make_nofilters(hid_t loc_id); + + + +/*------------------------------------------------------------------------- + * check SZIP parameters + *------------------------------------------------------------------------- + */ + +typedef struct +{ + int compression_mode; + int options_mask; + unsigned bits_per_pixel; + unsigned pixels_per_block; + unsigned pixels_per_scanline; + hsize_t pixels; + +}szip_comp_t; + +/* for SZIP */ +#if !defined (NN_OPTION_MASK) +#define NN_OPTION_MASK 32 +#endif +#if !defined (RAW_OPTION_MASK) +#define RAW_OPTION_MASK 128 +#endif +#if !defined (MAX_BLOCKS_PER_SCANLINE) +#define MAX_BLOCKS_PER_SCANLINE 128 +#endif +#if !defined (MAX_PIXELS_PER_BLOCK) +#define MAX_PIXELS_PER_BLOCK 32 +#endif +#if !defined (MAX_PIXELS_PER_SCANLINE) +#define MAX_PIXELS_PER_SCANLINE (MAX_BLOCKS_PER_SCANLINE)*(MAX_PIXELS_PER_BLOCK) +#endif +#if !defined (NN_MODE) +#define NN_MODE 1 +#endif + #endif /* H5REPACK_H__ */ diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c index 2afc94b..4d56ae6 100644 --- a/tools/h5repack/h5repack_copy.c +++ b/tools/h5repack/h5repack_copy.c @@ -19,6 +19,11 @@ #include "H5private.h" #include "h5repack.h" +static int filter_this(const char* name, + pack_opt_t *options, + pack_info_t *obj); /* info about object to filter */ + + /*------------------------------------------------------------------------- * Function: copy_file * @@ -39,7 +44,6 @@ int copy_file(const char* fnamein, { hid_t fidin; hid_t fidout; - int nobjects=0; trav_table_t *travt=NULL; /*------------------------------------------------------------------------- @@ -161,6 +165,8 @@ int do_copy_file(hid_t fidin, hsize_t nelmts; /* number of elements in dataset */ int rank; /* rank of dataset */ hsize_t dims[H5S_MAX_RANK];/* dimensions of dataset */ + hsize_t dsize_in; /* input dataset size before filter */ + hsize_t dsize_out; /* output dataset size after filter */ int i, j; /*------------------------------------------------------------------------- @@ -191,7 +197,7 @@ int do_copy_file(hid_t fidin, * copy attrs *------------------------------------------------------------------------- */ - if (copy_attr(grp_in,grp_out,options,travt,fidout)<0) + if (copy_attr(grp_in,grp_out,options)<0) goto error; if (H5Gclose(grp_out)<0) @@ -239,6 +245,12 @@ int do_copy_file(hid_t fidin, */ if ( ! H5Tequal(mtype_id, H5T_STD_REF_OBJ) ) { + + /* the information about the object to be filtered */ + pack_info_t filt_obj; + + /* get the storage size of the input dataset */ + dsize_in=H5Dget_storage_size(dset_in); /*------------------------------------------------------------------------- * read to memory @@ -254,21 +266,41 @@ int do_copy_file(hid_t fidin, goto error; /*------------------------------------------------------------------------- + * apply the filter; check first if the object is to be filtered. + * if the filter could not be applied, continue + *------------------------------------------------------------------------- + */ + if (filter_this(travt->objs[i].name,options,&filt_obj)) + { + if (apply_filter(dcpl_id,H5Tget_size(mtype_id),options,&filt_obj)<0) + continue; + } + + /*------------------------------------------------------------------------- * create/write dataset/close *------------------------------------------------------------------------- */ if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,ftype_id,space_id,dcpl_id))<0) goto error; - if (H5Dwrite(dset_out,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) - goto error; + if (dsize_in) { + if (H5Dwrite(dset_out,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) + goto error; + } /*------------------------------------------------------------------------- * copy attrs *------------------------------------------------------------------------- */ - if (copy_attr(dset_in,dset_out,options,travt,fidout)<0) + if (copy_attr(dset_in,dset_out,options)<0) goto error; + + /*------------------------------------------------------------------------- + * store the storage sizes + *------------------------------------------------------------------------- + */ + dsize_out=H5Dget_storage_size(dset_out); + /*close */ if (H5Dclose(dset_out)<0) goto error; @@ -317,7 +349,7 @@ int do_copy_file(hid_t fidin, * copy attrs *------------------------------------------------------------------------- */ - if (copy_attr(type_in,type_out,options,travt,fidout)<0) + if (copy_attr(type_in,type_out,options)<0) goto error; if (H5Tclose(type_in)<0) @@ -372,8 +404,6 @@ int do_copy_file(hid_t fidin, break; } } - - /*------------------------------------------------------------------------- * the root is a special case, we get an ID for the root group @@ -389,7 +419,7 @@ int do_copy_file(hid_t fidin, if ((grp_in = H5Gopen(fidin,"/"))<0) goto error; - if (copy_attr(grp_in,grp_out,options,travt,fidout)<0) + if (copy_attr(grp_in,grp_out,options)<0) goto error; if (H5Gclose(grp_out)<0) @@ -438,9 +468,7 @@ error: int copy_attr(hid_t loc_in, hid_t loc_out, - pack_opt_t *options, - trav_table_t *travt, - hid_t fidout /* for saving references */ + pack_opt_t *options ) { hid_t attr_id; /* attr ID */ @@ -456,7 +484,6 @@ int copy_attr(hid_t loc_in, char name[255]; int n, j; unsigned u; - int have_ref=0; if ((n = H5Aget_num_attrs(loc_in))<0) goto error; @@ -576,3 +603,61 @@ error: return -1; } + + + +/*------------------------------------------------------------------------- + * Function: filter_this + * + * Purpose: find the object name NAME (got from the traverse list) + * in the repack options list; assign the filter information OBJ + * + * Return: 0 not found, 1 found + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: December 19, 2003 + * + *------------------------------------------------------------------------- + */ + +static +int filter_this(const char* name, /* object name from traverse list */ + pack_opt_t *options, /* repack options */ + pack_info_t *obj) /* info about object to filter */ +{ + char *pdest; + int result; + int i; + + /* if we are applying to all objects just return true */ + if (options->all_filter) + { + /* assign the global filter and chunk info to the OBJ info */ + obj->filter=options->filter_g; + obj->chunk=options->chunk_g; + return 1; + } + + for ( i=0; iop_tbl->nelems; i++) + { + if (strcmp(options->op_tbl->objs[i].path,name)==0) + { + *obj=options->op_tbl->objs[i]; + return 1; + } + + pdest = strstr(name,options->op_tbl->objs[i].path); + result = (int)(pdest - name); + + /* found at position 1, meaning without '/' */ + if( pdest != NULL && result==1 ) + { + *obj=options->op_tbl->objs[i]; + return 1; + } + } + + return 0; +} + diff --git a/tools/h5repack/h5repack_filters.c b/tools/h5repack/h5repack_filters.c new file mode 100644 index 0000000..6c88b51 --- /dev/null +++ b/tools/h5repack/h5repack_filters.c @@ -0,0 +1,345 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "hdf5.h" +#include "h5test.h" +#include "h5repack.h" + + + +/*------------------------------------------------------------------------- + * Function: apply_filter + * + * Purpose: apply a filter to the property list; do extra checking + * in the case of SZIP + * + * Return: 0, ok, -1 no + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: December 19, 2003 + * + *------------------------------------------------------------------------- + */ + +int apply_filter(hid_t dcpl_id, + size_t size, /* size of datatype in bytes */ + pack_opt_t *options, /* repack options */ + pack_info_t *obj) /* info about object to filter */ +{ + int nfilters; /* number of filters */ + unsigned filt_flags; /* filter flags */ + H5Z_filter_t filtn; /* filter identification number */ + unsigned cd_values[20]; /* filter client data values */ + size_t cd_nelmts; /* filter client number of values */ + size_t cd_num; /* filter client data counter */ + char f_name[256]; /* filter/file name */ + char s[64]; /* temporary string buffer */ + int i; + unsigned aggression; /* the deflate level */ + unsigned szip_options_mask=H5_SZIP_NN_OPTION_MASK; + unsigned szip_pixels_per_block; + + + + /* get information about filters */ + if ((nfilters = H5Pget_nfilters(dcpl_id))<0) + return -1; + + for (i=0; iverbose) + { + f_name[sizeof(f_name)-1] = '\0'; + sprintf(s, "Filter-%d:", i); + printf(" %-10s %s-%u %s {", s, + f_name[0]?f_name:"method", + (unsigned)filtn, + filt_flags & H5Z_FLAG_OPTIONAL?"OPT":""); + for (cd_num=0; cd_numfilter.filtn) + { + case H5Z_FILTER_NONE: + + break; + + + case H5Z_FILTER_DEFLATE: + + + aggression=obj->filter.cd_values[0]; + + /* set up for deflated data */ + if(H5Pset_chunk(dcpl_id, obj->chunk.rank, obj->chunk.chunk_lengths)<0) + return -1; + if(H5Pset_deflate(dcpl_id,aggression)<0) + return -1; + + break; + + + case H5Z_FILTER_SZIP: + + szip_pixels_per_block=obj->filter.cd_values[0]; + + /* check szip parameters */ + if (check_szip(obj->chunk.rank, + obj->chunk.chunk_lengths, + size, + szip_options_mask, + szip_pixels_per_block)==1) + { + /* set up for szip data */ + if(H5Pset_chunk(dcpl_id,obj->chunk.rank,obj->chunk.chunk_lengths)<0) + return -1; + if (H5Pset_szip(dcpl_id, szip_options_mask, szip_pixels_per_block)<0) + return -1; + + } + else + { + printf("SZIP filter cannot be applied\n"); + } + + break; + + + default: + break; + + } /* switch */ + + return 0; +} + + + +/*------------------------------------------------------------------------- + * Function: check_szip + * + * Purpose: utility to check SZIP parameters + * + * SZIP compresses data block by block, with a user-tunable block size. + * This block size is passed in the parameter pixels_per_block and must be even, + * with typical values being 8, 10, 16, and 32. The more pixel values vary, + * the smaller this number should be. For optimal performance, the number of + * pixels per scan line (i.e., the size of the fastest-changing dimension in the chunk) + * should be an even multiple of the number of pixels per block. + * + * Return: 1=can apply the filter + * 0=cannot apply the filter + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: December 23, 2003 + * + *------------------------------------------------------------------------- + */ + +int check_szip(int rank, /* chunk rank */ + hsize_t *dims, /* chunk dims */ + size_t size, /* size of datatype in bytes */ + unsigned szip_options_mask, + unsigned szip_pixels_per_block) +{ + szip_comp_t szip; + int i; + + /* + pixels_per_scanline = size of the fastest-changing dimension + Must be <= MAX_PIXELS_PER_SCANLINE and <= pixels + */ + szip.pixels_per_scanline = (unsigned)dims[rank-1]; + szip.pixels = 1; + for ( i = 0; i < rank; i++) + { + szip.pixels *= dims[i]; + } + + if (szip.pixels_per_scanline > MAX_PIXELS_PER_SCANLINE) + { + printf("Warning: in SZIP setting, pixels per scanline was set to <%d>, \ + MAX_PIXELS_PER_SCANLINE\n",MAX_PIXELS_PER_SCANLINE); + szip.pixels_per_scanline = MAX_PIXELS_PER_SCANLINE; + } + + /* + pixels_per_block must be an even number, and <= pixels_per_scanline + and <= MAX_PIXELS_PER_BLOCK + */ + szip.pixels_per_block=szip_pixels_per_block; + + if (szip.pixels_per_block > szip.pixels_per_scanline) + { + printf("\n\tWarning: in SZIP setting, pixels per block <%d>, \ + cannot be greater than pixels per scanline<%d>\n", + szip.pixels_per_block, szip.pixels_per_scanline); + return 0; + } + + szip.options_mask = szip_options_mask; + szip.compression_mode = NN_MODE; + + /* + bits_per_pixel + Must be in range 1..24,32,64 + */ + switch(size) + { + case 0: + /* size was not provided for test */ + szip.bits_per_pixel = 0; + break; + case 1: + szip.bits_per_pixel = 8; + break; + case 2: + szip.bits_per_pixel = 16; + break; + case 4: + szip.bits_per_pixel = 32; + break; + case 8: + szip.bits_per_pixel = 64; + break; + default: + printf("Error: Bad numeric type of size <%d> in SZIP\n",size); + return 0; + } + + return check_szip_params( szip.bits_per_pixel, + szip.pixels_per_block, + szip.pixels_per_scanline, + szip.pixels); + +} + + +/*------------------------------------------------------------------------- + * Function: check_szip_params + * + * Purpose: Adapted from rice.c. Checks the SZIP parameters + * + * Return: 1=can apply the filter + * 0=cannot apply the filter + * + *------------------------------------------------------------------------- + */ + +int check_szip_params( unsigned bits_per_pixel, + unsigned pixels_per_block, + unsigned pixels_per_scanline, + hsize_t image_pixels) +{ + + if (pixels_per_block & 1) + { + printf("Pixels per block must be even.\n"); + return 0; + } + + if (pixels_per_block > pixels_per_scanline) + { + printf("Pixels per block is greater than pixels per scanline.\n"); + return 0; + } + + if (bits_per_pixel) /* if provided for test */ + { + if (bits_per_pixel >= 1 && bits_per_pixel <= 24) + ; + else if (bits_per_pixel == 32 || bits_per_pixel == 64) + ; + else + { + printf("bits per pixel must be in range 1..24,32,64"); + return 0; + } + } + + if (pixels_per_block > MAX_PIXELS_PER_BLOCK) + { + printf("maximum pixels per block exceeded"); + return 0; + } + + if (pixels_per_block & 1) + { + printf("pixels per block must be even"); + return 0; + } + + if (pixels_per_block > pixels_per_scanline) + { + printf("pixels per block > pixels per scanline"); + return 0; + } + + if (pixels_per_scanline > MAX_PIXELS_PER_SCANLINE) + { + printf("maximum pixels per scanline exceeded"); + return 0; + } + + if (image_pixels < pixels_per_scanline) + { + printf("image pixels less than pixels per scanline"); + return 0; + } + + if (image_pixels % pixels_per_scanline) + { + printf("Pixels (%d) must be integer multiple of pixels per scanline (%d)\n", + (unsigned)image_pixels,pixels_per_scanline); + return 0; + } + +#if 0 + if (pixels_per_scanline % pixels_per_block) + { + printf("Pixels per scanline (%d) must be an integer multiple of pixels per block (%d)\n", + pixels_per_scanline, pixels_per_block); + return 0; + } +#endif + + return 1; +} + diff --git a/tools/h5repack/h5repack_main.c b/tools/h5repack/h5repack_main.c index 5a5bb78..b7dd9bf 100644 --- a/tools/h5repack/h5repack_main.c +++ b/tools/h5repack/h5repack_main.c @@ -51,7 +51,7 @@ int main(int argc, char **argv) else if (strcmp(argv[i], "-t") == 0) { /* add the -t option */ - h5repack_addcomp(argv[i+1],&options); + h5repack_addfilter(argv[i+1],&options); /* jump to next */ ++i; diff --git a/tools/h5repack/h5repack_opttable.c b/tools/h5repack/h5repack_opttable.c index 0dcebd4..0571656 100644 --- a/tools/h5repack/h5repack_opttable.c +++ b/tools/h5repack/h5repack_opttable.c @@ -30,7 +30,7 @@ int options_table_init( pack_opttbl_t **tbl ) { - int i; + int i, j; pack_opttbl_t* table = (pack_opttbl_t*) malloc(sizeof(pack_opttbl_t)); if (table==NULL) { printf("Error: not enough memory for options table\n"); @@ -45,11 +45,13 @@ int options_table_init( pack_opttbl_t **tbl ) return -1; } - for (i = 0; i < table->size; i++) { + for ( i=0; isize; i++) { strcpy(table->objs[i].path,"\0"); - table->objs[i].comp.info = -1; - table->objs[i].comp.type = -1; + table->objs[i].filter.filtn = -1; + for ( j=0; jobjs[i].filter.cd_values[j] = -1; table->objs[i].chunk.rank = -1; + table->objs[i].refobj_id = -1; } *tbl = table; @@ -101,9 +103,11 @@ int options_add_chunk( obj_list_t *obj_list, } for (i = table->nelems; i < table->size; i++) { strcpy(table->objs[i].path,"\0"); - table->objs[i].comp.info = -1; - table->objs[i].comp.type = -1; + table->objs[i].filter.filtn = -1; + for ( j=0; jobjs[i].filter.cd_values[j] = -1; table->objs[i].chunk.rank = -1; + table->objs[i].refobj_id = -1; } } @@ -173,19 +177,19 @@ int options_add_chunk( obj_list_t *obj_list, /*------------------------------------------------------------------------- - * Function: options_add_comp + * Function: options_add_filter * - * Purpose: add a compression -t option to the option list + * Purpose: add a compression -f option to the option list * * Return: 0, ok, -1, fail * *------------------------------------------------------------------------- */ -int options_add_comp( obj_list_t *obj_list, - int n_objs, - comp_info_t comp, - pack_opttbl_t *table ) +int options_add_filter(obj_list_t *obj_list, + int n_objs, + filter_info_t filt, + pack_opttbl_t *table ) { int i, j, I, added=0, found=0; @@ -199,9 +203,11 @@ int options_add_comp( obj_list_t *obj_list, } for (i = table->nelems; i < table->size; i++) { strcpy(table->objs[i].path,"\0"); - table->objs[i].comp.info = -1; - table->objs[i].comp.type = -1; + table->objs[i].filter.filtn = -1; + for ( j=0; jobjs[i].filter.cd_values[j] = -1; table->objs[i].chunk.rank = -1; + table->objs[i].refobj_id = -1; } } @@ -217,19 +223,11 @@ int options_add_comp( obj_list_t *obj_list, /*already on the table */ if (strcmp(obj_list[j].obj,table->objs[i].path)==0) { - /* already COMP info inserted for this one; exit */ - if (table->objs[i].comp.type>0) - { - printf("Input Error: compression information already inserted for <%s>\n",obj_list[j].obj); - exit(1); - } - /* insert the comp info */ - else - { - table->objs[i].comp = comp; - found=1; - break; - } + /* insert */ + + table->objs[i].filter = filt; + found=1; + break; } /* if */ } /* i */ @@ -239,7 +237,7 @@ int options_add_comp( obj_list_t *obj_list, I = table->nelems + added; added++; strcpy(table->objs[I].path,obj_list[j].obj); - table->objs[I].comp = comp; + table->objs[I].filter = filt; } } /* j */ } @@ -253,7 +251,7 @@ int options_add_comp( obj_list_t *obj_list, I = table->nelems + added; added++; strcpy(table->objs[I].path,obj_list[j].obj); - table->objs[I].comp = comp; + table->objs[I].filter = filt; } } diff --git a/tools/h5repack/h5repack_parse.c b/tools/h5repack/h5repack_parse.c index f401e1c..9f331b2 100644 --- a/tools/h5repack/h5repack_parse.c +++ b/tools/h5repack/h5repack_parse.c @@ -20,15 +20,13 @@ /*------------------------------------------------------------------------- - * Function: parse_comp + * Function: parse_filter * - * Purpose: read compression info + * Purpose: read filter information * * Return: a list of names, the number of names and its compression type - * NULL, on error * * Examples: - * "AA,B,CDE:RLE" * "GZIP 6" * "A,B:NONE" * @@ -40,9 +38,10 @@ */ -obj_list_t* parse_comp(const char *str, - int *n_objs, - comp_info_t *comp) +obj_list_t* parse_filter(const char *str, + int *n_objs, + filter_info_t *filt, + pack_opt_t *options) { unsigned i, u; char c; @@ -52,9 +51,10 @@ obj_list_t* parse_comp(const char *str, char scomp[10]; char stype[5]; obj_list_t* obj_list=NULL; + unsigned pixels_per_block; /* initialize compression info */ - memset(comp,0,sizeof(comp_info_t)); + memset(filt,0,sizeof(filter_info_t)); /* check for the end of object list and number of objects */ for ( i=0, n=0; i\n",str); - exit(1); + /* apply to all objects */ + options->all_filter=1; } n++; @@ -107,9 +107,9 @@ obj_list_t* parse_comp(const char *str, } - /* get compression type */ + /* get filter additional parameters */ m=0; - for ( i=end_obj+1, k=0; iinfo=atoi(stype); + filt->cd_values[j++]=atoi(stype); i+=m; /* jump */ } else if (i==len-1) { /*no more parameters */ @@ -142,7 +142,7 @@ obj_list_t* parse_comp(const char *str, *------------------------------------------------------------------------- */ if (strcmp(scomp,"NONE")==0) - comp->type=H5Z_FILTER_NONE; + filt->filtn=H5Z_FILTER_NONE; /*------------------------------------------------------------------------- * H5Z_FILTER_DEFLATE @@ -150,7 +150,7 @@ obj_list_t* parse_comp(const char *str, */ else if (strcmp(scomp,"GZIP")==0) { - comp->type=H5Z_FILTER_DEFLATE; + filt->filtn=H5Z_FILTER_DEFLATE; if (no_param) { /*no more parameters, GZIP must have parameter */ if (obj_list) free(obj_list); printf("Input Error: Missing compression parameter in <%s>\n",str); @@ -164,60 +164,108 @@ obj_list_t* parse_comp(const char *str, */ else if (strcmp(scomp,"SZIP")==0) { - comp->type=H5Z_FILTER_SZIP; - if (m>0){ /*SZIP does not have parameter */ + filt->filtn=H5Z_FILTER_SZIP; + if (no_param) { /*no more parameters, SZIP must have parameter */ if (obj_list) free(obj_list); - printf("Input Error: Extra compression parameter in SZIP <%s>\n",str); + printf("Input Error: Missing compression parameter in <%s>\n",str); + exit(1); + } + } + +/*------------------------------------------------------------------------- + * H5Z_FILTER_SHUFFLE + *------------------------------------------------------------------------- + */ + else if (strcmp(scomp,"SHUF")==0) + { + filt->filtn=H5Z_FILTER_SHUFFLE; + if (m>0){ /*shuffle does not have parameter */ + if (obj_list) free(obj_list); + printf("Input Error: Extra parameter in SHUF <%s>\n",str); + exit(1); + } + } +/*------------------------------------------------------------------------- + * H5Z_FILTER_FLETCHER32 + *------------------------------------------------------------------------- + */ + else if (strcmp(scomp,"FLET")==0) + { + filt->filtn=H5Z_FILTER_FLETCHER32; + if (m>0){ /*shuffle does not have parameter */ + if (obj_list) free(obj_list); + printf("Input Error: Extra parameter in FLET <%s>\n",str); exit(1); } } else { if (obj_list) free(obj_list); - printf("Input Error: Invalid compression type in <%s>\n",str); + printf("Input Error: Invalid filter type in <%s>\n",str); exit(1); } } } /*i*/ +/*------------------------------------------------------------------------- + * check valid parameters + *------------------------------------------------------------------------- + */ - /* check valid parameters */ - switch (comp->type) - { - case H5Z_FILTER_DEFLATE: - if (comp->info<0 || comp->info>9 ){ - if (obj_list) free(obj_list); - printf("Input Error: Invalid compression parameter in <%s>\n",str); - exit(1); - } - break; - case H5Z_FILTER_SZIP: - break; - }; + switch (filt->filtn) + { + + case H5Z_FILTER_DEFLATE: + if (filt->cd_values[0]<0 || filt->cd_values[0]>9 ){ + if (obj_list) free(obj_list); + printf("Input Error: Invalid compression parameter in <%s>\n",str); + exit(1); + } + break; + + + case H5Z_FILTER_SZIP: + pixels_per_block=filt->cd_values[0]; + if ((pixels_per_block%2)==1) { + if (obj_list) free(obj_list); + printf("Input Error: pixels_per_block is not even in <%s>\n",str); + exit(1); + } + if (pixels_per_block>H5_SZIP_MAX_PIXELS_PER_BLOCK) { + if (obj_list) free(obj_list); + printf("Input Error: pixels_per_block is too large in <%s>\n",str); + exit(1); + } + break; + }; return obj_list; } /*------------------------------------------------------------------------- - * Function: get_scomp + * Function: get_sfilter * - * Purpose: return the compression type as a string + * Purpose: return the filter as a string name * * Return: name of filter, exit on error * *------------------------------------------------------------------------- */ -const char* get_scomp(int code) +const char* get_sfilter(H5Z_filter_t filtn) { - if (code==H5Z_FILTER_NONE) + if (filtn==H5Z_FILTER_NONE) return "NONE"; - else if (code==H5Z_FILTER_DEFLATE) + else if (filtn==H5Z_FILTER_DEFLATE) return "GZIP"; - else if (code==H5Z_FILTER_SZIP) + else if (filtn==H5Z_FILTER_SZIP) return "SZIP"; + else if (filtn==H5Z_FILTER_SHUFFLE) + return "SHUFFLE"; + else if (filtn==H5Z_FILTER_FLETCHER32) + return "FLETCHERP"; else { - printf("Input Error in compression type\n"); + printf("Input error in filter type\n"); exit(1); } return NULL; @@ -247,7 +295,8 @@ const char* get_scomp(int code) obj_list_t* parse_chunk(const char *str, int *n_objs, hsize_t *chunk_lengths, - int *chunk_rank) + int *chunk_rank, + pack_opt_t *options) { obj_list_t* obj_list=NULL; unsigned i; @@ -271,9 +320,8 @@ obj_list_t* parse_chunk(const char *str, } } - if (end_obj==-1) { /* missing : */ - printf("Input Error: Invalid chunking input in <%s>\n",str); - exit(1); + if (end_obj==-1) { /* missing : chunk all */ + options->all_chunk=1; } n++; diff --git a/tools/h5repack/h5repack_refs.c b/tools/h5repack/h5repack_refs.c index 81bafab..d02038d 100644 --- a/tools/h5repack/h5repack_refs.c +++ b/tools/h5repack/h5repack_refs.c @@ -130,6 +130,8 @@ int do_copy_refobjs(hid_t fidin, hobj_ref_t *refbuf; const char* refname; hobj_ref_t *buf; + unsigned u; + /*------------------------------------------------------------------------- * read to memory @@ -152,17 +154,17 @@ int do_copy_refobjs(hid_t fidin, printf( "cannot allocate memory\n" ); goto error; } - for ( j=0; jverbose) @@ -412,7 +414,7 @@ int do_copy_refobjs_inattr(hid_t loc_in, H5G_obj_t obj_type; hid_t refobj_id; hobj_ref_t *refbuf; - int i; + unsigned k; const char* refname; hobj_ref_t *buf; @@ -432,17 +434,17 @@ int do_copy_refobjs_inattr(hid_t loc_in, printf( "cannot allocate memory\n" ); goto error; } - for ( i=0; iverbose) @@ -503,7 +505,7 @@ error: *------------------------------------------------------------------------- */ -static void close_obj(H5G_obj_t obj_type, hid_t obj_id) +void close_obj(H5G_obj_t obj_type, hid_t obj_id) { switch (obj_type) { @@ -531,8 +533,7 @@ static void close_obj(H5G_obj_t obj_type, hid_t obj_id) */ const char* MapIdToName(hid_t refobj_id, - trav_table_t *travt, - pack_opt_t *options) /* repack options */ + trav_table_t *travt) { hid_t id; hid_t fid; diff --git a/tools/h5repack/h5repack_verify.c b/tools/h5repack/h5repack_verify.c new file mode 100644 index 0000000..f3c1583 --- /dev/null +++ b/tools/h5repack/h5repack_verify.c @@ -0,0 +1,241 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "hdf5.h" +#include "h5test.h" +#include "h5repack.h" + + + +/*------------------------------------------------------------------------- + * Function: has_filter + * + * Purpose: verify if a filter is present in the property list DCPL_ID + * + * Return: 1 has, 0 does not, -1 error + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: December 19, 2003 + * + *------------------------------------------------------------------------- + */ + +int has_filter(hid_t dcpl_id, + H5Z_filter_t filtnin) +{ + + int nfilters; /* number of filters */ + unsigned filt_flags; /* filter flags */ + H5Z_filter_t filtn; /* filter identification number */ + unsigned cd_values[20]; /* filter client data values */ + size_t cd_nelmts; /* filter client number of values */ + char f_name[256]; /* filter name */ + int have=0; /* flag, filter is present */ + hsize_t chsize[64]; /* chunk size in elements */ + H5D_layout_t layout; /* layout */ + int rank; /* rank */ + int i; /* index */ + + /* if no information about the input filter is requested return exit */ + if (filtnin==-1) + return 1; + + /* get information about filters */ + if ((nfilters = H5Pget_nfilters(dcpl_id))<0) + return -1; + +/* + H5D_COMPACT = 0 + H5D_CONTIGUOUS = 1 + H5D_CHUNKED = 2 + */ + layout = H5Pget_layout(dcpl_id); + if (layout==H5D_CHUNKED) + { + if ((rank = H5Pget_chunk(dcpl_id,NELMTS(chsize),chsize/*out*/))<0) + return -1; + } + + for (i=0; iop_tbl->nelems; i++) + { + char* name=options->op_tbl->objs[i].path; + pack_info_t obj=options->op_tbl->objs[i]; + +/*------------------------------------------------------------------------- + * open + *------------------------------------------------------------------------- + */ + if ((dset_id=H5Dopen(fid,name))<0) + goto error; + if ((space_id=H5Dget_space(dset_id))<0) + goto error; + if ((dcpl_id=H5Dget_create_plist(dset_id))<0) + goto error; + +/*------------------------------------------------------------------------- + * filter check + *------------------------------------------------------------------------- + */ + if (has_filter(dcpl_id,obj.filter.filtn)==0) + ret=0; + +/*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + if (H5Pclose(dcpl_id)<0) + goto error; + if (H5Sclose(space_id)<0) + goto error; + if (H5Dclose(dset_id)<0) + goto error; + + } + + +/*------------------------------------------------------------------------- + * check for the "all" objects option + *------------------------------------------------------------------------- + */ + + if (options->all_filter==1) + { + + /* init table */ + trav_table_init(&travt); + + /* get the list of objects in the file */ + if (h5trav_gettable(fid,travt)<0) + goto error; + + for ( i=0; inobjs; i++) + { + char* name=travt->objs[i].name; + + switch ( travt->objs[i].type ) + { + case H5G_DATASET: + + /*------------------------------------------------------------------------- + * open + *------------------------------------------------------------------------- + */ + if ((dset_id=H5Dopen(fid,name))<0) + goto error; + if ((space_id=H5Dget_space(dset_id))<0) + goto error; + if ((dcpl_id=H5Dget_create_plist(dset_id))<0) + goto error; + + /*------------------------------------------------------------------------- + * filter check + *------------------------------------------------------------------------- + */ + if (has_filter(dcpl_id,options->filter_g.filtn)==0) + ret=0; + + /*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + if (H5Pclose(dcpl_id)<0) + goto error; + if (H5Sclose(space_id)<0) + goto error; + if (H5Dclose(dset_id)<0) + goto error; + + break; + default: + break; + } /* switch */ + + } /* i */ + + /* free table */ + trav_table_free(travt); + } + +/*------------------------------------------------------------------------- + * close + *------------------------------------------------------------------------- + */ + + if (H5Fclose(fid)<0) + return -1; + + return ret; + +error: + H5E_BEGIN_TRY { + H5Pclose(dcpl_id); + H5Sclose(space_id); + H5Dclose(dset_id); + H5Fclose(fid); + trav_table_free(travt); + } H5E_END_TRY; + return -1; +} + diff --git a/tools/h5repack/testh5repack_dset.c b/tools/h5repack/testh5repack_dset.c index 88363f7..42ad644 100644 --- a/tools/h5repack/testh5repack_dset.c +++ b/tools/h5repack/testh5repack_dset.c @@ -16,6 +16,9 @@ #include "h5test.h" #include "h5repack.h" + + + /*------------------------------------------------------------------------- * Function: write_dset * diff --git a/tools/h5repack/testh5repack_filters.c b/tools/h5repack/testh5repack_filters.c index 9067ff0..5a35d67 100644 --- a/tools/h5repack/testh5repack_filters.c +++ b/tools/h5repack/testh5repack_filters.c @@ -16,80 +16,110 @@ #include "h5test.h" #include "h5repack.h" +#define DIM1 40 +#define DIM2 20 +#define CDIM1 DIM1/2 +#define CDIM2 DIM2/2 +#define RANK 2 + + /*------------------------------------------------------------------------- * Function: make_deflate * - * Purpose: make a dataset using DEFLATE (GZIP) compression in FID + * Purpose: make a dataset using DEFLATE (GZIP) compression in LOC_ID * *------------------------------------------------------------------------- */ -static int -make_deflate(hid_t fid) +int make_deflate(hid_t loc_id) { hid_t dcpl; /* dataset creation property list */ hid_t dsid; /* dataset ID */ hid_t sid; /* dataspace ID */ - int rank=2; - hsize_t dims[2]={4,2}; - hsize_t chunk_dims[2]={2,1}; - int buf[4][2]={{1,2},{3,4},{5,6},{7,8}}; + hsize_t dims[RANK]={DIM1,DIM2}; + hsize_t chunk_dims[RANK]={CDIM1,CDIM2}; + int buf[40][20]; + int i, j, n=0; + + for (i=0; i + * December, 19, 2003 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static int +test_filter_szip(void) +{ + pack_opt_t pack_options; + diff_opt_t diff_options; + memset(&diff_options, 0, sizeof (diff_opt_t)); + memset(&pack_options, 0, sizeof (pack_opt_t)); + + TESTING(" szip filter"); + +#ifdef H5_HAVE_FILTER_SZIP + + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack_addfilter("dset2:SZIP 8",&pack_options)<0) + TEST_ERROR; + if (h5repack_addchunk("dset2:20x10",&pack_options)<0) + TEST_ERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options)<0) + TEST_ERROR; if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) == 1) TEST_ERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; - PASSED(); + PASSED(); +#else + SKIPPED(); +#endif return 0; +#ifdef H5_HAVE_FILTER_SZIP error: return 1; +#endif + } + /*------------------------------------------------------------------------- * Function: main * @@ -157,7 +244,8 @@ int main (void) puts("Testing h5repack:"); /* make the test files */ - nerrors += make_testfiles(); + if (make_testfiles()<0) + goto error; /* test a copy with no filters */ nerrors += test_copy(); @@ -165,6 +253,8 @@ int main (void) /* test a copy with the deflate filter */ nerrors += test_filter_deflate(); + /* test a copy with the szip filter */ + nerrors += test_filter_szip(); /* check for errors */ if (nerrors) diff --git a/tools/h5repack/testh5repack_make.c b/tools/h5repack/testh5repack_make.c new file mode 100644 index 0000000..34d3029 --- /dev/null +++ b/tools/h5repack/testh5repack_make.c @@ -0,0 +1,318 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "hdf5.h" +#include "h5test.h" +#include "h5repack.h" + + +int make_all_objects(hid_t loc_id); +int make_attributes(hid_t loc_id); +int make_special_objects(hid_t loc_id); + + +/*------------------------------------------------------------------------- + * Function: make_testfiles + * + * Purpose: make a test file with all types of HDF5 objects, + * datatypes and filters + * + *------------------------------------------------------------------------- + */ +int make_testfiles(void) +{ + hid_t loc_id; /* file ID */ + + TESTING(" generating datasets"); + + /* create a file for general copy test */ + if((loc_id = H5Fcreate(FNAME1,H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT))<0) + return -1; + + if (make_all_objects(loc_id)<0) + goto out; + + /* close */ + if(H5Fclose(loc_id)<0) + return -1; + + /* create a file for attributes copy test */ + if((loc_id = H5Fcreate(FNAME2,H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT))<0) + return -1; + + if (make_attributes(loc_id)<0) + goto out; + + /* close */ + if(H5Fclose(loc_id)<0) + return -1; + + /* create a file for special items test */ + if((loc_id = H5Fcreate(FNAME3,H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT))<0) + return -1; + + if (make_special_objects(loc_id)<0) + goto out; + + /* close */ + if(H5Fclose(loc_id)<0) + return -1; + + /* create a file for the filters test */ + if((loc_id = H5Fcreate(FNAME4,H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT))<0) + return -1; + + if (make_nofilters(loc_id)<0) + goto out; + + if (make_deflate(loc_id)<0) + goto out; + + if (make_szip(loc_id)<0) + goto out; + + /* close */ + if(H5Fclose(loc_id)<0) + return -1; + + PASSED(); + return 0; + +out: + H5E_BEGIN_TRY { + H5Fclose(loc_id); + } H5E_END_TRY; + return -1; +} + + + +/*------------------------------------------------------------------------- + * Function: make_all_objects + * + * Purpose: make a test file with all types of HDF5 objects, datatypes + * + *------------------------------------------------------------------------- + */ +int make_all_objects(hid_t loc_id) +{ + hid_t dset_id; + hid_t group_id; + hid_t type_id; + hid_t root_id; + hid_t space_id; + hsize_t dims[1]={2}; + /* Compound datatype */ + typedef struct s_t + { + int a; + float b; + } s_t; + +/*------------------------------------------------------------------------- + * H5G_DATASET + *------------------------------------------------------------------------- + */ + + space_id = H5Screate_simple(1,dims,NULL); + dset_id = H5Dcreate(loc_id,"dset_ref",H5T_NATIVE_INT,space_id,H5P_DEFAULT); + H5Sclose(space_id); + +/*------------------------------------------------------------------------- + * H5G_GROUP + *------------------------------------------------------------------------- + */ + group_id = H5Gcreate(loc_id,"g1",0); + root_id = H5Gopen(loc_id, "/"); + +/*------------------------------------------------------------------------- + * H5G_TYPE + *------------------------------------------------------------------------- + */ + + /* Create a memory compound datatype */ + type_id = H5Tcreate (H5T_COMPOUND, sizeof(s_t)); + H5Tinsert(type_id, "a", HOFFSET(s_t, a), H5T_NATIVE_INT); + H5Tinsert(type_id, "b", HOFFSET(s_t, b), H5T_NATIVE_FLOAT); + /* Commit compound datatype and close it */ + H5Tcommit(loc_id, "type", type_id); + H5Tclose(type_id); + +/*------------------------------------------------------------------------- + * H5G_LINK + *------------------------------------------------------------------------- + */ + + H5Glink(loc_id, H5G_LINK_SOFT, "dset", "link"); + +/*------------------------------------------------------------------------- + * write a series of datasetes on the group, and root group + *------------------------------------------------------------------------- + */ + + write_dset_in(root_id,"dset_ref",loc_id,0); + write_dset_in(group_id,"dset_ref",loc_id,0); + + + /* Close */ + H5Dclose(dset_id); + H5Gclose(group_id); + H5Gclose(root_id); + + return 0; + +} + + +/*------------------------------------------------------------------------- + * Function: make_attributes + * + * Purpose: make a test file with all types of attributes + * + *------------------------------------------------------------------------- + */ +int make_attributes(hid_t loc_id) +{ + hid_t dset_id; + hid_t group_id; + hid_t root_id; + hid_t space_id; + hsize_t dims[1]={2}; + + +/*------------------------------------------------------------------------- + * H5G_DATASET + *------------------------------------------------------------------------- + */ + + space_id = H5Screate_simple(1,dims,NULL); + dset_id = H5Dcreate(loc_id,"dset",H5T_NATIVE_INT,space_id,H5P_DEFAULT); + H5Sclose(space_id); + +/*------------------------------------------------------------------------- + * H5G_GROUP + *------------------------------------------------------------------------- + */ + group_id = H5Gcreate(loc_id,"g1",0); + root_id = H5Gopen(loc_id, "/"); + +/*------------------------------------------------------------------------- + * write a series of attributes on the dataset, group, and root group + *------------------------------------------------------------------------- + */ + + write_attr_in(dset_id,"dset",loc_id,0); + write_attr_in(group_id,"dset",loc_id,0); + write_attr_in(root_id,"dset",loc_id,0); + + /* Close */ + H5Dclose(dset_id); + H5Gclose(group_id); + H5Gclose(root_id); + + return 0; + +} + + + +/*------------------------------------------------------------------------- + * Function: make_special_objects + * + * Purpose: make a test file with non common items + * + *------------------------------------------------------------------------- + */ +int make_special_objects(hid_t loc_id) +{ + hid_t group1_id; + hid_t group2_id; + hid_t group3_id; + hsize_t dims[2]={3,2}; + int buf[3][2]= {{1,1},{1,2},{2,2}}; + hid_t dset_id; + hid_t space_id; + hid_t plist_id; + herr_t status; + int fillvalue=2; + +/*------------------------------------------------------------------------- + * create a dataset and some hard links to it + *------------------------------------------------------------------------- + */ + + if (write_dset(loc_id,2,dims,"dset",H5T_NATIVE_INT,buf)<0) + return -1; + if (H5Glink(loc_id, H5G_LINK_HARD, "dset", "link1 to dset")<0) + return -1; + if (H5Glink(loc_id, H5G_LINK_HARD, "dset", "link2 to dset")<0) + return -1; + if (H5Glink(loc_id, H5G_LINK_HARD, "dset", "link3 to dset")<0) + return -1; + + +/*------------------------------------------------------------------------- + * create a group and some hard links to it + *------------------------------------------------------------------------- + */ + + if ((group1_id = H5Gcreate(loc_id,"g1",0))<0) + return -1; + if ((group2_id = H5Gcreate(group1_id,"g2",0))<0) + return -1; + if ((group3_id = H5Gcreate(group2_id,"g3",0))<0) + return -1; + + /* + H5Glink2(curr_loc_id, current_name, link_type, new_loc_id, new_name ) + hid_t curr_loc_id + IN: The file or group identifier for the original object. + const char * current_name + IN: Name of the existing object if link is a hard link. + H5G_link_t link_type + IN: Link type. Possible values are H5G_LINK_HARD and H5G_LINK_SOFT. + hid_t new_loc_id + IN: The file or group identifier for the new link. + const char * new_name + IN: New name for the object. + */ + + if (H5Glink2(loc_id, "g1", H5G_LINK_HARD, group2_id, "link1 to g1")<0) + return -1; + if (H5Glink2(group1_id, "g2", H5G_LINK_HARD, group3_id, "link1 to g2")<0) + return -1; + + H5Gclose(group1_id); + H5Gclose(group2_id); + H5Gclose(group3_id); + +/*------------------------------------------------------------------------- + * H5T_INTEGER, write a fill value + *------------------------------------------------------------------------- + */ + + plist_id = H5Pcreate(H5P_DATASET_CREATE); + status = H5Pset_fill_value(plist_id, H5T_NATIVE_INT, &fillvalue); + space_id = H5Screate_simple(2,dims,NULL); + dset_id = H5Dcreate(loc_id,"dset_fill",H5T_NATIVE_INT,space_id,plist_id); + status = H5Dwrite(dset_id,H5T_NATIVE_INT,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf); + status = H5Pclose(plist_id); + status = H5Dclose(dset_id); + status = H5Sclose(space_id); + + return 0; + +} + + -- cgit v0.12