diff options
author | Pedro Vicente Nunes <pvn@hdfgroup.org> | 2006-11-17 20:00:46 (GMT) |
---|---|---|
committer | Pedro Vicente Nunes <pvn@hdfgroup.org> | 2006-11-17 20:00:46 (GMT) |
commit | 04e7a0ba36f017acde63ff48e98070233294719b (patch) | |
tree | 78a2ab5efdcfc1aff6cbdc4d0cf5a8f349e4fc99 | |
parent | e215d22a79d497518040d74c1d1bd3fce34780cb (diff) | |
download | hdf5-04e7a0ba36f017acde63ff48e98070233294719b.zip hdf5-04e7a0ba36f017acde63ff48e98070233294719b.tar.gz hdf5-04e7a0ba36f017acde63ff48e98070233294719b.tar.bz2 |
[svn-r12936]
make h5repacktst clean a big file which name was changed to "h5repack_big_out.h5", do not use H5Ocopy only when the original DCPL has filters or a request is made for such, more code cleaning
-rw-r--r-- | tools/h5repack/h5repack_copy.c | 11 | ||||
-rw-r--r-- | tools/h5repack/h5repack_filters.c | 643 | ||||
-rw-r--r-- | tools/h5repack/h5repack_parse.c | 24 | ||||
-rw-r--r-- | tools/h5repack/h5repacktst.c | 2 |
4 files changed, 339 insertions, 341 deletions
diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c index f6b738c..76129ad 100644 --- a/tools/h5repack/h5repack_copy.c +++ b/tools/h5repack/h5repack_copy.c @@ -399,18 +399,17 @@ int do_copy_objects(hid_t fidin, /*------------------------------------------------------------------------- * check if we should use H5Ocopy or not - * if the DCPL has filters or a non default layout, we read/write the object + * if the DCPL has filters, we read/write the object * otherwise we do a copy using H5Ocopy *------------------------------------------------------------------------- */ - - if (nfilters || layout!=H5D_CONTIGUOUS - /* does the dcpl has filters or non default layout ? */ + if (nfilters + /* does the dcpl has filters ? */ || options->op_tbl->nelems - /* do we have input request for filter/chunking */ + /* do we have input request for filter/chunk ? */ || - options->all_filter==1 || options->all_layout==1 + options->all_filter==1 || options->all_layout==1 ) { int j; diff --git a/tools/h5repack/h5repack_filters.c b/tools/h5repack/h5repack_filters.c index a525e0d..08b0a3d 100644 --- a/tools/h5repack/h5repack_filters.c +++ b/tools/h5repack/h5repack_filters.c @@ -28,29 +28,29 @@ static int aux_find_obj(const char* name, /* object name from traverse list */ pack_opt_t *options, /* repack options */ - pack_info_t *obj /*OUT*/) /* info about object to filter */ + pack_info_t *obj /*OUT*/) /* info about object to filter */ { char *pdest; int result; - unsigned int i; + unsigned int i; for ( i=0; i<options->op_tbl->nelems; i++) { - if (strcmp(options->op_tbl->objs[i].path,name)==0) - { - *obj = options->op_tbl->objs[i]; - return i; - } - - pdest = strstr(name,options->op_tbl->objs[i].path); - result = (int)(pdest - name); - - /* found at position 1, meaning without '/' */ - if( pdest != NULL && result==1 ) - { - *obj = options->op_tbl->objs[i]; - return i; - } + if (strcmp(options->op_tbl->objs[i].path,name)==0) + { + *obj = options->op_tbl->objs[i]; + return i; + } + + pdest = strstr(name,options->op_tbl->objs[i].path); + result = (int)(pdest - name); + + /* found at position 1, meaning without '/' */ + if( pdest != NULL && result==1 ) + { + *obj = options->op_tbl->objs[i]; + return i; + } }/*i*/ return -1; @@ -72,103 +72,104 @@ int aux_assign_obj(const char* name, /* object name from traverse lis pack_opt_t *options, /* repack options */ pack_info_t *obj /*OUT*/) /* info about object to filter */ { - - int idx, i; - pack_info_t tmp; - - init_packobject(&tmp); - - idx = aux_find_obj(name,options,&tmp); - - /* name was on input */ - if (idx>=0) - { - - /* applying to all objects */ - if (options->all_layout) - { - /* assign the global layout info to the OBJ info */ - tmp.layout=options->layout_g; - switch (options->layout_g) - { - case H5D_CHUNKED: - tmp.chunk.rank=options->chunk_g.rank; - for ( i=0; i<tmp.chunk.rank; i++) - tmp.chunk.chunk_lengths[i]=options->chunk_g.chunk_lengths[i]; - break; - default: - break; - }/*switch*/ - } - else - { - tmp.layout = options->op_tbl->objs[idx].layout; - switch (tmp.layout) - { - case H5D_CHUNKED: - tmp.chunk.rank = options->op_tbl->objs[idx].chunk.rank; - for ( i=0; i<tmp.chunk.rank; i++) - tmp.chunk.chunk_lengths[i]=options->op_tbl->objs[idx].chunk.chunk_lengths[i]; - break; - default: - break; - }/*switch*/ - - } - - /* applying to all objects */ - if (options->all_filter) - { - /* assign the global filter */ - tmp.nfilters=1; - tmp.filter[0]=options->filter_g; - } /* if all */ - else - { - tmp.nfilters=options->op_tbl->objs[idx].nfilters; - for ( i=0; i<tmp.nfilters; i++) - { - tmp.filter[i] = options->op_tbl->objs[idx].filter[i]; - } - } - - - } /* if idx */ - - - /* no input name */ - - else - { - - if (options->all_filter) - { - /* assign the global filter */ - tmp.nfilters=1; - tmp.filter[0]=options->filter_g; - } - if (options->all_layout) - { - /* assign the global layout info to the OBJ info */ - tmp.layout=options->layout_g; - switch (options->layout_g) - { - case H5D_CHUNKED: - tmp.chunk.rank=options->chunk_g.rank; - for ( i=0; i<tmp.chunk.rank; i++) - tmp.chunk.chunk_lengths[i]=options->chunk_g.chunk_lengths[i]; - break; - default: - break; - }/*switch*/ - } - } - - *obj = tmp; - return 1; - + + int idx, i; + pack_info_t tmp; + + init_packobject(&tmp); + + idx = aux_find_obj(name,options,&tmp); + + /* name was on input */ + if (idx>=0) + { + + + /* applying to all objects */ + if (options->all_layout) + { + /* assign the global layout info to the OBJ info */ + tmp.layout=options->layout_g; + switch (options->layout_g) + { + case H5D_CHUNKED: + tmp.chunk.rank=options->chunk_g.rank; + for ( i=0; i<tmp.chunk.rank; i++) + tmp.chunk.chunk_lengths[i]=options->chunk_g.chunk_lengths[i]; + break; + default: + break; + }/*switch*/ + } + else + { + tmp.layout = options->op_tbl->objs[idx].layout; + switch (tmp.layout) + { + case H5D_CHUNKED: + tmp.chunk.rank = options->op_tbl->objs[idx].chunk.rank; + for ( i=0; i<tmp.chunk.rank; i++) + tmp.chunk.chunk_lengths[i]=options->op_tbl->objs[idx].chunk.chunk_lengths[i]; + break; + default: + break; + }/*switch*/ + + } + + /* applying to all objects */ + if (options->all_filter) + { + /* assign the global filter */ + tmp.nfilters=1; + tmp.filter[0]=options->filter_g; + } /* if all */ + else + { + tmp.nfilters=options->op_tbl->objs[idx].nfilters; + for ( i=0; i<tmp.nfilters; i++) + { + tmp.filter[i] = options->op_tbl->objs[idx].filter[i]; + } + } + + + } /* if idx */ + + + /* no input name */ + + else + { + + if (options->all_filter) + { + /* assign the global filter */ + tmp.nfilters=1; + tmp.filter[0]=options->filter_g; + } + if (options->all_layout) + { + /* assign the global layout info to the OBJ info */ + tmp.layout=options->layout_g; + switch (options->layout_g) + { + case H5D_CHUNKED: + tmp.chunk.rank=options->chunk_g.rank; + for ( i=0; i<tmp.chunk.rank; i++) + tmp.chunk.chunk_lengths[i]=options->chunk_g.chunk_lengths[i]; + break; + default: + break; + }/*switch*/ + } + } + + *obj = tmp; + return 1; + } - + /*------------------------------------------------------------------------- @@ -192,215 +193,213 @@ int apply_filters(const char* name, /* object name from traverse list */ hid_t dcpl_id, /* dataset creation property list */ pack_opt_t *options) /* repack options */ { - int nfilters; /* number of filters in DCPL */ - hsize_t chsize[64]; /* chunk size in elements */ - H5D_layout_t layout; - int i; - pack_info_t obj; - - if (rank==0) /* scalar dataset, do not apply */ - return 0; - - -/*------------------------------------------------------------------------- - * initialize the assigment object - *------------------------------------------------------------------------- - */ - init_packobject(&obj); - -/*------------------------------------------------------------------------- - * find options - *------------------------------------------------------------------------- - */ - if (aux_assign_obj(name,options,&obj)==0) - return 0; - - /* get information about input filters */ - if ((nfilters = H5Pget_nfilters(dcpl_id))<0) - return -1; - - /*------------------------------------------------------------------------- - * check if we have filters in the pipeline - * we want to replace them with the input filters - * only remove if we are inserting new ones - *------------------------------------------------------------------------- - */ - if (nfilters && obj.nfilters ) { - if (H5Premove_filter(dcpl_id,H5Z_FILTER_ALL)<0) - return -1; - } - - - /*------------------------------------------------------------------------- - * check if there is an existent chunk - * read it only if there is not a requested layout - *------------------------------------------------------------------------- - */ - if (obj.layout == -1 ) - { - if ((layout = H5Pget_layout(dcpl_id))<0) - return -1; - - if (layout==H5D_CHUNKED) - { - if ((rank = H5Pget_chunk(dcpl_id,NELMTS(chsize),chsize/*out*/))<0) - return -1; - obj.layout=H5D_CHUNKED; - obj.chunk.rank=rank; - for ( i=0; i<rank; i++) - obj.chunk.chunk_lengths[i] = chsize[i]; - } - } - -/*------------------------------------------------------------------------- - * the type of filter and additional parameter - * type can be one of the filters - * H5Z_FILTER_NONE 0, uncompress if compressed - * H5Z_FILTER_DEFLATE 1 , deflation like gzip - * H5Z_FILTER_SHUFFLE 2 , shuffle the data - * H5Z_FILTER_FLETCHER32 3 , fletcher32 checksum of EDC - * H5Z_FILTER_SZIP 4 , szip compression - * H5Z_FILTER_NBIT 5 , nbit compression - * H5Z_FILTER_SCALEOFFSET 6 , scaleoffset compression - *------------------------------------------------------------------------- - */ - - if (obj.nfilters) - { - - /*------------------------------------------------------------------------- - * filters require CHUNK layout; if we do not have one define a default - *------------------------------------------------------------------------- - */ - if (obj.layout==-1) - { - obj.chunk.rank=rank; - for (i=0; i<rank; i++) - obj.chunk.chunk_lengths[i] = dims[i]; - } - - for ( i=0; i<obj.nfilters; i++) - { - switch (obj.filter[i].filtn) - { - default: - break; - - /*------------------------------------------------------------------------- - * H5Z_FILTER_DEFLATE 1 , deflation like gzip - *------------------------------------------------------------------------- - */ - case H5Z_FILTER_DEFLATE: - { - unsigned aggression; /* the deflate level */ - - aggression = obj.filter[i].cd_values[0]; - /* set up for deflated data */ - if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) - return -1; - if(H5Pset_deflate(dcpl_id,aggression)<0) - return -1; - } - break; - - /*------------------------------------------------------------------------- - * H5Z_FILTER_SZIP 4 , szip compression - *------------------------------------------------------------------------- - */ - case H5Z_FILTER_SZIP: - { - unsigned options_mask; - unsigned pixels_per_block; - - options_mask = obj.filter[i].cd_values[0]; - pixels_per_block = obj.filter[i].cd_values[1]; - - /* set up for szip data */ - if(H5Pset_chunk(dcpl_id,obj.chunk.rank,obj.chunk.chunk_lengths)<0) - return -1; - if (H5Pset_szip(dcpl_id,options_mask,pixels_per_block)<0) - return -1; - - } - break; - - /*------------------------------------------------------------------------- - * H5Z_FILTER_SHUFFLE 2 , shuffle the data - *------------------------------------------------------------------------- - */ - case H5Z_FILTER_SHUFFLE: - if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) - return -1; - if (H5Pset_shuffle(dcpl_id)<0) - return -1; - break; - - /*------------------------------------------------------------------------- - * H5Z_FILTER_FLETCHER32 3 , fletcher32 checksum of EDC - *------------------------------------------------------------------------- - */ - case H5Z_FILTER_FLETCHER32: - if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) - return -1; - if (H5Pset_fletcher32(dcpl_id)<0) - return -1; - break; - /*----------- ------------------------------------------------------------- - * H5Z_FILTER_NBIT , NBIT compression - *------------------------------------------------------------------------- - */ - case H5Z_FILTER_NBIT: - if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) - return -1; - if (H5Pset_nbit(dcpl_id)<0) - return -1; - break; - /*----------- ------------------------------------------------------------- - * H5Z_FILTER_SCALEOFFSET , scale+offset compression - *------------------------------------------------------------------------- - */ - - case H5Z_FILTER_SCALEOFFSET: - { - H5Z_SO_scale_type_t scale_type; - int scale_factor; - - scale_type = obj.filter[i].cd_values[0]; - scale_factor = obj.filter[i].cd_values[1]; - - if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) - return -1; - if (H5Pset_scaleoffset(dcpl_id,scale_type,scale_factor)<0) - return -1; - } - break; - } /* switch */ - }/*i*/ - - } - /*obj.nfilters*/ - - /*------------------------------------------------------------------------- - * layout - *------------------------------------------------------------------------- - */ - - if (obj.layout>=0) - { - /* a layout was defined */ - if (H5Pset_layout(dcpl_id, obj.layout)<0) - return -1; - - if (H5D_CHUNKED==obj.layout) { /* set up chunk */ - if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) - return -1; - } - else if (H5D_COMPACT==obj.layout) { - if (H5Pset_alloc_time(dcpl_id, H5D_ALLOC_TIME_EARLY)<0) - return -1; - } - - } + int nfilters; /* number of filters in DCPL */ + hsize_t chsize[64]; /* chunk size in elements */ + H5D_layout_t layout; + int i; + pack_info_t obj; + + if (rank==0) /* scalar dataset, do not apply */ + return 0; + + /*------------------------------------------------------------------------- + * initialize the assigment object + *------------------------------------------------------------------------- + */ + init_packobject(&obj); + + /*------------------------------------------------------------------------- + * find options + *------------------------------------------------------------------------- + */ + if (aux_assign_obj(name,options,&obj)==0) + return 0; + + /* get information about input filters */ + if ((nfilters = H5Pget_nfilters(dcpl_id))<0) + return -1; + + /*------------------------------------------------------------------------- + * check if we have filters in the pipeline + * we want to replace them with the input filters + * only remove if we are inserting new ones + *------------------------------------------------------------------------- + */ + if (nfilters && obj.nfilters ) { + if (H5Premove_filter(dcpl_id,H5Z_FILTER_ALL)<0) + return -1; + } + + /*------------------------------------------------------------------------- + * check if there is an existent chunk + * read it only if there is not a requested layout + *------------------------------------------------------------------------- + */ + if (obj.layout == -1 ) + { + if ((layout = H5Pget_layout(dcpl_id))<0) + return -1; + + if (layout==H5D_CHUNKED) + { + if ((rank = H5Pget_chunk(dcpl_id,NELMTS(chsize),chsize/*out*/))<0) + return -1; + obj.layout=H5D_CHUNKED; + obj.chunk.rank=rank; + for ( i=0; i<rank; i++) + obj.chunk.chunk_lengths[i] = chsize[i]; + } + } + + /*------------------------------------------------------------------------- + * the type of filter and additional parameter + * type can be one of the filters + * H5Z_FILTER_NONE 0, uncompress if compressed + * H5Z_FILTER_DEFLATE 1 , deflation like gzip + * H5Z_FILTER_SHUFFLE 2 , shuffle the data + * H5Z_FILTER_FLETCHER32 3 , fletcher32 checksum of EDC + * H5Z_FILTER_SZIP 4 , szip compression + * H5Z_FILTER_NBIT 5 , nbit compression + * H5Z_FILTER_SCALEOFFSET 6 , scaleoffset compression + *------------------------------------------------------------------------- + */ + + if (obj.nfilters) + { + + /*------------------------------------------------------------------------- + * filters require CHUNK layout; if we do not have one define a default + *------------------------------------------------------------------------- + */ + if (obj.layout==-1) + { + obj.chunk.rank=rank; + for (i=0; i<rank; i++) + obj.chunk.chunk_lengths[i] = dims[i]; + } + + for ( i=0; i<obj.nfilters; i++) + { + switch (obj.filter[i].filtn) + { + default: + break; + + /*------------------------------------------------------------------------- + * H5Z_FILTER_DEFLATE 1 , deflation like gzip + *------------------------------------------------------------------------- + */ + case H5Z_FILTER_DEFLATE: + { + unsigned aggression; /* the deflate level */ + + aggression = obj.filter[i].cd_values[0]; + /* set up for deflated data */ + if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) + return -1; + if(H5Pset_deflate(dcpl_id,aggression)<0) + return -1; + } + break; + + /*------------------------------------------------------------------------- + * H5Z_FILTER_SZIP 4 , szip compression + *------------------------------------------------------------------------- + */ + case H5Z_FILTER_SZIP: + { + unsigned options_mask; + unsigned pixels_per_block; + + options_mask = obj.filter[i].cd_values[0]; + pixels_per_block = obj.filter[i].cd_values[1]; + + /* set up for szip data */ + if(H5Pset_chunk(dcpl_id,obj.chunk.rank,obj.chunk.chunk_lengths)<0) + return -1; + if (H5Pset_szip(dcpl_id,options_mask,pixels_per_block)<0) + return -1; + + } + break; + + /*------------------------------------------------------------------------- + * H5Z_FILTER_SHUFFLE 2 , shuffle the data + *------------------------------------------------------------------------- + */ + case H5Z_FILTER_SHUFFLE: + if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) + return -1; + if (H5Pset_shuffle(dcpl_id)<0) + return -1; + break; + + /*------------------------------------------------------------------------- + * H5Z_FILTER_FLETCHER32 3 , fletcher32 checksum of EDC + *------------------------------------------------------------------------- + */ + case H5Z_FILTER_FLETCHER32: + if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) + return -1; + if (H5Pset_fletcher32(dcpl_id)<0) + return -1; + break; + /*----------- ------------------------------------------------------------- + * H5Z_FILTER_NBIT , NBIT compression + *------------------------------------------------------------------------- + */ + case H5Z_FILTER_NBIT: + if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) + return -1; + if (H5Pset_nbit(dcpl_id)<0) + return -1; + break; + /*----------- ------------------------------------------------------------- + * H5Z_FILTER_SCALEOFFSET , scale+offset compression + *------------------------------------------------------------------------- + */ + + case H5Z_FILTER_SCALEOFFSET: + { + H5Z_SO_scale_type_t scale_type; + int scale_factor; + + scale_type = obj.filter[i].cd_values[0]; + scale_factor = obj.filter[i].cd_values[1]; + + if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) + return -1; + if (H5Pset_scaleoffset(dcpl_id,scale_type,scale_factor)<0) + return -1; + } + break; + } /* switch */ + }/*i*/ + + } + /*obj.nfilters*/ + + /*------------------------------------------------------------------------- + * layout + *------------------------------------------------------------------------- + */ + + if (obj.layout>=0) + { + /* a layout was defined */ + if (H5Pset_layout(dcpl_id, obj.layout)<0) + return -1; + + if (H5D_CHUNKED==obj.layout) { /* set up chunk */ + if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) + return -1; + } + else if (H5D_COMPACT==obj.layout) { + if (H5Pset_alloc_time(dcpl_id, H5D_ALLOC_TIME_EARLY)<0) + return -1; + } + + } return 0; } diff --git a/tools/h5repack/h5repack_parse.c b/tools/h5repack/h5repack_parse.c index 7b173cc..c049026 100644 --- a/tools/h5repack/h5repack_parse.c +++ b/tools/h5repack/h5repack_parse.c @@ -187,20 +187,20 @@ obj_list_t* parse_filter(const char *str, /*------------------------------------------------------------------------- * H5Z_FILTER_SCALEOFFSET - * scaleoffset has the format SOFF=<scale_factor,scale_type> - * scale_type can be - * integer datatype, H5Z_SO_INT (IN) - * float datatype using D-scaling method, H5Z_SO_FLOAT_DSCALE (DS) - * float datatype using E-scaling method, H5Z_SO_FLOAT_ESCALE (ES) , not yet implemented - * for integer datatypes, scale_factor denotes Minimum Bits - * for float datatypes, scale_factor denotes decimal scale factor + * scaleoffset has the format SOFF=<scale_factor,scale_type> + * scale_type can be + * integer datatype, H5Z_SO_INT (IN) + * float datatype using D-scaling method, H5Z_SO_FLOAT_DSCALE (DS) + * float datatype using E-scaling method, H5Z_SO_FLOAT_ESCALE (ES) , not yet implemented + * for integer datatypes, scale_factor denotes Minimum Bits + * for float datatypes, scale_factor denotes decimal scale factor * examples - * SOFF=31,IN - * SOFF=3,DF + * SOFF=31,IN + * SOFF=3,DF *------------------------------------------------------------------------- */ - else if (strcmp(scomp,"SOFF")==0) + else if (strcmp(scomp,"SOFF")==0) { l=-1; /* mask index check */ for ( m=0,u=i+1; u<len; u++,m++) @@ -229,9 +229,9 @@ obj_list_t* parse_filter(const char *str, i=len-1; /* end */ (*n_objs)--; /* we counted an extra ',' */ if (strcmp(smask,"IN")==0) - filt->cd_values[j++]=H5Z_SO_INT; + filt->cd_values[j++]=H5Z_SO_INT; else if (strcmp(smask,"DS")==H5Z_SO_FLOAT_DSCALE) - filt->cd_values[j++]=H5Z_SO_FLOAT_DSCALE; + filt->cd_values[j++]=H5Z_SO_FLOAT_DSCALE; else { error_msg(progname, "scale type must be 'IN' or 'DS' \n"); diff --git a/tools/h5repack/h5repacktst.c b/tools/h5repack/h5repacktst.c index df231aa..91cd215 100644 --- a/tools/h5repack/h5repacktst.c +++ b/tools/h5repack/h5repacktst.c @@ -64,7 +64,7 @@ const char *H5REPACK_FILENAMES[] = { - "test_bigout", + "h5repack_big_out", NULL }; |