diff options
Diffstat (limited to 'tools/src/h5repack/h5repack_filters.c')
-rw-r--r-- | tools/src/h5repack/h5repack_filters.c | 493 |
1 files changed, 493 insertions, 0 deletions
diff --git a/tools/src/h5repack/h5repack_filters.c b/tools/src/h5repack/h5repack_filters.c new file mode 100644 index 0000000..e21b829 --- /dev/null +++ b/tools/src/h5repack/h5repack_filters.c @@ -0,0 +1,493 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "h5repack.h" +#include "h5tools.h" +#include "h5tools_utils.h" + +/* number of members in an array */ +#ifndef NELMTS +# define NELMTS(X) (sizeof(X)/sizeof(X[0])) +#endif + +/* minimum of two values */ +#undef MIN +#define MIN(a,b) (((a)<(b)) ? (a) : (b)) + +/*------------------------------------------------------------------------- + * Function: aux_find_obj + * + * Purpose: find the object name NAME (got from the traverse list) + * in the repack options list + * + *------------------------------------------------------------------------- + */ +static int +aux_find_obj(const char* name, /* object name from traverse list */ + pack_opt_t *options, /* repack options */ + pack_info_t *obj /*OUT*/) /* info about object to filter */ +{ + char *pdest; + int result; + unsigned int i; + + for ( i=0; i<options->op_tbl->nelems; i++) + { + if (HDstrcmp(options->op_tbl->objs[i].path,name)==0) + { + *obj = options->op_tbl->objs[i]; + return (int)i; + } + + pdest = HDstrstr(name,options->op_tbl->objs[i].path); + result = (int)(pdest - name); + + /* found at position 1, meaning without '/' */ + if( pdest != NULL && result==1 ) + { + *obj = options->op_tbl->objs[i]; + return (int)i; + } + }/*i*/ + + return -1; +} + + +/*------------------------------------------------------------------------- + * Function: aux_assign_obj + * + * Purpose: find the object name NAME (got from the traverse list) + * in the repack options list; assign the filter information OBJ + * + * Return: 0 not found, 1 found + * + *------------------------------------------------------------------------- + */ +static int +aux_assign_obj(const char* name, /* object name from traverse list */ + pack_opt_t *options, /* repack options */ + pack_info_t *obj /*OUT*/) /* info about object to filter */ +{ + + int idx, i; + pack_info_t tmp; + + init_packobject(&tmp); + + idx = aux_find_obj(name,options,&tmp); + + /* name was on input */ + if (idx>=0) + { + + + /* applying to all objects */ + if (options->all_layout) + { + /* assign the global layout info to the OBJ info */ + tmp.layout=options->layout_g; + switch (options->layout_g) + { + case H5D_CHUNKED: + tmp.chunk.rank=options->chunk_g.rank; + for ( i=0; i<tmp.chunk.rank; i++) + tmp.chunk.chunk_lengths[i]=options->chunk_g.chunk_lengths[i]; + break; + case H5D_LAYOUT_ERROR: + case H5D_COMPACT: + case H5D_CONTIGUOUS: + case H5D_VIRTUAL: + case H5D_NLAYOUTS: + break; + default: + break; + }/*switch*/ + } + else + { + tmp.layout = options->op_tbl->objs[idx].layout; + switch (tmp.layout) + { + case H5D_CHUNKED: + tmp.chunk.rank = options->op_tbl->objs[idx].chunk.rank; + for ( i=0; i<tmp.chunk.rank; i++) + tmp.chunk.chunk_lengths[i]=options->op_tbl->objs[idx].chunk.chunk_lengths[i]; + break; + case H5D_LAYOUT_ERROR: + case H5D_COMPACT: + case H5D_CONTIGUOUS: + case H5D_VIRTUAL: + case H5D_NLAYOUTS: + break; + default: + break; + }/*switch*/ + + } + + /* applying to all objects */ + if (options->all_filter) + { + /* assign the global filter */ + tmp.nfilters=1; + tmp.filter[0]=options->filter_g[0]; + } /* if all */ + else + { + tmp.nfilters=options->op_tbl->objs[idx].nfilters; + for ( i=0; i<tmp.nfilters; i++) + { + tmp.filter[i] = options->op_tbl->objs[idx].filter[i]; + } + } + + + } /* if idx */ + + + /* no input name */ + + else + { + + if (options->all_filter) + { + int k; + + /* assign the global filters */ + tmp.nfilters=options->n_filter_g; + for ( k = 0; k < options->n_filter_g; k++) + tmp.filter[k]=options->filter_g[k]; + } + if (options->all_layout) + { + /* assign the global layout info to the OBJ info */ + tmp.layout=options->layout_g; + switch (options->layout_g) + { + case H5D_CHUNKED: + tmp.chunk.rank=options->chunk_g.rank; + for ( i=0; i<tmp.chunk.rank; i++) + tmp.chunk.chunk_lengths[i]=options->chunk_g.chunk_lengths[i]; + break; + case H5D_LAYOUT_ERROR: + case H5D_COMPACT: + case H5D_CONTIGUOUS: + case H5D_VIRTUAL: + case H5D_NLAYOUTS: + break; + default: + break; + }/*switch*/ + } + } + + *obj = tmp; + return 1; + +} + + +/*------------------------------------------------------------------------- + * Function: apply_filters + * + * Purpose: apply the filters in the object to the property list; + * do extra checking in the case of SZIP; delete all filters in the case + * of H5Z_FILTER_NONE present in the PACK_INFO_T filter array + * + * Return: 0 success, -1 an error occured + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: December 19, 2003 + * + *------------------------------------------------------------------------- + */ + +int apply_filters(const char* name, /* object name from traverse list */ + int rank, /* rank of dataset */ + hsize_t *dims, /* dimensions of dataset */ + size_t msize, /* size of type */ + hid_t dcpl_id, /* dataset creation property list */ + pack_opt_t *options, /* repack options */ + int *has_filter) /* (OUT) object NAME has a filter */ + + +{ + int nfilters; /* number of filters in DCPL */ + hsize_t chsize[64]; /* chunk size in elements */ + H5D_layout_t layout; + int i; + pack_info_t obj; + + *has_filter = 0; + + if (rank==0) /* scalar dataset, do not apply */ + return 0; + + /*------------------------------------------------------------------------- + * initialize the assigment object + *------------------------------------------------------------------------- + */ + init_packobject(&obj); + + /*------------------------------------------------------------------------- + * find options + *------------------------------------------------------------------------- + */ + if (aux_assign_obj(name,options,&obj)==0) + return 0; + + /* get information about input filters */ + if ((nfilters = H5Pget_nfilters(dcpl_id))<0) + return -1; + + /*------------------------------------------------------------------------- + * check if we have filters in the pipeline + * we want to replace them with the input filters + * only remove if we are inserting new ones + *------------------------------------------------------------------------- + */ + if (nfilters && obj.nfilters ) + { + *has_filter = 1; + if (H5Premove_filter(dcpl_id,H5Z_FILTER_ALL)<0) + return -1; + } + + /*------------------------------------------------------------------------- + * check if there is an existent chunk + * read it only if there is not a requested layout + *------------------------------------------------------------------------- + */ + if (obj.layout == -1 ) + { + if ((layout = H5Pget_layout(dcpl_id))<0) + return -1; + + if (layout == H5D_CHUNKED) + { + if ((rank = H5Pget_chunk(dcpl_id,NELMTS(chsize),chsize/*out*/))<0) + return -1; + obj.layout = H5D_CHUNKED; + obj.chunk.rank = rank; + for ( i = 0; i < rank; i++) + obj.chunk.chunk_lengths[i] = chsize[i]; + } + } + + /*------------------------------------------------------------------------- + * the type of filter and additional parameter + * type can be one of the filters + * H5Z_FILTER_NONE 0 , uncompress if compressed + * H5Z_FILTER_DEFLATE 1 , deflation like gzip + * H5Z_FILTER_SHUFFLE 2 , shuffle the data + * H5Z_FILTER_FLETCHER32 3 , fletcher32 checksum of EDC + * H5Z_FILTER_SZIP 4 , szip compression + * H5Z_FILTER_NBIT 5 , nbit compression + * H5Z_FILTER_SCALEOFFSET 6 , scaleoffset compression + *------------------------------------------------------------------------- + */ + + if (obj.nfilters) + { + + /*------------------------------------------------------------------------- + * filters require CHUNK layout; if we do not have one define a default + *------------------------------------------------------------------------- + */ + if (obj.layout==-1) + { + + /* stripmine info */ + hsize_t sm_size[H5S_MAX_RANK]; /*stripmine size */ + hsize_t sm_nbytes; /*bytes per stripmine */ + + obj.chunk.rank = rank; + + /* + * determine the strip mine size. The strip mine is + * a hyperslab whose size is manageable. + */ + + + + sm_nbytes = msize; + for ( i = rank; i > 0; --i) + { + hsize_t size = H5TOOLS_BUFSIZE / sm_nbytes; + if ( size == 0) /* datum size > H5TOOLS_BUFSIZE */ + size = 1; + sm_size[i - 1] = MIN(dims[i - 1], size); + sm_nbytes *= sm_size[i - 1]; + HDassert(sm_nbytes > 0); + + } + + for ( i = 0; i < rank; i++) + { + obj.chunk.chunk_lengths[i] = sm_size[i]; + } + + } + + for ( i=0; i<obj.nfilters; i++) + { + switch (obj.filter[i].filtn) + { + + /*------------------------------------------------------------------------- + * H5Z_FILTER_NONE 0 , uncompress if compressed + *------------------------------------------------------------------------- + */ + case H5Z_FILTER_NONE: + break; + + /*------------------------------------------------------------------------- + * H5Z_FILTER_DEFLATE 1 , deflation like gzip + *------------------------------------------------------------------------- + */ + case H5Z_FILTER_DEFLATE: + { + unsigned aggression; /* the deflate level */ + + aggression = obj.filter[i].cd_values[0]; + /* set up for deflated data */ + if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) + return -1; + if(H5Pset_deflate(dcpl_id,aggression)<0) + return -1; + } + break; + + /*------------------------------------------------------------------------- + * H5Z_FILTER_SZIP 4 , szip compression + *------------------------------------------------------------------------- + */ + case H5Z_FILTER_SZIP: + { + unsigned options_mask; + unsigned pixels_per_block; + + options_mask = obj.filter[i].cd_values[0]; + pixels_per_block = obj.filter[i].cd_values[1]; + + /* set up for szip data */ + if(H5Pset_chunk(dcpl_id,obj.chunk.rank,obj.chunk.chunk_lengths)<0) + return -1; + if (H5Pset_szip(dcpl_id,options_mask,pixels_per_block)<0) + return -1; + + } + break; + + /*------------------------------------------------------------------------- + * H5Z_FILTER_SHUFFLE 2 , shuffle the data + *------------------------------------------------------------------------- + */ + case H5Z_FILTER_SHUFFLE: + if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) + return -1; + if (H5Pset_shuffle(dcpl_id)<0) + return -1; + break; + + /*------------------------------------------------------------------------- + * H5Z_FILTER_FLETCHER32 3 , fletcher32 checksum of EDC + *------------------------------------------------------------------------- + */ + case H5Z_FILTER_FLETCHER32: + if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) + return -1; + if (H5Pset_fletcher32(dcpl_id)<0) + return -1; + break; + /*----------- ------------------------------------------------------------- + * H5Z_FILTER_NBIT , NBIT compression + *------------------------------------------------------------------------- + */ + case H5Z_FILTER_NBIT: + if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) + return -1; + if (H5Pset_nbit(dcpl_id)<0) + return -1; + break; + /*----------- ------------------------------------------------------------- + * H5Z_FILTER_SCALEOFFSET , scale+offset compression + *------------------------------------------------------------------------- + */ + + case H5Z_FILTER_SCALEOFFSET: + { + H5Z_SO_scale_type_t scale_type; + int scale_factor; + + scale_type = (H5Z_SO_scale_type_t)obj.filter[i].cd_values[0]; + scale_factor = (int)obj.filter[i].cd_values[1]; + + if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) + return -1; + if (H5Pset_scaleoffset(dcpl_id,scale_type,scale_factor)<0) + return -1; + } + break; + default: + { + if (H5Pset_filter (dcpl_id, obj.filter[i].filtn, H5Z_FLAG_MANDATORY, obj.filter[i].cd_nelmts, obj.filter[i].cd_values)<0) + return -1; + if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) + return -1; + } + break; + } /* switch */ + }/*i*/ + + } + /*obj.nfilters*/ + + /*------------------------------------------------------------------------- + * layout + *------------------------------------------------------------------------- + */ + + if (obj.layout>=0) + { + /* a layout was defined */ + if (H5Pset_layout(dcpl_id, obj.layout)<0) + return -1; + + if (H5D_CHUNKED == obj.layout) + { + if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0) + return -1; + } + else if (H5D_COMPACT == obj.layout) + { + if (H5Pset_alloc_time(dcpl_id, H5D_ALLOC_TIME_EARLY)<0) + return -1; + } + /* remove filters for the H5D_CONTIGUOUS case */ + else if (H5D_CONTIGUOUS == obj.layout) + { + if (H5Premove_filter(dcpl_id,H5Z_FILTER_ALL)<0) + return -1; + } + + } + + return 0; +} + |