From 92b6e817897817df03ad9f9b171ea4e11b901834 Mon Sep 17 00:00:00 2001 From: Quincey Koziol Date: Tue, 8 Apr 2003 21:34:21 -0500 Subject: [svn-r6611] Purpose: Code cleanup/new features Description: Switch over to a new style for registering filters with the library - instead passing in an ID, a string and a callback function to H5Zregister, the client should pass in a single pointer to a H5Z_claass_t struct which contains the ID, the description string and all the function callbacks as fields. Added support for a new "can apply" callback for each filter, which is called when a dataset is created to check whether the parameters for that filter apply correctly to the combination of the datatype and the chunk size (i.e. dataspace) for the dataset. Added support for a new "set local" callback for each filter, which is called when a dataset is created (after the "can apply" filter callback) and sets filter parameters that are specific to that particular dataset. Switched the filters we ship over to use the new H5Z_class_t struct for their internal registrations and also added "set local" callbacks to the szip and shuffle filters and a "can apply" callback to the szip filter. Lots of other code cleanups, etc. also Solution: Platforms tested: FreeBSD 4.8 (sleipnir) w/szip Linux 2.4 (sleipnir) w/szip Solaris 2.7 (arabica) w/FORTRAN IRIX64 6.5 (modi4) w/szip, FORTRAN & parallel Misc. update: --- src/H5Z.c | 648 ++++++++++++++++++++++++++++++++++++++++++++++------ src/H5Zdeflate.c | 19 +- src/H5Zfletcher32.c | 27 ++- src/H5Zpkg.h | 56 +++++ src/H5Zprivate.h | 63 ++--- src/H5Zpublic.h | 69 +++++- src/H5Zshuffle.c | 78 ++++++- src/H5Zszip.c | 189 ++++++++++++++- src/Makefile.in | 4 +- 9 files changed, 1027 insertions(+), 126 deletions(-) create mode 100644 src/H5Zpkg.h diff --git a/src/H5Z.c b/src/H5Z.c index cb96be5..b4b63fd 100644 --- a/src/H5Z.c +++ b/src/H5Z.c @@ -12,11 +12,17 @@ * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include "H5private.h" -#include "H5Eprivate.h" -#include "H5MMprivate.h" -#include "H5Oprivate.h" -#include "H5Zprivate.h" +#define H5Z_PACKAGE /*suppress error about including H5Zpkg */ + +#include "H5private.h" /* Generic Functions */ +#include "H5Dprivate.h" /* Dataset functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Iprivate.h" /* IDs */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5Oprivate.h" /* Object headers */ +#include "H5Pprivate.h" /* Property lists */ +#include "H5Sprivate.h" /* Dataspace functions */ +#include "H5Zpkg.h" /* Data filters */ /* Pablo mask */ #define PABLO_MASK H5Z_mask @@ -26,12 +32,33 @@ static int interface_initialize_g = 0; static herr_t H5Z_init_interface (void); +/* Local typedefs */ +#ifdef H5Z_DEBUG +typedef struct H5Z_stats_t { + struct { + hsize_t total; /*total number of bytes processed */ + hsize_t errors; /*bytes of total attributable to errors */ + H5_timer_t timer; /*execution time including errors */ + } stats[2]; /*0=output, 1=input */ +} H5Z_stats_t; +#endif /* H5Z_DEBUG */ + +/* Enumerated type for dataset creation prelude callbacks */ +typedef enum { + H5Z_PRELUDE_CAN_APPLY, /* Call "can apply" callback */ + H5Z_PRELUDE_SET_LOCAL /* Call "set local" callback */ +} H5Z_prelude_type_t; + /* Local variables */ static size_t H5Z_table_alloc_g = 0; static size_t H5Z_table_used_g = 0; static H5Z_class_t *H5Z_table_g = NULL; +#ifdef H5Z_DEBUG +static H5Z_stats_t *H5Z_stat_table_g = NULL; +#endif /* H5Z_DEBUG */ /* Local functions */ +static int H5Z_find_idx(H5Z_filter_t id); /*------------------------------------------------------------------------- @@ -51,22 +78,31 @@ static H5Z_class_t *H5Z_table_g = NULL; static herr_t H5Z_init_interface (void) { + herr_t ret_value=SUCCEED; /* Return value */ + FUNC_ENTER_NOINIT(H5Z_init_interface); #ifdef H5_HAVE_FILTER_DEFLATE - H5Z_register (H5Z_FILTER_DEFLATE, "deflate", H5Z_filter_deflate); + if (H5Z_register (H5Z_DEFLATE)<0) + HGOTO_ERROR (H5E_PLINE, H5E_CANTINIT, FAIL, "unable to register deflate filter"); #endif /* H5_HAVE_FILTER_DEFLATE */ #ifdef H5_HAVE_FILTER_SHUFFLE - H5Z_register (H5Z_FILTER_SHUFFLE, "shuffle", H5Z_filter_shuffle); + if (H5Z_register (H5Z_SHUFFLE)<0) + HGOTO_ERROR (H5E_PLINE, H5E_CANTINIT, FAIL, "unable to register shuffle filter"); #endif /* H5_HAVE_FILTER_SHUFFLE */ #ifdef H5_HAVE_FILTER_FLETCHER32 - H5Z_register (H5Z_FILTER_FLETCHER32, "fletcher32", H5Z_filter_fletcher32); + if (H5Z_register (H5Z_FLETCHER32)<0) + HGOTO_ERROR (H5E_PLINE, H5E_CANTINIT, FAIL, "unable to register fletcher32 filter"); #endif /* H5_HAVE_FILTER_FLETCHER32 */ #ifdef H5_HAVE_FILTER_SZIP - H5Z_register (H5Z_FILTER_SZIP, "szip", H5Z_filter_szip); + if (H5Z_register (H5Z_SZIP)<0) + HGOTO_ERROR (H5E_PLINE, H5E_CANTINIT, FAIL, "unable to register szip filter"); #endif /* H5_HAVE_FILTER_SZIP */ - FUNC_LEAVE_NOAPI(SUCCEED); +#if (defined H5_HAVE_FILTER_DEFLATE | defined H5_HAVE_FILTER_FLETCHER32 | defined H5_HAVE_FILTER_SHUFFLE | defined H5_HAVE_FILTER_SZIP) +done: +#endif /* (defined H5_HAVE_FILTER_DEFLATE | defined H5_HAVE_FILTER_FLETCHER32 | defined H5_HAVE_FILTER_SHUFFLE | defined H5_HAVE_FILTER_SZIP) */ + FUNC_LEAVE_NOAPI(ret_value); } @@ -98,7 +134,7 @@ H5Z_term_interface (void) if (H5DEBUG(Z)) { for (i=0; i", comment, - H5Z_table_g[i].stats[dir].total, - H5Z_table_g[i].stats[dir].errors, - H5Z_table_g[i].stats[dir].timer.utime, - H5Z_table_g[i].stats[dir].timer.stime, - H5Z_table_g[i].stats[dir].timer.etime, + H5Z_stat_table_g[i].stats[dir].total, + H5Z_stat_table_g[i].stats[dir].errors, + H5Z_stat_table_g[i].stats[dir].timer.utime, + H5Z_stat_table_g[i].stats[dir].timer.stime, + H5Z_stat_table_g[i].stats[dir].timer.etime, bandwidth); } } } -#endif - /* Free the table */ - for (i=0; iid<0 || cls->id>H5Z_FILTER_MAX) + HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, "invalid filter identification number"); + if (cls->idfilter==NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no filter function specified"); + /* Do it */ - if (H5Z_register (id, comment, func)<0) + if (H5Z_register (cls)<0) HGOTO_ERROR (H5E_PLINE, H5E_CANTINIT, FAIL, "unable to register filter"); done: FUNC_LEAVE_API(ret_value); } +#endif /* H5_WANT_H5_V1_4_COMPAT */ /*------------------------------------------------------------------------- @@ -210,42 +299,54 @@ done: *------------------------------------------------------------------------- */ herr_t -H5Z_register (H5Z_filter_t id, const char *comment, H5Z_func_t func) +H5Z_register (const H5Z_class_t *cls) { size_t i; herr_t ret_value=SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(H5Z_register, FAIL); - assert (id>=0 && id<=H5Z_FILTER_MAX); + assert (cls); + assert (cls->id>=0 && cls->id<=H5Z_FILTER_MAX); /* Is the filter already registered? */ - for (i=0; iid) + break; + + /* Filter not already registered */ if (i>=H5Z_table_used_g) { if (H5Z_table_used_g>=H5Z_table_alloc_g) { size_t n = MAX(H5Z_MAX_NFILTERS, 2*H5Z_table_alloc_g); H5Z_class_t *table = H5MM_realloc(H5Z_table_g, n*sizeof(H5Z_class_t)); +#ifdef H5Z_DEBUG + H5Z_stats_t *stat_table = H5MM_realloc(H5Z_stat_table_g, + n*sizeof(H5Z_stats_t)); +#endif /* H5Z_DEBUG */ if (!table) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to extend filter table"); H5Z_table_g = table; +#ifdef H5Z_DEBUG + if (!stat_table) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to extend filter statistics table"); + H5Z_stat_table_g = stat_table; +#endif /* H5Z_DEBUG */ H5Z_table_alloc_g = n; - } + } /* end if */ /* Initialize */ i = H5Z_table_used_g++; - HDmemset(H5Z_table_g+i, 0, sizeof(H5Z_class_t)); - H5Z_table_g[i].id = id; - H5Z_table_g[i].name = H5MM_xstrdup(comment); - H5Z_table_g[i].func = func; - } else { + HDmemcpy(H5Z_table_g+i, cls, sizeof(H5Z_class_t)); +#ifdef H5Z_DEBUG + HDmemset(H5Z_stat_table_g+i, 0, sizeof(H5Z_stats_t)); +#endif /* H5Z_DEBUG */ + } /* end if */ + /* Filter already registered */ + else { /* Replace old contents */ - H5MM_xfree(H5Z_table_g[i].name); - H5Z_table_g[i].name = H5MM_xstrdup(comment); - H5Z_table_g[i].func = func; - } + HDmemcpy(H5Z_table_g+i, cls, sizeof(H5Z_class_t)); + } /* end else */ done: FUNC_LEAVE_NOAPI(ret_value); @@ -310,7 +411,7 @@ H5Z_unregister (H5Z_filter_t id) size_t i; /* Local index variable */ herr_t ret_value=SUCCEED; /* Return value */ - FUNC_ENTER_NOINIT(H5Z_unregister); + FUNC_ENTER_NOAPI(H5Z_unregister,FAIL); assert (id>=0 && id<=H5Z_FILTER_MAX); @@ -326,6 +427,9 @@ H5Z_unregister (H5Z_filter_t id) /* Remove filter from table */ /* Don't worry about shrinking table size (for now) */ HDmemmove(&H5Z_table_g[i],&H5Z_table_g[i+1],sizeof(H5Z_class_t)*((H5Z_table_used_g-1)-i)); +#ifdef H5Z_DEBUG + HDmemmove(&H5Z_stat_table_g[i],&H5Z_stat_table_g[i+1],sizeof(H5Z_stats_t)*((H5Z_table_used_g-1)-i)); +#endif /* H5Z_DEBUG */ H5Z_table_used_g--; done: @@ -373,6 +477,295 @@ done: /*------------------------------------------------------------------------- + * Function: H5Z_prelude_callback + * + * Purpose: Makes a dataset creation "prelude" callback for the "can_apply" + * or "set_local" routines. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Friday, April 4, 2003 + * + * Notes: + * The chunk dimensions are used to create a dataspace, instead + * of passing in the dataset's dataspace, since the chunk + * dimensions are what the I/O filter will actually see + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_prelude_callback(hid_t dcpl_id, hid_t type_id, H5Z_prelude_type_t prelude_type) +{ + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOINIT(H5Z_prelude_callback); + + assert (H5I_GENPROP_LST==H5I_get_type(dcpl_id)); + assert (H5I_DATATYPE==H5I_get_type(type_id)); + + /* Check if the property list is non-default */ + if(dcpl_id!=H5P_DATASET_CREATE_DEFAULT) { + H5P_genplist_t *dc_plist; /* Dataset creation property list object */ + H5D_layout_t dcpl_layout; /* Dataset's layout information */ + + /* Get dataset creation property list object */ + if (NULL == (dc_plist = H5I_object(dcpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get dataset creation property list"); + + /* Get layout information */ + if(H5P_get(dc_plist, H5D_CRT_LAYOUT_NAME, &dcpl_layout) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't retrieve layout"); + + /* Check if the dataset is chunked */ + if(H5D_CHUNKED == dcpl_layout) { + H5O_pline_t dcpl_pline; /* Dataset's I/O pipeline information */ + + /* Get I/O pipeline information */ + if(H5P_get(dc_plist, H5D_CRT_DATA_PIPELINE_NAME, &dcpl_pline) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't retrieve pipeline filter"); + + /* Check if the chunks have filters */ + if(dcpl_pline.nfilters > 0) { + int chunk_ndims; /* # of chunk dimensions */ + hsize_t chunk_size[H5O_LAYOUT_NDIMS]; /* Size of chunk dimensions */ + H5S_t *space; /* Dataspace describing chunk */ + hid_t space_id; /* ID for dataspace describing chunk */ + size_t u; /* Local index variable */ + + /* Get chunk information */ + if(H5P_get(dc_plist, H5D_CRT_CHUNK_DIM_NAME, &chunk_ndims) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't retrieve chunk dimensions"); + if(H5P_get(dc_plist, H5D_CRT_CHUNK_SIZE_NAME, chunk_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't retrieve chunk size"); + + /* Create a data space for a chunk & set the extent */ + if(NULL == (space = H5S_create_simple(chunk_ndims,chunk_size,NULL))) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCREATE, FAIL, "can't create simple dataspace"); + + /* Get ID for dataspace to pass to filter routines */ + if ((space_id=H5I_register (H5I_DATASPACE, space))<0) { + H5S_close(space); + HGOTO_ERROR (H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to register dataspace ID"); + } /* end if */ + + /* Iterate over filters */ + for (u=0; ucan_apply) { + /* Make callback to filter's "can apply" function */ + herr_t status=(fclass->can_apply)(dcpl_id, type_id, space_id); + + /* Check return value */ + if(status<=0) { + /* We're leaving, so close dataspace */ + if(H5Sclose(space_id)<0) + HGOTO_ERROR (H5E_PLINE, H5E_CANTRELEASE, FAIL, "unable to close dataspace"); + + /* Indicate filter can't apply to this combination of parameters */ + if(status==0) { + HGOTO_ERROR(H5E_PLINE, H5E_CANAPPLY, FAIL, "filter parameters not appropriate"); + } /* end if */ + /* Indicate error during filter callback */ + else { + HGOTO_ERROR(H5E_PLINE, H5E_CANAPPLY, FAIL, "error during user callback"); + } /* end if */ + } /* end if */ + } /* end if */ + break; + + case H5Z_PRELUDE_SET_LOCAL: + /* Check if there is a "set local" callback */ + if(fclass->set_local) { + /* Make callback to filter's "set local" function */ + if((fclass->set_local)(dcpl_id, type_id, space_id)<0) { + /* We're leaving, so close dataspace */ + if(H5Sclose(space_id)<0) + HGOTO_ERROR (H5E_PLINE, H5E_CANTRELEASE, FAIL, "unable to close dataspace"); + + /* Indicate error during filter callback */ + HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "error during user callback"); + } /* end if */ + } /* end if */ + break; + + default: + assert("invalid prelude type" && 0); + } /* end switch */ + } /* end else */ + } /* end for */ + + /* Close dataspace */ + if(H5Sclose(space_id)<0) + HGOTO_ERROR (H5E_PLINE, H5E_CANTRELEASE, FAIL, "unable to close dataspace"); + } /* end if */ + } /* end if */ + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5Z_prelude_callback() */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_can_apply + * + * Purpose: Checks if all the filters defined in the dataset creation + * property list can be applied to a particular combination of + * datatype and dataspace for a dataset. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Thursday, April 3, 2003 + * + * Notes: + * The chunk dimensions are used to create a dataspace, instead + * of passing in the dataset's dataspace, since the chunk + * dimensions are what the I/O filter will actually see + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5Z_can_apply (hid_t dcpl_id, hid_t type_id) +{ + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_can_apply,FAIL); + + assert (H5I_GENPROP_LST==H5I_get_type(dcpl_id)); + assert (H5I_DATATYPE==H5I_get_type(type_id)); + + /* Make "can apply" callbacks for filters in pipeline */ + if(H5Z_prelude_callback(dcpl_id, type_id, H5Z_PRELUDE_CAN_APPLY)<0) + HGOTO_ERROR(H5E_PLINE, H5E_CANAPPLY, FAIL, "filter parameters not appropriate"); + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5Z_can_apply() */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_set_local + * + * Purpose: Makes callbacks to modify dataset creation list property + * settings for filters on a new dataset, based on the datatype + * and dataspace of that dataset (chunk). + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Friday, April 4, 2003 + * + * Notes: + * The chunk dimensions are used to create a dataspace, instead + * of passing in the dataset's dataspace, since the chunk + * dimensions are what the I/O filter will actually see + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5Z_set_local (hid_t dcpl_id, hid_t type_id) +{ + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_set_local,FAIL); + + assert (H5I_GENPROP_LST==H5I_get_type(dcpl_id)); + assert (H5I_DATATYPE==H5I_get_type(type_id)); + + /* Make "set local" callbacks for filters in pipeline */ + if(H5Z_prelude_callback(dcpl_id, type_id, H5Z_PRELUDE_SET_LOCAL)<0) + HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "local filter parameters not set"); + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5Z_set_local() */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_modify + * + * Purpose: Modify filter parameters for specified pipeline. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Friday, April 5, 2003 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5Z_modify(H5O_pline_t *pline, H5Z_filter_t filter, unsigned flags, + size_t cd_nelmts, const unsigned int cd_values[/*cd_nelmts*/]) +{ + size_t idx; /* Index of filter in pipeline */ + size_t i; /* Local index variable */ + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_modify, FAIL); + + assert(pline); + assert(filter>=0 && filter<=H5Z_FILTER_MAX); + assert(0==(flags & ~((unsigned)H5Z_FLAG_DEFMASK))); + assert(0==cd_nelmts || cd_values); + + /* Locate the filter in the pipeline */ + for(idx=0; idxnfilters; idx++) + if(pline->filter[idx].id==filter) + break; + + /* Check if the filter was not already in the pipeline */ + if(idx>pline->nfilters) + HGOTO_ERROR(H5E_PLINE, H5E_NOTFOUND, FAIL, "filter not in pipeline"); + + /* Change parameters for filter */ + pline->filter[idx].flags = flags; + pline->filter[idx].cd_nelmts = cd_nelmts; + + /* Free any existing parameters */ + if(pline->filter[idx].cd_values!=NULL) + H5MM_xfree(pline->filter[idx].cd_values); + + /* Set parameters */ + if (cd_nelmts>0) { + pline->filter[idx].cd_values = H5MM_malloc(cd_nelmts*sizeof(unsigned)); + if (NULL==pline->filter[idx].cd_values) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for filter parameters"); + for (i=0; ifilter[idx].cd_values[i] = cd_values[i]; + } /* end if */ + else + pline->filter[idx].cd_values = NULL; + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5Z_modify() */ + + +/*------------------------------------------------------------------------- * Function: H5Z_append * * Purpose: Append another filter to the specified pipeline. @@ -441,38 +834,74 @@ done: /*------------------------------------------------------------------------- + * Function: H5Z_find_idx + * + * Purpose: Given a filter ID return the offset in the global array + * that holds all the registered filters. + * + * Return: Success: Non-negative index of entry in global filter table. + * Failure: Negative + * + * Programmer: Quincey Koziol + * Friday, April 5, 2003 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static int +H5Z_find_idx(H5Z_filter_t id) +{ + size_t i; /* Local index variable */ + int ret_value=FAIL; /* Return value */ + + FUNC_ENTER_NOINIT(H5Z_find_idx); + + for (i=0; i0); @@ -536,22 +966,24 @@ H5Z_pipeline(H5F_t UNUSED *f, const H5O_pline_t *pline, unsigned flags, failed |= (unsigned)1 << idx; continue;/*filter excluded*/ } - if (NULL==(fclass=H5Z_find(pline->filter[idx].id))) { + if ((fclass_idx=H5Z_find_idx(pline->filter[idx].id))<0) { failed |= (unsigned)1 << idx; HGOTO_ERROR(H5E_PLINE, H5E_READERROR, FAIL, "required filter is not registered"); } + fclass=&H5Z_table_g[fclass_idx]; #ifdef H5Z_DEBUG + fstats=&H5Z_stat_table_g[fclass_idx]; H5_timer_begin(&timer); #endif tmp_flags=flags|(pline->filter[idx].flags); tmp_flags|=(edc_read== H5Z_DISABLE_EDC) ? H5Z_FLAG_SKIP_EDC : 0; - new_nbytes = (fclass->func)(tmp_flags, pline->filter[idx].cd_nelmts, + new_nbytes = (fclass->filter)(tmp_flags, pline->filter[idx].cd_nelmts, pline->filter[idx].cd_values, *nbytes, buf_size, buf); #ifdef H5Z_DEBUG - H5_timer_end(&(fclass->stats[1].timer), &timer); - fclass->stats[1].total += MAX(*nbytes, new_nbytes); - if (0==new_nbytes) fclass->stats[1].errors += *nbytes; + H5_timer_end(&(fstats->stats[1].timer), &timer); + fstats->stats[1].total += MAX(*nbytes, new_nbytes); + if (0==new_nbytes) fstats->stats[1].errors += *nbytes; #endif if(0==new_nbytes) { @@ -572,7 +1004,7 @@ H5Z_pipeline(H5F_t UNUSED *f, const H5O_pline_t *pline, unsigned flags, failed |= (unsigned)1 << idx; continue; /*filter excluded*/ } - if (NULL==(fclass=H5Z_find(pline->filter[idx].id))) { + if ((fclass_idx=H5Z_find_idx(pline->filter[idx].id))<0) { failed |= (unsigned)1 << idx; if (pline->filter[idx].flags & H5Z_FLAG_OPTIONAL) { H5E_clear(); @@ -581,15 +1013,17 @@ H5Z_pipeline(H5F_t UNUSED *f, const H5O_pline_t *pline, unsigned flags, HGOTO_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL, "required filter is not registered"); } } + fclass=&H5Z_table_g[fclass_idx]; #ifdef H5Z_DEBUG + fstats=&H5Z_stat_table_g[fclass_idx]; H5_timer_begin(&timer); #endif - new_nbytes = (fclass->func)(flags|(pline->filter[idx].flags), pline->filter[idx].cd_nelmts, + new_nbytes = (fclass->filter)(flags|(pline->filter[idx].flags), pline->filter[idx].cd_nelmts, pline->filter[idx].cd_values, *nbytes, buf_size, buf); #ifdef H5Z_DEBUG - H5_timer_end(&(fclass->stats[0].timer), &timer); - fclass->stats[0].total += MAX(*nbytes, new_nbytes); - if (0==new_nbytes) fclass->stats[0].errors += *nbytes; + H5_timer_end(&(fstats->stats[0].timer), &timer); + fstats->stats[0].total += MAX(*nbytes, new_nbytes); + if (0==new_nbytes) fstats->stats[0].errors += *nbytes; #endif if(0==new_nbytes) { if (0==(pline->filter[idx].flags & H5Z_FLAG_OPTIONAL)) { @@ -616,3 +1050,89 @@ done: FUNC_LEAVE_NOAPI(ret_value); } + +/*------------------------------------------------------------------------- + * Function: H5Z_filter_info + * + * Purpose: Get pointer to filter info for pipeline + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Friday, April 5, 2003 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +H5Z_filter_info_t * +H5Z_filter_info(H5O_pline_t *pline, H5Z_filter_t filter) +{ + size_t idx; /* Index of filter in pipeline */ + H5Z_filter_info_t *ret_value; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_filter_info, NULL); + + assert(pline); + assert(filter>=0 && filter<=H5Z_FILTER_MAX); + + /* Locate the filter in the pipeline */ + for(idx=0; idxnfilters; idx++) + if(pline->filter[idx].id==filter) + break; + + /* Check if the filter was not already in the pipeline */ + if(idx>pline->nfilters) + HGOTO_ERROR(H5E_PLINE, H5E_NOTFOUND, NULL, "filter not in pipeline"); + + /* Set return value */ + ret_value=&pline->filter[idx]; + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5Z_filter_info() */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_all_filters_avail + * + * Purpose: Verify that all the filters in a pipeline are currently + * available (i.e. registered) + * + * Return: Non-negative (TRUE/FALSE) on success/Negative on failure + * + * Programmer: Quincey Koziol + * Tuesday, April 8, 2003 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +htri_t +H5Z_all_filters_avail(H5O_pline_t *pline) +{ + size_t i,j; /* Local index variable */ + htri_t ret_value=TRUE; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_all_filters_avail, UFAIL); + + /* Check args */ + assert(pline); + + /* Iterate through all the filters in pipeline */ + for(i=0; infilters; i++) { + + /* Look for each filter in the list of registered filters */ + for(j=0; jfilter[i].id) + break; + + /* Check if we didn't find the filter */ + if(j==H5Z_table_used_g) + HGOTO_DONE(FALSE); + } /* end for */ + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5Z_all_filters_avail() */ + diff --git a/src/H5Zdeflate.c b/src/H5Zdeflate.c index 144c873..89f15cc 100644 --- a/src/H5Zdeflate.c +++ b/src/H5Zdeflate.c @@ -17,10 +17,12 @@ * Friday, August 27, 1999 */ +#define H5Z_PACKAGE /*suppress error about including H5Zpkg */ + #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ #include "H5MMprivate.h" /* Memory management */ -#include "H5Zprivate.h" /* Data filters */ +#include "H5Zpkg.h" /* Data filters */ #ifdef H5_HAVE_FILTER_DEFLATE @@ -33,6 +35,19 @@ #define INTERFACE_INIT NULL static int interface_initialize_g = 0; +/* Local function prototypes */ +static size_t H5Z_filter_deflate (unsigned flags, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); + +/* This message derives from H5Z */ +const H5Z_class_t H5Z_DEFLATE[1] = {{ + H5Z_FILTER_DEFLATE, /* Filter id number */ + "deflate", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + NULL, /* The "set local" callback */ + H5Z_filter_deflate, /* The actual filter function */ +}}; + #define H5Z_DEFLATE_SIZE_ADJUST(s) (HDceil((double)((s)*1.001))+12) @@ -52,7 +67,7 @@ static int interface_initialize_g = 0; * *------------------------------------------------------------------------- */ -size_t +static size_t H5Z_filter_deflate (unsigned flags, size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf) diff --git a/src/H5Zfletcher32.c b/src/H5Zfletcher32.c index 62fbfa5..690c1c3 100644 --- a/src/H5Zfletcher32.c +++ b/src/H5Zfletcher32.c @@ -17,20 +17,36 @@ * Jan 3, 2003 */ +#define H5Z_PACKAGE /*suppress error about including H5Zpkg */ + #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* File access */ #include "H5MMprivate.h" /* Memory management */ -#include "H5Zprivate.h" /* Data filters */ +#include "H5Zpkg.h" /* Data filters */ #ifdef H5_HAVE_FILTER_FLETCHER32 -#define FLETCHER_LEN 4 - /* Interface initialization */ #define PABLO_MASK H5Z_fletcher32_mask #define INTERFACE_INIT NULL static int interface_initialize_g = 0; +/* Local function prototypes */ +static size_t H5Z_filter_fletcher32 (unsigned flags, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); + +/* This message derives from H5Z */ +const H5Z_class_t H5Z_FLETCHER32[1] = {{ + H5Z_FILTER_FLETCHER32, /* Filter id number */ + "fletcher32", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + NULL, /* The "set local" callback */ + H5Z_filter_fletcher32, /* The actual filter function */ +}}; + +#define FLETCHER_LEN 4 + /*------------------------------------------------------------------------- * Function: H5Z_filter_fletcher32_compute @@ -48,7 +64,8 @@ static int interface_initialize_g = 0; * *------------------------------------------------------------------------- */ -static uint32_t H5Z_filter_fletcher32_compute(unsigned short *src, size_t len) +static uint32_t +H5Z_filter_fletcher32_compute(unsigned short *src, size_t len) { size_t count = len; /* Number of bytes left to checksum */ uint32_t s1 = 0, s2 = 0; /* Temporary partial checksums */ @@ -102,7 +119,7 @@ static uint32_t H5Z_filter_fletcher32_compute(unsigned short *src, size_t len) * *------------------------------------------------------------------------- */ -size_t +static size_t H5Z_filter_fletcher32 (unsigned flags, size_t UNUSED cd_nelmts, const unsigned UNUSED cd_values[], size_t nbytes, size_t *buf_size, void **buf) { diff --git a/src/H5Zpkg.h b/src/H5Zpkg.h new file mode 100644 index 0000000..4643bbc --- /dev/null +++ b/src/H5Zpkg.h @@ -0,0 +1,56 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef H5Z_PACKAGE +#error "Do not include this file outside the H5Z package!" +#endif + +#ifndef _H5Zpkg_H +#define _H5Zpkg_H + +/* Include private header file */ +#include "H5Zprivate.h" /* Filter functions */ + +#ifdef H5_HAVE_FILTER_DEFLATE +/* + * Deflate filter + */ +H5_DLLVAR const H5Z_class_t H5Z_DEFLATE[1]; +#endif /* H5_HAVE_FILTER_DEFLATE */ + +#ifdef H5_HAVE_FILTER_SHUFFLE +/* + * Shuffle filter + */ +H5_DLLVAR const H5Z_class_t H5Z_SHUFFLE[1]; +#endif /* H5_HAVE_FILTER_SHUFFLE */ + +#ifdef H5_HAVE_FILTER_FLETCHER32 +/* + * Fletcher32 filter + */ +H5_DLLVAR const H5Z_class_t H5Z_FLETCHER32[1]; +#endif /* H5_HAVE_FILTER_FLETCHER32 */ + +#ifdef H5_HAVE_FILTER_SZIP +/* + * szip filter + */ +H5_DLLVAR const H5Z_class_t H5Z_SZIP[1]; +#endif /* H5_HAVE_FILTER_SZIP */ + +/* Package-local function prototypes */ + +#endif /* _H5Zpkg_H */ + diff --git a/src/H5Zprivate.h b/src/H5Zprivate.h index 6f90e76..84b2eeb 100644 --- a/src/H5Zprivate.h +++ b/src/H5Zprivate.h @@ -20,58 +20,35 @@ #define _H5Zprivate_H #include "H5Zpublic.h" -#include "H5Fprivate.h" -#include "H5Ppublic.h" -/* - * The filter table maps filter identification numbers to structs that - * contain a pointers to the filter function and timing statistics. - */ -typedef struct H5Z_class_t { - H5Z_filter_t id; /*filter ID number */ - char *name; /*comment for debugging */ - H5Z_func_t func; /*the filter function */ - -#ifdef H5Z_DEBUG - struct { - hsize_t total; /*total number of bytes processed */ - hsize_t errors; /*bytes of total attributable to errors */ - H5_timer_t timer; /*execution time including errors */ - } stats[2]; /*0=output, 1=input */ -#endif -} H5Z_class_t; +/* Structure to store information about each filter's parameters */ +typedef struct { + H5Z_filter_t id; /*filter identification number */ + unsigned flags; /*defn and invocation flags */ + char *name; /*optional filter name */ + size_t cd_nelmts; /*number of elements in cd_values[] */ + unsigned *cd_values; /*client data values */ +} H5Z_filter_info_t; struct H5O_pline_t; /*forward decl*/ -H5_DLL herr_t H5Z_register(H5Z_filter_t id, const char *comment, - H5Z_func_t filter); -H5_DLL herr_t H5Z_unregister (H5Z_filter_t id); +/* Internal API routines */ +H5_DLL herr_t H5Z_register(const H5Z_class_t *cls); +H5_DLL herr_t H5Z_unregister(H5Z_filter_t id); H5_DLL herr_t H5Z_append(struct H5O_pline_t *pline, H5Z_filter_t filter, - unsigned flags, size_t cd_nelmts, - const unsigned int cd_values[]); -H5_DLL herr_t H5Z_pipeline(H5F_t *f, const struct H5O_pline_t *pline, + unsigned flags, size_t cd_nelmts, const unsigned int cd_values[]); +H5_DLL herr_t H5Z_modify(struct H5O_pline_t *pline, H5Z_filter_t filter, + unsigned flags, size_t cd_nelmts, const unsigned int cd_values[]); +H5_DLL herr_t H5Z_pipeline(const struct H5O_pline_t *pline, unsigned flags, unsigned *filter_mask/*in,out*/, H5Z_EDC_t edc_read, H5Z_cb_t cb_struct, size_t *nbytes/*in,out*/, size_t *buf_size/*in,out*/, void **buf/*in,out*/); H5_DLL H5Z_class_t *H5Z_find(H5Z_filter_t id); - - -/* Filter routines */ -H5_DLL size_t H5Z_filter_deflate(unsigned flags, size_t cd_nelmts, - const unsigned cd_values[], size_t nbytes, - size_t *buf_size, void **buf); - -H5_DLL size_t H5Z_filter_szip(unsigned flags, size_t cd_nelmts, - const unsigned cd_values[], size_t nbytes, - size_t *buf_size, void **buf); - -H5_DLL size_t H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts, - const unsigned cd_values[], size_t nbytes, - size_t *buf_size, void **buf); - -H5_DLL size_t H5Z_filter_fletcher32(unsigned flags, size_t cd_nelmts, - const unsigned cd_values[], size_t nbytes, - size_t *buf_size, void **buf); +H5_DLL herr_t H5Z_can_apply(hid_t dcpl_id, hid_t type_id); +H5_DLL herr_t H5Z_set_local(hid_t dcpl_id, hid_t type_id); +H5_DLL H5Z_filter_info_t *H5Z_filter_info(struct H5O_pline_t *pline, + H5Z_filter_t filter); +H5_DLL htri_t H5Z_all_filters_avail(struct H5O_pline_t *pline); #endif diff --git a/src/H5Zpublic.h b/src/H5Zpublic.h index 237ab39..c455700 100644 --- a/src/H5Zpublic.h +++ b/src/H5Zpublic.h @@ -48,10 +48,12 @@ typedef int H5Z_filter_t; #define H5Z_FLAG_SKIP_EDC 0x0200 /*skip EDC filters for read */ /* Special parameters for szip compression */ -#ifdef H5_HAVE_FILTER_SZIP +/* [These are aliases for the similar definitions in szlib.h, which we can't + * include directly due to the duplication of various symbols with the zlib.h + * header file] */ #define H5_SZIP_RAW_OPTION_MASK 128 #define H5_SZIP_NN_OPTION_MASK 32 -#endif /* H5_HAVE_FILTER_SZIP */ +#define H5_SZIP_MAX_PIXELS_PER_BLOCK 32 /* Values to decide if EDC is enabled for reading data */ typedef enum H5Z_EDC_t { @@ -84,6 +86,51 @@ extern "C" { #endif /* + * Before a dataset gets created, the "can_apply" callbacks for any filters used + * in the dataset creation property list are called + * with the dataset's dataset creation property list, the dataset's datatype and + * a dataspace describing a chunk (for chunked dataset storage). + * + * The "can_apply" callback must determine if the combination of the dataset + * creation property list setting, the datatype and the dataspace represent a + * valid combination to apply this filter to. For example, some cases of + * invalid combinations may involve the filter not operating correctly on + * certain datatypes (or certain datatype sizes), or certain sizes of the chunk + * dataspace. + * + * The "can_apply" callback can be the NULL pointer, in which case, the library + * will assume that it can apply to any combination of dataset creation + * property list values, datatypes and dataspaces. + * + * The "can_apply" callback returns positive a valid combination, zero for an + * invalid combination and negative for an error. + */ +typedef herr_t (*H5Z_can_apply_func_t)(hid_t dcpl_id, hid_t type_id, hid_t space_id); + +/* + * After the "can_apply" callbacks are checked for new datasets, the "set_local" + * callbacks for any filters used in the dataset creation property list are + * called. These callbacks receive the dataset's private copy of the dataset + * creation property list passed in to H5Dcreate (i.e. not the actual property + * list passed in to H5Dcreate) and the datatype ID passed in to H5Dcreate + * (which is not copied and should not be modified) and a dataspace describing + * the chunk (for chunked dataset storage) (which should also not be modified). + * + * The "set_local" callback must set any parameters that are specific to this + * dataset, based on the combination of the dataset creation property list + * values, the datatype and the dataspace. For example, some filters perform + * different actions based on different datatypes (or datatype sizes) or + * different number of dimensions or dataspace sizes. + * + * The "set_local" callback can be the NULL pointer, in which case, the library + * will assume that there are no dataset-specific settings for this filter. + * + * The "set_local" callback must return non-negative on success and negative + * for an error. + */ +typedef herr_t (*H5Z_set_local_func_t)(hid_t dcpl_id, hid_t type_id, hid_t space_id); + +/* * A filter gets definition flags and invocation flags (defined above), the * client data array and size defined when the filter was added to the * pipeline, the size in bytes of the data on which to operate, and pointers @@ -102,11 +149,25 @@ typedef size_t (*H5Z_func_t)(unsigned int flags, size_t cd_nelmts, const unsigned int cd_values[], size_t nbytes, size_t *buf_size, void **buf); +/* + * The filter table maps filter identification numbers to structs that + * contain a pointers to the filter function and timing statistics. + */ +typedef struct H5Z_class_t { + H5Z_filter_t id; /* Filter ID number */ + const char *name; /* Comment for debugging */ + H5Z_can_apply_func_t can_apply; /* The "can apply" callback for a filter */ + H5Z_set_local_func_t set_local; /* The "set local" callback for a filter */ + H5Z_func_t filter; /* The actual filter function */ +} H5Z_class_t; + +#ifdef H5_WANT_H5_V1_4_COMPAT H5_DLL herr_t H5Zregister(H5Z_filter_t id, const char *comment, H5Z_func_t filter); - +#else /* H5_WANT_H5_V1_4_COMPAT */ +H5_DLL herr_t H5Zregister(const H5Z_class_t *cls); +#endif /* H5_WANT_H5_V1_4_COMPAT */ H5_DLL herr_t H5Zunregister(H5Z_filter_t id); - H5_DLL htri_t H5Zfilter_avail(H5Z_filter_t id); diff --git a/src/H5Zshuffle.c b/src/H5Zshuffle.c index bcadbc4..e82f1dc 100644 --- a/src/H5Zshuffle.c +++ b/src/H5Zshuffle.c @@ -12,10 +12,14 @@ * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +#define H5Z_PACKAGE /*suppress error about including H5Zpkg */ + #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ #include "H5MMprivate.h" /* Memory management */ -#include "H5Zprivate.h" /* Data filters */ +#include "H5Ppublic.h" /* Property lists */ +#include "H5Tpublic.h" /* Datatype functions */ +#include "H5Zpkg.h" /* Data filters */ #ifdef H5_HAVE_FILTER_SHUFFLE @@ -24,6 +28,72 @@ #define INTERFACE_INIT NULL static int interface_initialize_g = 0; +/* Local function prototypes */ +static herr_t H5Z_set_local_shuffle(hid_t dcpl_id, hid_t type_id, hid_t space_id); +static size_t H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); + +/* This message derives from H5Z */ +const H5Z_class_t H5Z_SHUFFLE[1] = {{ + H5Z_FILTER_SHUFFLE, /* Filter id number */ + "shuffle", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + H5Z_set_local_shuffle, /* The "set local" callback */ + H5Z_filter_shuffle, /* The actual filter function */ +}}; + +/* Local macros */ +#define H5Z_SHUFFLE_USER_NPARMS 0 /* Number of parameters that users can set */ +#define H5Z_SHUFFLE_TOTAL_NPARMS 1 /* Total number of parameters for filter */ +#define H5Z_SHUFFLE_PARM_SIZE 0 /* "Local" parameter for shuffling size */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_set_local_shuffle + * + * Purpose: Set the "local" dataset parameter for data shuffling to be + * the size of the datatype. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Quincey Koziol + * Monday, April 7, 2003 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_set_local_shuffle(hid_t dcpl_id, hid_t type_id, hid_t UNUSED space_id) +{ + unsigned flags; /* Filter flags */ + size_t cd_nelmts=H5Z_SHUFFLE_USER_NPARMS; /* Number of filter parameters */ + unsigned cd_values[H5Z_SHUFFLE_TOTAL_NPARMS]; /* Filter parameters */ + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_set_local_shuffle, FAIL); + + /* Get the filter's current parameters */ + if(H5Pget_filter_by_id(dcpl_id,H5Z_FILTER_SHUFFLE,&flags,&cd_nelmts, cd_values,0,NULL)<0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "can't get shuffle parameters"); + + /* Check that no parameters are currently set */ + if(cd_nelmts!=H5Z_SHUFFLE_USER_NPARMS) + HGOTO_ERROR(H5E_PLINE, H5E_BADVALUE, FAIL, "incorrect # of shuffle parameters"); + + /* Set "local" parameter for this dataset */ + if((cd_values[H5Z_SHUFFLE_PARM_SIZE]=H5Tget_size(type_id))==0) + HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype size"); + + /* Modify the filter's parameters for this dataset */ + if(H5Pmodify_filter(dcpl_id, H5Z_FILTER_SHUFFLE, flags, H5Z_SHUFFLE_TOTAL_NPARMS, cd_values)<0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTSET, FAIL, "can't set local shuffle parameters"); + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5Z_set_local_shuffle() */ + /*------------------------------------------------------------------------- * Function: H5Z_filter_shuffle @@ -47,7 +117,7 @@ static int interface_initialize_g = 0; * *------------------------------------------------------------------------- */ -size_t +static size_t H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf) { @@ -63,11 +133,11 @@ H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], FUNC_ENTER_NOAPI(H5Z_filter_shuffle, 0); /* Check arguments */ - if (cd_nelmts!=1 || cd_values[0]==0) + if (cd_nelmts!=H5Z_SHUFFLE_TOTAL_NPARMS || cd_values[H5Z_SHUFFLE_PARM_SIZE]==0) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid shuffle parameters"); /* Get the number of bytes per element from the parameter block */ - bytesoftype=cd_values[0]; + bytesoftype=cd_values[H5Z_SHUFFLE_PARM_SIZE]; /* Don't do anything for 1-byte elements */ if(bytesoftype>1) { diff --git a/src/H5Zszip.c b/src/H5Zszip.c index b6e9972..1124d1e 100644 --- a/src/H5Zszip.c +++ b/src/H5Zszip.c @@ -12,10 +12,16 @@ * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +#define H5Z_PACKAGE /*suppress error about including H5Zpkg */ + #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* File access */ #include "H5MMprivate.h" /* Memory management */ -#include "H5Zprivate.h" /* Data filters */ +#include "H5Oprivate.h" /* Object headers */ +#include "H5Ppublic.h" /* Property lists */ +#include "H5Tpublic.h" /* Datatype functions */ +#include "H5Zpkg.h" /* Data filters */ #ifdef H5_HAVE_FILTER_SZIP @@ -28,6 +34,185 @@ #define INTERFACE_INIT NULL static int interface_initialize_g = 0; +/* Local function prototypes */ +static herr_t H5Z_can_apply_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id); +static herr_t H5Z_set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id); +static size_t H5Z_filter_szip (unsigned flags, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); + +/* This message derives from H5Z */ +const H5Z_class_t H5Z_SZIP[1] = {{ + H5Z_FILTER_SZIP, /* Filter id number */ + "szip", /* Filter name for debugging */ + H5Z_can_apply_szip, /* The "can apply" callback */ + H5Z_set_local_szip, /* The "set local" callback */ + H5Z_filter_szip, /* The actual filter function */ +}}; + +/* Local macros */ +#define H5Z_SZIP_USER_NPARMS 2 /* Number of parameters that users can set */ +#define H5Z_SZIP_TOTAL_NPARMS 4 /* Total number of parameters for filter */ +#define H5Z_SZIP_PARM_MASK 0 /* "User" parameter for option mask */ +#define H5Z_SZIP_PARM_PPB 1 /* "User" parameter for pixels-per-block */ +#define H5Z_SZIP_PARM_BPP 2 /* "Local" parameter for bits-per-pixel */ +#define H5Z_SZIP_PARM_PPS 3 /* "Local" parameter for pixels-per-scanline */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_can_apply_szip + * + * Purpose: Check the parameters for szip compression for validity and + * whether they fit a particular dataset. + * + * Note: This function currently range-checks for datatypes with + * 8-bit boundaries (8, 16, 24, etc.). It appears that the szip + * library can actually handle 1-24, 32 & 64 bit samples. If + * this becomes important, we should make the checks below more + * sophisticated and have them check for n-bit datatypes of the + * correct size, etc. - QAK + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Quincey Koziol + * Monday, April 7, 2003 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_can_apply_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id) +{ + unsigned flags; /* Filter flags */ + size_t cd_nelmts=H5Z_SZIP_USER_NPARMS; /* Number of filter parameters */ + unsigned cd_values[H5Z_SZIP_TOTAL_NPARMS]; /* Filter parameters */ + hsize_t dims[H5O_LAYOUT_NDIMS]; /* Dataspace (i.e. chunk) dimensions */ + int dtype_size; /* Datatype's size (in bits) */ + H5T_order_t dtype_order; /* Datatype's endianness order */ + hsize_t scanline; /* Size of dataspace's fastest changing dimension */ + herr_t ret_value=TRUE; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_can_apply_szip, FAIL); + + /* Get the filter's current parameters */ + if(H5Pget_filter_by_id(dcpl_id,H5Z_FILTER_SZIP,&flags,&cd_nelmts, cd_values,0,NULL)<0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "can't get szip parameters"); + + /* Check that no parameters are currently set */ + if(cd_nelmts!=H5Z_SZIP_USER_NPARMS) + HGOTO_ERROR(H5E_PLINE, H5E_BADVALUE, FAIL, "incorrect # of szip parameters"); + + /* Get datatype's size, for checking the "bits-per-pixel" */ + if((dtype_size=(sizeof(unsigned char)*H5Tget_size(type_id)))==0) + HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype size"); + + /* Range check datatype's size */ + if(dtype_size>32 && dtype_size!=64) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FALSE, "invalid datatype size"); + + /* Get datatype's endianness order */ + if((dtype_order=H5Tget_order(type_id))==H5T_ORDER_ERROR) + HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "can't retrieve datatype endianness order"); + + /* Range check datatype's endianness order */ + /* (Note: this may not handle non-atomic datatypes well) */ + if(dtype_order != H5T_ORDER_LE && dtype_order != H5T_ORDER_BE) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FALSE, "invalid datatype endianness order"); + + /* Get dimensions for dataspace */ + if (H5Sget_simple_extent_dims(space_id, dims, NULL)<0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "unable to get dataspace dimensions"); + + /* Get "local" parameter for this dataset's "pixels-per-scanline" */ + scanline=dims[0]; + + /* Range check the scanline's size */ + if(scanline > SZ_MAX_PIXELS_PER_SCANLINE) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FALSE, "invalid scanline size"); + + /* Range check the scanline's number of blocks */ + if((scanline/cd_values[H5Z_SZIP_PARM_PPB]) > SZ_MAX_BLOCKS_PER_SCANLINE) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FALSE, "invalid number of blocks per scanline"); + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5Z_can_apply_szip() */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_set_local_szip + * + * Purpose: Set the "local" dataset parameters for szip compression. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Quincey Koziol + * Monday, April 7, 2003 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id) +{ + unsigned flags; /* Filter flags */ + size_t cd_nelmts=H5Z_SZIP_USER_NPARMS; /* Number of filter parameters */ + unsigned cd_values[H5Z_SZIP_TOTAL_NPARMS]; /* Filter parameters */ + hsize_t dims[H5O_LAYOUT_NDIMS]; /* Dataspace (i.e. chunk) dimensions */ + H5T_order_t dtype_order; /* Datatype's endianness order */ + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_set_local_szip, FAIL); + + /* Get the filter's current parameters */ + if(H5Pget_filter_by_id(dcpl_id,H5Z_FILTER_SZIP,&flags,&cd_nelmts, cd_values,0,NULL)<0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "can't get szip parameters"); + + /* Check that no parameters are currently set */ + if(cd_nelmts!=H5Z_SZIP_USER_NPARMS) + HGOTO_ERROR(H5E_PLINE, H5E_BADVALUE, FAIL, "incorrect # of szip parameters"); + + /* Get dimensions for dataspace */ + if (H5Sget_simple_extent_dims(space_id, dims, NULL)<0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "unable to get dataspace dimensions"); + + /* Set "local" parameter for this dataset's "bits-per-pixel" */ + if((cd_values[H5Z_SZIP_PARM_BPP]=(sizeof(unsigned char)*H5Tget_size(type_id)))==0) + HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype size"); + + /* Set "local" parameter for this dataset's "pixels-per-scanline" */ + cd_values[H5Z_SZIP_PARM_PPS]=dims[0]; + + /* Get datatype's endianness order */ + if((dtype_order=H5Tget_order(type_id))==H5T_ORDER_ERROR) + HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype endianness order"); + + /* Set the correct endianness flag for szip */ + /* (Note: this may not handle non-atomic datatypes well) */ + switch(dtype_order) { + case H5T_ORDER_LE: /* Little-endian byte order */ + cd_values[H5Z_SZIP_PARM_MASK] |= SZ_LSB_OPTION_MASK; + break; + + case H5T_ORDER_BE: /* Big-endian byte order */ + cd_values[H5Z_SZIP_PARM_MASK] |= SZ_MSB_OPTION_MASK; + break; + + default: + HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype endianness order"); + } /* end switch */ + + /* Modify the filter's parameters for this dataset */ + if(H5Pmodify_filter(dcpl_id, H5Z_FILTER_SZIP, flags, H5Z_SZIP_TOTAL_NPARMS, cd_values)<0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTSET, FAIL, "can't set local szip parameters"); + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5Z_set_local_szip() */ + /*------------------------------------------------------------------------- * Function: H5Z_filter_szip @@ -47,7 +232,7 @@ static int interface_initialize_g = 0; * *------------------------------------------------------------------------- */ -size_t +static size_t H5Z_filter_szip (unsigned flags, size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf) { diff --git a/src/Makefile.in b/src/Makefile.in index 3f82d8e..08c7186 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -42,7 +42,7 @@ LIB_SRC=H5.c H5A.c H5AC.c H5B.c H5D.c H5E.c H5F.c H5Farray.c H5Fcontig.c \ H5Tcompound.c H5Tconv.c H5Tcset.c H5Tenum.c H5Tfields.c H5Tfixed.c \ H5Tfloat.c H5Tinit.c H5Tnative.c H5Toffset.c H5Topaque.c H5Torder.c \ H5Tpad.c H5Tprecis.c H5Tstrpad.c H5Tvlen.c H5TB.c H5TS.c H5V.c H5Z.c \ - H5Zdeflate.c H5Zshuffle.c H5Zszip.c H5Zfletcher32.c + H5Zdeflate.c H5Zfletcher32.c H5Zshuffle.c H5Zszip.c LIB_OBJ=$(LIB_SRC:.c=.lo) @@ -66,7 +66,7 @@ PRIVATE_HDR=H5private.h H5Aprivate.h H5Apkg.h H5ACprivate.h H5Bprivate.h \ H5MMprivate.h H5Oprivate.h H5Opkg.h H5Pprivate.h H5Ppkg.h \ H5Rprivate.h H5RSprivate.h H5Sprivate.h H5STprivate.h \ H5Tprivate.h H5TBprivate.h H5Tpkg.h H5TSprivate.h H5Vprivate.h \ - H5Zprivate.h H5config.h + H5Zprivate.h H5Zpkg.h H5config.h ## Number format detection ## The LD_LIBRARY_PATH setting is a klutch. -- cgit v0.12