diff options
| -rw-r--r-- | release_docs/RELEASE.txt | 4 | ||||
| -rw-r--r-- | src/H5Dprivate.h | 1 | ||||
| -rw-r--r-- | src/H5Pdxpl.c | 83 | ||||
| -rw-r--r-- | src/H5Ppublic.h | 2 | ||||
| -rw-r--r-- | src/H5Tconv.c | 86 | ||||
| -rw-r--r-- | test/dtypes.c | 107 | ||||
| -rw-r--r-- | test/enum.c | 22 |
7 files changed, 295 insertions, 10 deletions
diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index d09d99c..147f0e5 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -91,6 +91,10 @@ New Features Library: -------- + - The library added two new dataset transfer property to control whether + to fill 0xff in the destination data or convert to the destination data + when overflow happens to ENUM data (Issue 7579). The two new functions + are H5Pset(get)_enum_conv_overflow. (SLU - 2011/05/27) - H5Tcreate now supports string type (fixed-length and variable-length). (SLU - 2011/05/20) - Added ability to cache files opened through external links. Added new diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h index 5849598..cc770df 100644 --- a/src/H5Dprivate.h +++ b/src/H5Dprivate.h @@ -76,6 +76,7 @@ #define H5D_XFER_EDC_NAME "err_detect" /* EDC */ #define H5D_XFER_FILTER_CB_NAME "filter_cb" /* Filter callback function */ #define H5D_XFER_CONV_CB_NAME "type_conv_cb" /* Type conversion callback function */ +#define H5D_XFER_CONV_ENUM_OVERFLOW_NAME "conv_enum_overflow" /* conversion of enum overflow values */ #define H5D_XFER_XFORM_NAME "data_transform" /* Data transform */ #ifdef H5_HAVE_INSTRUMENTED_LIBRARY /* Collective chunk instrumentation properties */ diff --git a/src/H5Pdxpl.c b/src/H5Pdxpl.c index 042951e..775eba9 100644 --- a/src/H5Pdxpl.c +++ b/src/H5Pdxpl.c @@ -109,6 +109,9 @@ /* Definitions for type conversion callback function property */ #define H5D_XFER_CONV_CB_SIZE sizeof(H5T_conv_cb_t) #define H5D_XFER_CONV_CB_DEF {NULL,NULL} +/* Definition for the property of converting enum overflowing values */ +#define H5D_XFER_CONV_ENUM_OVERFLOW_SIZE sizeof(hbool_t) +#define H5D_XFER_CONV_ENUM_OVERFLOW_DEF TRUE /* Definitions for data transform property */ #define H5D_XFER_XFORM_SIZE sizeof(void *) #define H5D_XFER_XFORM_DEF NULL @@ -206,6 +209,7 @@ H5P_dxfr_reg_prop(H5P_genclass_t *pclass) H5Z_EDC_t enable_edc = H5D_XFER_EDC_DEF; /* Default value for EDC property */ H5Z_cb_t filter_cb = H5D_XFER_FILTER_CB_DEF; /* Default value for filter callback */ H5T_conv_cb_t conv_cb = H5D_XFER_CONV_CB_DEF; /* Default value for datatype conversion callback */ + hbool_t enum_conv = H5D_XFER_CONV_ENUM_OVERFLOW_DEF;/* Default value for enum overflow values handling*/ void *def_xfer_xform = H5D_XFER_XFORM_DEF; /* Default value for data transform */ herr_t ret_value = SUCCEED; /* Return value */ @@ -289,6 +293,10 @@ H5P_dxfr_reg_prop(H5P_genclass_t *pclass) if(H5P_register_real(pclass, H5D_XFER_CONV_CB_NAME, H5D_XFER_CONV_CB_SIZE, &conv_cb, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the enum overflow handling property */ + if(H5P_register_real(pclass, H5D_XFER_CONV_ENUM_OVERFLOW_NAME, H5D_XFER_CONV_ENUM_OVERFLOW_SIZE, &enum_conv, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the data transform property */ if(H5P_register_real(pclass, H5D_XFER_XFORM_NAME, H5D_XFER_XFORM_SIZE, &def_xfer_xform, NULL, NULL, NULL, H5D_XFER_XFORM_DEL, H5D_XFER_XFORM_COPY, NULL, H5D_XFER_XFORM_CLOSE) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") @@ -1070,6 +1078,81 @@ done: /*------------------------------------------------------------------------- + * Function: H5Pset_enum_conv_overflow + * + * Purpose: Sets the property of converting overflowing enum values + * for dataset transfer property list. It indicates whether + * to convert the values or fill in the default value. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * 26 May 2011 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_enum_conv_overflow(hid_t plist_id, hbool_t conv_overflow) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value=SUCCEED; /* return value */ + + FUNC_ENTER_API(H5Pset_enum_conv_overflow, FAIL) + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id,H5P_DATASET_XFER))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Update property list */ + if (H5P_set(plist,H5D_XFER_CONV_ENUM_OVERFLOW_NAME,&conv_overflow)<0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set value") + +done: + FUNC_LEAVE_API(ret_value) +} + + +/*------------------------------------------------------------------------- + * Function: H5Pget_enum_conv_overflow + * + * Purpose: Gets the property of converting overflowing enum values + * for dataset transfer property list. It indicates whether + * to convert the values or fill in the default value. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * 26 May 2011 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_enum_conv_overflow(hid_t plist_id, hbool_t *conv_overflow/*out*/) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value=SUCCEED; /* return value */ + + FUNC_ENTER_API(H5Pget_enum_conv_overflow, FAIL) + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id,H5P_DATASET_XFER))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Return values */ + if (conv_overflow) + if (H5P_get(plist,H5D_XFER_CONV_ENUM_OVERFLOW_NAME,conv_overflow)<0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get value") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_enum_conv_overflow() */ + + +/*------------------------------------------------------------------------- * Function: H5Pget_btree_ratios * * Purpose: Queries B-tree split ratios. See H5Pset_btree_ratios(). diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 6867b63..80c028a 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -357,6 +357,8 @@ H5_DLL herr_t H5Pset_hyper_vector_size(hid_t fapl_id, size_t size); H5_DLL herr_t H5Pget_hyper_vector_size(hid_t fapl_id, size_t *size/*out*/); H5_DLL herr_t H5Pset_type_conv_cb(hid_t dxpl_id, H5T_conv_except_func_t op, void* operate_data); H5_DLL herr_t H5Pget_type_conv_cb(hid_t dxpl_id, H5T_conv_except_func_t *op, void** operate_data); +H5_DLL herr_t H5Pset_enum_conv_overflow(hid_t plist_id, hbool_t conv_overflow); +H5_DLL herr_t H5Pget_enum_conv_overflow(hid_t plist_id, hbool_t *conv_overflow/*out*/); /* Link creation property list (LCPL) routines */ H5_DLL herr_t H5Pset_create_intermediate_group(hid_t plist_id, unsigned crt_intmd); diff --git a/src/H5Tconv.c b/src/H5Tconv.c index e7295d7..a3fb856 100644 --- a/src/H5Tconv.c +++ b/src/H5Tconv.c @@ -2706,6 +2706,12 @@ done: * then convert one value at each memory location advancing * BUF_STRIDE bytes each time; otherwise assume both source and * destination values are packed. + * + * Raymond Lu, 2011-05-26 + * When overflow happened, the old way was to fill in the destination + * with 0xff. The new default way is to convert the source value + * to the destination. If the user sets the property through + * H5Pset_enum_conv_overflow, the library still uses the old way. *------------------------------------------------------------------------- */ herr_t @@ -2715,6 +2721,11 @@ H5T_conv_enum(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts, { uint8_t *buf = (uint8_t*)_buf; /*cast for pointer arithmetic */ H5T_t *src = NULL, *dst = NULL; /*src and dst datatypes */ + H5T_t *src_super = NULL, *dst_super = NULL; /*parent types for src and dst*/ + hid_t src_super_id = NULL, dst_super_id = NULL; /*parent type IDs for src and dst*/ + size_t src_super_size, dst_super_size; /*size of parent types for src and dst*/ + H5T_path_t *tpath; /*type conversion info */ + void *tmp_buf = NULL; /*small conversion buffer */ uint8_t *s = NULL, *d = NULL; /*src and dst BUF pointers */ int src_delta, dst_delta; /*conversion strides */ int n; /*src value cast as native int */ @@ -2722,6 +2733,7 @@ H5T_conv_enum(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts, H5P_genplist_t *plist; /*property list pointer */ H5T_conv_cb_t cb_struct; /*conversion callback structure */ H5T_conv_ret_t except_ret; /*return of callback function */ + hbool_t conv_overflow = TRUE; /* library's default behavior when overflow happens */ size_t i; /*counters */ herr_t ret_value = SUCCEED; /* Return value */ @@ -2802,6 +2814,43 @@ H5T_conv_enum(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts, if(H5P_get(plist, H5D_XFER_CONV_CB_NAME, &cb_struct) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get conversion exception callback") + /* Get the property of overflow handling */ + if(H5P_get(plist, H5D_XFER_CONV_ENUM_OVERFLOW_NAME, &conv_overflow) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get conversion exception callback") + + /* Preparation for converting the data when overflow happens */ + if(conv_overflow) { + /* Retrieve base type for the source type */ + if(NULL == (src_super = H5T_get_super(src))) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADTYPE, FAIL, "unable to get base type for the source type") + + /* Retrieve base type for the destination type */ + if(NULL == (dst_super = H5T_get_super(dst))) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADTYPE, FAIL, "unable to get base type for the destination type") + + /* Find the conversion function */ + if(NULL == (tpath = H5T_path_find(src_super, dst_super, NULL, NULL, dxpl_id, FALSE))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to find convert path between src and dst types") + + /* The conversion function needs the type IDs */ + if((src_super_id = H5I_register(H5I_DATATYPE, src_super, FALSE)) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register data type") + + if((dst_super_id = H5I_register(H5I_DATATYPE, dst_super, FALSE)) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register data type") + + /* Get the size of the base types of src & dst */ + if(!(src_super_size = H5T_get_size(src_super))) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADTYPE, FAIL, "unable to get type size") + + if(!(dst_super_size = H5T_get_size(dst_super))) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADTYPE, FAIL, "unable to get type size") + + /* Conversion buffer for overflowing data */ + if(!(tmp_buf = HDmalloc(MAX(src_super_size, dst_super_size)))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to allocate buffer") + } + for(i = 0; i < nelmts; i++, s += src_delta, d += dst_delta) { if(priv->length) { /* Use O(1) lookup */ @@ -2827,7 +2876,17 @@ H5T_conv_enum(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts, } if(except_ret == H5T_CONV_UNHANDLED) { - HDmemset(d, 0xff, dst->shared->size); + if(conv_overflow) { + /* Copy the source data to a small buffer, convert it, then copy it back to the + * destination */ + HDmemcpy(tmp_buf, s, src_super_size); + if(H5T_convert(tpath, src_super_id, dst_super_id, 1, (size_t)0, (size_t)0, + tmp_buf, NULL, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "data type conversion failed") + HDmemcpy(d, tmp_buf, dst_super_size); + } else + /* Fill the destination with 0xff if we don't convert it */ + HDmemset(d, 0xff, dst->shared->size); } else if(except_ret == H5T_CONV_ABORT) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCONVERT, FAIL, "can't handle conversion exception") } else { @@ -2860,7 +2919,18 @@ H5T_conv_enum(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts, } if(except_ret == H5T_CONV_UNHANDLED) { - HDmemset(d, 0xff, dst->shared->size); + if(conv_overflow) { + /* Copy the source data to a small buffer, convert it, then copy it back to the + * destination */ + HDmemcpy(tmp_buf, s, src_super_size); + if(H5T_convert(tpath, src_super_id, dst_super_id, 1, (size_t)0, (size_t)0, + tmp_buf, NULL, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "data type conversion failed") + HDmemcpy(d, tmp_buf, dst_super_size); + } else + /* Fill the destination with 0xff if we don't convert it */ + HDmemset(d, 0xff, dst->shared->size); + } else if(except_ret == H5T_CONV_ABORT) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCONVERT, FAIL, "can't handle conversion exception") } /* end if */ @@ -2871,6 +2941,18 @@ H5T_conv_enum(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts, } /* end else */ } } + + /* Close the type IDs. Have to use public function here */ + if(conv_overflow) { + if(H5Tclose(src_super_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CLOSEERROR, FAIL, "problem closing datatype") + + if(H5Tclose(dst_super_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CLOSEERROR, FAIL, "problem closing datatype") + + HDfree(tmp_buf); + } + break; default: diff --git a/test/dtypes.c b/test/dtypes.c index 51cfed2..fbb552d 100644 --- a/test/dtypes.c +++ b/test/dtypes.c @@ -4482,12 +4482,20 @@ error: * Programmer: Robb Matzke, LLNL, 2003-06-09 * * Modifications: + * Raymond Lu + * 26 May 2011 + * I added a few overflowing values (beyond the range of enumerate + * values) to make sure the library retains these values. The test + * for overflowing values when no conversion happens is in + * test_noconv of enum.c. *------------------------------------------------------------------------- */ static int test_conv_enum_2(void) { hid_t srctype=-1, dsttype=-1, oddsize=-1; + hid_t dxpl_id=-1; + hbool_t conv_overflow; int *data=NULL, i, nerrors=0; const char *mname[] = { "RED", "GREEN", @@ -4497,7 +4505,7 @@ test_conv_enum_2(void) "PURPLE", "ORANGE", "WHITE" }; - + TESTING("non-native enumeration type conversion"); /* Source enum type */ @@ -4539,11 +4547,108 @@ test_conv_enum_2(void) } } + /* Now make the source data overflow and see the conversion retain the original values by default. + * The initialization makes the first 128 values be -1, -2, -3,..., -128. In hexadecimal, + * they are 0xffffff, 0xfffffe, 0xfffffd,..., 0xffff7f. The rest values are between 0 + * and 127. */ + for (i=0; i<NTESTELEM; i++) { + if(i > 127) { + ((char*)data)[i*3+2] = i % 128; + ((char*)data)[i*3+0] = 0; + ((char*)data)[i*3+1] = 0; + } else { + ((char*)data)[i*3+2] = -(i+1); + ((char*)data)[i*3+0] = -1; + ((char*)data)[i*3+1] = -1; + } + } + + /* Convert to destination type */ + H5Tconvert(srctype, dsttype, (size_t)NTESTELEM, data, NULL, H5P_DEFAULT); + + /* Check results */ + for (i=0; i<NTESTELEM; i++) { + if(i > 127) { + if (data[i] != i%128) { + if (!nerrors++) { + H5_FAILED(); + printf("element %d is %d but should have been %d\n", + i, data[i], i%128); + } + } + } else { + if (data[i] != -(i+1)) { + if (!nerrors++) { + H5_FAILED(); + printf("element %d is %d but should have been %d\n", + i, data[i], -(i+1)); + } + } + } + } + + /* Now make the source data overflow and set the property for handling overflowing data to fill them + * them with 0xff (-1). The initialization makes the first 128 values be -1, -2, -3,..., -128. In + * hexadecimal, they are 0xffffff, 0xfffffe, 0xfffffd,..., 0xffff7f. The next 128 values are 128 to + * 255. The rest values are between 0 and 7. */ + for (i=0; i<NTESTELEM; i++) { + if(i > 255) { + ((char*)data)[i*3+2] = i % 8; + ((char*)data)[i*3+0] = 0; + ((char*)data)[i*3+1] = 0; + } else if(i > 127) { + ((char*)data)[i*3+2] = i; + ((char*)data)[i*3+0] = 0; + ((char*)data)[i*3+1] = 0; + + } else { + ((char*)data)[i*3+2] = -(i+1); + ((char*)data)[i*3+0] = -1; + ((char*)data)[i*3+1] = -1; + } + } + + /* Set the property for handling overflowing data */ + dxpl_id = H5Pcreate(H5P_DATASET_XFER); + + /* First verify it's TRUE by default */ + H5Pget_enum_conv_overflow(dxpl_id, &conv_overflow); + if(!conv_overflow) + nerrors++; + + /* Then set it to FALSE */ + H5Pset_enum_conv_overflow(dxpl_id, FALSE); + + /* Convert to destination type */ + H5Tconvert(srctype, dsttype, (size_t)NTESTELEM, data, NULL, dxpl_id); + + /* Check results. All overflowing values should turn to -1. */ + for (i=0; i<NTESTELEM; i++) { + if(i > 255) { + if (data[i] != i%8) { + if (!nerrors++) { + H5_FAILED(); + printf("element %d is %d but should have been %d\n", + i, data[i], i%8); + } + } + } else { + if (data[i] != -1) { + if (!nerrors++) { + H5_FAILED(); + printf("element %d is %d but should have been -1\n", + i, data[i]); + } + } + } + } + /* Cleanup */ free(data); H5Tclose(srctype); H5Tclose(dsttype); H5Tclose(oddsize); + H5Pclose(dxpl_id); /* Failure */ if (nerrors) { diff --git a/test/enum.c b/test/enum.c index e9f607d..df3ae3d 100644 --- a/test/enum.c +++ b/test/enum.c @@ -26,11 +26,13 @@ const char *FILENAME[] = { }; typedef enum { - E1_RED, - E1_GREEN, - E1_BLUE, - E1_WHITE, - E1_BLACK + E1_ERROR = -1, + E1_RED = 0, + E1_GREEN = 1, + E1_BLUE = 2, + E1_WHITE = 3, + E1_BLACK = 4, + E1_NO_COLOR } c_e1; @@ -121,7 +123,12 @@ test_named(hid_t file) * Monday, January 4, 1999 * * Modifications: - * + * Raymond Lu + * 26 May 2011 + * I added a few overflowing values (beyond the range of enumerate + * values) to make sure the library retains these values. The + * test for overflowing values when conversion happens is in the + * test_conv_enum_2() of dtypes.c. *------------------------------------------------------------------------- */ static int @@ -132,7 +139,8 @@ test_noconv(hid_t file) static c_e1 data1[]={E1_RED, E1_GREEN, E1_BLUE, E1_GREEN, E1_WHITE, E1_WHITE, E1_BLACK, E1_GREEN, E1_BLUE, E1_RED, E1_RED, E1_BLUE, E1_GREEN, E1_BLACK, E1_WHITE, - E1_RED, E1_WHITE, E1_GREEN, E1_GREEN, E1_BLUE}; + E1_RED, E1_WHITE, E1_GREEN, E1_GREEN, E1_BLUE, + E1_ERROR, E1_ERROR, E1_NO_COLOR, E1_NO_COLOR, E1_NO_COLOR}; c_e1 data2[NELMTS(data1)]; hsize_t ds_size[1]={NELMTS(data1)}; size_t i; |