[svn-r20920] Issue 7579 - The overflowing ENUM values are inconsistent. When no conversion is involved in reading or writing the

data, overflowing values are retained.  When conversion happens, the values become -1. The conversion function puts -1
when overflow happens.  I added two new dataset transfer property to control whether to fill 0xff in the destination 
data or convert to the destination data when overflow happens.  The two new functions are H5Pset(get)_enum_conv_overflow.  I also added test cases in enum.c and dtypes.c.

Tested on jam, koala, and heiwa.

4 files changed, 171 insertions, 2 deletions

diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h
index 5849598..cc770df 100644
--- a/src/H5Dprivate.h
+++ b/src/H5Dprivate.h
@@ -76,6 +76,7 @@
 #define H5D_XFER_EDC_NAME               "err_detect"    /* EDC */
 #define H5D_XFER_FILTER_CB_NAME         "filter_cb"     /* Filter callback function */
 #define H5D_XFER_CONV_CB_NAME           "type_conv_cb"  /* Type conversion callback function */
+#define H5D_XFER_CONV_ENUM_OVERFLOW_NAME "conv_enum_overflow"  /* conversion of enum overflow values */
 #define H5D_XFER_XFORM_NAME             "data_transform" /* Data transform */
 #ifdef H5_HAVE_INSTRUMENTED_LIBRARY
 /* Collective chunk instrumentation properties */
diff --git a/src/H5Pdxpl.c b/src/H5Pdxpl.c
index 1186868..6ed2353 100644
--- a/src/H5Pdxpl.c
+++ b/src/H5Pdxpl.c
@@ -108,6 +108,9 @@
 /* Definitions for type conversion callback function property */
 #define H5D_XFER_CONV_CB_SIZE       sizeof(H5T_conv_cb_t)
 #define H5D_XFER_CONV_CB_DEF        {NULL,NULL}
+/* Definition for the property of converting enum overflowing values */
+#define H5D_XFER_CONV_ENUM_OVERFLOW_SIZE        sizeof(hbool_t)
+#define H5D_XFER_CONV_ENUM_OVERFLOW_DEF         TRUE
 /* Definitions for data transform property */
 #define H5D_XFER_XFORM_SIZE         sizeof(void *)
 #define H5D_XFER_XFORM_DEF          NULL
@@ -204,6 +207,7 @@ H5P_dxfr_reg_prop(H5P_genclass_t *pclass)
     H5Z_EDC_t enable_edc = H5D_XFER_EDC_DEF;            /* Default value for EDC property */
     H5Z_cb_t filter_cb = H5D_XFER_FILTER_CB_DEF;        /* Default value for filter callback */
     H5T_conv_cb_t conv_cb = H5D_XFER_CONV_CB_DEF;       /* Default value for datatype conversion callback */
+    hbool_t enum_conv = H5D_XFER_CONV_ENUM_OVERFLOW_DEF;/* Default value for enum overflow values handling*/
     void *def_xfer_xform = H5D_XFER_XFORM_DEF;          /* Default value for data transform */
     herr_t ret_value = SUCCEED;         /* Return value */
 
@@ -283,6 +287,10 @@ H5P_dxfr_reg_prop(H5P_genclass_t *pclass)
     if(H5P_register_real(pclass, H5D_XFER_CONV_CB_NAME, H5D_XFER_CONV_CB_SIZE, &conv_cb, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0)
         HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class")
 
+    /* Register the enum overflow handling property */
+    if(H5P_register_real(pclass, H5D_XFER_CONV_ENUM_OVERFLOW_NAME, H5D_XFER_CONV_ENUM_OVERFLOW_SIZE, &enum_conv, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0)
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class")
+
     /* Register the data transform property */
     if(H5P_register_real(pclass, H5D_XFER_XFORM_NAME, H5D_XFER_XFORM_SIZE, &def_xfer_xform, NULL, NULL, NULL, H5D_XFER_XFORM_DEL, H5D_XFER_XFORM_COPY, NULL, H5D_XFER_XFORM_CLOSE) < 0)
         HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class")
@@ -1064,6 +1072,81 @@ done:
 
 
 /*-------------------------------------------------------------------------
+ * Function:	H5Pset_enum_conv_overflow
+ *
+ * Purpose:     Sets the property of converting overflowing enum values 
+ *              for dataset transfer property list. It indicates whether
+ *              to convert the values or fill in the default value.
+ *
+ * Return:	Non-negative on success/Negative on failure
+ *
+ * Programmer:	Raymond Lu
+ *              26 May 2011
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pset_enum_conv_overflow(hid_t plist_id, hbool_t conv_overflow)
+{
+    H5P_genplist_t      *plist;      /* Property list pointer */
+    herr_t              ret_value=SUCCEED;   /* return value */
+
+    FUNC_ENTER_API(H5Pset_enum_conv_overflow, FAIL)
+
+    /* Get the plist structure */
+    if(NULL == (plist = H5P_object_verify(plist_id,H5P_DATASET_XFER)))
+        HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID")
+
+    /* Update property list */
+    if (H5P_set(plist,H5D_XFER_CONV_ENUM_OVERFLOW_NAME,&conv_overflow)<0)
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set value")
+
+done:
+    FUNC_LEAVE_API(ret_value)
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function:	H5Pget_enum_conv_overflow
+ *
+ * Purpose:     Gets the property of converting overflowing enum values 
+ *              for dataset transfer property list. It indicates whether
+ *              to convert the values or fill in the default value.
+ *
+ * Return:	Non-negative on success/Negative on failure
+ *
+ * Programmer:	Raymond Lu
+ *              26 May 2011
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pget_enum_conv_overflow(hid_t plist_id, hbool_t *conv_overflow/*out*/)
+{
+    H5P_genplist_t *plist;      /* Property list pointer */
+    herr_t ret_value=SUCCEED;   /* return value */
+
+    FUNC_ENTER_API(H5Pget_enum_conv_overflow, FAIL)
+
+    /* Get the plist structure */
+    if(NULL == (plist = H5P_object_verify(plist_id,H5P_DATASET_XFER)))
+        HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID")
+
+    /* Return values */
+    if (conv_overflow)
+        if (H5P_get(plist,H5D_XFER_CONV_ENUM_OVERFLOW_NAME,conv_overflow)<0)
+            HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get value")
+
+done:
+    FUNC_LEAVE_API(ret_value)
+} /* end H5Pget_enum_conv_overflow() */
+
+
+/*-------------------------------------------------------------------------
  * Function:	H5Pget_btree_ratios
  *
  * Purpose:	Queries B-tree split ratios.  See H5Pset_btree_ratios().
diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h
index 75781cd..0481b05 100644
--- a/src/H5Ppublic.h
+++ b/src/H5Ppublic.h
@@ -358,6 +358,8 @@ H5_DLL herr_t H5Pset_hyper_vector_size(hid_t fapl_id, size_t size);
 H5_DLL herr_t H5Pget_hyper_vector_size(hid_t fapl_id, size_t *size/*out*/);
 H5_DLL herr_t H5Pset_type_conv_cb(hid_t dxpl_id, H5T_conv_except_func_t op, void* operate_data);
 H5_DLL herr_t H5Pget_type_conv_cb(hid_t dxpl_id, H5T_conv_except_func_t *op, void** operate_data);
+H5_DLL herr_t H5Pset_enum_conv_overflow(hid_t plist_id, hbool_t conv_overflow);
+H5_DLL herr_t H5Pget_enum_conv_overflow(hid_t plist_id, hbool_t *conv_overflow/*out*/);
 
 /* Link creation property list (LCPL) routines */
 H5_DLL herr_t H5Pset_create_intermediate_group(hid_t plist_id, unsigned crt_intmd);
diff --git a/src/H5Tconv.c b/src/H5Tconv.c
index 579094a..4df9f03 100644
--- a/src/H5Tconv.c
+++ b/src/H5Tconv.c
@@ -2706,6 +2706,12 @@ done:
  *		then convert one value at each memory location advancing
  *		BUF_STRIDE bytes each time; otherwise assume both source and
  *		destination values are packed.
+ *              
+ *              Raymond Lu, 2011-05-26
+ *              When overflow happened, the old way was to fill in the destination
+ *              with 0xff.  The new default way is to convert the source value
+ *              to the destination.  If the user sets the property through 
+ *              H5Pset_enum_conv_overflow, the library still uses the old way.
  *-------------------------------------------------------------------------
  */
 herr_t
@@ -2715,6 +2721,11 @@ H5T_conv_enum(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts,
 {
     uint8_t	*buf = (uint8_t*)_buf;	/*cast for pointer arithmetic	*/
     H5T_t	*src = NULL, *dst = NULL;	/*src and dst datatypes	*/
+    H5T_t	*src_super = NULL, *dst_super = NULL;	/*parent types for src and dst*/
+    hid_t	src_super_id = NULL, dst_super_id = NULL;	/*parent type IDs for src and dst*/
+    size_t      src_super_size, dst_super_size; /*size of parent types for src and dst*/
+    H5T_path_t	*tpath;			/*type conversion info	*/
+    void        *tmp_buf = NULL;     /*small conversion buffer */
     uint8_t	*s = NULL, *d = NULL;	/*src and dst BUF pointers	*/
     int	src_delta, dst_delta;	/*conversion strides		*/
     int	n;			/*src value cast as native int	*/
@@ -2722,6 +2733,7 @@ H5T_conv_enum(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts,
     H5P_genplist_t      *plist;         /*property list pointer         */
     H5T_conv_cb_t       cb_struct;      /*conversion callback structure */
     H5T_conv_ret_t      except_ret;     /*return of callback function   */
+    hbool_t             conv_overflow = TRUE; /* library's default behavior when overflow happens */
     size_t	i;			/*counters			*/
     herr_t      ret_value = SUCCEED;    /* Return value                 */
 
@@ -2802,6 +2814,43 @@ H5T_conv_enum(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts,
             if(H5P_get(plist, H5D_XFER_CONV_CB_NAME, &cb_struct) < 0)
                 HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get conversion exception callback")
 
+            /* Get the property of overflow handling */
+            if(H5P_get(plist, H5D_XFER_CONV_ENUM_OVERFLOW_NAME, &conv_overflow) < 0)
+                HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get conversion exception callback")
+
+            /* Preparation for converting the data when overflow happens */
+	    if(conv_overflow) {
+		/* Retrieve base type for the source type */
+		if(NULL == (src_super = H5T_get_super(src)))
+		    HGOTO_ERROR(H5E_DATATYPE, H5E_BADTYPE, FAIL, "unable to get base type for the source type")
+
+		/* Retrieve base type for the destination type */
+		if(NULL == (dst_super = H5T_get_super(dst)))
+		    HGOTO_ERROR(H5E_DATATYPE, H5E_BADTYPE, FAIL, "unable to get base type for the destination type")
+
+		/* Find the conversion function */
+		if(NULL == (tpath = H5T_path_find(src_super, dst_super, NULL, NULL, dxpl_id, FALSE)))
+		    HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to find convert path between src and dst types")
+
+                /* The conversion function needs the type IDs */
+		if((src_super_id = H5I_register(H5I_DATATYPE, src_super, FALSE)) < 0) 
+		    HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register data type")
+
+		if((dst_super_id = H5I_register(H5I_DATATYPE, dst_super, FALSE)) < 0) 
+		    HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register data type")
+
+		/* Get the size of the base types of src & dst */
+		if(!(src_super_size = H5T_get_size(src_super)))
+		    HGOTO_ERROR(H5E_DATATYPE, H5E_BADTYPE, FAIL, "unable to get type size")
+
+		if(!(dst_super_size = H5T_get_size(dst_super)))
+		    HGOTO_ERROR(H5E_DATATYPE, H5E_BADTYPE, FAIL, "unable to get type size")
+
+                /* Conversion buffer for overflowing data */
+		if(!(tmp_buf = HDmalloc(MAX(src_super_size, dst_super_size))))
+		    HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to allocate buffer")
+	    }
+
             for(i = 0; i < nelmts; i++, s += src_delta, d += dst_delta) {
                 if(priv->length) {
                     /* Use O(1) lookup */
@@ -2827,7 +2876,17 @@ H5T_conv_enum(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts,
                         }
 
                         if(except_ret == H5T_CONV_UNHANDLED) {
-                            HDmemset(d, 0xff, dst->shared->size);
+			    if(conv_overflow) {
+                                /* Copy the source data to a small buffer, convert it, then copy it back to the 
+                                 * destination */
+				HDmemcpy(tmp_buf, s, src_super_size);
+				if(H5T_convert(tpath, src_super_id, dst_super_id, 1, (size_t)0, (size_t)0, 
+                                    tmp_buf, NULL, dxpl_id) < 0)
+				    HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "data type conversion failed")
+				HDmemcpy(d, tmp_buf, dst_super_size);
+			    } else
+                                /* Fill the destination with 0xff if we don't convert it */
+				HDmemset(d, 0xff, dst->shared->size);
                         } else if(except_ret == H5T_CONV_ABORT)
                             HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCONVERT, FAIL, "can't handle conversion exception")
                     } else {
@@ -2860,7 +2919,18 @@ H5T_conv_enum(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts,
                         }
 
                         if(except_ret == H5T_CONV_UNHANDLED) {
-                            HDmemset(d, 0xff, dst->shared->size);
+			    if(conv_overflow) {
+                                /* Copy the source data to a small buffer, convert it, then copy it back to the 
+                                 * destination */
+			        HDmemcpy(tmp_buf, s, src_super_size);
+			        if(H5T_convert(tpath, src_super_id, dst_super_id, 1, (size_t)0, (size_t)0, 
+                                    tmp_buf, NULL, dxpl_id) < 0)
+				    HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "data type conversion failed")
+			        HDmemcpy(d, tmp_buf, dst_super_size);
+                            } else
+                                /* Fill the destination with 0xff if we don't convert it */
+                                HDmemset(d, 0xff, dst->shared->size);
+
                         } else if(except_ret == H5T_CONV_ABORT)
                             HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCONVERT, FAIL, "can't handle conversion exception")
                     } /* end if */
@@ -2871,6 +2941,19 @@ H5T_conv_enum(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts,
                     } /* end else */
                 }
             }
+
+            /* Close the type IDs.  Have to use public function here */
+	    if(conv_overflow) {
+                /* Disable the error stack for these two public functions. Otherwise they would
+                 * clear it */
+                H5E_BEGIN_TRY {
+		    H5Tclose(src_super_id);
+		    H5Tclose(dst_super_id);
+                } H5E_END_TRY;
+
+		HDfree(tmp_buf);
+            }
+
             break;
 
         default: