[svn-r17977] Bug fix for 1584. H5Tdetect_class said a VL string is a string type. But when it's in a

compound type, it says it's a VL type.  We want to tell user a VL string is a string.  But
internally we treat it as a VL type.  I added a flag as a parameter of H5T_detect_class.  
It tells whether the caller is the public function H5Tdetect_class.  I also added a 
detection for VL string in the private function for the compound case (or array or nested 
VL type).

Tested on jam and amani - I tested the same change for 1.8 with h5committest.

13 files changed, 48 insertions, 25 deletions

diff --git a/src/H5Aint.c b/src/H5Aint.c
index c50126a..d63b5f0 100644
--- a/src/H5Aint.c
+++ b/src/H5Aint.c
@@ -944,7 +944,7 @@ H5A_attr_copy_file(const H5A_t *attr_src, H5F_t *file_dst, hbool_t *recompute_si
             HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
 
         /* Check if we need to convert data */
-        if(H5T_detect_class(attr_src->shared->dt, H5T_VLEN) > 0) {
+        if(H5T_detect_class(attr_src->shared->dt, H5T_VLEN, FALSE) > 0) {
             H5T_path_t  *tpath_src_mem, *tpath_mem_dst;   /* Datatype conversion paths */
             H5T_t *dt_mem;              /* Memory datatype */
             size_t src_dt_size;         /* Source datatype size */
diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c
index a358320..cb7901e 100644
--- a/src/H5Dchunk.c
+++ b/src/H5Dchunk.c
@@ -4150,7 +4150,7 @@ H5D_chunk_copy_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata)
 
     /* Check parameter for type conversion */
     if(udata->do_convert) {
-        if(H5T_detect_class(udata->dt_src, H5T_VLEN) > 0)
+        if(H5T_detect_class(udata->dt_src, H5T_VLEN, FALSE) > 0)
             is_vlen = TRUE;
         else if((H5T_get_class(udata->dt_src, FALSE) == H5T_REFERENCE) && (udata->file_src != udata->idx_info_dst->f))
             fix_ref = TRUE;
@@ -4384,7 +4384,7 @@ H5D_chunk_copy(H5F_t *f_src, H5O_storage_chunk_t *storage_src,
         HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register source file datatype")
 
     /* If there's a VLEN source datatype, set up type conversion information */
-    if(H5T_detect_class(dt_src, H5T_VLEN) > 0) {
+    if(H5T_detect_class(dt_src, H5T_VLEN, FALSE) > 0) {
         H5T_t *dt_dst;              /* Destination datatype */
         H5T_t *dt_mem;              /* Memory datatype */
         size_t mem_dt_size;         /* Memory datatype size */
diff --git a/src/H5Dcompact.c b/src/H5Dcompact.c
index 33b92de..edd2fe6 100644
--- a/src/H5Dcompact.c
+++ b/src/H5Dcompact.c
@@ -412,7 +412,7 @@ H5D_compact_copy(H5F_t *f_src, H5O_storage_compact_t *storage_src, H5F_t *f_dst,
         HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL, "unable to register source file datatype")
 
     /* If there's a VLEN source datatype, do type conversion information */
-    if(H5T_detect_class(dt_src, H5T_VLEN) > 0) {
+    if(H5T_detect_class(dt_src, H5T_VLEN, FALSE) > 0) {
         H5T_path_t  *tpath_src_mem, *tpath_mem_dst;   /* Datatype conversion paths */
         H5T_t *dt_dst;              /* Destination datatype */
         H5T_t *dt_mem;              /* Memory datatype */
diff --git a/src/H5Dcontig.c b/src/H5Dcontig.c
index 3e88be6..282cecb 100644
--- a/src/H5Dcontig.c
+++ b/src/H5Dcontig.c
@@ -1275,7 +1275,7 @@ H5D_contig_copy(H5F_t *f_src, const H5O_storage_contig_t *storage_src,
         HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register source file datatype")
 
     /* If there's a VLEN source datatype, set up type conversion information */
-    if(H5T_detect_class(dt_src, H5T_VLEN) > 0) {
+    if(H5T_detect_class(dt_src, H5T_VLEN, FALSE) > 0) {
         /* create a memory copy of the variable-length datatype */
         if(NULL == (dt_mem = H5T_copy(dt_src, H5T_COPY_TRANSIENT)))
             HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to copy")
diff --git a/src/H5Dfill.c b/src/H5Dfill.c
index bacaca2..2f3f112 100644
--- a/src/H5Dfill.c
+++ b/src/H5Dfill.c
@@ -240,7 +240,7 @@ H5D_fill(const void *fill, const H5T_t *fill_type, void *buf,
          * then do conversion on each element so that each of them has a copy
          * of the VL data.
          */
-        if(TRUE == H5T_detect_class(fill_type, H5T_VLEN)) {
+        if(TRUE == H5T_detect_class(fill_type, H5T_VLEN, FALSE)) {
             H5D_dxpl_cache_t _dxpl_cache;       /* Data transfer property cache buffer */
             H5D_dxpl_cache_t *dxpl_cache = &_dxpl_cache;   /* Data transfer property cache */
             H5S_sel_iter_t mem_iter;            /* Memory selection iteration info */
@@ -397,7 +397,7 @@ H5D_fill_init(H5D_fill_buf_info_t *fb_info, void *caller_fill_buf,
         htri_t has_vlen_type;   /* Whether the datatype has a VL component */
 
         /* Detect whether the datatype has a VL component */
-        if((has_vlen_type = H5T_detect_class(dset_type, H5T_VLEN)) < 0)
+        if((has_vlen_type = H5T_detect_class(dset_type, H5T_VLEN, FALSE)) < 0)
             HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "unable to detect vlen datatypes?")
         fb_info->has_vlen_fill_type = (hbool_t)has_vlen_type;
 
diff --git a/src/H5Dint.c b/src/H5Dint.c
index b92200b..c11c7bd 100644
--- a/src/H5Dint.c
+++ b/src/H5Dint.c
@@ -736,7 +736,7 @@ H5D_update_oh_info(H5F_t *file, hid_t dxpl_id, H5D_t *dset, hid_t dapl_id)
         HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't tell if fill value defined")
 
     /* Special case handling for variable-length types */
-    if(H5T_detect_class(type, H5T_VLEN)) {
+    if(H5T_detect_class(type, H5T_VLEN, FALSE)) {
         /* If the default fill value is chosen for variable-length types, always write it */
         if(fill_prop->fill_time == H5D_FILL_TIME_IFSET && fill_status == H5D_FILL_VALUE_DEFAULT) {
             /* Update dataset creation property */
@@ -927,7 +927,7 @@ H5D_create(H5F_t *file, hid_t type_id, const H5S_t *space, hid_t dcpl_id,
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "datatype is not sensible")
 
     /* Check if the datatype is/contains a VL-type */
-    if(H5T_detect_class(type, H5T_VLEN))
+    if(H5T_detect_class(type, H5T_VLEN, FALSE))
         has_vl_type = TRUE;
 
     /* Check if the dataspace has an extent set (or is NULL) */
diff --git a/src/H5Dio.c b/src/H5Dio.c
index 62d2177..406b5ce 100644
--- a/src/H5Dio.c
+++ b/src/H5Dio.c
@@ -485,7 +485,7 @@ H5D_write(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space,
         /* If MPI based VFD is used, no VL datatype support yet. */
         /* This is because they use the global heap in the file and we don't */
         /* support parallel access of that yet */
-        if(H5T_detect_class(type_info.mem_type, H5T_VLEN) > 0)
+        if(H5T_detect_class(type_info.mem_type, H5T_VLEN, FALSE) > 0)
             HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "Parallel IO does not support writing VL datatypes yet")
 
         /* If MPI based VFD is used, no VL datatype support yet. */
@@ -543,7 +543,7 @@ H5D_write(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space,
             HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "can't retrieve number of elements in file dataset")
 
         /* Always allow fill values to be written if the dataset has a VL datatype */
-        if(H5T_detect_class(dataset->shared->type, H5T_VLEN))
+        if(H5T_detect_class(dataset->shared->type, H5T_VLEN, FALSE))
             full_overwrite = FALSE;
         else
             full_overwrite = (hbool_t)((hsize_t)file_nelmts == nelmts ? TRUE : FALSE);
@@ -751,7 +751,7 @@ H5D_typeinfo_init(const H5D_t *dset, const H5D_dxpl_cache_t *dxpl_cache,
         type_info->cmpd_subset = H5T_path_compound_subset(type_info->tpath);
 
         /* Check if we need a background buffer */
-        if(do_write && H5T_detect_class(dset->shared->type, H5T_VLEN))
+        if(do_write && H5T_detect_class(dset->shared->type, H5T_VLEN, FALSE))
             type_info->need_bkg = H5T_BKG_YES;
         else {
             H5T_bkg_t path_bkg;     /* Type conversion's background info */
diff --git a/src/H5Ofill.c b/src/H5Ofill.c
index 8cf901f..d133a0a 100644
--- a/src/H5Ofill.c
+++ b/src/H5Ofill.c
@@ -689,7 +689,7 @@ H5O_fill_reset_dyn(H5O_fill_t *fill)
     HDassert(fill);
 
     if(fill->buf) {
-        if(fill->type && H5T_detect_class(fill->type, H5T_VLEN) > 0) {
+        if(fill->type && H5T_detect_class(fill->type, H5T_VLEN, FALSE) > 0) {
             H5T_t *fill_type;           /* Copy of fill value datatype */
             H5S_t *fill_space;          /* Scalar dataspace for fill value element */
 
diff --git a/src/H5T.c b/src/H5T.c
index 64e39d6..bbd3854 100644
--- a/src/H5T.c
+++ b/src/H5T.c
@@ -1918,7 +1918,7 @@ H5Tdetect_class(hid_t type, H5T_class_t cls)
     if(H5T_IS_VL_STRING(dt->shared))
         ret_value = (H5T_STRING == cls);
     else
-        ret_value = H5T_detect_class(dt, cls);
+        ret_value = H5T_detect_class(dt, cls, TRUE);
 
 done:
     FUNC_LEAVE_API(ret_value)
@@ -1937,11 +1937,18 @@ done:
  *		Wednesday, November 29, 2000
  *
  * Modifications:
- *
+ *              Raymond Lu
+ *              4 December 2009
+ *              Added a flag as a parameter to indicate whether the caller is
+ *              H5Tdetect_class.  I also added the check for VL string type 
+ *              just like the public function.  Because we want to tell users
+ *              VL string is a string type but we treat it as a VL type 
+ *              internally, H5T_detect_class needs to know where the caller
+ *              is from.
  *-------------------------------------------------------------------------
  */
 htri_t
-H5T_detect_class (const H5T_t *dt, H5T_class_t cls)
+H5T_detect_class (const H5T_t *dt, H5T_class_t cls, hbool_t from_api)
 {
     unsigned	i;
     htri_t      ret_value=FALSE;        /* Return value */
@@ -1951,6 +1958,12 @@ H5T_detect_class (const H5T_t *dt, H5T_class_t cls)
     assert(dt);
     assert(cls>H5T_NO_CLASS && cls<H5T_NCLASSES);
 
+    /* Consider VL string as a string for API, as a VL for internal use.  The same
+     * check is also in H5Tdetect_class.  Do it again to check members of nested 
+     * compound type and base type of array and VL type. */
+    if(from_api && H5T_IS_VL_STRING(dt->shared))
+        HGOTO_DONE(H5T_STRING == cls);
+ 
     /* Check if this type is the correct type */
     if(dt->shared->type==cls)
         HGOTO_DONE(TRUE);
@@ -1961,13 +1974,18 @@ H5T_detect_class (const H5T_t *dt, H5T_class_t cls)
             for (i=0; i<dt->shared->u.compnd.nmembs; i++) {
                 htri_t nested_ret;      /* Return value from nested call */
 
+                /* Consider VL string as a string for API, as a VL for internal use.  
+                 * Do it again here to check members of compound type. */
+                if(from_api && H5T_IS_VL_STRING(dt->shared->u.compnd.memb[i].type->shared))
+                    HGOTO_DONE(H5T_STRING == cls);
+
                 /* Check if this field's type is the correct type */
                 if(dt->shared->u.compnd.memb[i].type->shared->type==cls)
                     HGOTO_DONE(TRUE);
 
                 /* Recurse if it's VL, compound, enum or array */
                 if(H5T_IS_COMPLEX(dt->shared->u.compnd.memb[i].type->shared->type))
-                    if((nested_ret=H5T_detect_class(dt->shared->u.compnd.memb[i].type,cls))!=FALSE)
+                    if((nested_ret=H5T_detect_class(dt->shared->u.compnd.memb[i].type, cls, from_api))!=FALSE)
                         HGOTO_DONE(nested_ret);
             } /* end for */
             break;
@@ -1975,7 +1993,7 @@ H5T_detect_class (const H5T_t *dt, H5T_class_t cls)
         case H5T_ARRAY:
         case H5T_VLEN:
         case H5T_ENUM:
-            HGOTO_DONE(H5T_detect_class(dt->shared->parent,cls));
+            HGOTO_DONE(H5T_detect_class(dt->shared->parent, cls, from_api));
 
         default:
             break;
@@ -5125,7 +5143,7 @@ H5T_is_relocatable(const H5T_t *dt)
     HDassert(dt);
 
     /* VL and reference datatypes are relocatable */
-    if(H5T_detect_class(dt, H5T_VLEN) || H5T_detect_class(dt, H5T_REFERENCE))
+    if(H5T_detect_class(dt, H5T_VLEN, FALSE) || H5T_detect_class(dt, H5T_REFERENCE, FALSE))
         ret_value = TRUE;
 
 done:
diff --git a/src/H5Tcompound.c b/src/H5Tcompound.c
index db7a701b..48192d2 100644
--- a/src/H5Tcompound.c
+++ b/src/H5Tcompound.c
@@ -403,7 +403,7 @@ H5Tpack(hid_t type_id)
     H5TRACE1("e", "i", type_id);
 
     /* Check args */
-    if(NULL == (dt = (H5T_t *)H5I_object_verify(type_id, H5I_DATATYPE)) || H5T_detect_class(dt, H5T_COMPOUND) <= 0)
+    if(NULL == (dt = (H5T_t *)H5I_object_verify(type_id, H5I_DATATYPE)) || H5T_detect_class(dt, H5T_COMPOUND, TRUE) <= 0)
 	HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a compound datatype")
 
     /* Pack */
@@ -534,7 +534,7 @@ H5T_pack(const H5T_t *dt)
 
     HDassert(dt);
 
-    if(H5T_detect_class(dt, H5T_COMPOUND) > 0) {
+    if(H5T_detect_class(dt, H5T_COMPOUND, FALSE) > 0) {
         /* If datatype has been packed, skip packing it and indicate success */
         if(TRUE == H5T_is_packed(dt))
             HGOTO_DONE(SUCCEED)
diff --git a/src/H5Tconv.c b/src/H5Tconv.c
index ab9e703..ecf93e1 100644
--- a/src/H5Tconv.c
+++ b/src/H5Tconv.c
@@ -2996,7 +2996,7 @@ H5T_conv_vlen(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts,
                 noop_conv = TRUE;
 
             /* Check if we need a temporary buffer for this conversion */
-            parent_is_vlen = H5T_detect_class(dst->shared->parent, H5T_VLEN);
+            parent_is_vlen = H5T_detect_class(dst->shared->parent, H5T_VLEN, FALSE);
             if(tpath->cdata.need_bkg || parent_is_vlen) {
                 /* Set up initial background buffer */
                 tmp_buf_size = MAX(src_base_size,dst_base_size);
diff --git a/src/H5Tprivate.h b/src/H5Tprivate.h
index 9189f04..afa6ceb 100644
--- a/src/H5Tprivate.h
+++ b/src/H5Tprivate.h
@@ -111,7 +111,7 @@ H5_DLL herr_t H5T_lock(H5T_t *dt, hbool_t immutable);
 H5_DLL herr_t H5T_close(H5T_t *dt);
 H5_DLL H5T_t *H5T_get_super(const H5T_t *dt);
 H5_DLL H5T_class_t H5T_get_class(const H5T_t *dt, htri_t internal);
-H5_DLL htri_t H5T_detect_class(const H5T_t *dt, H5T_class_t cls);
+H5_DLL htri_t H5T_detect_class(const H5T_t *dt, H5T_class_t cls, hbool_t from_api);
 H5_DLL size_t H5T_get_size(const H5T_t *dt);
 H5_DLL int    H5T_cmp(const H5T_t *dt1, const H5T_t *dt2, hbool_t superset);
 H5_DLL herr_t H5T_debug(const H5T_t *dt, FILE * stream);
diff --git a/test/dtypes.c b/test/dtypes.c
index 2619f1b..d2db20b 100644
--- a/test/dtypes.c
+++ b/test/dtypes.c
@@ -325,7 +325,10 @@ test_copy(void)
  *              Saturday, August 30, 2003
  *
  * Modifications:
- *
+ *              Raymond Lu
+ *              8 December 2009
+ *              I added a field of VL string in the compound type to test 
+ *              H5Tdetect_class correctly detect it as string type.
  *-------------------------------------------------------------------------
  */
 static int
@@ -342,6 +345,7 @@ test_detect(void)
         hobj_ref_t arr_r[3][3];
         int i;
         hvl_t vl_f;
+        hvl_t vl_s;
         char c;
         short s;
     };
@@ -441,6 +445,7 @@ test_detect(void)
     if (H5Tinsert(cplx_cmpd_id, "arr_r", HOFFSET(struct complex, arr_r), atom_arr_id) < 0) TEST_ERROR
     if (H5Tinsert(cplx_cmpd_id, "i", HOFFSET(struct complex, i), H5T_NATIVE_INT) < 0) TEST_ERROR
     if (H5Tinsert(cplx_cmpd_id, "vl_f", HOFFSET(struct complex, vl_f), atom_vlf_id) < 0) TEST_ERROR
+    if (H5Tinsert(cplx_cmpd_id, "vl_s", HOFFSET(struct complex, vl_s), atom_vls_id) < 0) TEST_ERROR
     if (H5Tinsert(cplx_cmpd_id, "c", HOFFSET(struct complex, c), H5T_NATIVE_CHAR) < 0) TEST_ERROR
     if (H5Tinsert(cplx_cmpd_id, "s", HOFFSET(struct complex, s), H5T_NATIVE_SHORT) < 0) TEST_ERROR
 
@@ -450,12 +455,12 @@ test_detect(void)
     if(H5Tdetect_class(cplx_cmpd_id,H5T_REFERENCE)!=TRUE) TEST_ERROR
     if(H5Tdetect_class(cplx_cmpd_id,H5T_INTEGER)!=TRUE) TEST_ERROR
     if(H5Tdetect_class(cplx_cmpd_id,H5T_FLOAT)!=TRUE) TEST_ERROR
+    if(H5Tdetect_class(cplx_cmpd_id,H5T_STRING)!=TRUE) TEST_ERROR
     if(H5Tdetect_class(cplx_cmpd_id,H5T_VLEN)!=TRUE) TEST_ERROR
 
     /* Make certain that an incorrect class is not detected */
     if(H5Tdetect_class(cplx_cmpd_id,H5T_TIME)!=FALSE) TEST_ERROR
     if(H5Tdetect_class(cplx_cmpd_id,H5T_ENUM)!=FALSE) TEST_ERROR
-    if(H5Tdetect_class(cplx_cmpd_id,H5T_STRING)!=FALSE) TEST_ERROR
 
     /* Close complex compound datatype */
     if(H5Tclose(cplx_cmpd_id) < 0) TEST_ERROR