[svn-r20285] Purpose:

    Improve the previous fix for Bug 2216 - GMQS: h5diff - memory leak when 
    compares vlen string in dataset or attributes

Description:
    Related to the previous checkin r20270 and r20266.
    Improve h5tools_detect_vlen() code for better performance. H5Tdetect_class
    already recusive on given type so don't need to be part of recusive call
    again. Also improve error handlings in h5tools_detect_vlen and 
    h5tools_detect_vlen_str functions.
    Also updated h5ls and h5dump code accordingly.

     
Tested:
 jam (linux32-LE), amani (linux64-LE), heiwa (linuxppc64-BE), tejeda (mac32-LE), linew (solaris-BE), Cmake - jam

4 files changed, 72 insertions, 55 deletions

diff --git a/tools/lib/h5diff_attr.c b/tools/lib/h5diff_attr.c
index 01fa896..097c809 100644
--- a/tools/lib/h5diff_attr.c
+++ b/tools/lib/h5diff_attr.c
@@ -223,12 +223,12 @@ hsize_t diff_attr(hid_t loc1_id,
 
         /* Free buf1 and buf2, check both VLEN-data VLEN-string to reclaim any 
          * VLEN memory first */
-        if(TRUE == h5tools_detect_vlen_data(mtype1_id))
+        if(TRUE == h5tools_detect_vlen(mtype1_id))
             H5Dvlen_reclaim(mtype1_id, space1_id, H5P_DEFAULT, buf1);
         HDfree(buf1);
 
         buf1 = NULL;
-        if(TRUE == h5tools_detect_vlen_data(mtype2_id))
+        if(TRUE == h5tools_detect_vlen(mtype2_id))
             H5Dvlen_reclaim(mtype2_id, space2_id, H5P_DEFAULT, buf2);
         HDfree(buf2);
         buf2 = NULL;
@@ -258,12 +258,12 @@ hsize_t diff_attr(hid_t loc1_id,
 error:
     H5E_BEGIN_TRY {
         if(buf1) {
-            if(TRUE == h5tools_detect_vlen_data(mtype1_id))
+            if(TRUE == h5tools_detect_vlen(mtype1_id))
                 H5Dvlen_reclaim(mtype1_id, space1_id, H5P_DEFAULT, buf1);
             HDfree(buf1);
         } /* end if */
         if(buf2) {
-            if(TRUE == h5tools_detect_vlen_data(mtype2_id))
+            if(TRUE == h5tools_detect_vlen(mtype2_id))
                 H5Dvlen_reclaim(mtype2_id, space2_id, H5P_DEFAULT, buf2);
             HDfree(buf2);
         } /* end if */
diff --git a/tools/lib/h5diff_dset.c b/tools/lib/h5diff_dset.c
index b1193c6..230e7bb 100644
--- a/tools/lib/h5diff_dset.c
+++ b/tools/lib/h5diff_dset.c
@@ -330,7 +330,7 @@ hsize_t diff_datasetid( hid_t did1,
     
     /* Check if type is either VLEN-data or VLEN-string to reclaim any
      * VLEN memory buffer later */
-    if( TRUE == h5tools_detect_vlen_data(m_tid1) )
+    if( TRUE == h5tools_detect_vlen(m_tid1) )
         vl_data = TRUE;
 
     /*-------------------------------------------------------------------------
diff --git a/tools/lib/h5tools.c b/tools/lib/h5tools.c
index 84a164b..7389351 100644
--- a/tools/lib/h5tools.c
+++ b/tools/lib/h5tools.c
@@ -622,51 +622,42 @@ h5tools_ncols(const char *s)
 }
 
 /*-------------------------------------------------------------------------
- * Function: h5tools_detect_vlen_data
+ * Function: h5tools_detect_vlen
  *
  * Purpose: Recursive check for any variable length data in given type.
  *
- * Return: TRUE if type conatains any variable length data, else FALSE
+ * Return: 
+ *    TRUE : type conatains any variable length data
+ *    FALSE : type doesn't contain any variable length data
+ *    Negative value: error occur
  * 
  * Programmer: Jonathan Kim  March 18, 2011
- * 
- * Note: Adopted from h5tools_detect_vlen_str() which only check
- *       vlen-string data in type.
  *-------------------------------------------------------------------------
  */
 htri_t
-h5tools_detect_vlen_data(hid_t tid)
+h5tools_detect_vlen(hid_t tid)
 {
-    int i = 0;
-    int n = 0;
-    htri_t has_vlen_str = FALSE;
-    H5T_class_t tclass = -1;
+    htri_t status;
+    htri_t ret = FALSE;
+    /* recursive detect any vlen data values in type (compound, array ...) */
+    status = H5Tdetect_class(tid, H5T_VLEN);
+    if ( (status == TRUE) || (status < 0) )
+    {
+        ret = status;
+        goto done;
+    }
 
-    /* detect any vlen data in type */
-    if (H5Tdetect_class(tid, H5T_VLEN) == TRUE ||           /* VLEN-data */
-        H5Tis_variable_str(tid) == TRUE)                    /* VLEN-string*/
-        return TRUE;
+    /* recursive detect any vlen string in type (compound, array ...) */
+    status = h5tools_detect_vlen_str(tid);
+    if ( (status == TRUE) || (status < 0) )
 
-    tclass = H5Tget_class(tid);
-    if (tclass == H5T_ARRAY) {
-        hid_t btid = H5Tget_super(tid);
-        has_vlen_str = h5tools_detect_vlen_data(btid);
-        H5Tclose(btid);
-        return has_vlen_str;
-    }
-    else if (tclass == H5T_COMPOUND) {
-        n = H5Tget_nmembers(tid);
-        for (i = 0; i < n; i++) {
-            hid_t mtid = H5Tget_member_type(tid, i);
-            has_vlen_str = h5tools_detect_vlen_data(mtid);
-            if (has_vlen_str == TRUE) {
-                H5Tclose(mtid);
-                return TRUE;
-            }
-            H5Tclose(mtid);
-        }
+    {
+        ret = status;
+        goto done;
     }
-    return FALSE;
+
+done:
+    return ret;
 }
 
 
@@ -675,7 +666,10 @@ h5tools_detect_vlen_data(hid_t tid)
  *
  * Purpose: Recursive check for variable length string of a datatype.
  *
- * Return: TRUE if type conatains a variable string type, else FALSE
+ * Return: 
+ *    TRUE : type conatains any variable length string
+ *    FALSE : type doesn't contain any variable length string
+ *    Negative value: error occur
  *
  *-------------------------------------------------------------------------
  */
@@ -684,32 +678,55 @@ h5tools_detect_vlen_str(hid_t tid)
 {
     int i = 0;
     int n = 0;
-    htri_t has_vlen_str = FALSE;
+    htri_t ret = FALSE;
     H5T_class_t tclass = -1;
+    hid_t btid;
+    hid_t mtid;
 
-    if (H5Tis_variable_str(tid) == TRUE)
-        return TRUE;
+    ret = H5Tis_variable_str(tid);
+    if ( (ret == TRUE) || (ret < 0) )
+        goto done;
 
     tclass = H5Tget_class(tid);
-    if (tclass == H5T_ARRAY) {
-        hid_t btid = H5Tget_super(tid);
-        has_vlen_str = h5tools_detect_vlen_str(btid);
-        H5Tclose(btid);
-        return has_vlen_str;
+    if (tclass == H5T_ARRAY) 
+    {
+        btid = H5Tget_super(tid);
+        if (btid < 0)
+        {
+            ret = (htri_t) btid;
+            goto done;
+        }
+        ret = h5tools_detect_vlen_str(btid);
+        if ( (ret == TRUE) || (ret < 0) ) 
+        {
+            H5Tclose(btid);
+            goto done;
+        }
     }
-    else if (tclass == H5T_COMPOUND) {
+    else if (tclass == H5T_COMPOUND) 
+    {
         n = H5Tget_nmembers(tid);
-        for (i = 0; i < n; i++) {
-            hid_t mtid = H5Tget_member_type(tid, i);
-            has_vlen_str = h5tools_detect_vlen_str(mtid);
-            if (has_vlen_str == TRUE) {
+        if (n < 0) 
+        {
+            n = ret;
+            goto done;
+        }
+
+        for (i = 0; i < n; i++) 
+        {
+            mtid = H5Tget_member_type(tid, i);
+            ret = h5tools_detect_vlen_str(mtid);
+            if ( (ret == TRUE) || (ret < 0) ) 
+            {
                 H5Tclose(mtid);
-                return TRUE;
+                goto done;
             }
             H5Tclose(mtid);
         }
     }
-    return FALSE;
+
+done:
+    return ret;
 }
 
 /*-------------------------------------------------------------------------
diff --git a/tools/lib/h5tools.h b/tools/lib/h5tools.h
index 376f05a..d08edf7 100644
--- a/tools/lib/h5tools.h
+++ b/tools/lib/h5tools.h
@@ -553,7 +553,7 @@ H5TOOLS_DLL hid_t    h5tools_get_native_type(hid_t type);
 H5TOOLS_DLL hid_t    h5tools_get_little_endian_type(hid_t type);
 H5TOOLS_DLL hid_t    h5tools_get_big_endian_type(hid_t type);
 
-H5TOOLS_DLL htri_t   h5tools_detect_vlen_data(hid_t tid);
+H5TOOLS_DLL htri_t   h5tools_detect_vlen(hid_t tid);
 H5TOOLS_DLL htri_t   h5tools_detect_vlen_str(hid_t tid);
 
 H5TOOLS_DLL void     h5tools_dump_simple_data(FILE *stream, const h5tool_format_t *info, hid_t container,