[svn-r21358] Purpose:

    HDFFV-7712 - h5diff: segfault over combinations of complex container types (compound, array, vlen)


Description:
    - Fixed segfault over dataset with container types (array,lven) with multiple nested compound types. (ex: compound->array->compound, compound->vlen->compound)

Tested:
    jam (linux32-LE), koala (linux64-LE), heiwa (linuxppc64-BE), tejeda (mac32-LE), linew (solaris-BE), Cmake (jam)

10 files changed, 728 insertions, 35 deletions

diff --git a/MANIFEST b/MANIFEST
index 50a40a4..caacb8c 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -1856,6 +1856,7 @@
 ./tools/h5diff/testfiles/h5diff_517.txt
 ./tools/h5diff/testfiles/h5diff_518.txt
 ./tools/h5diff/testfiles/h5diff_530.txt
+./tools/h5diff/testfiles/h5diff_540.txt
 
 ./tools/h5diff/testfiles/h5diff_attr1.h5
 ./tools/h5diff/testfiles/h5diff_attr2.h5
@@ -1893,6 +1894,8 @@
 ./tools/h5diff/testfiles/h5diff_dset_zero_dim_size1.h5
 ./tools/h5diff/testfiles/h5diff_dset_zero_dim_size2.h5
 ./tools/h5diff/testfiles/h5diff_enum_invalid_values.h5
+./tools/h5diff/testfiles/compounds_array_vlen1.h5
+./tools/h5diff/testfiles/compounds_array_vlen2.h5
 
 #test files for h5repack
 ./tools/h5repack/testfiles/h5repack_fill.h5
diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt
index 2efea2d..31f6d4b 100644
--- a/release_docs/RELEASE.txt
+++ b/release_docs/RELEASE.txt
@@ -571,6 +571,10 @@ Bug Fixes since HDF5-1.8.0 release
 
     Tools
     -----
+        - h5diff: fixed segfault over dataset with container types
+          (array,lven) with multiple nested compound types. 
+          (ex: compound->array->compound, compound->vlen->compound)
+          HDFFV-7712  JKM (09/01/2011)
         - h5repack: added macro to handle failure in H5Dread/write when memory 
           allocation failed inside the library. (PC -- 2011/08/19)
         - Fixed h5jam not to allow specifying an HDF5 formatted file as input 
diff --git a/tools/h5diff/CMakeLists.txt b/tools/h5diff/CMakeLists.txt
index 52a58eb..12dc388 100644
--- a/tools/h5diff/CMakeLists.txt
+++ b/tools/h5diff/CMakeLists.txt
@@ -158,6 +158,7 @@ IF (BUILD_TESTING)
       h5diff_517.txt
       h5diff_518.txt
       h5diff_530.txt
+      h5diff_540.txt
       h5diff_600.txt
       h5diff_601.txt
       h5diff_603.txt
@@ -245,6 +246,8 @@ IF (BUILD_TESTING)
       h5diff_comp_vl_strs.h5
       h5diff_attr_v_level1.h5
       h5diff_attr_v_level2.h5
+      compounds_array_vlen1.h5
+      compounds_array_vlen2.h5
   )
 
   FOREACH (txt_file ${HDF5_REFERENCE_FILES})
@@ -477,6 +480,9 @@ IF (BUILD_TESTING)
   SET (EXCLUDE_FILE2_2 h5diff_exclude2-2.h5)
   # compound type with multiple vlen string types
   SET (COMP_VL_STRS_FILE h5diff_comp_vl_strs.h5)
+  # container types (array,vlen) with multiple nested compound types
+  SET (COMPS_ARRAY_VLEN_FILE1 compounds_array_vlen1.h5)
+  SET (COMPS_ARRAY_VLEN_FILE2 compounds_array_vlen2.h5)
   # attrs with verbose option level
   SET (ATTR_VERBOSE_LEVEL_FILE1 h5diff_attr_v_level1.h5)
   SET (ATTR_VERBOSE_LEVEL_FILE2 h5diff_attr_v_level2.h5)
@@ -703,6 +709,8 @@ IF (BUILD_TESTING)
           h5diff_518.out.err
           h5diff_530.out
           h5diff_530.out.err
+          h5diff_540.out
+          h5diff_540.out.err
           h5diff_600.out
           h5diff_600.out.err
           h5diff_601.out
@@ -1303,6 +1311,12 @@ ADD_H5_TEST (h5diff_484 0 -v --exclude-path "/dset3" ${EXCLUDE_FILE1_1} ${EXCLUD
 ADD_H5_TEST (h5diff_530 0 -v  ${COMP_VL_STRS_FILE} ${COMP_VL_STRS_FILE} /group /group_copy)
 
 # ##############################################################################
+# # Test container types (array,vlen) with multiple nested compound types
+# # Complex compound types in dataset and attribute
+# ##############################################################################
+ADD_H5_TEST (h5diff_540 1 -v ${COMPS_ARRAY_VLEN_FILE1} ${COMPS_ARRAY_VLEN_FILE2})
+
+# ##############################################################################
 # # Test mutually exclusive options 
 # ##############################################################################
 #
diff --git a/tools/h5diff/h5diffgentest.c b/tools/h5diff/h5diffgentest.c
index 719707f..562c579 100644
--- a/tools/h5diff/h5diffgentest.c
+++ b/tools/h5diff/h5diffgentest.c
@@ -77,6 +77,9 @@
 #define ATTR_VERBOSE_LEVEL_FILE2 "h5diff_attr_v_level2.h5"
 /* file containing valid/invalid enum value mix */
 #define ENUM_INVALID_VALUES "h5diff_enum_invalid_values.h5"
+/* file with container types (array,vlen) with multiple compounds */
+#define COMPS_COMPLEX1   "compounds_array_vlen1.h5"
+#define COMPS_COMPLEX2   "compounds_array_vlen2.h5"
 
 #define UIMAX    4294967295u /*Maximum value for a variable of type unsigned int */
 #define STR_SIZE 3
@@ -134,6 +137,10 @@ static int test_exclude_obj2(const char *fname1, const char *fname2);
 static int test_comp_vlen_strings(const char *fname1, const char *grp_name, int is_file_new);
 static int test_attributes_verbose_level(const char *fname1, const char *fname2);
 static int test_enums(const char *fname);
+static void test_comps_array (const char *fname, const char *dset, const char *attr,int diff, int is_file_new);
+static void test_comps_vlen (const char *fname, const char *dset,const char *attr, int diff, int is_file_new);
+static void test_comps_array_vlen (const char *fname, const char *dset, const char *attr, int diff, int is_file_new);
+static void test_comps_vlen_arry (const char *fname, const char *dset,const char *attr, int diff, int is_file_new);
 
 /* called by test_attributes() and test_datasets() */
 static void write_attr_in(hid_t loc_id,const char* dset_name,hid_t fid,int make_diffs);
@@ -207,6 +214,21 @@ int main(void)
      */
     test_enums(ENUM_INVALID_VALUES);
 
+    /* -------------------------------------------------
+     * Create test files with dataset and attribute with container types 
+     * (array, vlen) with multiple nested compound types.
+     */
+    /* file1 */
+    test_comps_array(COMPS_COMPLEX1,"dset1", "attr1", 0, 1);
+    test_comps_vlen(COMPS_COMPLEX1,"dset2", "attr2", 0, 0);
+    test_comps_array_vlen(COMPS_COMPLEX1,"dset3", "attr3", 0, 0);
+    test_comps_vlen_arry(COMPS_COMPLEX1,"dset4", "attr4", 0, 0);
+    /* file2 */
+    test_comps_array(COMPS_COMPLEX2,"dset1", "attr1", 5, 1);
+    test_comps_vlen(COMPS_COMPLEX2,"dset2", "attr2",5, 0);
+    test_comps_array_vlen(COMPS_COMPLEX2,"dset3", "attr3", 5, 0);
+    test_comps_vlen_arry(COMPS_COMPLEX2,"dset4", "attr4", 5, 0);
+
     return 0;
 }
 
@@ -940,7 +962,6 @@ int test_attributes(const char *file,
 */
 static int test_attributes_verbose_level(const char *fname1, const char *fname2)
 {
-    int i;
     herr_t  status = SUCCEED;
     hid_t   fid1, fid2;
     hid_t   f1_gid, f2_gid;
@@ -2713,18 +2734,15 @@ static int test_group_recurse2(void)
     hid_t       grp1=0, grp2;
     hid_t       grp3=0;
     hid_t       grp4=0;
-    hid_t       tid;
     hid_t       dset1, dset2;
     hid_t       datatype, dataspace;   /* handles */
     hid_t       fileid2;
     hid_t       fileid3;
     hid_t       fileid4;
-    hid_t       fileid4_1;
     hsize_t     dimsf[2];              /* dataset dimensions */
     herr_t      status=0;
     int data1[4][2] = {{0,0},{1,1},{2,2},{3,3}};
     int data2[4][2] = {{0,0},{0,1},{0,2},{3,3}};
-    int         i, j;
 
    /*-----------------------------------------------------------------------
     * FILE 1
@@ -4110,6 +4128,545 @@ out:
     return status;
 }
 
+/*-------------------------------------------------------------------------
+*
+* Purpose: 
+*   Create test files with dataset and attribute with container types 
+*   (array, vlen) with multiple nested compound types.
+*
+* Function: test_comps_array()
+*  - type: compound->array->compound
+*
+* Function: test_comps_vlen()
+*  - type: compound->vlen->compound
+*
+* Function: test_comps_array_vlen()
+*  - type: compound->array->compound->vlen->compound
+*
+* Function: test_comps_vlen_arry()
+*  - type: compound->vlen->compound->array->compound
+*
+* Programmer: Jonathan Kim (Sep, 1, 2011)
+*
+*-------------------------------------------------------------------------*/
+#define SDIM_DSET 2
+#define SDIM_CMPD_ARRAY 2
+
+static void test_comps_array (const char *fname, const char *dset, const char *attr,int diff, int is_file_new)
+{
+    /* sub compound 2 */
+    typedef struct {        
+        int i2;
+        float f2;
+    } cmpd2_t;
+
+    /* top compound 1 */
+    typedef struct {
+        int i1;
+        cmpd2_t cmpd2[SDIM_CMPD_ARRAY];
+    } cmpd1_t;
+
+    cmpd1_t wdata[SDIM_DSET];   /* dataset with compound1 */
+
+    hid_t  fid;  /* HDF5 File IDs  */
+    hid_t  did_dset; /* Dataset ID   */
+    hid_t  sid_dset;       /* Dataset space ID   */
+    hid_t  tid_cmpd1;       /* Compound1 type ID   */
+    hid_t  tid_arry1;       /* Array type ID in compound1   */
+    hid_t  tid_cmpd2;       /* Compound2 type ID   */
+    hid_t  tid_attr;
+    hsize_t  sdims_dset[] = {SDIM_DSET};
+    hsize_t  sdims_cmpd_arry[] = {SDIM_CMPD_ARRAY};
+    int     i,j;       
+    herr_t  ret;  /* Generic return value  */
+
+    /* Initialize array data to write */
+    for(i=0; i < SDIM_DSET; i++)
+    {
+        wdata[i].i1 = i;
+        for(j=0; j < SDIM_CMPD_ARRAY; j++) 
+        {
+            wdata[i].cmpd2[j].i2 = i*10 + diff;
+            wdata[i].cmpd2[j].f2 = (float)(i*10.5) + diff;
+        } /* end for */
+    }
+
+   /*-----------------------------------------------------------------------
+    * Create file(s)
+    *------------------------------------------------------------------------*/
+    if (is_file_new == 1)
+        fid = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
+    else
+        fid = H5Fopen (fname, H5F_ACC_RDWR, H5P_DEFAULT);
+
+
+    /* -------------------------------
+     * Create a sub compound2 datatype  */
+    tid_cmpd2 = H5Tcreate(H5T_COMPOUND, sizeof(cmpd2_t));
+
+    /* Insert integer field */
+    ret = H5Tinsert(tid_cmpd2, "int2", HOFFSET(cmpd2_t, i2), H5T_NATIVE_INT);
+    assert(ret >= 0);
+
+    /* Insert float field */
+    ret = H5Tinsert(tid_cmpd2, "float2", HOFFSET(cmpd2_t, f2), H5T_NATIVE_FLOAT);
+    assert(ret >= 0);
+
+    /*-----------------------------------
+     * Create a top compound1.
+     */
+    tid_cmpd1 = H5Tcreate (H5T_COMPOUND, sizeof(cmpd1_t));
+
+    ret = H5Tinsert(tid_cmpd1, "int1", HOFFSET(cmpd1_t, i1), H5T_NATIVE_INT);
+    assert(ret >= 0);
+
+    /* Create an array datatype */
+    tid_arry1 = H5Tarray_create2(tid_cmpd2, 1, sdims_cmpd_arry);
+    /* insert the array */
+    ret = H5Tinsert(tid_cmpd1, "array_cmpd1", HOFFSET(cmpd1_t, cmpd2), tid_arry1);
+    assert(ret >= 0);
+
+
+    /* -------------------
+     * Create a dataset 
+     */
+    /* Create dataspace for datasets */
+    sid_dset = H5Screate_simple(1, sdims_dset, NULL);
+
+    did_dset = H5Dcreate2(fid, dset, tid_cmpd1, sid_dset, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+
+    /* Write dataset to disk */
+    ret = H5Dwrite(did_dset, tid_cmpd1, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata);
+    assert(ret >= 0);
+
+    /*-----------------------------------
+     * Create an attribute in root group
+     */
+    tid_attr = H5Acreate2(fid, attr, tid_cmpd1, sid_dset, H5P_DEFAULT, H5P_DEFAULT);
+    ret = H5Awrite(tid_attr, tid_cmpd1, wdata);
+    assert(ret >= 0);
+
+    /* ----------------
+     * Close Dataset */
+    ret = H5Aclose(tid_attr);
+    assert(ret >= 0);
+    ret = H5Tclose(tid_arry1);
+    assert(ret >= 0);
+    ret = H5Dclose(did_dset);
+    assert(ret >= 0);
+    ret = H5Tclose(tid_cmpd1);
+    assert(ret >= 0);
+    ret = H5Tclose(tid_cmpd2);
+    assert(ret >= 0);
+    ret = H5Sclose(sid_dset);
+    assert(ret >= 0);
+    ret = H5Fclose(fid);
+    assert(ret >= 0);
+}
+
+
+
+static void test_comps_vlen (const char * fname, const char *dset, const char *attr, int diff, int is_file_new)
+{
+    /* sub compound 2 */
+    typedef struct {       
+        int i2;
+        float f2;
+    } cmpd2_t;
+
+    /* top compound 1 */
+    typedef struct { 
+        int i1;
+        hvl_t vl;   /* VL information for compound2 */
+    } cmpd1_t;
+
+    cmpd1_t wdata[SDIM_DSET];   /* Dataset for compound1 */ 
+
+    hid_t  fid;  /* HDF5 File ID */
+    hid_t  did_dset; /* dataset ID   */
+    hid_t  sid_dset;  /* dataset space ID */
+    hid_t  tid_cmpd2; /* compound2 type ID */
+    hid_t  tid_cmpd1; /* compound1 type ID */
+    hid_t  tid_cmpd1_vlen;
+    hsize_t  sdims_dset[] = {SDIM_DSET};
+
+    unsigned       i,j;        /* counting variables */
+    herr_t  ret;  /* Generic return value  */
+
+    /* Allocate and initialize VL data to write */
+    for(i=0; i<SDIM_DSET; i++)
+    {
+        wdata[i].i1 = i;
+        wdata[i].vl.p = malloc((i+1)*sizeof(cmpd2_t));
+        wdata[i].vl.len = i+1;
+        for(j=0; j<(i+1); j++)
+        {
+            ((cmpd2_t *)wdata[i].vl.p)[j].i2 = i*10 + diff;
+            ((cmpd2_t *)wdata[i].vl.p)[j].f2 = (float)(i*10.5) + diff;
+        } /* end for */
+    } /* end for */
+
+    /*-----------------------------------------------------------------------
+     * Create file(s)
+     *------------------------------------------------------------------------*/
+    if (is_file_new == 1)
+        fid = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
+    else
+        fid = H5Fopen (fname, H5F_ACC_RDWR, H5P_DEFAULT);
+
+    /* -----------------------------
+     * Create sub compound2 type */
+    tid_cmpd2 = H5Tcreate(H5T_COMPOUND, sizeof(cmpd2_t));
+
+    /* Insert fields */
+    ret = H5Tinsert(tid_cmpd2, "int2", HOFFSET(cmpd2_t, i2), H5T_NATIVE_INT);
+    assert(ret >= 0);
+    ret = H5Tinsert(tid_cmpd2, "float2", HOFFSET(cmpd2_t, f2), H5T_NATIVE_FLOAT);
+    assert(ret >= 0);
+
+    /* ---------------------------
+     * Create top compound1 type */
+    tid_cmpd1 = H5Tcreate(H5T_COMPOUND, sizeof(cmpd1_t));
+    /* Insert fields */
+    ret = H5Tinsert(tid_cmpd1, "int1", HOFFSET(cmpd1_t, i1), H5T_NATIVE_INT);
+    assert(ret >= 0);
+    /* Create a VL datatype */
+    tid_cmpd1_vlen = H5Tvlen_create(tid_cmpd2);
+
+    ret = H5Tinsert(tid_cmpd1, "vlen_cmpd1", HOFFSET(cmpd1_t, vl), tid_cmpd1_vlen);
+    assert(ret >= 0);
+
+    /* -------------------------------
+     * Create dataset with compound1 
+     */
+    /* Create dataspace for dataset */
+    sid_dset = H5Screate_simple(1, sdims_dset, NULL);
+
+    /* Create a dataset */
+    did_dset = H5Dcreate2(fid, dset, tid_cmpd1, sid_dset, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+
+    /* Write dataset to disk */
+    ret = H5Dwrite(did_dset, tid_cmpd1, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata);
+    assert(ret >= 0);
+
+    /* Reclaim the write VL data */
+    ret = H5Dvlen_reclaim(tid_cmpd1, sid_dset, H5P_DEFAULT, wdata);
+    assert(ret >= 0);
+
+    /*-----------------------------------
+     * Create an attribute in root group
+     */
+    /* TODO: creating vlen with compound type doesn't work for attribute now. 
+     * so add this later when it's fixed 
+    tid_attr = H5Acreate2(fid, attr, tid_cmpd1, sid_dset, H5P_DEFAULT, H5P_DEFAULT);
+    ret = H5Awrite(tid_attr, tid_cmpd1, wdata);
+    assert(ret >= 0);
+    */
+
+    /* ----------------
+     * Close IDs */
+    ret = H5Dclose(did_dset);
+    assert(ret >= 0);
+    ret = H5Tclose(tid_cmpd2);
+    assert(ret >= 0);
+    ret = H5Tclose(tid_cmpd1);
+    assert(ret >= 0);
+    ret = H5Sclose(sid_dset);
+    assert(ret >= 0);
+    ret = H5Tclose(tid_cmpd1_vlen);
+    assert(ret >= 0);
+    ret = H5Fclose(fid);
+    assert(ret >= 0);
+}
+
+
+static void test_comps_array_vlen (const char * fname, const char *dset,const char *attr, int diff, int is_file_new)
+{
+    typedef struct {       
+        int i3;
+        float f3;
+    } cmpd3_t;
+ 
+    typedef struct {        /* Typedef for compound datatype */
+        int i2;
+        hvl_t vl;   /* VL information to write */
+    } cmpd2_t;
+
+    typedef struct {
+        int i1;
+        cmpd2_t cmpd2[SDIM_CMPD_ARRAY];
+    } cmpd1_t;
+
+    cmpd1_t wdata[SDIM_DSET];   /* Information to write */
+    hid_t  fid;  /* HDF5 File IDs  */
+    hid_t  did_dset; /* Dataset ID   */
+    hid_t  sid_dset;       /* Dataspace ID   */
+    hid_t  tid_cmpd1;       /* Compound1 Datatype ID   */
+    hid_t  tid_arry1;       /* Array Datatype ID   */
+    hid_t  tid_cmpd2;       /* Compound2 Datatype ID   */
+    hid_t  tid_cmpd2_vlen;
+    hid_t  tid_cmpd3;       /* Compound3 Datatype ID   */
+    hsize_t  sdims_dset[] = {SDIM_DSET};
+    hsize_t  sdims_arry[] = {SDIM_CMPD_ARRAY};
+    int        i,j,k;        /* counting variables */
+    herr_t  ret;  /* Generic return value  */
+
+
+
+    /* Initialize array data to write in compound1 */
+    for(i=0; i < SDIM_DSET; i++)
+    {
+        wdata[i].i1 = i;
+        /* Allocate and initialize VL data to write in compound2 */
+        for(j=0; j < SDIM_CMPD_ARRAY; j++) 
+        {
+            wdata[i].cmpd2[j].i2 = j*10;
+            wdata[i].cmpd2[j].vl.p = malloc((j+1)*sizeof(cmpd3_t));
+            wdata[i].cmpd2[j].vl.len = j+1;
+            for(k=0; k<(j+1); k++) 
+            {
+                /* Initialize data of compound3 */
+                ((cmpd3_t *)wdata[i].cmpd2[j].vl.p)[k].i3 = j*10 + diff;
+                ((cmpd3_t *)wdata[i].cmpd2[j].vl.p)[k].f3 = (float)(j*10.5) + diff;
+            } /* end for */
+        } /* end for */
+    }
+
+    /*-----------------------------------------------------------------------
+     * Create file(s)
+     *------------------------------------------------------------------------*/
+    if (is_file_new == 1)
+        fid = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
+    else
+        fid = H5Fopen (fname, H5F_ACC_RDWR, H5P_DEFAULT);
+
+    /* Create dataspace for datasets */
+    sid_dset = H5Screate_simple(1, sdims_dset, NULL);
+
+    /*-------------------------------------
+     * Create a sub compound3 datatype  */
+    tid_cmpd3 = H5Tcreate(H5T_COMPOUND, sizeof(cmpd3_t));
+
+    /* Insert integer field */
+    ret = H5Tinsert(tid_cmpd3, "int", HOFFSET(cmpd3_t, i3), H5T_NATIVE_INT);
+    assert(ret >= 0);
+
+    /* Insert float field */
+    ret = H5Tinsert(tid_cmpd3, "float", HOFFSET(cmpd3_t, f3), H5T_NATIVE_FLOAT);
+    assert(ret >= 0);
+
+
+    /*-------------------------------------
+     * Create a sub compound2 datatype  */
+    tid_cmpd2 = H5Tcreate(H5T_COMPOUND, sizeof(cmpd2_t));
+
+    /* Insert integer field */
+    ret = H5Tinsert(tid_cmpd2, "int", HOFFSET(cmpd2_t, i2), H5T_NATIVE_INT);
+    assert(ret >= 0);
+    /* Create a VL datatype */
+    tid_cmpd2_vlen = H5Tvlen_create(tid_cmpd3);
+    ret = H5Tinsert(tid_cmpd2, "vlen", HOFFSET(cmpd2_t, vl), tid_cmpd2_vlen);
+    assert(ret >= 0);
+
+
+    /*-----------------------------------
+     * Create a top compound1 datatype for dataset.
+     */
+    tid_cmpd1 = H5Tcreate (H5T_COMPOUND, sizeof(cmpd1_t));
+
+    /* Create an array datatype */
+    tid_arry1 = H5Tarray_create2(tid_cmpd2, 1, sdims_arry);
+    /* insert the array */
+    H5Tinsert(tid_cmpd1, "array_comp", HOFFSET(cmpd1_t, cmpd2), tid_arry1);
+
+
+    /* ----------------------
+     * Create a dataset */
+    did_dset = H5Dcreate2(fid, dset, tid_cmpd1, sid_dset, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+
+    /* Write dataset to disk */
+    ret = H5Dwrite(did_dset, tid_cmpd1, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata);
+    assert(ret >= 0);
+
+    /* Reclaim the write VL data */
+    ret = H5Dvlen_reclaim(tid_cmpd1, sid_dset, H5P_DEFAULT, wdata);
+    assert(ret >= 0);
+
+    /*-----------------------------------
+     * Create an attribute in root group
+     */
+    /* TODO: creating vlen with compound type doesn't work for attribute now. 
+     * so add this later when it's fixed
+    tid_attr = H5Acreate2(fid, attr, tid_cmpd1, sid_dset, H5P_DEFAULT, H5P_DEFAULT);
+    ret = H5Awrite(tid_attr, tid_cmpd1, wdata);
+    assert(ret >= 0);
+    */
+
+    /*-------------------
+     * Close IDs */
+    ret = H5Tclose(tid_arry1);
+    assert(ret >= 0);
+    ret = H5Dclose(did_dset);
+    assert(ret >= 0);
+    ret = H5Sclose(sid_dset);
+    assert(ret >= 0);
+    ret = H5Tclose(tid_cmpd3);
+    assert(ret >= 0);
+    ret = H5Tclose(tid_cmpd2);
+    assert(ret >= 0);
+    ret = H5Tclose(tid_cmpd2_vlen);
+    assert(ret >= 0);
+    ret = H5Tclose(tid_cmpd1);
+    assert(ret >= 0);
+    ret = H5Fclose(fid);
+    assert(ret >= 0);
+}
+
+
+static void test_comps_vlen_arry (const char * fname, const char *dset, const char *attr, int diff, int is_file_new)
+{
+    /* sub compound 3 */
+    typedef struct {       
+        int i3;
+        float f3;
+    } cmpd3_t;
+
+    /* sub compound 2 */
+    typedef struct {       
+        int i2;
+        cmpd3_t cmpd3[SDIM_CMPD_ARRAY];
+    } cmpd2_t;
+
+    /* top compound 1 */
+    typedef struct { 
+        int i1;
+        hvl_t vl;   /* VL information for compound2 */
+    } cmpd1_t;
+
+    cmpd1_t wdata[SDIM_DSET];   /* Dataset for compound1 */
+
+    hid_t  fid;  /* HDF5 File ID */
+    hid_t  did_dset; /* dataset ID   */
+    hid_t  sid_dset;  /* dataset space ID */
+    hid_t  tid_cmpd3; /* compound3 type ID */
+    hid_t  tid_cmpd2; /* compound2 type ID */
+    hid_t  tid_cmpd2_arry;
+    hid_t  tid_cmpd1; /* compound1 type ID */
+    hid_t  tid_cmpd1_vlen;
+    hsize_t  sdims_dset[] = {SDIM_DSET};
+    hsize_t  sdims_cmpd_arry[] = {SDIM_CMPD_ARRAY};
+
+    unsigned       i,j,k;        /* counting variables */
+    herr_t  ret;  /* Generic return value  */
+
+    /* Allocate and initialize VL data to write */
+    for(i=0; i<SDIM_DSET; i++)
+    {
+        /* compound 1 data */
+        wdata[i].i1 = i;
+        wdata[i].vl.p = malloc((i+1)*sizeof(cmpd2_t));
+        wdata[i].vl.len = i+1;
+        for(j=0; j<(i+1); j++)
+        {
+            /* compound2 data */
+            ((cmpd2_t *)wdata[i].vl.p)[j].i2 = i*10 + diff;
+            for (k=0; k < SDIM_CMPD_ARRAY; k++)
+            {
+                /* compound 3 data */
+                ((cmpd2_t *)(wdata[i].vl.p))[j].cmpd3[k].i3 = k*10.5 + diff;
+                ((cmpd2_t *)(wdata[i].vl.p))[j].cmpd3[k].f3 = (float)(k*10.5) + diff;
+            }
+        } /* end for */
+    } /* end for */
+
+    /*-----------------------------------------------------------------------
+     * Create file(s)
+     *------------------------------------------------------------------------*/
+    if (is_file_new == 1)
+        fid = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
+    else
+        fid = H5Fopen (fname, H5F_ACC_RDWR, H5P_DEFAULT);
+
+    /* -----------------------------
+     * Create sub compound3 type */
+    tid_cmpd3 = H5Tcreate(H5T_COMPOUND, sizeof(cmpd3_t));
+
+    /* Insert fields */
+    ret = H5Tinsert(tid_cmpd3, "int3", HOFFSET(cmpd3_t, i3), H5T_NATIVE_INT);
+    assert(ret >= 0);
+    ret = H5Tinsert(tid_cmpd3, "float3", HOFFSET(cmpd3_t, f3), H5T_NATIVE_FLOAT);
+    assert(ret >= 0);
+
+    /* -----------------------------
+     * Create sub compound2 type */
+    tid_cmpd2 = H5Tcreate(H5T_COMPOUND, sizeof(cmpd2_t));
+
+    ret = H5Tinsert(tid_cmpd2, "int2", HOFFSET(cmpd2_t, i2), H5T_NATIVE_INT);
+    assert(ret >= 0);
+
+    tid_cmpd2_arry = H5Tarray_create2(tid_cmpd3, 1, sdims_cmpd_arry);
+    ret = H5Tinsert(tid_cmpd2, "array_cmpd2", HOFFSET(cmpd2_t, cmpd3), tid_cmpd2_arry);
+    assert(ret >= 0);
+
+    /* ---------------------------
+     * Create top compound1 type
+     */
+    /* Create a VL datatype */
+    tid_cmpd1 = H5Tcreate(H5T_COMPOUND, sizeof(cmpd1_t));
+    /* Insert fields */
+    ret = H5Tinsert(tid_cmpd1, "int1", HOFFSET(cmpd1_t, i1), H5T_NATIVE_INT);
+    assert(ret >= 0);
+    tid_cmpd1_vlen = H5Tvlen_create(tid_cmpd2);
+    ret = H5Tinsert(tid_cmpd1, "vlen_cmpd1", HOFFSET(cmpd1_t, vl), tid_cmpd1_vlen);
+    assert(ret >= 0);
+
+    /* -------------------------------
+     * Create dataset with compound1 
+     */
+    /* Create dataspace for dataset */
+    sid_dset = H5Screate_simple(1, sdims_dset, NULL);
+
+    /* Create a dataset */
+    did_dset = H5Dcreate2(fid, dset, tid_cmpd1, sid_dset, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+
+    /* Write dataset to disk */
+    ret = H5Dwrite(did_dset, tid_cmpd1, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata);
+    assert(ret >= 0);
+
+    /* Reclaim the write VL data */
+    ret = H5Dvlen_reclaim(tid_cmpd1, sid_dset, H5P_DEFAULT, wdata);
+    assert(ret >= 0);
+
+    /*-----------------------------------
+     * Create an attribute in root group
+     */
+    /* TODO: creating vlen with compound type doesn't work for attribute now.
+     * so add this later when it's fixed
+    tid_attr = H5Acreate2(fid, attr, tid_cmpd1, sid_dset, H5P_DEFAULT, H5P_DEFAULT);
+    ret = H5Awrite(tid_attr, tid_cmpd1, wdata);
+    assert(ret >= 0);
+    */
+
+    /* ----------------
+     * Close IDs */
+    ret = H5Dclose(did_dset);
+    assert(ret >= 0);
+    ret = H5Sclose(sid_dset);
+    assert(ret >= 0);
+    ret = H5Tclose(tid_cmpd3);
+    assert(ret >= 0);
+    ret = H5Tclose(tid_cmpd2_arry);
+    assert(ret >= 0);
+    ret = H5Tclose(tid_cmpd2);
+    assert(ret >= 0);
+    ret = H5Tclose(tid_cmpd1_vlen);
+    assert(ret >= 0);
+    ret = H5Tclose(tid_cmpd1);
+    assert(ret >= 0);
+    ret = H5Fclose(fid);
+    assert(ret >= 0);
+}
+
 
 /*-------------------------------------------------------------------------
 * Function: write_attr_in
diff --git a/tools/h5diff/testfiles/compounds_array_vlen1.h5 b/tools/h5diff/testfiles/compounds_array_vlen1.h5
new file mode 100644
index 0000000..398026c
--- /dev/null
+++ b/tools/h5diff/testfiles/compounds_array_vlen1.h5
diff --git a/tools/h5diff/testfiles/compounds_array_vlen2.h5 b/tools/h5diff/testfiles/compounds_array_vlen2.h5
new file mode 100644
index 0000000..f6f0868
--- /dev/null
+++ b/tools/h5diff/testfiles/compounds_array_vlen2.h5
diff --git a/tools/h5diff/testfiles/h5diff_540.txt b/tools/h5diff/testfiles/h5diff_540.txt
new file mode 100644
index 0000000..a6903c8
--- /dev/null
+++ b/tools/h5diff/testfiles/h5diff_540.txt
@@ -0,0 +1,86 @@
+
+file1     file2
+---------------------------------------
+    x      x    /              
+    x      x    /dset1         
+    x      x    /dset2         
+    x      x    /dset3         
+    x      x    /dset4         
+
+group  : </> and </>
+0 differences found
+attribute: <attr1 of </>> and <attr1 of </>>
+size:           [2]           [2]
+position        attr1 of </>    attr1 of </>    difference          
+------------------------------------------------------------
+[ 0 ]          0               5               5              
+[ 0 ]          0               5               5              
+[ 0 ]          0               5               5              
+[ 0 ]          0               5               5              
+[ 1 ]          10              15              5              
+[ 1 ]          10.5            15.5            5              
+[ 1 ]          10              15              5              
+[ 1 ]          10.5            15.5            5              
+8 differences found
+dataset: </dset1> and </dset1>
+size:           [2]           [2]
+position        dset1           dset1           difference          
+------------------------------------------------------------
+[ 0 ]          0               5               5              
+[ 0 ]          0               5               5              
+[ 0 ]          0               5               5              
+[ 0 ]          0               5               5              
+[ 1 ]          10              15              5              
+[ 1 ]          10.5            15.5            5              
+[ 1 ]          10              15              5              
+[ 1 ]          10.5            15.5            5              
+8 differences found
+dataset: </dset2> and </dset2>
+size:           [2]           [2]
+position        dset2           dset2           difference          
+------------------------------------------------------------
+[ 0 ]          0               5               5              
+[ 0 ]          0               5               5              
+[ 1 ]          10              15              5              
+[ 1 ]          10.5            15.5            5              
+[ 1 ]          10              15              5              
+[ 1 ]          10.5            15.5            5              
+6 differences found
+dataset: </dset3> and </dset3>
+size:           [2]           [2]
+position        dset3           dset3           difference          
+------------------------------------------------------------
+[ 0 ]          0               5               5              
+[ 0 ]          0               5               5              
+[ 0 ]          10              15              5              
+[ 0 ]          10.5            15.5            5              
+[ 0 ]          10              15              5              
+[ 0 ]          10.5            15.5            5              
+[ 1 ]          0               5               5              
+[ 1 ]          0               5               5              
+[ 1 ]          10              15              5              
+[ 1 ]          10.5            15.5            5              
+[ 1 ]          10              15              5              
+[ 1 ]          10.5            15.5            5              
+12 differences found
+dataset: </dset4> and </dset4>
+size:           [2]           [2]
+position        dset4           dset4           difference          
+------------------------------------------------------------
+[ 0 ]          0               5               5              
+[ 0 ]          0               5               5              
+[ 0 ]          0               5               5              
+[ 0 ]          10              15              5              
+[ 0 ]          10.5            15.5            5              
+[ 1 ]          10              15              5              
+[ 1 ]          0               5               5              
+[ 1 ]          0               5               5              
+[ 1 ]          10              15              5              
+[ 1 ]          10.5            15.5            5              
+[ 1 ]          10              15              5              
+[ 1 ]          0               5               5              
+[ 1 ]          0               5               5              
+[ 1 ]          10              15              5              
+[ 1 ]          10.5            15.5            5              
+15 differences found
+EXIT CODE: 1
diff --git a/tools/h5diff/testh5diff.sh b/tools/h5diff/testh5diff.sh
index 8d38171..e318a15 100755
--- a/tools/h5diff/testh5diff.sh
+++ b/tools/h5diff/testh5diff.sh
@@ -112,6 +112,8 @@ $SRC_H5DIFF_TESTFILES/h5diff_exclude1-2.h5
 $SRC_H5DIFF_TESTFILES/h5diff_exclude2-1.h5
 $SRC_H5DIFF_TESTFILES/h5diff_exclude2-2.h5
 $SRC_H5DIFF_TESTFILES/h5diff_comp_vl_strs.h5
+$SRC_H5DIFF_TESTFILES/compounds_array_vlen1.h5
+$SRC_H5DIFF_TESTFILES/compounds_array_vlen2.h5
 $SRC_H5DIFF_TESTFILES/h5diff_attr_v_level1.h5
 $SRC_H5DIFF_TESTFILES/h5diff_attr_v_level2.h5
 $SRC_H5DIFF_TESTFILES/h5diff_enum_invalid_values.h5
@@ -227,6 +229,7 @@ $SRC_H5DIFF_TESTFILES/h5diff_516.txt
 $SRC_H5DIFF_TESTFILES/h5diff_517.txt
 $SRC_H5DIFF_TESTFILES/h5diff_518.txt
 $SRC_H5DIFF_TESTFILES/h5diff_530.txt
+$SRC_H5DIFF_TESTFILES/h5diff_540.txt
 $SRC_H5DIFF_TESTFILES/h5diff_600.txt
 $SRC_H5DIFF_TESTFILES/h5diff_601.txt
 $SRC_H5DIFF_TESTFILES/h5diff_603.txt
@@ -989,9 +992,14 @@ TOOLTEST h5diff_484.txt -v --exclude-path "/dset3" h5diff_exclude1-1.h5 h5diff_e
 TOOLTEST h5diff_530.txt -v  h5diff_comp_vl_strs.h5 h5diff_comp_vl_strs.h5 /group /group_copy
 
 # ##############################################################################
+# # Test container types (array,vlen) with multiple nested compound types
+# # Complex compound types in dataset and attribute
+# ##############################################################################
+TOOLTEST h5diff_540.txt -v compounds_array_vlen1.h5 compounds_array_vlen2.h5
+
+# ##############################################################################
 # # Test mutually exclusive options 
 # ##############################################################################
-#
 # Test with -d , -p and --use-system-epsilon. 
 TOOLTEST h5diff_640.txt -v -d 5 -p 0.05 --use-system-epsilon h5diff_basic1.h5 h5diff_basic2.h5 /g1/dset3 /g1/dset4
 TOOLTEST h5diff_641.txt -v -d 5 -p 0.05 h5diff_basic1.h5 h5diff_basic2.h5 /g1/dset3 /g1/dset4
diff --git a/tools/lib/h5diff_array.c b/tools/lib/h5diff_array.c
index 3e3db02..91b94b8 100644
--- a/tools/lib/h5diff_array.c
+++ b/tools/lib/h5diff_array.c
@@ -186,8 +186,8 @@ static int my_isnan(dtype_t type, void *val);
  * XCAO, 11/10/2010
  * added to improve performance for compound datasets
  */
-static void set_comp_members(hid_t tid, mcomp_t *members);
-static void free_comp_members(mcomp_t *members);
+static void get_member_types(hid_t tid, mcomp_t *members);
+static void close_member_types(mcomp_t *members);
 
 
 
@@ -318,7 +318,7 @@ hsize_t diff_array( void *_mem1,
     case H5T_VLEN:
     case H5T_REFERENCE:
         HDmemset(&members, 0, sizeof (mcomp_t));
-        set_comp_members(m_type, &members);
+        get_member_types(m_type, &members);
         for ( i = 0; i < nelmts; i++)
         {
             nfound+=diff_datum(
@@ -338,11 +338,11 @@ hsize_t diff_array( void *_mem1,
                 &ph, &members);
             if (options->n && nfound>=options->count)
             {
-                free_comp_members(&members);
+                close_member_types(&members);
                 return nfound;
             }
         } /* i */
-        free_comp_members(&members);
+        close_member_types(&members);
     } /* switch */
 
     return nfound;
@@ -726,7 +726,7 @@ hsize_t diff_datum(void       *_mem1,
                    obj2,
                    container1_id,
                    container2_id,
-                   ph, NULL);
+                   ph, members);
            }
             H5Tclose(memb_type);
         }
@@ -873,7 +873,7 @@ hsize_t diff_datum(void       *_mem1,
             obj2,
             container1_id,
             container2_id,
-            ph, NULL);
+            ph, members);
 
         H5Tclose(memb_type);
 
@@ -6117,36 +6117,46 @@ static void h5diff_print_char(char ch)
  * added to improve performance for compound datasets
  * set up compound datatype structures.
  */
-static void set_comp_members(hid_t tid, mcomp_t *members)
+static void get_member_types(hid_t tid, mcomp_t *members)
 {
     int i;
+    int tclass;
 
     if (tid <=0 || !members)
         return;
 
-    if (H5Tget_class(tid) != H5T_COMPOUND)
-        return;
-
-    members->n = H5Tget_nmembers( tid );
-    if (members->n <=0)
-        return;
+    tclass = H5Tget_class(tid);
+    if (tclass == H5T_ARRAY || tclass == H5T_VLEN)
+    {
+        hid_t base_tid = H5Tget_super(tid);
+        get_member_types(base_tid, members);
+        H5Tclose(base_tid);
+    } 
+    else if (tclass == H5T_COMPOUND) 
+    {
+        members->n = H5Tget_nmembers( tid );
+        if (members->n <=0)
+            return;
 
-    members->ids = HDcalloc(members->n, sizeof(hid_t));
-    members->flags = HDcalloc(members->n, sizeof(unsigned char));
-    members->offsets = HDcalloc(members->n, sizeof(size_t));
-    members->m = HDcalloc(members->n, sizeof(mcomp_t *));
+        members->ids = HDcalloc(members->n, sizeof(hid_t));
+        members->flags = HDcalloc(members->n, sizeof(unsigned char));
+        members->offsets = HDcalloc(members->n, sizeof(size_t));
+        members->m = HDcalloc(members->n, sizeof(mcomp_t *));
 
-    for (i=0; i< members->n; i++)
-    {
-         members->ids[i] = H5Tget_member_type( tid, i );
-         members->flags[i] = H5Tis_variable_str( members->ids[i] );
-         members->offsets[i] = H5Tget_member_offset( tid, i );
-         if (H5Tget_class( members->ids[i])==H5T_COMPOUND)
-         {
-              members->m[i] = (mcomp_t *)HDmalloc(sizeof(mcomp_t));
-              set_comp_members(members->ids[i], members->m[i]);
-         }
+        for (i=0; i< members->n; i++)
+        {
+             members->ids[i] = H5Tget_member_type( tid, i );
+             members->flags[i] = H5Tis_variable_str( members->ids[i] );
+             members->offsets[i] = H5Tget_member_offset( tid, i );
+             members->m[i] = (mcomp_t *)HDmalloc(sizeof(mcomp_t));
+             HDmemset(members->m[i], 0, sizeof(mcomp_t));
+             get_member_types(members->ids[i], members->m[i]);
+        }  
     }
+     
+   return;
+
+
 }
 
 /*-------------------------------------------------------------------------
@@ -6154,7 +6164,7 @@ static void set_comp_members(hid_t tid, mcomp_t *members)
  * added to improve performance for compound datasets
  * clean and close compound members.
  */
-static void free_comp_members(mcomp_t *members)
+static void close_member_types(mcomp_t *members)
 {
     int i;
 
@@ -6165,7 +6175,7 @@ static void free_comp_members(mcomp_t *members)
     {
         if (members->m[i])
         {
-            free_comp_members(members->m[i]);
+            close_member_types(members->m[i]);
             HDfree(members->m[i]);
         }
         H5Tclose(members->ids[i]);
diff --git a/windows/tools/h5diff/testh5diff.bat b/windows/tools/h5diff/testh5diff.bat
index 0235c44..491332f 100644
--- a/windows/tools/h5diff/testh5diff.bat
+++ b/windows/tools/h5diff/testh5diff.bat
@@ -59,6 +59,8 @@ set srcexclude1_2=h5diff_exclude1-2.h5
 set srcexclude2_1=h5diff_exclude2-1.h5

 set srcexclude2_2=h5diff_exclude2-2.h5

 set src_comp_vl_strs=h5diff_comp_vl_strs.h5

+set src_COMPS_ARRAY_VLEN1=compounds_array_vlen1.h5

+set src_COMPS_ARRAY_VLEN2=compounds_array_vlen2.h5

 set src_ATTR_VERBOSE_LEVEL_FILE1=h5diff_attr_v_level1.h5

 set src_ATTR_VERBOSE_LEVEL_FILE2=h5diff_attr_v_level2.h5

 

@@ -93,6 +95,8 @@ set exclude1_2=%indir%\h5diff_exclude1-2.h5
 set exclude2_1=%indir%\h5diff_exclude2-1.h5

 set exclude2_2=%indir%\h5diff_exclude2-2.h5

 set comp_vl_strs=%indir%\h5diff_comp_vl_strs.h5

+set COMPS_ARRAY_VLEN1=%indir%\compounds_array_vlen1.h5

+set COMPS_ARRAY_VLEN2=%indir%\compounds_array_vlen2.h5

 set ATTR_VERBOSE_LEVEL_FILE1=%indir%\h5diff_attr_v_level1.h5

 set ATTR_VERBOSE_LEVEL_FILE2=%indir%\h5diff_attr_v_level2.h5

 

@@ -929,6 +933,13 @@ rem ############################################################################
     call :testing %h5diff% -v %src_comp_vl_strs% %src_comp_vl_strs% /group /group_copy

     call :tooltest h5diff_530.txt -v  %comp_vl_strs% %comp_vl_strs% /group /group_copy

 

+    rem # #####################################################################

+    rem # # Test container types (array,vlen) with multiple nested compound types

+    rem # # Complex compound types in dataset and attribute

+    rem # #####################################################################

+    call :testing %h5diff% -v %src_COMPS_ARRAY_VLEN1% %src_COMPS_ARRAY_VLEN2%

+    call :tooltest h5diff_540.txt -v %COMPS_ARRAY_VLEN1% %COMPS_ARRAY_VLEN2%

+

     rem #######################################################################

     rem # Test mutually exclusive options 

     rem #######################################################################