[svn-r5812]

Purpose:
    Bug fix.
Description:
    This should fix the nested VL datatype bug.
Platforms tested:
    modi4, eirene, impact

3 files changed, 234 insertions, 21 deletions

diff --git a/src/H5Tconv.c b/src/H5Tconv.c
index 65b7c93..e61e2cb 100644
--- a/src/H5Tconv.c
+++ b/src/H5Tconv.c
@@ -2178,11 +2178,10 @@ H5T_conv_vlen(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, hsize_t nelmts,
     uint8_t	*s, *sp, *d, *dp;	/*source and dest traversal ptrs     */
     uint8_t 	**dptr;		        /*pointer to correct destination pointer*/
     uint8_t	*bg_ptr=NULL;		/*background buf traversal pointer   */
-    uint8_t	*bg=NULL;		
-    H5HG_t	bg_hobjid;
+    H5HG_t	bg_hobjid, parent_hobjid;
     size_t	src_delta, dst_delta, bkg_delta;/*source & destination stride*/
     hssize_t 	seq_len;                /*the number of elements in the current sequence*/
-    hsize_t	bg_seq_len=0;
+    hsize_t	bg_seq_len=0, parent_seq_len=0;
     size_t	src_base_size, dst_base_size;/*source & destination base size*/
     size_t	src_size, dst_size;     /*source & destination total size in bytes*/
     void	*conv_buf=NULL;     	/*temporary conversion buffer 	     */
@@ -2191,7 +2190,9 @@ H5T_conv_vlen(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, hsize_t nelmts,
     size_t	tmp_buf_size=0;	        /*size of temporary bkg buffer	     */
     uint8_t	dbuf[64],*dbuf_ptr=dbuf;/*temp destination buffer	     */
     int	        direction;		/*direction of traversal	     */
+    int         nested=0;               /*flag of nested VL case             */
     hsize_t	elmtno;			/*element number counter	     */
+    hsize_t     i;
 
     FUNC_ENTER_NOAPI(H5T_conv_vlen, FAIL);
 
@@ -2241,16 +2242,14 @@ H5T_conv_vlen(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, hsize_t nelmts,
             if (src->size==dst->size || buf_stride>0) {
                 olap = nelmts;
                 sp = dp = (uint8_t*)_buf;
-                if(_bkg!=NULL)
-                    bg_ptr  = (uint8_t*)_bkg;
+                bg_ptr  = (uint8_t*)_bkg;
                 direction = 1;
             } else if (src->size>=dst->size) {
                 /* potentially this uses the destination buffer 1 extra
                  * time, but its faster that floating-point calcs */
                 olap = ((dst->size)/(src->size-dst->size))+1;
                 sp = dp = (uint8_t*)_buf;
-                if(_bkg!=NULL)
-                    bg_ptr  = (uint8_t*)_bkg;
+                bg_ptr  = (uint8_t*)_bkg;
                 direction = 1;
             } else {
                 /* potentially this uses the destination buffer 1 extra
@@ -2309,14 +2308,16 @@ H5T_conv_vlen(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, hsize_t nelmts,
                 if ((tmp_buf=H5FL_BLK_ALLOC(vlen_seq,tmp_buf_size,1))==NULL)
                     HRETURN_ERROR (H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for type conversion");
             } /* end if */
+    
+            /* Set the flag for nested VL case */
+            if(dst->u.vlen.f!=NULL && H5T_detect_class(dst->parent,H5T_VLEN) && bg_ptr!=NULL)
+                nested=1;
 
             for (elmtno=0; elmtno<nelmts; elmtno++) {
                 s = sp;
                 d = *dptr;
-                if(bg_ptr!=NULL)
-                    bg = bg_ptr;
 
-                /* Get length of sequences in bytes */
+                /* Get length of element sequences */
                 seq_len=(*(src->u.vlen.getlen))(src->u.vlen.f,s);
                 assert(seq_len>=0);
                 H5_CHECK_OVERFLOW(seq_len,hssize_t,size_t);
@@ -2345,15 +2346,23 @@ H5T_conv_vlen(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, hsize_t nelmts,
                     tmp_buf_size=conv_buf_size;
                     if((tmp_buf=H5FL_BLK_REALLOC(vlen_seq,tmp_buf,tmp_buf_size))==NULL)
                         HRETURN_ERROR (H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for type conversion");
-		    HDmemset(tmp_buf,0,tmp_buf_size);	
                 } /* end if */
 
 		/* If we are writing and there is a nested VL type, read 
 		 * the sequence into the background buffer */
-		if(dst->u.vlen.f!=NULL && H5T_detect_class(dst->parent,H5T_VLEN) 		    && bg!=NULL) {
-                    uint8_t *tmp=bg;
+		if(nested) {
+                    uint8_t *tmp=bg_ptr;
 		    UINT32DECODE(tmp, bg_seq_len);
 		    if(bg_seq_len>0) {
+			if(tmp_buf_size<bg_seq_len*MAX(src_base_size,
+			    dst_base_size)) {
+			    tmp_buf_size=bg_seq_len*MAX(src_base_size,
+			    		 	dst_base_size);
+                    	    if((tmp_buf=H5FL_BLK_REALLOC(vlen_seq,tmp_buf,
+				tmp_buf_size))==NULL)
+                        	HRETURN_ERROR (H5E_RESOURCE, H5E_NOSPACE, FAIL, 
+				"memory allocation failed for type conversion");
+                        }
 			H5F_addr_decode(dst->u.vlen.f, (const uint8_t **)&tmp,
 					&(bg_hobjid.addr));
 		    	INT32DECODE(tmp, bg_hobjid.idx);
@@ -2361,6 +2370,10 @@ H5T_conv_vlen(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, hsize_t nelmts,
 			    HRETURN_ERROR (H5E_DATATYPE, H5E_READERROR, FAIL, 
 			    "can't read VL sequence into background buffer");
 		    } /* end if */
+
+                    /* If the sequence gets shorter, pad out the original sequence with zeros */
+                    if(bg_seq_len<seq_len)
+                        HDmemset((uint8_t *)tmp_buf+dst_base_size*bg_seq_len,0,(seq_len-bg_seq_len)*dst_base_size);
 		} /* end if */
 
                 /* Convert VL sequence */
@@ -2370,10 +2383,25 @@ H5T_conv_vlen(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, hsize_t nelmts,
                                   "datatype conversion failed");
 
                 /* Write sequence to destination location */
-                if((*(dst->u.vlen.write))(dset_xfer_plist,dst->u.vlen.f,d,conv_buf, bg, (hsize_t)seq_len,(hsize_t)dst_base_size)<0)
+                if((*(dst->u.vlen.write))(dset_xfer_plist,dst->u.vlen.f,d,conv_buf, bg_ptr, (hsize_t)seq_len,(hsize_t)dst_base_size)<0)
                     HRETURN_ERROR(H5E_DATATYPE, H5E_WRITEERROR, FAIL,
                                   "can't write VL data");
 
+                /* For nested VL case, free leftover heap objects from the deeper level if the length of new data elements is shorted than the old data elements.*/
+                if(nested && seq_len<bg_seq_len) {
+                    uint8_t *tmp_p=tmp_buf;
+		    tmp_p += seq_len*dst_base_size;
+                    for(i=0; i<(bg_seq_len-seq_len); i++) {
+                        UINT32DECODE(tmp_p, parent_seq_len);
+                        if(parent_seq_len>0) {
+                            H5F_addr_decode(dst->u.vlen.f, (const uint8_t **)&tmp_p, &(parent_hobjid.addr));
+                            INT32DECODE(tmp_p, parent_hobjid.idx);
+                            if(H5HG_remove(dst->u.vlen.f, &parent_hobjid)<0)
+                                HRETURN_ERROR(H5E_DATATYPE, H5E_WRITEERROR, FAIL, "Unable to remove heap object");
+                        }
+                    }
+                }
+                
                 /*
                  * If we had used a temporary buffer for the destination
                  * then we should copy the value to the true destination
diff --git a/src/H5Tvlen.c b/src/H5Tvlen.c
index cc38b3c..78e5b68 100644
--- a/src/H5Tvlen.c
+++ b/src/H5Tvlen.c
@@ -489,10 +489,11 @@ herr_t H5T_vlen_disk_write(hid_t UNUSED plist_id, H5F_t *f, void *vl_addr, void
     /* Get the length of the sequence and heap object ID from background data.
      * Free heap object for old data.  */
     if(bg!=NULL) {
+	HDmemset(&bg_hobjid,0,sizeof(H5HG_t));
         UINT32DECODE(bg, bg_seq_len);
 
         /* Free heap object for old data */
-        if(bg_seq_len!=0) {
+        if(bg_seq_len>0) {
             /* Get heap information */
             H5F_addr_decode(f, (const uint8_t **)&bg, &(bg_hobjid.addr));
             INT32DECODE(bg, bg_hobjid.idx);
diff --git a/test/tvltypes.c b/test/tvltypes.c
index 3aa7d65..1c51789 100644
--- a/test/tvltypes.c
+++ b/test/tvltypes.c
@@ -1305,12 +1305,12 @@ test_vltypes_vlen_vlen_atomic(void)
 
 /****************************************************************
 **
-**  rewrite_vltypes_vlen_vlen_atomic(): Test basic VL datatype code.
+**  rewrite_longer_vltypes_vlen_vlen_atomic(): Test basic VL datatype code.
 **      Tests VL datatype with VL datatypes of atomic datatypes.
 **
 ****************************************************************/
 static void
-rewrite_vltypes_vlen_vlen_atomic(void)
+rewrite_longer_vltypes_vlen_vlen_atomic(void)
 {
     hvl_t wdata[SPACE1_DIM1];   /* Information to write */
     hvl_t rdata[SPACE1_DIM1];   /* Information read in */
@@ -1444,7 +1444,7 @@ rewrite_vltypes_vlen_vlen_atomic(void)
             } /* end if */
             for(k=0; k<t2->len; k++) {
                 if( ((unsigned int *)t1->p)[k] != ((unsigned int *)t2->p)[k] ) {                    num_errs++;
-                    printf("VL data values don't match!, t1->p[%d]=%d, t2->p[%d]=%d, i=%d, j=%d\n",(int)k, (int)((unsigned int *)t1->p)[k], (int)k, (int)((unsigned int *)t2->p)[k], i, j);
+                    printf("VL data values don't match!, t1->p[%d]=%d, t2->p[%d]=%d\n",(int)k, (int)((unsigned int *)t1->p)[k], (int)k, (int)((unsigned int *)t2->p)[k]);
                     continue;
                 } /* end if */
             } /* end for */
@@ -1482,7 +1482,190 @@ rewrite_vltypes_vlen_vlen_atomic(void)
     ret = H5Fclose(fid1);
     CHECK(ret, FAIL, "H5Fclose");
 
-} /* end rewrite_vltypes_vlen_vlen_atomic() */
+} /* end rewrite_longer_vltypes_vlen_vlen_atomic() */
+
+/****************************************************************
+**
+**  rewrite_shorter_vltypes_vlen_vlen_atomic(): Test basic VL datatype code.
+**      Tests VL datatype with VL datatypes of atomic datatypes.
+**
+****************************************************************/
+static void
+rewrite_shorter_vltypes_vlen_vlen_atomic(void)
+{
+    hvl_t wdata[SPACE1_DIM1];   /* Information to write */
+    hvl_t rdata[SPACE1_DIM1];   /* Information read in */
+    hvl_t *t1, *t2;             /* Temporary pointer to VL information */
+    hid_t               fid1;           /* HDF5 File IDs                */
+    hid_t               dataset;        /* Dataset ID                   */
+    hid_t               sid1;       /* Dataspace ID                     */
+    hid_t               tid2;   /* Datatype IDs         */
+    hid_t       xfer_pid;   /* Dataset transfer property list ID */
+    hsize_t     size;       /* Number of bytes which will be used */
+    unsigned       i,j,k;      /* counting variables */
+    size_t         mem_used=0; /* Memory used during allocation */
+    int                 increment=1;
+    herr_t              ret;            /* Generic return value         */
+
+    /* Output message about test being performed */
+    MESSAGE(5, ("Check memory leak for VL Datatypes with VL Atomic Datatype Component Functionality\n"));
+
+    /* Allocate and initialize VL data to write */
+    for(i=0; i<SPACE1_DIM1; i++) {
+        wdata[i].p=malloc((i+increment)*sizeof(hvl_t));
+        if(wdata[i].p==NULL) {
+            printf("Cannot allocate memory for VL data! i=%u\n",i);
+            num_errs++;
+            return;
+        } /* end if */
+        wdata[i].len=i+increment;
+        for(t1=wdata[i].p,j=0; j<(i+increment); j++, t1++) {
+            t1->p=malloc((j+1)*sizeof(unsigned int));
+            if(t1->p==NULL) {
+                printf("Cannot allocate memory for VL data! i=%u, j=%u\n",i,j);
+                num_errs++;
+                return;
+            } /* end if */
+            t1->len=j+1;
+            for(k=0; k<(j+1); k++)
+                ((unsigned int *)t1->p)[k]=i*100000+j*1000+k*10;
+        } /* end for */
+    } /* end for */
+
+    /* Open file */
+    fid1 = H5Fopen(FILENAME, H5F_ACC_RDWR, H5P_DEFAULT);
+    CHECK(fid1, FAIL, "H5Fopen");
+
+    /* Open the dataset */
+    dataset=H5Dopen(fid1,"Dataset1");
+    CHECK(dataset, FAIL, "H5Dopen");
+
+    /* Get dataspace for datasets */
+    sid1 = H5Dget_space(dataset);
+    CHECK(sid1, FAIL, "H5Dget_space");
+
+    /* Open datatype of the dataset */
+    tid2 = H5Dget_type(dataset);
+    CHECK(tid2, FAIL, "H5Dget_type");
+
+    /* Write dataset to disk */
+    ret=H5Dwrite(dataset,tid2,H5S_ALL,H5S_ALL,H5P_DEFAULT,wdata);
+    CHECK(ret, FAIL, "H5Dwrite");
+
+    /* Close Dataset */
+    ret = H5Dclose(dataset);
+    CHECK(ret, FAIL, "H5Dclose");
+
+    /* Close datatype */
+    ret = H5Tclose(tid2);
+    CHECK(ret, FAIL, "H5Tclose");
+
+    /* Close disk dataspace */
+    ret = H5Sclose(sid1);
+    CHECK(ret, FAIL, "H5Sclose");
+
+    /* Close file */
+    ret = H5Fclose(fid1);
+    CHECK(ret, FAIL, "H5Fclose");
+
+
+    /* Open the file for data checking */
+    fid1 = H5Fopen(FILENAME, H5F_ACC_RDONLY, H5P_DEFAULT);
+    CHECK(fid1, FAIL, "H5Fopen");
+
+    /* Open a dataset */
+    dataset=H5Dopen(fid1,"Dataset1");
+    CHECK(dataset, FAIL, "H5Dopen");
+
+    /* Get dataspace for datasets */
+    sid1 = H5Dget_space(dataset);
+    CHECK(sid1, FAIL, "H5Dget_space");
+
+    /* Get datatype for dataset */
+    tid2 = H5Dget_type(dataset);
+    CHECK(tid2, FAIL, "H5Dget_type");
+
+    /* Change to the custom memory allocation routines for reading VL data */
+    xfer_pid=H5Pcreate(H5P_DATASET_XFER);
+    CHECK(xfer_pid, FAIL, "H5Pcreate");
+
+    ret=H5Pset_vlen_mem_manager(xfer_pid,test_vltypes_alloc_custom,&mem_used,test_vltypes_free_custom,&mem_used);
+    CHECK(ret, FAIL, "H5Pset_vlen_mem_manager");
+
+    /* Make certain the correct amount of memory was used */
+    ret=H5Dvlen_get_buf_size(dataset,tid2,sid1,&size);
+    CHECK(ret, FAIL, "H5Dvlen_get_buf_size");
+
+    /* 10 hvl_t elements allocated = 1 + 2 + 3 + 4 elements for each array position */
+    /* 20 unsigned int elements allocated = 1 + 3 + 6 + 10 elements */
+    VERIFY(size,((SPACE1_DIM1*(SPACE1_DIM1+1))/2)*sizeof(hvl_t)+vlen_size_func(SPACE1_DIM1)*sizeof(unsigned int),"H5Dvlen_get_buf_size");
+
+    /* Read dataset from disk */
+    ret=H5Dread(dataset,tid2,H5S_ALL,H5S_ALL,xfer_pid,rdata);
+    CHECK(ret, FAIL, "H5Dread");
+
+    /* Make certain the correct amount of memory has been used */
+    /* 10 hvl_t elements allocated = 1 + 2 + 3 + 4 elements for each array position */
+    /* 20 unsigned int elements allocated = 1 + 3 + 6 + 10 elements */
+    VERIFY(mem_used,((SPACE1_DIM1*(SPACE1_DIM1+1))/2)*sizeof(hvl_t)+vlen_size_func(SPACE1_DIM1)*sizeof(unsigned int),"H5Dread");
+
+    /* Compare data read in */
+    for(i=0; i<SPACE1_DIM1; i++) {
+        if(wdata[i].len!=rdata[i].len) {
+            num_errs++;
+            printf("VL data length don't match!, wdata[%d].len=%d, rdata[%d].len=%d\n",(int)i,(int)wdata[i].len,(int)i,(int)rdata[i].len);
+            continue;
+        } /* end if */
+        for(t1=wdata[i].p, t2=rdata[i].p, j=0; j<rdata[i].len; j++, t1++, t2++)
+{
+            if(t1->len!=t2->len) {
+                num_errs++;
+                printf("VL data length don't match!, i=%d, j=%d, t1->len=%d, t2->len=%d\n",(int)i,(int)j,(int)t1->len,(int)t2->len);
+                continue;
+            } /* end if */
+            for(k=0; k<t2->len; k++) {
+                if( ((unsigned int *)t1->p)[k] != ((unsigned int *)t2->p)[k] ) { 
+                    num_errs++;
+/*printf("rdata[i].len=%d, t2->len=%d\n", rdata[i].len, t2->len);*/
+                    printf("VL data values don't match!, t1->p[%d]=%d, t2->p[%d]=%d\n",(int)k, (int)((unsigned int *)t1->p)[k], (int)k, (int)((unsigned int *)t2->p)[k]);
+                    continue;
+                } /* end if */
+            } /* end for */
+        } /* end for */
+    } /* end for */
+
+    /* Reclaim all the (nested) VL data */
+    ret=H5Dvlen_reclaim(tid2,sid1,xfer_pid,rdata);
+    CHECK(ret, FAIL, "H5Dvlen_reclaim");
+
+    /* Make certain the VL memory has been freed */
+    VERIFY(mem_used,0,"H5Dvlen_reclaim");
+
+    /* Reclaim the write VL data */
+    ret=H5Dvlen_reclaim(tid2,sid1,H5P_DEFAULT,wdata);
+    CHECK(ret, FAIL, "H5Dvlen_reclaim");
+
+    /* Close Dataset */
+    ret = H5Dclose(dataset);
+    CHECK(ret, FAIL, "H5Dclose");
+
+    /* Close datatype */
+    ret = H5Tclose(tid2);
+    CHECK(ret, FAIL, "H5Tclose");
+
+    /* Close disk dataspace */
+    ret = H5Sclose(sid1);
+    CHECK(ret, FAIL, "H5Sclose");
+
+    /* Close dataset transfer property list */
+    ret = H5Pclose(xfer_pid);
+    CHECK(ret, FAIL, "H5Pclose");
+
+    /* Close file */
+    ret = H5Fclose(fid1);
+    CHECK(ret, FAIL, "H5Fclose");
+
+} /* end rewrite_shorter_vltypes_vlen_vlen_atomic() */
 
 /****************************************************************
 **
@@ -1504,8 +1687,9 @@ test_vltypes(void)
     rewrite_vltypes_vlen_compound();  /* Check VL memory leak	  */
     test_vltypes_compound_vlen_atomic(); /* Test compound datatypes with VL atomic components */
     rewrite_vltypes_compound_vlen_atomic();/* Check VL memory leak	*/
-    test_vltypes_vlen_vlen_atomic();  /* Test VL datatype with VL atomic components */
-    rewrite_vltypes_vlen_vlen_atomic();  /* Check VL memory leak	*/
+    test_vltypes_vlen_vlen_atomic();  	   /* Test VL datatype with VL atomic components */
+    rewrite_longer_vltypes_vlen_vlen_atomic();  /*overwrite with VL data of longer sequence*/
+    rewrite_shorter_vltypes_vlen_vlen_atomic();  /*overwrite with VL data of shorted sequence*/
 }   /* test_vltypes() */