From 814ea8b962fec329f6a440d1f52f6b682de47524 Mon Sep 17 00:00:00 2001 From: Raymond Lu Date: Wed, 17 Jul 2002 16:11:47 -0500 Subject: [svn-r5812] Purpose: Bug fix. Description: This should fix the nested VL datatype bug. Platforms tested: modi4, eirene, impact --- src/H5Tconv.c | 56 ++++++++++++---- src/H5Tvlen.c | 3 +- test/tvltypes.c | 196 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 234 insertions(+), 21 deletions(-) diff --git a/src/H5Tconv.c b/src/H5Tconv.c index 65b7c93..e61e2cb 100644 --- a/src/H5Tconv.c +++ b/src/H5Tconv.c @@ -2178,11 +2178,10 @@ H5T_conv_vlen(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, hsize_t nelmts, uint8_t *s, *sp, *d, *dp; /*source and dest traversal ptrs */ uint8_t **dptr; /*pointer to correct destination pointer*/ uint8_t *bg_ptr=NULL; /*background buf traversal pointer */ - uint8_t *bg=NULL; - H5HG_t bg_hobjid; + H5HG_t bg_hobjid, parent_hobjid; size_t src_delta, dst_delta, bkg_delta;/*source & destination stride*/ hssize_t seq_len; /*the number of elements in the current sequence*/ - hsize_t bg_seq_len=0; + hsize_t bg_seq_len=0, parent_seq_len=0; size_t src_base_size, dst_base_size;/*source & destination base size*/ size_t src_size, dst_size; /*source & destination total size in bytes*/ void *conv_buf=NULL; /*temporary conversion buffer */ @@ -2191,7 +2190,9 @@ H5T_conv_vlen(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, hsize_t nelmts, size_t tmp_buf_size=0; /*size of temporary bkg buffer */ uint8_t dbuf[64],*dbuf_ptr=dbuf;/*temp destination buffer */ int direction; /*direction of traversal */ + int nested=0; /*flag of nested VL case */ hsize_t elmtno; /*element number counter */ + hsize_t i; FUNC_ENTER_NOAPI(H5T_conv_vlen, FAIL); @@ -2241,16 +2242,14 @@ H5T_conv_vlen(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, hsize_t nelmts, if (src->size==dst->size || buf_stride>0) { olap = nelmts; sp = dp = (uint8_t*)_buf; - if(_bkg!=NULL) - bg_ptr = (uint8_t*)_bkg; + bg_ptr = (uint8_t*)_bkg; direction = 1; } else if (src->size>=dst->size) { /* potentially this uses the destination buffer 1 extra * time, but its faster that floating-point calcs */ olap = ((dst->size)/(src->size-dst->size))+1; sp = dp = (uint8_t*)_buf; - if(_bkg!=NULL) - bg_ptr = (uint8_t*)_bkg; + bg_ptr = (uint8_t*)_bkg; direction = 1; } else { /* potentially this uses the destination buffer 1 extra @@ -2309,14 +2308,16 @@ H5T_conv_vlen(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, hsize_t nelmts, if ((tmp_buf=H5FL_BLK_ALLOC(vlen_seq,tmp_buf_size,1))==NULL) HRETURN_ERROR (H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for type conversion"); } /* end if */ + + /* Set the flag for nested VL case */ + if(dst->u.vlen.f!=NULL && H5T_detect_class(dst->parent,H5T_VLEN) && bg_ptr!=NULL) + nested=1; for (elmtno=0; elmtnou.vlen.getlen))(src->u.vlen.f,s); assert(seq_len>=0); H5_CHECK_OVERFLOW(seq_len,hssize_t,size_t); @@ -2345,15 +2346,23 @@ H5T_conv_vlen(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, hsize_t nelmts, tmp_buf_size=conv_buf_size; if((tmp_buf=H5FL_BLK_REALLOC(vlen_seq,tmp_buf,tmp_buf_size))==NULL) HRETURN_ERROR (H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for type conversion"); - HDmemset(tmp_buf,0,tmp_buf_size); } /* end if */ /* If we are writing and there is a nested VL type, read * the sequence into the background buffer */ - if(dst->u.vlen.f!=NULL && H5T_detect_class(dst->parent,H5T_VLEN) && bg!=NULL) { - uint8_t *tmp=bg; + if(nested) { + uint8_t *tmp=bg_ptr; UINT32DECODE(tmp, bg_seq_len); if(bg_seq_len>0) { + if(tmp_buf_sizeu.vlen.f, (const uint8_t **)&tmp, &(bg_hobjid.addr)); INT32DECODE(tmp, bg_hobjid.idx); @@ -2361,6 +2370,10 @@ H5T_conv_vlen(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, hsize_t nelmts, HRETURN_ERROR (H5E_DATATYPE, H5E_READERROR, FAIL, "can't read VL sequence into background buffer"); } /* end if */ + + /* If the sequence gets shorter, pad out the original sequence with zeros */ + if(bg_seq_lenu.vlen.write))(dset_xfer_plist,dst->u.vlen.f,d,conv_buf, bg, (hsize_t)seq_len,(hsize_t)dst_base_size)<0) + if((*(dst->u.vlen.write))(dset_xfer_plist,dst->u.vlen.f,d,conv_buf, bg_ptr, (hsize_t)seq_len,(hsize_t)dst_base_size)<0) HRETURN_ERROR(H5E_DATATYPE, H5E_WRITEERROR, FAIL, "can't write VL data"); + /* For nested VL case, free leftover heap objects from the deeper level if the length of new data elements is shorted than the old data elements.*/ + if(nested && seq_len0) { + H5F_addr_decode(dst->u.vlen.f, (const uint8_t **)&tmp_p, &(parent_hobjid.addr)); + INT32DECODE(tmp_p, parent_hobjid.idx); + if(H5HG_remove(dst->u.vlen.f, &parent_hobjid)<0) + HRETURN_ERROR(H5E_DATATYPE, H5E_WRITEERROR, FAIL, "Unable to remove heap object"); + } + } + } + /* * If we had used a temporary buffer for the destination * then we should copy the value to the true destination diff --git a/src/H5Tvlen.c b/src/H5Tvlen.c index cc38b3c..78e5b68 100644 --- a/src/H5Tvlen.c +++ b/src/H5Tvlen.c @@ -489,10 +489,11 @@ herr_t H5T_vlen_disk_write(hid_t UNUSED plist_id, H5F_t *f, void *vl_addr, void /* Get the length of the sequence and heap object ID from background data. * Free heap object for old data. */ if(bg!=NULL) { + HDmemset(&bg_hobjid,0,sizeof(H5HG_t)); UINT32DECODE(bg, bg_seq_len); /* Free heap object for old data */ - if(bg_seq_len!=0) { + if(bg_seq_len>0) { /* Get heap information */ H5F_addr_decode(f, (const uint8_t **)&bg, &(bg_hobjid.addr)); INT32DECODE(bg, bg_hobjid.idx); diff --git a/test/tvltypes.c b/test/tvltypes.c index 3aa7d65..1c51789 100644 --- a/test/tvltypes.c +++ b/test/tvltypes.c @@ -1305,12 +1305,12 @@ test_vltypes_vlen_vlen_atomic(void) /**************************************************************** ** -** rewrite_vltypes_vlen_vlen_atomic(): Test basic VL datatype code. +** rewrite_longer_vltypes_vlen_vlen_atomic(): Test basic VL datatype code. ** Tests VL datatype with VL datatypes of atomic datatypes. ** ****************************************************************/ static void -rewrite_vltypes_vlen_vlen_atomic(void) +rewrite_longer_vltypes_vlen_vlen_atomic(void) { hvl_t wdata[SPACE1_DIM1]; /* Information to write */ hvl_t rdata[SPACE1_DIM1]; /* Information read in */ @@ -1444,7 +1444,7 @@ rewrite_vltypes_vlen_vlen_atomic(void) } /* end if */ for(k=0; klen; k++) { if( ((unsigned int *)t1->p)[k] != ((unsigned int *)t2->p)[k] ) { num_errs++; - printf("VL data values don't match!, t1->p[%d]=%d, t2->p[%d]=%d, i=%d, j=%d\n",(int)k, (int)((unsigned int *)t1->p)[k], (int)k, (int)((unsigned int *)t2->p)[k], i, j); + printf("VL data values don't match!, t1->p[%d]=%d, t2->p[%d]=%d\n",(int)k, (int)((unsigned int *)t1->p)[k], (int)k, (int)((unsigned int *)t2->p)[k]); continue; } /* end if */ } /* end for */ @@ -1482,7 +1482,190 @@ rewrite_vltypes_vlen_vlen_atomic(void) ret = H5Fclose(fid1); CHECK(ret, FAIL, "H5Fclose"); -} /* end rewrite_vltypes_vlen_vlen_atomic() */ +} /* end rewrite_longer_vltypes_vlen_vlen_atomic() */ + +/**************************************************************** +** +** rewrite_shorter_vltypes_vlen_vlen_atomic(): Test basic VL datatype code. +** Tests VL datatype with VL datatypes of atomic datatypes. +** +****************************************************************/ +static void +rewrite_shorter_vltypes_vlen_vlen_atomic(void) +{ + hvl_t wdata[SPACE1_DIM1]; /* Information to write */ + hvl_t rdata[SPACE1_DIM1]; /* Information read in */ + hvl_t *t1, *t2; /* Temporary pointer to VL information */ + hid_t fid1; /* HDF5 File IDs */ + hid_t dataset; /* Dataset ID */ + hid_t sid1; /* Dataspace ID */ + hid_t tid2; /* Datatype IDs */ + hid_t xfer_pid; /* Dataset transfer property list ID */ + hsize_t size; /* Number of bytes which will be used */ + unsigned i,j,k; /* counting variables */ + size_t mem_used=0; /* Memory used during allocation */ + int increment=1; + herr_t ret; /* Generic return value */ + + /* Output message about test being performed */ + MESSAGE(5, ("Check memory leak for VL Datatypes with VL Atomic Datatype Component Functionality\n")); + + /* Allocate and initialize VL data to write */ + for(i=0; ip=malloc((j+1)*sizeof(unsigned int)); + if(t1->p==NULL) { + printf("Cannot allocate memory for VL data! i=%u, j=%u\n",i,j); + num_errs++; + return; + } /* end if */ + t1->len=j+1; + for(k=0; k<(j+1); k++) + ((unsigned int *)t1->p)[k]=i*100000+j*1000+k*10; + } /* end for */ + } /* end for */ + + /* Open file */ + fid1 = H5Fopen(FILENAME, H5F_ACC_RDWR, H5P_DEFAULT); + CHECK(fid1, FAIL, "H5Fopen"); + + /* Open the dataset */ + dataset=H5Dopen(fid1,"Dataset1"); + CHECK(dataset, FAIL, "H5Dopen"); + + /* Get dataspace for datasets */ + sid1 = H5Dget_space(dataset); + CHECK(sid1, FAIL, "H5Dget_space"); + + /* Open datatype of the dataset */ + tid2 = H5Dget_type(dataset); + CHECK(tid2, FAIL, "H5Dget_type"); + + /* Write dataset to disk */ + ret=H5Dwrite(dataset,tid2,H5S_ALL,H5S_ALL,H5P_DEFAULT,wdata); + CHECK(ret, FAIL, "H5Dwrite"); + + /* Close Dataset */ + ret = H5Dclose(dataset); + CHECK(ret, FAIL, "H5Dclose"); + + /* Close datatype */ + ret = H5Tclose(tid2); + CHECK(ret, FAIL, "H5Tclose"); + + /* Close disk dataspace */ + ret = H5Sclose(sid1); + CHECK(ret, FAIL, "H5Sclose"); + + /* Close file */ + ret = H5Fclose(fid1); + CHECK(ret, FAIL, "H5Fclose"); + + + /* Open the file for data checking */ + fid1 = H5Fopen(FILENAME, H5F_ACC_RDONLY, H5P_DEFAULT); + CHECK(fid1, FAIL, "H5Fopen"); + + /* Open a dataset */ + dataset=H5Dopen(fid1,"Dataset1"); + CHECK(dataset, FAIL, "H5Dopen"); + + /* Get dataspace for datasets */ + sid1 = H5Dget_space(dataset); + CHECK(sid1, FAIL, "H5Dget_space"); + + /* Get datatype for dataset */ + tid2 = H5Dget_type(dataset); + CHECK(tid2, FAIL, "H5Dget_type"); + + /* Change to the custom memory allocation routines for reading VL data */ + xfer_pid=H5Pcreate(H5P_DATASET_XFER); + CHECK(xfer_pid, FAIL, "H5Pcreate"); + + ret=H5Pset_vlen_mem_manager(xfer_pid,test_vltypes_alloc_custom,&mem_used,test_vltypes_free_custom,&mem_used); + CHECK(ret, FAIL, "H5Pset_vlen_mem_manager"); + + /* Make certain the correct amount of memory was used */ + ret=H5Dvlen_get_buf_size(dataset,tid2,sid1,&size); + CHECK(ret, FAIL, "H5Dvlen_get_buf_size"); + + /* 10 hvl_t elements allocated = 1 + 2 + 3 + 4 elements for each array position */ + /* 20 unsigned int elements allocated = 1 + 3 + 6 + 10 elements */ + VERIFY(size,((SPACE1_DIM1*(SPACE1_DIM1+1))/2)*sizeof(hvl_t)+vlen_size_func(SPACE1_DIM1)*sizeof(unsigned int),"H5Dvlen_get_buf_size"); + + /* Read dataset from disk */ + ret=H5Dread(dataset,tid2,H5S_ALL,H5S_ALL,xfer_pid,rdata); + CHECK(ret, FAIL, "H5Dread"); + + /* Make certain the correct amount of memory has been used */ + /* 10 hvl_t elements allocated = 1 + 2 + 3 + 4 elements for each array position */ + /* 20 unsigned int elements allocated = 1 + 3 + 6 + 10 elements */ + VERIFY(mem_used,((SPACE1_DIM1*(SPACE1_DIM1+1))/2)*sizeof(hvl_t)+vlen_size_func(SPACE1_DIM1)*sizeof(unsigned int),"H5Dread"); + + /* Compare data read in */ + for(i=0; ilen!=t2->len) { + num_errs++; + printf("VL data length don't match!, i=%d, j=%d, t1->len=%d, t2->len=%d\n",(int)i,(int)j,(int)t1->len,(int)t2->len); + continue; + } /* end if */ + for(k=0; klen; k++) { + if( ((unsigned int *)t1->p)[k] != ((unsigned int *)t2->p)[k] ) { + num_errs++; +/*printf("rdata[i].len=%d, t2->len=%d\n", rdata[i].len, t2->len);*/ + printf("VL data values don't match!, t1->p[%d]=%d, t2->p[%d]=%d\n",(int)k, (int)((unsigned int *)t1->p)[k], (int)k, (int)((unsigned int *)t2->p)[k]); + continue; + } /* end if */ + } /* end for */ + } /* end for */ + } /* end for */ + + /* Reclaim all the (nested) VL data */ + ret=H5Dvlen_reclaim(tid2,sid1,xfer_pid,rdata); + CHECK(ret, FAIL, "H5Dvlen_reclaim"); + + /* Make certain the VL memory has been freed */ + VERIFY(mem_used,0,"H5Dvlen_reclaim"); + + /* Reclaim the write VL data */ + ret=H5Dvlen_reclaim(tid2,sid1,H5P_DEFAULT,wdata); + CHECK(ret, FAIL, "H5Dvlen_reclaim"); + + /* Close Dataset */ + ret = H5Dclose(dataset); + CHECK(ret, FAIL, "H5Dclose"); + + /* Close datatype */ + ret = H5Tclose(tid2); + CHECK(ret, FAIL, "H5Tclose"); + + /* Close disk dataspace */ + ret = H5Sclose(sid1); + CHECK(ret, FAIL, "H5Sclose"); + + /* Close dataset transfer property list */ + ret = H5Pclose(xfer_pid); + CHECK(ret, FAIL, "H5Pclose"); + + /* Close file */ + ret = H5Fclose(fid1); + CHECK(ret, FAIL, "H5Fclose"); + +} /* end rewrite_shorter_vltypes_vlen_vlen_atomic() */ /**************************************************************** ** @@ -1504,8 +1687,9 @@ test_vltypes(void) rewrite_vltypes_vlen_compound(); /* Check VL memory leak */ test_vltypes_compound_vlen_atomic(); /* Test compound datatypes with VL atomic components */ rewrite_vltypes_compound_vlen_atomic();/* Check VL memory leak */ - test_vltypes_vlen_vlen_atomic(); /* Test VL datatype with VL atomic components */ - rewrite_vltypes_vlen_vlen_atomic(); /* Check VL memory leak */ + test_vltypes_vlen_vlen_atomic(); /* Test VL datatype with VL atomic components */ + rewrite_longer_vltypes_vlen_vlen_atomic(); /*overwrite with VL data of longer sequence*/ + rewrite_shorter_vltypes_vlen_vlen_atomic(); /*overwrite with VL data of shorted sequence*/ } /* test_vltypes() */ -- cgit v0.12