[svn-r7842] Purpose:

    Bug fix

Description:
    Variable length strings and sequences with NULL pointers were not handled
by library, causing problems access the data.  This also affected fill values
for variable-length datatypes.

Solution:
    Address the issues in the library by detecting NULL sequences/strings
and avoid trying to convert them.

    Patched up dumper to display NULL sequences/strings.

Platforms tested:
    FreeBSD 4.9 (sleipnir)
    h5committest

2 files changed, 293 insertions, 11 deletions

diff --git a/test/tvlstr.c b/test/tvlstr.c
index 8a4ac99..ef35267 100644
--- a/test/tvlstr.c
+++ b/test/tvlstr.c
@@ -258,10 +258,12 @@ test_vlstrings_special(void)
     const char *wdata[SPACE1_DIM1] = {"one", "two", "", "four"};
     const char *wdata2[SPACE1_DIM1] = {NULL, NULL, NULL, NULL};
     char *rdata[SPACE1_DIM1];   /* Information read in */
+    char *fill;                 /* Fill value */
     hid_t		fid1;		/* HDF5 File IDs		*/
     hid_t		dataset;	/* Dataset ID			*/
     hid_t		sid1;       /* Dataspace ID			*/
     hid_t		tid1;       /* Datatype ID			*/
+    hid_t		dcpl;       /* Dataset creation property list ID */
     hsize_t		dims1[] = {SPACE1_DIM1};
     unsigned       i;          /* counting variable */
     herr_t		ret;		/* Generic return value		*/
@@ -288,6 +290,18 @@ test_vlstrings_special(void)
     dataset=H5Dcreate(fid1,"Dataset3",tid1,sid1,H5P_DEFAULT);
     CHECK(dataset, FAIL, "H5Dcreate");
 
+    /* Read from dataset before writing data */
+    ret=H5Dread(dataset,tid1,H5S_ALL,H5S_ALL,H5P_DEFAULT,rdata);
+    CHECK(ret, FAIL, "H5Dread");
+
+    /* Check data read in */
+    for(i=0; i<SPACE1_DIM1; i++) {
+        if(rdata[i]!=NULL) {
+            num_errs++;
+            printf("VL doesn't match!, rdata[%d]=%p\n",(int)i,rdata[i]);
+        } /* end if */
+    } /* end for */
+    
     /* Write dataset to disk */
     ret=H5Dwrite(dataset,tid1,H5S_ALL,H5S_ALL,H5P_DEFAULT,wdata);
     CHECK(ret, FAIL, "H5Dwrite");
@@ -319,14 +333,49 @@ test_vlstrings_special(void)
     CHECK(ret, FAIL, "H5Dclose");
 
     /* Create another dataset to test nil strings */
-    dataset=H5Dcreate(fid1,"Dataset4",tid1,sid1,H5P_DEFAULT);
+    dcpl=H5Pcreate(H5P_DATASET_CREATE);
+    CHECK(dcpl, FAIL, "H5Pcreate");
+
+    /* Set the fill value for the second dataset */
+    fill=NULL;
+    ret=H5Pset_fill_value(dcpl, tid1, &fill);
+    CHECK(ret, FAIL, "H5Pset_fill_value");
+
+    dataset=H5Dcreate(fid1,"Dataset4",tid1,sid1,dcpl);
     CHECK(dataset, FAIL, "H5Dcreate");
 
-    /* Try to write nil strings to disk.  Should fail. */
-    H5E_BEGIN_TRY {
-        H5Dwrite(dataset,tid1,H5S_ALL,H5S_ALL,H5P_DEFAULT,wdata2);
-    } H5E_END_TRY;
+    /* Close dataset creation property list */
+    ret = H5Pclose(dcpl);
+    CHECK(ret, FAIL, "H5Pclose");
+    
+    /* Read from dataset before writing data */
+    ret=H5Dread(dataset,tid1,H5S_ALL,H5S_ALL,H5P_DEFAULT,rdata);
+    CHECK(ret, FAIL, "H5Dread");
 
+    /* Check data read in */
+    for(i=0; i<SPACE1_DIM1; i++) {
+        if(rdata[i]!=NULL) {
+            num_errs++;
+            printf("VL doesn't match!, rdata[%d]=%p\n",(int)i,rdata[i]);
+        } /* end if */
+    } /* end for */
+    
+    /* Try to write nil strings to disk. */
+    ret=H5Dwrite(dataset,tid1,H5S_ALL,H5S_ALL,H5P_DEFAULT,wdata2);
+    CHECK(ret, FAIL, "H5Dwrite");
+
+    /* Read nil strings back from disk */
+    ret=H5Dread(dataset,tid1,H5S_ALL,H5S_ALL,H5P_DEFAULT,rdata);
+    CHECK(ret, FAIL, "H5Dread");
+
+    /* Check data read in */
+    for(i=0; i<SPACE1_DIM1; i++) {
+        if(rdata[i]!=NULL) {
+            num_errs++;
+            printf("VL doesn't match!, rdata[%d]=%p\n",(int)i,rdata[i]);
+        } /* end if */
+    } /* end for */
+    
     /* Close Dataset */
     ret = H5Dclose(dataset);
     CHECK(ret, FAIL, "H5Dclose");
@@ -432,8 +481,8 @@ static void test_vlstring_type(void)
 
 /****************************************************************
 **
-**  test_vlstrings_special(): Test VL string code for special 
-**      string cases, nil and zero-sized.
+**  test_compact_vlstring(): Test code for storing VL strings in
+**      compact datasets.
 ** 
 ****************************************************************/
 static void
diff --git a/test/tvltypes.c b/test/tvltypes.c
index c924db6..a2f561f 100644
--- a/test/tvltypes.c
+++ b/test/tvltypes.c
@@ -170,11 +170,14 @@ static void
 test_vltypes_vlen_atomic(void)
 {
     hvl_t wdata[SPACE1_DIM1];   /* Information to write */
+    hvl_t wdata2[SPACE1_DIM1];  /* Information to write */
     hvl_t rdata[SPACE1_DIM1];   /* Information read in */
+    hvl_t fill;                 /* Fill value */
     hid_t		fid1;		/* HDF5 File IDs		*/
     hid_t		dataset;	/* Dataset ID			*/
     hid_t		sid1;       /* Dataspace ID			*/
     hid_t		tid1;       /* Datatype ID			*/
+    hid_t       dcpl_pid;   /* Dataset creation property list ID */
     hid_t       xfer_pid;   /* Dataset transfer property list ID */
     hsize_t		dims1[] = {SPACE1_DIM1};
     hsize_t     size;       /* Number of bytes which will be used */
@@ -191,6 +194,9 @@ test_vltypes_vlen_atomic(void)
         wdata[i].len=i+1;
         for(j=0; j<(i+1); j++)
             ((unsigned int *)wdata[i].p)[j]=i*10+j;
+
+        wdata2[i].p=NULL;
+        wdata2[i].len=0;
     } /* end for */
 
     /* Create file */
@@ -209,6 +215,34 @@ test_vltypes_vlen_atomic(void)
     dataset=H5Dcreate(fid1,"Dataset1",tid1,sid1,H5P_DEFAULT);
     CHECK(dataset, FAIL, "H5Dcreate");
 
+    /* Read from dataset before writing data */
+    ret=H5Dread(dataset,tid1,H5S_ALL,H5S_ALL,H5P_DEFAULT,rdata);
+    CHECK(ret, FAIL, "H5Dread");
+
+    /* Check data read in */
+    for(i=0; i<SPACE1_DIM1; i++) {
+        if(rdata[i].len!=0 || rdata[i].p!=NULL) {
+            num_errs++;
+            printf("VL doesn't match!, rdata[%d].len=%u, rdata[%d].p=%p\n",(int)i,(unsigned)rdata[i].len,(int)i,rdata[i].p);
+        } /* end if */
+    } /* end for */
+    
+    /* Write "nil" data to disk */
+    ret=H5Dwrite(dataset,tid1,H5S_ALL,H5S_ALL,H5P_DEFAULT,wdata2);
+    CHECK(ret, FAIL, "H5Dwrite");
+
+    /* Read from dataset with "nil" data */
+    ret=H5Dread(dataset,tid1,H5S_ALL,H5S_ALL,H5P_DEFAULT,rdata);
+    CHECK(ret, FAIL, "H5Dread");
+
+    /* Check data read in */
+    for(i=0; i<SPACE1_DIM1; i++) {
+        if(rdata[i].len!=0 || rdata[i].p!=NULL) {
+            num_errs++;
+            printf("VL doesn't match!, rdata[%d].len=%u, rdata[%d].p=%p\n",(int)i,(unsigned)rdata[i].len,(int)i,rdata[i].p);
+        } /* end if */
+    } /* end for */
+    
     /* Write dataset to disk */
     ret=H5Dwrite(dataset,tid1,H5S_ALL,H5S_ALL,H5P_DEFAULT,wdata);
     CHECK(ret, FAIL, "H5Dwrite");
@@ -217,6 +251,59 @@ test_vltypes_vlen_atomic(void)
     ret = H5Dclose(dataset);
     CHECK(ret, FAIL, "H5Dclose");
 
+    /* Create second dataset, with fill value */
+    dcpl_pid=H5Pcreate(H5P_DATASET_CREATE);
+    CHECK(dcpl_pid, FAIL, "H5Pcreate");
+
+    /* Set the fill value for the second dataset */
+    fill.p=NULL; fill.len=0;
+    ret=H5Pset_fill_value(dcpl_pid, tid1, &fill);
+    CHECK(ret, FAIL, "H5Pset_fill_value");
+
+    /* Create a second dataset */
+    dataset=H5Dcreate(fid1,"Dataset2",tid1,sid1,dcpl_pid);
+    CHECK(dataset, FAIL, "H5Dcreate");
+
+    /* Close dataset creation property list */
+    ret = H5Pclose(dcpl_pid);
+    CHECK(ret, FAIL, "H5Pclose");
+    
+    /* Read from dataset before writing data */
+    ret=H5Dread(dataset,tid1,H5S_ALL,H5S_ALL,H5P_DEFAULT,rdata);
+    CHECK(ret, FAIL, "H5Dread");
+
+    /* Check data read in */
+    for(i=0; i<SPACE1_DIM1; i++) {
+        if(rdata[i].len!=0 || rdata[i].p!=NULL) {
+            num_errs++;
+            printf("VL doesn't match!, rdata[%d].len=%u, rdata[%d].p=%p\n",(int)i,(unsigned)rdata[i].len,(int)i,rdata[i].p);
+        } /* end if */
+    } /* end for */
+    
+    /* Write "nil" data to disk */
+    ret=H5Dwrite(dataset,tid1,H5S_ALL,H5S_ALL,H5P_DEFAULT,wdata2);
+    CHECK(ret, FAIL, "H5Dwrite");
+
+    /* Read from dataset with "nil" data */
+    ret=H5Dread(dataset,tid1,H5S_ALL,H5S_ALL,H5P_DEFAULT,rdata);
+    CHECK(ret, FAIL, "H5Dread");
+
+    /* Check data read in */
+    for(i=0; i<SPACE1_DIM1; i++) {
+        if(rdata[i].len!=0 || rdata[i].p!=NULL) {
+            num_errs++;
+            printf("VL doesn't match!, rdata[%d].len=%u, rdata[%d].p=%p\n",(int)i,(unsigned)rdata[i].len,(int)i,rdata[i].p);
+        } /* end if */
+    } /* end for */
+    
+    /* Write data to disk */
+    ret=H5Dwrite(dataset,tid1,H5S_ALL,H5S_ALL,H5P_DEFAULT,wdata);
+    CHECK(ret, FAIL, "H5Dwrite");
+
+    /* Close Dataset */
+    ret = H5Dclose(dataset);
+    CHECK(ret, FAIL, "H5Dclose");
+
     /* Close datatype */
     ret = H5Tclose(tid1);
     CHECK(ret, FAIL, "H5Tclose");
@@ -291,14 +378,88 @@ test_vltypes_vlen_atomic(void)
     /* Make certain the VL memory has been freed */
     VERIFY(mem_used,0,"H5Dvlen_reclaim");
 
-    /* Reclaim the write VL data */
-    ret=H5Dvlen_reclaim(tid1,sid1,H5P_DEFAULT,wdata);
+    /* Close Dataset */
+    ret = H5Dclose(dataset);
+    CHECK(ret, FAIL, "H5Dclose");
+
+    /* Close datatype */
+    ret = H5Tclose(tid1);
+    CHECK(ret, FAIL, "H5Tclose");
+
+    /* Close disk dataspace */
+    ret = H5Sclose(sid1);
+    CHECK(ret, FAIL, "H5Sclose");
+    
+    /* Close dataset transfer property list */
+    ret = H5Pclose(xfer_pid);
+    CHECK(ret, FAIL, "H5Pclose");
+    
+
+    /* Open second dataset */
+    dataset=H5Dopen(fid1,"Dataset2");
+    CHECK(dataset, FAIL, "H5Dopen");
+
+    /* Get dataspace for datasets */
+    sid1 = H5Dget_space(dataset);
+    CHECK(sid1, FAIL, "H5Dget_space");
+
+    /* Get datatype for dataset */
+    tid1 = H5Dget_type(dataset);
+    CHECK(tid1, FAIL, "H5Dget_type");
+
+    /* Change to the custom memory allocation routines for reading VL data */
+    xfer_pid=H5Pcreate(H5P_DATASET_XFER);
+    CHECK(xfer_pid, FAIL, "H5Pcreate");
+
+    ret=H5Pset_vlen_mem_manager(xfer_pid,test_vltypes_alloc_custom,&mem_used,test_vltypes_free_custom,&mem_used);
+    CHECK(ret, FAIL, "H5Pset_vlen_mem_manager");
+
+    /* Make certain the correct amount of memory will be used */
+    ret=H5Dvlen_get_buf_size(dataset,tid1,sid1,&size);
+    CHECK(ret, FAIL, "H5Dvlen_get_buf_size");
+
+    /* 10 elements allocated = 1 + 2 + 3 + 4 elements for each array position */
+    VERIFY(size,((SPACE1_DIM1*(SPACE1_DIM1+1))/2)*sizeof(unsigned int),"H5Dvlen_get_buf_size");
+
+    /* Read dataset from disk */
+    ret=H5Dread(dataset,tid1,H5S_ALL,H5S_ALL,xfer_pid,rdata);
+    CHECK(ret, FAIL, "H5Dread");
+
+    /* Make certain the correct amount of memory has been used */
+    /* 10 elements allocated = 1 + 2 + 3 + 4 elements for each array position */
+    VERIFY(mem_used,((SPACE1_DIM1*(SPACE1_DIM1+1))/2)*sizeof(unsigned int),"H5Dread");
+
+    /* Compare data read in */
+    for(i=0; i<SPACE1_DIM1; i++) {
+        if(wdata[i].len!=rdata[i].len) {
+            num_errs++;
+            printf("%d: VL data lengths don't match!, wdata[%d].len=%d, rdata[%d].len=%d\n",__LINE__,(int)i,(int)wdata[i].len,(int)i,(int)rdata[i].len);
+            continue;
+        } /* end if */
+        for(j=0; j<rdata[i].len; j++) {
+            if( ((unsigned int *)wdata[i].p)[j] != ((unsigned int *)rdata[i].p)[j] ) {
+                num_errs++;
+                printf("VL data values don't match!, wdata[%d].p[%d]=%d, rdata[%d].p[%d]=%d\n",(int)i,(int)j, (int)((unsigned int *)wdata[i].p)[j], (int)i,(int)j, (int)((unsigned int *)rdata[i].p)[j]);
+                continue;
+            } /* end if */
+        } /* end for */
+    } /* end for */
+
+    /* Reclaim the read VL data */
+    ret=H5Dvlen_reclaim(tid1,sid1,xfer_pid,rdata);
     CHECK(ret, FAIL, "H5Dvlen_reclaim");
 
+    /* Make certain the VL memory has been freed */
+    VERIFY(mem_used,0,"H5Dvlen_reclaim");
+
     /* Close Dataset */
     ret = H5Dclose(dataset);
     CHECK(ret, FAIL, "H5Dclose");
 
+    /* Reclaim the write VL data */
+    ret=H5Dvlen_reclaim(tid1,sid1,H5P_DEFAULT,wdata);
+    CHECK(ret, FAIL, "H5Dvlen_reclaim");
+
     /* Close datatype */
     ret = H5Tclose(tid1);
     CHECK(ret, FAIL, "H5Tclose");
@@ -789,11 +950,13 @@ test_vltypes_compound_vlen_atomic(void)
     } s1;
     s1 wdata[SPACE1_DIM1];   /* Information to write */
     s1 rdata[SPACE1_DIM1];   /* Information read in */
+    s1 fill;                    /* Fill value */
     hid_t		fid1;		/* HDF5 File IDs		*/
     hid_t		dataset;	/* Dataset ID			*/
     hid_t		sid1;       /* Dataspace ID			*/
     hid_t		tid1, tid2; /* Datatype IDs         */
     hid_t       xfer_pid;   /* Dataset transfer property list ID */
+    hid_t       dcpl_pid;   /* Dataset creation property list ID */
     hsize_t		dims1[] = {SPACE1_DIM1};
     hsize_t     size;       /* Number of bytes which will be used */
     unsigned       i,j;        /* counting variables */
@@ -900,14 +1063,84 @@ test_vltypes_compound_vlen_atomic(void)
     /* Make certain the VL memory has been freed */
     VERIFY(mem_used,0,"H5Dvlen_reclaim");
 
-    /* Reclaim the write VL data */
-    ret=H5Dvlen_reclaim(tid2,sid1,H5P_DEFAULT,wdata);
+    /* Close Dataset */
+    ret = H5Dclose(dataset);
+    CHECK(ret, FAIL, "H5Dclose");
+
+    /* Create a second dataset, with a fill value */
+    dcpl_pid=H5Pcreate(H5P_DATASET_CREATE);
+    CHECK(dcpl_pid, FAIL, "H5Pcreate");
+
+    /* Set the fill value for the second dataset */
+    HDmemset(&fill,0,sizeof(s1));
+    ret=H5Pset_fill_value(dcpl_pid, tid2, &fill);
+    CHECK(ret, FAIL, "H5Pset_fill_value");
+
+    dataset=H5Dcreate(fid1,"Dataset2",tid2,sid1,H5P_DEFAULT);
+    CHECK(dataset, FAIL, "H5Dcreate");
+
+    /* Close dataset creation property list */
+    ret = H5Pclose(dcpl_pid);
+    CHECK(ret, FAIL, "H5Pclose");
+    
+    /* Read from dataset before writing data */
+    ret=H5Dread(dataset,tid2,H5S_ALL,H5S_ALL,H5P_DEFAULT,rdata);
+    CHECK(ret, FAIL, "H5Dread");
+
+    /* Check data read in */
+    for(i=0; i<SPACE1_DIM1; i++) {
+        if(rdata[i].i!=0 || rdata[i].f!=0.0 || rdata[i].v.len!=0 || rdata[i].v.p!=NULL) {
+            num_errs++;
+            printf("VL doesn't match!, rdata[%d].i=%d, rdata[%d].f=%f, rdata[%d].v.len=%u, rdata[%d].v.p=%p\n",(int)i,rdata[i].i,(int)i,rdata[i].f,(int)i,(unsigned)rdata[i].v.len,(int)i,rdata[i].v.p);
+        } /* end if */
+    } /* end for */
+    
+    /* Write dataset to disk */
+    ret=H5Dwrite(dataset,tid2,H5S_ALL,H5S_ALL,H5P_DEFAULT,wdata);
+    CHECK(ret, FAIL, "H5Dwrite");
+
+    /* Read dataset from disk */
+    ret=H5Dread(dataset,tid2,H5S_ALL,H5S_ALL,H5P_DEFAULT,rdata);
+    CHECK(ret, FAIL, "H5Dread");
+
+    /* Compare data read in */
+    for(i=0; i<SPACE1_DIM1; i++) {
+        if(wdata[i].i!=rdata[i].i) {
+            num_errs++;
+            printf("Integer components don't match!, wdata[%d].i=%d, rdata[%d].i=%d\n",(int)i,(int)wdata[i].i,(int)i,(int)rdata[i].i);
+            continue;
+        } /* end if */
+        if(wdata[i].f!=rdata[i].f) {
+            num_errs++;
+            printf("Float components don't match!, wdata[%d].f=%f, rdata[%d].f=%f\n",(int)i,(double)wdata[i].f,(int)i,(double)rdata[i].f);
+            continue;
+        } /* end if */
+        if(wdata[i].v.len!=rdata[i].v.len) {
+            num_errs++;
+            printf("%d: VL data length don't match!, wdata[%d].v.len=%d, rdata[%d].v.len=%d\n",__LINE__,(int)i,(int)wdata[i].v.len,(int)i,(int)rdata[i].v.len);
+            continue;
+        } /* end if */
+        for(j=0; j<rdata[i].v.len; j++) {
+            if( ((unsigned int *)wdata[i].v.p)[j] != ((unsigned int *)rdata[i].v.p)[j] ) {
+                num_errs++;
+                printf("VL data values don't match!, wdata[%d].v.p[%d]=%d, rdata[%d].v.p[%d]=%d\n",(int)i,(int)j, (int)((unsigned int *)wdata[i].v.p)[j], (int)i,(int)j, (int)((unsigned int *)rdata[i].v.p)[j]);
+                continue;
+            } /* end if */
+        } /* end for */
+    } /* end for */
+
+    /* Reclaim the VL data */
+    ret=H5Dvlen_reclaim(tid2,sid1,H5P_DEFAULT,rdata);
     CHECK(ret, FAIL, "H5Dvlen_reclaim");
 
     /* Close Dataset */
     ret = H5Dclose(dataset);
     CHECK(ret, FAIL, "H5Dclose");
 
+    /* Reclaim the write VL data */
+    ret=H5Dvlen_reclaim(tid2,sid1,H5P_DEFAULT,wdata);
+    CHECK(ret, FAIL, "H5Dvlen_reclaim");
+
     /* Close datatype */
     ret = H5Tclose(tid2);
     CHECK(ret, FAIL, "H5Tclose");