[svn-r306] Changes since 19980305

----------------------

./src/H5D.c
./test/external.c
        A contiguous dataset can now be stored in external files in
        such a way that the slowest varying dimension can be increased
        after the dataset is created and even written.  The user must
        have reserved enough space in the external file and the data
        space must allow an increase the the slowest varying dimension.
        See test_3() of ./test/external.c

./src/H5Oefl.c
./configure.in
./src/H5Fprivate.h
        Fixed an overflow bug with unlimited external storage.

./src/H5V.c
        Fixed a cast warning.

./test/dsets.c
        Make better use of automatic error reporting.

5 files changed, 95 insertions, 35 deletions

diff --git a/src/H5D.c b/src/H5D.c
index da470d3..8d20c85 100644
--- a/src/H5D.c
+++ b/src/H5D.c
@@ -746,7 +746,9 @@ H5D_create(H5F_t *f, const char *name, const H5T_t *type, const H5S_t *space,
 {
     H5D_t		*new_dset = NULL;
     H5D_t		*ret_value = NULL;
-    intn		i;
+    intn		i, ndims;
+    size_t		max_dim[H5O_LAYOUT_NDIMS];
+    H5O_efl_t		*efl = NULL;
 
     FUNC_ENTER(H5D_create, NULL);
 
@@ -763,6 +765,7 @@ H5D_create(H5F_t *f, const char *name, const H5T_t *type, const H5S_t *space,
     new_dset->type = H5T_copy(type);
     new_dset->space = H5S_copy(space);
     new_dset->create_parms = H5P_copy (H5P_DATASET_CREATE, create_parms);
+    efl = &(new_dset->create_parms->efl);
 
     /* Total raw data size */
     new_dset->layout.type = new_dset->create_parms->layout;
@@ -772,26 +775,53 @@ H5D_create(H5F_t *f, const char *name, const H5T_t *type, const H5S_t *space,
 
     switch (new_dset->create_parms->layout) {
     case H5D_CONTIGUOUS:
-	if (H5S_get_dims(space, new_dset->layout.dim, NULL) < 0) {
+	/*
+	 * The maximum size of the dataset cannot exceed the storage size.
+	 * Also, only the slowest varying dimension of a simple data space
+	 * can be extendible.
+	 */
+	if ((ndims=H5S_get_dims(space, new_dset->layout.dim, max_dim)) < 0) {
 	    HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL,
 			"unable to initialize contiguous storage");
 	}
-	if (new_dset->create_parms->efl.nused>0) {
+	for (i=1; i<ndims; i++) {
+	    if (max_dim[i]>new_dset->layout.dim[i]) {
+		HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, NULL,
+			     "only the first dimension can be extendible");
+	    }
+	}
+	if (efl->nused>0) {
 	    size_t max_points = H5S_get_npoints_max (space);
-	    if (max_points*H5T_get_size (type) >
-		H5O_efl_total_size (&(new_dset->create_parms->efl))) {
+	    size_t max_storage = H5O_efl_total_size (efl);
+
+	    if (H5S_UNLIMITED==max_points) {
+		if (H5O_EFL_UNLIMITED!=max_storage) {
+		    HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, NULL,
+				 "unlimited data space but finite storage");
+		}
+	    } else if (max_points * H5T_get_size (type) < max_points) {
 		HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, NULL,
-			     "max size exceeds external storage size");
+			     "data space * type size overflowed");
+	    } else if (max_points * H5T_get_size (type) > max_storage) {
+		HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, NULL,
+			     "data space size exceeds external storage size");
 	    }
+	} else if (max_dim[0]>new_dset->layout.dim[0]) {
+	    HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, NULL,
+			 "extendible contiguous non-external dataset");
 	}
 	break;
 
     case H5D_CHUNKED:
+	/*
+	 * Chunked storage allows any type of data space extension, so we
+	 * don't even bother checking.
+	 */
 	if (new_dset->create_parms->chunk_ndims != H5S_get_ndims(space)) {
 	    HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, NULL,
 		   "dimensionality of chunks doesn't match the data space");
 	}
-	if (new_dset->create_parms->efl.nused>0) {
+	if (efl->nused>0) {
 	    HGOTO_ERROR (H5E_DATASET, H5E_BADVALUE, NULL,
 			 "external storage not supported with chunked layout");
 	}
@@ -818,8 +848,12 @@ H5D_create(H5F_t *f, const char *name, const H5T_t *type, const H5S_t *space,
 		    "can't update type or space header messages");
     }
 
-    /* Initialize storage */
-    if (0==new_dset->create_parms->efl.nused) {
+    /*
+     * Initialize storage.  We assume that external storage is already
+     * initialized by the caller, or at least will be before I/O is
+     * performed.
+     */
+    if (0==efl->nused) {
 	if (H5F_arr_create(f, &(new_dset->layout)) < 0) {
 	    HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL,
 			"unable to initialize storage");
@@ -827,26 +861,27 @@ H5D_create(H5F_t *f, const char *name, const H5T_t *type, const H5S_t *space,
     } else {
 	H5F_addr_undef (&(new_dset->layout.addr));
     }
+
+    /* Update layout message */
     if (H5O_modify (&(new_dset->ent), H5O_LAYOUT, 0, H5O_FLAG_CONSTANT,
 		    &(new_dset->layout)) < 0) {
 	HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, NULL,
 		     "unable to update layout message");
     }
-    if (new_dset->create_parms->efl.nused>0) {
+
+    /* Update external storage message */
+    if (efl->nused>0) {
 	size_t heap_size = H5H_ALIGN (1);
-	for (i=0; i<new_dset->create_parms->efl.nused; i++) {
-	    size_t n = strlen (new_dset->create_parms->efl.slot[i].name)+1;
-	    heap_size += H5H_ALIGN (n);
+	for (i=0; i<efl->nused; i++) {
+	    heap_size += H5H_ALIGN (strlen (efl->slot[i].name)+1);
 	}
-	if (H5H_create (f, H5H_LOCAL, heap_size,
-			&(new_dset->create_parms->efl.heap_addr))<0 ||
-	    H5H_insert (f, &(new_dset->create_parms->efl.heap_addr),
-			1, "")<0) {
+	if (H5H_create (f, H5H_LOCAL, heap_size, &(efl->heap_addr))<0 ||
+	    (size_t)(-1)==H5H_insert (f, &(efl->heap_addr), 1, "")) {
 	    HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, NULL,
 			 "unable to create external file list name heap");
 	}
 	if (H5O_modify (&(new_dset->ent), H5O_EFL, 0, H5O_FLAG_CONSTANT,
-			&(new_dset->create_parms->efl))<0) {
+			efl)<0) {
 	    HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, NULL,
 			 "unable to update external file list message");
 	}
@@ -1368,16 +1403,12 @@ H5D_extend (H5D_t *dataset, const size_t *size)
     assert (dataset);
     assert (size);
 
-    if (dataset->create_parms->efl.nused>0) {
-	HRETURN_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL,
-		       "extending externally-stored data is not implemente yet");
-    }
-    
-    /* This is only allowed for data spaces with chunked layout */
-    if (H5D_CHUNKED!=dataset->layout.type) {
-	HRETURN_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL,
-		       "size can only be increased for chunked datasets");
-    }
+    /*
+     * Restrictions on extensions were checked when the dataset was created.
+     * All extensions are allowed here since none should be able to muck
+     * things up.
+     */
+    /*void*/
 
     /* Increase the size of the data space */
     if ((changed=H5S_extend (dataset->space, size))<0) {
diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h
index 497a9a0..77a299f 100644
--- a/src/H5Fprivate.h
+++ b/src/H5Fprivate.h
@@ -209,6 +209,17 @@
 #define NBYTEDECODE(s, d, n) {	 HDmemcpy(d,s,n); p+=n }
 
 /*
+ * Macros that check for overflows.  These are somewhat dangerous to fiddle
+ * with.
+ */
+#if (SIZEOF_SIZE_T >= SIZEOF_OFF_T)
+#   define H5F_OVERFLOW_SIZET2OFFT(X)					      \
+    ((size_t)(X)>=(size_t)((size_t)1<<(8*sizeof(off_t)-1)))
+#else
+#   define H5F_OVERFLOW_SIZET2OFFT(X) 0
+#endif
+
+/*
  * File-creation property list.
  */
 typedef struct H5F_create_t {
diff --git a/src/H5Oefl.c b/src/H5Oefl.c
index ac65101..1b3fa6a 100644
--- a/src/H5Oefl.c
+++ b/src/H5Oefl.c
@@ -386,7 +386,8 @@ H5O_efl_read (H5F_t *f, const H5O_efl_t *efl, haddr_t *addr, size_t size,
 
     /* Find the first efl member from which to read */
     for (i=0, cur=0; i<efl->nused; i++) {
-	if (addr->offset < cur+efl->slot[i].size) {
+	if (H5O_EFL_UNLIMITED==efl->slot[i].size ||
+	    addr->offset < cur+efl->slot[i].size) {
 	    skip = addr->offset - cur;
 	    break;
 	}
@@ -399,11 +400,15 @@ H5O_efl_read (H5F_t *f, const H5O_efl_t *efl, haddr_t *addr, size_t size,
 	    HGOTO_ERROR (H5E_EFL, H5E_OVERFLOW, FAIL,
 			 "read past logical end of file");
 	}
+	if (H5F_OVERFLOW_SIZET2OFFT (efl->slot[i].offset+skip)) {
+	    HGOTO_ERROR (H5E_EFL, H5E_OVERFLOW, FAIL,
+			 "external file address overflowed");
+	}
 	if ((fd=open (efl->slot[i].name, O_RDONLY))<0) {
 	    HGOTO_ERROR (H5E_EFL, H5E_CANTOPENFILE, FAIL,
 			 "unable to open external raw data file");
 	}
-	if (lseek (fd, efl->slot[i].offset+skip, SEEK_SET)<0) {
+	if (lseek (fd, (off_t)(efl->slot[i].offset+skip), SEEK_SET)<0) {
 	    HGOTO_ERROR (H5E_EFL, H5E_SEEKERROR, FAIL,
 			 "unable to seek in external raw data file");
 	}
@@ -454,7 +459,6 @@ H5O_efl_write (H5F_t *f, const H5O_efl_t *efl, haddr_t *addr, size_t size,
 {
     int		i, fd=-1;
     size_t	to_write, cur, skip;
-    ssize_t	n;
     herr_t	ret_value = FAIL;
     
     FUNC_ENTER (H5O_efl_write, FAIL);
@@ -466,7 +470,8 @@ H5O_efl_write (H5F_t *f, const H5O_efl_t *efl, haddr_t *addr, size_t size,
 
     /* Find the first efl member in which to write */
     for (i=0, cur=0; i<efl->nused; i++) {
-	if (addr->offset < cur+efl->slot[i].size) {
+	if (H5O_EFL_UNLIMITED==efl->slot[i].size ||
+	    addr->offset < cur+efl->slot[i].size) {
 	    skip = addr->offset - cur;
 	    break;
 	}
@@ -479,6 +484,10 @@ H5O_efl_write (H5F_t *f, const H5O_efl_t *efl, haddr_t *addr, size_t size,
 	    HGOTO_ERROR (H5E_EFL, H5E_OVERFLOW, FAIL,
 			 "write past logical end of file");
 	}
+	if (H5F_OVERFLOW_SIZET2OFFT (efl->slot[i].offset+skip)) {
+	    HGOTO_ERROR (H5E_EFL, H5E_OVERFLOW, FAIL,
+			 "external file address overflowed");
+	}
 	if ((fd=open (efl->slot[i].name, O_RDWR))<0) {
 	    if (access (efl->slot[i].name, F_OK)<0) {
 		HGOTO_ERROR (H5E_EFL, H5E_CANTOPENFILE, FAIL,
@@ -488,12 +497,12 @@ H5O_efl_write (H5F_t *f, const H5O_efl_t *efl, haddr_t *addr, size_t size,
 			     "unable to open external raw data file");
 	    }
 	}
-	if (lseek (fd, efl->slot[i].offset+skip, SEEK_SET)<0) {
+	if (lseek (fd, (off_t)(efl->slot[i].offset+skip), SEEK_SET)<0) {
 	    HGOTO_ERROR (H5E_EFL, H5E_SEEKERROR, FAIL,
 			 "unable to seek in external raw data file");
 	}
 	to_write = MIN(efl->slot[i].size-skip, size);
-	if ((n=write (fd, buf, to_write))!=to_write) {
+	if (write (fd, buf, to_write)!=to_write) {
 	    HGOTO_ERROR (H5E_EFL, H5E_READERROR, FAIL,
 			 "write error in external raw data file");
 	} 
diff --git a/src/H5V.c b/src/H5V.c
index e686e29..85d8752 100644
--- a/src/H5V.c
+++ b/src/H5V.c
@@ -166,7 +166,7 @@ H5V_hyper_stride(intn n, const size_t *size,
     for (i = n - 2, acc = 1; i >= 0; --i) {
 	size_t tmp = acc * (total_size[i+1] - size[i+1]);
 	assert (tmp<((size_t)1<<(8*sizeof(ssize_t)-1)));
-	stride[i] = (size_t)tmp; /*overflow checked*/
+	stride[i] = (ssize_t)tmp; /*overflow checked*/
 	acc *= total_size[i+1];
 	skip += acc * (offset ? offset[i] : 0);
     }
diff --git a/src/H5config.h.in b/src/H5config.h.in
index 6f8eef1..ebba10f 100644
--- a/src/H5config.h.in
+++ b/src/H5config.h.in
@@ -43,9 +43,15 @@
 /* The number of bytes in a long long.  */
 #undef SIZEOF_LONG_LONG
 
+/* The number of bytes in a off_t.  */
+#undef SIZEOF_OFF_T
+
 /* The number of bytes in a short.  */
 #undef SIZEOF_SHORT
 
+/* The number of bytes in a size_t.  */
+#undef SIZEOF_SIZE_T
+
 /* Define if you have the fseek64 function.  */
 #undef HAVE_FSEEK64
 
@@ -58,6 +64,9 @@
 /* Define if you have the lseek64 function.  */
 #undef HAVE_LSEEK64
 
+/* Define if you have the <unistd.h> header file.  */
+#undef HAVE_UNISTD_H
+
 /* Define if you have the mpi library (-lmpi).  */
 #undef HAVE_LIBMPI