[svn-r5875] Purpose:

    Additional regression tests & bug fixes

Description:
    There was no testing for the H5Dget_storage_size function and it seemed to
    be having problems with compressed, chunked datasets, so write some tests
    to verify that its working correctly.

    Also, fix case for allocating storage early for chunked datasets

Platforms tested:
    FreeBSD 4.6 (sleipnir) serial & parallel

4 files changed, 120 insertions, 67 deletions

diff --git a/src/H5D.c b/src/H5D.c
index d4f28db..b033e48 100644
--- a/src/H5D.c
+++ b/src/H5D.c
@@ -3196,10 +3196,12 @@ H5D_init_storage(H5D_t *dset, const H5S_t *space)
     hsize_t		addr;           /* Offset in dataset */
     void		*buf = NULL;    /* Buffer for fill value writing */
     H5O_fill_t          fill;           /* Fill value information */
+    H5D_space_time_t    space_time;     /* When to allocate space */
     H5P_genplist_t     *plist;          /* Property list */
     herr_t		ret_value = SUCCEED;    /* Return value */
 
     FUNC_ENTER_NOINIT(H5D_init_storage);
+
     assert(dset);
     assert(space);
 
@@ -3208,6 +3210,8 @@ H5D_init_storage(H5D_t *dset, const H5S_t *space)
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset creation property list");
     if(H5P_get(plist, H5D_CRT_FILL_VALUE_NAME, &fill) < 0)
         HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get fill value");
+    if(H5P_get(plist, H5D_CRT_SPACE_TIME_NAME, &space_time) < 0)
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't retrieve space allocation time");
 
     switch (dset->layout.type) {
         case H5D_CONTIGUOUS:
@@ -3252,12 +3256,15 @@ H5D_init_storage(H5D_t *dset, const H5S_t *space)
             break;
 
         case H5D_CHUNKED:
-#ifdef H5_HAVE_PARALLEL
             /*
              * If the dataset is accessed via parallel I/O, allocate file space
              * for all chunks now and initialize each chunk with the fill value.
              */
-            if (IS_H5FD_MPIO(dset->ent.file) || IS_H5FD_MPIPOSIX(dset->ent.file)) {
+            if (space_time==H5D_SPACE_ALLOC_EARLY
+#ifdef H5_HAVE_PARALLEL
+                || (IS_H5FD_MPIO(dset->ent.file) || IS_H5FD_MPIPOSIX(dset->ent.file))
+#endif /*H5_HAVE_PARALLEL*/
+                ) {
                 /* We only handle simple data spaces so far */
                 int		ndims;
                 hsize_t		dim[H5O_LAYOUT_NDIMS];
@@ -3271,13 +3278,13 @@ H5D_init_storage(H5D_t *dset, const H5S_t *space)
                         &(dset->layout), dim, plist)<0)
                     HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to allocate all chunks of dataset");
             } /* end if */
-#endif /*H5_HAVE_PARALLEL*/
             break;
     } /* end switch */
 
 done:
     if (buf)
         H5FL_BLK_FREE(type_conv,buf);
+
     FUNC_LEAVE(ret_value);
 }
 
diff --git a/src/H5Distore.c b/src/H5Distore.c
index 2c6a403..c9dc766 100644
--- a/src/H5Distore.c
+++ b/src/H5Distore.c
@@ -2320,7 +2320,6 @@ done:
  *              that they should never be written.
  *-------------------------------------------------------------------------
  */
-#ifdef H5_HAVE_PARALLEL
 herr_t
 H5F_istore_allocate(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout,
 		    const hsize_t *space_dim, H5P_genplist_t *dc_plist)
@@ -2334,11 +2333,14 @@ H5F_istore_allocate(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout,
     void *chunk=NULL;           /* Chunk buffer for writing fill values */
     H5P_genplist_t *dx_plist;   /* Data xfer property list */
     double	split_ratios[3];/* B-tree node splitting ratios		*/
-    int         mpi_rank;       /* This process's rank  */
-    int         mpi_size;       /* Total # of processes */
+#ifdef H5_HAVE_PARALLEL
+    int         mpi_rank=(-1);  /* This process's rank  */
+    int         mpi_size=(-1);  /* Total # of processes */
     int         mpi_round=0;    /* Current process responsible for I/O */
     int         mpi_code;       /* MPI return code */
     unsigned    blocks_written=0; /* Flag to indicate that chunk was actually written */
+    unsigned    using_mpi=0;    /* Flag to indicate that the file is being accessed with an MPI-capable file driver */
+#endif /* H5_HAVE_PARALLEL */
     int		carry;          /* Flag to indicate that chunk increment carrys to higher dimension (sorta) */
     int		i;              /* Local index variable */
     unsigned	u;              /* Local index variable */
@@ -2369,9 +2371,36 @@ H5F_istore_allocate(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout,
     if(H5P_get(dx_plist,H5D_XFER_BTREE_SPLIT_RATIO_NAME,split_ratios)<0)
         HGOTO_ERROR(H5E_STORAGE, H5E_CANTGET, FAIL, "can't get B-tree split ratios");
 
+#ifdef H5_HAVE_PARALLEL
+    /* Retrieve up MPI parameters */
+    if(IS_H5FD_MPIO(f)) {
+        if ((mpi_rank=H5FD_mpio_mpi_rank(f->shared->lf))<0)
+            HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Can't retrieve MPI rank");
+        if ((mpi_size=H5FD_mpio_mpi_size(f->shared->lf))<0)
+            HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Can't retrieve MPI size");
+
+        /* Set the MPI-capable file driver flag */
+        using_mpi=1;
+    } /* end if */
+    else {
+        if(IS_H5FD_MPIPOSIX(f)) {
+            /* Get the MPI rank & size */
+            if ((mpi_rank=H5FD_mpiposix_mpi_rank(f->shared->lf))<0)
+                HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Can't retrieve MPI rank");
+            if ((mpi_size=H5FD_mpiposix_mpi_size(f->shared->lf))<0)
+                HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Can't retrieve MPI size");
+
+            /* Set the MPI-capable file driver flag */
+            using_mpi=1;
+        } /* end if */
+    } /* end else */
+#endif /* H5_HAVE_PARALLEL */
+
+#ifdef H5_HAVE_PARALLEL
     /* Can't use data I/O pipeline in parallel (yet) */
-    if (pline.nfilters>0)
+    if (using_mpi && pline.nfilters>0)
         HGOTO_ERROR(H5E_STORAGE, H5E_UNSUPPORTED, FAIL, "can't use data pipeline in parallel");
+#endif /* H5_HAVE_PARALLEL */
 
     /*
      * Setup indice to go through all chunks. (Future improvement
@@ -2404,24 +2433,6 @@ H5F_istore_allocate(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout,
         } /* end else */
     } /* end if */
 
-    /* Retrieve up MPI parameters */
-    if(IS_H5FD_MPIO(f)) {
-        if ((mpi_rank=H5FD_mpio_mpi_rank(f->shared->lf))<0)
-            HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Can't retrieve MPI rank");
-        if ((mpi_size=H5FD_mpio_mpi_size(f->shared->lf))<0)
-            HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Can't retrieve MPI size");
-    } /* end if */
-    else {
-        /* Sanity Check */
-        assert(IS_H5FD_MPIPOSIX(f));
-
-        /* Get the MPI rank & size */
-        if ((mpi_rank=H5FD_mpiposix_mpi_rank(f->shared->lf))<0)
-            HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Can't retrieve MPI rank");
-        if ((mpi_size=H5FD_mpiposix_mpi_size(f->shared->lf))<0)
-            HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Can't retrieve MPI size");
-    } /* end else */
-
     /* Loop over all chunks */
     carry=0;
     while (carry==0) {
@@ -2441,15 +2452,26 @@ H5F_istore_allocate(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout,
 
             /* Check if fill values should be written to blocks */
             if(fill_time != H5D_FILL_TIME_NEVER) {
-                /* Round-robin write the chunks out from only one process */
-                if(mpi_round==mpi_rank) {
+#ifdef H5_HAVE_PARALLEL
+                /* Check if this file is accessed with an MPI-capable file driver */
+                if(using_mpi) {
+                    /* Round-robin write the chunks out from only one process */
+                    if(mpi_round==mpi_rank) {
+                        if (H5F_block_write(f, H5FD_MEM_DRAW, udata.addr, udata.key.nbytes, dxpl_id, chunk)<0)
+                            HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to write raw data to file");
+                    } /* end if */
+                    mpi_round=(++mpi_round)%mpi_size;
+
+                    /* Indicate that blocks are being written */
+                    blocks_written=1;
+                } /* end if */
+                else {
+#endif /* H5_HAVE_PARALLEL */
                     if (H5F_block_write(f, H5FD_MEM_DRAW, udata.addr, udata.key.nbytes, dxpl_id, chunk)<0)
                         HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to write raw data to file");
-                } /* end if */
-                mpi_round=(++mpi_round)%mpi_size;
-
-                /* Indicate that blocks are being written */
-                blocks_written=1;
+#ifdef H5_HAVE_PARALLEL
+                } /* end else */
+#endif /* H5_HAVE_PARALLEL */
             } /* end if */
         } /* end if */
 	
@@ -2463,8 +2485,10 @@ H5F_istore_allocate(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout,
         } /* end for */
     } /* end while */
 
+#ifdef H5_HAVE_PARALLEL
     /* Only need to block at the barrier if we actually allocated a chunk */
-    if(blocks_written) {
+    /* And if we are using an MPI-capable file driver */
+    if(using_mpi && blocks_written) {
         /* Wait at barrier to avoid race conditions where some processes are
          * still writing out chunks and other processes race ahead to read
          * them in, getting bogus data.
@@ -2481,6 +2505,7 @@ H5F_istore_allocate(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout,
                 HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mpi_code);
         } /* end else */
     } /* end if */
+#endif /* H5_HAVE_PARALLEL */
 
 done:
     /* Free the chunk for fill values */
@@ -2489,7 +2514,6 @@ done:
 
     FUNC_LEAVE(ret_value);
 }
-#endif /* H5_HAVE_PARALLEL */
 
 
 /*-------------------------------------------------------------------------
diff --git a/src/H5Fistore.c b/src/H5Fistore.c
index 2c6a403..c9dc766 100644
--- a/src/H5Fistore.c
+++ b/src/H5Fistore.c
@@ -2320,7 +2320,6 @@ done:
  *              that they should never be written.
  *-------------------------------------------------------------------------
  */
-#ifdef H5_HAVE_PARALLEL
 herr_t
 H5F_istore_allocate(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout,
 		    const hsize_t *space_dim, H5P_genplist_t *dc_plist)
@@ -2334,11 +2333,14 @@ H5F_istore_allocate(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout,
     void *chunk=NULL;           /* Chunk buffer for writing fill values */
     H5P_genplist_t *dx_plist;   /* Data xfer property list */
     double	split_ratios[3];/* B-tree node splitting ratios		*/
-    int         mpi_rank;       /* This process's rank  */
-    int         mpi_size;       /* Total # of processes */
+#ifdef H5_HAVE_PARALLEL
+    int         mpi_rank=(-1);  /* This process's rank  */
+    int         mpi_size=(-1);  /* Total # of processes */
     int         mpi_round=0;    /* Current process responsible for I/O */
     int         mpi_code;       /* MPI return code */
     unsigned    blocks_written=0; /* Flag to indicate that chunk was actually written */
+    unsigned    using_mpi=0;    /* Flag to indicate that the file is being accessed with an MPI-capable file driver */
+#endif /* H5_HAVE_PARALLEL */
     int		carry;          /* Flag to indicate that chunk increment carrys to higher dimension (sorta) */
     int		i;              /* Local index variable */
     unsigned	u;              /* Local index variable */
@@ -2369,9 +2371,36 @@ H5F_istore_allocate(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout,
     if(H5P_get(dx_plist,H5D_XFER_BTREE_SPLIT_RATIO_NAME,split_ratios)<0)
         HGOTO_ERROR(H5E_STORAGE, H5E_CANTGET, FAIL, "can't get B-tree split ratios");
 
+#ifdef H5_HAVE_PARALLEL
+    /* Retrieve up MPI parameters */
+    if(IS_H5FD_MPIO(f)) {
+        if ((mpi_rank=H5FD_mpio_mpi_rank(f->shared->lf))<0)
+            HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Can't retrieve MPI rank");
+        if ((mpi_size=H5FD_mpio_mpi_size(f->shared->lf))<0)
+            HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Can't retrieve MPI size");
+
+        /* Set the MPI-capable file driver flag */
+        using_mpi=1;
+    } /* end if */
+    else {
+        if(IS_H5FD_MPIPOSIX(f)) {
+            /* Get the MPI rank & size */
+            if ((mpi_rank=H5FD_mpiposix_mpi_rank(f->shared->lf))<0)
+                HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Can't retrieve MPI rank");
+            if ((mpi_size=H5FD_mpiposix_mpi_size(f->shared->lf))<0)
+                HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Can't retrieve MPI size");
+
+            /* Set the MPI-capable file driver flag */
+            using_mpi=1;
+        } /* end if */
+    } /* end else */
+#endif /* H5_HAVE_PARALLEL */
+
+#ifdef H5_HAVE_PARALLEL
     /* Can't use data I/O pipeline in parallel (yet) */
-    if (pline.nfilters>0)
+    if (using_mpi && pline.nfilters>0)
         HGOTO_ERROR(H5E_STORAGE, H5E_UNSUPPORTED, FAIL, "can't use data pipeline in parallel");
+#endif /* H5_HAVE_PARALLEL */
 
     /*
      * Setup indice to go through all chunks. (Future improvement
@@ -2404,24 +2433,6 @@ H5F_istore_allocate(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout,
         } /* end else */
     } /* end if */
 
-    /* Retrieve up MPI parameters */
-    if(IS_H5FD_MPIO(f)) {
-        if ((mpi_rank=H5FD_mpio_mpi_rank(f->shared->lf))<0)
-            HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Can't retrieve MPI rank");
-        if ((mpi_size=H5FD_mpio_mpi_size(f->shared->lf))<0)
-            HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Can't retrieve MPI size");
-    } /* end if */
-    else {
-        /* Sanity Check */
-        assert(IS_H5FD_MPIPOSIX(f));
-
-        /* Get the MPI rank & size */
-        if ((mpi_rank=H5FD_mpiposix_mpi_rank(f->shared->lf))<0)
-            HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Can't retrieve MPI rank");
-        if ((mpi_size=H5FD_mpiposix_mpi_size(f->shared->lf))<0)
-            HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Can't retrieve MPI size");
-    } /* end else */
-
     /* Loop over all chunks */
     carry=0;
     while (carry==0) {
@@ -2441,15 +2452,26 @@ H5F_istore_allocate(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout,
 
             /* Check if fill values should be written to blocks */
             if(fill_time != H5D_FILL_TIME_NEVER) {
-                /* Round-robin write the chunks out from only one process */
-                if(mpi_round==mpi_rank) {
+#ifdef H5_HAVE_PARALLEL
+                /* Check if this file is accessed with an MPI-capable file driver */
+                if(using_mpi) {
+                    /* Round-robin write the chunks out from only one process */
+                    if(mpi_round==mpi_rank) {
+                        if (H5F_block_write(f, H5FD_MEM_DRAW, udata.addr, udata.key.nbytes, dxpl_id, chunk)<0)
+                            HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to write raw data to file");
+                    } /* end if */
+                    mpi_round=(++mpi_round)%mpi_size;
+
+                    /* Indicate that blocks are being written */
+                    blocks_written=1;
+                } /* end if */
+                else {
+#endif /* H5_HAVE_PARALLEL */
                     if (H5F_block_write(f, H5FD_MEM_DRAW, udata.addr, udata.key.nbytes, dxpl_id, chunk)<0)
                         HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to write raw data to file");
-                } /* end if */
-                mpi_round=(++mpi_round)%mpi_size;
-
-                /* Indicate that blocks are being written */
-                blocks_written=1;
+#ifdef H5_HAVE_PARALLEL
+                } /* end else */
+#endif /* H5_HAVE_PARALLEL */
             } /* end if */
         } /* end if */
 	
@@ -2463,8 +2485,10 @@ H5F_istore_allocate(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout,
         } /* end for */
     } /* end while */
 
+#ifdef H5_HAVE_PARALLEL
     /* Only need to block at the barrier if we actually allocated a chunk */
-    if(blocks_written) {
+    /* And if we are using an MPI-capable file driver */
+    if(using_mpi && blocks_written) {
         /* Wait at barrier to avoid race conditions where some processes are
          * still writing out chunks and other processes race ahead to read
          * them in, getting bogus data.
@@ -2481,6 +2505,7 @@ H5F_istore_allocate(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout,
                 HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mpi_code);
         } /* end else */
     } /* end if */
+#endif /* H5_HAVE_PARALLEL */
 
 done:
     /* Free the chunk for fill values */
@@ -2489,7 +2514,6 @@ done:
 
     FUNC_LEAVE(ret_value);
 }
-#endif /* H5_HAVE_PARALLEL */
 
 
 /*-------------------------------------------------------------------------
diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h
index a4a2a3d..dd7cbd9 100644
--- a/src/H5Fprivate.h
+++ b/src/H5Fprivate.h
@@ -391,12 +391,10 @@ __DLL__ herr_t H5F_seq_writev(H5F_t *f, hid_t dxpl_id,
 
 
 /* Functions that operate on indexed storage */
-#ifdef H5_HAVE_PARALLEL
 __DLL__ herr_t H5F_istore_allocate (H5F_t *f, hid_t dxpl_id,
                                 const struct H5O_layout_t *layout,
                                 const hsize_t *space_dim,
                                 struct H5P_genplist_t *dc_plist);
-#endif /* H5_HAVE_PARALLEL */
 __DLL__ hsize_t H5F_istore_allocated(H5F_t *f, unsigned ndims, haddr_t addr);
 __DLL__ herr_t H5F_istore_dump_btree(H5F_t *f, FILE *stream, unsigned ndims,
         haddr_t addr);