[svn-r23775] Implement SWMR-5:

Implemented use case 1.9: Appending n-1 dimensional planes

Adding the option "-y" which allows chunks to be thicker
(more than 1 plane). This supports use case 1.9 which have chunks
of multiple planes but writing is still appending one plane at a time.
Using -y with "use_append_chunk", would provide the use case
of writing a plane to a partial chunk.
Using -y with "use_append_mchunks", would provide the use case
of writing a plane to multiple partial chunks.

Also added the option -n which specifies how many planes to write
instead of the defaults of chunksize number of planes. This allows
the final dataset to be other than a cube.

test/use_append_mchunks.c:
test/use_common.c:
test/use.h:
test/use_append_chunk.c:
I also overhauled code in use_append_mchunks and use_append_chunk:
    1. creating a set_parameter() to customize individual use case;
    2. combine the create_uc_file(), read_uc_file() and write_uc_file()
       of both use cases appending one plane per chunks and one plane per
       multiple chunks.  Moved these combined create/read/write functions
       into use_common.c to make future modification and maintence easier.
test/test_usecases.sh.in:
Added a simple test to demonstarte how to use -y to run use case 1.9.

Tested: h5committest, except cmake, passed. Hand tested in Koala for various options.

1 files changed, 25 insertions, 401 deletions

diff --git a/test/use_append_mchunks.c b/test/use_append_mchunks.c
index 85abb74..113f75d 100644
--- a/test/use_append_mchunks.c
+++ b/test/use_append_mchunks.c
@@ -60,407 +60,38 @@
 options_t UC_opts;	/* Use Case Options */
 const char *progname_g="use_append_mchunks";	/* program name */
 
-/* Create the skeleton use case file for testing.
- * It has one 3d dataset using chunked storage.
- * The dataset is (unlimited, 2*chunksize, 2*chunksize).
- * Dataset type is 2 bytes integer.
- * It starts out "empty", i.e., first dimension is 0.
- *
+/* Setup parameters for the use case.
  * Return: 0 succeed; -1 fail.
  */
-int create_uc_file(void)
+int setup_parameters(int argc, char * const argv[])
 {
-    hsize_t dims[3];		/* Dataset starting dimensions */
-    hsize_t max_dims[3];	/* Dataset maximum dimensions */
-    hsize_t chunk_dims[3];	/* Chunk dimensions */
-    hid_t fid;          /* File ID for new HDF5 file */
-    hid_t dcpl;         /* Dataset creation property list */
-    hid_t sid;          /* Dataspace ID */
-    hid_t dsid;         /* Dataset ID */
-
-    /* Create the file */
-    if((fid = H5Fcreate(UC_opts.filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
-        return -1;
-
-    /* Set up dimension sizes */
-    chunk_dims[0] = 1;
-    dims[0] = 0;
-    max_dims[0] = H5S_UNLIMITED;
-    chunk_dims[1] = chunk_dims[2] = UC_opts.chunksize;
-    max_dims[1] = max_dims[2] = dims[1] = dims[2] = 2*UC_opts.chunksize;
-
-    /* Create dataspace for creating datasets */
-    if((sid = H5Screate_simple(3, dims, max_dims)) < 0)
-        return -1;
-
-    /* Create dataset creation property list */
-    if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
-        return -1;
-    if(H5Pset_chunk(dcpl, 3, chunk_dims) < 0)
-        return -1;
-
-    /* create dataset of progname */
-    if((dsid = H5Dcreate2(fid, progname_g, UC_DATATYPE, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
-	return -1;
-
-    /* Close everythign */
-    if(H5Dclose(dsid) < 0)
-	return -1;
-
-    if(H5Sclose(sid) < 0)
-        return -1;
-    if(H5Fclose(fid) < 0)
-        return -1;
-
-    return 0;
-}
-
-
-/* Append planes, each of (1,2*chunksize,2*chunksize) to the dataset.
- * In other words, 4 chunks are appended to the dataset at a time.
- * Fill each plan with the plane number and then write it at the nth plane.
- * Increase the plane number and repeat till the end of dataset, when it
- * reaches chunksize long. End product is a (2*chunksize)^3 cube.
- *
- * Return: 0 succeed; -1 fail.
- */
-int write_uc_file(void)
-{
-    hid_t	fid;          /* File ID for new HDF5 file */
-    hid_t	dsid;         /* dataset ID */
-    hid_t	dcpl;         /* Dataset creation property list */
-    char	*name;
-    UC_CTYPE	*buffer, *bufptr;	/* data buffer */
-    int		cz=UC_opts.chunksize;		/* Chunk size */
-    hid_t	f_sid;	    /* dataset file space id */
-    hid_t	m_sid;	    /* memory space id */
-    int		rank;	    /* rank */
-    hsize_t 	chunk_dims[3];	/* Chunk dimensions */
-    hsize_t	dims[3];    /* Dataspace dimensions */
-    hsize_t	memdims[3]; /* Memory space dimensions */
-    hsize_t	start[3] = {0,0,0}, count[3];    /* Hyperslab selection values */
-    int		i, j, k;
-
-    name = UC_opts.filename;
-
-    /* Open the file */
-    if((fid = H5Fopen(name, H5F_ACC_RDWR | (UC_opts.use_swmr ? H5F_ACC_SWMR_WRITE : 0), H5P_DEFAULT)) < 0){
-	fprintf(stderr, "H5Fopen failed\n");
-        return -1;
-    }
-
-    /* Open the dataset of the program name */
-    if((dsid = H5Dopen2(fid, progname_g, H5P_DEFAULT)) < 0){
-	fprintf(stderr, "H5Dopen2 failed\n");
-	return -1;
-    }
-
-    /* Find chunksize used */
-    if ((dcpl = H5Dget_create_plist(dsid)) < 0){
-	fprintf(stderr, "H5Dget_create_plist failed\n");
-	return -1;
-    }
-    if (H5D_CHUNKED != H5Pget_layout(dcpl)){
-	fprintf(stderr, "storage layout is not chunked\n");
-	return -1;
-    }
-    if ((rank = H5Pget_chunk(dcpl, 3, chunk_dims)) != 3){
-	fprintf(stderr, "storage rank is not 3\n");
-	return -1;
-    }
-
-    /* verify chunk_dims is (1, chunkszie, chunksize) */
-    if (chunk_dims[0]!=1 || chunk_dims[1] != cz || chunk_dims[2] != cz){
-	fprintf(stderr, "chunk size is not as expected. Got dims=(%ld,%ld,%ld)\n",
-	    (long)chunk_dims[0], (long)chunk_dims[1], (long)chunk_dims[2]);
-	return -1;
-    }
-
-    /* allocate space for data buffer 1 X 2*chunksize X 2*chunksize of UC_CTYPE */
-    memdims[0]=1;
-    memdims[1] = memdims[2] = 2*cz;
-    if ((buffer=(UC_CTYPE*)HDmalloc(4*cz*cz*sizeof(UC_CTYPE)))==NULL) {
-	fprintf(stderr, "malloc: failed\n");
-	return -1;
-    };
-
-    /*
-     * Get dataset rank and dimension.
-     */
-    f_sid = H5Dget_space(dsid);    /* Get filespace handle first. */
-    rank  = H5Sget_simple_extent_ndims(f_sid);
-    if (rank != 3){
-	fprintf(stderr, "rank(%d) of dataset does not match\n", rank);
-	return -1;
-    }
-    if (H5Sget_simple_extent_dims(f_sid, dims, NULL) < 0){
-	fprintf(stderr, "H5Sget_simple_extent_dims got error\n");
-	return -1;
-    }
-    printf("dataset rank %d, dimensions %lu x %lu x %lu\n",
-	   rank, (unsigned long)(dims[0]), (unsigned long)(dims[1]), (unsigned long)(dims[2]));
-    /* verify that file space dims are as expected and are consistent with memory space dims */
-    if (dims[0] != 0 || dims[1] != memdims[1] || dims[2] != memdims[2]){
-	fprintf(stderr, "dataset is not empty. Got dims=(%ld,%ld,%ld)\n",
-	    (long)dims[0], (long)dims[1], (long)dims[2]);
-	return -1;
-    }
-    
-    /* setup mem-space for buffer */
-    if ((m_sid=H5Screate_simple(rank, memdims, NULL))<0){
-	fprintf(stderr, "H5Screate_simple for memory failed\n");
-	return -1;
-    };
-
-    /* write planes */
-    count[0]=1;
-    count[1]=count[2]=2*cz;
-    for (i=0; i<UC_opts.nplanes; i++){
-	/* fill buffer with value i+1 */
-	bufptr = buffer;
-	for (j=0; j<2*cz; j++)
-	    for (k=0; k<2*cz; k++)
-		*bufptr++ = i;
-
-	/* extend the dataset by one for new plane */
-	dims[0]=i+1;
-        if(H5Dset_extent(dsid, dims) < 0){
-	    fprintf(stderr, "H5Dset_extent failed\n");
-            return -1;
-	}
-
-        /* Get the dataset's dataspace */
-        if((f_sid = H5Dget_space(dsid)) < 0){
-	    fprintf(stderr, "H5Dset_extent failed\n");
-            return -1;
-	}
-
-	start[0]=i;
-        /* Choose the next plane to write */
-        if(H5Sselect_hyperslab(f_sid, H5S_SELECT_SET, start, NULL, count, NULL) < 0){
-	    fprintf(stderr, "Failed H5Sselect_hyperslab\n");
-            return -1;
-	}
-
-        /* Write plane to the dataset */
-        if(H5Dwrite(dsid, UC_DATATYPE, m_sid, f_sid, H5P_DEFAULT, buffer) < 0){
-	    fprintf(stderr, "Failed H5Dwrite\n");
-            return -1;
-	}
-	/* flush file to make the just written plane available. */
-	if(H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0){
-	    fprintf(stderr, "Failed to H5Fflush file\n");
-	    return -1;
-	}
-    }
-
-    /* Done writing. Free/Close all resources including data file */
-    HDfree(buffer);
-    if (H5Dclose(dsid) < 0){
-	fprintf(stderr, "Failed to close datasete\n");
-	return -1;
-    }
-    if (H5Sclose(m_sid) < 0){
-	fprintf(stderr, "Failed to close memory space\n");
-	return -1;
-    }
-    if (H5Sclose(f_sid) < 0){
-	fprintf(stderr, "Failed to close file space\n");
-	return -1;
-    }
-    if (H5Fclose(fid) < 0){
-	fprintf(stderr, "Failed to close file id\n");
-	return -1;
-    }
-
-    return 0;
-}
-
-
-/* Read planes from the dataset.
- * It expects the dataset is being changed (growing).
- * It checks the unlimited dimension (1st one). When it increases,
- * it will read in the new planes, one by one, and verify the data correctness.
- * (The nth plan should contain all "n".)
- * When the unlimited dimension grows to the chunksize (it becomes a cube),
- * that is the expected end of data, the reader exits.
- *
- * Return: 0 succeed; -1 fail.
- */
-int read_uc_file(void)
-{
-    hid_t	fid;          /* File ID for new HDF5 file */
-    hid_t	dsid;         /* dataset ID */
-    char	*name;
-    UC_CTYPE	*buffer, *bufptr;	/* read data buffer */
-    int		cz=UC_opts.chunksize;		/* Chunk size */
-    hid_t	f_sid;	    /* dataset file space id */
-    hid_t	m_sid;	    /* memory space id */
-    int		rank;	    /* rank */
-    hsize_t	dims[3];    /* Dataspace dimensions */
-    hsize_t	memdims[3]; /* Memory space dimensions */
-    int		nplane=0, nplane_old=0;	/* nth plane, last nth plane */
-    hsize_t	start[3] = {0,0,0}, count[3];    /* Hyperslab selection values */
-    int		j, k;
-    int		nreadererr=0;
-    int		nerrs;
-    int		nonewplane;
-
-    name = UC_opts.filename;
-
-    /* Open the file */
-    if((fid = H5Fopen(name, H5F_ACC_RDONLY | (UC_opts.use_swmr ? H5F_ACC_SWMR_READ : 0), H5P_DEFAULT)) < 0){
-	fprintf(stderr, "H5Fopen failed\n");
-        return -1;
-    }
+    /* use case defaults */
+    HDmemset(&UC_opts, 0, sizeof(options_t));
+    UC_opts.chunksize = Chunksize_DFT;
+    UC_opts.use_swmr = 1;	/* use swmr open */
+    UC_opts.iterations = 1;
+    UC_opts.chunkplanes = 1;
 
-    /* Open the dataset of the program name */
-    if((dsid = H5Dopen2(fid, progname_g, H5P_DEFAULT)) < 0){
-	fprintf(stderr, "H5Dopen2 failed\n");
-	return -1;
+    /* parse options */
+    if (parse_option(argc, argv) < 0){
+	return(-1);
     }
+    /* set chunk dims */
+    UC_opts.chunkdims[0] = UC_opts.chunkplanes;
+    UC_opts.chunkdims[1]=UC_opts.chunkdims[2]=UC_opts.chunksize;
 
-    /* allocate space for data buffer 1 X 2*chunksize X 2*chunksize of UC_CTYPE */
-    memdims[0]=1;
-    memdims[1] = memdims[2] = 2*cz;
-    if ((buffer=(UC_CTYPE*)HDmalloc(4*cz*cz*sizeof(UC_CTYPE)))==NULL) {
-	fprintf(stderr, "malloc: failed\n");
-	return -1;
-    };
+    /* set dataset initial and max dims */
+    UC_opts.dims[0] = 0;
+    UC_opts.max_dims[0] = H5S_UNLIMITED;
+    UC_opts.dims[1] = UC_opts.dims[2] = UC_opts.max_dims[1]=UC_opts.max_dims[2]=2*UC_opts.chunksize;
 
-    /*
-     * Get dataset rank and dimension.
-     * Verify dimension is as expected (unlimited,2*chunksize,2*chunksize).
-     */
-    f_sid = H5Dget_space(dsid);    /* Get filespace handle first. */
-    rank  = H5Sget_simple_extent_ndims(f_sid);
-    if (rank != 3){
-	fprintf(stderr, "rank(%d) of dataset does not match\n", rank);
-	return -1;
-    }
-    if (H5Sget_simple_extent_dims(f_sid, dims, NULL) < 0){
-	fprintf(stderr, "H5Sget_simple_extent_dims got error\n");
-	return -1;
-    }
-    printf("dataset rank %d, dimensions %lu x %lu x %lu\n",
-	   rank, (unsigned long)(dims[0]), (unsigned long)(dims[1]), (unsigned long)(dims[2]));
-    /* verify that file space dims are as expected and are consistent with memory space dims */
-    if (dims[1] != memdims[1] || dims[2] != memdims[2]){
-	fprintf(stderr, "dataset dimension is not as expected. Got dims=(%ld,%ld,%ld)\n",
-	    (long)dims[0], (long)dims[1], (long)dims[2]);
-	fprintf(stderr, "But memdims=(%ld,%ld,%ld)\n",
-	    (long)memdims[0], (long)memdims[1], (long)memdims[2]);
-	return -1;
-    }
-    
-    /* setup mem-space for buffer */
-    if ((m_sid=H5Screate_simple(rank, memdims, NULL))<0){
-	fprintf(stderr, "H5Screate_simple for memory failed\n");
-	return -1;
-    };
+    /* set nplanes */
+    if (UC_opts.nplanes == 0)
+        UC_opts.nplanes = 2*UC_opts.chunksize;
 
-    /* Read 1 plane at a time whenever the dataset grows larger
-     * (along dim[0]) */
-    count[0]=1;
-    count[1]=count[2]=2*cz;
-    /* quit when all nplanes, default cz, have been read */
-    nonewplane=0;
-    while (nplane_old < UC_opts.nplanes ){
-	/* print progress message according to if new planes are availalbe */
-	if (nplane_old < dims[0]) {
-	    if (nonewplane){
-		/* end the previous message */
-		printf("\n");
-		nonewplane=0;
-	    }
-	    printf("reading planes %d to %d\n", nplane_old, (int)dims[0]);
-	}else{
-	    if (nonewplane){
-		printf(".");
-		if (nonewplane>=30){
-		    fprintf(stderr, "waited too long for new plane, quit.\n");
-		    return -1;
-		}
-	    }else{
-		/* print mesg only the first time; dots still no new plane */
-		printf("no new planes to read ");
-	    }
-	    nonewplane++;
-	    /* pause for a second */
-	    sleep(1);
-	}
-	for (nplane=nplane_old; nplane < dims[0]; nplane++){
-	    /* read planes between last old nplanes and current extent */
-	    /* Get the dataset's dataspace */
-	    if((f_sid = H5Dget_space(dsid)) < 0){
-		fprintf(stderr, "H5Dget_space failed\n");
-		return -1;
-	    }
-
-	    start[0]=nplane;
-	    /* Choose the next plane to read */
-	    if(H5Sselect_hyperslab(f_sid, H5S_SELECT_SET, start, NULL, count, NULL) < 0){
-		fprintf(stderr, "H5Sselect_hyperslab failed\n");
-		return -1;
-	    }
-
-	    /* Read the plane from the dataset */
-	    if(H5Dread(dsid, UC_DATATYPE, m_sid, f_sid, H5P_DEFAULT, buffer) < 0){
-		fprintf(stderr, "H5Dread failed\n");
-		return -1;
-	    }
-
-	    /* compare read data with expected data value which is nplane */
-	    bufptr = buffer;
-	    nerrs=0;
-	    for (j=0; j<2*cz; j++){
-		for (k=0; k<2*cz; k++){
-		    if (*bufptr++ != nplane){
-			if (++nerrs < ErrorReportMax){
-			    fprintf(stderr,
-				"found error %d plane(%d,%d), expected %d, got %d\n",
-				nplane, j, k, nplane, (int)*(bufptr-1));
-			}
-		    }
-		}
-	    }
-	    if (nerrs){
-		nreadererr++;
-		fprintf(stderr, "found %d unexpected values in plane %d\n", nerrs, nplane);
-	    }
-	}
-	/* Have read all current planes */
-	nplane_old=dims[0];
-
-	/* check if dataset has grown since last time */
-	/* close dsid and file, then reopen them */
-	if (H5Dclose(dsid) < 0){
-	    fprintf(stderr, "H5Dclose failed\n");
-	    return -1;
-	}
-	if (H5Fclose(fid) < 0){
-	    fprintf(stderr, "H5Fclose failed\n");
-	    return -1;
-	}
-	if((fid = H5Fopen(name, H5F_ACC_RDONLY | (UC_opts.use_swmr ? H5F_ACC_SWMR_READ : 0), H5P_DEFAULT)) < 0){
-	    fprintf(stderr, "H5Fopen failed\n");
-	    return -1;
-	}
-	if((dsid = H5Dopen2(fid, progname_g, H5P_DEFAULT)) < 0){
-	    fprintf(stderr, "H5Dopen2 failed\n");
-	    return -1;
-	}
-	f_sid = H5Dget_space(dsid);    /* Get filespace handle first. */
-	if (H5Sget_simple_extent_dims(f_sid, dims, NULL) < 0){
-	    fprintf(stderr, "H5Sget_simple_extent_dims got error\n");
-	    return -1;
-	}
-    }
-
-    if (nreadererr)
-	return -1;
-    else
-	return 0;
+    /* show parameters and return */
+    show_parameters();
+    return(0);
 }
 
 
@@ -482,14 +113,7 @@ main(int argc, char *argv[])
     int child_ret_value;
 
     /* initialization */
-    /* use case defaults */
-    HDmemset(&UC_opts, 0, sizeof(options_t));
-    UC_opts.h5_use_chunks = 1;	/* use chunked datasets */
-    UC_opts.chunksize = Chunksize_DFT;
-    UC_opts.use_swmr = 1;	/* use swmr open */
-
-    /* parse options */
-    if (parse_option(argc, argv) < 0){
+    if (setup_parameters(argc, argv) < 0){
 	Hgoto_error(1);
     }