diff options
author | Albert Cheng <acheng@hdfgroup.org> | 2013-06-14 21:55:01 (GMT) |
---|---|---|
committer | Albert Cheng <acheng@hdfgroup.org> | 2013-06-14 21:55:01 (GMT) |
commit | b3526d3b90565e7224d84ada66d55a3d27c76866 (patch) | |
tree | 35dc00d67e0615ed893ddacff4752ff95fe574ae | |
parent | 6e46183e3580866e77bccc2021707ac40d95fa52 (diff) | |
download | hdf5-b3526d3b90565e7224d84ada66d55a3d27c76866.zip hdf5-b3526d3b90565e7224d84ada66d55a3d27c76866.tar.gz hdf5-b3526d3b90565e7224d84ada66d55a3d27c76866.tar.bz2 |
[svn-r23775] Implement SWMR-5:
Implemented use case 1.9: Appending n-1 dimensional planes
Adding the option "-y" which allows chunks to be thicker
(more than 1 plane). This supports use case 1.9 which have chunks
of multiple planes but writing is still appending one plane at a time.
Using -y with "use_append_chunk", would provide the use case
of writing a plane to a partial chunk.
Using -y with "use_append_mchunks", would provide the use case
of writing a plane to multiple partial chunks.
Also added the option -n which specifies how many planes to write
instead of the defaults of chunksize number of planes. This allows
the final dataset to be other than a cube.
test/use_append_mchunks.c:
test/use_common.c:
test/use.h:
test/use_append_chunk.c:
I also overhauled code in use_append_mchunks and use_append_chunk:
1. creating a set_parameter() to customize individual use case;
2. combine the create_uc_file(), read_uc_file() and write_uc_file()
of both use cases appending one plane per chunks and one plane per
multiple chunks. Moved these combined create/read/write functions
into use_common.c to make future modification and maintence easier.
test/test_usecases.sh.in:
Added a simple test to demonstarte how to use -y to run use case 1.9.
Tested: h5committest, except cmake, passed. Hand tested in Koala for various options.
-rw-r--r-- | test/test_usecases.sh.in | 4 | ||||
-rw-r--r-- | test/use.h | 14 | ||||
-rw-r--r-- | test/use_append_chunk.c | 402 | ||||
-rw-r--r-- | test/use_append_mchunks.c | 426 | ||||
-rw-r--r-- | test/use_common.c | 458 |
5 files changed, 516 insertions, 788 deletions
diff --git a/test/test_usecases.sh.in b/test/test_usecases.sh.in index 771be71..18b1554 100644 --- a/test/test_usecases.sh.in +++ b/test/test_usecases.sh.in @@ -101,11 +101,13 @@ TOOLTEST() { # main body for p in $USECASES_PROGRAMS; do TOOLTEST $p - TOOLTEST $p -z 512 + TOOLTEST $p -z 256 tmpfile=/tmp/datatfile.$$ TOOLTEST $p -f $tmpfile; rm -f $tmpfile TOOLTEST $p -l w TOOLTEST $p -l r + # use case 1.9, testing with multi-planes chunks + TOOLTEST $p -z 256 -y 5 # 5 planes chunks # cleanup temp datafile if test -z "$HDF5_NOCLEANUP"; then rm -f $p.h5 @@ -32,6 +32,7 @@ /* these two definitions must match each other */ #define UC_DATATYPE H5T_NATIVE_SHORT /* use case HDF5 data type */ #define UC_CTYPE short /* use case C data type */ +#define UC_RANK 3 /* use case dataset rank */ /* type declarations */ typedef enum part_t { @@ -40,13 +41,16 @@ typedef enum part_t { UC_READER /* reader only */ } part_t; typedef struct options_t { - int use_swmr; /* use swmr open (1) or not */ - int compress; /* 0: no compress */ - int h5_use_chunks; /* 0/1: Not use/use chunked dataset */ int chunksize; /* chunks are chunksize^2 planes */ - int nplanes; /* number of planes, default chunksize */ + int chunkplanes; /* number of planes per chunk, default 1*/ + hsize_t chunkdims[UC_RANK]; /* chunk dims is (chunkplan, chunksize, chunksize) */ + hsize_t dims[UC_RANK]; /* dataset initial dims */ + hsize_t max_dims[UC_RANK]; /* dataset max dims */ + int nplanes; /* number of planes to write, default proportional to chunksize */ char *filename; /* use case data filename */ part_t launch; /* launch writer, reader or both */ + int use_swmr; /* use swmr open (1) or not */ + int iterations; /* iterations, default 1 */ } options_t; /* global variables declarations */ @@ -55,6 +59,8 @@ extern const char *progname_g; /* Program name */ /* prototype declarations */ int parse_option(int argc, char * const argv[]); +int setup_parameters(int argc, char * const argv[]); +void show_parameters(void); void usage(const char *prog); int create_uc_file(void); int write_uc_file(void); diff --git a/test/use_append_chunk.c b/test/use_append_chunk.c index d4de8da..2ec645e 100644 --- a/test/use_append_chunk.c +++ b/test/use_append_chunk.c @@ -67,383 +67,38 @@ options_t UC_opts; /* Use Case Options */ const char *progname_g="use_append_chunk"; /* program name */ -/* Create the skeleton use case file for testing. - * It has one 3d dataset using chunked storage. - * The dataset is (unlimited, chunksize, chunksize). - * Dataset type is 2 bytes integer. - * It starts out "empty", i.e., first dimension is 0. - * +/* Setup parameters for the use case. * Return: 0 succeed; -1 fail. */ -int create_uc_file(void) +int setup_parameters(int argc, char * const argv[]) { - hsize_t dims[3]; /* Dataset starting dimensions */ - hsize_t max_dims[3]; /* Dataset maximum dimensions */ - hsize_t chunk_dims[3]; /* Chunk dimensions */ - hid_t fid; /* File ID for new HDF5 file */ - hid_t dcpl; /* Dataset creation property list */ - hid_t sid; /* Dataspace ID */ - hid_t dsid; /* Dataset ID */ - - /* Create the file */ - if((fid = H5Fcreate(UC_opts.filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) - return -1; - - /* Set up dimension sizes */ - chunk_dims[0] = 1; - dims[0] = 0; - max_dims[0] = H5S_UNLIMITED; - max_dims[1] = max_dims[2] = dims[1] = dims[2] = chunk_dims[1] = chunk_dims[2] = UC_opts.chunksize; - - /* Create dataspace for creating datasets */ - if((sid = H5Screate_simple(3, dims, max_dims)) < 0) - return -1; - - /* Create dataset creation property list */ - if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) - return -1; - if(H5Pset_chunk(dcpl, 3, chunk_dims) < 0) - return -1; - - /* create dataset of progname */ - if((dsid = H5Dcreate2(fid, progname_g, UC_DATATYPE, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) - return -1; - - /* Close everythign */ - if(H5Dclose(dsid) < 0) - return -1; - - if(H5Sclose(sid) < 0) - return -1; - if(H5Fclose(fid) < 0) - return -1; - - return 0; -} - - -/* Append planes, each of (1,chunksize,chunksize) to the dataset. - * Fill each plan with the plane number and then write it at the nth plane. - * Increase the plane number and repeat till the end of dataset, when it - * reaches chunksize long. End product is a chunksize^3 cube. - * - * Return: 0 succeed; -1 fail. - */ -int write_uc_file(void) -{ - hid_t fid; /* File ID for new HDF5 file */ - hid_t dsid; /* dataset ID */ - char *name; - UC_CTYPE *buffer, *bufptr; /* data buffer */ - int cz=UC_opts.chunksize; /* Chunk size */ - hid_t f_sid; /* dataset file space id */ - hid_t m_sid; /* memory space id */ - int rank; /* rank */ - hsize_t dims[3]; /* Dataspace dimensions */ - hsize_t memdims[3]; /* Memory space dimensions */ - hsize_t start[3] = {0,0,0}, count[3]; /* Hyperslab selection values */ - int i, j, k; - - name = UC_opts.filename; - - /* Open the file */ - if((fid = H5Fopen(name, H5F_ACC_RDWR | (UC_opts.use_swmr ? H5F_ACC_SWMR_WRITE : 0), H5P_DEFAULT)) < 0){ - fprintf(stderr, "H5Fopen failed\n"); - return -1; - } - - /* Open the dataset of the program name */ - if((dsid = H5Dopen2(fid, progname_g, H5P_DEFAULT)) < 0){ - fprintf(stderr, "H5Dopen2 failed\n"); - return -1; - } - - /* allocate space for data buffer chunksize X chunksize of UC_CTYPE */ - memdims[0]=1; - memdims[1] = memdims[2] = cz; - if ((buffer=(UC_CTYPE*)HDmalloc(cz*cz*sizeof(UC_CTYPE)))==NULL) { - fprintf(stderr, "malloc: failed\n"); - return -1; - }; - - /* - * Get dataset rank and dimension. - */ - f_sid = H5Dget_space(dsid); /* Get filespace handle first. */ - rank = H5Sget_simple_extent_ndims(f_sid); - if (rank != 3){ - fprintf(stderr, "rank(%d) of dataset does not match\n", rank); - return -1; - } - if (H5Sget_simple_extent_dims(f_sid, dims, NULL) < 0){ - fprintf(stderr, "H5Sget_simple_extent_dims got error\n"); - return -1; - } - printf("dataset rank %d, dimensions %lu x %lu x %lu\n", - rank, (unsigned long)(dims[0]), (unsigned long)(dims[1]), (unsigned long)(dims[2])); - /* verify that file space dims are as expected and are consistent with memory space dims */ - if (dims[0] != 0 || dims[1] != memdims[1] || dims[2] != memdims[2]){ - fprintf(stderr, "dataset is not empty. Got dims=(%ld,%ld,%ld)\n", - (long)dims[0], (long)dims[1], (long)dims[2]); - return -1; - } - - /* setup mem-space for buffer */ - if ((m_sid=H5Screate_simple(rank, memdims, NULL))<0){ - fprintf(stderr, "H5Screate_simple for memory failed\n"); - return -1; - }; - - /* write planes */ - count[0]=1; - count[1]=count[2]=cz; - for (i=0; i<UC_opts.nplanes; i++){ - /* fill buffer with value i+1 */ - bufptr = buffer; - for (j=0; j<cz; j++) - for (k=0; k<cz; k++) - *bufptr++ = i; - - /* extend the dataset by one for new plane */ - dims[0]=i+1; - if(H5Dset_extent(dsid, dims) < 0){ - fprintf(stderr, "H5Dset_extent failed\n"); - return -1; - } - - /* Get the dataset's dataspace */ - if((f_sid = H5Dget_space(dsid)) < 0){ - fprintf(stderr, "H5Dset_extent failed\n"); - return -1; - } - - start[0]=i; - /* Choose the next plane to write */ - if(H5Sselect_hyperslab(f_sid, H5S_SELECT_SET, start, NULL, count, NULL) < 0){ - fprintf(stderr, "Failed H5Sselect_hyperslab\n"); - return -1; - } - - /* Write plane to the dataset */ - if(H5Dwrite(dsid, UC_DATATYPE, m_sid, f_sid, H5P_DEFAULT, buffer) < 0){ - fprintf(stderr, "Failed H5Dwrite\n"); - return -1; - } - /* flush file to make the just written plane available. */ - if(H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0){ - fprintf(stderr, "Failed to H5Fflush file\n"); - return -1; - } - } - - /* Done writing. Free/Close all resources including data file */ - HDfree(buffer); - if (H5Dclose(dsid) < 0){ - fprintf(stderr, "Failed to close datasete\n"); - return -1; - } - if (H5Sclose(m_sid) < 0){ - fprintf(stderr, "Failed to close memory space\n"); - return -1; - } - if (H5Sclose(f_sid) < 0){ - fprintf(stderr, "Failed to close file space\n"); - return -1; - } - if (H5Fclose(fid) < 0){ - fprintf(stderr, "Failed to close file id\n"); - return -1; - } - - return 0; -} - - -/* Read planes from the dataset. - * It expects the dataset is being changed (growing). - * It checks the unlimited dimension (1st one). When it increases, - * it will read in the new planes, one by one, and verify the data correctness. - * (The nth plan should contain all "n".) - * When the unlimited dimension grows to the chunksize (it becomes a cube), - * that is the expected end of data, the reader exits. - * - * Return: 0 succeed; -1 fail. - */ -int read_uc_file(void) -{ - hid_t fid; /* File ID for new HDF5 file */ - hid_t dsid; /* dataset ID */ - char *name; - UC_CTYPE *buffer, *bufptr; /* read data buffer */ - int cz=UC_opts.chunksize; /* Chunk size */ - hid_t f_sid; /* dataset file space id */ - hid_t m_sid; /* memory space id */ - int rank; /* rank */ - hsize_t dims[3]; /* Dataspace dimensions */ - hsize_t memdims[3]; /* Memory space dimensions */ - int nplane=0, nplane_old=0; /* nth plane, last nth plane */ - hsize_t start[3] = {0,0,0}, count[3]; /* Hyperslab selection values */ - int j, k; - int nreadererr=0; - int nerrs; - int nonewplane; - - name = UC_opts.filename; - - /* Open the file */ - if((fid = H5Fopen(name, H5F_ACC_RDONLY | (UC_opts.use_swmr ? H5F_ACC_SWMR_READ : 0), H5P_DEFAULT)) < 0){ - fprintf(stderr, "H5Fopen failed\n"); - return -1; - } - - /* Open the dataset of the program name */ - if((dsid = H5Dopen2(fid, progname_g, H5P_DEFAULT)) < 0){ - fprintf(stderr, "H5Dopen2 failed\n"); - return -1; - } - - /* allocate space for data buffer chunksize X chunksize of UC_CTYPE */ - /*cz = UC_opts.chunksize;*/ - memdims[0]=1; - memdims[1] = memdims[2] = cz; - if ((buffer=(UC_CTYPE*)HDmalloc(cz*cz*sizeof(UC_CTYPE)))==NULL) { - fprintf(stderr, "malloc: failed\n"); - return -1; - }; + /* use case defaults */ + HDmemset(&UC_opts, 0, sizeof(options_t)); + UC_opts.chunksize = Chunksize_DFT; + UC_opts.use_swmr = 1; /* use swmr open */ + UC_opts.iterations = 1; + UC_opts.chunkplanes = 1; - /* - * Get dataset rank and dimension. - * Verify dimension is as expected (unlimited,chunksize,chunksize). - */ - f_sid = H5Dget_space(dsid); /* Get filespace handle first. */ - rank = H5Sget_simple_extent_ndims(f_sid); - if (rank != 3){ - fprintf(stderr, "rank(%d) of dataset does not match\n", rank); - return -1; - } - if (H5Sget_simple_extent_dims(f_sid, dims, NULL) < 0){ - fprintf(stderr, "H5Sget_simple_extent_dims got error\n"); - return -1; - } - printf("dataset rank %d, dimensions %lu x %lu x %lu\n", - rank, (unsigned long)(dims[0]), (unsigned long)(dims[1]), (unsigned long)(dims[2])); - /* verify that file space dims are as expected and are consistent with memory space dims */ - if (dims[1] != memdims[1] || dims[2] != memdims[2]){ - fprintf(stderr, "dataset dimension is not as expected. Got dims=(%ld,%ld,%ld)\n", - (long)dims[0], (long)dims[1], (long)dims[2]); - fprintf(stderr, "But memdims=(%ld,%ld,%ld)\n", - (long)memdims[0], (long)memdims[1], (long)memdims[2]); - return -1; + /* parse options */ + if (parse_option(argc, argv) < 0){ + return(-1); } - - /* setup mem-space for buffer */ - if ((m_sid=H5Screate_simple(rank, memdims, NULL))<0){ - fprintf(stderr, "H5Screate_simple for memory failed\n"); - return -1; - }; - - /* Read 1 plane at a time whenever the dataset grows larger - * (along dim[0]) */ - count[0]=1; - count[1]=count[2]=cz; - /* quit when all nplanes, default cz, have been read */ - nonewplane=0; - while (nplane_old < UC_opts.nplanes ){ - /* print progress message according to if new planes are availalbe */ - if (nplane_old < dims[0]) { - if (nonewplane){ - /* end the previous message */ - printf("\n"); - nonewplane=0; - } - printf("reading planes %d to %d\n", nplane_old, (int)dims[0]); - }else{ - if (nonewplane){ - printf("."); - if (nonewplane>=30){ - fprintf(stderr, "waited too long for new plane, quit.\n"); - return -1; - } - }else{ - /* print mesg only the first time; dots still no new plane */ - printf("no new planes to read "); - } - nonewplane++; - /* pause for a second */ - sleep(1); - } - for (nplane=nplane_old; nplane < dims[0]; nplane++){ - /* read planes between last old nplanes and current extent */ - /* Get the dataset's dataspace */ - if((f_sid = H5Dget_space(dsid)) < 0){ - fprintf(stderr, "H5Dget_space failed\n"); - return -1; - } + /* set chunk dims */ + UC_opts.chunkdims[0] = UC_opts.chunkplanes; + UC_opts.chunkdims[1]=UC_opts.chunkdims[2]=UC_opts.chunksize; - start[0]=nplane; - /* Choose the next plane to read */ - if(H5Sselect_hyperslab(f_sid, H5S_SELECT_SET, start, NULL, count, NULL) < 0){ - fprintf(stderr, "H5Sselect_hyperslab failed\n"); - return -1; - } - - /* Read the plane from the dataset */ - if(H5Dread(dsid, UC_DATATYPE, m_sid, f_sid, H5P_DEFAULT, buffer) < 0){ - fprintf(stderr, "H5Dread failed\n"); - return -1; - } + /* set dataset initial and max dims */ + UC_opts.dims[0] = 0; + UC_opts.max_dims[0] = H5S_UNLIMITED; + UC_opts.dims[1] = UC_opts.dims[2] = UC_opts.max_dims[1]=UC_opts.max_dims[2]=UC_opts.chunksize; - /* compare read data with expected data value which is nplane */ - bufptr = buffer; - nerrs=0; - for (j=0; j<cz; j++){ - for (k=0; k<cz; k++){ - if (*bufptr++ != nplane){ - if (++nerrs < ErrorReportMax){ - fprintf(stderr, - "found error %d plane(%d,%d), expected %d, got %d\n", - nplane, j, k, nplane, (int)*(bufptr-1)); - } - } - } - } - if (nerrs){ - nreadererr++; - fprintf(stderr, "found %d unexpected values in plane %d\n", nerrs, nplane); - } - } - /* Have read all current planes */ - nplane_old=dims[0]; + /* set nplanes */ + if (UC_opts.nplanes == 0) + UC_opts.nplanes = UC_opts.chunksize; - /* check if dataset has grown since last time */ - /* close dsid and file, then reopen them */ - if (H5Dclose(dsid) < 0){ - fprintf(stderr, "H5Dclose failed\n"); - return -1; - } - if (H5Fclose(fid) < 0){ - fprintf(stderr, "H5Fclose failed\n"); - return -1; - } - if((fid = H5Fopen(name, H5F_ACC_RDONLY | (UC_opts.use_swmr ? H5F_ACC_SWMR_READ : 0), H5P_DEFAULT)) < 0){ - fprintf(stderr, "H5Fopen failed\n"); - return -1; - } - if((dsid = H5Dopen2(fid, progname_g, H5P_DEFAULT)) < 0){ - fprintf(stderr, "H5Dopen2 failed\n"); - return -1; - } - f_sid = H5Dget_space(dsid); /* Get filespace handle first. */ - if (H5Sget_simple_extent_dims(f_sid, dims, NULL) < 0){ - fprintf(stderr, "H5Sget_simple_extent_dims got error\n"); - return -1; - } - } - - if (nreadererr) - return -1; - else - return 0; + /* show parameters and return */ + show_parameters(); + return(0); } @@ -465,14 +120,7 @@ main(int argc, char *argv[]) int child_ret_value; /* initialization */ - /* use case defaults */ - HDmemset(&UC_opts, 0, sizeof(options_t)); - UC_opts.h5_use_chunks = 1; /* use chunked datasets */ - UC_opts.chunksize = Chunksize_DFT; - UC_opts.use_swmr = 1; /* use swmr open */ - - /* parse options */ - if (parse_option(argc, argv) < 0){ + if (setup_parameters(argc, argv) < 0){ Hgoto_error(1); } diff --git a/test/use_append_mchunks.c b/test/use_append_mchunks.c index 85abb74..113f75d 100644 --- a/test/use_append_mchunks.c +++ b/test/use_append_mchunks.c @@ -60,407 +60,38 @@ options_t UC_opts; /* Use Case Options */ const char *progname_g="use_append_mchunks"; /* program name */ -/* Create the skeleton use case file for testing. - * It has one 3d dataset using chunked storage. - * The dataset is (unlimited, 2*chunksize, 2*chunksize). - * Dataset type is 2 bytes integer. - * It starts out "empty", i.e., first dimension is 0. - * +/* Setup parameters for the use case. * Return: 0 succeed; -1 fail. */ -int create_uc_file(void) +int setup_parameters(int argc, char * const argv[]) { - hsize_t dims[3]; /* Dataset starting dimensions */ - hsize_t max_dims[3]; /* Dataset maximum dimensions */ - hsize_t chunk_dims[3]; /* Chunk dimensions */ - hid_t fid; /* File ID for new HDF5 file */ - hid_t dcpl; /* Dataset creation property list */ - hid_t sid; /* Dataspace ID */ - hid_t dsid; /* Dataset ID */ - - /* Create the file */ - if((fid = H5Fcreate(UC_opts.filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) - return -1; - - /* Set up dimension sizes */ - chunk_dims[0] = 1; - dims[0] = 0; - max_dims[0] = H5S_UNLIMITED; - chunk_dims[1] = chunk_dims[2] = UC_opts.chunksize; - max_dims[1] = max_dims[2] = dims[1] = dims[2] = 2*UC_opts.chunksize; - - /* Create dataspace for creating datasets */ - if((sid = H5Screate_simple(3, dims, max_dims)) < 0) - return -1; - - /* Create dataset creation property list */ - if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) - return -1; - if(H5Pset_chunk(dcpl, 3, chunk_dims) < 0) - return -1; - - /* create dataset of progname */ - if((dsid = H5Dcreate2(fid, progname_g, UC_DATATYPE, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) - return -1; - - /* Close everythign */ - if(H5Dclose(dsid) < 0) - return -1; - - if(H5Sclose(sid) < 0) - return -1; - if(H5Fclose(fid) < 0) - return -1; - - return 0; -} - - -/* Append planes, each of (1,2*chunksize,2*chunksize) to the dataset. - * In other words, 4 chunks are appended to the dataset at a time. - * Fill each plan with the plane number and then write it at the nth plane. - * Increase the plane number and repeat till the end of dataset, when it - * reaches chunksize long. End product is a (2*chunksize)^3 cube. - * - * Return: 0 succeed; -1 fail. - */ -int write_uc_file(void) -{ - hid_t fid; /* File ID for new HDF5 file */ - hid_t dsid; /* dataset ID */ - hid_t dcpl; /* Dataset creation property list */ - char *name; - UC_CTYPE *buffer, *bufptr; /* data buffer */ - int cz=UC_opts.chunksize; /* Chunk size */ - hid_t f_sid; /* dataset file space id */ - hid_t m_sid; /* memory space id */ - int rank; /* rank */ - hsize_t chunk_dims[3]; /* Chunk dimensions */ - hsize_t dims[3]; /* Dataspace dimensions */ - hsize_t memdims[3]; /* Memory space dimensions */ - hsize_t start[3] = {0,0,0}, count[3]; /* Hyperslab selection values */ - int i, j, k; - - name = UC_opts.filename; - - /* Open the file */ - if((fid = H5Fopen(name, H5F_ACC_RDWR | (UC_opts.use_swmr ? H5F_ACC_SWMR_WRITE : 0), H5P_DEFAULT)) < 0){ - fprintf(stderr, "H5Fopen failed\n"); - return -1; - } - - /* Open the dataset of the program name */ - if((dsid = H5Dopen2(fid, progname_g, H5P_DEFAULT)) < 0){ - fprintf(stderr, "H5Dopen2 failed\n"); - return -1; - } - - /* Find chunksize used */ - if ((dcpl = H5Dget_create_plist(dsid)) < 0){ - fprintf(stderr, "H5Dget_create_plist failed\n"); - return -1; - } - if (H5D_CHUNKED != H5Pget_layout(dcpl)){ - fprintf(stderr, "storage layout is not chunked\n"); - return -1; - } - if ((rank = H5Pget_chunk(dcpl, 3, chunk_dims)) != 3){ - fprintf(stderr, "storage rank is not 3\n"); - return -1; - } - - /* verify chunk_dims is (1, chunkszie, chunksize) */ - if (chunk_dims[0]!=1 || chunk_dims[1] != cz || chunk_dims[2] != cz){ - fprintf(stderr, "chunk size is not as expected. Got dims=(%ld,%ld,%ld)\n", - (long)chunk_dims[0], (long)chunk_dims[1], (long)chunk_dims[2]); - return -1; - } - - /* allocate space for data buffer 1 X 2*chunksize X 2*chunksize of UC_CTYPE */ - memdims[0]=1; - memdims[1] = memdims[2] = 2*cz; - if ((buffer=(UC_CTYPE*)HDmalloc(4*cz*cz*sizeof(UC_CTYPE)))==NULL) { - fprintf(stderr, "malloc: failed\n"); - return -1; - }; - - /* - * Get dataset rank and dimension. - */ - f_sid = H5Dget_space(dsid); /* Get filespace handle first. */ - rank = H5Sget_simple_extent_ndims(f_sid); - if (rank != 3){ - fprintf(stderr, "rank(%d) of dataset does not match\n", rank); - return -1; - } - if (H5Sget_simple_extent_dims(f_sid, dims, NULL) < 0){ - fprintf(stderr, "H5Sget_simple_extent_dims got error\n"); - return -1; - } - printf("dataset rank %d, dimensions %lu x %lu x %lu\n", - rank, (unsigned long)(dims[0]), (unsigned long)(dims[1]), (unsigned long)(dims[2])); - /* verify that file space dims are as expected and are consistent with memory space dims */ - if (dims[0] != 0 || dims[1] != memdims[1] || dims[2] != memdims[2]){ - fprintf(stderr, "dataset is not empty. Got dims=(%ld,%ld,%ld)\n", - (long)dims[0], (long)dims[1], (long)dims[2]); - return -1; - } - - /* setup mem-space for buffer */ - if ((m_sid=H5Screate_simple(rank, memdims, NULL))<0){ - fprintf(stderr, "H5Screate_simple for memory failed\n"); - return -1; - }; - - /* write planes */ - count[0]=1; - count[1]=count[2]=2*cz; - for (i=0; i<UC_opts.nplanes; i++){ - /* fill buffer with value i+1 */ - bufptr = buffer; - for (j=0; j<2*cz; j++) - for (k=0; k<2*cz; k++) - *bufptr++ = i; - - /* extend the dataset by one for new plane */ - dims[0]=i+1; - if(H5Dset_extent(dsid, dims) < 0){ - fprintf(stderr, "H5Dset_extent failed\n"); - return -1; - } - - /* Get the dataset's dataspace */ - if((f_sid = H5Dget_space(dsid)) < 0){ - fprintf(stderr, "H5Dset_extent failed\n"); - return -1; - } - - start[0]=i; - /* Choose the next plane to write */ - if(H5Sselect_hyperslab(f_sid, H5S_SELECT_SET, start, NULL, count, NULL) < 0){ - fprintf(stderr, "Failed H5Sselect_hyperslab\n"); - return -1; - } - - /* Write plane to the dataset */ - if(H5Dwrite(dsid, UC_DATATYPE, m_sid, f_sid, H5P_DEFAULT, buffer) < 0){ - fprintf(stderr, "Failed H5Dwrite\n"); - return -1; - } - /* flush file to make the just written plane available. */ - if(H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0){ - fprintf(stderr, "Failed to H5Fflush file\n"); - return -1; - } - } - - /* Done writing. Free/Close all resources including data file */ - HDfree(buffer); - if (H5Dclose(dsid) < 0){ - fprintf(stderr, "Failed to close datasete\n"); - return -1; - } - if (H5Sclose(m_sid) < 0){ - fprintf(stderr, "Failed to close memory space\n"); - return -1; - } - if (H5Sclose(f_sid) < 0){ - fprintf(stderr, "Failed to close file space\n"); - return -1; - } - if (H5Fclose(fid) < 0){ - fprintf(stderr, "Failed to close file id\n"); - return -1; - } - - return 0; -} - - -/* Read planes from the dataset. - * It expects the dataset is being changed (growing). - * It checks the unlimited dimension (1st one). When it increases, - * it will read in the new planes, one by one, and verify the data correctness. - * (The nth plan should contain all "n".) - * When the unlimited dimension grows to the chunksize (it becomes a cube), - * that is the expected end of data, the reader exits. - * - * Return: 0 succeed; -1 fail. - */ -int read_uc_file(void) -{ - hid_t fid; /* File ID for new HDF5 file */ - hid_t dsid; /* dataset ID */ - char *name; - UC_CTYPE *buffer, *bufptr; /* read data buffer */ - int cz=UC_opts.chunksize; /* Chunk size */ - hid_t f_sid; /* dataset file space id */ - hid_t m_sid; /* memory space id */ - int rank; /* rank */ - hsize_t dims[3]; /* Dataspace dimensions */ - hsize_t memdims[3]; /* Memory space dimensions */ - int nplane=0, nplane_old=0; /* nth plane, last nth plane */ - hsize_t start[3] = {0,0,0}, count[3]; /* Hyperslab selection values */ - int j, k; - int nreadererr=0; - int nerrs; - int nonewplane; - - name = UC_opts.filename; - - /* Open the file */ - if((fid = H5Fopen(name, H5F_ACC_RDONLY | (UC_opts.use_swmr ? H5F_ACC_SWMR_READ : 0), H5P_DEFAULT)) < 0){ - fprintf(stderr, "H5Fopen failed\n"); - return -1; - } + /* use case defaults */ + HDmemset(&UC_opts, 0, sizeof(options_t)); + UC_opts.chunksize = Chunksize_DFT; + UC_opts.use_swmr = 1; /* use swmr open */ + UC_opts.iterations = 1; + UC_opts.chunkplanes = 1; - /* Open the dataset of the program name */ - if((dsid = H5Dopen2(fid, progname_g, H5P_DEFAULT)) < 0){ - fprintf(stderr, "H5Dopen2 failed\n"); - return -1; + /* parse options */ + if (parse_option(argc, argv) < 0){ + return(-1); } + /* set chunk dims */ + UC_opts.chunkdims[0] = UC_opts.chunkplanes; + UC_opts.chunkdims[1]=UC_opts.chunkdims[2]=UC_opts.chunksize; - /* allocate space for data buffer 1 X 2*chunksize X 2*chunksize of UC_CTYPE */ - memdims[0]=1; - memdims[1] = memdims[2] = 2*cz; - if ((buffer=(UC_CTYPE*)HDmalloc(4*cz*cz*sizeof(UC_CTYPE)))==NULL) { - fprintf(stderr, "malloc: failed\n"); - return -1; - }; + /* set dataset initial and max dims */ + UC_opts.dims[0] = 0; + UC_opts.max_dims[0] = H5S_UNLIMITED; + UC_opts.dims[1] = UC_opts.dims[2] = UC_opts.max_dims[1]=UC_opts.max_dims[2]=2*UC_opts.chunksize; - /* - * Get dataset rank and dimension. - * Verify dimension is as expected (unlimited,2*chunksize,2*chunksize). - */ - f_sid = H5Dget_space(dsid); /* Get filespace handle first. */ - rank = H5Sget_simple_extent_ndims(f_sid); - if (rank != 3){ - fprintf(stderr, "rank(%d) of dataset does not match\n", rank); - return -1; - } - if (H5Sget_simple_extent_dims(f_sid, dims, NULL) < 0){ - fprintf(stderr, "H5Sget_simple_extent_dims got error\n"); - return -1; - } - printf("dataset rank %d, dimensions %lu x %lu x %lu\n", - rank, (unsigned long)(dims[0]), (unsigned long)(dims[1]), (unsigned long)(dims[2])); - /* verify that file space dims are as expected and are consistent with memory space dims */ - if (dims[1] != memdims[1] || dims[2] != memdims[2]){ - fprintf(stderr, "dataset dimension is not as expected. Got dims=(%ld,%ld,%ld)\n", - (long)dims[0], (long)dims[1], (long)dims[2]); - fprintf(stderr, "But memdims=(%ld,%ld,%ld)\n", - (long)memdims[0], (long)memdims[1], (long)memdims[2]); - return -1; - } - - /* setup mem-space for buffer */ - if ((m_sid=H5Screate_simple(rank, memdims, NULL))<0){ - fprintf(stderr, "H5Screate_simple for memory failed\n"); - return -1; - }; + /* set nplanes */ + if (UC_opts.nplanes == 0) + UC_opts.nplanes = 2*UC_opts.chunksize; - /* Read 1 plane at a time whenever the dataset grows larger - * (along dim[0]) */ - count[0]=1; - count[1]=count[2]=2*cz; - /* quit when all nplanes, default cz, have been read */ - nonewplane=0; - while (nplane_old < UC_opts.nplanes ){ - /* print progress message according to if new planes are availalbe */ - if (nplane_old < dims[0]) { - if (nonewplane){ - /* end the previous message */ - printf("\n"); - nonewplane=0; - } - printf("reading planes %d to %d\n", nplane_old, (int)dims[0]); - }else{ - if (nonewplane){ - printf("."); - if (nonewplane>=30){ - fprintf(stderr, "waited too long for new plane, quit.\n"); - return -1; - } - }else{ - /* print mesg only the first time; dots still no new plane */ - printf("no new planes to read "); - } - nonewplane++; - /* pause for a second */ - sleep(1); - } - for (nplane=nplane_old; nplane < dims[0]; nplane++){ - /* read planes between last old nplanes and current extent */ - /* Get the dataset's dataspace */ - if((f_sid = H5Dget_space(dsid)) < 0){ - fprintf(stderr, "H5Dget_space failed\n"); - return -1; - } - - start[0]=nplane; - /* Choose the next plane to read */ - if(H5Sselect_hyperslab(f_sid, H5S_SELECT_SET, start, NULL, count, NULL) < 0){ - fprintf(stderr, "H5Sselect_hyperslab failed\n"); - return -1; - } - - /* Read the plane from the dataset */ - if(H5Dread(dsid, UC_DATATYPE, m_sid, f_sid, H5P_DEFAULT, buffer) < 0){ - fprintf(stderr, "H5Dread failed\n"); - return -1; - } - - /* compare read data with expected data value which is nplane */ - bufptr = buffer; - nerrs=0; - for (j=0; j<2*cz; j++){ - for (k=0; k<2*cz; k++){ - if (*bufptr++ != nplane){ - if (++nerrs < ErrorReportMax){ - fprintf(stderr, - "found error %d plane(%d,%d), expected %d, got %d\n", - nplane, j, k, nplane, (int)*(bufptr-1)); - } - } - } - } - if (nerrs){ - nreadererr++; - fprintf(stderr, "found %d unexpected values in plane %d\n", nerrs, nplane); - } - } - /* Have read all current planes */ - nplane_old=dims[0]; - - /* check if dataset has grown since last time */ - /* close dsid and file, then reopen them */ - if (H5Dclose(dsid) < 0){ - fprintf(stderr, "H5Dclose failed\n"); - return -1; - } - if (H5Fclose(fid) < 0){ - fprintf(stderr, "H5Fclose failed\n"); - return -1; - } - if((fid = H5Fopen(name, H5F_ACC_RDONLY | (UC_opts.use_swmr ? H5F_ACC_SWMR_READ : 0), H5P_DEFAULT)) < 0){ - fprintf(stderr, "H5Fopen failed\n"); - return -1; - } - if((dsid = H5Dopen2(fid, progname_g, H5P_DEFAULT)) < 0){ - fprintf(stderr, "H5Dopen2 failed\n"); - return -1; - } - f_sid = H5Dget_space(dsid); /* Get filespace handle first. */ - if (H5Sget_simple_extent_dims(f_sid, dims, NULL) < 0){ - fprintf(stderr, "H5Sget_simple_extent_dims got error\n"); - return -1; - } - } - - if (nreadererr) - return -1; - else - return 0; + /* show parameters and return */ + show_parameters(); + return(0); } @@ -482,14 +113,7 @@ main(int argc, char *argv[]) int child_ret_value; /* initialization */ - /* use case defaults */ - HDmemset(&UC_opts, 0, sizeof(options_t)); - UC_opts.h5_use_chunks = 1; /* use chunked datasets */ - UC_opts.chunksize = Chunksize_DFT; - UC_opts.use_swmr = 1; /* use swmr open */ - - /* parse options */ - if (parse_option(argc, argv) < 0){ + if (setup_parameters(argc, argv) < 0){ Hgoto_error(1); } diff --git a/test/use_common.c b/test/use_common.c index cdbeca3..98448c3 100644 --- a/test/use_common.c +++ b/test/use_common.c @@ -21,9 +21,12 @@ usage(const char *prog) fprintf(stderr, " OPTIONS\n"); fprintf(stderr, " -h, --help Print a usage message and exit\n"); fprintf(stderr, " -f FN Test file name [default: %s.h5]\n", prog); + fprintf(stderr, " -i N, --iteration=N Number of iterations to repeat the whole thing. [default: 1]\n"); fprintf(stderr, " -l w|r launch writer or reader only. [default: launch both]\n"); + fprintf(stderr, " -n N, --nplanes=N Number of planes to write/read. [default: 1000]\n"); fprintf(stderr, " -s N, --swmr=N Use SWMR mode (0: no, non-0: yes) default is yes\n"); fprintf(stderr, " -z N, --chunksize=N Chunk size [default: %d]\n", Chunksize_DFT); + fprintf(stderr, " -y N, --chunkplanes=N Number of planes per chunk [default: 1]\n"); fprintf(stderr, "\n"); } @@ -35,7 +38,7 @@ parse_option(int argc, char * const argv[]) int ret_value=0; int c; /* command line options: See function usage for a description */ - const char *nagg_options = "f:hl:s:z:"; + const char *nagg_options = "f:hi:l:n:s:y:z:"; /* suppress getopt from printing error */ opterr = 0; @@ -51,6 +54,13 @@ parse_option(int argc, char * const argv[]) case 'f': /* usecase data file name */ UC_opts.filename = optarg; break; + case 'i': /* iterations */ + if ((UC_opts.iterations = atoi(optarg)) <= 0){ + fprintf(stderr, "bad iterations number %s, must be a positive integer\n", optarg); + usage(progname_g); + Hgoto_error(-1); + }; + break; case 'l': /* launch reader or writer only */ switch (*optarg) { case 'r': /* reader only */ @@ -66,6 +76,13 @@ parse_option(int argc, char * const argv[]) break; } break; + case 'n': /* number of planes to write/read */ + if ((UC_opts.nplanes = atoi(optarg)) <= 0){ + fprintf(stderr, "bad number of planes %s, must be a positive integer\n", optarg); + usage(progname_g); + Hgoto_error(-1); + }; + break; case 's': /* use swmr file open mode */ if ((UC_opts.use_swmr = atoi(optarg)) < 0){ fprintf(stderr, "swmr value should be 0(no) or 1(yes)\n"); @@ -73,6 +90,13 @@ parse_option(int argc, char * const argv[]) Hgoto_error(-1); }; break; + case 'y': /* Number of planes per chunk */ + if ((UC_opts.chunkplanes = atoi(optarg)) <= 0){ + fprintf(stderr, "bad number of planes per chunk %s, must be a positive integer\n", optarg); + usage(progname_g); + Hgoto_error(-1); + }; + break; case 'z': /* size of chunk=(z,z) */ if ((UC_opts.chunksize = atoi(optarg)) <= 0){ fprintf(stderr, "bad chunksize %s, must be a positive integer\n", optarg); @@ -90,10 +114,6 @@ parse_option(int argc, char * const argv[]) } } - /* set nplanes */ - if (UC_opts.nplanes == 0) - UC_opts.nplanes = UC_opts.chunksize; - /* set test file name if not given */ if (!UC_opts.filename){ /* default data file name is <progname>.h5 */ @@ -109,3 +129,431 @@ done: /* All done. */ return(ret_value); } + +/* Show parameters used for this use case */ +void show_parameters(void){ + printf("===Parameters used:===\n"); + printf("chunk dims=(%llu, %llu, %llu)\n", UC_opts.chunkdims[0], UC_opts.chunkdims[1], UC_opts.chunkdims[2]); + printf("dataset max dims=(%llu, %llu, %llu)\n", UC_opts.max_dims[0], UC_opts.max_dims[1], UC_opts.max_dims[2]); + printf("number of planes to write=%d\n", UC_opts.nplanes); + printf("using SWMR mode=%s\n", UC_opts.use_swmr ? "yes(1)" : "no(0)"); + printf("data filename=%s\n", UC_opts.filename); + printf("launch part="); + switch (UC_opts.launch){ + case UC_READWRITE: + printf("Reader/Writer\n"); + break; + case UC_WRITER: + printf("Writer\n"); + break; + case UC_READER: + printf("Reader\n"); + break; + default: + /* should not happen */ + printf("Illegal part(%d)\n", UC_opts.launch); + }; + printf("number of iterations=%d (not used yet)\n", UC_opts.iterations); + printf("===Parameters shown===\n"); +} + +/* Create the skeleton use case file for testing. + * It has one 3d dataset using chunked storage. + * The dataset is (unlimited, chunksize, chunksize). + * Dataset type is 2 bytes integer. + * It starts out "empty", i.e., first dimension is 0. + * + * Return: 0 succeed; -1 fail. + */ +int create_uc_file(void) +{ + hsize_t dims[3]; /* Dataset starting dimensions */ + hid_t fid; /* File ID for new HDF5 file */ + hid_t dcpl; /* Dataset creation property list */ + hid_t sid; /* Dataspace ID */ + hid_t dsid; /* Dataset ID */ + + /* Create the file */ + if((fid = H5Fcreate(UC_opts.filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + return -1; + + /* Set up dimension sizes */ + dims[0] = 0; + dims[1] = dims[2] = UC_opts.max_dims[1]; + + /* Create dataspace for creating datasets */ + if((sid = H5Screate_simple(3, dims, UC_opts.max_dims)) < 0) + return -1; + + /* Create dataset creation property list */ + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + return -1; + if(H5Pset_chunk(dcpl, 3, UC_opts.chunkdims) < 0) + return -1; + + /* create dataset of progname */ + if((dsid = H5Dcreate2(fid, progname_g, UC_DATATYPE, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + return -1; + + /* Close everythign */ + if(H5Dclose(dsid) < 0) + return -1; + + if(H5Sclose(sid) < 0) + return -1; + if(H5Fclose(fid) < 0) + return -1; + + return 0; +} + +/* Append planes, each of (1,2*chunksize,2*chunksize) to the dataset. + * In other words, 4 chunks are appended to the dataset at a time. + * Fill each plan with the plane number and then write it at the nth plane. + * Increase the plane number and repeat till the end of dataset, when it + * reaches chunksize long. End product is a (2*chunksize)^3 cube. + * + * Return: 0 succeed; -1 fail. + */ +int write_uc_file(void) +{ + hid_t fid; /* File ID for new HDF5 file */ + hid_t dsid; /* dataset ID */ + hid_t dcpl; /* Dataset creation property list */ + char *name; + UC_CTYPE *buffer, *bufptr; /* data buffer */ + int cz=UC_opts.chunksize; /* Chunk size */ + hid_t f_sid; /* dataset file space id */ + hid_t m_sid; /* memory space id */ + int rank; /* rank */ + hsize_t chunk_dims[3]; /* Chunk dimensions */ + hsize_t dims[3]; /* Dataspace dimensions */ + hsize_t memdims[3]; /* Memory space dimensions */ + hsize_t start[3] = {0,0,0}, count[3]; /* Hyperslab selection values */ + int i, j, k; + + name = UC_opts.filename; + + /* Open the file */ + if((fid = H5Fopen(name, H5F_ACC_RDWR | (UC_opts.use_swmr ? H5F_ACC_SWMR_WRITE : 0), H5P_DEFAULT)) < 0){ + fprintf(stderr, "H5Fopen failed\n"); + return -1; + } + + /* Open the dataset of the program name */ + if((dsid = H5Dopen2(fid, progname_g, H5P_DEFAULT)) < 0){ + fprintf(stderr, "H5Dopen2 failed\n"); + return -1; + } + + /* Find chunksize used */ + if ((dcpl = H5Dget_create_plist(dsid)) < 0){ + fprintf(stderr, "H5Dget_create_plist failed\n"); + return -1; + } + if (H5D_CHUNKED != H5Pget_layout(dcpl)){ + fprintf(stderr, "storage layout is not chunked\n"); + return -1; + } + if ((rank = H5Pget_chunk(dcpl, 3, chunk_dims)) != 3){ + fprintf(stderr, "storage rank is not 3\n"); + return -1; + } + + /* verify chunk_dims against set paramenters */ + if (chunk_dims[0]!=UC_opts.chunkdims[0] || chunk_dims[1] != cz || chunk_dims[2] != cz){ + fprintf(stderr, "chunk size is not as expected. Got dims=(%ld,%ld,%ld)\n", + (long)chunk_dims[0], (long)chunk_dims[1], (long)chunk_dims[2]); + return -1; + } + + /* allocate space for data buffer 1 X dims[1] X dims[2] of UC_CTYPE */ + memdims[0]=1; + memdims[1] = UC_opts.dims[1]; + memdims[2] = UC_opts.dims[2]; + if ((buffer=(UC_CTYPE*)HDmalloc(memdims[1]*memdims[2]*sizeof(UC_CTYPE)))==NULL) { + fprintf(stderr, "malloc: failed\n"); + return -1; + }; + + /* + * Get dataset rank and dimension. + */ + f_sid = H5Dget_space(dsid); /* Get filespace handle first. */ + rank = H5Sget_simple_extent_ndims(f_sid); + if (rank != UC_RANK){ + fprintf(stderr, "rank(%d) of dataset does not match\n", rank); + return -1; + } + if (H5Sget_simple_extent_dims(f_sid, dims, NULL) < 0){ + fprintf(stderr, "H5Sget_simple_extent_dims got error\n"); + return -1; + } + printf("dataset rank %d, dimensions %lu x %lu x %lu\n", + rank, (unsigned long)(dims[0]), (unsigned long)(dims[1]), (unsigned long)(dims[2])); + /* verify that file space dims are as expected and are consistent with memory space dims */ + if (dims[0] != 0 || dims[1] != memdims[1] || dims[2] != memdims[2]){ + fprintf(stderr, "dataset is not empty. Got dims=(%ld,%ld,%ld)\n", + (long)dims[0], (long)dims[1], (long)dims[2]); + return -1; + } + + /* setup mem-space for buffer */ + if ((m_sid=H5Screate_simple(rank, memdims, NULL))<0){ + fprintf(stderr, "H5Screate_simple for memory failed\n"); + return -1; + }; + + /* write planes */ + count[0]=1; + count[1]=dims[1]; + count[2]=dims[2]; + for (i=0; i<UC_opts.nplanes; i++){ + /* fill buffer with value i+1 */ + bufptr = buffer; + for (j=0; j<dims[1]; j++) + for (k=0; k<dims[2]; k++) + *bufptr++ = i; + + /* extend the dataset by one for new plane */ + dims[0]=i+1; + if(H5Dset_extent(dsid, dims) < 0){ + fprintf(stderr, "H5Dset_extent failed\n"); + return -1; + } + + /* Get the dataset's dataspace */ + if((f_sid = H5Dget_space(dsid)) < 0){ + fprintf(stderr, "H5Dset_extent failed\n"); + return -1; + } + + start[0]=i; + /* Choose the next plane to write */ + if(H5Sselect_hyperslab(f_sid, H5S_SELECT_SET, start, NULL, count, NULL) < 0){ + fprintf(stderr, "Failed H5Sselect_hyperslab\n"); + return -1; + } + + /* Write plane to the dataset */ + if(H5Dwrite(dsid, UC_DATATYPE, m_sid, f_sid, H5P_DEFAULT, buffer) < 0){ + fprintf(stderr, "Failed H5Dwrite\n"); + return -1; + } + /* flush file to make the just written plane available. */ + if(H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0){ + fprintf(stderr, "Failed to H5Fflush file\n"); + return -1; + } + } + + /* Done writing. Free/Close all resources including data file */ + HDfree(buffer); + if (H5Dclose(dsid) < 0){ + fprintf(stderr, "Failed to close datasete\n"); + return -1; + } + if (H5Sclose(m_sid) < 0){ + fprintf(stderr, "Failed to close memory space\n"); + return -1; + } + if (H5Sclose(f_sid) < 0){ + fprintf(stderr, "Failed to close file space\n"); + return -1; + } + if (H5Fclose(fid) < 0){ + fprintf(stderr, "Failed to close file id\n"); + return -1; + } + + return 0; +} + + +/* Read planes from the dataset. + * It expects the dataset is being changed (growing). + * It checks the unlimited dimension (1st one). When it increases, + * it will read in the new planes, one by one, and verify the data correctness. + * (The nth plan should contain all "n".) + * When the unlimited dimension grows to the chunksize (it becomes a cube), + * that is the expected end of data, the reader exits. + * + * Return: 0 succeed; -1 fail. + */ +int read_uc_file(void) +{ + hid_t fid; /* File ID for new HDF5 file */ + hid_t dsid; /* dataset ID */ + char *name; + UC_CTYPE *buffer, *bufptr; /* read data buffer */ + int cz=UC_opts.chunksize; /* Chunk size */ + hid_t f_sid; /* dataset file space id */ + hid_t m_sid; /* memory space id */ + int rank; /* rank */ + hsize_t dims[3]; /* Dataspace dimensions */ + hsize_t memdims[3]; /* Memory space dimensions */ + int nplane=0, nplane_old=0; /* nth plane, last nth plane */ + hsize_t start[3] = {0,0,0}, count[3]; /* Hyperslab selection values */ + int j, k; + int nreadererr=0; + int nerrs; + int nonewplane; + + name = UC_opts.filename; + + /* Open the file */ + if((fid = H5Fopen(name, H5F_ACC_RDONLY | (UC_opts.use_swmr ? H5F_ACC_SWMR_READ : 0), H5P_DEFAULT)) < 0){ + fprintf(stderr, "H5Fopen failed\n"); + return -1; + } + + /* Open the dataset of the program name */ + if((dsid = H5Dopen2(fid, progname_g, H5P_DEFAULT)) < 0){ + fprintf(stderr, "H5Dopen2 failed\n"); + return -1; + } + + /* allocate space for data buffer 1 X dims[1] X dims[2] of UC_CTYPE */ + memdims[0]=1; + memdims[1] = UC_opts.dims[1]; + memdims[2] = UC_opts.dims[2]; + if ((buffer=(UC_CTYPE*)HDmalloc(memdims[1]*memdims[2]*sizeof(UC_CTYPE)))==NULL) { + fprintf(stderr, "malloc: failed\n"); + return -1; + }; + + /* + * Get dataset rank and dimension. + * Verify dimension is as expected (unlimited,2*chunksize,2*chunksize). + */ + f_sid = H5Dget_space(dsid); /* Get filespace handle first. */ + rank = H5Sget_simple_extent_ndims(f_sid); + if (rank != UC_RANK){ + fprintf(stderr, "rank(%d) of dataset does not match\n", rank); + return -1; + } + if (H5Sget_simple_extent_dims(f_sid, dims, NULL) < 0){ + fprintf(stderr, "H5Sget_simple_extent_dims got error\n"); + return -1; + } + printf("dataset rank %d, dimensions %lu x %lu x %lu\n", + rank, (unsigned long)(dims[0]), (unsigned long)(dims[1]), (unsigned long)(dims[2])); + /* verify that file space dims are as expected and are consistent with memory space dims */ + if (dims[1] != memdims[1] || dims[2] != memdims[2]){ + fprintf(stderr, "dataset dimension is not as expected. Got dims=(%ld,%ld,%ld)\n", + (long)dims[0], (long)dims[1], (long)dims[2]); + fprintf(stderr, "But memdims=(%ld,%ld,%ld)\n", + (long)memdims[0], (long)memdims[1], (long)memdims[2]); + return -1; + } + + /* setup mem-space for buffer */ + if ((m_sid=H5Screate_simple(rank, memdims, NULL))<0){ + fprintf(stderr, "H5Screate_simple for memory failed\n"); + return -1; + }; + + /* Read 1 plane at a time whenever the dataset grows larger + * (along dim[0]) */ + count[0]=1; + count[1]=dims[1]; + count[2]=dims[2]; + /* quit when all nplanes, default cz, have been read */ + nonewplane=0; + while (nplane_old < UC_opts.nplanes ){ + /* print progress message according to if new planes are availalbe */ + if (nplane_old < dims[0]) { + if (nonewplane){ + /* end the previous message */ + printf("\n"); + nonewplane=0; + } + printf("reading planes %d to %d\n", nplane_old, (int)dims[0]); + }else{ + if (nonewplane){ + printf("."); + if (nonewplane>=30){ + fprintf(stderr, "waited too long for new plane, quit.\n"); + return -1; + } + }else{ + /* print mesg only the first time; dots still no new plane */ + printf("no new planes to read "); + } + nonewplane++; + /* pause for a second */ + sleep(1); + } + for (nplane=nplane_old; nplane < dims[0]; nplane++){ + /* read planes between last old nplanes and current extent */ + /* Get the dataset's dataspace */ + if((f_sid = H5Dget_space(dsid)) < 0){ + fprintf(stderr, "H5Dget_space failed\n"); + return -1; + } + + start[0]=nplane; + /* Choose the next plane to read */ + if(H5Sselect_hyperslab(f_sid, H5S_SELECT_SET, start, NULL, count, NULL) < 0){ + fprintf(stderr, "H5Sselect_hyperslab failed\n"); + return -1; + } + + /* Read the plane from the dataset */ + if(H5Dread(dsid, UC_DATATYPE, m_sid, f_sid, H5P_DEFAULT, buffer) < 0){ + fprintf(stderr, "H5Dread failed\n"); + return -1; + } + + /* compare read data with expected data value which is nplane */ + bufptr = buffer; + nerrs=0; + for (j=0; j<dims[1]; j++){ + for (k=0; k<dims[2]; k++){ + if (*bufptr++ != nplane){ + if (++nerrs < ErrorReportMax){ + fprintf(stderr, + "found error %d plane(%d,%d), expected %d, got %d\n", + nplane, j, k, nplane, (int)*(bufptr-1)); + } + } + } + } + if (nerrs){ + nreadererr++; + fprintf(stderr, "found %d unexpected values in plane %d\n", nerrs, nplane); + } + } + /* Have read all current planes */ + nplane_old=dims[0]; + + /* check if dataset has grown since last time */ + /* close dsid and file, then reopen them */ + if (H5Dclose(dsid) < 0){ + fprintf(stderr, "H5Dclose failed\n"); + return -1; + } + if (H5Fclose(fid) < 0){ + fprintf(stderr, "H5Fclose failed\n"); + return -1; + } + if((fid = H5Fopen(name, H5F_ACC_RDONLY | (UC_opts.use_swmr ? H5F_ACC_SWMR_READ : 0), H5P_DEFAULT)) < 0){ + fprintf(stderr, "H5Fopen failed\n"); + return -1; + } + if((dsid = H5Dopen2(fid, progname_g, H5P_DEFAULT)) < 0){ + fprintf(stderr, "H5Dopen2 failed\n"); + return -1; + } + f_sid = H5Dget_space(dsid); /* Get filespace handle first. */ + if (H5Sget_simple_extent_dims(f_sid, dims, NULL) < 0){ + fprintf(stderr, "H5Sget_simple_extent_dims got error\n"); + return -1; + } + } + + if (nreadererr) + return -1; + else + return 0; +} |