diff options
| author | Neil Fortner <nfortne2@hdfgroup.org> | 2012-02-27 16:34:55 (GMT) |
|---|---|---|
| committer | Neil Fortner <nfortne2@hdfgroup.org> | 2012-02-27 16:34:55 (GMT) |
| commit | 73c139e29b45941dfc4e558d9096a0869a184260 (patch) | |
| tree | 422cc12e6675238cd92edadc078fa427a4f6bcd0 /test | |
| parent | e2cd00055204f6b264002e70b21a6e2ca4c98c77 (diff) | |
| download | hdf5-73c139e29b45941dfc4e558d9096a0869a184260.zip hdf5-73c139e29b45941dfc4e558d9096a0869a184260.tar.gz hdf5-73c139e29b45941dfc4e558d9096a0869a184260.tar.bz2 | |
[svn-r21989] Purpose: Add SWMR capability to v1 b-tree
Description:
Adds SWMR capability to v1 b-trees, and the chunk index using v1 b-trees. With
this implementation, flush dependencies are always on when in the cache. This
will allow attritbutes to be used for "checkpointing" data when object header
dependencies are fixed - i.e. if a writer writes data before an attribute in
that dataset's object header, then if a reader sees the updated attribute the
written data is guaranteed to be visible, as long as that dataset's b-tree nodes
are evicted from the reader's cache.
Also adds support for compression with SWMR.
Also fixes earray implementation to not free (reuse) the file space for deleted
chunks and outdated versions of compressed chunks when doing SWMR writes. These
should eventually be added to a timeout list.
Adds testing for these cases.
Tested: durandal
Diffstat (limited to 'test')
| -rw-r--r-- | test/Makefile.am | 3 | ||||
| -rw-r--r-- | test/Makefile.in | 46 | ||||
| -rw-r--r-- | test/swmr_addrem_writer.c | 322 | ||||
| -rw-r--r-- | test/swmr_generator.c | 54 | ||||
| -rw-r--r-- | test/swmr_reader.c | 8 | ||||
| -rw-r--r-- | test/swmr_remove_reader.c | 366 | ||||
| -rw-r--r-- | test/swmr_remove_writer.c | 243 | ||||
| -rw-r--r-- | test/swmr_sparse_reader.c | 319 | ||||
| -rw-r--r-- | test/swmr_sparse_writer.c | 345 | ||||
| -rw-r--r-- | test/swmr_writer.c | 24 | ||||
| -rwxr-xr-x | test/testswmr.sh | 276 |
11 files changed, 1949 insertions, 57 deletions
diff --git a/test/Makefile.am b/test/Makefile.am index 70f49f1..7c8eeed 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -45,7 +45,8 @@ TEST_PROG= testhdf5 lheap ohdr stab gheap cache cache_api cache_tagging \ getname vfd ntypes dangle dtransform reserved cross_read \ freespace mf farray earray btree2 fheap -bin_PROGRAMS=swmr_generator swmr_reader swmr_writer +bin_PROGRAMS=swmr_generator swmr_reader swmr_writer swmr_remove_reader \ + swmr_remove_writer swmr_addrem_writer swmr_sparse_reader swmr_sparse_writer # List programs to be built when testing here. error_test and err_compat are # built at the same time as the other tests, but executed by testerror.sh. diff --git a/test/Makefile.in b/test/Makefile.in index 268d334..23c6ca8 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -59,7 +59,7 @@ DIST_COMMON = $(srcdir)/H5srcdir_str.h.in $(srcdir)/Makefile.am \ $(top_srcdir)/config/commence.am \ $(top_srcdir)/config/conclude.am COPYING bin_PROGRAMS = swmr_generator$(EXEEXT) swmr_reader$(EXEEXT) \ - swmr_writer$(EXEEXT) + swmr_writer$(EXEEXT) swmr_remove_reader$(EXEEXT) swmr_remove_writer$(EXEEXT) swmr_addrem_writer$(EXEEXT) swmr_sparse_reader$(EXEEXT) swmr_sparse_writer$(EXEEXT) check_PROGRAMS = $(am__EXEEXT_1) error_test$(EXEEXT) \ err_compat$(EXEEXT) tcheck_version$(EXEEXT) testmeta$(EXEEXT) \ links_env$(EXEEXT) flushrefresh$(EXEEXT) @@ -380,6 +380,26 @@ swmr_writer_SOURCES = swmr_writer.c swmr_writer_OBJECTS = swmr_writer.$(OBJEXT) swmr_writer_LDADD = $(LDADD) swmr_writer_DEPENDENCIES = libh5test.la $(LIBHDF5) +swmr_remove_reader_SOURCES = swmr_remove_reader.c +swmr_remove_reader_OBJECTS = swmr_remove_reader.$(OBJEXT) +swmr_remove_reader_LDADD = $(LDADD) +swmr_remove_reader_DEPENDENCIES = libh5test.la $(LIBHDF5) +swmr_remove_writer_SOURCES = swmr_remove_writer.c +swmr_remove_writer_OBJECTS = swmr_remove_writer.$(OBJEXT) +swmr_remove_writer_LDADD = $(LDADD) +swmr_remove_writer_DEPENDENCIES = libh5test.la $(LIBHDF5) +swmr_addrem_writer_SOURCES = swmr_addrem_writer.c +swmr_addrem_writer_OBJECTS = swmr_addrem_writer.$(OBJEXT) +swmr_addrem_writer_LDADD = $(LDADD) +swmr_addrem_writer_DEPENDENCIES = libh5test.la $(LIBHDF5) +swmr_sparse_reader_SOURCES = swmr_sparse_reader.c +swmr_sparse_reader_OBJECTS = swmr_sparse_reader.$(OBJEXT) +swmr_sparse_reader_LDADD = $(LDADD) +swmr_sparse_reader_DEPENDENCIES = libh5test.la $(LIBHDF5) +swmr_sparse_writer_SOURCES = swmr_sparse_writer.c +swmr_sparse_writer_OBJECTS = swmr_sparse_writer.$(OBJEXT) +swmr_sparse_writer_LDADD = $(LDADD) +swmr_sparse_writer_DEPENDENCIES = libh5test.la $(LIBHDF5) tcheck_version_SOURCES = tcheck_version.c tcheck_version_OBJECTS = tcheck_version.$(OBJEXT) tcheck_version_LDADD = $(LDADD) @@ -453,7 +473,7 @@ SOURCES = $(libh5test_la_SOURCES) accum.c app_ref.c big.c bittests.c \ gen_udlinks.c getname.c gheap.c hyperslab.c istore.c lheap.c \ links.c links_env.c mf.c mount.c mtime.c ntypes.c objcopy.c \ ohdr.c pool.c reserved.c set_extent.c space_overflow.c stab.c \ - swmr_generator.c swmr_reader.c swmr_writer.c tcheck_version.c \ + swmr_generator.c swmr_reader.c swmr_writer.c swmr_remove_reader.c swmr_remove_writer.c swmr_addrem_writer.c swmr_sparse_reader.c swmr_sparse_writer.c tcheck_version.c \ $(testhdf5_SOURCES) testmeta.c $(ttsafe_SOURCES) unlink.c \ vfd.c DIST_SOURCES = $(libh5test_la_SOURCES) accum.c app_ref.c big.c \ @@ -470,7 +490,7 @@ DIST_SOURCES = $(libh5test_la_SOURCES) accum.c app_ref.c big.c \ istore.c lheap.c links.c links_env.c mf.c mount.c mtime.c \ ntypes.c objcopy.c ohdr.c pool.c reserved.c set_extent.c \ space_overflow.c stab.c swmr_generator.c swmr_reader.c \ - swmr_writer.c tcheck_version.c $(testhdf5_SOURCES) testmeta.c \ + swmr_writer.c swmr_remove_reader.c swmr_remove_writer.c swmr_addrem_writer.c swmr_sparse_reader.c swmr_sparse_writer.c tcheck_version.c $(testhdf5_SOURCES) testmeta.c \ $(ttsafe_SOURCES) unlink.c vfd.c ETAGS = etags CTAGS = ctags @@ -1188,6 +1208,21 @@ swmr_reader$(EXEEXT): $(swmr_reader_OBJECTS) $(swmr_reader_DEPENDENCIES) swmr_writer$(EXEEXT): $(swmr_writer_OBJECTS) $(swmr_writer_DEPENDENCIES) @rm -f swmr_writer$(EXEEXT) $(AM_V_CCLD)$(LINK) $(swmr_writer_OBJECTS) $(swmr_writer_LDADD) $(LIBS) +swmr_remove_reader$(EXEEXT): $(swmr_remove_reader_OBJECTS) $(swmr_remove_reader_DEPENDENCIES) + @rm -f swmr_remove_reader$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(swmr_remove_reader_OBJECTS) $(swmr_remove_reader_LDADD) $(LIBS) +swmr_remove_writer$(EXEEXT): $(swmr_remove_writer_OBJECTS) $(swmr_remove_writer_DEPENDENCIES) + @rm -f swmr_remove_writer$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(swmr_remove_writer_OBJECTS) $(swmr_remove_writer_LDADD) $(LIBS) +swmr_addrem_writer$(EXEEXT): $(swmr_addrem_writer_OBJECTS) $(swmr_addrem_writer_DEPENDENCIES) + @rm -f swmr_addrem_writer$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(swmr_addrem_writer_OBJECTS) $(swmr_addrem_writer_LDADD) $(LIBS) +swmr_sparse_reader$(EXEEXT): $(swmr_sparse_reader_OBJECTS) $(swmr_sparse_reader_DEPENDENCIES) + @rm -f swmr_sparse_reader$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(swmr_sparse_reader_OBJECTS) $(swmr_sparse_reader_LDADD) $(LIBS) +swmr_sparse_writer$(EXEEXT): $(swmr_sparse_writer_OBJECTS) $(swmr_sparse_writer_DEPENDENCIES) + @rm -f swmr_sparse_writer$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(swmr_sparse_writer_OBJECTS) $(swmr_sparse_writer_LDADD) $(LIBS) tcheck_version$(EXEEXT): $(tcheck_version_OBJECTS) $(tcheck_version_DEPENDENCIES) @rm -f tcheck_version$(EXEEXT) $(AM_V_CCLD)$(LINK) $(tcheck_version_OBJECTS) $(tcheck_version_LDADD) $(LIBS) @@ -1284,6 +1319,11 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/swmr_generator.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/swmr_reader.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/swmr_writer.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/swmr_remove_reader.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/swmr_remove_writer.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/swmr_addrem_writer.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/swmr_sparse_reader.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/swmr_sparse_writer.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tarray.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tattr.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tcheck_version.Po@am__quote@ diff --git a/test/swmr_addrem_writer.c b/test/swmr_addrem_writer.c new file mode 100644 index 0000000..ddfcd5c --- /dev/null +++ b/test/swmr_addrem_writer.c @@ -0,0 +1,322 @@ +#include "swmr_common.h"
+
+#define MAX_SIZE_CHANGE 10
+
+static hid_t
+open_skeleton(const char *filename, unsigned verbose)
+{
+ hid_t fid; /* File ID for new HDF5 file */
+ hid_t fapl; /* File access property list */
+ hid_t sid; /* Dataspace ID */
+ hsize_t dim; /* Dataspace dimension */
+ unsigned u, v; /* Local index variable */
+
+ /* Create file access property list */
+ if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
+ return(-1);
+
+#ifdef QAK
+/* Increase the initial size of the metadata cache */
+{
+ H5AC_cache_config_t mdc_config;
+
+ mdc_config.version = H5AC__CURR_CACHE_CONFIG_VERSION;
+ H5Pget_mdc_config(fapl, &mdc_config);
+printf("mdc_config.initial_size = %lu\n", (unsigned long)mdc_config.initial_size);
+printf("mdc_config.epoch_length = %lu\n", (unsigned long)mdc_config.epoch_length);
+ mdc_config.set_initial_size = 1;
+ mdc_config.initial_size = 16 * 1024 * 1024;
+/* mdc_config.epoch_length = 5000; */
+ H5Pset_mdc_config(fapl, &mdc_config);
+}
+#endif /* QAK */
+
+#ifdef QAK
+ H5Pset_fapl_log(fapl, "append.log", H5FD_LOG_ALL, (size_t)(512 * 1024 * 1024));
+#endif /* QAK */
+
+ /* Open the file */
+ if((fid = H5Fopen(filename, H5F_ACC_RDWR | H5F_ACC_SWMR_WRITE, fapl)) < 0)
+ return(-1);
+
+ /* Close file access property list */
+ if(H5Pclose(fapl) < 0)
+ return(-1);
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Opening datasets\n");
+
+ /* Open the datasets */
+ for(u = 0; u < NLEVELS; u++)
+ for(v = 0; v < symbol_count[u]; v++) {
+ if((symbol_info[u][v].dsid = H5Dopen2(fid, symbol_info[u][v].name, H5P_DEFAULT)) < 0)
+ return(-1);
+ if((sid = H5Dget_space(symbol_info[u][v].dsid)) < 0)
+ return -1;
+ if(1 != H5Sget_simple_extent_ndims(sid))
+ return -1;
+ if(H5Sget_simple_extent_dims(sid, &dim, NULL) < 0)
+ return -1;
+ symbol_info[u][v].nrecords = (hsize_t)dim;
+ } /* end for */
+
+ return(fid);
+}
+
+static int
+addrem_records(hid_t fid, unsigned verbose, unsigned long nops, unsigned long flush_count)
+{
+ hid_t tid; /* Datatype ID for records */
+ hid_t mem_sid; /* Memory dataspace ID */
+ hsize_t start, count; /* Hyperslab selection values */
+ symbol_t buf[MAX_SIZE_CHANGE]; /* Write buffer */
+ H5AC_cache_config_t mdc_config_orig; /* Original metadata cache configuration */
+ H5AC_cache_config_t mdc_config_cork; /* Corked metadata cache configuration */
+ unsigned long op_to_flush; /* # of operations before flush */
+ unsigned long u, v; /* Local index variables */
+
+ /* Reset the buffer */
+ memset(&buf, 0, sizeof(buf));
+
+ /* Create a dataspace for the record to add */
+ count = 1;
+ if((mem_sid = H5Screate_simple(1, &count, NULL)) < 0)
+ return(-1);
+
+ /* Create datatype for appending records */
+ if((tid = create_symbol_datatype()) < 0)
+ return(-1);
+
+ /* Get the current metadata cache configuration, and set up the corked
+ * configuration */
+ mdc_config_orig.version = H5AC__CURR_CACHE_CONFIG_VERSION;
+ if(H5Fget_mdc_config(fid, &mdc_config_orig) < 0)
+ return(-1);
+ memcpy(&mdc_config_cork, &mdc_config_orig, sizeof(mdc_config_cork));
+ mdc_config_cork.evictions_enabled = FALSE;
+ mdc_config_cork.incr_mode = H5C_incr__off;
+ mdc_config_cork.flash_incr_mode = H5C_flash_incr__off;
+ mdc_config_cork.decr_mode = H5C_decr__off;
+
+ /* Add and remove records to random datasets, according to frequency
+ * distribution */
+ op_to_flush = flush_count;
+ for(u=0; u<nops; u++) {
+ symbol_info_t *symbol; /* Symbol to write record to */
+ hid_t file_sid; /* Dataset's space ID */
+
+ /* Get a random dataset, according to the symbol distribution */
+ symbol = choose_dataset();
+
+ /* Decide whether to shrink or expand, and by how much */
+ count = (hsize_t)random() % (MAX_SIZE_CHANGE * 2) + 1;
+
+ if(count > MAX_SIZE_CHANGE) {
+ /* Add records */
+ count -= MAX_SIZE_CHANGE;
+
+ /* Set the buffer's IDs (equal to its position) */
+ for(v=0; v<count; v++)
+ buf[v].rec_id = (uint64_t)symbol->nrecords + (uint64_t)v;
+
+ /* Set the memory space to the correct size */
+ if(H5Sset_extent_simple(mem_sid, 1, &count, NULL) < 0)
+ return(-1);
+
+ /* Get the coordinates to write */
+ start = symbol->nrecords;
+
+ /* Cork the metadata cache, to prevent the object header from being
+ * flushed before the data has been written */
+ /*if(H5Fset_mdc_config(fid, &mdc_config_cork) < 0)
+ return(-1);*/
+
+ /* Extend the dataset's dataspace to hold the new record */
+ symbol->nrecords+= count;
+ if(H5Dset_extent(symbol->dsid, &symbol->nrecords) < 0)
+ return(-1);
+
+ /* Get the dataset's dataspace */
+ if((file_sid = H5Dget_space(symbol->dsid)) < 0)
+ return(-1);
+
+ /* Choose the last record in the dataset */
+ if(H5Sselect_hyperslab(file_sid, H5S_SELECT_SET, &start, NULL, &count, NULL) < 0)
+ return(-1);
+
+ /* Write record to the dataset */
+ if(H5Dwrite(symbol->dsid, tid, mem_sid, file_sid, H5P_DEFAULT, &buf) < 0)
+ return(-1);
+
+ /* Uncork the metadata cache */
+ /*if(H5Fset_mdc_config(fid, &mdc_config_orig) < 0)
+ return(-1);*/
+
+ /* Close the dataset's dataspace */
+ if(H5Sclose(file_sid) < 0)
+ return(-1);
+ } /* end if */
+ else {
+ /* Shrink the dataset's dataspace */
+ if(count > symbol->nrecords)
+ symbol->nrecords = 0;
+ else
+ symbol->nrecords -= count;
+ if(H5Dset_extent(symbol->dsid, &symbol->nrecords) < 0)
+ return(-1);
+ } /* end else */
+
+ /* Check for flushing file */
+ if(flush_count > 0) {
+ /* Decrement count of records to write before flushing */
+ op_to_flush--;
+
+ /* Check for counter being reached */
+ if(0 == op_to_flush) {
+ /* Flush contents of file */
+ if(H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0)
+ return(-1);
+
+ /* Reset flush counter */
+ op_to_flush = flush_count;
+ } /* end if */
+ } /* end if */
+ } /* end for */
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Closing datasets\n");
+
+ /* Close the datasets */
+ for(u = 0; u < NLEVELS; u++)
+ for(v = 0; v < symbol_count[u]; v++)
+ if(H5Dclose(symbol_info[u][v].dsid) < 0)
+ return(-1);
+
+ return(0);
+}
+
+static void
+usage(void)
+{
+ printf("Usage error!\n");
+ printf("Usage: swmr_addrem_writer [-q] [-f <# of operations between flushing file contents>] <# of shrinks>\n");
+ printf("<# of operations between flushing file contents> should be 0 (for no flushing) or between 1 and (<# of shrinks> - 1)\n");
+ printf("Defaults to verbose (no '-q' given) and flushing every 1000 operations('-f 1000')\n");
+ exit(1);
+}
+
+int main(int argc, const char *argv[])
+{
+ hid_t fid; /* File ID for file opened */
+ time_t curr_time; /* Current time, for seeding random number generator */
+ long nops = 0; /* # of times to grow or shrink the dataset */
+ long flush_count = 1000; /* # of records to write between flushing file */
+ unsigned verbose = 1; /* Whether to emit some informational messages */
+ unsigned u; /* Local index variable */
+
+ /* Parse command line options */
+ if(argc < 2)
+ usage();
+ if(argc > 1) {
+ u = 1;
+ while(u < (unsigned)argc) {
+ if(argv[u][0] == '-') {
+ switch(argv[u][1]) {
+ /* # of records to write between flushing file */
+ case 'f':
+ flush_count = atol(argv[u + 1]);
+ if(flush_count < 0)
+ usage();
+ u += 2;
+ break;
+
+ /* Be quiet */
+ case 'q':
+ verbose = 0;
+ u++;
+ break;
+
+ default:
+ usage();
+ break;
+ } /* end switch */
+ } /* end if */
+ else {
+ /* Get the number of records to append */
+ nops = atol(argv[u]);
+ if(nops <= 0)
+ usage();
+
+ u++;
+ } /* end else */
+ } /* end while */
+ } /* end if */
+ if(nops <= 0)
+ usage();
+ if(flush_count >= nops)
+ usage();
+
+ /* Emit informational message */
+ if(verbose) {
+ printf("Parameters:\n");
+ printf("\t# of operations between flushes = %ld\n", flush_count);
+ printf("\t# of operations = %ld\n", nops);
+ } /* end if */
+
+ /* Create randomized set of numbers */
+ curr_time = time(NULL);
+ srandom((unsigned)curr_time);
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Generating symbol names\n");
+
+ /* Generate dataset names */
+ if(generate_symbols() < 0)
+ return(-1);
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Opening skeleton file: %s\n", FILENAME);
+
+ /* Open file skeleton */
+ if((fid = open_skeleton(FILENAME, verbose)) < 0) {
+ printf("Error opening skeleton file!\n");
+ exit(1);
+ } /* end if */
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Adding and removing records\n");
+
+ /* Grow and shrink datasets */
+ if(addrem_records(fid, verbose, (unsigned long)nops, (unsigned long)flush_count) < 0) {
+ printf("Error adding and removing records from datasets!\n");
+ exit(1);
+ } /* end if */
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Releasing symbols\n");
+
+ /* Clean up the symbols */
+ if(shutdown_symbols() < 0) {
+ printf("Error releasing symbols!\n");
+ exit(1);
+ } /* end if */
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Closing objects\n");
+
+ /* Close objects opened */
+ if(H5Fclose(fid) < 0) {
+ printf("Error closing file!\n");
+ exit(1);
+ } /* end if */
+
+ return(0);
+}
+
diff --git a/test/swmr_generator.c b/test/swmr_generator.c index 3b4d7ef..a19f70f 100644 --- a/test/swmr_generator.c +++ b/test/swmr_generator.c @@ -3,7 +3,8 @@ #define CHUNK_SIZE 50
static int
-gen_skeleton(const char *filename, unsigned verbose, int comp_level)
+gen_skeleton(const char *filename, unsigned verbose, int comp_level,
+ const char *index_type)
{
hid_t fid; /* File ID for new HDF5 file */
hid_t fcpl; /* File creation property list */
@@ -11,16 +12,24 @@ gen_skeleton(const char *filename, unsigned verbose, int comp_level) hid_t dcpl; /* Dataset creation property list */
hid_t tid; /* Datatype for dataset elements */
hid_t sid; /* Dataspace ID */
+ hid_t aid; /* Attribute ID */
hsize_t dims = 0; /* Dataset starting dimensions */
hsize_t max_dims = H5S_UNLIMITED; /* Dataset maximum dimensions */
hsize_t chunk_dims = CHUNK_SIZE; /* Chunk dimensions */
+#ifdef FILLVAL_WORKS
+ symbol_t fillval; /* Dataset fill value */
+#endif /* FILLVAL_WORKS */
+ unsigned seed; /* Random seed to write to root group attribute */
unsigned u, v; /* Local index variable */
/* Create file access property list */
if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
return(-1);
- if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
- return(-1);
+
+ /* Select the correct index type */
+ if(strcmp(index_type, "b1"))
+ if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
+ return(-1);
#ifdef QAK
/* Increase the initial size of the metadata cache */
@@ -70,6 +79,17 @@ printf("mdc_config.epoch_length = %lu\n", (unsigned long)mdc_config.epoch_length if(H5Pclose(fapl) < 0)
return(-1);
+ /* Create attribute with (shared) random number seed - for sparse test */
+ seed = (unsigned)time(NULL);
+ if((sid = H5Screate(H5S_SCALAR)) < 0)
+ return(-1);
+ if((aid = H5Acreate2(fid, "seed", H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+ return(-1);
+ if(H5Awrite(aid, H5T_NATIVE_UINT, &seed) < 0)
+ return(-1);
+ if(H5Sclose(sid) < 0)
+ return(-1);
+
/* Create datatype for creating datasets */
if((tid = create_symbol_datatype()) < 0)
return(-1);
@@ -87,6 +107,15 @@ printf("mdc_config.epoch_length = %lu\n", (unsigned long)mdc_config.epoch_length if(H5Pset_deflate(dcpl, (unsigned)comp_level) < 0)
return(-1);
} /* end if */
+#ifdef FILLVAL_WORKS
+ /* Currently fill values do not work because they can bump the dataspace
+ * message to the second object header chunk. We should enable the fillval
+ * here when this is fixed. -NAF 8/11/11 */
+ memset(&fillval, 0, sizeof(fillval));
+ fillval.rec_id = (uint64_t)ULLONG_MAX;
+ if(H5Pset_fill_value(dcpl, tid, &fillval) < 0)
+ return(-1);
+#endif /* FILLVAL_WORKS */
/* Emit informational message */
if(verbose)
@@ -127,9 +156,11 @@ static void usage(void)
{
printf("Usage error!\n");
- printf("Usage: swmr_generator [-q] [-c <deflate compression level>]\n");
+ printf("Usage: swmr_generator [-q] [-c <deflate compression level>] [-i <index type>]\n");
printf("<deflate compression level> should be -1 (for no compression) or 0-9\n");
- printf("Defaults to verbose (no '-q' given) and no compression ('-c -1')\n");
+ printf("<index type> should be b1, b2, fa, or ea (fa and b2 not yet implemented)\n");
+ printf("Defaults to verbose (no '-q' given), no compression ('-c -1') and v1 b-tree\n");
+ printf(" (-i b1)");
exit(1);
} /* end usage() */
@@ -137,6 +168,7 @@ int main(int argc, const char *argv[]) {
int comp_level = (-1); /* Compression level (-1 is no compression) */
unsigned verbose = 1; /* Whether to emit some informational messages */
+ const char *index_type = "b1"; /* Chunk index type */
unsigned u; /* Local index variables */
/* Parse command line options */
@@ -153,6 +185,15 @@ int main(int argc, const char *argv[]) u += 2;
break;
+ /* Chunk index type */
+ case 'i':
+ index_type = argv[u + 1];
+ if(strcmp(index_type, "b1")
+ && strcmp(index_type, "ea"))
+ usage();
+ u += 2;
+ break;
+
/* Be quiet */
case 'q':
verbose = 0;
@@ -171,6 +212,7 @@ int main(int argc, const char *argv[]) if(verbose) {
printf("Parameters:\n");
printf("\tcompression level = %d\n", comp_level);
+ printf("\tindex_type = %s\n", index_type);
} /* end if */
/* Emit informational message */
@@ -178,7 +220,7 @@ int main(int argc, const char *argv[]) printf("Generating skeleton file: %s\n", FILENAME);
/* Generate file skeleton */
- if(gen_skeleton(FILENAME, verbose, comp_level) < 0) {
+ if(gen_skeleton(FILENAME, verbose, comp_level, index_type) < 0) {
printf("Error generating skeleton file!\n");
exit(1);
} /* end if */
diff --git a/test/swmr_reader.c b/test/swmr_reader.c index 394a46e..48cf4bd 100644 --- a/test/swmr_reader.c +++ b/test/swmr_reader.c @@ -81,7 +81,7 @@ read_records(const char *filename, unsigned verbose, unsigned long nseconds, /* Allocate space for 'common' datasets, if any */
if(ncommon > 0) {
/* Allocate array to hold pointers to symbols for common datasets */
- if(NULL == (sym_com = malloc(sizeof(symbol_info_t *) * ncommon)))
+ if(NULL == (sym_com = (symbol_info_t **)malloc(sizeof(symbol_info_t *) * ncommon)))
return(-1);
/* Open the common datasets */
@@ -90,7 +90,7 @@ read_records(const char *filename, unsigned verbose, unsigned long nseconds, /* Determine the offset of the symbol, within level 0 symbols */
/* (level 0 symbols are the most common symbols) */
- offset = random() % symbol_count[0];
+ offset = (unsigned)(random() % symbol_count[0]);
sym_com[v] = &symbol_info[0][offset];
/* Emit informational message */
@@ -102,7 +102,7 @@ read_records(const char *filename, unsigned verbose, unsigned long nseconds, /* Allocate space for 'random' datasets, if any */
if(nrandom > 0) {
/* Allocate array to hold pointers to symbols for random datasets */
- if(NULL == (sym_rand = malloc(sizeof(symbol_info_t *) * nrandom)))
+ if(NULL == (sym_rand = (symbol_info_t **)malloc(sizeof(symbol_info_t *) * nrandom)))
return(-1);
/* Determine the random datasets */
@@ -133,7 +133,7 @@ read_records(const char *filename, unsigned verbose, unsigned long nseconds, curr_time = start_time;
/* Loop over reading records until [at least] the correct # of seconds have passed */
- while(curr_time < (time_t)(start_time + nseconds)) {
+ while(curr_time < (time_t)(start_time + (time_t)nseconds)) {
hid_t fid; /* File ID */
/* Emit informational message */
diff --git a/test/swmr_remove_reader.c b/test/swmr_remove_reader.c new file mode 100644 index 0000000..61d5aab --- /dev/null +++ b/test/swmr_remove_reader.c @@ -0,0 +1,366 @@ +#include "swmr_common.h"
+#include <unistd.h>
+
+static hid_t symbol_tid = (-1);
+
+static int
+check_dataset(hid_t fid, unsigned verbose, const char *sym_name, symbol_t *record,
+ hid_t rec_sid)
+{
+ hid_t dsid; /* Dataset ID */
+ hid_t file_sid; /* Dataset's space ID */
+ hssize_t snpoints; /* Number of elements in dataset */
+ hsize_t start, count = 1; /* Hyperslab selection values */
+
+ /* Open dataset for symbol */
+ if((dsid = H5Dopen2(fid, sym_name, H5P_DEFAULT)) < 0)
+ return(-1);
+
+ /* Get the dataset's dataspace */
+ if((file_sid = H5Dget_space(dsid)) < 0)
+ return(-1);
+
+ /* Get the number of elements (= records, for 1-D datasets) */
+ if((snpoints = H5Sget_simple_extent_npoints(file_sid)) < 0)
+ return(-1);
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Symbol = '%s', # of records = %lld\n", sym_name, (long long)snpoints);
+
+ /* Check if there are records for symbol */
+ if(snpoints > 0) {
+ /* Choose a random record in the dataset, choosing the last record half
+ * the time */
+ start = (hsize_t)(random() % (snpoints * 2));
+ if(start > (hsize_t)(snpoints - 1))
+ start = (hsize_t)(snpoints - 1);
+ if(H5Sselect_hyperslab(file_sid, H5S_SELECT_SET, &start, NULL, &count, NULL) < 0)
+ return(-1);
+
+ /* Read record from dataset */
+#ifdef FILLVAL_WORKS
+ /* When shrinking the dataset, we cannot guarantee that the buffer will
+ * even be touched, unless there is a fill value. Since fill values do
+ * not work with SWMR currently (see note in swmr_generator.c), we
+ * simply initialize rec_id to 0. */
+ record->rec_id = (uint64_t)ULLONG_MAX - 1;
+#else /* FILLVAL_WORKS */
+ record->rec_id = (uint64_t)0;
+#endif /* FILLVAL_WORKS */
+ if(H5Dread(dsid, symbol_tid, rec_sid, file_sid, H5P_DEFAULT, record) < 0)
+ return(-1);
+
+ /* Verify record value - note that it may be the fill value, because the
+ * chunk may be deleted before the object header has the updated
+ * dimensions */
+ if(record->rec_id != start && record->rec_id != (uint64_t)0) {
+ printf("Incorrect record value!\n");
+ printf("Symbol = '%s', # of records = %lld, record->rec_id = %llx\n", sym_name, (long long)snpoints, (unsigned long long)record->rec_id);
+ return(-1);
+ } /* end if */
+ } /* end if */
+
+ /* Close the dataset's dataspace */
+ if(H5Sclose(file_sid) < 0)
+ return(-1);
+
+ /* Close dataset for symbol */
+ if(H5Dclose(dsid) < 0)
+ return(-1);
+
+ return(0);
+} /* end check_dataset() */
+
+static int
+read_records(const char *filename, unsigned verbose, unsigned long nseconds,
+ unsigned poll_time, unsigned ncommon, unsigned nrandom)
+{
+ time_t start_time; /* Starting time */
+ time_t curr_time; /* Current time */
+ symbol_info_t **sym_com = NULL, **sym_rand = NULL; /* Pointers to arrays of common & random dataset IDs */
+ hid_t mem_sid; /* Memory dataspace ID */
+ symbol_t record; /* The record to add to the dataset */
+ unsigned v; /* Local index variable */
+
+ /* Reset the record */
+ /* (record's 'info' field might need to change for each record written, also) */
+ memset(&record, 0, sizeof(record));
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Choosing datasets\n");
+
+ /* Allocate space for 'common' datasets, if any */
+ if(ncommon > 0) {
+ /* Allocate array to hold pointers to symbols for common datasets */
+ if(NULL == (sym_com = (symbol_info_t **)malloc(sizeof(symbol_info_t *) * ncommon)))
+ return(-1);
+
+ /* Open the common datasets */
+ for(v = 0; v < ncommon; v++) {
+ unsigned offset; /* Offset of symbol to use */
+
+ /* Determine the offset of the symbol, within level 0 symbols */
+ /* (level 0 symbols are the most common symbols) */
+ offset = (unsigned)(random() % symbol_count[0]);
+ sym_com[v] = &symbol_info[0][offset];
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Common symbol #%u = '%s'\n", v, symbol_info[0][offset].name);
+ } /* end for */
+ } /* end if */
+
+ /* Allocate space for 'random' datasets, if any */
+ if(nrandom > 0) {
+ /* Allocate array to hold pointers to symbols for random datasets */
+ if(NULL == (sym_rand = (symbol_info_t **)malloc(sizeof(symbol_info_t *) * nrandom)))
+ return(-1);
+
+ /* Determine the random datasets */
+ for(v = 0; v < nrandom; v++) {
+ symbol_info_t *sym; /* Symbol to use */
+
+ /* Determine the symbol, within all symbols */
+ if(NULL == (sym = choose_dataset()))
+ return(-1);
+ sym_rand[v] = sym;
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Random symbol #%u = '%s'\n", v, sym->name);
+ } /* end for */
+ } /* end if */
+
+ /* Create a dataspace for the record to read */
+ if((mem_sid = H5Screate(H5S_SCALAR)) < 0)
+ return(-1);
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Reading records\n");
+
+ /* Get the starting time */
+ start_time = time(NULL);
+ curr_time = start_time;
+
+ /* Loop over reading records until [at least] the correct # of seconds have passed */
+ while(curr_time < (time_t)(start_time + (time_t)nseconds)) {
+ hid_t fid; /* File ID */
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Opening file: %s\n", filename);
+
+ /* Open the file */
+ if((fid = H5Fopen(filename, H5F_ACC_RDONLY | H5F_ACC_SWMR_READ, H5P_DEFAULT)) < 0)
+ return(-1);
+
+ /* Check 'common' datasets, if any */
+ if(ncommon > 0) {
+ /* Emit informational message */
+ if(verbose)
+ printf("Checking common symbols\n");
+
+ /* Iterate over common datasets */
+ for(v = 0; v < ncommon; v++) {
+ /* Check common dataset */
+ if(check_dataset(fid, verbose, sym_com[v]->name, &record, mem_sid) < 0)
+ return(-1);
+ } /* end for */
+ } /* end if */
+
+ /* Check 'random' datasets, if any */
+ if(nrandom > 0) {
+ /* Emit informational message */
+ if(verbose)
+ printf("Checking random symbols\n");
+
+ /* Iterate over random datasets */
+ for(v = 0; v < nrandom; v++) {
+ /* Check random dataset */
+ if(check_dataset(fid, verbose, sym_rand[v]->name, &record, mem_sid) < 0)
+ return(-1);
+ } /* end for */
+ } /* end if */
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Closing file\n");
+
+ /* Close the file */
+ if(H5Fclose(fid) < 0)
+ return(-1);
+
+ /* Sleep for the appropriate # of seconds */
+ sleep(poll_time);
+
+ /* Retrieve the current time */
+ curr_time = time(NULL);
+ } /* end while */
+
+ /* Close the memory dataspace */
+ if(H5Sclose(mem_sid) < 0)
+ return(-1);
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Closing datasets\n");
+
+ /* Close 'random' datasets, if any */
+ if(nrandom > 0) {
+ /* Release array holding dataset ID's for random datasets */
+ free(sym_rand);
+ } /* end if */
+
+ /* Close 'common' datasets, if any */
+ if(ncommon > 0) {
+ /* Release array holding dataset ID's for common datasets */
+ free(sym_com);
+ } /* end if */
+
+ return(0);
+} /* end read_records() */
+
+static void
+usage(void)
+{
+ printf("Usage error!\n");
+ printf("Usage: swmr_reader [-q] [-s <# of seconds to sleep between polling>] [-h <# of common symbols to poll>] [-l <# of random symbols to poll>] [-r <random # seed>] <# of seconds to test>\n");
+ printf("Defaults to verbose (no '-q' given), 1 second between polling ('-s 1'), 5 common symbols to poll ('-h 5') and 10 random symbols to poll ('-l 10')\n");
+ exit(1);
+}
+
+int main(int argc, const char *argv[])
+{
+ long nseconds = 0; /* # of seconds to test */
+ int poll_time = 1; /* # of seconds between polling */
+ int ncommon = 5; /* # of common symbols to poll */
+ int nrandom = 10; /* # of random symbols to poll */
+ unsigned verbose = 1; /* Whether to emit some informational messages */
+ int random_seed = 0; /* Random # seed */
+ unsigned u; /* Local index variables */
+
+ /* Parse command line options */
+ if(argc < 2)
+ usage();
+ if(argc > 1) {
+ u = 1;
+ while(u < (unsigned)argc) {
+ if(argv[u][0] == '-') {
+ switch(argv[u][1]) {
+ /* # of common symbols to poll */
+ case 'h':
+ ncommon = atoi(argv[u + 1]);
+ if(ncommon < 0)
+ usage();
+ u += 2;
+ break;
+
+ /* # of random symbols to poll */
+ case 'l':
+ nrandom = atoi(argv[u + 1]);
+ if(nrandom < 0)
+ usage();
+ u += 2;
+ break;
+
+ /* Be quiet */
+ case 'q':
+ verbose = 0;
+ u++;
+ break;
+
+ /* Random # seed */
+ case 'r':
+ random_seed = atoi(argv[u + 1]);
+ if(random_seed < 0)
+ usage();
+ u += 2;
+ break;
+
+ /* # of seconds between polling */
+ case 's':
+ poll_time = atoi(argv[u + 1]);
+ if(poll_time < 0)
+ usage();
+ u += 2;
+ break;
+
+ default:
+ usage();
+ break;
+ } /* end switch */
+ } /* end if */
+ else {
+ /* Get the number of records to append */
+ nseconds = atol(argv[u]);
+ if(nseconds <= 0)
+ usage();
+
+ u++;
+ } /* end else */
+ } /* end while */
+ } /* end if */
+ if(nseconds <= 0)
+ usage();
+ if(poll_time >= nseconds)
+ usage();
+
+ /* Emit informational message */
+ if(verbose) {
+ printf("Parameters:\n");
+ printf("\t# of seconds between polling = %d\n", poll_time);
+ printf("\t# of common symbols to poll = %d\n", ncommon);
+ printf("\t# of random symbols to poll = %d\n", nrandom);
+ printf("\t# of seconds to test = %ld\n", nseconds);
+ } /* end if */
+
+ /* Create randomized set of numbers */
+ random_seed += (int)time(NULL);
+ srandom((unsigned)random_seed);
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Generating symbol names\n");
+
+ /* Generate dataset names */
+ if(generate_symbols() < 0) {
+ printf("Error generating symbol names!\n");
+ exit(1);
+ } /* end if */
+
+ /* Create datatype for creating datasets */
+ if((symbol_tid = create_symbol_datatype()) < 0)
+ return(-1);
+
+ /* Reading records from datasets */
+ if(read_records(FILENAME, verbose, (unsigned long)nseconds, (unsigned)poll_time, (unsigned)ncommon, (unsigned)nrandom) < 0) {
+ printf("Error reading records from datasets!\n");
+ exit(1);
+ } /* end if */
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Releasing symbols\n");
+
+ /* Clean up the symbols */
+ if(shutdown_symbols() < 0) {
+ printf("Error releasing symbols!\n");
+ exit(1);
+ } /* end if */
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Closing objects\n");
+
+ /* Close objects created */
+ if(H5Tclose(symbol_tid) < 0) {
+ printf("Error closing symbol datatype!\n");
+ exit(1);
+ } /* end if */
+
+ return(0);
+}
+
diff --git a/test/swmr_remove_writer.c b/test/swmr_remove_writer.c new file mode 100644 index 0000000..576fae8 --- /dev/null +++ b/test/swmr_remove_writer.c @@ -0,0 +1,243 @@ +#include "swmr_common.h"
+
+#define MAX_REMOVE_SIZE 10
+
+static hid_t
+open_skeleton(const char *filename, unsigned verbose)
+{
+ hid_t fid; /* File ID for new HDF5 file */
+ hid_t fapl; /* File access property list */
+ hid_t sid; /* Dataspace ID */
+ hsize_t dim; /* Dataspace dimension */
+ unsigned u, v; /* Local index variable */
+
+ /* Create file access property list */
+ if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
+ return(-1);
+
+#ifdef QAK
+/* Increase the initial size of the metadata cache */
+{
+ H5AC_cache_config_t mdc_config;
+
+ mdc_config.version = H5AC__CURR_CACHE_CONFIG_VERSION;
+ H5Pget_mdc_config(fapl, &mdc_config);
+printf("mdc_config.initial_size = %lu\n", (unsigned long)mdc_config.initial_size);
+printf("mdc_config.epoch_length = %lu\n", (unsigned long)mdc_config.epoch_length);
+ mdc_config.set_initial_size = 1;
+ mdc_config.initial_size = 16 * 1024 * 1024;
+/* mdc_config.epoch_length = 5000; */
+ H5Pset_mdc_config(fapl, &mdc_config);
+}
+#endif /* QAK */
+
+#ifdef QAK
+ H5Pset_fapl_log(fapl, "append.log", H5FD_LOG_ALL, (size_t)(512 * 1024 * 1024));
+#endif /* QAK */
+
+ /* Open the file */
+ if((fid = H5Fopen(filename, H5F_ACC_RDWR | H5F_ACC_SWMR_WRITE, fapl)) < 0)
+ return(-1);
+
+ /* Close file access property list */
+ if(H5Pclose(fapl) < 0)
+ return(-1);
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Opening datasets\n");
+
+ /* Open the datasets */
+ for(u = 0; u < NLEVELS; u++)
+ for(v = 0; v < symbol_count[u]; v++) {
+ if((symbol_info[u][v].dsid = H5Dopen2(fid, symbol_info[u][v].name, H5P_DEFAULT)) < 0)
+ return(-1);
+ if((sid = H5Dget_space(symbol_info[u][v].dsid)) < 0)
+ return -1;
+ if(1 != H5Sget_simple_extent_ndims(sid))
+ return -1;
+ if(H5Sget_simple_extent_dims(sid, &dim, NULL) < 0)
+ return -1;
+ symbol_info[u][v].nrecords = (hsize_t)dim;
+ } /* end for */
+
+ return(fid);
+}
+
+static int
+remove_records(hid_t fid, unsigned verbose, unsigned long nshrinks, unsigned long flush_count)
+{
+ unsigned long shrink_to_flush; /* # of removals before flush */
+ unsigned long u, v; /* Local index variables */
+
+ /* Remove records from random datasets, according to frequency distribution */
+ shrink_to_flush = flush_count;
+ for(u = 0; u < nshrinks; u++) {
+ symbol_info_t *symbol; /* Symbol to remove record from */
+ hsize_t remove_size; /* Size to reduce dataset dimension by */
+
+ /* Get a random dataset, according to the symbol distribution */
+ symbol = choose_dataset();
+
+ /* Shrink the dataset's dataspace */
+ remove_size = (hsize_t)random() % MAX_REMOVE_SIZE + 1;
+ if(remove_size > symbol->nrecords)
+ symbol->nrecords = 0;
+ else
+ symbol->nrecords -= remove_size;
+ if(H5Dset_extent(symbol->dsid, &symbol->nrecords) < 0)
+ return(-1);
+
+ /* Check for flushing file */
+ if(flush_count > 0) {
+ /* Decrement count of records to write before flushing */
+ shrink_to_flush--;
+
+ /* Check for counter being reached */
+ if(0 == shrink_to_flush) {
+ /* Flush contents of file */
+ if(H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0)
+ return(-1);
+
+ /* Reset flush counter */
+ shrink_to_flush = flush_count;
+ } /* end if */
+ } /* end if */
+ } /* end for */
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Closing datasets\n");
+
+ /* Close the datasets */
+ for(u = 0; u < NLEVELS; u++)
+ for(v = 0; v < symbol_count[u]; v++)
+ if(H5Dclose(symbol_info[u][v].dsid) < 0)
+ return(-1);
+
+ return(0);
+}
+
+static void
+usage(void)
+{
+ printf("Usage error!\n");
+ printf("Usage: swmr_remove_writer [-q] [-f <# of shrinks between flushing file contents>] <# of shrinks>\n");
+ printf("<# of shrinks between flushing file contents> should be 0 (for no flushing) or between 1 and (<# of shrinks> - 1)\n");
+ printf("Defaults to verbose (no '-q' given) and flushing every 1000 shrinks('-f 1000')\n");
+ exit(1);
+}
+
+int main(int argc, const char *argv[])
+{
+ hid_t fid; /* File ID for file opened */
+ time_t curr_time; /* Current time, for seeding random number generator */
+ long nshrinks = 0; /* # of times to shrink the dataset */
+ long flush_count = 1000; /* # of records to write between flushing file */
+ unsigned verbose = 1; /* Whether to emit some informational messages */
+ unsigned u; /* Local index variable */
+
+ /* Parse command line options */
+ if(argc < 2)
+ usage();
+ if(argc > 1) {
+ u = 1;
+ while(u < (unsigned)argc) {
+ if(argv[u][0] == '-') {
+ switch(argv[u][1]) {
+ /* # of records to write between flushing file */
+ case 'f':
+ flush_count = atol(argv[u + 1]);
+ if(flush_count < 0)
+ usage();
+ u += 2;
+ break;
+
+ /* Be quiet */
+ case 'q':
+ verbose = 0;
+ u++;
+ break;
+
+ default:
+ usage();
+ break;
+ } /* end switch */
+ } /* end if */
+ else {
+ /* Get the number of records to append */
+ nshrinks = atol(argv[u]);
+ if(nshrinks <= 0)
+ usage();
+
+ u++;
+ } /* end else */
+ } /* end while */
+ } /* end if */
+ if(nshrinks <= 0)
+ usage();
+ if(flush_count >= nshrinks)
+ usage();
+
+ /* Emit informational message */
+ if(verbose) {
+ printf("Parameters:\n");
+ printf("\t# of shrinks between flushes = %ld\n", flush_count);
+ printf("\t# of shrinks = %ld\n", nshrinks);
+ } /* end if */
+
+ /* Create randomized set of numbers */
+ curr_time = time(NULL);
+ srandom((unsigned)curr_time);
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Generating symbol names\n");
+
+ /* Generate dataset names */
+ if(generate_symbols() < 0)
+ return(-1);
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Opening skeleton file: %s\n", FILENAME);
+
+ /* Open file skeleton */
+ if((fid = open_skeleton(FILENAME, verbose)) < 0) {
+ printf("Error opening skeleton file!\n");
+ exit(1);
+ } /* end if */
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Removing records\n");
+
+ /* Remove records from datasets */
+ if(remove_records(fid, verbose, (unsigned long)nshrinks, (unsigned long)flush_count) < 0) {
+ printf("Error removing records from datasets!\n");
+ exit(1);
+ } /* end if */
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Releasing symbols\n");
+
+ /* Clean up the symbols */
+ if(shutdown_symbols() < 0) {
+ printf("Error releasing symbols!\n");
+ exit(1);
+ } /* end if */
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Closing objects\n");
+
+ /* Close objects opened */
+ if(H5Fclose(fid) < 0) {
+ printf("Error closing file!\n");
+ exit(1);
+ } /* end if */
+
+ return(0);
+}
+
diff --git a/test/swmr_sparse_reader.c b/test/swmr_sparse_reader.c new file mode 100644 index 0000000..53bd877 --- /dev/null +++ b/test/swmr_sparse_reader.c @@ -0,0 +1,319 @@ +#include "swmr_common.h"
+#include <unistd.h>
+
+#define TIMEOUT 300
+
+static hid_t symbol_tid = (-1);
+
+static int
+check_dataset(hid_t fid, unsigned verbose, const symbol_info_t *symbol, symbol_t *record,
+ hid_t rec_sid)
+{
+ hid_t dsid; /* Dataset ID */
+ hid_t file_sid; /* Dataset's space ID */
+ hsize_t start, count = 1; /* Hyperslab selection values */
+
+ /* Open dataset for symbol */
+ if((dsid = H5Dopen2(fid, symbol->name, H5P_DEFAULT)) < 0)
+ return(-1);
+
+ /* Get the dataset's dataspace */
+ if((file_sid = H5Dget_space(dsid)) < 0)
+ return(-1);
+
+ /* Choose the random record in the dataset (will be the same as chosen by
+ * the writer) */
+ start = (hsize_t)random() % symbol->nrecords;;
+ if(H5Sselect_hyperslab(file_sid, H5S_SELECT_SET, &start, NULL, &count, NULL) < 0)
+ return(-1);
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Symbol = '%s', location = %lld\n", symbol->name, (long long)start);
+
+ /* Read record from dataset */
+#ifdef OHDR_DEPS_WORK
+ /* Even with the sequence number attribute and all the flush dependencues,
+ * it is still currently possible for the attribute to be updated before the
+ * index and/or raw data, because the attribute may reside in an object
+ * header chunk afer the first. Until this is fixed, just allow the read
+ * value to be 0. */
+ record->rec_id = (uint64_t)ULLONG_MAX;
+#else /* FILLVAL_WORKS */
+ record->rec_id = (uint64_t)0;
+#endif /* FILLVAL_WORKS */
+ if(H5Dread(dsid, symbol_tid, rec_sid, file_sid, H5P_DEFAULT, record) < 0)
+ return(-1);
+
+ /* Verify record value */
+ if(record->rec_id != start
+#ifndef OHDR_DEPS_WORK
+ && record->rec_id != (uint64_t)0
+#endif
+ ) {
+ printf("Incorrect record value!\n");
+ printf("Symbol = '%s', location = %lld, record->rec_id = %llu\n", symbol->name, (long long)start, (unsigned long long)record->rec_id);
+ return(-1);
+ } /* end if */
+
+ /* Close the dataset's dataspace */
+ if(H5Sclose(file_sid) < 0)
+ return(-1);
+
+ /* Close dataset for symbol */
+ if(H5Dclose(dsid) < 0)
+ return(-1);
+
+ return(0);
+} /* end check_dataset() */
+
+static int
+read_records(const char *filename, unsigned verbose, unsigned long nrecords,
+ unsigned poll_time, unsigned reopen_count)
+{
+ hid_t fid; /* File ID */
+ hid_t aid; /* Attribute ID */
+ time_t start_time; /* Starting time */
+ hid_t mem_sid; /* Memory dataspace ID */
+ symbol_t record; /* The record to add to the dataset */
+ unsigned seed; /* Seed for random number generator */
+ unsigned iter_to_reopen = reopen_count; /* # of iterations until reopen */
+ unsigned long u; /* Local index variable */
+hid_t fapl;
+fapl = H5Pcreate(H5P_FILE_ACCESS);
+H5Pset_fclose_degree(fapl, H5F_CLOSE_SEMI);
+ /* Emit informational message */
+ if(verbose)
+ printf("Opening file: %s\n", filename);
+
+ /* Open the file */
+ if((fid = H5Fopen(filename, H5F_ACC_RDONLY | H5F_ACC_SWMR_READ, fapl)) < 0)
+ return(-1);
+
+ /* Seed the random number generator with the attribute in the file */
+ if((aid = H5Aopen(fid, "seed", H5P_DEFAULT)) < 0)
+ return(-1);
+ if(H5Aread(aid, H5T_NATIVE_UINT, &seed) < 0)
+ return(-1);
+ if(H5Aclose(aid) < 0)
+ return(-1);
+ srandom(seed);
+
+ /* Reset the record */
+ /* (record's 'info' field might need to change for each record written, also) */
+ memset(&record, 0, sizeof(record));
+
+ /* Create a dataspace for the record to read */
+ if((mem_sid = H5Screate(H5S_SCALAR)) < 0)
+ return(-1);
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Reading records\n");
+
+ /* Get the starting time */
+ start_time = time(NULL);
+
+ /* Read records */
+ for(u = 0; u < nrecords; u++) {
+ symbol_info_t *symbol = NULL; /* Symbol (dataset) */
+ int can_read; /* Boolean: whether we can read the dataset */
+ htri_t attr_exists; /* Whether the sequence number attribute exists */
+ unsigned long file_u; /* Attribute sequence number (writer's "u") */
+
+ /* Get a random dataset, according to the symbol distribution */
+ symbol = choose_dataset();
+
+ /* Fill in "nrecords" field. Note that this depends on the writer
+ * using the same algorithm and "nrecords" */
+ symbol->nrecords = nrecords / 5;
+
+ /* Wait until we can read the dataset */
+ can_read = 0;
+ do {
+ /* Check if sequence attribute exists */
+ if((attr_exists = H5Aexists_by_name(fid, symbol->name, "seq", H5P_DEFAULT)) < 0)
+ return(-1);
+
+ if(attr_exists) {
+ /* Read sequence number attribute */
+ if((aid = H5Aopen_by_name(fid, symbol->name, "seq", H5P_DEFAULT, H5P_DEFAULT)) < 0)
+ return(-1);
+ if(H5Aread(aid, H5T_NATIVE_ULONG, &file_u) < 0)
+ return(-1);
+ if(H5Aclose(aid) < 0)
+ return(-1);
+
+ /* Check if sequence number is at least u - if so, this should
+ * guarantee that this record has been written */
+ if(file_u >= u)
+ break;
+ } /* end if */
+
+ /* Check for timeout */
+ if(time(NULL) >= (time_t)(start_time + (time_t)TIMEOUT)) {
+ printf("Reader timed out\n");
+ return(-1);
+ } /* end if */
+
+ /* Pause */
+ sleep(poll_time);
+
+ /* Reopen the file */
+ if(H5Fclose(fid) < 0)
+ return(-1);
+ if((fid = H5Fopen(filename, H5F_ACC_RDONLY | H5F_ACC_SWMR_READ, fapl)) < 0)
+ return(-1);
+ iter_to_reopen = reopen_count;
+ } while(1);
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Checking dataset %lu\n", u);
+
+ /* Check dataset */
+ if(check_dataset(fid, verbose, symbol, &record, mem_sid) < 0)
+ return(-1);
+
+ /* Check for reopen */
+ iter_to_reopen--;
+ if(iter_to_reopen == 0) {
+ /* Emit informational message */
+ if(verbose)
+ printf("Reopening file: %s\n", filename);
+
+ /* Reopen the file */
+ if(H5Fclose(fid) < 0)
+ return(-1);
+ if((fid = H5Fopen(filename, H5F_ACC_RDONLY | H5F_ACC_SWMR_READ, fapl)) < 0)
+ return(-1);
+ iter_to_reopen = reopen_count;
+ } /* end if */
+ } /* end while */
+
+ /* Close file */
+ if(H5Fclose(fid) < 0)
+ return(-1);
+
+ /* Close the memory dataspace */
+ if(H5Sclose(mem_sid) < 0)
+ return(-1);
+
+ return(0);
+} /* end read_records() */
+
+static void
+usage(void)
+{
+ printf("Usage error!\n");
+ printf("Usage: swmr_sparse_reader [-q] [-s <# of seconds to wait for writer>] [-r <# of reads between reopens>] <# of records>\n");
+ printf("Defaults to verbose (no '-q' given), 1 second wait ('-s 1') and 1 read between reopens ('-r 1')\n");
+ printf("Note that the # of records *must* be the same as that supplied to swmr_sparse_writer\n");
+ exit(1);
+} /* end usage() */
+
+int main(int argc, const char *argv[])
+{
+ long nrecords = 0; /* # of records to read */
+ int poll_time = 1; /* # of seconds to sleep when waiting for writer */
+ int reopen_count = 1; /* # of reads between reopens */
+ unsigned verbose = 1; /* Whether to emit some informational messages */
+ unsigned u; /* Local index variables */
+
+ /* Parse command line options */
+ if(argc < 2)
+ usage();
+ if(argc > 1) {
+ u = 1;
+ while(u < (unsigned)argc) {
+ if(argv[u][0] == '-') {
+ switch(argv[u][1]) {
+ /* # of reads between reopens */
+ case 'r':
+ reopen_count = atoi(argv[u + 1]);
+ if(reopen_count < 0)
+ usage();
+ u += 2;
+ break;
+
+ /* Be quiet */
+ case 'q':
+ verbose = 0;
+ u++;
+ break;
+
+ /* # of seconds between polling */
+ case 's':
+ poll_time = atoi(argv[u + 1]);
+ if(poll_time < 0)
+ usage();
+ u += 2;
+ break;
+
+ default:
+ usage();
+ break;
+ } /* end switch */
+ } /* end if */
+ else {
+ /* Get the number of records to read */
+ nrecords = atol(argv[u]);
+ if(nrecords <= 0)
+ usage();
+
+ u++;
+ } /* end else */
+ } /* end while */
+ } /* end if */
+
+ /* Emit informational message */
+ if(verbose) {
+ printf("Parameters:\n");
+ printf("\t# of seconds between polling = %d\n", poll_time);
+ printf("\t# of reads between reopens = %d\n", reopen_count);
+ printf("\t# of records to read = %ld\n", nrecords);
+ } /* end if */
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Generating symbol names\n");
+
+ /* Generate dataset names */
+ if(generate_symbols() < 0) {
+ printf("Error generating symbol names!\n");
+ exit(1);
+ } /* end if */
+
+ /* Create datatype for creating datasets */
+ if((symbol_tid = create_symbol_datatype()) < 0)
+ return(-1);
+
+ /* Reading records from datasets */
+ if(read_records(FILENAME, verbose, (unsigned long) nrecords, (unsigned)poll_time, (unsigned)reopen_count) < 0) {
+ printf("Error reading records from datasets!\n");
+ exit(1);
+ } /* end if */
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Releasing symbols\n");
+
+ /* Clean up the symbols */
+ if(shutdown_symbols() < 0) {
+ printf("Error releasing symbols!\n");
+ exit(1);
+ } /* end if */
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Closing objects\n");
+
+ /* Close objects created */
+ if(H5Tclose(symbol_tid) < 0) {
+ printf("Error closing symbol datatype!\n");
+ exit(1);
+ } /* end if */
+
+ return(0);
+}
+
diff --git a/test/swmr_sparse_writer.c b/test/swmr_sparse_writer.c new file mode 100644 index 0000000..97e2674 --- /dev/null +++ b/test/swmr_sparse_writer.c @@ -0,0 +1,345 @@ +#include "swmr_common.h"
+
+#define BUSY_WAIT 100000
+
+static hid_t
+open_skeleton(const char *filename, unsigned verbose)
+{
+ hid_t fid; /* File ID for new HDF5 file */
+ hid_t fapl; /* File access property list */
+ hid_t aid; /* Attribute ID */
+ unsigned seed; /* Seed for random number generator */
+ unsigned u, v; /* Local index variable */
+
+ /* Create file access property list */
+ if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
+ return(-1);
+
+#ifdef QAK
+/* Increase the initial size of the metadata cache */
+{
+ H5AC_cache_config_t mdc_config;
+
+ mdc_config.version = H5AC__CURR_CACHE_CONFIG_VERSION;
+ H5Pget_mdc_config(fapl, &mdc_config);
+printf("mdc_config.initial_size = %lu\n", (unsigned long)mdc_config.initial_size);
+printf("mdc_config.epoch_length = %lu\n", (unsigned long)mdc_config.epoch_length);
+ mdc_config.set_initial_size = 1;
+ mdc_config.initial_size = 16 * 1024 * 1024;
+/* mdc_config.epoch_length = 5000; */
+ H5Pset_mdc_config(fapl, &mdc_config);
+}
+#endif /* QAK */
+
+#ifdef QAK
+ H5Pset_fapl_log(fapl, "append.log", H5FD_LOG_ALL, (size_t)(512 * 1024 * 1024));
+#endif /* QAK */
+
+ /* Open the file */
+ if((fid = H5Fopen(filename, H5F_ACC_RDWR | H5F_ACC_SWMR_WRITE, fapl)) < 0)
+ return(-1);
+
+ /* Close file access property list */
+ if(H5Pclose(fapl) < 0)
+ return(-1);
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Opening datasets\n");
+
+ /* Seed the random number generator with the attribute in the file */
+ if((aid = H5Aopen(fid, "seed", H5P_DEFAULT)) < 0)
+ return(-1);
+ if(H5Aread(aid, H5T_NATIVE_UINT, &seed) < 0)
+ return(-1);
+ if(H5Aclose(aid) < 0)
+ return(-1);
+ srandom(seed);
+
+ /* Open the datasets */
+ for(u = 0; u < NLEVELS; u++)
+ for(v = 0; v < symbol_count[u]; v++) {
+ if((symbol_info[u][v].dsid = H5Dopen2(fid, symbol_info[u][v].name, H5P_DEFAULT)) < 0)
+ return(-1);
+ symbol_info[u][v].nrecords = 0;
+ } /* end for */
+
+ return(fid);
+}
+
+static int
+add_records(hid_t fid, unsigned verbose, unsigned long nrecords, unsigned long flush_count)
+{
+ hid_t tid; /* Datatype ID for records */
+ hid_t mem_sid; /* Memory dataspace ID */
+ hsize_t start, count = 1; /* Hyperslab selection values */
+ symbol_t record; /* The record to add to the dataset */
+ H5AC_cache_config_t mdc_config_orig; /* Original metadata cache configuration */
+ H5AC_cache_config_t mdc_config_cork; /* Corked metadata cache configuration */
+ unsigned long rec_to_flush; /* # of records left to write before flush */
+ volatile int dummy; /* Dummy varialbe for busy sleep */
+ unsigned long u, v; /* Local index variables */
+
+ /* Reset the record */
+ /* (record's 'info' field might need to change for each record written, also) */
+ memset(&record, 0, sizeof(record));
+
+ /* Create a dataspace for the record to add */
+ if((mem_sid = H5Screate(H5S_SCALAR)) < 0)
+ return(-1);
+
+ /* Create datatype for appending records */
+ if((tid = create_symbol_datatype()) < 0)
+ return(-1);
+
+ /* Get the current metadata cache configuration, and set up the corked
+ * configuration */
+ mdc_config_orig.version = H5AC__CURR_CACHE_CONFIG_VERSION;
+ if(H5Fget_mdc_config(fid, &mdc_config_orig) < 0)
+ return(-1);
+ memcpy(&mdc_config_cork, &mdc_config_orig, sizeof(mdc_config_cork));
+ mdc_config_cork.evictions_enabled = FALSE;
+ mdc_config_cork.incr_mode = H5C_incr__off;
+ mdc_config_cork.flash_incr_mode = H5C_flash_incr__off;
+ mdc_config_cork.decr_mode = H5C_decr__off;
+
+ /* Add records to random datasets, according to frequency distribution */
+ rec_to_flush = flush_count;
+ for(u = 0; u < nrecords; u++) {
+ symbol_info_t *symbol; /* Symbol to write record to */
+ hid_t file_sid; /* Dataset's space ID */
+ hid_t aid; /* Attribute ID */
+
+ /* Get a random dataset, according to the symbol distribution */
+ symbol = choose_dataset();
+
+ /* Cork the metadata cache, to prevent the object header from being
+ * flushed before the data has been written */
+ /*if(H5Fset_mdc_config(fid, &mdc_config_cork) < 0)
+ return(-1);*/
+
+ /* If this is the first time the dataset has been opened, extend it and
+ * add the sequence attribute */
+ if(symbol->nrecords == 0) {
+ symbol->nrecords = nrecords / 5;
+
+ if(H5Dset_extent(symbol->dsid, &symbol->nrecords) < 0)
+ return(-1);
+
+ if((file_sid = H5Screate(H5S_SCALAR)) < 0)
+ return(-1);
+ if((aid = H5Acreate2(symbol->dsid, "seq", H5T_NATIVE_ULONG, file_sid, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+ return(-1);
+ if(H5Sclose(file_sid) < 0)
+ return(-1);
+ } /* end if */
+ else if((aid = H5Aopen(symbol->dsid, "seq", H5P_DEFAULT)) < 0)
+ return(-1);
+
+ /* Get the coordinate to write */
+ start = (hsize_t)random() % symbol->nrecords;
+
+ /* Set the record's ID (equal to its position) */
+ record.rec_id = start;
+
+ /* Extend the dataset's dataspace to hold the new record */
+ symbol->nrecords++;
+ if(H5Dset_extent(symbol->dsid, &symbol->nrecords) < 0)
+ return(-1);
+
+ /* Get the dataset's dataspace */
+ if((file_sid = H5Dget_space(symbol->dsid)) < 0)
+ return(-1);
+
+ /* Choose the last record in the dataset */
+ if(H5Sselect_hyperslab(file_sid, H5S_SELECT_SET, &start, NULL, &count, NULL) < 0)
+ return(-1);
+
+ /* Write record to the dataset */
+ if(H5Dwrite(symbol->dsid, tid, mem_sid, file_sid, H5P_DEFAULT, &record) < 0)
+ return(-1);
+
+ /* Write the sequence number attribute. Since we synchronize the random
+ * number seed, the readers will always generate the same sequence of
+ * randomly chosen datasets and offsets. Therefore, and because of the
+ * flush dependencies on the object header, the reader will be
+ * guaranteed to see the written data if the sequence attribute is >=u.
+ */
+ if(H5Awrite(aid, H5T_NATIVE_ULONG, &u) < 0)
+ return(-1);
+
+ /* Close the attribute */
+ if(H5Aclose(aid) < 0)
+ return(-1);
+
+ /* Uncork the metadata cache */
+ /*if(H5Fset_mdc_config(fid, &mdc_config_orig) < 0)
+ return(-1);*/
+
+ /* Close the dataset's dataspace */
+ if(H5Sclose(file_sid) < 0)
+ return(-1);
+
+ /* Check for flushing file */
+ if(flush_count > 0) {
+ /* Decrement count of records to write before flushing */
+ rec_to_flush--;
+
+ /* Check for counter being reached */
+ if(0 == rec_to_flush) {
+ /* Flush contents of file */
+ if(H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0)
+ return(-1);
+
+ /* Reset flush counter */
+ rec_to_flush = flush_count;
+ } /* end if */
+ } /* end if */
+
+ /* Busy wait, to let readers catch up */
+ dummy = 0;
+ for(v=0; v<BUSY_WAIT; v++)
+ dummy++;
+ if((unsigned long)dummy != v)
+ return(-1);
+ } /* end for */
+
+ /* Close the memory dataspace */
+ if(H5Sclose(mem_sid) < 0)
+ return(-1);
+
+ /* Close the datatype */
+ if(H5Tclose(tid) < 0)
+ return(-1);
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Closing datasets\n");
+
+ /* Close the datasets */
+ for(u = 0; u < NLEVELS; u++)
+ for(v = 0; v < symbol_count[u]; v++)
+ if(H5Dclose(symbol_info[u][v].dsid) < 0)
+ return(-1);
+
+ return(0);
+}
+
+static void
+usage(void)
+{
+ printf("Usage error!\n");
+ printf("Usage: swmr_sparse_writer [-q] [-f <# of records to write between flushing file contents>] <# of records>\n");
+ printf("<# of records to write between flushing file contents> should be 0 (for no flushing) or between 1 and (<# of records> - 1)\n");
+ printf("Defaults to verbose (no '-q' given) and flushing every 1000 records('-f 1000')\n");
+ exit(1);
+}
+
+int main(int argc, const char *argv[])
+{
+ hid_t fid; /* File ID for file opened */\
+ long nrecords = 0; /* # of records to append */
+ long flush_count = 1000; /* # of records to write between flushing file */
+ unsigned verbose = 1; /* Whether to emit some informational messages */
+ unsigned u; /* Local index variable */
+
+ /* Parse command line options */
+ if(argc < 2)
+ usage();
+ if(argc > 1) {
+ u = 1;
+ while(u < (unsigned)argc) {
+ if(argv[u][0] == '-') {
+ switch(argv[u][1]) {
+ /* # of records to write between flushing file */
+ case 'f':
+ flush_count = atol(argv[u + 1]);
+ if(flush_count < 0)
+ usage();
+ u += 2;
+ break;
+
+ /* Be quiet */
+ case 'q':
+ verbose = 0;
+ u++;
+ break;
+
+ default:
+ usage();
+ break;
+ } /* end switch */
+ } /* end if */
+ else {
+ /* Get the number of records to append */
+ nrecords = atol(argv[u]);
+ if(nrecords <= 0)
+ usage();
+
+ u++;
+ } /* end else */
+ } /* end while */
+ } /* end if */
+ if(nrecords <= 0)
+ usage();
+ if(flush_count >= nrecords)
+ usage();
+
+ /* Emit informational message */
+ if(verbose) {
+ printf("Parameters:\n");
+ printf("\t# of records between flushes = %ld\n", flush_count);
+ printf("\t# of records to write = %ld\n", nrecords);
+ } /* end if */
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Generating symbol names\n");
+
+ /* Generate dataset names */
+ if(generate_symbols() < 0)
+ return(-1);
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Opening skeleton file: %s\n", FILENAME);
+
+ /* Open file skeleton */
+ if((fid = open_skeleton(FILENAME, verbose)) < 0) {
+ printf("Error opening skeleton file!\n");
+ exit(1);
+ } /* end if */
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Adding records\n");
+
+ /* Append records to datasets */
+ if(add_records(fid, verbose, (unsigned long)nrecords, (unsigned long)flush_count) < 0) {
+ printf("Error appending records to datasets!\n");
+ exit(1);
+ } /* end if */
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Releasing symbols\n");
+
+ /* Clean up the symbols */
+ if(shutdown_symbols() < 0) {
+ printf("Error releasing symbols!\n");
+ exit(1);
+ } /* end if */
+
+ /* Emit informational message */
+ if(verbose)
+ printf("Closing objects\n");
+
+ /* Close objects opened */
+ if(H5Fclose(fid) < 0) {
+ printf("Error closing file!\n");
+ exit(1);
+ } /* end if */
+
+ return(0);
+}
+
diff --git a/test/swmr_writer.c b/test/swmr_writer.c index 0949872..c584f60 100644 --- a/test/swmr_writer.c +++ b/test/swmr_writer.c @@ -61,6 +61,8 @@ add_records(hid_t fid, unsigned verbose, unsigned long nrecords, unsigned long f hid_t mem_sid; /* Memory dataspace ID */
hsize_t start, count = 1; /* Hyperslab selection values */
symbol_t record; /* The record to add to the dataset */
+ H5AC_cache_config_t mdc_config_orig; /* Original metadata cache configuration */
+ H5AC_cache_config_t mdc_config_cork; /* Corked metadata cache configuration */
unsigned long rec_to_flush; /* # of records left to write before flush */
unsigned long u, v; /* Local index variables */
@@ -76,6 +78,17 @@ add_records(hid_t fid, unsigned verbose, unsigned long nrecords, unsigned long f if((tid = create_symbol_datatype()) < 0)
return(-1);
+ /* Get the current metadata cache configuration, and set up the corked
+ * configuration */
+ mdc_config_orig.version = H5AC__CURR_CACHE_CONFIG_VERSION;
+ if(H5Fget_mdc_config(fid, &mdc_config_orig) < 0)
+ return(-1);
+ memcpy(&mdc_config_cork, &mdc_config_orig, sizeof(mdc_config_cork));
+ mdc_config_cork.evictions_enabled = FALSE;
+ mdc_config_cork.incr_mode = H5C_incr__off;
+ mdc_config_cork.flash_incr_mode = H5C_flash_incr__off;
+ mdc_config_cork.decr_mode = H5C_decr__off;
+
/* Add records to random datasets, according to frequency distribution */
rec_to_flush = flush_count;
for(u = 0; u < nrecords; u++) {
@@ -86,11 +99,16 @@ add_records(hid_t fid, unsigned verbose, unsigned long nrecords, unsigned long f symbol = choose_dataset();
/* Set the record's ID (equal to its position) */
- record.rec_id = symbol->nrecords;;
+ record.rec_id = symbol->nrecords;
/* Get the coordinate to write */
start = symbol->nrecords;
+ /* Cork the metadata cache, to prevent the object header from being
+ * flushed before the data has been written */
+ /*if(H5Fset_mdc_config(fid, &mdc_config_cork) < 0)
+ return(-1);*/
+
/* Extend the dataset's dataspace to hold the new record */
symbol->nrecords++;
if(H5Dset_extent(symbol->dsid, &symbol->nrecords) < 0)
@@ -108,6 +126,10 @@ add_records(hid_t fid, unsigned verbose, unsigned long nrecords, unsigned long f if(H5Dwrite(symbol->dsid, tid, mem_sid, file_sid, H5P_DEFAULT, &record) < 0)
return(-1);
+ /* Uncork the metadata cache */
+ /*if(H5Fset_mdc_config(fid, &mdc_config_orig) < 0)
+ return(-1);*/
+
/* Close the dataset's dataspace */
if(H5Sclose(file_sid) < 0)
return(-1);
diff --git a/test/testswmr.sh b/test/testswmr.sh index c0d2e08..090a131 100755 --- a/test/testswmr.sh +++ b/test/testswmr.sh @@ -24,8 +24,13 @@ ############################################################################### Nreaders=5 # number of readers to launch +Nrdrs_spa=3 # number of sparse readers to launch Nrecords=200000 # number of records to write -Nsecs=5 # number of seconds per read interval +Nrecs_rem=40000 # number of times to shrink +Nrecs_spa=20000 # number of records to write in the sparse test +Nsecs_add=5 # number of seconds per read interval +Nsecs_rem=3 # number of seconds per read interval +Nsecs_addrem=8 # number of seconds per read interval nerrors=0 ############################################################################### @@ -63,49 +68,236 @@ while [ $# -gt 0 ]; do esac done -# Launch the Generator -echo launch the swmr_generator -./swmr_generator -if test $? -ne 0; then - echo generator had error - nerrors=`expr $nerrors + 1` -fi +# Loop over index types +for index_type in "-i b1" "-i ea" +do + # Try with and without compression + for compress in "" "-c 1" + do + ############################################################################### + ## Writer test - test expanding the dataset + ############################################################################### -# Launch the Writer -echo launch the swmr_writer -./swmr_writer $Nrecords & -pid_writer=$! -$DPRINT pid_writer=$pid_writer - -# Launch the Readers -n=0 -echo launch $Nreaders swmr_readers -while [ $n -lt $Nreaders ]; do - ./swmr_reader -r $n $Nsecs & - pid_readers="$pid_readers $!" - n=`expr $n + 1` -done -$DPRINT pid_readers=$pid_readers -$IFDEBUG ps - -# Collect exit code of the readers first because they usually finish -# before the writer. -for xpid in $pid_readers; do - $DPRINT checked reader $xpid - wait $xpid - if test $? -ne 0; then - echo reader had error - nerrors=`expr $nerrors + 1` - fi -done + # Launch the Generator + echo launch the swmr_generator + ./swmr_generator $compress $index_type + if test $? -ne 0; then + echo generator had error + nerrors=`expr $nerrors + 1` + fi -# Collect exit code of the writer -$DPRINT checked writer $pid_writer -wait $pid_writer -if test $? -ne 0; then - echo writer had error - nerrors=`expr $nerrors + 1` -fi + # Launch the Writer + echo launch the swmr_writer + ./swmr_writer $Nrecords & + pid_writer=$! + $DPRINT pid_writer=$pid_writer + + # Launch the Readers + n=0 + echo launch $Nreaders swmr_readers + pid_readers="" + while [ $n -lt $Nreaders ]; do + ./swmr_reader -r $n $Nsecs_add & + pid_readers="$pid_readers $!" + n=`expr $n + 1` + done + $DPRINT pid_readers=$pid_readers + $IFDEBUG ps + + # Collect exit code of the readers first because they usually finish + # before the writer. + for xpid in $pid_readers; do + $DPRINT checked reader $xpid + wait $xpid + if test $? -ne 0; then + echo reader had error + nerrors=`expr $nerrors + 1` + fi + done + + # Collect exit code of the writer + $DPRINT checked writer $pid_writer + wait $pid_writer + if test $? -ne 0; then + echo writer had error + nerrors=`expr $nerrors + 1` + fi + + # Check for error and exit if one occured + $DPRINT nerrors=$nerrors + if test $nerrors -ne 0 ; then + echo "SWMR tests failed with $nerrors errors." + exit 1 + fi + + ############################################################################### + ## Remove test - test shrinking the dataset + ############################################################################### + + # Launch the Remove Writer + echo launch the swmr_remove_writer + ./swmr_remove_writer $Nrecs_rem & + pid_writer=$! + $DPRINT pid_writer=$pid_writer + + # Launch the Remove Readers + n=0 + pid_readers="" + echo launch $Nreaders swmr_remove_readers + while [ $n -lt $Nreaders ]; do + ./swmr_remove_reader -r $n $Nsecs_rem & + pid_readers="$pid_readers $!" + n=`expr $n + 1` + done + $DPRINT pid_readers=$pid_readers + $IFDEBUG ps + + # Collect exit code of the readers first because they usually finish + # before the writer. + for xpid in $pid_readers; do + $DPRINT checked reader $xpid + wait $xpid + if test $? -ne 0; then + echo reader had error + nerrors=`expr $nerrors + 1` + fi + done + + # Collect exit code of the writer + $DPRINT checked writer $pid_writer + wait $pid_writer + if test $? -ne 0; then + echo writer had error + nerrors=`expr $nerrors + 1` + fi + + # Check for error and exit if one occured + $DPRINT nerrors=$nerrors + if test $nerrors -ne 0 ; then + echo "SWMR tests failed with $nerrors errors." + exit 1 + fi + + ############################################################################### + ## Add/remove test - randomly grow or shrink the dataset + ############################################################################### + + # Launch the Generator + echo launch the swmr_generator + ./swmr_generator $compress $index_type + if test $? -ne 0; then + echo generator had error + nerrors=`expr $nerrors + 1` + fi + + # Launch the Writer (not in parallel - just to rebuild the datasets) + echo launch the swmr_writer + ./swmr_writer $Nrecords + if test $? -ne 0; then + echo writer had error + nerrors=`expr $nerrors + 1` + fi + + # Launch the Add/Remove Writer + echo launch the swmr_addrem_writer + ./swmr_addrem_writer $Nrecords & + pid_writer=$! + $DPRINT pid_writer=$pid_writer + + # Launch the Add/Remove Readers + n=0 + pid_readers="" + echo launch $Nreaders swmr_remove_readers + while [ $n -lt $Nreaders ]; do + ./swmr_remove_reader -r $n $Nsecs_addrem & + pid_readers="$pid_readers $!" + n=`expr $n + 1` + done + $DPRINT pid_readers=$pid_readers + $IFDEBUG ps + + # Collect exit code of the readers first because they usually finish + # before the writer. + for xpid in $pid_readers; do + $DPRINT checked reader $xpid + wait $xpid + if test $? -ne 0; then + echo reader had error + nerrors=`expr $nerrors + 1` + fi + done + + # Collect exit code of the writer + $DPRINT checked writer $pid_writer + wait $pid_writer + if test $? -ne 0; then + echo writer had error + nerrors=`expr $nerrors + 1` + fi + + # Check for error and exit if one occured + $DPRINT nerrors=$nerrors + if test $nerrors -ne 0 ; then + echo "SWMR tests failed with $nerrors errors." + exit 1 + fi + + ############################################################################### + ## Sparse writer test - test writing to random locations in the dataset + ############################################################################### + + # Launch the Generator + echo launch the swmr_generator + ./swmr_generator $compress $index_type + if test $? -ne 0; then + echo generator had error + nerrors=`expr $nerrors + 1` + fi + + # Launch the Sparse writer + echo launch the swmr_sparse_writer + nice -n 20 ./swmr_sparse_writer -q $Nrecs_spa & + pid_writer=$! + $DPRINT pid_writer=$pid_writer + + # Launch the Sparse readers + n=0 + pid_readers="" + echo launch $Nrdrs_spa swmr_sparse_readers + while [ $n -lt $Nrdrs_spa ]; do + ./swmr_sparse_reader -q $Nrecs_spa & + pid_readers="$pid_readers $!" + n=`expr $n + 1` + done + $DPRINT pid_readers=$pid_readers + $IFDEBUG ps + + # Collect exit code of the writer + $DPRINT checked writer $pid_writer + wait $pid_writer + if test $? -ne 0; then + echo writer had error + nerrors=`expr $nerrors + 1` + fi + + # Collect exit code of the readers + for xpid in $pid_readers; do + $DPRINT checked reader $xpid + wait $xpid + if test $? -ne 0; then + echo reader had error + nerrors=`expr $nerrors + 1` + fi + done + + # Check for error and exit if one occured + $DPRINT nerrors=$nerrors + if test $nerrors -ne 0 ; then + echo "SWMR tests failed with $nerrors errors." + exit 1 + fi + done +done ############################################################################### ## Report and exit |