From be4c660148eab2cebbadee134d6afda8df64e092 Mon Sep 17 00:00:00 2001 From: Albert Cheng Date: Wed, 22 Jul 2009 15:17:42 -0500 Subject: [svn-r17224] Installed SWMR testings. Tested: amani. --- test/Makefile.am | 6 +- test/Makefile.in | 115 +++++++++++++---- test/swmr_common.c | 93 ++++++++++++++ test/swmr_common.h | 56 ++++++++ test/swmr_generator.c | 188 +++++++++++++++++++++++++++ test/swmr_reader.c | 345 ++++++++++++++++++++++++++++++++++++++++++++++++++ test/swmr_writer.c | 275 ++++++++++++++++++++++++++++++++++++++++ test/testswmr.sh | 76 +++++++++++ 8 files changed, 1127 insertions(+), 27 deletions(-) create mode 100644 test/swmr_common.c create mode 100644 test/swmr_common.h create mode 100644 test/swmr_generator.c create mode 100644 test/swmr_reader.c create mode 100644 test/swmr_writer.c create mode 100755 test/testswmr.sh diff --git a/test/Makefile.am b/test/Makefile.am index ebf4fa3..ec6df85 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -24,7 +24,7 @@ include $(top_srcdir)/config/commence.am INCLUDES=-I$(top_srcdir)/src -I$(top_builddir)/src # Test script for error_test and err_compat -TEST_SCRIPT = testerror.sh +TEST_SCRIPT = testerror.sh $(srcdir)/testswmr.sh check_SCRIPTS = $(TEST_SCRIPT) SCRIPT_DEPEND = error_test$(EXEEXT) err_compat$(EXEEXT) @@ -44,6 +44,8 @@ TEST_PROG=testhdf5 lheap ohdr stab gheap cache cache_api \ getname vfd ntypes dangle dtransform reserved cross_read \ freespace mf earray btree2 fheap +bin_PROGRAMS=swmr_generator swmr_reader swmr_writer + # List programs to be built when testing here. error_test and err_compat are # built at the same time as the other tests, but executed by testerror.sh. # 'make check' doesn't run them directly, so they are not included in TEST_PROG. @@ -69,7 +71,7 @@ endif # The libh5test library provides common support code for the tests. noinst_LTLIBRARIES=libh5test.la -libh5test_la_SOURCES=h5test.c testframe.c cache_common.c +libh5test_la_SOURCES=h5test.c testframe.c cache_common.c swmr_common.c # Use libhd5test.la to compile all of the tests LDADD=libh5test.la $(LIBHDF5) diff --git a/test/Makefile.in b/test/Makefile.in index 1475d50..130cd44 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -53,6 +53,8 @@ host_triplet = @host@ DIST_COMMON = $(srcdir)/Makefile.am $(srcdir)/Makefile.in \ $(srcdir)/testerror.sh.in $(top_srcdir)/config/commence.am \ $(top_srcdir)/config/conclude.am COPYING +bin_PROGRAMS = swmr_generator$(EXEEXT) swmr_reader$(EXEEXT) \ + swmr_writer$(EXEEXT) check_PROGRAMS = $(am__EXEEXT_1) error_test$(EXEEXT) \ err_compat$(EXEEXT) testmeta$(EXEEXT) @BUILD_ALL_CONDITIONAL_TRUE@noinst_PROGRAMS = $(am__EXEEXT_2) @@ -67,8 +69,11 @@ CONFIG_HEADER = $(top_builddir)/src/H5config.h CONFIG_CLEAN_FILES = testerror.sh LTLIBRARIES = $(noinst_LTLIBRARIES) libh5test_la_LIBADD = -am_libh5test_la_OBJECTS = h5test.lo testframe.lo cache_common.lo +am_libh5test_la_OBJECTS = h5test.lo testframe.lo cache_common.lo \ + swmr_common.lo libh5test_la_OBJECTS = $(am_libh5test_la_OBJECTS) +am__installdirs = "$(DESTDIR)$(bindir)" +binPROGRAMS_INSTALL = $(INSTALL_PROGRAM) am__EXEEXT_1 = testhdf5$(EXEEXT) lheap$(EXEEXT) ohdr$(EXEEXT) \ stab$(EXEEXT) gheap$(EXEEXT) cache$(EXEEXT) cache_api$(EXEEXT) \ pool$(EXEEXT) hyperslab$(EXEEXT) istore$(EXEEXT) \ @@ -89,7 +94,7 @@ am__EXEEXT_2 = gen_bad_ohdr$(EXEEXT) gen_bogus$(EXEEXT) \ gen_new_super$(EXEEXT) gen_noencoder$(EXEEXT) \ gen_nullspace$(EXEEXT) gen_udlinks$(EXEEXT) \ space_overflow$(EXEEXT) -PROGRAMS = $(noinst_PROGRAMS) +PROGRAMS = $(bin_PROGRAMS) $(noinst_PROGRAMS) app_ref_SOURCES = app_ref.c app_ref_OBJECTS = app_ref.$(OBJEXT) app_ref_LDADD = $(LDADD) @@ -306,6 +311,18 @@ stab_SOURCES = stab.c stab_OBJECTS = stab.$(OBJEXT) stab_LDADD = $(LDADD) stab_DEPENDENCIES = libh5test.la $(LIBHDF5) +swmr_generator_SOURCES = swmr_generator.c +swmr_generator_OBJECTS = swmr_generator.$(OBJEXT) +swmr_generator_LDADD = $(LDADD) +swmr_generator_DEPENDENCIES = libh5test.la $(LIBHDF5) +swmr_reader_SOURCES = swmr_reader.c +swmr_reader_OBJECTS = swmr_reader.$(OBJEXT) +swmr_reader_LDADD = $(LDADD) +swmr_reader_DEPENDENCIES = libh5test.la $(LIBHDF5) +swmr_writer_SOURCES = swmr_writer.c +swmr_writer_OBJECTS = swmr_writer.$(OBJEXT) +swmr_writer_LDADD = $(LDADD) +swmr_writer_DEPENDENCIES = libh5test.la $(LIBHDF5) am_testhdf5_OBJECTS = testhdf5.$(OBJEXT) tarray.$(OBJEXT) \ tattr.$(OBJEXT) tchecksum.$(OBJEXT) tconfig.$(OBJEXT) \ tfile.$(OBJEXT) tgenprop.$(OBJEXT) th5o.$(OBJEXT) \ @@ -358,8 +375,9 @@ SOURCES = $(libh5test_la_SOURCES) app_ref.c big.c bittests.c btree2.c \ gen_noencoder.c gen_nullspace.c gen_udlinks.c getname.c \ gheap.c hyperslab.c istore.c lheap.c links.c mf.c mount.c \ mtime.c ntypes.c objcopy.c ohdr.c pool.c reserved.c \ - set_extent.c space_overflow.c stab.c $(testhdf5_SOURCES) \ - testmeta.c $(ttsafe_SOURCES) unlink.c vfd.c + set_extent.c space_overflow.c stab.c swmr_generator.c \ + swmr_reader.c swmr_writer.c $(testhdf5_SOURCES) testmeta.c \ + $(ttsafe_SOURCES) unlink.c vfd.c DIST_SOURCES = $(libh5test_la_SOURCES) app_ref.c big.c bittests.c \ btree2.c cache.c cache_api.c cmpd_dset.c cross_read.c dangle.c \ dsets.c dt_arith.c dtransform.c dtypes.c earray.c enum.c \ @@ -370,8 +388,9 @@ DIST_SOURCES = $(libh5test_la_SOURCES) app_ref.c big.c bittests.c \ gen_new_super.c gen_noencoder.c gen_nullspace.c gen_udlinks.c \ getname.c gheap.c hyperslab.c istore.c lheap.c links.c mf.c \ mount.c mtime.c ntypes.c objcopy.c ohdr.c pool.c reserved.c \ - set_extent.c space_overflow.c stab.c $(testhdf5_SOURCES) \ - testmeta.c $(ttsafe_SOURCES) unlink.c vfd.c + set_extent.c space_overflow.c stab.c swmr_generator.c \ + swmr_reader.c swmr_writer.c $(testhdf5_SOURCES) testmeta.c \ + $(ttsafe_SOURCES) unlink.c vfd.c ETAGS = etags CTAGS = ctags DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) @@ -653,7 +672,7 @@ CHECK_CLEANFILES = *.chkexe *.chklog *.clog cmpd_dset.h5 \ INCLUDES = -I$(top_srcdir)/src -I$(top_builddir)/src # Test script for error_test and err_compat -TEST_SCRIPT = testerror.sh +TEST_SCRIPT = testerror.sh $(srcdir)/testswmr.sh check_SCRIPTS = $(TEST_SCRIPT) SCRIPT_DEPEND = error_test$(EXEEXT) err_compat$(EXEEXT) @@ -687,7 +706,7 @@ BUILD_ALL_PROGS = gen_bad_ohdr gen_bogus gen_cross gen_deflate gen_filters gen_n # The libh5test library provides common support code for the tests. noinst_LTLIBRARIES = libh5test.la -libh5test_la_SOURCES = h5test.c testframe.c cache_common.c +libh5test_la_SOURCES = h5test.c testframe.c cache_common.c swmr_common.c # Use libhd5test.la to compile all of the tests LDADD = libh5test.la $(LIBHDF5) @@ -770,6 +789,34 @@ clean-noinstLTLIBRARIES: done libh5test.la: $(libh5test_la_OBJECTS) $(libh5test_la_DEPENDENCIES) $(LINK) $(libh5test_la_OBJECTS) $(libh5test_la_LIBADD) $(LIBS) +install-binPROGRAMS: $(bin_PROGRAMS) + @$(NORMAL_INSTALL) + test -z "$(bindir)" || $(MKDIR_P) "$(DESTDIR)$(bindir)" + @list='$(bin_PROGRAMS)'; for p in $$list; do \ + p1=`echo $$p|sed 's/$(EXEEXT)$$//'`; \ + if test -f $$p \ + || test -f $$p1 \ + ; then \ + f=`echo "$$p1" | sed 's,^.*/,,;$(transform);s/$$/$(EXEEXT)/'`; \ + echo " $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(binPROGRAMS_INSTALL) '$$p' '$(DESTDIR)$(bindir)/$$f'"; \ + $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(binPROGRAMS_INSTALL) "$$p" "$(DESTDIR)$(bindir)/$$f" || exit 1; \ + else :; fi; \ + done + +uninstall-binPROGRAMS: + @$(NORMAL_UNINSTALL) + @list='$(bin_PROGRAMS)'; for p in $$list; do \ + f=`echo "$$p" | sed 's,^.*/,,;s/$(EXEEXT)$$//;$(transform);s/$$/$(EXEEXT)/'`; \ + echo " rm -f '$(DESTDIR)$(bindir)/$$f'"; \ + rm -f "$(DESTDIR)$(bindir)/$$f"; \ + done + +clean-binPROGRAMS: + @list='$(bin_PROGRAMS)'; for p in $$list; do \ + f=`echo $$p|sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f $$p $$f"; \ + rm -f $$p $$f ; \ + done clean-checkPROGRAMS: @list='$(check_PROGRAMS)'; for p in $$list; do \ @@ -946,6 +993,15 @@ space_overflow$(EXEEXT): $(space_overflow_OBJECTS) $(space_overflow_DEPENDENCIES stab$(EXEEXT): $(stab_OBJECTS) $(stab_DEPENDENCIES) @rm -f stab$(EXEEXT) $(LINK) $(stab_OBJECTS) $(stab_LDADD) $(LIBS) +swmr_generator$(EXEEXT): $(swmr_generator_OBJECTS) $(swmr_generator_DEPENDENCIES) + @rm -f swmr_generator$(EXEEXT) + $(LINK) $(swmr_generator_OBJECTS) $(swmr_generator_LDADD) $(LIBS) +swmr_reader$(EXEEXT): $(swmr_reader_OBJECTS) $(swmr_reader_DEPENDENCIES) + @rm -f swmr_reader$(EXEEXT) + $(LINK) $(swmr_reader_OBJECTS) $(swmr_reader_LDADD) $(LIBS) +swmr_writer$(EXEEXT): $(swmr_writer_OBJECTS) $(swmr_writer_DEPENDENCIES) + @rm -f swmr_writer$(EXEEXT) + $(LINK) $(swmr_writer_OBJECTS) $(swmr_writer_LDADD) $(LIBS) testhdf5$(EXEEXT): $(testhdf5_OBJECTS) $(testhdf5_DEPENDENCIES) @rm -f testhdf5$(EXEEXT) $(LINK) $(testhdf5_OBJECTS) $(testhdf5_LDADD) $(LIBS) @@ -1024,6 +1080,10 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/set_extent.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/space_overflow.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/stab.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/swmr_common.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/swmr_generator.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/swmr_reader.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/swmr_writer.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tarray.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tattr.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tchecksum.Po@am__quote@ @@ -1165,6 +1225,9 @@ check-am: all-am check: check-am all-am: Makefile $(LTLIBRARIES) $(PROGRAMS) all-local installdirs: + for dir in "$(DESTDIR)$(bindir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done install: install-am install-exec: install-exec-am install-data: install-data-am @@ -1192,8 +1255,9 @@ maintainer-clean-generic: @echo "it deletes files that may require special tools to rebuild." clean: clean-am -clean-am: clean-checkPROGRAMS clean-generic clean-libtool \ - clean-noinstLTLIBRARIES clean-noinstPROGRAMS mostlyclean-am +clean-am: clean-binPROGRAMS clean-checkPROGRAMS clean-generic \ + clean-libtool clean-noinstLTLIBRARIES clean-noinstPROGRAMS \ + mostlyclean-am distclean: distclean-am -rm -rf ./$(DEPDIR) @@ -1215,7 +1279,7 @@ install-data-am: install-dvi: install-dvi-am -install-exec-am: +install-exec-am: install-binPROGRAMS install-html: install-html-am @@ -1247,24 +1311,25 @@ ps: ps-am ps-am: -uninstall-am: +uninstall-am: uninstall-binPROGRAMS .MAKE: install-am install-strip .PHONY: CTAGS GTAGS all all-am all-local check check-TESTS check-am \ - clean clean-checkPROGRAMS clean-generic clean-libtool \ - clean-noinstLTLIBRARIES clean-noinstPROGRAMS ctags distclean \ - distclean-compile distclean-generic distclean-libtool \ - distclean-tags distdir dvi dvi-am html html-am info info-am \ - install install-am install-data install-data-am install-dvi \ - install-dvi-am install-exec install-exec-am install-html \ - install-html-am install-info install-info-am install-man \ - install-pdf install-pdf-am install-ps install-ps-am \ - install-strip installcheck installcheck-am installdirs \ - maintainer-clean maintainer-clean-generic mostlyclean \ - mostlyclean-compile mostlyclean-generic mostlyclean-libtool \ - mostlyclean-local pdf pdf-am ps ps-am tags uninstall \ - uninstall-am + clean clean-binPROGRAMS clean-checkPROGRAMS clean-generic \ + clean-libtool clean-noinstLTLIBRARIES clean-noinstPROGRAMS \ + ctags distclean distclean-compile distclean-generic \ + distclean-libtool distclean-tags distdir dvi dvi-am html \ + html-am info info-am install install-am install-binPROGRAMS \ + install-data install-data-am install-dvi install-dvi-am \ + install-exec install-exec-am install-html install-html-am \ + install-info install-info-am install-man install-pdf \ + install-pdf-am install-ps install-ps-am install-strip \ + installcheck installcheck-am installdirs maintainer-clean \ + maintainer-clean-generic mostlyclean mostlyclean-compile \ + mostlyclean-generic mostlyclean-libtool mostlyclean-local pdf \ + pdf-am ps ps-am tags uninstall uninstall-am \ + uninstall-binPROGRAMS # List all build rules defined by HDF5 Makefiles as "PHONY" targets here. diff --git a/test/swmr_common.c b/test/swmr_common.c new file mode 100644 index 0000000..bb75c0d --- /dev/null +++ b/test/swmr_common.c @@ -0,0 +1,93 @@ +#include "swmr_common.h" + +static unsigned symbol_mapping[NMAPPING] = {0, 0, 0, 0, 1, 1, 2, 3, 4}; +unsigned symbol_count[NLEVELS] = {100, 200, 400, 800, 1600}; + +symbol_info_t *symbol_info[NLEVELS]; + +symbol_info_t * +choose_dataset(void) +{ + unsigned level; /* The level of the dataset */ + unsigned offset; /* The "offset" of the dataset at that level */ + + /* Determine level of dataset */ + level = symbol_mapping[random() % NMAPPING]; + + /* Determine the offset of the level */ + offset = random() % symbol_count[level]; + + return(&symbol_info[level][offset]); +} /* end choose_dataset() */ + +hid_t +create_symbol_datatype(void) +{ + hid_t sym_type_id; /* Datatype ID for symbol */ + hid_t opaq_type_id; /* Datatype ID for opaque part of record */ + + /* Create opaque datatype to represent other information for this record */ + if((opaq_type_id = H5Tcreate(H5T_OPAQUE, DTYPE_SIZE)) < 0) + return(-1); + + /* Create compound datatype for symbol */ + if((sym_type_id = H5Tcreate(H5T_COMPOUND, sizeof(symbol_t))) < 0) + return(-1); + + /* Insert fields in symbol datatype */ + if(H5Tinsert(sym_type_id, "rec_id", HOFFSET(symbol_t, rec_id), H5T_NATIVE_UINT64) < 0) + return(-1); + if(H5Tinsert(sym_type_id, "info", HOFFSET(symbol_t, info), opaq_type_id) < 0) + return(-1); + + /* Close opaque datatype */ + if(H5Tclose(opaq_type_id) < 0) + return(-1); + + return(sym_type_id); +} /* end create_symbol_datatype() */ + +int +generate_name(char *name_buf, unsigned level, unsigned count) +{ + sprintf(name_buf, "%u-%04u", level, count); + + return(0); +} /* end generate_name() */ + +int +generate_symbols(void) +{ + unsigned u, v; /* Local index variables */ + + for(u = 0; u < NLEVELS; u++) { + symbol_info[u] = malloc(symbol_count[u] * sizeof(symbol_info_t)); + for(v = 0; v < symbol_count[u]; v++) { + char name_buf[64]; + + generate_name(name_buf, u, v); + symbol_info[u][v].name = malloc(strlen(name_buf) + 1); + strcpy(symbol_info[u][v].name, name_buf); + symbol_info[u][v].dsid = -1; + symbol_info[u][v].nrecords = 0; + } /* end for */ + } /* end for */ + + return(0); +} /* end generate_symbols() */ + +int +shutdown_symbols(void) +{ + unsigned u, v; /* Local index variables */ + + /* Clean up the symbols */ + for(u = 0; u < NLEVELS; u++) { + for(v = 0; v < symbol_count[u]; v++) + free(symbol_info[u][v].name); + free(symbol_info[u]); + } /* end for */ + + return(0); +} /* end shutdown_symbols() */ + diff --git a/test/swmr_common.h b/test/swmr_common.h new file mode 100644 index 0000000..edf0f3b --- /dev/null +++ b/test/swmr_common.h @@ -0,0 +1,56 @@ +#ifndef __BENCHMARK_COMMON_H +#define __BENCHMARK_COMMON_H + +/* Headers needed */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include "hdf5.h" + +/* Macros */ + +#ifndef TRUE +#define TRUE 1 +#endif /* TRUE */ +#ifndef FALSE +#define FALSE 0 +#endif /* FALSE */ + +#define NLEVELS 5 +#define NMAPPING 9 + +#define FILENAME "swmr_data.h5" +#define DTYPE_SIZE 150 + +/* Typedefs */ + +/* Information about a symbol/dataset */ +typedef struct { + char *name; /* Dataset name for symbol */ + hid_t dsid; /* Dataset ID for symbol */ + hsize_t nrecords; /* Number of records for the symbol */ +} symbol_info_t; + +/* A symbol's record */ +typedef struct { + uint64_t rec_id; /* ID for this record (unique in symbol) */ + uint8_t info[DTYPE_SIZE]; /* "Other" information for this record */ +} symbol_t; + +/* Global variables */ +extern symbol_info_t *symbol_info[NLEVELS]; +extern unsigned symbol_count[NLEVELS]; + +/* Prototypes */ +extern symbol_info_t * choose_dataset(void); +extern hid_t create_symbol_datatype(void); +extern int generate_name(char *name_buf, unsigned level, unsigned count); +extern int generate_symbols(void); +extern int shutdown_symbols(void); + +#endif /* __BENCHMARK_COMMON_H */ + diff --git a/test/swmr_generator.c b/test/swmr_generator.c new file mode 100644 index 0000000..3b4d7ef --- /dev/null +++ b/test/swmr_generator.c @@ -0,0 +1,188 @@ +#include "swmr_common.h" + +#define CHUNK_SIZE 50 + +static int +gen_skeleton(const char *filename, unsigned verbose, int comp_level) +{ + hid_t fid; /* File ID for new HDF5 file */ + hid_t fcpl; /* File creation property list */ + hid_t fapl; /* File access property list */ + hid_t dcpl; /* Dataset creation property list */ + hid_t tid; /* Datatype for dataset elements */ + hid_t sid; /* Dataspace ID */ + hsize_t dims = 0; /* Dataset starting dimensions */ + hsize_t max_dims = H5S_UNLIMITED; /* Dataset maximum dimensions */ + hsize_t chunk_dims = CHUNK_SIZE; /* Chunk dimensions */ + unsigned u, v; /* Local index variable */ + + /* Create file access property list */ + if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + return(-1); + if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + return(-1); + +#ifdef QAK +/* Increase the initial size of the metadata cache */ +{ + H5AC_cache_config_t mdc_config; + + mdc_config.version = H5AC__CURR_CACHE_CONFIG_VERSION; + H5Pget_mdc_config(fapl, &mdc_config); +printf("mdc_config.initial_size = %lu\n", (unsigned long)mdc_config.initial_size); +printf("mdc_config.epoch_length = %lu\n", (unsigned long)mdc_config.epoch_length); + mdc_config.set_initial_size = 1; + mdc_config.initial_size = 16 * 1024 * 1024; +/* mdc_config.epoch_length = 5000; */ + H5Pset_mdc_config(fapl, &mdc_config); +} +#endif /* QAK */ + +#ifdef QAK + H5Pset_small_data_block_size(fapl, (hsize_t)(50 * CHUNK_SIZE * DTYPE_SIZE)); +#endif /* QAK */ + +#ifdef QAK + H5Pset_fapl_log(fapl, "append.log", H5FD_LOG_ALL, (size_t)(512 * 1024 * 1024)); +#endif /* QAK */ + + /* Create file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + return(-1); + +#ifdef QAK + H5Pset_link_phase_change(fcpl, 0, 0); +#endif /* QAK */ + + /* Emit informational message */ + if(verbose) + printf("Creating file\n"); + + /* Create the file */ + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + return(-1); + + /* Close file creation property list */ + if(H5Pclose(fcpl) < 0) + return(-1); + + /* Close file access property list */ + if(H5Pclose(fapl) < 0) + return(-1); + + /* Create datatype for creating datasets */ + if((tid = create_symbol_datatype()) < 0) + return(-1); + + /* Create dataspace for creating datasets */ + if((sid = H5Screate_simple(1, &dims, &max_dims)) < 0) + return(-1); + + /* Create dataset creation property list */ + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + return(-1); + if(H5Pset_chunk(dcpl, 1, &chunk_dims) < 0) + return(-1); + if(comp_level >= 0) { + if(H5Pset_deflate(dcpl, (unsigned)comp_level) < 0) + return(-1); + } /* end if */ + + /* Emit informational message */ + if(verbose) + printf("Creating datasets\n"); + + /* Create the datasets */ + for(u = 0; u < NLEVELS; u++) + for(v = 0; v < symbol_count[u]; v++) { + hid_t dsid; /* Dataset ID */ + char name_buf[64]; + + generate_name(name_buf, u, v); + if((dsid = H5Dcreate2(fid, name_buf, tid, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + return(-1); + + if(H5Dclose(dsid) < 0) + return(-1); + } /* end for */ + + /* Emit informational message */ + if(verbose) + printf("Closing objects\n"); + + /* Close everythign */ + if(H5Pclose(dcpl) < 0) + return(-1); + if(H5Sclose(sid) < 0) + return(-1); + if(H5Tclose(tid) < 0) + return(-1); + if(H5Fclose(fid) < 0) + return(-1); + + return(0); +} /* end gen_skeleton() */ + +static void +usage(void) +{ + printf("Usage error!\n"); + printf("Usage: swmr_generator [-q] [-c ]\n"); + printf(" should be -1 (for no compression) or 0-9\n"); + printf("Defaults to verbose (no '-q' given) and no compression ('-c -1')\n"); + exit(1); +} /* end usage() */ + +int main(int argc, const char *argv[]) +{ + int comp_level = (-1); /* Compression level (-1 is no compression) */ + unsigned verbose = 1; /* Whether to emit some informational messages */ + unsigned u; /* Local index variables */ + + /* Parse command line options */ + if(argc > 1) { + u = 1; + while(u < (unsigned)argc) { + if(argv[u][0] == '-') { + switch(argv[u][1]) { + /* Compress dataset chunks */ + case 'c': + comp_level = atoi(argv[u + 1]); + if(comp_level < -1 || comp_level > 9) + usage(); + u += 2; + break; + + /* Be quiet */ + case 'q': + verbose = 0; + u++; + break; + + default: + usage(); + break; + } /* end switch */ + } /* end if */ + } /* end while */ + } /* end if */ + + /* Emit informational message */ + if(verbose) { + printf("Parameters:\n"); + printf("\tcompression level = %d\n", comp_level); + } /* end if */ + + /* Emit informational message */ + if(verbose) + printf("Generating skeleton file: %s\n", FILENAME); + + /* Generate file skeleton */ + if(gen_skeleton(FILENAME, verbose, comp_level) < 0) { + printf("Error generating skeleton file!\n"); + exit(1); + } /* end if */ + + return(0); +} + diff --git a/test/swmr_reader.c b/test/swmr_reader.c new file mode 100644 index 0000000..7045954 --- /dev/null +++ b/test/swmr_reader.c @@ -0,0 +1,345 @@ +#include "swmr_common.h" +#include + +static hid_t symbol_tid = (-1); + +static int +check_dataset(hid_t fid, unsigned verbose, const char *sym_name, symbol_t *record, + hid_t rec_sid) +{ + hid_t dsid; /* Dataset ID */ + hid_t file_sid; /* Dataset's space ID */ + hssize_t snpoints; /* Number of elements in dataset */ + hsize_t start, count = 1; /* Hyperslab selection values */ + + /* Open dataset for symbol */ + if((dsid = H5Dopen(fid, sym_name, H5P_DEFAULT)) < 0) + return(-1); + + /* Get the dataset's dataspace */ + if((file_sid = H5Dget_space(dsid)) < 0) + return(-1); + + /* Get the number of elements (= records, for 1-D datasets) */ + if((snpoints = H5Sget_simple_extent_npoints(file_sid)) < 0) + return(-1); + + /* Emit informational message */ + if(verbose) + printf("Symbol = '%s', # of records = %lld\n", sym_name, (long long)snpoints); + + /* Check if there are records for symbol */ + if(snpoints > 0) { + /* Choose the last record in the dataset */ + start = (hsize_t)(snpoints - 1); + if(H5Sselect_hyperslab(file_sid, H5S_SELECT_SET, &start, NULL, &count, NULL) < 0) + return(-1); + + /* Read record from dataset */ + record->rec_id = (uint64_t)ULLONG_MAX; + if(H5Dread(dsid, symbol_tid, rec_sid, file_sid, H5P_DEFAULT, record) < 0) + return(-1); + + /* Verify record value */ + if(record->rec_id != start) { + printf("Incorrect record value!\n"); + printf("Symbol = '%s', # of records = %lld, record->rec_id = %llu\n", sym_name, (long long)snpoints, (unsigned long long)record->rec_id); + return(-1); + } /* end if */ + } /* end if */ + + /* Close the dataset's dataspace */ + if(H5Sclose(file_sid) < 0) + return(-1); + + /* Close dataset for symbol */ + if(H5Dclose(dsid) < 0) + return(-1); + + return(0); +} /* end check_dataset() */ + +static int +read_records(const char *filename, unsigned verbose, unsigned long nseconds, + unsigned poll_time, unsigned ncommon, unsigned nrandom) +{ + time_t start_time; /* Starting time */ + time_t curr_time; /* Current time */ + symbol_info_t **sym_com = NULL, **sym_rand = NULL; /* Pointers to arrays of common & random dataset IDs */ + hid_t mem_sid; /* Memory dataspace ID */ + symbol_t record; /* The record to add to the dataset */ + unsigned v; /* Local index variable */ + + /* Reset the record */ + /* (record's 'info' field might need to change for each record written, also) */ + memset(&record, 0, sizeof(record)); + + /* Emit informational message */ + if(verbose) + printf("Choosing datasets\n"); + + /* Allocate space for 'common' datasets, if any */ + if(ncommon > 0) { + /* Allocate array to hold pointers to symbols for common datasets */ + if(NULL == (sym_com = malloc(sizeof(symbol_info_t *) * ncommon))) + return(-1); + + /* Open the common datasets */ + for(v = 0; v < ncommon; v++) { + unsigned offset; /* Offset of symbol to use */ + + /* Determine the offset of the symbol, within level 0 symbols */ + /* (level 0 symbols are the most common symbols) */ + offset = random() % symbol_count[0]; + sym_com[v] = &symbol_info[0][offset]; + + /* Emit informational message */ + if(verbose) + printf("Common symbol #%u = '%s'\n", v, symbol_info[0][offset].name); + } /* end for */ + } /* end if */ + + /* Allocate space for 'random' datasets, if any */ + if(nrandom > 0) { + /* Allocate array to hold pointers to symbols for random datasets */ + if(NULL == (sym_rand = malloc(sizeof(symbol_info_t *) * nrandom))) + return(-1); + + /* Determine the random datasets */ + for(v = 0; v < nrandom; v++) { + symbol_info_t *sym; /* Symbol to use */ + + /* Determine the symbol, within all symbols */ + if(NULL == (sym = choose_dataset())) + return(-1); + sym_rand[v] = sym; + + /* Emit informational message */ + if(verbose) + printf("Random symbol #%u = '%s'\n", v, sym->name); + } /* end for */ + } /* end if */ + + /* Create a dataspace for the record to read */ + if((mem_sid = H5Screate(H5S_SCALAR)) < 0) + return(-1); + + /* Emit informational message */ + if(verbose) + printf("Reading records\n"); + + /* Get the starting time */ + start_time = time(NULL); + curr_time = start_time; + + /* Loop over reading records until [at least] the correct # of seconds have passed */ + while(curr_time < (time_t)(start_time + nseconds)) { + hid_t fid; /* File ID */ + + /* Emit informational message */ + if(verbose) + printf("Opening file: %s\n", filename); + + /* Open the file */ + if((fid = H5Fopen(filename, H5F_ACC_RDONLY | H5F_ACC_SWMR_READ, H5P_DEFAULT)) < 0) + return(-1); + + /* Check 'common' datasets, if any */ + if(ncommon > 0) { + /* Emit informational message */ + if(verbose) + printf("Checking common symbols\n"); + + /* Iterate over common datasets */ + for(v = 0; v < ncommon; v++) { + /* Check common dataset */ + if(check_dataset(fid, verbose, sym_com[v]->name, &record, mem_sid) < 0) + return(-1); + } /* end for */ + } /* end if */ + + /* Check 'random' datasets, if any */ + if(nrandom > 0) { + /* Emit informational message */ + if(verbose) + printf("Checking random symbols\n"); + + /* Iterate over random datasets */ + for(v = 0; v < nrandom; v++) { + /* Check random dataset */ + if(check_dataset(fid, verbose, sym_rand[v]->name, &record, mem_sid) < 0) + return(-1); + } /* end for */ + } /* end if */ + + /* Emit informational message */ + if(verbose) + printf("Closing file\n"); + + /* Close the file */ + if(H5Fclose(fid) < 0) + return(-1); + + /* Sleep for the appropriate # of seconds */ + sleep(poll_time); + + /* Retrieve the current time */ + curr_time = time(NULL); + } /* end while */ + + /* Close the memory dataspace */ + if(H5Sclose(mem_sid) < 0) + return(-1); + + /* Emit informational message */ + if(verbose) + printf("Closing datasets\n"); + + /* Close 'random' datasets, if any */ + if(nrandom > 0) { + /* Release array holding dataset ID's for random datasets */ + free(sym_rand); + } /* end if */ + + /* Close 'common' datasets, if any */ + if(ncommon > 0) { + /* Release array holding dataset ID's for common datasets */ + free(sym_com); + } /* end if */ + + return(0); +} /* end read_records() */ + +static void +usage(void) +{ + printf("Usage error!\n"); + printf("Usage: swmr_reader [-q] [-s <# of seconds to sleep between polling>] [-h <# of common symbols to poll>] [-l <# of random symbols to poll>] <# of seconds to test>\n"); + printf("Defaults to verbose (no '-q' given), 1 second between polling ('-s 1'), 5 common symbols to poll ('-h 5') and 10 random symbols to poll ('-l 10')\n"); + exit(1); +} + +int main(int argc, const char *argv[]) +{ + time_t curr_time; /* Current time, for seeding random number generator */ + long nseconds = 0; /* # of seconds to test */ + int poll_time = 1; /* # of seconds between polling */ + int ncommon = 5; /* # of common symbols to poll */ + int nrandom = 10; /* # of random symbols to poll */ + unsigned verbose = 1; /* Whether to emit some informational messages */ + unsigned u; /* Local index variables */ + + /* Parse command line options */ + if(argc < 2) + usage(); + if(argc > 1) { + u = 1; + while(u < (unsigned)argc) { + if(argv[u][0] == '-') { + switch(argv[u][1]) { + /* # of seconds between polling */ + case 's': + poll_time = atoi(argv[u + 1]); + if(poll_time < 0) + usage(); + u += 2; + break; + + /* # of common symbols to poll */ + case 'h': + ncommon = atoi(argv[u + 1]); + if(ncommon < 0) + usage(); + u += 2; + break; + + /* # of random symbols to poll */ + case 'l': + nrandom = atoi(argv[u + 1]); + if(nrandom < 0) + usage(); + u += 2; + break; + + /* Be quiet */ + case 'q': + verbose = 0; + u++; + break; + + default: + usage(); + break; + } /* end switch */ + } /* end if */ + else { + /* Get the number of records to append */ + nseconds = atol(argv[u]); + if(nseconds <= 0) + usage(); + + u++; + } /* end else */ + } /* end while */ + } /* end if */ + if(nseconds <= 0) + usage(); + if(poll_time >= nseconds) + usage(); + + /* Emit informational message */ + if(verbose) { + printf("Parameters:\n"); + printf("\t# of seconds between polling = %d\n", poll_time); + printf("\t# of common symbols to poll = %d\n", ncommon); + printf("\t# of random symbols to poll = %d\n", nrandom); + printf("\t# of seconds to test = %ld\n", nseconds); + } /* end if */ + + /* Create randomized set of numbers */ + curr_time = time(NULL); + srandom((unsigned)curr_time); + + /* Emit informational message */ + if(verbose) + printf("Generating symbol names\n"); + + /* Generate dataset names */ + if(generate_symbols() < 0) { + printf("Error generating symbol names!\n"); + exit(1); + } /* end if */ + + /* Create datatype for creating datasets */ + if((symbol_tid = create_symbol_datatype()) < 0) + return(-1); + + /* Reading records from datasets */ + if(read_records(FILENAME, verbose, (unsigned long)nseconds, (unsigned)poll_time, (unsigned)ncommon, (unsigned)nrandom) < 0) { + printf("Error reading records from datasets!\n"); + exit(1); + } /* end if */ + + /* Emit informational message */ + if(verbose) + printf("Releasing symbols\n"); + + /* Clean up the symbols */ + if(shutdown_symbols() < 0) { + printf("Error releasing symbols!\n"); + exit(1); + } /* end if */ + + /* Emit informational message */ + if(verbose) + printf("Closing objects\n"); + + /* Close objects created */ + if(H5Tclose(symbol_tid) < 0) { + printf("Error closing symbol datatype!\n"); + exit(1); + } /* end if */ + + return(0); +} + diff --git a/test/swmr_writer.c b/test/swmr_writer.c new file mode 100644 index 0000000..0949872 --- /dev/null +++ b/test/swmr_writer.c @@ -0,0 +1,275 @@ +#include "swmr_common.h" + +static hid_t +open_skeleton(const char *filename, unsigned verbose) +{ + hid_t fid; /* File ID for new HDF5 file */ + hid_t fapl; /* File access property list */ + unsigned u, v; /* Local index variable */ + + /* Create file access property list */ + if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + return(-1); + +#ifdef QAK +/* Increase the initial size of the metadata cache */ +{ + H5AC_cache_config_t mdc_config; + + mdc_config.version = H5AC__CURR_CACHE_CONFIG_VERSION; + H5Pget_mdc_config(fapl, &mdc_config); +printf("mdc_config.initial_size = %lu\n", (unsigned long)mdc_config.initial_size); +printf("mdc_config.epoch_length = %lu\n", (unsigned long)mdc_config.epoch_length); + mdc_config.set_initial_size = 1; + mdc_config.initial_size = 16 * 1024 * 1024; +/* mdc_config.epoch_length = 5000; */ + H5Pset_mdc_config(fapl, &mdc_config); +} +#endif /* QAK */ + +#ifdef QAK + H5Pset_fapl_log(fapl, "append.log", H5FD_LOG_ALL, (size_t)(512 * 1024 * 1024)); +#endif /* QAK */ + + /* Open the file */ + if((fid = H5Fopen(filename, H5F_ACC_RDWR | H5F_ACC_SWMR_WRITE, fapl)) < 0) + return(-1); + + /* Close file access property list */ + if(H5Pclose(fapl) < 0) + return(-1); + + /* Emit informational message */ + if(verbose) + printf("Opening datasets\n"); + + /* Open the datasets */ + for(u = 0; u < NLEVELS; u++) + for(v = 0; v < symbol_count[u]; v++) { + if((symbol_info[u][v].dsid = H5Dopen2(fid, symbol_info[u][v].name, H5P_DEFAULT)) < 0) + return(-1); + symbol_info[u][v].nrecords = 0; + } /* end for */ + + return(fid); +} + +static int +add_records(hid_t fid, unsigned verbose, unsigned long nrecords, unsigned long flush_count) +{ + hid_t tid; /* Datatype ID for records */ + hid_t mem_sid; /* Memory dataspace ID */ + hsize_t start, count = 1; /* Hyperslab selection values */ + symbol_t record; /* The record to add to the dataset */ + unsigned long rec_to_flush; /* # of records left to write before flush */ + unsigned long u, v; /* Local index variables */ + + /* Reset the record */ + /* (record's 'info' field might need to change for each record written, also) */ + memset(&record, 0, sizeof(record)); + + /* Create a dataspace for the record to add */ + if((mem_sid = H5Screate(H5S_SCALAR)) < 0) + return(-1); + + /* Create datatype for appending records */ + if((tid = create_symbol_datatype()) < 0) + return(-1); + + /* Add records to random datasets, according to frequency distribution */ + rec_to_flush = flush_count; + for(u = 0; u < nrecords; u++) { + symbol_info_t *symbol; /* Symbol to write record to */ + hid_t file_sid; /* Dataset's space ID */ + + /* Get a random dataset, according to the symbol distribution */ + symbol = choose_dataset(); + + /* Set the record's ID (equal to its position) */ + record.rec_id = symbol->nrecords;; + + /* Get the coordinate to write */ + start = symbol->nrecords; + + /* Extend the dataset's dataspace to hold the new record */ + symbol->nrecords++; + if(H5Dset_extent(symbol->dsid, &symbol->nrecords) < 0) + return(-1); + + /* Get the dataset's dataspace */ + if((file_sid = H5Dget_space(symbol->dsid)) < 0) + return(-1); + + /* Choose the last record in the dataset */ + if(H5Sselect_hyperslab(file_sid, H5S_SELECT_SET, &start, NULL, &count, NULL) < 0) + return(-1); + + /* Write record to the dataset */ + if(H5Dwrite(symbol->dsid, tid, mem_sid, file_sid, H5P_DEFAULT, &record) < 0) + return(-1); + + /* Close the dataset's dataspace */ + if(H5Sclose(file_sid) < 0) + return(-1); + + /* Check for flushing file */ + if(flush_count > 0) { + /* Decrement count of records to write before flushing */ + rec_to_flush--; + + /* Check for counter being reached */ + if(0 == rec_to_flush) { + /* Flush contents of file */ + if(H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0) + return(-1); + + /* Reset flush counter */ + rec_to_flush = flush_count; + } /* end if */ + } /* end if */ + } /* end for */ + + /* Close the memory dataspace */ + if(H5Sclose(mem_sid) < 0) + return(-1); + + /* Close the datatype */ + if(H5Tclose(tid) < 0) + return(-1); + + /* Emit informational message */ + if(verbose) + printf("Closing datasets\n"); + + /* Close the datasets */ + for(u = 0; u < NLEVELS; u++) + for(v = 0; v < symbol_count[u]; v++) + if(H5Dclose(symbol_info[u][v].dsid) < 0) + return(-1); + + return(0); +} + +static void +usage(void) +{ + printf("Usage error!\n"); + printf("Usage: swmr_writer [-q] [-f <# of records to write between flushing file contents>] <# of records>\n"); + printf("<# of records to write between flushing file contents> should be 0 (for no flushing) or between 1 and (<# of records> - 1)\n"); + printf("Defaults to verbose (no '-q' given) and flushing every 10000 records('-f 10000')\n"); + exit(1); +} + +int main(int argc, const char *argv[]) +{ + hid_t fid; /* File ID for file opened */ + time_t curr_time; /* Current time, for seeding random number generator */ + long nrecords = 0; /* # of records to append */ + long flush_count = 10000; /* # of records to write between flushing file */ + unsigned verbose = 1; /* Whether to emit some informational messages */ + unsigned u; /* Local index variable */ + + /* Parse command line options */ + if(argc < 2) + usage(); + if(argc > 1) { + u = 1; + while(u < (unsigned)argc) { + if(argv[u][0] == '-') { + switch(argv[u][1]) { + /* # of records to write between flushing file */ + case 'f': + flush_count = atol(argv[u + 1]); + if(flush_count < 0) + usage(); + u += 2; + break; + + /* Be quiet */ + case 'q': + verbose = 0; + u++; + break; + + default: + usage(); + break; + } /* end switch */ + } /* end if */ + else { + /* Get the number of records to append */ + nrecords = atol(argv[u]); + if(nrecords <= 0) + usage(); + + u++; + } /* end else */ + } /* end while */ + } /* end if */ + if(nrecords <= 0) + usage(); + if(flush_count >= nrecords) + usage(); + + /* Emit informational message */ + if(verbose) { + printf("Parameters:\n"); + printf("\t# of records between flushes = %ld\n", flush_count); + printf("\t# of records to write = %ld\n", nrecords); + } /* end if */ + + /* Create randomized set of numbers */ + curr_time = time(NULL); + srandom((unsigned)curr_time); + + /* Emit informational message */ + if(verbose) + printf("Generating symbol names\n"); + + /* Generate dataset names */ + if(generate_symbols() < 0) + return(-1); + + /* Emit informational message */ + if(verbose) + printf("Opening skeleton file: %s\n", FILENAME); + + /* Open file skeleton */ + if((fid = open_skeleton(FILENAME, verbose)) < 0) { + printf("Error opening skeleton file!\n"); + exit(1); + } /* end if */ + + /* Emit informational message */ + if(verbose) + printf("Adding records\n"); + + /* Append records to datasets */ + if(add_records(fid, verbose, (unsigned long)nrecords, (unsigned long)flush_count) < 0) { + printf("Error appending records to datasets!\n"); + exit(1); + } /* end if */ + + /* Emit informational message */ + if(verbose) + printf("Releasing symbols\n"); + + /* Clean up the symbols */ + if(shutdown_symbols() < 0) { + printf("Error releasing symbols!\n"); + exit(1); + } /* end if */ + + /* Emit informational message */ + if(verbose) + printf("Closing objects\n"); + + /* Close objects opened */ + if(H5Fclose(fid) < 0) { + printf("Error closing file!\n"); + exit(1); + } /* end if */ + + return(0); +} + diff --git a/test/testswmr.sh b/test/testswmr.sh new file mode 100755 index 0000000..445b82e --- /dev/null +++ b/test/testswmr.sh @@ -0,0 +1,76 @@ +#! /bin/sh -x +# +# Copyright by The HDF Group. +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the files COPYING and Copyright.html. COPYING can be found at the root +# of the source code distribution tree; Copyright.html can be found at the +# root level of an installed copy of the electronic HDF5 document set and +# is linked from the top-level documents page. It can also be found at +# http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have +# access to either file, you may request a copy from help@hdfgroup.org. +# +# Tests for the swmr feature. +# +# Created: +# Albert Cheng, 2009/07/22 + + +############################################################################### +## test parameters +############################################################################### + +Nreaders=5 # number of readers to launch +Nrecords=1000000 # number of records to write +Nsecs=5 # number of seconds per read interval +nerrors=0 + +# The build (current) directory might be different than the source directory. +if test -z "$srcdir"; then + srcdir=. +fi + +# Parse option +# -p run ph5diff tests +# -h print help page +while [ $# -gt 0 ]; do + case "$1" in + *) # unknown option + echo "$0: Unknown option ($1)" + exit 1 + ;; + esac +done + +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Testing". +# +TESTING() { + SPACES=" " + echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' +} + +./swmr_generator +echo launch the swmr_writer +./swmr_writer $Nrecords & +# launch readers +n=0 +echo launch $Nreaders swmr_readers +while [ $n -lt $Nreaders ]; do + ./swmr_reader $Nsecs & + n=`expr $n + 1` +done +wait + +# ############################################################################## +# # END +# ############################################################################## + +if test $nerrors -eq 0 ; then + echo "SWMR tests passed." +fi + +exit $nerrors -- cgit v0.12