diff options
Diffstat (limited to 'test')
79 files changed, 14123 insertions, 2667 deletions
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index e3bfeb3..0474d49 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -16,6 +16,8 @@ set (TEST_LIB_SOURCES ${HDF5_TEST_SOURCE_DIR}/cache_common.c ${HDF5_TEST_SOURCE_DIR}/external_common.c ${HDF5_TEST_SOURCE_DIR}/swmr_common.c + ${HDF5_TEST_SOURCE_DIR}/vfd_swmr_common.c + ${HDF5_TEST_SOURCE_DIR}/vds_swmr_common.c ) set (TEST_LIB_HEADERS @@ -24,6 +26,7 @@ set (TEST_LIB_HEADERS ${HDF5_TEST_SOURCE_DIR}/external_common.h ${HDF5_TEST_SOURCE_DIR}/external_fname.h ${HDF5_TEST_SOURCE_DIR}/swmr_common.h + ${HDF5_TEST_SOURCE_DIR}/vfd_swmr_common.h ) if (NOT ONLY_SHARED_LIBS) @@ -317,8 +320,8 @@ set (H5_TESTS_MULTIPLE testhdf5 cache_image ttsafe - thread_id # special link mirror_vfd + thread_id # special link ) # Only build single source tests here foreach (h5_test ${H5_TESTS}) @@ -498,6 +501,20 @@ foreach (h5_test ${H5_VDS_SWMR_TESTS}) ADD_H5_EXE(${h5_test}) endforeach () +set (H5_VFD_SWMR_TESTS + vds_swmr_addrem_writer + vfd_swmr_generator + vfd_swmr_reader + vfd_swmr_remove_reader + vfd_swmr_remove_writer + vfd_swmr_sparse_writer + vfd_swmr_writer +) + +foreach (h5_test ${H5_VFD_SWMR_TESTS}) + ADD_H5_EXE(${h5_test}) +endforeach () + #-- Adding test for accum_swmr_reader # This has to be copied to the test directory for execve() to find it # and it can't be renamed (i.e., no <foo>-shared). diff --git a/test/Makefile.am b/test/Makefile.am index 805b482..a6049f3 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -37,6 +37,7 @@ AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_builddir)/src # test_usecases.sh: use_append_chunk, use_append_mchunks, use_disable_mdc_flushes TEST_SCRIPT = testerror.sh testlibinfo.sh testcheck_version.sh testlinks_env.sh testexternal_env.sh \ testswmr.sh testvds_env.sh testvdsswmr.sh testflushrefresh.sh test_usecases.sh testabort_fail.sh \ + testvfdswmr.sh \ test_mirror.sh SCRIPT_DEPEND = error_test$(EXEEXT) err_compat$(EXEEXT) links_env$(EXEEXT) \ external_env$(EXEEXT) filenotclosed$(EXEEXT) del_many_dense_attrs$(EXEEXT) \ @@ -44,7 +45,16 @@ SCRIPT_DEPEND = error_test$(EXEEXT) err_compat$(EXEEXT) links_env$(EXEEXT) \ swmr_generator$(EXEEXT) swmr_reader$(EXEEXT) swmr_writer$(EXEEXT) \ swmr_remove_reader$(EXEEXT) swmr_remove_writer$(EXEEXT) swmr_addrem_writer$(EXEEXT) \ swmr_sparse_reader$(EXEEXT) swmr_sparse_writer$(EXEEXT) swmr_start_write$(EXEEXT) \ - vds_env$(EXEEXT) vds_swmr_gen$(EXEEXT) vds_swmr_reader$(EXEEXT) vds_swmr_writer$(EXEEXT) + vfd_swmr_generator$(EXEEXT) vfd_swmr_reader$(EXEEXT) vfd_swmr_writer$(EXEEXT) \ + vfd_swmr_remove_reader$(EXEEXT) vfd_swmr_remove_writer$(EXEEXT) \ + vfd_swmr_addrem_writer$(EXEEXT) vfd_swmr_sparse_reader$(EXEEXT) \ + vfd_swmr_sparse_writer$(EXEEXT) \ + vfd_swmr_bigset_reader$(EXEEXT) vfd_swmr_bigset_writer$(EXEEXT) \ + vfd_swmr_group_reader$(EXEEXT) vfd_swmr_group_writer$(EXEEXT) \ + vfd_swmr_vlstr_reader$(EXEEXT) vfd_swmr_vlstr_writer$(EXEEXT) \ + vfd_swmr_zoo_reader$(EXEEXT) vfd_swmr_zoo_writer$(EXEEXT) \ + vds_env$(EXEEXT) \ + vds_swmr_gen$(EXEEXT) vds_swmr_reader$(EXEEXT) vds_swmr_writer$(EXEEXT) if HAVE_SHARED_CONDITIONAL TEST_SCRIPT += test_filter_plugin.sh test_vol_plugin.sh SCRIPT_DEPEND += filter_plugin$(EXEEXT) vol_plugin$(EXEEXT) @@ -59,7 +69,7 @@ check_SCRIPTS = $(TEST_SCRIPT) TEST_PROG= testhdf5 \ cache cache_api cache_image cache_tagging lheap ohdr \ stab gheap evict_on_close farray earray btree2 fheap \ - pool accum hyperslab istore bittests dt_arith page_buffer \ + pool accum hyperslab istore bittests dt_arith page_buffer vfd_swmr \ dtypes dsets chunk_info cmpd_dset filter_fail extend direct_chunk \ external efc objcopy objcopy_ref links unlink twriteorder big mtime fillval mount \ flush1 flush2 app_ref enum set_extent ttsafe enc_dec_plist \ @@ -90,6 +100,13 @@ check_PROGRAMS=$(TEST_PROG) error_test err_compat tcheck_version \ use_append_chunk use_append_chunk_mirror use_append_mchunks use_disable_mdc_flushes \ swmr_generator swmr_start_write swmr_reader swmr_writer swmr_remove_reader \ swmr_remove_writer swmr_addrem_writer swmr_sparse_reader swmr_sparse_writer \ + vfd_swmr_generator vfd_swmr_reader vfd_swmr_writer \ + vfd_swmr_remove_reader vfd_swmr_remove_writer vfd_swmr_addrem_writer \ + vfd_swmr_sparse_reader vfd_swmr_sparse_writer \ + vfd_swmr_bigset_reader vfd_swmr_bigset_writer \ + vfd_swmr_group_reader vfd_swmr_group_writer \ + vfd_swmr_vlstr_reader vfd_swmr_vlstr_writer \ + vfd_swmr_zoo_reader vfd_swmr_zoo_writer \ swmr_check_compat_vfd vds_env vds_swmr_gen vds_swmr_reader vds_swmr_writer \ mirror_vfd if HAVE_SHARED_CONDITIONAL @@ -139,7 +156,7 @@ else noinst_LTLIBRARIES=libh5test.la endif -libh5test_la_SOURCES=h5test.c testframe.c cache_common.c swmr_common.c external_common.c +libh5test_la_SOURCES=h5test.c testframe.c cache_common.c swmr_common.c external_common.c stubs.c vfd_swmr_common.c # Use libhd5test.la to compile all of the tests LDADD=libh5test.la $(LIBHDF5) @@ -150,6 +167,12 @@ ttsafe_SOURCES=ttsafe.c ttsafe_dcreate.c ttsafe_error.c ttsafe_cancel.c \ cache_image_SOURCES=cache_image.c genall5.c mirror_vfd_SOURCES=mirror_vfd.c genall5.c +vfd_swmr_zoo_writer_SOURCES=vfd_swmr_zoo_writer.c genall5.c +vfd_swmr_zoo_reader_SOURCES=vfd_swmr_zoo_writer.c genall5.c + +vfd_swmr_bigset_reader_SOURCES=vfd_swmr_bigset_writer.c +vfd_swmr_group_reader_SOURCES=vfd_swmr_group_writer.c + VFD_LIST = sec2 stdio core core_paged split multi family if DIRECT_VFD_CONDITIONAL VFD_LIST += direct diff --git a/test/cache_common.c b/test/cache_common.c index 1dc13a1..651534d 100644 --- a/test/cache_common.c +++ b/test/cache_common.c @@ -269,6 +269,7 @@ static const H5C_class_t pico_class[1] = {{ NULL, pico_free_icr, NULL, + NULL, }}; static const H5C_class_t nano_class[1] = {{ @@ -286,6 +287,7 @@ static const H5C_class_t nano_class[1] = {{ NULL, nano_free_icr, NULL, + NULL, }}; static const H5C_class_t micro_class[1] = {{ @@ -303,6 +305,7 @@ static const H5C_class_t micro_class[1] = {{ NULL, micro_free_icr, NULL, + NULL, }}; static const H5C_class_t tiny_class[1] = {{ @@ -320,6 +323,7 @@ static const H5C_class_t tiny_class[1] = {{ NULL, tiny_free_icr, NULL, + NULL, }}; static const H5C_class_t small_class[1] = {{ @@ -337,6 +341,7 @@ static const H5C_class_t small_class[1] = {{ NULL, small_free_icr, NULL, + NULL, }}; static const H5C_class_t medium_class[1] = {{ @@ -354,6 +359,7 @@ static const H5C_class_t medium_class[1] = {{ NULL, medium_free_icr, NULL, + NULL, }}; static const H5C_class_t large_class[1] = {{ @@ -371,6 +377,7 @@ static const H5C_class_t large_class[1] = {{ NULL, large_free_icr, NULL, + NULL, }}; static const H5C_class_t huge_class[1] = {{ @@ -388,6 +395,7 @@ static const H5C_class_t huge_class[1] = {{ NULL, huge_free_icr, NULL, + NULL, }}; static const H5C_class_t monster_class[1] = {{ @@ -405,6 +413,7 @@ static const H5C_class_t monster_class[1] = {{ NULL, monster_free_icr, NULL, + NULL, }}; static const H5C_class_t variable_class[1] = {{ @@ -422,6 +431,7 @@ static const H5C_class_t variable_class[1] = {{ NULL, variable_free_icr, NULL, + NULL, }}; static const H5C_class_t notify_class[1] = {{ @@ -439,6 +449,7 @@ static const H5C_class_t notify_class[1] = {{ notify_notify, notify_free_icr, NULL, + NULL, }}; /* callback table declaration */ diff --git a/test/cache_image.c b/test/cache_image.c index 59689a9..d4002c7 100644 --- a/test/cache_image.c +++ b/test/cache_image.c @@ -22,7 +22,6 @@ /* global variable declarations: */ - const char *FILENAMES[] = { "cache_image_test", NULL @@ -4820,7 +4819,9 @@ cache_image_smoke_check_5(hbool_t single_file_vfd) /* 3) Construct a "zoo" in the above group, and validate it. */ if ( pass ) - create_zoo(file_id, process_group_name, min_group); + pass = create_zoo(file_id, process_group_name, + (zoo_config_t){.proc_num = min_group, .skip_varlen = false, + .skip_compact = false}); #if H5C_COLLECT_CACHE_STATS if ( pass ) { @@ -4891,8 +4892,11 @@ cache_image_smoke_check_5(hbool_t single_file_vfd) /* 6) Validate the "zoo" created in the previous file open. */ - if ( pass ) - validate_zoo(file_id, process_group_name, max_group); + if ( pass ) { + pass = validate_zoo(file_id, process_group_name, + (zoo_config_t){.proc_num = max_group, .skip_varlen = false, + .skip_compact = false}); + } #if H5C_COLLECT_CACHE_STATS if ( pass ) { @@ -4932,8 +4936,11 @@ cache_image_smoke_check_5(hbool_t single_file_vfd) /* 8) Construct a "zoo" in the above group, and validate it. */ - if ( pass ) - create_zoo(file_id, process_group_name, max_group); + if ( pass ) { + pass = create_zoo(file_id, process_group_name, + (zoo_config_t){.proc_num = max_group, .skip_varlen = false, + .skip_compact = false}); + } if ( show_progress ) HDfprintf(stdout, "%s:L4 cp = %d, max_group = %d, pass = %d.\n", @@ -4994,7 +5001,9 @@ cache_image_smoke_check_5(hbool_t single_file_vfd) i = min_group; while(pass && i <= max_group) { HDsprintf(process_group_name, "/process_%d", i); - validate_zoo(file_id, process_group_name, i++); + pass = validate_zoo(file_id, process_group_name, + (zoo_config_t){.proc_num = i++, .skip_varlen = false, + .skip_compact = false}); } #if H5C_COLLECT_CACHE_STATS @@ -5047,7 +5056,9 @@ cache_image_smoke_check_5(hbool_t single_file_vfd) while ( ( pass ) && ( i <= max_group ) ) { HDsprintf(process_group_name, "/process_%d", i); - validate_zoo(file_id, process_group_name, i++); + pass = validate_zoo(file_id, process_group_name, + (zoo_config_t){.proc_num = i++, .skip_varlen = false, + .skip_compact = false}); } #if H5C_COLLECT_CACHE_STATS @@ -5111,7 +5122,9 @@ cache_image_smoke_check_5(hbool_t single_file_vfd) i = min_group; while ( ( pass ) && ( i <= max_group ) ) { HDsprintf(process_group_name, "/process_%d", i); - validate_zoo(file_id, process_group_name, i++); + pass = validate_zoo(file_id, process_group_name, + (zoo_config_t){.proc_num = i++, .skip_varlen = false, + .skip_compact = false}); } #if H5C_COLLECT_CACHE_STATS diff --git a/test/cache_tagging.c b/test/cache_tagging.c index cf0cd8d..ddd63e4 100644 --- a/test/cache_tagging.c +++ b/test/cache_tagging.c @@ -443,7 +443,7 @@ check_file_creation_tags(hid_t fcpl_id, int type) hid_t fid = -1; /* File Identifier */ #ifndef NDEBUG int verbose = FALSE; /* verbose test outout */ -#endif /* NDEBUG */ /* end debugging functions */ +#endif hid_t fapl = -1; /* File access prop list */ haddr_t root_tag = 0; haddr_t sbe_tag = 0; @@ -532,7 +532,7 @@ check_file_open_tags(hid_t fcpl, int type) hid_t fid = -1; /* File Identifier */ #ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ -#endif /* NDEBUG */ /* end debugging functions */ +#endif hid_t fapl = -1; /* File access prop list */ haddr_t root_tag; /* Root Group Tag */ haddr_t sbe_tag; /* Sblock Extension Tag */ @@ -647,7 +647,7 @@ check_group_creation_tags(void) hid_t gid = -1; /* Group Identifier */ #ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ -#endif /* NDEBUG */ /* end debugging functions */ +#endif hid_t fapl = -1; /* File access prop list */ haddr_t root_tag = HADDR_UNDEF; /* Root Group Tag */ haddr_t g_tag; /* Group Tag */ @@ -750,7 +750,7 @@ check_multi_group_creation_tags(void) hid_t gid = -1; /* Group Identifier */ #ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ -#endif /* NDEBUG */ /* end debugging functions */ +#endif char gname[16]; /* group name buffer */ int i = 0; /* iterator */ hid_t fapl = -1; /* File access prop list */ @@ -881,7 +881,7 @@ check_link_iteration_tags(void) hid_t did = -1; /* Group Identifier */ #ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ -#endif /* NDEBUG */ /* end debugging functions */ +#endif int i = 0; /* iterator */ haddr_t root_tag = 0; /* Root Group Tag Value */ char dsetname[500]; /* Name of dataset */ @@ -1003,7 +1003,7 @@ check_dense_attribute_tags(void) hid_t dcpl = -1; /* Group Identifier */ #ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ -#endif /* NDEBUG */ /* end debugging functions */ +#endif int i = 0; /* iterator */ hid_t fapl = -1; /* File access property list */ haddr_t d_tag = 0; /* Dataset tag value */ @@ -1187,7 +1187,7 @@ check_group_open_tags(void) hid_t gid = -1; /* Group Identifier */ #ifndef NDEBUG int verbose = FALSE; /* verbose file output */ -#endif /* NDEBUG */ /* end debugging functions */ +#endif hid_t fapl = -1; /* File access prop list */ haddr_t root_tag = HADDR_UNDEF; haddr_t g_tag; @@ -1298,7 +1298,7 @@ check_attribute_creation_tags(hid_t fcpl, int type) hid_t sid = -1; /* Dataspace Identifier */ #ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ -#endif /* NDEBUG */ /* end debugging functions */ +#endif hid_t fapl = -1; /* File access prop list */ haddr_t root_tag = 0; /* Root group tag */ haddr_t g_tag = 0; @@ -1434,7 +1434,7 @@ check_attribute_open_tags(hid_t fcpl, int type) hid_t sid = -1; /* Dataspace Identifier */ #ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ -#endif /* NDEBUG */ /* end debugging functions */ +#endif hid_t fapl = -1; /* File access prop list */ haddr_t root_tag = 0; haddr_t g_tag = 0; @@ -1573,7 +1573,7 @@ check_attribute_rename_tags(hid_t fcpl, int type) hid_t sid = -1; /* Dataset Identifier */ #ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ -#endif /* NDEBUG */ /* end debugging functions */ +#endif int *data = NULL; /* data buffer */ int i,j,k = 0; /* iterators */ hid_t fapl = -1; /* File access prop list */ @@ -1750,7 +1750,7 @@ check_attribute_delete_tags(hid_t fcpl, int type) hid_t sid = -1; /* Dataset Identifier */ #ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ -#endif /* NDEBUG */ /* end debugging functions */ +#endif int *data = NULL; /* data buffer */ int i,j,k = 0; /* iterators */ hid_t fapl = -1; /* File access prop list */ @@ -1918,7 +1918,7 @@ check_dataset_creation_tags(hid_t fcpl, int type) hid_t sid = -1; /* Dataspace Identifier */ #ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ -#endif /* NDEBUG */ /* end debugging functions */ +#endif hid_t dcpl = -1; /* dataset creation pl */ hsize_t cdims[2] = {1,1}; /* chunk dimensions */ int fillval = 0; @@ -2053,7 +2053,7 @@ check_dataset_creation_earlyalloc_tags(hid_t fcpl, int type) hid_t sid = -1; /* Dataspace Identifier */ #ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ -#endif /* NDEBUG */ /* end debugging functions */ +#endif hid_t dcpl = -1; /* dataset creation pl */ hsize_t cdims[2] = {1,1}; /* chunk dimensions */ int fillval = 0; @@ -2192,7 +2192,7 @@ check_dataset_open_tags(void) hid_t sid = -1; /* Dataspace Identifier */ #ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ -#endif /* NDEBUG */ /* end debugging functions */ +#endif hid_t dcpl = -1; /* dataset creation pl */ hsize_t cdims[2] = {1,1}; /* chunk dimensions */ int fillval = 0; @@ -2320,7 +2320,7 @@ check_dataset_write_tags(void) hid_t sid = -1; /* Dataspace Identifier */ #ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ -#endif /* NDEBUG */ /* end debugging functions */ +#endif hid_t dcpl = -1; /* dataset creation pl */ hsize_t cdims[2] = {1,1}; /* chunk dimensions */ int fillval = 0; @@ -2464,7 +2464,7 @@ check_attribute_write_tags(hid_t fcpl, int type) hid_t sid = -1; /* Dataset Identifier */ #ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ -#endif /* NDEBUG */ /* end debugging functions */ +#endif int *data = NULL; /* data buffer */ int i,j,k = 0; /* iterators */ hid_t fapl = -1; /* File access prop list */ @@ -2619,7 +2619,7 @@ check_dataset_read_tags(void) hid_t sid = -1; /* Dataspace Identifier */ #ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ -#endif /* NDEBUG */ /* end debugging functions */ +#endif hid_t dcpl = -1; /* dataset creation pl */ hsize_t cdims[2] = {1,1}; /* chunk dimensions */ int fillval = 0; @@ -2758,7 +2758,7 @@ check_dataset_size_retrieval(void) hid_t sid = -1; /* Dataspace Identifier */ #ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ -#endif /* NDEBUG */ /* end debugging functions */ +#endif hid_t dcpl = -1; /* dataset creation pl */ hsize_t cdims[2] = {1,1}; /* chunk dimensions */ int fillval = 0; @@ -2899,7 +2899,7 @@ check_dataset_extend_tags(void) hid_t sid = -1; /* Dataspace Identifier */ #ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ -#endif /* NDEBUG */ /* end debugging functions */ +#endif hid_t dcpl = -1; /* dataset creation pl */ hsize_t cdims[2] = {1,1}; /* chunk dimensions */ int fillval = 0; @@ -3038,7 +3038,7 @@ check_object_info_tags(void) hid_t gid = -1; /* Group Identifier */ #ifndef NDEBUG int verbose = FALSE; /* verbose file output */ -#endif /* NDEBUG */ /* end debugging functions */ +#endif hid_t fapl = -1; /* File access prop list */ haddr_t root_tag = HADDR_UNDEF; haddr_t g_tag; @@ -3152,7 +3152,7 @@ check_object_copy_tags(void) hid_t gid = -1; /* Group Identifier */ #ifndef NDEBUG int verbose = FALSE; /* verbose file output */ -#endif /* NDEBUG */ /* end debugging functions */ +#endif hid_t fapl = -1; /* File access prop list */ haddr_t root_tag = HADDR_UNDEF; haddr_t g_tag; @@ -3275,7 +3275,7 @@ check_link_removal_tags(hid_t fcpl, int type) hid_t gid = -1; /* Dataspace Identifier */ #ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ -#endif /* NDEBUG */ /* end debugging functions */ +#endif hid_t dcpl = -1; /* dataset creation pl */ hsize_t cdims[2] = {1,1}; /* chunk dimensions */ int fillval = 0; @@ -3436,7 +3436,7 @@ check_link_getname_tags(void) hid_t gid = -1; /* Dataspace Identifier */ #ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ -#endif /* NDEBUG */ /* end debugging functions */ +#endif hid_t dcpl = -1; /* dataset creation pl */ hsize_t cdims[2] = {1,1}; /* chunk dimensions */ int fillval = 0; @@ -3587,7 +3587,7 @@ check_external_link_creation_tags(void) hid_t gid = -1; /* Dataspace Identifier */ #ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ -#endif /* NDEBUG */ /* end debugging functions */ +#endif hid_t fapl = -1; /* File access prop list */ haddr_t root_tag = 0; @@ -3695,7 +3695,7 @@ check_external_link_open_tags(void) hid_t xid = -1; /* Dataspace Identifier */ #ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ -#endif /* NDEBUG */ /* end debugging functions */ +#endif H5O_native_info_t ninfo; /* Native object info struct */ hid_t fapl = -1; /* File access prop list */ haddr_t root_tag = 0; @@ -3900,10 +3900,9 @@ check_invalid_tag_application(void) #if H5C_DO_TAGGING_SANITY_CHECKS error: if(api_ctx_pushed) H5CX_pop(); - - return 1; #endif /* H5C_DO_TAGGING_SANITY_CHECKS */ + return 1; } /* check_invalid_tag_application */ diff --git a/test/earray.c b/test/earray.c index 6597afd..82c482d 100644 --- a/test/earray.c +++ b/test/earray.c @@ -2106,7 +2106,7 @@ test_set_elmts(hid_t fapl, H5EA_create_t *cparam, earray_test_param_t *tparam, /* * Display testing message */ - TESTING(test_str); + TESTING("%s", test_str); /* Create file & retrieve pointer to internal file object */ if(create_file(H5F_ACC_TRUNC, fapl, &file, &f) < 0) @@ -2280,7 +2280,7 @@ test_skip_elmts(hid_t fapl, H5EA_create_t *cparam, earray_test_param_t *tparam, /* * Display testing message */ - TESTING(test_str); + TESTING("%s", test_str); /* Create file & retrieve pointer to internal file object */ if(create_file(H5F_ACC_TRUNC, fapl, &file, &f) < 0) diff --git a/test/enc_dec_plist_cross_platform.c b/test/enc_dec_plist_cross_platform.c index f54f675..c4dec9f 100644 --- a/test/enc_dec_plist_cross_platform.c +++ b/test/enc_dec_plist_cross_platform.c @@ -156,73 +156,70 @@ error: return 1; } +static hid_t +read_and_decode_plist_file(const char *filename_prefix, unsigned config, char *filename, size_t filename_len) +{ + int fd; + size_t size; + const char *testfile; + void *buf = NULL; + hid_t plist = H5I_INVALID_HID; + + /* Generate filename from prefix and configuration word */ + if(HDsnprintf(filename, filename_len, "%s%s%s", filename_prefix, + config & CONFIG_64 ? "64" : "32", + config & CONFIG_LE ? "le" : "be") < 0) + TEST_ERROR + + /* Read file 1 */ + testfile = H5_get_srcdir_filename(filename); + if((fd = HDopen(testfile, O_RDONLY)) < 0) + TEST_ERROR + size = (size_t)HDlseek(fd, 0, SEEK_END); + HDlseek(fd, 0, SEEK_SET); + buf = HDmalloc(size); + if(HDread(fd, buf, size) < 0) + TEST_ERROR + HDclose(fd); + + /* Decode property lists */ + if((plist = H5Pdecode(buf)) < 0) + FAIL_STACK_ERROR +error: + if (buf != NULL) + HDfree(buf); + return plist; +} + static int test_plists(const char *filename_prefix) { - unsigned config_1, config_2; - int fd_1, fd_2; - size_t size_1 = 0, size_2 = 0; - void *buf_1 = NULL, *buf_2 = NULL; - hid_t plist_1, plist_2; - char filename[1024]; - const char *testfile; + int i; + unsigned config[2]; + hid_t plist[2]; + char filename[2][1024]; /* Iterate over all combinations of configurations */ - for(config_1 = 0; config_1 < (NCONFIG - 1); config_1++) - for(config_2 = config_1 + 1; config_2 < NCONFIG; config_2++) { - /* Generate filename for file 1 */ - if(HDsnprintf(filename, sizeof(filename), "%s%s%s", filename_prefix, - config_1 & CONFIG_64 ? "64" : "32", - config_1 & CONFIG_LE ? "le" : "be") < 0) - TEST_ERROR - - /* Read file 1 */ - testfile = H5_get_srcdir_filename(filename); - if((fd_1 = HDopen(testfile, O_RDONLY)) < 0) - TEST_ERROR - size_1 = (size_t)HDlseek(fd_1, (HDoff_t)0, SEEK_END); - HDlseek(fd_1, (HDoff_t)0, SEEK_SET); - buf_1 = (void *)HDmalloc(size_1); - if(HDread(fd_1, buf_1, size_1) < 0) - TEST_ERROR - HDclose(fd_1); - - /* Generate filename for file 2 */ - if(HDsnprintf(filename, sizeof(filename), "%s%s%s", filename_prefix, - config_2 & CONFIG_64 ? "64" : "32", - config_2 & CONFIG_LE ? "le" : "be") < 0) - TEST_ERROR - - /* Read file 1 */ - testfile = H5_get_srcdir_filename(filename); - if((fd_2 = HDopen(testfile, O_RDONLY)) < 0) - TEST_ERROR - size_2 = (size_t)HDlseek(fd_2, (HDoff_t)0, SEEK_END); - HDlseek(fd_2, (HDoff_t)0, SEEK_SET); - buf_2 = (void *)HDmalloc(size_2); - if(HDread(fd_2, buf_2, size_2) < 0) - TEST_ERROR - HDclose(fd_2); - - /* Decode property lists */ - if((plist_1 = H5Pdecode(buf_1)) < 0) - FAIL_STACK_ERROR - if((plist_2 = H5Pdecode(buf_2)) < 0) - FAIL_STACK_ERROR + for(config[0] = 0; config[0] < (NCONFIG - 1); config[0]++) { + for(config[1] = config[0] + 1; config[1] < NCONFIG; config[1]++) { + for (i = 0; i < 2; i++) { + plist[i] = read_and_decode_plist_file(filename_prefix, + config[i], filename[i], sizeof(filename[i])); + if (plist[i] == H5I_INVALID_HID) + goto error; + } /* Compare decoded property lists */ - if(!H5Pequal(plist_1, plist_2)) - FAIL_PUTS_ERROR("PLIST encoding/decoding comparison failed\n") - - /* Close */ - if((H5Pclose(plist_1)) < 0) - FAIL_STACK_ERROR - if((H5Pclose(plist_2)) < 0) - FAIL_STACK_ERROR - - HDfree(buf_1); - HDfree(buf_2); - } /* end for */ + if(!H5Pequal(plist[0], plist[1])) + FAIL_PRINTF_ERROR("PLIST encoding/decoding comparison failed, " + "%s != %s\n", filename[0], filename[1]) + + for (i = 0; i < 2; i++) { + if((H5Pclose(plist[i])) < 0) + FAIL_STACK_ERROR + } + } + } return 1; diff --git a/test/err_compat.c b/test/err_compat.c index eb86760..bd317bf 100644 --- a/test/err_compat.c +++ b/test/err_compat.c @@ -241,7 +241,7 @@ test_error1(void) herr_t ret; TESTING("error API H5Eset/get_auto"); - HDfprintf(stderr, "\n"); + printf("\n"); /* Create the data space */ dims[0] = DIM0; @@ -338,6 +338,7 @@ test_error1(void) if(dataset >= 0) TEST_ERROR; + fprintf(stderr, "\n"); return 0; error: @@ -370,7 +371,7 @@ test_error2(hid_t file) const char *FUNC_test_error="test_error2"; TESTING("error API based on data I/O"); - HDfprintf(stderr, "\n"); + printf("\n"); /* Create the data space */ dims[0] = DIM0; @@ -504,6 +505,7 @@ main(void) /* Print out the errors on stack */ dump_error(); + fprintf(stderr, "\n"); /* Empty error stack */ H5Eclear1(); diff --git a/test/farray.c b/test/farray.c index b647b51..2cc79a1 100644 --- a/test/farray.c +++ b/test/farray.c @@ -1376,7 +1376,7 @@ test_set_elmts(hid_t fapl, H5FA_create_t *cparam, farray_test_param_t *tparam, /* * Display testing message */ - TESTING(test_str); + TESTING("%s", test_str); /* Create file & retrieve pointer to internal file object */ if(create_file(fapl, &file, &f) < 0) @@ -1531,7 +1531,7 @@ test_skip_elmts(hid_t fapl, H5FA_create_t *cparam, farray_test_param_t *tparam, /* * Display testing message */ - TESTING(test_str); + TESTING("%s", test_str); /* Create file & retrieve pointer to internal file object */ if(create_file(fapl, &file, &f) < 0) diff --git a/test/fheap.c b/test/fheap.c index b1a0db0..b6d24f7 100644 --- a/test/fheap.c +++ b/test/fheap.c @@ -558,11 +558,8 @@ begin_test(fheap_test_param_t *tparam, const char *base_desc, */ del_str = get_del_string(tparam); HDassert(del_str); - test_desc = (char *)H5MM_malloc(HDstrlen(del_str) + HDstrlen(base_desc)); - HDsprintf(test_desc, base_desc, del_str); - TESTING(test_desc); + TESTING(base_desc, del_str); H5MM_xfree(del_str); - H5MM_xfree(test_desc); /* Initialize the heap ID structure */ HDmemset(keep_ids, 0, sizeof(fheap_heap_ids_t)); diff --git a/test/file_image.c b/test/file_image.c index 0d4873a..bb16e6d 100644 --- a/test/file_image.c +++ b/test/file_image.c @@ -710,7 +710,7 @@ test_get_file_image(const char * test_banner, hid_t fcpl = -1; herr_t ret; - TESTING(test_banner); + TESTING("%s", test_banner); /* set flag if we are dealing with a family file */ driver = H5Pget_driver(fapl); diff --git a/test/gen_plist.c b/test/gen_plist.c index b6f5597..7f64aa4 100644 --- a/test/gen_plist.c +++ b/test/gen_plist.c @@ -469,7 +469,7 @@ encode_plist(hid_t plist_id, int little_endian, int word_length, const char *fil if((ret = H5Pencode2(plist_id, NULL, &temp_size, H5P_DEFAULT)) < 0) HDassert(ret > 0); - temp_buf = (void *)HDmalloc(temp_size); + temp_buf = HDcalloc(1, temp_size); HDassert(temp_buf); if((ret = H5Pencode2(plist_id, temp_buf, &temp_size, H5P_DEFAULT)) < 0) diff --git a/test/genall5.c b/test/genall5.c index 3f55930..0a64f37 100644 --- a/test/genall5.c +++ b/test/genall5.c @@ -19,6 +19,8 @@ * of the same name. */ +#include <err.h> + #include "cache_common.h" #include "genall5.h" @@ -27,6 +29,70 @@ #define DSET_CHUNK_DIMS 1024 #define DSET_COMPACT_DIMS 4096 +typedef enum phase {PHASE_CREATE, PHASE_VALIDATE, PHASE_DELETE, + PHASE_VALIDATE_DELETION} phase_t; + +static bool rm_ns_grp_0(hid_t, const char *); +static bool rm_ns_grp_c(hid_t, const char *, unsigned); +static bool rm_ns_grp_d(hid_t, const char *, unsigned); +static bool rm_os_grp_0(hid_t, const char *); +static bool rm_os_grp_n(hid_t, const char *, int, unsigned); +static bool rm_ds_ctg_i(hid_t, const char *, hbool_t); +static bool rm_ds_chk_i(hid_t, const char *, hbool_t); +static bool rm_ds_cpt_i(hid_t, const char *, hbool_t); +static bool rm_ds_ctg_v(hid_t, const char *, hbool_t); + +static bool missing_ns_grp_0(hid_t, const char *); +static bool missing_ns_grp_c(hid_t, const char *, unsigned); +static bool missing_ns_grp_d(hid_t, const char *, unsigned); +static bool missing_os_grp_0(hid_t, const char *); +static bool missing_os_grp_n(hid_t, const char *, int, unsigned); +static bool missing_ds_ctg_i(hid_t, const char *, hbool_t); +static bool missing_ds_chk_i(hid_t, const char *, hbool_t); +static bool missing_ds_cpt_i(hid_t, const char *, hbool_t); +static bool missing_ds_ctg_v(hid_t, const char *, hbool_t); + +#define FN_ITEM_DEFN(__name, ...) \ + typedef bool (*__name##fn_t)(__VA_ARGS__); \ + static const __name##fn_t __name##_fntbl[] = \ + {__name, vrfy_##__name, rm_##__name, missing_##__name} + +FN_ITEM_DEFN(ns_grp_0, hid_t, const char *); +FN_ITEM_DEFN(ns_grp_c, hid_t, const char *, unsigned); +FN_ITEM_DEFN(ns_grp_d, hid_t, const char *, unsigned); +FN_ITEM_DEFN(os_grp_0, hid_t, const char *); +FN_ITEM_DEFN(os_grp_n, hid_t, const char *, int, unsigned); +FN_ITEM_DEFN(ds_ctg_i, hid_t, const char *, bool); +FN_ITEM_DEFN(ds_chk_i, hid_t, const char *, bool); +FN_ITEM_DEFN(ds_cpt_i, hid_t, const char *, bool); +FN_ITEM_DEFN(ds_ctg_v, hid_t, const char *, bool); + +#undef FN_ITEM_DEFN + +static bool +file_has_no_path(hid_t fid, const char *path) +{ + switch (H5Lexists(fid, path, H5P_DEFAULT)) { + case FALSE: + return true; + case TRUE: + failure_mssg = "H5Lexists unexpectedly true."; + return false; + default: + failure_mssg = "H5Lexists unexpectedly failed."; + return false; + } +} + +static bool +remove_from_file_path(hid_t fid, const char *path) +{ + if (H5Ldelete(fid, path, H5P_DEFAULT) < 0) { + failure_mssg = "H5Ldelete failed."; + return false; + } + return true; +} /*------------------------------------------------------------------------- * Function: ns_grp_0 @@ -34,12 +100,10 @@ * Purpose: Create an empty "new style" group at the specified location * in the specified file. * - * If pass is false on entry, do nothing. - * - * If an error is detected, set pass to FALSE, and set + * If an error is detected, return false, and set * failure_mssg to point to an appropriate error message. * - * Return: void + * Return: true on success, false on failure. * * Programmer: John Mainzer * 9/14/15 @@ -47,65 +111,62 @@ *------------------------------------------------------------------------- */ -void -ns_grp_0(hid_t fid, const char *group_name) { +static bool +missing_ns_grp_0(hid_t fid, const char *group_name) +{ + return file_has_no_path(fid, group_name); +} + +static bool +rm_ns_grp_0(hid_t fid, const char *group_name) +{ + return remove_from_file_path(fid, group_name); +} + +bool +ns_grp_0(hid_t fid, const char *group_name) +{ hid_t gid = -1; hid_t gcpl = -1; herr_t ret; - if (pass) { - gcpl = H5Pcreate(H5P_GROUP_CREATE); + gcpl = H5Pcreate(H5P_GROUP_CREATE); - if (gcpl <= 0) { - pass = FALSE; - failure_mssg = "ns_grp_0: H5Pcreate() failed"; - } - HDassert(gcpl > 0); + if (gcpl <= 0) { + failure_mssg = "ns_grp_0: H5Pcreate() failed"; + return false; } - if (pass) { - ret = H5Pset_link_creation_order(gcpl, H5P_CRT_ORDER_TRACKED); + ret = H5Pset_link_creation_order(gcpl, H5P_CRT_ORDER_TRACKED); - if (ret < 0) { - pass = FALSE; - failure_mssg = "ns_grp_0: H5Pset_link_creation_order() failed"; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "ns_grp_0: H5Pset_link_creation_order() failed"; + return false; } - if (pass) { - gid = H5Gcreate2(fid, group_name, H5P_DEFAULT, gcpl, H5P_DEFAULT); + gid = H5Gcreate2(fid, group_name, H5P_DEFAULT, gcpl, H5P_DEFAULT); - if (gid <= 0) { - pass = FALSE; - failure_mssg = "ns_grp_0: H5Gcreate2() failed"; - } - HDassert(gid > 0); + if (gid <= 0) { + failure_mssg = "ns_grp_0: H5Gcreate2() failed"; + return false; } - if (pass) { - ret = H5Pclose(gcpl); + ret = H5Pclose(gcpl); - if (ret < 0) { - pass = FALSE; - failure_mssg = "ns_grp_0: H5Pclose(gcpl) failed"; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "ns_grp_0: H5Pclose(gcpl) failed"; + return false; } - if (pass) { - ret = H5Gclose(gid); + ret = H5Gclose(gid); - if (ret < 0) { - pass = FALSE; - failure_mssg = "ns_grp_0: H5Gclose(gid) failed"; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "ns_grp_0: H5Gclose(gid) failed"; + return false; } - return; - -} /* ns_grp_0 */ + return true; +} /*------------------------------------------------------------------------- @@ -114,12 +175,10 @@ ns_grp_0(hid_t fid, const char *group_name) { * Purpose: verify an empty "new style" group at the specified location * in the specified file. * - * If pass is false on entry, do nothing. - * - * If an error is detected, set pass to FALSE, and set + * If an error is detected, return false, and set * failure_mssg to point to an appropriate error message. * - * Return: void + * Return: true on success, false on failure. * * Programmer: John Mainzer * 9/14/15 @@ -127,103 +186,76 @@ ns_grp_0(hid_t fid, const char *group_name) { *------------------------------------------------------------------------- */ -void vrfy_ns_grp_0(hid_t fid, const char *group_name) { +bool +vrfy_ns_grp_0(hid_t fid, const char *group_name) +{ hid_t gid = -1; hid_t gcpl = -1; H5G_info_t grp_info; unsigned crt_order_flags = 0; herr_t ret; - if (pass) { - gid = H5Gopen2(fid, group_name, H5P_DEFAULT); + gid = H5Gopen2(fid, group_name, H5P_DEFAULT); - if (gid <= 0) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_0: H5Gopen2() failed"; - } - HDassert(gid > 0); + if (gid <= 0) { + failure_mssg = "vrfy_ns_grp_0: H5Gopen2() failed"; + return false; } - if (pass) { - gcpl = H5Gget_create_plist(gid); + gcpl = H5Gget_create_plist(gid); - if (gcpl <= 0) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_0: H5Gget_create_plist() failed"; - } - HDassert(gcpl > 0); + if (gcpl <= 0) { + failure_mssg = "vrfy_ns_grp_0: H5Gget_create_plist() failed"; + return false; } - if (pass) { - ret = H5Pget_link_creation_order(gcpl, &crt_order_flags); + ret = H5Pget_link_creation_order(gcpl, &crt_order_flags); - if (gcpl <= 0) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_0: H5Pget_link_creation_order() failed"; - } - else if ( H5P_CRT_ORDER_TRACKED != crt_order_flags) { - pass = FALSE; - failure_mssg = - "vrfy_ns_grp_0: H5P_CRT_ORDER_TRACKED != crt_order_flags"; - } - HDassert(ret >= 0); - HDassert(H5P_CRT_ORDER_TRACKED == crt_order_flags); + if (ret < 0) { + failure_mssg = "vrfy_ns_grp_0: H5Pget_link_creation_order() failed"; + return false; + } else if ( H5P_CRT_ORDER_TRACKED != crt_order_flags) { + failure_mssg = + "vrfy_ns_grp_0: H5P_CRT_ORDER_TRACKED != crt_order_flags"; + return false; } - if (pass) { - ret = H5Pclose(gcpl); + ret = H5Pclose(gcpl); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_0: H5Pclose() failed"; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "vrfy_ns_grp_0: H5Pclose() failed"; + return false; } - if (pass) { - HDmemset(&grp_info, 0, sizeof(grp_info)); - ret = H5Gget_info(gid, &grp_info); - - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_0: H5Gget_info() failed"; - } - else if (H5G_STORAGE_TYPE_COMPACT != grp_info.storage_type) { - pass = FALSE; - failure_mssg = - "vrfy_ns_grp_0: H5G_STORAGE_TYPE_COMPACT != grp_info.storage_type"; - } - else if (0 != grp_info.nlinks) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_0: 0 != grp_info.nlinks"; - } - else if (0 != grp_info.max_corder) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_0: 0 != grp_info.max_corder"; - } - else if ( FALSE != grp_info.mounted) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_0: FALSE != grp_info.mounted"; - } + HDmemset(&grp_info, 0, sizeof(grp_info)); + ret = H5Gget_info(gid, &grp_info); - HDassert(ret >= 0); - HDassert(H5G_STORAGE_TYPE_COMPACT == grp_info.storage_type); - HDassert(0 == grp_info.nlinks); - HDassert(0 == grp_info.max_corder); - HDassert(false == grp_info.mounted); + if (ret < 0) { + failure_mssg = "vrfy_ns_grp_0: H5Gget_info() failed"; + return false; + } else if (H5G_STORAGE_TYPE_COMPACT != grp_info.storage_type) { + failure_mssg = + "vrfy_ns_grp_0: H5G_STORAGE_TYPE_COMPACT != grp_info.storage_type"; + return false; + } else if (0 != grp_info.nlinks) { + failure_mssg = "vrfy_ns_grp_0: 0 != grp_info.nlinks"; + return false; + } else if (0 != grp_info.max_corder) { + failure_mssg = "vrfy_ns_grp_0: 0 != grp_info.max_corder"; + return false; + } else if ( FALSE != grp_info.mounted) { + failure_mssg = "vrfy_ns_grp_0: FALSE != grp_info.mounted"; + return false; } - if (pass) { - ret = H5Gclose(gid); + ret = H5Gclose(gid); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_0: H5Gclose() failed"; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "vrfy_ns_grp_0: H5Gclose() failed"; + return false; } - return; + return true; } /* vrfy_ns_grp_0() */ @@ -233,12 +265,10 @@ void vrfy_ns_grp_0(hid_t fid, const char *group_name) { * Purpose: Create a compact "new style" group, with 'nlinks' * soft/hard/external links in it in the specified file. * - * If pass is false on entry, do nothing. - * - * If an error is detected, set pass to FALSE, and set + * If an error is detected, return false, and set * failure_mssg to point to an appropriate error message. * - * Return: void + * Return: true on success, false on failure. * * Programmer: John Mainzer * 9/14/15 @@ -246,127 +276,111 @@ void vrfy_ns_grp_0(hid_t fid, const char *group_name) { *------------------------------------------------------------------------- */ -void -ns_grp_c(hid_t fid, const char *group_name, unsigned nlinks) { +static bool +missing_ns_grp_c(hid_t fid, const char *group_name, + unsigned H5_ATTR_UNUSED nlinks) +{ + return file_has_no_path(fid, group_name); +} + +static bool +rm_ns_grp_c(hid_t fid, const char *group_name, unsigned H5_ATTR_UNUSED nlinks) +{ + return remove_from_file_path(fid, group_name); +} + +bool +ns_grp_c(hid_t fid, const char *group_name, unsigned nlinks) +{ hid_t gid = -1; hid_t gcpl = -1; unsigned max_compact; unsigned u; herr_t ret; - if (pass) { - gcpl = H5Pcreate(H5P_GROUP_CREATE); + gcpl = H5Pcreate(H5P_GROUP_CREATE); - if (gcpl <= 0) { - pass = FALSE; - failure_mssg = "ns_grp_c: H5Pcreate(H5P_GROUP_CREATE) failed"; - } - HDassert(gcpl > 0); + if (gcpl <= 0) { + failure_mssg = "ns_grp_c: H5Pcreate(H5P_GROUP_CREATE) failed"; + return false; } - if (pass) { - ret = H5Pset_link_creation_order(gcpl, H5P_CRT_ORDER_TRACKED); + ret = H5Pset_link_creation_order(gcpl, H5P_CRT_ORDER_TRACKED); - if (ret < 0) { - pass = FALSE; - failure_mssg = "ns_grp_c: H5Pset_link_creation_order() failed"; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "ns_grp_c: H5Pset_link_creation_order() failed"; + return false; } - if (pass) { - gid = H5Gcreate2(fid, group_name, H5P_DEFAULT, gcpl, H5P_DEFAULT); + gid = H5Gcreate2(fid, group_name, H5P_DEFAULT, gcpl, H5P_DEFAULT); - if (gid <= 0) { - pass = FALSE; - failure_mssg = "ns_grp_c: H5Gcreate2() failed"; - } - HDassert(gid > 0); + if (gid <= 0) { + failure_mssg = "ns_grp_c: H5Gcreate2() failed"; + return false; } - if (pass) { - max_compact = 0; - ret = H5Pget_link_phase_change(gcpl, &max_compact, NULL); - - if (ret < 0) { - pass = FALSE; - failure_mssg = "ns_grp_c: H5Pget_link_phase_change() failed"; - } - else if (nlinks <= 0) { - pass = FALSE; - failure_mssg = "ns_grp_c: nlinks <= 0"; - } - else if (nlinks >= max_compact) { - pass = FALSE; - failure_mssg = "ns_grp_c: nlinks >= max_compact"; - } + max_compact = 0; + ret = H5Pget_link_phase_change(gcpl, &max_compact, NULL); - HDassert(ret >= 0); - HDassert(nlinks > 0); - HDassert(nlinks < max_compact); + if (ret < 0) { + failure_mssg = "ns_grp_c: H5Pget_link_phase_change() failed"; + return false; + } else if (nlinks <= 0) { + failure_mssg = "ns_grp_c: nlinks <= 0"; + return false; + } else if (nlinks >= max_compact) { + failure_mssg = "ns_grp_c: nlinks >= max_compact"; + return false; } - u = 0; - while ((pass) && (u < nlinks)) { + for (u = 0; u < nlinks; u++) { char linkname[16]; HDsprintf(linkname, "%u", u); if (0 == (u % 3)) { - ret = H5Lcreate_soft(group_name, gid, linkname, H5P_DEFAULT, H5P_DEFAULT); + ret = H5Lcreate_soft(group_name, gid, linkname, H5P_DEFAULT, + H5P_DEFAULT); if (ret < 0) { - pass = FALSE; failure_mssg = "ns_grp_c: H5Lcreate_soft() failed"; + return false; } - HDassert(ret >= 0); - } /* end if */ - else if (1 == (u % 3)) { + } else if (1 == (u % 3)) { ret = H5Lcreate_hard(fid, "/", gid, linkname, H5P_DEFAULT, H5P_DEFAULT); if (ret < 0) { - pass = FALSE; failure_mssg = "ns_grp_c: H5Lcreate_hard() failed"; + return false; } - HDassert(ret >= 0); - } /* end else-if */ - else { + } else { HDassert(2 == (u % 3)); ret = H5Lcreate_external("external.h5", "/ext", gid, linkname, H5P_DEFAULT, H5P_DEFAULT); if (ret < 0) { - pass = FALSE; failure_mssg = "ns_grp_c: H5Lcreate_external() failed"; + return false; } - HDassert(ret >= 0); - } /* end else */ - - u++; - } /* end while() */ + } + } - if (pass) { - ret = H5Pclose(gcpl); + ret = H5Pclose(gcpl); - if (ret < 0) { - pass = FALSE; - failure_mssg = "ns_grp_c: H5Pclose(gcpl) failed"; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "ns_grp_c: H5Pclose(gcpl) failed"; + return false; } - if (pass) { - ret = H5Gclose(gid); + ret = H5Gclose(gid); - if (ret < 0) { - pass = FALSE; - failure_mssg = "ns_grp_c: H5Gclose(gid) failed"; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "ns_grp_c: H5Gclose(gid) failed"; + return false; } - return; + return true; } /* ns_grp_c() */ @@ -377,12 +391,10 @@ ns_grp_c(hid_t fid, const char *group_name, unsigned nlinks) { * Purpose: Verify a compact "new style" group, with 'nlinks' * soft/hard/external links in it in the specified file. * - * If pass is false on entry, do nothing. - * - * If an error is detected, set pass to FALSE, and set + * If an error is detected, return false, and set * failure_mssg to point to an appropriate error message. * - * Return: void + * Return: true on success, false on failure. * * Programmer: John Mainzer * 9/14/15 @@ -390,8 +402,9 @@ ns_grp_c(hid_t fid, const char *group_name, unsigned nlinks) { *------------------------------------------------------------------------- */ -void -vrfy_ns_grp_c(hid_t fid, const char *group_name, unsigned nlinks) { +bool +vrfy_ns_grp_c(hid_t fid, const char *group_name, unsigned nlinks) +{ hid_t gid = -1; hid_t gcpl = -1; H5G_info_t grp_info; @@ -399,86 +412,60 @@ vrfy_ns_grp_c(hid_t fid, const char *group_name, unsigned nlinks) { unsigned u; herr_t ret; - if (pass) { - gid = H5Gopen2(fid, group_name, H5P_DEFAULT); + gid = H5Gopen2(fid, group_name, H5P_DEFAULT); - if (gid <= 0) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_c: H5Gopen2() failed"; - } - HDassert(gid > 0); + if (gid <= 0) { + failure_mssg = "vrfy_ns_grp_c: H5Gopen2() failed"; + return false; } - if (pass) { - gcpl = H5Gget_create_plist(gid); + gcpl = H5Gget_create_plist(gid); - if (gcpl <= 0) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_c: H5Gget_create_plist(gid) failed"; - } - HDassert(gcpl > 0); + if (gcpl <= 0) { + failure_mssg = "vrfy_ns_grp_c: H5Gget_create_plist(gid) failed"; + return false; } - if (pass) { - ret = H5Pget_link_creation_order(gcpl, &crt_order_flags); + ret = H5Pget_link_creation_order(gcpl, &crt_order_flags); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_c: H5Pget_link_creation_order() failed"; - } - else if ( H5P_CRT_ORDER_TRACKED != crt_order_flags) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_c: H5P_CRT_ORDER_TRACKED != crt_order_flags"; - } - HDassert(ret >= 0); - HDassert(H5P_CRT_ORDER_TRACKED == crt_order_flags); + if (ret < 0) { + failure_mssg = "vrfy_ns_grp_c: H5Pget_link_creation_order() failed"; + return false; + } else if ( H5P_CRT_ORDER_TRACKED != crt_order_flags) { + failure_mssg = + "vrfy_ns_grp_c: H5P_CRT_ORDER_TRACKED != crt_order_flags"; + return false; } - if (pass) { - ret = H5Pclose(gcpl); + ret = H5Pclose(gcpl); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_c: H5Pclose() failed"; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "vrfy_ns_grp_c: H5Pclose() failed"; + return false; } - if (pass) { - HDmemset(&grp_info, 0, sizeof(grp_info)); - ret = H5Gget_info(gid, &grp_info); + HDmemset(&grp_info, 0, sizeof(grp_info)); + ret = H5Gget_info(gid, &grp_info); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_c: H5Gget_info() failed"; - } - else if (H5G_STORAGE_TYPE_COMPACT != grp_info.storage_type) { - pass = FALSE; - failure_mssg = - "vrfy_ns_grp_c: H5G_STORAGE_TYPE_COMPACT != grp_info.storage_type"; - } - else if (nlinks != grp_info.nlinks) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_c: nlinks != grp_info.nlinks"; - } - else if (nlinks != grp_info.max_corder) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_c: nlinks != grp_info.max_corder"; - } - else if ( FALSE != grp_info.mounted) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_c: FALSE != grp_info.mounted"; - } - - HDassert(ret >= 0); - HDassert(H5G_STORAGE_TYPE_COMPACT == grp_info.storage_type); - HDassert(nlinks == grp_info.nlinks); - HDassert(nlinks == grp_info.max_corder); - HDassert(false == grp_info.mounted); + if (ret < 0) { + failure_mssg = "vrfy_ns_grp_c: H5Gget_info() failed"; + return false; + } else if (H5G_STORAGE_TYPE_COMPACT != grp_info.storage_type) { + failure_mssg = + "vrfy_ns_grp_c: H5G_STORAGE_TYPE_COMPACT != grp_info.storage_type"; + return false; + } else if (nlinks != grp_info.nlinks) { + failure_mssg = "vrfy_ns_grp_c: nlinks != grp_info.nlinks"; + return false; + } else if (nlinks != grp_info.max_corder) { + failure_mssg = "vrfy_ns_grp_c: nlinks != grp_info.max_corder"; + return false; + } else if ( FALSE != grp_info.mounted) { + failure_mssg = "vrfy_ns_grp_c: FALSE != grp_info.mounted"; + return false; } - u = 0; - while ((pass) && (u < nlinks)) { + for (u = 0; u < nlinks; u++) { H5L_info2_t lnk_info; char linkname[16]; htri_t link_exists; @@ -487,104 +474,82 @@ vrfy_ns_grp_c(hid_t fid, const char *group_name, unsigned nlinks) { link_exists = H5Lexists(gid, linkname, H5P_DEFAULT); if (link_exists < 0) { - pass = FALSE; failure_mssg = "vrfy_ns_grp_c: H5Lexists() failed"; + return false; } - HDassert(link_exists >= 0); HDmemset(&lnk_info, 0, sizeof(grp_info)); ret = H5Lget_info2(gid, linkname, &lnk_info, H5P_DEFAULT); if (ret < 0) { - pass = FALSE; failure_mssg = "vrfy_ns_grp_c: H5Lget_info() failed"; - } - else if ( TRUE != lnk_info.corder_valid) { - pass = FALSE; + return false; + } else if ( TRUE != lnk_info.corder_valid) { failure_mssg = "vrfy_ns_grp_c: TRUE != lnk_info.corder_valid"; - } - else if (u != lnk_info.corder) { - pass = FALSE; + return false; + } else if (u != lnk_info.corder) { failure_mssg = "vrfy_ns_grp_c: u != lnk_info.corder"; - } - else if (H5T_CSET_ASCII != lnk_info.cset) { - pass = FALSE; + return false; + } else if (H5T_CSET_ASCII != lnk_info.cset) { failure_mssg = "vrfy_ns_grp_c: H5T_CSET_ASCII != lnk_info.cset"; + return false; } - HDassert(ret >= 0); - HDassert(true == lnk_info.corder_valid); - HDassert(u == lnk_info.corder); - HDassert(H5T_CSET_ASCII == lnk_info.cset); if (0 == (u % 3)) { char *slinkval; if (H5L_TYPE_SOFT != lnk_info.type) { - pass = FALSE; failure_mssg = "vrfy_ns_grp_c: H5L_TYPE_SOFT != lnk_info.type"; + return false; } else if ((HDstrlen(group_name) + 1) != lnk_info.u.val_size) { - pass = FALSE; failure_mssg = "vrfy_ns_grp_c: (HDstrlen(group_name) + 1) != lnk_info.u.val_size"; + return false; } - HDassert(H5L_TYPE_SOFT == lnk_info.type); - HDassert((HDstrlen(group_name) + 1) == lnk_info.u.val_size); - slinkval = (char *) HDmalloc(lnk_info.u.val_size); + slinkval = HDmalloc(lnk_info.u.val_size); if (!slinkval) { - pass = FALSE; failure_mssg = "vrfy_ns_grp_c: HDmalloc of slinkval failed"; + return false; } - HDassert(slinkval); ret = H5Lget_val(gid, linkname, slinkval, lnk_info.u.val_size, H5P_DEFAULT); if (ret < 0) { - pass = FALSE; failure_mssg = "vrfy_ns_grp_c: H5Lget_val() failed"; - } - else if (0 != HDstrcmp(slinkval, group_name)) { - pass = FALSE; + HDfree(slinkval); + return false; + } else if (0 != HDstrcmp(slinkval, group_name)) { failure_mssg = "vrfy_ns_grp_c: 0 != HDstrcmp(slinkval, group_name)"; + HDfree(slinkval); + return false; } - HDassert(ret >= 0); - HDassert(0 == HDstrcmp(slinkval, group_name)); HDfree(slinkval); - } /* end if */ - else if (1 == (u % 3)) { + } else if (1 == (u % 3)) { H5O_info2_t root_oinfo; int token_cmp = 0; if (H5L_TYPE_HARD != lnk_info.type) { - pass = FALSE; failure_mssg = "vrfy_ns_grp_c: H5L_TYPE_HARD != lnk_info.type"; + return false; } - HDassert(H5L_TYPE_HARD == lnk_info.type); HDmemset(&root_oinfo, 0, sizeof(root_oinfo)); ret = H5Oget_info3(fid, &root_oinfo, H5O_INFO_BASIC); if (ret < 0) { - pass = FALSE; failure_mssg = "vrfy_ns_grp_c: H5Oget_info() failed."; + return false; + } else if(H5Otoken_cmp(fid, &root_oinfo.token, &lnk_info.u.token, &token_cmp) < 0) { + failure_mssg = "vrfy_ns_grp_c: H5Otoken_cmp() failed."; + return false; + } else if (token_cmp) { + failure_mssg = "vrfy_ns_grp_c: root_oinfo.token != lnk_info.u.token"; + return false; } - else { - if(H5Otoken_cmp(fid, &root_oinfo.token, &lnk_info.u.token, &token_cmp) < 0) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_c: H5Otoken_cmp() failed."; - } - - if (token_cmp) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_c: root_oinfo.token != lnk_info.u.token"; - } - } - HDassert(ret >= 0); - HDassert(!token_cmp); - } /* end else-if */ - else { + } else { void *elinkval; const char *file = NULL; const char *path = NULL; @@ -592,61 +557,53 @@ vrfy_ns_grp_c(hid_t fid, const char *group_name, unsigned nlinks) { HDassert(2 == (u % 3)); if (H5L_TYPE_EXTERNAL != lnk_info.type) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_c: H5L_TYPE_EXTERNAL != lnk_info.type"; + failure_mssg = + "vrfy_ns_grp_c: H5L_TYPE_EXTERNAL != lnk_info.type"; + return false; } - HDassert(H5L_TYPE_EXTERNAL == lnk_info.type); elinkval = HDmalloc(lnk_info.u.val_size); if (!elinkval) { - pass = FALSE; failure_mssg = "vrfy_ns_grp_c: HDmalloc of elinkval failed."; + return false; } - HDassert(elinkval); ret = H5Lget_val(gid, linkname, elinkval, lnk_info.u.val_size, H5P_DEFAULT); if (ret < 0) { - pass = FALSE; failure_mssg = "vrfy_ns_grp_c: H5Lget_val() failed."; + return false; } - HDassert(ret >= 0); - ret = H5Lunpack_elink_val(elinkval, lnk_info.u.val_size, NULL, &file, &path); + ret = H5Lunpack_elink_val(elinkval, lnk_info.u.val_size, NULL, + &file, &path); if (ret < 0) { - pass = FALSE; failure_mssg = "vrfy_ns_grp_c: H5Lunpack_elink_val() failed."; - } - else if (0 != HDstrcmp(file, "external.h5")) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_c: 0 != HDstrcmp(file, \"external.h5\")"; - } - else if (0 != HDstrcmp(path, "/ext")) { - pass = FALSE; + HDfree(elinkval); + return false; + } else if (0 != HDstrcmp(file, "external.h5")) { + failure_mssg = + "vrfy_ns_grp_c: 0 != HDstrcmp(file, \"external.h5\")"; + HDfree(elinkval); + return false; + } else if (0 != HDstrcmp(path, "/ext")) { failure_mssg = "vrfy_ns_grp_c: 0 != HDstrcmp(path, \"/ext\")"; + HDfree(elinkval); + return false; } - HDassert(ret >= 0); - HDassert(0 == HDstrcmp(file, "external.h5")); - HDassert(0 == HDstrcmp(path, "/ext")); - HDfree(elinkval); - } /* end else */ - - u++; - } /* end while */ + } + } - if (pass) { - ret = H5Gclose(gid); + ret = H5Gclose(gid); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_c: H5Gclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "vrfy_ns_grp_c: H5Gclose() failed."; + return false; } - return; + return true; } /* vrfy_ns_grp_c() */ @@ -656,12 +613,10 @@ vrfy_ns_grp_c(hid_t fid, const char *group_name, unsigned nlinks) { * Purpose: Create a dense "new style" group, with 'nlinks' * (soft/hard/external) links in it in the specified file. * - * If pass is false on entry, do nothing. - * - * If an error is detected, set pass to FALSE, and set + * If an error is detected, return false, and set * failure_mssg to point to an appropriate error message. * - * Return: void + * Return: true on success, false on failure. * * Programmer: John Mainzer * 9/14/15 @@ -669,62 +624,62 @@ vrfy_ns_grp_c(hid_t fid, const char *group_name, unsigned nlinks) { *------------------------------------------------------------------------- */ -void -ns_grp_d(hid_t fid, const char *group_name, unsigned nlinks) { +static bool +missing_ns_grp_d(hid_t fid, const char *group_name, + unsigned H5_ATTR_UNUSED nlinks) +{ + return file_has_no_path(fid, group_name); +} + +static bool +rm_ns_grp_d(hid_t fid, const char *group_name, unsigned H5_ATTR_UNUSED nlinks) +{ + return remove_from_file_path(fid, group_name); +} + +bool +ns_grp_d(hid_t fid, const char *group_name, unsigned nlinks) +{ hid_t gid = -1; hid_t gcpl = -1; unsigned max_compact; unsigned u; herr_t ret; - if (pass) { - gcpl = H5Pcreate(H5P_GROUP_CREATE); + gcpl = H5Pcreate(H5P_GROUP_CREATE); - if (gcpl <= 0) { - pass = FALSE; - failure_mssg = "ns_grp_d: H5Pcreate() failed."; - } - HDassert(gcpl > 0); + if (gcpl <= 0) { + failure_mssg = "ns_grp_d: H5Pcreate() failed."; + return false; } - if (pass) { - ret = H5Pset_link_creation_order(gcpl, H5P_CRT_ORDER_TRACKED); + ret = H5Pset_link_creation_order(gcpl, H5P_CRT_ORDER_TRACKED); - if (ret < 0) { - pass = FALSE; - failure_mssg = "ns_grp_d: H5Pset_link_creation_order() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "ns_grp_d: H5Pset_link_creation_order() failed."; + return false; } - if (pass) { - gid = H5Gcreate2(fid, group_name, H5P_DEFAULT, gcpl, H5P_DEFAULT); + gid = H5Gcreate2(fid, group_name, H5P_DEFAULT, gcpl, H5P_DEFAULT); - if (gid <= 0) { - pass = FALSE; - failure_mssg = "ns_grp_d: H5Gcreate2() failed."; - } - HDassert(gid > 0); + if (gid <= 0) { + failure_mssg = "ns_grp_d: H5Gcreate2() failed."; + return false; } - if (pass) { - max_compact = 0; - ret = H5Pget_link_phase_change(gcpl, &max_compact, NULL); + max_compact = 0; + ret = H5Pget_link_phase_change(gcpl, &max_compact, NULL); - if (ret < 0) { - pass = FALSE; - failure_mssg = "ns_grp_d: H5Pget_link_phase_change() failed."; - } - else if (nlinks <= max_compact) { - pass = FALSE; - failure_mssg = "ns_grp_d: nlinks <= max_compact"; - } - HDassert(ret >= 0); - HDassert(nlinks > max_compact); + if (ret < 0) { + failure_mssg = "ns_grp_d: H5Pget_link_phase_change() failed."; + return false; + } + else if (nlinks <= max_compact) { + failure_mssg = "ns_grp_d: nlinks <= max_compact"; + return false; } - u = 0; - while ((pass) && (u < nlinks)) { + for (u = 0; u < nlinks; u++) { char linkname[16]; HDsprintf(linkname, "%u", u); @@ -734,58 +689,45 @@ ns_grp_d(hid_t fid, const char *group_name, unsigned nlinks) { H5P_DEFAULT, H5P_DEFAULT); if (ret < 0) { - pass = FALSE; failure_mssg = "ns_grp_d: H5Lcreate_soft() failed."; + return false; } - HDassert(ret >= 0); - } /* end if */ - else if (1 == (u % 3)) { + } else if (1 == (u % 3)) { ret = H5Lcreate_hard(fid, "/", gid, linkname, H5P_DEFAULT, H5P_DEFAULT); if (ret < 0) { - pass = FALSE; failure_mssg = "ns_grp_d: H5Lcreate_hard() failed."; + return false; } - HDassert(ret >= 0); - } /* end else-if */ - else { + } else { HDassert(2 == (u % 3)); ret = H5Lcreate_external("external.h5", "/ext", gid, linkname, H5P_DEFAULT, H5P_DEFAULT); if (ret < 0) { - pass = FALSE; failure_mssg = "ns_grp_d: H5Lcreate_external() failed."; + return false; } - HDassert(ret >= 0); - } /* end else */ - - u++; - } /* end while */ + } + } - if (pass) { - ret = H5Pclose(gcpl); + ret = H5Pclose(gcpl); - if (ret < 0) { - pass = FALSE; - failure_mssg = "ns_grp_d: H5Pclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "ns_grp_d: H5Pclose() failed."; + return false; } - if (pass) { - ret = H5Gclose(gid); + ret = H5Gclose(gid); - if (ret < 0) { - pass = FALSE; - failure_mssg = "ns_grp_d: H5Gclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "ns_grp_d: H5Gclose() failed."; + return false; } - return; + return true; } /* ns_grp_d() */ @@ -795,12 +737,10 @@ ns_grp_d(hid_t fid, const char *group_name, unsigned nlinks) { * Purpose: Verify a dense "new style" group, with 'nlinks' * soft/hard/external links in it in the specified file. * - * If pass is false on entry, do nothing. - * - * If an error is detected, set pass to FALSE, and set + * If an error is detected, return false, and set * failure_mssg to point to an appropriate error message. * - * Return: void + * Return: true on success, false on failure. * * Programmer: John Mainzer * 9/14/15 @@ -809,8 +749,9 @@ ns_grp_d(hid_t fid, const char *group_name, unsigned nlinks) { */ -void -vrfy_ns_grp_d(hid_t fid, const char *group_name, unsigned nlinks) { +bool +vrfy_ns_grp_d(hid_t fid, const char *group_name, unsigned nlinks) +{ hid_t gid = -1; hid_t gcpl = -1; H5G_info_t grp_info; @@ -818,84 +759,59 @@ vrfy_ns_grp_d(hid_t fid, const char *group_name, unsigned nlinks) { unsigned u; herr_t ret; - if (pass) { - gid = H5Gopen2(fid, group_name, H5P_DEFAULT); + gid = H5Gopen2(fid, group_name, H5P_DEFAULT); - if (gid <= 0) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_d: H5Gopen2() failed."; - } - HDassert(gid > 0); + if (gid <= 0) { + failure_mssg = "vrfy_ns_grp_d: H5Gopen2() failed."; + return false; } - if (pass) { - gcpl = H5Gget_create_plist(gid); + gcpl = H5Gget_create_plist(gid); - if (gcpl <= 0) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_d: H5Gget_create_plist() failed."; - } - HDassert(gcpl > 0); + if (gcpl <= 0) { + failure_mssg = "vrfy_ns_grp_d: H5Gget_create_plist() failed."; + return false; } - if (pass) { - ret = H5Pget_link_creation_order(gcpl, &crt_order_flags); + ret = H5Pget_link_creation_order(gcpl, &crt_order_flags); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_d: H5Pget_link_creation_order() failed."; - } - else if (H5P_CRT_ORDER_TRACKED != crt_order_flags) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_d: H5P_CRT_ORDER_TRACKED != crt_order_flags"; - } - HDassert(ret >= 0); - HDassert(H5P_CRT_ORDER_TRACKED == crt_order_flags); + if (ret < 0) { + failure_mssg = "vrfy_ns_grp_d: H5Pget_link_creation_order() failed."; + return false; + } else if (H5P_CRT_ORDER_TRACKED != crt_order_flags) { + failure_mssg = + "vrfy_ns_grp_d: H5P_CRT_ORDER_TRACKED != crt_order_flags"; + return false; } - if (pass) { - ret = H5Pclose(gcpl); + ret = H5Pclose(gcpl); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_d: H5Pclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "vrfy_ns_grp_d: H5Pclose() failed."; + return false; } - if (pass) { - HDmemset(&grp_info, 0, sizeof(grp_info)); - ret = H5Gget_info(gid, &grp_info); + HDmemset(&grp_info, 0, sizeof(grp_info)); + ret = H5Gget_info(gid, &grp_info); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_d: H5Gget_info() failed."; - } - else if (H5G_STORAGE_TYPE_DENSE != grp_info.storage_type) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_d: H5G_STORAGE_TYPE_DENSE != grp_info.storage_type"; - } - else if (nlinks != grp_info.nlinks) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_d: nlinks != grp_info.nlinks"; - } - else if (nlinks != grp_info.max_corder) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_d: nlinks != grp_info.max_corder"; - } - else if ( FALSE != grp_info.mounted) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_d: FALSE != grp_info.mounted"; - } - HDassert(ret >= 0); - HDassert(H5G_STORAGE_TYPE_DENSE == grp_info.storage_type); - HDassert(nlinks == grp_info.nlinks); - HDassert(nlinks == grp_info.max_corder); - HDassert(false == grp_info.mounted); + if (ret < 0) { + failure_mssg = "vrfy_ns_grp_d: H5Gget_info() failed."; + return false; + } else if (H5G_STORAGE_TYPE_DENSE != grp_info.storage_type) { + failure_mssg = "vrfy_ns_grp_d: H5G_STORAGE_TYPE_DENSE != grp_info.storage_type"; + return false; + } else if (nlinks != grp_info.nlinks) { + failure_mssg = "vrfy_ns_grp_d: nlinks != grp_info.nlinks"; + return false; + } else if (nlinks != grp_info.max_corder) { + failure_mssg = "vrfy_ns_grp_d: nlinks != grp_info.max_corder"; + return false; + } else if ( FALSE != grp_info.mounted) { + failure_mssg = "vrfy_ns_grp_d: FALSE != grp_info.mounted"; + return false; } - u = 0; - while ((pass) && (u < nlinks)) { + for (u = 0; u < nlinks; u++) { H5L_info2_t lnk_info; char linkname[16]; htri_t link_exists; @@ -904,103 +820,80 @@ vrfy_ns_grp_d(hid_t fid, const char *group_name, unsigned nlinks) { link_exists = H5Lexists(gid, linkname, H5P_DEFAULT); if (link_exists < 0) { - pass = FALSE; failure_mssg = "vrfy_ns_grp_d: H5Lexists() failed."; + return false; } - HDassert(link_exists >= 0); HDmemset(&lnk_info, 0, sizeof(grp_info)); ret = H5Lget_info2(gid, linkname, &lnk_info, H5P_DEFAULT); if (ret < 0) { - pass = FALSE; failure_mssg = "vrfy_ns_grp_d: H5Lget_info() failed."; - } - else if (TRUE != lnk_info.corder_valid) { - pass = FALSE; + return false; + } else if (TRUE != lnk_info.corder_valid) { failure_mssg = "vrfy_ns_grp_d: TRUE != lnk_info.corder_valid"; - } - else if (u != lnk_info.corder) { - pass = FALSE; + return false; + } else if (u != lnk_info.corder) { failure_mssg = "vrfy_ns_grp_d: u != lnk_info.corder"; - } - else if (H5T_CSET_ASCII != lnk_info.cset) { - pass = FALSE; + return false; + } else if (H5T_CSET_ASCII != lnk_info.cset) { failure_mssg = "vrfy_ns_grp_d: H5T_CSET_ASCII != lnk_info.cset"; + return false; } - HDassert(ret >= 0); - HDassert(true == lnk_info.corder_valid); - HDassert(u == lnk_info.corder); - HDassert(H5T_CSET_ASCII == lnk_info.cset); if (0 == (u % 3)) { char *slinkval; if (H5L_TYPE_SOFT != lnk_info.type) { - pass = FALSE; failure_mssg = "vrfy_ns_grp_d: H5L_TYPE_SOFT != lnk_info.type"; + return false; } else if ((HDstrlen(group_name) + 1) != lnk_info.u.val_size) { - pass = FALSE; failure_mssg = "vrfy_ns_grp_d: H5L_TYPE_SOFT != lnk_info.type"; + return false; } - HDassert(H5L_TYPE_SOFT == lnk_info.type); - HDassert((HDstrlen(group_name) + 1) == lnk_info.u.val_size); - slinkval = (char *) HDmalloc(lnk_info.u.val_size); + slinkval = HDmalloc(lnk_info.u.val_size); if (!slinkval) { - pass = FALSE; failure_mssg = "vrfy_ns_grp_d: HDmalloc of slinkval failed"; + return false; } - HDassert(slinkval); ret = H5Lget_val(gid, linkname, slinkval, lnk_info.u.val_size, - H5P_DEFAULT); + H5P_DEFAULT); if (ret < 0) { - pass = FALSE; failure_mssg = "vrfy_ns_grp_d: H5Lget_val() failed"; - } - else if (0 != HDstrcmp(slinkval, group_name)) { - pass = FALSE; + HDfree(slinkval); + return false; + } else if (0 != HDstrcmp(slinkval, group_name)) { failure_mssg = "vrfy_ns_grp_d: 0 != HDstrcmp(slinkval, group_name)"; + HDfree(slinkval); + return false; } - HDassert(ret >= 0); - HDassert(0 == HDstrcmp(slinkval, group_name)); - HDfree(slinkval); - } /* end if */ - else if (1 == (u % 3)) { + } else if (1 == (u % 3)) { H5O_info2_t root_oinfo; int token_cmp = 0; if (H5L_TYPE_HARD != lnk_info.type) { - pass = FALSE; failure_mssg = "vrfy_ns_grp_d: H5L_TYPE_HARD != lnk_info.type"; + return false; } - HDassert(H5L_TYPE_HARD == lnk_info.type); HDmemset(&root_oinfo, 0, sizeof(root_oinfo)); ret = H5Oget_info3(fid, &root_oinfo, H5O_INFO_BASIC); if (ret < 0) { - pass = FALSE; failure_mssg = "vrfy_ns_grp_d: H5Oget_info() failed."; + return false; + } else if(H5Otoken_cmp(fid, &root_oinfo.token, &lnk_info.u.token, &token_cmp) < 0) { + failure_mssg = "vrfy_ns_grp_d: H5Otoken_cmp() failed."; + return false; + } else if (token_cmp) { + failure_mssg = "vrfy_ns_grp_d: root_oinfo.token != lnk_info.u.token"; + return false; } - else { - if(H5Otoken_cmp(fid, &root_oinfo.token, &lnk_info.u.token, &token_cmp) < 0) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_d: H5Otoken_cmp() failed."; - } - - if (token_cmp) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_d: root_oinfo.token != lnk_info.u.token"; - } - } - HDassert(ret >= 0); - HDassert(!token_cmp); - } /* end else-if */ - else { + } else { void *elinkval; const char *file = NULL; const char *path = NULL; @@ -1008,61 +901,53 @@ vrfy_ns_grp_d(hid_t fid, const char *group_name, unsigned nlinks) { HDassert(2 == (u % 3)); if (H5L_TYPE_EXTERNAL != lnk_info.type) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_d: H5L_TYPE_EXTERNAL != lnk_info.type"; + failure_mssg = + "vrfy_ns_grp_d: H5L_TYPE_EXTERNAL != lnk_info.type"; + return false; } - HDassert(H5L_TYPE_EXTERNAL == lnk_info.type); elinkval = HDmalloc(lnk_info.u.val_size); if (!elinkval) { - pass = FALSE; failure_mssg = "vrfy_ns_grp_d: HDmalloc of elinkval failed."; + return false; } - HDassert(elinkval); ret = H5Lget_val(gid, linkname, elinkval, lnk_info.u.val_size, - H5P_DEFAULT); + H5P_DEFAULT); if (ret < 0) { - pass = FALSE; failure_mssg = "vrfy_ns_grp_d: H5Lget_val failed."; + return false; } - HDassert(ret >= 0); - ret = H5Lunpack_elink_val(elinkval, lnk_info.u.val_size, NULL, &file, &path); + ret = H5Lunpack_elink_val(elinkval, lnk_info.u.val_size, NULL, + &file, &path); if (ret < 0) { - pass = FALSE; failure_mssg = "vrfy_ns_grp_d: H5Lunpack_elink_val failed."; - } - else if (0 != HDstrcmp(file, "external.h5")) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_d: 0 != HDstrcmp(file, \"external.h5\")."; - } - else if (0 != HDstrcmp(path, "/ext")) { - pass = FALSE; + HDfree(elinkval); + return false; + } else if (0 != HDstrcmp(file, "external.h5")) { + failure_mssg = + "vrfy_ns_grp_d: 0 != HDstrcmp(file, \"external.h5\")."; + HDfree(elinkval); + return false; + } else if (0 != HDstrcmp(path, "/ext")) { failure_mssg = "vrfy_ns_grp_d: 0 != HDstrcmp(path, \"/ext\")"; + HDfree(elinkval); + return false; } - HDassert(ret >= 0); - HDassert(0 == HDstrcmp(file, "external.h5")); - HDassert(0 == HDstrcmp(path, "/ext")); - HDfree(elinkval); - } /* end else */ - - u++; - } /* end while() */ + } + } - if (pass) { - ret = H5Gclose(gid); + ret = H5Gclose(gid); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ns_grp_d: H5Gclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "vrfy_ns_grp_d: H5Gclose() failed."; + return false; } - return; + return true; } /* vrfy_ns_grp_d() */ @@ -1072,12 +957,10 @@ vrfy_ns_grp_d(hid_t fid, const char *group_name, unsigned nlinks) { * * Purpose: Create an empty "old style" group. * - * If pass is false on entry, do nothing. - * - * If an error is detected, set pass to FALSE, and set + * If an error is detected, return false, and set * failure_mssg to point to an appropriate error message. * - * Return: void + * Return: true on success, false on failure. * * Programmer: John Mainzer * 9/14/15 @@ -1085,7 +968,19 @@ vrfy_ns_grp_d(hid_t fid, const char *group_name, unsigned nlinks) { *------------------------------------------------------------------------- */ -void +static bool +missing_os_grp_0(hid_t fid, const char *group_name) +{ + return file_has_no_path(fid, group_name); +} + +static bool +rm_os_grp_0(hid_t fid, const char *group_name) +{ + return remove_from_file_path(fid, group_name); +} + +bool os_grp_0(hid_t fid, const char *group_name) { hid_t gid = -1; @@ -1094,66 +989,52 @@ os_grp_0(hid_t fid, const char *group_name) herr_t ret; - if ( pass ) { /* get the file's file access property list */ - fapl = H5Fget_access_plist(fid); - if ( fapl <= 0 ) { - pass = FALSE; - failure_mssg = "os_grp_0: H5Fget_access_plist() failed."; - } - HDassert(fapl > 0); + /* get the file's file access property list */ + fapl = H5Fget_access_plist(fid); + if ( fapl <= 0 ) { + failure_mssg = "os_grp_0: H5Fget_access_plist() failed."; + return false; } - if ( pass ) { /* get low and high bounds from fapl */ - ret = H5Pget_libver_bounds(fapl, &low, &high); - if ( ret < 0 ) { - pass = FALSE; - failure_mssg = "os_grp_0: H5Pget_libver_bounds() failed(1)."; - } - HDassert(ret >= 0); + /* get low and high bounds from fapl */ + ret = H5Pget_libver_bounds(fapl, &low, &high); + if ( ret < 0 ) { + failure_mssg = "os_grp_0: H5Pget_libver_bounds() failed(1)."; + return false; } - if ( pass ) { /* turn file format latest off */ - if(low >= H5F_LIBVER_V18) { - ret = H5Fset_libver_bounds(fid, H5F_LIBVER_EARLIEST, high); - if ( ret < 0 ) { - pass = FALSE; - failure_mssg = "os_grp_0: H5Fset_libver_bounds() failed(1)."; - } - HDassert(ret >= 0); + /* turn file format latest off */ + if(low >= H5F_LIBVER_V18) { + ret = H5Fset_libver_bounds(fid, H5F_LIBVER_EARLIEST, high); + if ( ret < 0 ) { + failure_mssg = "os_grp_0: H5Fset_libver_bounds() failed(1)."; + return false; } } - if ( pass ) { - gid = H5Gcreate2(fid, group_name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - if ( gid <= 0 ) { - pass = FALSE; - failure_mssg = "os_grp_0: H5Gcreate2() failed."; - } - HDassert(gid > 0); + gid = H5Gcreate2(fid, group_name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + if ( gid <= 0 ) { + failure_mssg = "os_grp_0: H5Gcreate2() failed."; + return false; } - if ( pass ) { - ret = H5Gclose(gid); + ret = H5Gclose(gid); - if ( ret < 0 ) { - pass = FALSE; - failure_mssg = "os_grp_0: H5Gclose() failed."; - } - HDassert(ret >= 0); + if ( ret < 0 ) { + failure_mssg = "os_grp_0: H5Gclose() failed."; + return false; } - if ( pass ) { /* restore low and high bounds */ - if(low >= H5F_LIBVER_V18) { - ret = H5Fset_libver_bounds(fid, low, high); - if ( ret < 0 ) { - pass = FALSE; - failure_mssg = "os_grp_0: H5Fset_libver_bounds() failed(1)."; - } - HDassert(ret >= 0); + /* restore low and high bounds */ + if(low >= H5F_LIBVER_V18) { + ret = H5Fset_libver_bounds(fid, low, high); + if ( ret < 0 ) { + failure_mssg = "os_grp_0: H5Fset_libver_bounds() failed(1)."; + return false; } } - return; + return true; } /* os_grp_0() */ @@ -1162,12 +1043,10 @@ os_grp_0(hid_t fid, const char *group_name) * * Purpose: Validate an empty "old style" group. * - * If pass is false on entry, do nothing. - * - * If an error is detected, set pass to FALSE, and set + * If an error is detected, return false, and set * failure_mssg to point to an appropriate error message. * - * Return: void + * Return: true on success, false on failure. * * Programmer: John Mainzer * 9/14/15 @@ -1175,102 +1054,74 @@ os_grp_0(hid_t fid, const char *group_name) *------------------------------------------------------------------------- */ -void -vrfy_os_grp_0(hid_t fid, const char *group_name) { +bool +vrfy_os_grp_0(hid_t fid, const char *group_name) +{ hid_t gid = -1; hid_t gcpl = -1; H5G_info_t grp_info; unsigned crt_order_flags = 0; herr_t ret; - if (pass) { - gid = H5Gopen2(fid, group_name, H5P_DEFAULT); + gid = H5Gopen2(fid, group_name, H5P_DEFAULT); - if (gid <= 0) { - pass = FALSE; - failure_mssg = "vrfy_os_grp_0: H5Gopen2() failed."; - } - HDassert(gid > 0); + if (gid <= 0) { + failure_mssg = "vrfy_os_grp_0: H5Gopen2() failed."; + return false; } - if (pass) { - gcpl = H5Gget_create_plist(gid); + gcpl = H5Gget_create_plist(gid); - if (gcpl <= 0) { - pass = FALSE; - failure_mssg = "vrfy_os_grp_0: H5Gget_create_plist() failed."; - } - HDassert(gcpl > 0); + if (gcpl <= 0) { + failure_mssg = "vrfy_os_grp_0: H5Gget_create_plist() failed."; + return false; } - if (pass) { - - ret = H5Pget_link_creation_order(gcpl, &crt_order_flags); + ret = H5Pget_link_creation_order(gcpl, &crt_order_flags); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_os_grp_0: H5Pget_link_creation_order() failed"; - } - else if (0 != crt_order_flags) { - pass = FALSE; - failure_mssg = "vrfy_os_grp_0: 0 != crt_order_flags"; - } - HDassert(ret >= 0); - HDassert(0 == crt_order_flags); + if (ret < 0) { + failure_mssg = "vrfy_os_grp_0: H5Pget_link_creation_order() failed"; + return false; + } else if (0 != crt_order_flags) { + failure_mssg = "vrfy_os_grp_0: 0 != crt_order_flags"; + return false; } - if (pass) { - ret = H5Pclose(gcpl); + ret = H5Pclose(gcpl); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_os_grp_0: H5Pclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "vrfy_os_grp_0: H5Pclose() failed."; + return false; } - if (pass) { - HDmemset(&grp_info, 0, sizeof(grp_info)); - ret = H5Gget_info(gid, &grp_info); + HDmemset(&grp_info, 0, sizeof(grp_info)); + ret = H5Gget_info(gid, &grp_info); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_os_grp_0: H5Gget_info() failed."; - } - else if (H5G_STORAGE_TYPE_SYMBOL_TABLE != grp_info.storage_type) { - pass = FALSE; - failure_mssg = "vrfy_os_grp_0: H5G_STORAGE_TYPE_SYMBOL_TABLE != grp_info.storage_type"; - } - else if (0 != grp_info.nlinks) { - pass = FALSE; - failure_mssg = "vrfy_os_grp_0: 0 != grp_info.nlinks"; - } - else if (0 != grp_info.max_corder) { - pass = FALSE; - failure_mssg = "vrfy_os_grp_0: 0 != grp_info.max_corder"; - } - else if ( FALSE != grp_info.mounted) { - pass = FALSE; - failure_mssg = "vrfy_os_grp_0: FALSE != grp_info.mounted"; - } - HDassert(ret >= 0); - HDassert(H5G_STORAGE_TYPE_SYMBOL_TABLE == grp_info.storage_type); - HDassert(0 == grp_info.nlinks); - HDassert(0 == grp_info.max_corder); - HDassert(false == grp_info.mounted); + if (ret < 0) { + failure_mssg = "vrfy_os_grp_0: H5Gget_info() failed."; + return false; + } else if (H5G_STORAGE_TYPE_SYMBOL_TABLE != grp_info.storage_type) { + failure_mssg = "vrfy_os_grp_0: H5G_STORAGE_TYPE_SYMBOL_TABLE != grp_info.storage_type"; + return false; + } else if (0 != grp_info.nlinks) { + failure_mssg = "vrfy_os_grp_0: 0 != grp_info.nlinks"; + return false; + } else if (0 != grp_info.max_corder) { + failure_mssg = "vrfy_os_grp_0: 0 != grp_info.max_corder"; + return false; + } else if ( FALSE != grp_info.mounted) { + failure_mssg = "vrfy_os_grp_0: FALSE != grp_info.mounted"; + return false; } - if (pass) { - ret = H5Gclose(gid); + ret = H5Gclose(gid); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_os_grp_0: H5Gclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "vrfy_os_grp_0: H5Gclose() failed."; + return false; } - return; + return true; } /* vrfy_os_grp_0() */ @@ -1280,12 +1131,10 @@ vrfy_os_grp_0(hid_t fid, const char *group_name) { * Purpose: Create an "old style" group, with 'nlinks' soft/hard * links in it. * - * If pass is false on entry, do nothing. - * - * If an error is detected, set pass to FALSE, and set + * If an error is detected, return false, and set * failure_mssg to point to an appropriate error message. * - * Return: void + * Return: true on success, false on failure. * * Programmer: John Mainzer * 9/14/15 @@ -1293,7 +1142,21 @@ vrfy_os_grp_0(hid_t fid, const char *group_name) { *------------------------------------------------------------------------- */ -void +static bool +missing_os_grp_n(hid_t fid, const char *group_name, int H5_ATTR_UNUSED proc_num, + unsigned H5_ATTR_UNUSED nlinks) +{ + return file_has_no_path(fid, group_name); +} + +static bool +rm_os_grp_n(hid_t fid, const char *group_name, int H5_ATTR_UNUSED proc_num, + unsigned H5_ATTR_UNUSED nlinks) +{ + return remove_from_file_path(fid, group_name); +} + +bool os_grp_n(hid_t fid, const char *group_name, int proc_num, unsigned nlinks) { hid_t gid = -1; @@ -1302,48 +1165,38 @@ os_grp_n(hid_t fid, const char *group_name, int proc_num, unsigned nlinks) H5F_libver_t low, high; herr_t ret; - if ( pass ) { /* get the file's file access property list */ - fapl = H5Fget_access_plist(fid); - if ( fapl <= 0 ) { - pass = FALSE; - failure_mssg = "os_grp_n: H5Fget_access_plist() failed."; - } - HDassert(fapl > 0); + /* get the file's file access property list */ + fapl = H5Fget_access_plist(fid); + if ( fapl <= 0 ) { + failure_mssg = "os_grp_n: H5Fget_access_plist() failed."; + return false; } - if ( pass ) { /* get low and high bounds from fapl */ - ret = H5Pget_libver_bounds(fapl, &low, &high); - if ( ret < 0 ) { - pass = FALSE; - failure_mssg = "os_grp_0: H5Pget_libver_bounds() failed(1)."; - } - HDassert(ret >= 0); + /* get low and high bounds from fapl */ + ret = H5Pget_libver_bounds(fapl, &low, &high); + if ( ret < 0 ) { + failure_mssg = "os_grp_0: H5Pget_libver_bounds() failed(1)."; + return false; } - if ( pass ) { /* turn file format latest off */ - if(low >= H5F_LIBVER_V18) { - ret = H5Fset_libver_bounds(fid, H5F_LIBVER_EARLIEST, high); - if ( ret < 0 ) { - pass = FALSE; - failure_mssg = "os_grp_0: H5Fset_libver_bounds() failed(1)."; - } - HDassert(ret >= 0); + /* turn file format latest off */ + if(low >= H5F_LIBVER_V18) { + ret = H5Fset_libver_bounds(fid, H5F_LIBVER_EARLIEST, high); + if ( ret < 0 ) { + failure_mssg = "os_grp_0: H5Fset_libver_bounds() failed(1)."; + return false; } } - if ( pass ) { - gid = H5Gcreate2(fid, group_name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - if ( gid <= 0 ) { - pass = FALSE; - failure_mssg = "os_grp_n: H5Gcreate2() failed."; - } - HDassert(gid > 0); + gid = H5Gcreate2(fid, group_name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + if ( gid <= 0 ) { + failure_mssg = "os_grp_n: H5Gcreate2() failed."; + return false; } HDassert(nlinks > 0); - u = 0; - while ( ( pass ) && ( u < nlinks ) ) { + for (u = 0; u < nlinks; u++) { char linkname[32]; HDsprintf(linkname, "ln%d_%u", proc_num, u); @@ -1351,47 +1204,37 @@ os_grp_n(hid_t fid, const char *group_name, int proc_num, unsigned nlinks) if(0 == (u % 2)) { ret = H5Lcreate_soft(group_name, gid, linkname, H5P_DEFAULT, H5P_DEFAULT); if ( ret < 0 ) { - pass = FALSE; failure_mssg = "os_grp_n: H5Lcreate_soft() failed."; + return false; } - HDassert(ret >= 0); - } /* end if */ - else { + } else { HDassert(1 == (u % 2)); ret = H5Lcreate_hard(fid, "/", gid, linkname, H5P_DEFAULT, H5P_DEFAULT); if ( ret < 0 ) { - pass = FALSE; failure_mssg = "os_grp_n: H5Lcreate_hard() failed."; + return false; } - HDassert(ret >= 0); - } /* end else */ - - u++; - } /* end while */ + } + } - if ( pass ) { - ret = H5Gclose(gid); + ret = H5Gclose(gid); - if ( ret < 0 ) { - pass = FALSE; - failure_mssg = "os_grp_n: H5Gclose() failed."; - } - HDassert(ret >= 0); + if ( ret < 0 ) { + failure_mssg = "os_grp_n: H5Gclose() failed."; + return false; } - if ( pass ) { /* restore low and high bounds */ - if(low >= H5F_LIBVER_V18) { - ret = H5Fset_libver_bounds(fid, low, high); - if ( ret < 0 ) { - pass = FALSE; - failure_mssg = "os_grp_n: H5Fset_libver_bounds() failed(2)."; - } - HDassert(ret >= 0); + /* restore low and high bounds */ + if(low >= H5F_LIBVER_V18) { + ret = H5Fset_libver_bounds(fid, low, high); + if ( ret < 0 ) { + failure_mssg = "os_grp_n: H5Fset_libver_bounds() failed(2)."; + return false; } } - return; + return true; } /* os_grp_n() */ @@ -1401,21 +1244,19 @@ os_grp_n(hid_t fid, const char *group_name, int proc_num, unsigned nlinks) * Purpose: Validate an "old style" group with 'nlinks' soft/hard * links in it. * - * If pass is false on entry, do nothing. - * - * If an error is detected, set pass to FALSE, and set + * If an error is detected, return false, and set * failure_mssg to point to an appropriate error message. * - * Return: void + * Return: true on success, false on failure. * * Programmer: John Mainzer * 9/14/15 * *------------------------------------------------------------------------- */ -void -vrfy_os_grp_n(hid_t fid, const char *group_name, int proc_num, - unsigned nlinks) { +bool +vrfy_os_grp_n(hid_t fid, const char *group_name, int proc_num, unsigned nlinks) +{ hid_t gid = -1; hid_t gcpl = -1; H5G_info_t grp_info; @@ -1423,85 +1264,59 @@ vrfy_os_grp_n(hid_t fid, const char *group_name, int proc_num, unsigned u; herr_t ret; - if (pass) { - gid = H5Gopen2(fid, group_name, H5P_DEFAULT); + gid = H5Gopen2(fid, group_name, H5P_DEFAULT); - if (gid <= 0) { - pass = FALSE; - failure_mssg = "vrfy_os_grp_n: H5Gopen2() failed"; - } - HDassert(gid > 0); + if (gid <= 0) { + failure_mssg = "vrfy_os_grp_n: H5Gopen2() failed"; + return false; } - if (pass) { - gcpl = H5Gget_create_plist(gid); + gcpl = H5Gget_create_plist(gid); - if (gcpl <= 0) { - pass = FALSE; - failure_mssg = "vrfy_os_grp_n: H5Gget_create_plist() failed"; - } - HDassert(gcpl > 0); + if (gcpl <= 0) { + failure_mssg = "vrfy_os_grp_n: H5Gget_create_plist() failed"; + return false; } - if (pass) { - ret = H5Pget_link_creation_order(gcpl, &crt_order_flags); + ret = H5Pget_link_creation_order(gcpl, &crt_order_flags); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_os_grp_n: H5Pget_link_creation_order"; - } - else if (0 != crt_order_flags) { - pass = FALSE; - failure_mssg = "vrfy_os_grp_n: 0 != crt_order_flags"; - } - HDassert(ret >= 0); - HDassert(0 == crt_order_flags); + if (ret < 0) { + failure_mssg = "vrfy_os_grp_n: H5Pget_link_creation_order"; + return false; + } else if (0 != crt_order_flags) { + failure_mssg = "vrfy_os_grp_n: 0 != crt_order_flags"; + return false; } - if (pass) { - ret = H5Pclose(gcpl); + ret = H5Pclose(gcpl); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_os_grp_n: H5Pclose() failed"; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "vrfy_os_grp_n: H5Pclose() failed"; + return false; } - if (pass) { - HDmemset(&grp_info, 0, sizeof(grp_info)); + HDmemset(&grp_info, 0, sizeof(grp_info)); - ret = H5Gget_info(gid, &grp_info); + ret = H5Gget_info(gid, &grp_info); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_os_grp_n: H5Gget_info() failed"; - } - else if (H5G_STORAGE_TYPE_SYMBOL_TABLE != grp_info.storage_type) { - pass = FALSE; - failure_mssg = "vrfy_os_grp_n: H5G_STORAGE_TYPE_SYMBOL_TABLE != grp_info.storage_type"; - } - else if (nlinks != grp_info.nlinks) { - pass = FALSE; - failure_mssg = "vrfy_os_grp_n: nlinks != grp_info.nlinks"; - } - else if (0 != grp_info.max_corder) { - pass = FALSE; - failure_mssg = "vrfy_os_grp_n: 0 != grp_info.max_corder"; - } - else if ( FALSE != grp_info.mounted) { - pass = FALSE; - failure_mssg = "vrfy_os_grp_n: FALSE != grp_info.mounted"; - } - HDassert(ret >= 0); - HDassert(H5G_STORAGE_TYPE_SYMBOL_TABLE == grp_info.storage_type); - HDassert(nlinks == grp_info.nlinks); - HDassert(0 == grp_info.max_corder); - HDassert(false == grp_info.mounted); + if (ret < 0) { + failure_mssg = "vrfy_os_grp_n: H5Gget_info() failed"; + return false; + } else if (H5G_STORAGE_TYPE_SYMBOL_TABLE != grp_info.storage_type) { + failure_mssg = "vrfy_os_grp_n: H5G_STORAGE_TYPE_SYMBOL_TABLE != grp_info.storage_type"; + return false; + } else if (nlinks != grp_info.nlinks) { + failure_mssg = "vrfy_os_grp_n: nlinks != grp_info.nlinks"; + return false; + } else if (0 != grp_info.max_corder) { + failure_mssg = "vrfy_os_grp_n: 0 != grp_info.max_corder"; + return false; + } else if ( FALSE != grp_info.mounted) { + failure_mssg = "vrfy_os_grp_n: FALSE != grp_info.mounted"; + return false; } - u = 0; - while ((pass) && (u < nlinks)) { + for (u = 0; u < nlinks; u++) { H5L_info2_t lnk_info; char linkname[32]; htri_t link_exists; @@ -1510,8 +1325,8 @@ vrfy_os_grp_n(hid_t fid, const char *group_name, int proc_num, link_exists = H5Lexists(gid, linkname, H5P_DEFAULT); if (link_exists < 0) { - pass = FALSE; failure_mssg = "vrfy_os_grp_n: H5Lexists() failed"; + return false; } HDassert(link_exists >= 0); @@ -1519,107 +1334,84 @@ vrfy_os_grp_n(hid_t fid, const char *group_name, int proc_num, ret = H5Lget_info2(gid, linkname, &lnk_info, H5P_DEFAULT); if (ret < 0) { - pass = FALSE; failure_mssg = "vrfy_os_grp_n: H5Lget_info() failed"; + return false; } else if ( FALSE != lnk_info.corder_valid) { - pass = FALSE; failure_mssg = "vrfy_os_grp_n: FALSE != lnk_info.corder_valid"; + return false; } else if (H5T_CSET_ASCII != lnk_info.cset) { - pass = FALSE; failure_mssg = "vrfy_os_grp_n: H5T_CSET_ASCII != lnk_info.cset"; + return false; } - HDassert(ret >= 0); - HDassert(false == lnk_info.corder_valid); - HDassert(H5T_CSET_ASCII == lnk_info.cset); if (0 == (u % 2)) { char *slinkval; if (H5L_TYPE_SOFT != lnk_info.type) { - pass = FALSE; failure_mssg = "vrfy_os_grp_n: H5L_TYPE_SOFT != lnk_info.type"; - } - else if ((HDstrlen(group_name) + 1) != lnk_info.u.val_size) { - pass = FALSE; + return false; + } else if ((HDstrlen(group_name) + 1) != lnk_info.u.val_size) { failure_mssg = "vrfy_os_grp_n: (HDstrlen(group_name) + 1) != lnk_info.u.val_size"; + return false; } - HDassert(H5L_TYPE_SOFT == lnk_info.type); - HDassert((HDstrlen(group_name) + 1) == lnk_info.u.val_size); - slinkval = (char *) HDmalloc(lnk_info.u.val_size); + slinkval = HDmalloc(lnk_info.u.val_size); if (!slinkval) { - pass = FALSE; failure_mssg = "vrfy_os_grp_n: HDmalloc of slinkval failed"; + return false; } - HDassert(slinkval); ret = H5Lget_val(gid, linkname, slinkval, lnk_info.u.val_size, - H5P_DEFAULT); + H5P_DEFAULT); if (ret < 0) { - pass = FALSE; failure_mssg = "vrfy_os_grp_n: H5Lget_val() failed"; - } - else if (0 != HDstrcmp(slinkval, group_name)) { - pass = FALSE; + HDfree(slinkval); + return false; + } else if (0 != HDstrcmp(slinkval, group_name)) { failure_mssg = "vrfy_os_grp_n: 0 != HDstrcmp(slinkval, group_name)"; + HDfree(slinkval); + return false; } - HDassert(ret >= 0); - HDassert(0 == HDstrcmp(slinkval, group_name)); - HDfree(slinkval); - } /* end if */ - else { + } else { H5O_info2_t root_oinfo; int token_cmp = 0; HDassert(1 == (u % 2)); if (H5L_TYPE_HARD != lnk_info.type) { - pass = FALSE; failure_mssg = "vrfy_os_grp_n: H5L_TYPE_HARD != lnk_info.type"; + return false; } - HDassert(H5L_TYPE_HARD == lnk_info.type); HDmemset(&root_oinfo, 0, sizeof(root_oinfo)); ret = H5Oget_info3(fid, &root_oinfo, H5O_INFO_BASIC); if (ret < 0) { - pass = FALSE; failure_mssg = "vrfy_os_grp_n: H5Oget_info() failed."; + return false; + } else if(H5Otoken_cmp(fid, &root_oinfo.token, &lnk_info.u.token, &token_cmp) < 0) { + failure_mssg = "vrfy_os_grp_n: H5Otoken_cmp() failed."; + return false; + } else if (token_cmp) { + failure_mssg = "vrfy_os_grp_n: root_oinfo.token != lnk_info.u.token"; + return false; } - else { - if(H5Otoken_cmp(fid, &root_oinfo.token, &lnk_info.u.token, &token_cmp) < 0) { - pass = FALSE; - failure_mssg = "vrfy_os_grp_n: H5Otoken_cmp() failed."; - } - - if (token_cmp) { - pass = FALSE; - failure_mssg = "vrfy_os_grp_n: root_oinfo.token != lnk_info.u.token"; - } - } - HDassert(ret >= 0); - HDassert(!token_cmp); - } /* end else */ - - u++; - } /* end while */ + } + } - if (pass) { - ret = H5Gclose(gid); + ret = H5Gclose(gid); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_os_grp_n: H5Gclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "vrfy_os_grp_n: H5Gclose() failed."; + return false; } - return; + return true; } /* vrfy_os_grp_n() */ @@ -1630,20 +1422,32 @@ vrfy_os_grp_n(hid_t fid, const char *group_name, int proc_num, * to the data set or not as indicated by the write_data * parameter. * - * If pass is false on entry, do nothing. - * - * If an error is detected, set pass to FALSE, and set + * If an error is detected, return false, and set * failure_mssg to point to an appropriate error message. * - * Return: void + * Return: true on success, false on failure. * * Programmer: John Mainzer * 9/14/15 * *------------------------------------------------------------------------- */ -void -ds_ctg_i(hid_t fid, const char *dset_name, hbool_t write_data) { +static bool +missing_ds_ctg_i(hid_t fid, const char *dset_name, + hbool_t H5_ATTR_UNUSED write_data) +{ + return file_has_no_path(fid, dset_name); +} + +static bool +rm_ds_ctg_i(hid_t fid, const char *dset_name, hbool_t H5_ATTR_UNUSED write_data) +{ + return remove_from_file_path(fid, dset_name); +} + +bool +ds_ctg_i(hid_t fid, const char *dset_name, hbool_t write_data) +{ int *wdata = NULL; unsigned u; hid_t dsid = -1; @@ -1651,74 +1455,58 @@ ds_ctg_i(hid_t fid, const char *dset_name, hbool_t write_data) { hsize_t dims[1] = { DSET_DIMS }; herr_t ret; - if (pass) { - sid = H5Screate_simple(1, dims, NULL); + sid = H5Screate_simple(1, dims, NULL); - if (sid <= 0) { - pass = FALSE; - failure_mssg = "ds_ctg_i: H5Screate_simple() failed"; - } - HDassert(sid > 0); + if (sid <= 0) { + failure_mssg = "ds_ctg_i: H5Screate_simple() failed"; + return false; } - if (pass) { - dsid = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, - H5P_DEFAULT, H5P_DEFAULT); + dsid = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, + H5P_DEFAULT, H5P_DEFAULT); - if (dsid <= 0) { - pass = FALSE; - failure_mssg = "ds_ctg_i: H5Dcreate2() failed"; - } - HDassert(dsid > 0); + if (dsid <= 0) { + failure_mssg = "ds_ctg_i: H5Dcreate2() failed"; + return false; } - if (pass) { - ret = H5Sclose(sid); + ret = H5Sclose(sid); - if (ret < 0) { - pass = FALSE; - failure_mssg = "ds_ctg_i: H5Sclose() failed"; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "ds_ctg_i: H5Sclose() failed"; + return false; } - if ((pass) && (write_data)) { - wdata = (int *) HDmalloc(sizeof(int) * DSET_DIMS); + if (write_data) { + wdata = HDmalloc(sizeof(int) * DSET_DIMS); if (!wdata) { - pass = FALSE; failure_mssg = "ds_ctg_i: HDmalloc of wdata failed."; + return false; } - HDassert(wdata); - } - if ((pass) && (write_data)) { for (u = 0; u < DSET_DIMS; u++) wdata[u] = (int) u; - ret = H5Dwrite(dsid, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, - H5P_DEFAULT, wdata); + ret = H5Dwrite(dsid, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, + wdata); + + HDfree(wdata); if (ret < 0) { - pass = FALSE; failure_mssg = "ds_ctg_i: H5Dwrite() failed."; + return false; } - HDassert(ret >= 0); - - HDfree(wdata); } - if (pass) { - ret = H5Dclose(dsid); + ret = H5Dclose(dsid); - if (ret < 0) { - pass = FALSE; - failure_mssg = "ds_ctg_i: H5Dclose() failed"; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "ds_ctg_i: H5Dclose() failed"; + return false; } - return; + return true; } /* ds_ctg_i */ @@ -1728,20 +1516,19 @@ ds_ctg_i(hid_t fid, const char *dset_name, hbool_t write_data) { * Purpose: Validate a contiguous datasets w/int datatypes. Validate * data if indicated via the write_data parameter. * - * If pass is false on entry, do nothing. - * - * If an error is detected, set pass to FALSE, and set + * If an error is detected, return false, and set * failure_mssg to point to an appropriate error message. * - * Return: void + * Return: true on success, false on failure. * * Programmer: John Mainzer * 9/14/15 * *------------------------------------------------------------------------- */ -void -vrfy_ds_ctg_i(hid_t fid, const char *dset_name, hbool_t write_data) { +bool +vrfy_ds_ctg_i(hid_t fid, const char *dset_name, hbool_t write_data) +{ int *rdata = NULL; unsigned u; hid_t dsid = -1; @@ -1755,191 +1542,137 @@ vrfy_ds_ctg_i(hid_t fid, const char *dset_name, hbool_t write_data) { htri_t type_equal; herr_t ret; - if (pass) { - dsid = H5Dopen2(fid, dset_name, H5P_DEFAULT); + dsid = H5Dopen2(fid, dset_name, H5P_DEFAULT); - if (dsid <= 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_i: H5Dopen2() failed."; - } - HDassert(dsid > 0); + if (dsid <= 0) { + failure_mssg = "vrfy_ds_ctg_i: H5Dopen2() failed."; + return false; } - if (pass) { - sid = H5Dget_space(dsid); + sid = H5Dget_space(dsid); - if (sid <= 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_i: H5Dget_space() failed."; - } - HDassert(sid > 0); + if (sid <= 0) { + failure_mssg = "vrfy_ds_ctg_i: H5Dget_space() failed."; + return false; } - if (pass) { - ndims = H5Sget_simple_extent_ndims(sid); + ndims = H5Sget_simple_extent_ndims(sid); - if (1 != ndims) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_i: 1 != ndims"; - } - HDassert(1 == ndims); + if (1 != ndims) { + failure_mssg = "vrfy_ds_ctg_i: 1 != ndims"; + return false; } - if (pass) { - ret = H5Sget_simple_extent_dims(sid, dims, max_dims); + ret = H5Sget_simple_extent_dims(sid, dims, max_dims); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_i: H5Sget_simple_extent_dims() failed"; - } - else if ( DSET_DIMS != dims[0]) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_i: DSET_DIMS != dims[0]"; - } - else if ( DSET_DIMS != max_dims[0]) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_i: DSET_DIMS != max_dims[0]"; - } - HDassert(ret >= 0); - HDassert(DSET_DIMS == dims[0]); - HDassert(DSET_DIMS == max_dims[0]); + if (ret < 0) { + failure_mssg = "vrfy_ds_ctg_i: H5Sget_simple_extent_dims() failed"; + return false; + } else if ( DSET_DIMS != dims[0]) { + failure_mssg = "vrfy_ds_ctg_i: DSET_DIMS != dims[0]"; + return false; + } else if ( DSET_DIMS != max_dims[0]) { + failure_mssg = "vrfy_ds_ctg_i: DSET_DIMS != max_dims[0]"; + return false; } - if (pass) { - ret = H5Sclose(sid); + ret = H5Sclose(sid); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_i: H5Sclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "vrfy_ds_ctg_i: H5Sclose() failed."; + return false; } - if (pass) { - tid = H5Dget_type(dsid); + tid = H5Dget_type(dsid); - if (tid <= 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_i: H5Dget_type() failed."; - } - HDassert(tid > 0); + if (tid <= 0) { + failure_mssg = "vrfy_ds_ctg_i: H5Dget_type() failed."; + return false; } - if (pass) { - type_equal = H5Tequal(tid, H5T_NATIVE_INT); + type_equal = H5Tequal(tid, H5T_NATIVE_INT); - if (1 != type_equal) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_i: type not H5T_NATIVE_INT"; - } - HDassert(1 == type_equal); + if (1 != type_equal) { + failure_mssg = "vrfy_ds_ctg_i: type not H5T_NATIVE_INT"; + return false; } - if (pass) { - ret = H5Tclose(tid); + ret = H5Tclose(tid); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_i: H5Tclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "vrfy_ds_ctg_i: H5Tclose() failed."; + return false; } - if (pass) { - ret = H5Dget_space_status(dsid, &allocation); + ret = H5Dget_space_status(dsid, &allocation); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_i: H5Dget_space_status() failed."; - } - else if (write_data && (allocation != H5D_SPACE_STATUS_ALLOCATED)) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_i: write_data && allocation != H5D_SPACE_STATUS_ALLOCATED"; - } - else if (!write_data - && (allocation != H5D_SPACE_STATUS_NOT_ALLOCATED)) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_i: !write_data && allocation != H5D_SPACE_STATUS_NOT_ALLOCATED"; - } - HDassert(ret >= 0); - HDassert((write_data && allocation == H5D_SPACE_STATUS_ALLOCATED) - || (!write_data && allocation == H5D_SPACE_STATUS_NOT_ALLOCATED)); + if (ret < 0) { + failure_mssg = "vrfy_ds_ctg_i: H5Dget_space_status() failed."; + return false; + } else if (write_data && (allocation != H5D_SPACE_STATUS_ALLOCATED)) { + failure_mssg = "vrfy_ds_ctg_i: " + "write_data && allocation != H5D_SPACE_STATUS_ALLOCATED"; + return false; + } else if (!write_data && (allocation != H5D_SPACE_STATUS_NOT_ALLOCATED)) { + failure_mssg = "vrfy_ds_ctg_i: " + "!write_data && allocation != H5D_SPACE_STATUS_NOT_ALLOCATED"; + return false; } - if (pass) { - dcpl = H5Dget_create_plist(dsid); + dcpl = H5Dget_create_plist(dsid); - if (dcpl <= 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_i: H5Dget_create_plist() failed."; - } - HDassert(dcpl > 0); + if (dcpl <= 0) { + failure_mssg = "vrfy_ds_ctg_i: H5Dget_create_plist() failed."; + return false; } - if (pass) { - layout = H5Pget_layout(dcpl); + layout = H5Pget_layout(dcpl); - if (H5D_CONTIGUOUS != layout) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_i: H5D_CONTIGUOUS != layout"; - } - HDassert(H5D_CONTIGUOUS == layout); + if (H5D_CONTIGUOUS != layout) { + failure_mssg = "vrfy_ds_ctg_i: H5D_CONTIGUOUS != layout"; + return false; } - if (pass) { - ret = H5Pclose(dcpl); + ret = H5Pclose(dcpl); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_i: H5Pclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "vrfy_ds_ctg_i: H5Pclose() failed."; + return false; } - if ((pass) && (write_data)) { - rdata = (int *) HDmalloc(sizeof(int) * DSET_DIMS); + if (write_data) { + rdata = HDmalloc(sizeof(int) * DSET_DIMS); if (!rdata) { - pass = FALSE; failure_mssg = "vrfy_ds_ctg_i: HDmalloc of rdata failed."; + return false; } - HDassert(rdata); - } - if ((pass) && (write_data)) { - ret = H5Dread(dsid, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, - H5P_DEFAULT, rdata); + ret = H5Dread(dsid, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, + rdata); if (ret < 0) { - pass = FALSE; failure_mssg = "vrfy_ds_ctg_i: H5Dread() failed."; + return false; } - HDassert(ret >= 0); - } - if ((pass) && (write_data)) { for (u = 0; u < DSET_DIMS; u++) { if ((int) u != rdata[u]) { - pass = FALSE; failure_mssg = "vrfy_ds_ctg_i: u != rdata[u]."; - break; + HDfree(rdata); + return false; } - HDassert((int )u == rdata[u]); } - HDfree(rdata); - } /* end if */ + } - if (pass) { - ret = H5Dclose(dsid); + ret = H5Dclose(dsid); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_i: H5Dclose() failed"; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "vrfy_ds_ctg_i: H5Dclose() failed"; + return false; } - return; + return true; } /* vrfy_ds_ctg_i() */ @@ -1951,20 +1684,32 @@ vrfy_ds_ctg_i(hid_t fid, const char *dset_name, hbool_t write_data) { * to the data set or not as indicated by the write_data * parameter. * - * If pass is false on entry, do nothing. - * - * If an error is detected, set pass to FALSE, and set + * If an error is detected, return false, and set * failure_mssg to point to an appropriate error message. * - * Return: void + * Return: true on success, false on failure. * * Programmer: John Mainzer * 9/14/15 * *------------------------------------------------------------------------- */ -void -ds_chk_i(hid_t fid, const char *dset_name, hbool_t write_data) { +static bool +missing_ds_chk_i(hid_t fid, const char *dset_name, + hbool_t H5_ATTR_UNUSED write_data) +{ + return file_has_no_path(fid, dset_name); +} + +static bool +rm_ds_chk_i(hid_t fid, const char *dset_name, hbool_t H5_ATTR_UNUSED write_data) +{ + return remove_from_file_path(fid, dset_name); +} + +bool +ds_chk_i(hid_t fid, const char *dset_name, hbool_t write_data) +{ int *wdata = NULL; unsigned u; hid_t dsid = -1; @@ -1974,102 +1719,77 @@ ds_chk_i(hid_t fid, const char *dset_name, hbool_t write_data) { hsize_t chunk_dims[1] = { DSET_CHUNK_DIMS }; herr_t ret; - if (pass) { - sid = H5Screate_simple(1, dims, NULL); + sid = H5Screate_simple(1, dims, NULL); - if (sid <= 0) { - pass = FALSE; - failure_mssg = "ds_chk_i: H5Screate_simple() failed."; - } - HDassert(sid > 0); + if (sid <= 0) { + failure_mssg = "ds_chk_i: H5Screate_simple() failed."; + return false; } - if (pass) { - dcpl = H5Pcreate(H5P_DATASET_CREATE); + dcpl = H5Pcreate(H5P_DATASET_CREATE); - if (dcpl <= 0) { - pass = FALSE; - failure_mssg = "ds_chk_i: H5Pcreate() failed."; - } - HDassert(dcpl > 0); + if (dcpl <= 0) { + failure_mssg = "ds_chk_i: H5Pcreate() failed."; + return false; } - if (pass) { - ret = H5Pset_chunk(dcpl, 1, chunk_dims); + ret = H5Pset_chunk(dcpl, 1, chunk_dims); - if (ret < 0) { - pass = FALSE; - failure_mssg = "ds_chk_i: H5Pset_chunk() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "ds_chk_i: H5Pset_chunk() failed."; + return false; } - if (pass) { - dsid = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, - H5P_DEFAULT, dcpl, H5P_DEFAULT); + dsid = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, + H5P_DEFAULT); - if (dsid <= 0) { - pass = FALSE; - failure_mssg = "ds_chk_i: H5Dcreate2() failed"; - } - HDassert(dsid > 0); + if (dsid <= 0) { + failure_mssg = "ds_chk_i: H5Dcreate2() failed"; + return false; } - if (pass) { - ret = H5Pclose(dcpl); + ret = H5Pclose(dcpl); - if (ret < 0) { - pass = FALSE; - failure_mssg = "ds_chk_i: H5Pclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "ds_chk_i: H5Pclose() failed."; + return false; } - if (pass) { - ret = H5Sclose(sid); + ret = H5Sclose(sid); - if (ret < 0) { - pass = FALSE; - failure_mssg = "ds_chk_i: H5Sclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "ds_chk_i: H5Sclose() failed."; + return false; } - if ((pass) && (write_data)) { - wdata = (int *) HDmalloc(sizeof(int) * DSET_DIMS); + if (write_data) { + wdata = HDmalloc(sizeof(int) * DSET_DIMS); if (!wdata) { - pass = FALSE; failure_mssg = "ds_chk_i: HDmalloc of wdata failed."; + return false; } - HDassert(wdata); - } - if ((pass) && (write_data)) { for (u = 0; u < DSET_DIMS; u++) wdata[u] = (int) u; - ret = H5Dwrite(dsid, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, - H5P_DEFAULT, wdata); + ret = H5Dwrite(dsid, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, + wdata); + HDfree(wdata); if (ret < 0) { - pass = FALSE; failure_mssg = "ds_chk_i: H5Dwrite() failed."; + return false; } - HDassert(ret >= 0); - HDfree(wdata); - } /* end if */ + } - if (pass) { - ret = H5Dclose(dsid); + ret = H5Dclose(dsid); - if (ret < 0) { - pass = FALSE; - failure_mssg = "ds_chk_i: H5Dclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "ds_chk_i: H5Dclose() failed."; + return false; } - return; + return true; } /* ds_chk_i */ @@ -2079,20 +1799,19 @@ ds_chk_i(hid_t fid, const char *dset_name, hbool_t write_data) { * Purpose: Validate a chunked datasets w/int datatypes. Validate * data if indicated via the write_data parameter. * - * If pass is false on entry, do nothing. - * - * If an error is detected, set pass to FALSE, and set + * If an error is detected, return false, and set * failure_mssg to point to an appropriate error message. * - * Return: void + * Return: true on success, false on failure. * * Programmer: John Mainzer * 9/14/15 * *------------------------------------------------------------------------- */ -void -vrfy_ds_chk_i(hid_t fid, const char *dset_name, hbool_t write_data) { +bool +vrfy_ds_chk_i(hid_t fid, const char *dset_name, hbool_t write_data) +{ int *rdata = NULL; unsigned u; hid_t dsid = -1; @@ -2106,204 +1825,145 @@ vrfy_ds_chk_i(hid_t fid, const char *dset_name, hbool_t write_data) { htri_t type_equal; herr_t ret; - if (pass) { - dsid = H5Dopen2(fid, dset_name, H5P_DEFAULT); + dsid = H5Dopen2(fid, dset_name, H5P_DEFAULT); - if (dsid <= 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_chk_i: H5Dopen2() failed."; - } - HDassert(dsid > 0); + if (dsid <= 0) { + failure_mssg = "vrfy_ds_chk_i: H5Dopen2() failed."; + return false; } - if (pass) { - sid = H5Dget_space(dsid); + sid = H5Dget_space(dsid); - if (sid <= 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_chk_i: H5Dget_space() failed."; - } - HDassert(sid > 0); + if (sid <= 0) { + failure_mssg = "vrfy_ds_chk_i: H5Dget_space() failed."; + return false; } - if (pass) { - ndims = H5Sget_simple_extent_ndims(sid); + ndims = H5Sget_simple_extent_ndims(sid); - if (1 != ndims) { - pass = FALSE; - failure_mssg = "vrfy_ds_chk_i: 1 != ndims"; - } - HDassert(1 == ndims); + if (1 != ndims) { + failure_mssg = "vrfy_ds_chk_i: 1 != ndims"; + return false; } - if (pass) { - ret = H5Sget_simple_extent_dims(sid, dims, max_dims); + ret = H5Sget_simple_extent_dims(sid, dims, max_dims); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_chk_i: H5Sget_simple_extent_dims() failed"; - } - else if ( DSET_DIMS != dims[0]) { - pass = FALSE; - failure_mssg = "vrfy_ds_chk_i: DSET_DIMS != dims[0]"; - } - else if ( DSET_DIMS != max_dims[0]) { - pass = FALSE; - failure_mssg = "vrfy_ds_chk_i: DSET_DIMS != max_dims[0]"; - } - HDassert(ret >= 0); - HDassert(DSET_DIMS == dims[0]); - HDassert(DSET_DIMS == max_dims[0]); + if (ret < 0) { + failure_mssg = "vrfy_ds_chk_i: H5Sget_simple_extent_dims() failed"; + return false; + } else if ( DSET_DIMS != dims[0]) { + failure_mssg = "vrfy_ds_chk_i: DSET_DIMS != dims[0]"; + return false; + } else if ( DSET_DIMS != max_dims[0]) { + failure_mssg = "vrfy_ds_chk_i: DSET_DIMS != max_dims[0]"; + return false; } - if (pass) { - ret = H5Sclose(sid); + ret = H5Sclose(sid); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_chk_i: H5Sclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "vrfy_ds_chk_i: H5Sclose() failed."; + return false; } - if (pass) { - tid = H5Dget_type(dsid); + tid = H5Dget_type(dsid); - if (tid <= 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_chk_i: H5Dget_type() failed."; - } - HDassert(tid > 0); + if (tid <= 0) { + failure_mssg = "vrfy_ds_chk_i: H5Dget_type() failed."; + return false; } - if (pass) { - type_equal = H5Tequal(tid, H5T_NATIVE_INT); + type_equal = H5Tequal(tid, H5T_NATIVE_INT); - if (1 != type_equal) { - pass = FALSE; - failure_mssg = "vrfy_ds_chk_i: tid != H5T_NATIVE_INT"; - } - HDassert(1 == type_equal); + if (1 != type_equal) { + failure_mssg = "vrfy_ds_chk_i: tid != H5T_NATIVE_INT"; + return false; } - if (pass) { - ret = H5Tclose(tid); + ret = H5Tclose(tid); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_chk_i: H5Tclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "vrfy_ds_chk_i: H5Tclose() failed."; + return false; } - if (pass) { - ret = H5Dget_space_status(dsid, &allocation); + ret = H5Dget_space_status(dsid, &allocation); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_chk_i: H5Dget_space_status() failed."; - } - else if (write_data && (allocation != H5D_SPACE_STATUS_ALLOCATED)) { - pass = FALSE; - failure_mssg = "vrfy_ds_chk_i: write_data && allocation != H5D_SPACE_STATUS_ALLOCATED"; - } - else if (!write_data && (allocation != H5D_SPACE_STATUS_NOT_ALLOCATED)) { - pass = FALSE; - failure_mssg = "vrfy_ds_chk_i: !write_data && allocation != H5D_SPACE_STATUS_NOT_ALLOCATED"; - } - HDassert(ret >= 0); - HDassert((write_data && allocation == H5D_SPACE_STATUS_ALLOCATED) - || (!write_data && allocation == H5D_SPACE_STATUS_NOT_ALLOCATED)); + if (ret < 0) { + failure_mssg = "vrfy_ds_chk_i: H5Dget_space_status() failed."; + return false; + } else if (write_data && (allocation != H5D_SPACE_STATUS_ALLOCATED)) { + failure_mssg = "vrfy_ds_chk_i: write_data && allocation != H5D_SPACE_STATUS_ALLOCATED"; + return false; + } else if (!write_data && (allocation != H5D_SPACE_STATUS_NOT_ALLOCATED)) { + failure_mssg = "vrfy_ds_chk_i: !write_data && allocation != H5D_SPACE_STATUS_NOT_ALLOCATED"; + return false; } - if (pass) { - dcpl = H5Dget_create_plist(dsid); + dcpl = H5Dget_create_plist(dsid); - if (dcpl <= 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_chk_i: H5Dget_create_plist() failed."; - } - HDassert(dcpl > 0); + if (dcpl <= 0) { + failure_mssg = "vrfy_ds_chk_i: H5Dget_create_plist() failed."; + return false; } - if (pass) { - layout = H5Pget_layout(dcpl); + layout = H5Pget_layout(dcpl); - if (H5D_CHUNKED != layout) { - pass = FALSE; - failure_mssg = "vrfy_ds_chk_i: H5D_CHUNKED != layout"; - } - HDassert(H5D_CHUNKED == layout); + if (H5D_CHUNKED != layout) { + failure_mssg = "vrfy_ds_chk_i: H5D_CHUNKED != layout"; + return false; } - if (pass) { - ret = H5Pget_chunk(dcpl, 1, chunk_dims); + ret = H5Pget_chunk(dcpl, 1, chunk_dims); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_chk_i: H5Pget_chunk"; - } - else if ( DSET_CHUNK_DIMS != chunk_dims[0]) { - pass = FALSE; - failure_mssg = "vrfy_ds_chk_i: "; - } - HDassert(ret >= 0); - HDassert(DSET_CHUNK_DIMS == chunk_dims[0]); + if (ret < 0) { + failure_mssg = "vrfy_ds_chk_i: H5Pget_chunk"; + return false; + } else if ( DSET_CHUNK_DIMS != chunk_dims[0]) { + failure_mssg = "vrfy_ds_chk_i: "; + return false; } - if (pass) { - ret = H5Pclose(dcpl); + ret = H5Pclose(dcpl); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_chk_i: H5Pclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "vrfy_ds_chk_i: H5Pclose() failed."; + return false; } - if ((pass) && (write_data)) { - rdata = (int *) HDmalloc(sizeof(int) * DSET_DIMS); + if (write_data) { + rdata = HDmalloc(sizeof(int) * DSET_DIMS); if (!rdata) { - pass = FALSE; failure_mssg = "vrfy_ds_chk_i: HDmalloc of rdata failed."; + return false; } - HDassert(rdata); - } - if ((pass) && (write_data)) { - ret = H5Dread(dsid, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rdata); + ret = H5Dread(dsid, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, + rdata); if (ret < 0) { - pass = FALSE; failure_mssg = "vrfy_ds_chk_i: H5Dread() failed."; + return false; } - HDassert(ret >= 0); - } - if ((pass) && (write_data)) { for (u = 0; u < DSET_DIMS; u++) { if ((int) u != rdata[u]) { - pass = FALSE; failure_mssg = "vrfy_ds_chk_i: u != rdata[u]"; - break; + HDfree(rdata); + return false; } - HDassert((int )u == rdata[u]); } - HDfree(rdata); - } /* end if */ + } - if (pass) { - ret = H5Dclose(dsid); + ret = H5Dclose(dsid); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_chk_i: H5Dclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "vrfy_ds_chk_i: H5Dclose() failed."; + return false; } - return; + return true; } /* vrfy_ds_chk_i() */ @@ -2314,20 +1974,32 @@ vrfy_ds_chk_i(hid_t fid, const char *dset_name, hbool_t write_data) { * to the data set or not as indicated by the write_data * parameter. * - * If pass is false on entry, do nothing. - * - * If an error is detected, set pass to FALSE, and set + * If an error is detected, return false, and set * failure_mssg to point to an appropriate error message. * - * Return: void + * Return: true on success, false on failure. * * Programmer: John Mainzer * 9/14/15 * *------------------------------------------------------------------------- */ -void -ds_cpt_i(hid_t fid, const char *dset_name, hbool_t write_data) { +static bool +missing_ds_cpt_i(hid_t fid, const char *dset_name, + hbool_t H5_ATTR_UNUSED write_data) +{ + return file_has_no_path(fid, dset_name); +} + +static bool +rm_ds_cpt_i(hid_t fid, const char *dset_name, hbool_t H5_ATTR_UNUSED write_data) +{ + return remove_from_file_path(fid, dset_name); +} + +bool +ds_cpt_i(hid_t fid, const char *dset_name, hbool_t write_data) +{ int *wdata = NULL; unsigned u; hid_t dsid = -1; @@ -2336,104 +2008,78 @@ ds_cpt_i(hid_t fid, const char *dset_name, hbool_t write_data) { hsize_t dims[1] = { DSET_COMPACT_DIMS }; herr_t ret; - if (pass) { - sid = H5Screate_simple(1, dims, NULL); + sid = H5Screate_simple(1, dims, NULL); - if (sid <= 0) { - pass = FALSE; - failure_mssg = "ds_cpt_i: H5Screate_simple() failed."; - } - HDassert(sid > 0); + if (sid <= 0) { + failure_mssg = "ds_cpt_i: H5Screate_simple() failed."; + return false; } - if (pass) { - dcpl = H5Pcreate(H5P_DATASET_CREATE); + dcpl = H5Pcreate(H5P_DATASET_CREATE); - if (dcpl <= 0) { - pass = FALSE; - failure_mssg = "ds_cpt_i: H5Pcreate() failed."; - } - HDassert(dcpl > 0); + if (dcpl <= 0) { + failure_mssg = "ds_cpt_i: H5Pcreate() failed."; + return false; } - if (pass) { - ret = H5Pset_layout(dcpl, H5D_COMPACT); + ret = H5Pset_layout(dcpl, H5D_COMPACT); - if (ret < 0) { - pass = FALSE; - failure_mssg = "ds_cpt_i: H5Pset_layout() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "ds_cpt_i: H5Pset_layout() failed."; + return false; } - if (pass) { - dsid = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, - H5P_DEFAULT, dcpl, H5P_DEFAULT); + dsid = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, + H5P_DEFAULT); - if (dsid <= 0) { - pass = FALSE; - failure_mssg = "ds_cpt_i: H5Dcreate2() failed."; - } - HDassert(dsid > 0); + if (dsid <= 0) { + failure_mssg = "ds_cpt_i: H5Dcreate2() failed."; + return false; } - if (pass) { - ret = H5Pclose(dcpl); + ret = H5Pclose(dcpl); - if (ret < 0) { - pass = FALSE; - failure_mssg = "ds_cpt_i: H5Pclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "ds_cpt_i: H5Pclose() failed."; + return false; } - if (pass) { - ret = H5Sclose(sid); + ret = H5Sclose(sid); - if (ret < 0) { - pass = FALSE; - failure_mssg = "ds_cpt_i: H5Sclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "ds_cpt_i: H5Sclose() failed."; + return false; } - if ((pass) && (write_data)) { - wdata = (int *) HDmalloc(sizeof(int) * DSET_COMPACT_DIMS); + if (write_data) { + wdata = HDmalloc(sizeof(int) * DSET_COMPACT_DIMS); if (!wdata) { - pass = FALSE; failure_mssg = "ds_cpt_i: HDmalloc of wdata failed."; + return false; } - HDassert(wdata); - } - if ((pass) && (write_data)) { for (u = 0; u < DSET_COMPACT_DIMS; u++) wdata[u] = (int) u; - ret = H5Dwrite(dsid, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, - H5P_DEFAULT, wdata); + ret = H5Dwrite(dsid, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, + wdata); + HDfree(wdata); if (ret < 0) { - pass = FALSE; failure_mssg = "ds_cpt_i: H5Dwrite() failed."; + return false; } - HDassert(ret >= 0); - - HDfree(wdata); - } /* end if */ + } - if (pass) { - ret = H5Dclose(dsid); + ret = H5Dclose(dsid); - if (ret < 0) { - pass = FALSE; - failure_mssg = "ds_cpt_i: H5Dclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "ds_cpt_i: H5Dclose() failed."; + return false; } - return; + return true; } /* ds_cpt_i() */ @@ -2444,20 +2090,19 @@ ds_cpt_i(hid_t fid, const char *dset_name, hbool_t write_data) { * Purpose: Validate a compact datasets w/int datatypes. Validate * data if indicated via the write_data parameter. * - * If pass is false on entry, do nothing. - * - * If an error is detected, set pass to FALSE, and set + * If an error is detected, return false, and set * failure_mssg to point to an appropriate error message. * - * Return: void + * Return: true on success, false on failure. * * Programmer: John Mainzer * 9/14/15 * *------------------------------------------------------------------------- */ -void -vrfy_ds_cpt_i(hid_t fid, const char *dset_name, hbool_t write_data) { +bool +vrfy_ds_cpt_i(hid_t fid, const char *dset_name, hbool_t write_data) +{ int *rdata = NULL; unsigned u; hid_t dsid = -1; @@ -2471,186 +2116,133 @@ vrfy_ds_cpt_i(hid_t fid, const char *dset_name, hbool_t write_data) { htri_t type_equal; herr_t ret; - if (pass) { - dsid = H5Dopen2(fid, dset_name, H5P_DEFAULT); + dsid = H5Dopen2(fid, dset_name, H5P_DEFAULT); - if (dsid <= 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_cpt_i: H5Dopen2() failed."; - } - HDassert(dsid > 0); + if (dsid <= 0) { + failure_mssg = "vrfy_ds_cpt_i: H5Dopen2() failed."; + return false; } - if (pass) { - sid = H5Dget_space(dsid); + sid = H5Dget_space(dsid); - if (sid <= 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_cpt_i: H5Dget_space() failed."; - } - HDassert(sid > 0); + if (sid <= 0) { + failure_mssg = "vrfy_ds_cpt_i: H5Dget_space() failed."; + return false; } - if (pass) { - ndims = H5Sget_simple_extent_ndims(sid); + ndims = H5Sget_simple_extent_ndims(sid); - if (1 != ndims) { - pass = FALSE; - failure_mssg = "vrfy_ds_cpt_i: 1 != ndims"; - } - HDassert(1 == ndims); + if (1 != ndims) { + failure_mssg = "vrfy_ds_cpt_i: 1 != ndims"; + return false; } - if (pass) { - ret = H5Sget_simple_extent_dims(sid, dims, max_dims); + ret = H5Sget_simple_extent_dims(sid, dims, max_dims); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_cpt_i: H5Sget_simple_extent_dims() failed"; - } - else if ( DSET_COMPACT_DIMS != dims[0]) { - pass = FALSE; - failure_mssg = "vrfy_ds_cpt_i: DSET_COMPACT_DIMS != dims[0]"; - } - else if ( DSET_COMPACT_DIMS != max_dims[0]) { - pass = FALSE; - failure_mssg = "vrfy_ds_cpt_i: DSET_COMPACT_DIMS != max_dims[0]"; - } - HDassert(ret >= 0); - HDassert(DSET_COMPACT_DIMS == dims[0]); - HDassert(DSET_COMPACT_DIMS == max_dims[0]); + if (ret < 0) { + failure_mssg = "vrfy_ds_cpt_i: H5Sget_simple_extent_dims() failed"; + return false; + } else if ( DSET_COMPACT_DIMS != dims[0]) { + failure_mssg = "vrfy_ds_cpt_i: DSET_COMPACT_DIMS != dims[0]"; + return false; + } else if ( DSET_COMPACT_DIMS != max_dims[0]) { + failure_mssg = "vrfy_ds_cpt_i: DSET_COMPACT_DIMS != max_dims[0]"; + return false; } - if (pass) { - ret = H5Sclose(sid); + ret = H5Sclose(sid); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_cpt_i: H5Sclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "vrfy_ds_cpt_i: H5Sclose() failed."; + return false; } - if (pass) { - tid = H5Dget_type(dsid); + tid = H5Dget_type(dsid); - if (tid <= 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_cpt_i: H5Dget_type() failed."; - } - HDassert(tid > 0); + if (tid <= 0) { + failure_mssg = "vrfy_ds_cpt_i: H5Dget_type() failed."; + return false; } - if (pass) { - type_equal = H5Tequal(tid, H5T_NATIVE_INT); + type_equal = H5Tequal(tid, H5T_NATIVE_INT); - if (1 != type_equal) { - pass = FALSE; - failure_mssg = "vrfy_ds_cpt_i: type != H5T_NATIVE_INT"; - } - HDassert(1 == type_equal); + if (1 != type_equal) { + failure_mssg = "vrfy_ds_cpt_i: type != H5T_NATIVE_INT"; + return false; } - if (pass) { - ret = H5Tclose(tid); + ret = H5Tclose(tid); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_cpt_i: H5Tclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "vrfy_ds_cpt_i: H5Tclose() failed."; + return false; } - if (pass) { - ret = H5Dget_space_status(dsid, &allocation); + ret = H5Dget_space_status(dsid, &allocation); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_cpt_i: H5Dget_space_status() failed."; - } - else if (H5D_SPACE_STATUS_ALLOCATED != allocation) { - pass = FALSE; - failure_mssg = - "vrfy_ds_cpt_i: H5D_SPACE_STATUS_ALLOCATED != allocation"; - } - HDassert(ret >= 0); - HDassert(H5D_SPACE_STATUS_ALLOCATED == allocation); + if (ret < 0) { + failure_mssg = "vrfy_ds_cpt_i: H5Dget_space_status() failed."; + return false; + } else if (H5D_SPACE_STATUS_ALLOCATED != allocation) { + failure_mssg = + "vrfy_ds_cpt_i: H5D_SPACE_STATUS_ALLOCATED != allocation"; + return false; } - if (pass) { - dcpl = H5Dget_create_plist(dsid); + dcpl = H5Dget_create_plist(dsid); - if (dcpl <= 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_cpt_i: H5Dget_create_plist() failed."; - } - HDassert(dcpl > 0); + if (dcpl <= 0) { + failure_mssg = "vrfy_ds_cpt_i: H5Dget_create_plist() failed."; + return false; } - if (pass) { - layout = H5Pget_layout(dcpl); + layout = H5Pget_layout(dcpl); - if (H5D_COMPACT != layout) { - pass = FALSE; - failure_mssg = "vrfy_ds_cpt_i: H5D_COMPACT != layout"; - } - HDassert(H5D_COMPACT == layout); + if (H5D_COMPACT != layout) { + failure_mssg = "vrfy_ds_cpt_i: H5D_COMPACT != layout"; + return false; } - if (pass) { - ret = H5Pclose(dcpl); + ret = H5Pclose(dcpl); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_cpt_i: H5Pclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "vrfy_ds_cpt_i: H5Pclose() failed."; + return false; } - if ((pass) && (write_data)) { - rdata = (int *) HDmalloc(sizeof(int) * DSET_COMPACT_DIMS); + if (write_data) { + rdata = HDmalloc(sizeof(int) * DSET_COMPACT_DIMS); if (!rdata) { - pass = FALSE; failure_mssg = "vrfy_ds_cpt_i: HDmalloc of rdata failed."; + return false; } - HDassert(rdata); - } - if ((pass) && (write_data)) { - ret = H5Dread(dsid, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, - H5P_DEFAULT, rdata); + ret = H5Dread(dsid, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, + rdata); if (ret < 0) { - pass = FALSE; failure_mssg = "vrfy_ds_cpt_i: H5Dread() failed."; + return false; } - HDassert(ret >= 0); - } - if ((pass) && (write_data)) { for (u = 0; u < DSET_COMPACT_DIMS; u++) { if ((int) u != rdata[u]) { - pass = FALSE; failure_mssg = "vrfy_ds_cpt_i: (int)u != rdata[u]"; - break; + HDfree(rdata); + return false; } - HDassert((int )u == rdata[u]); } - HDfree(rdata); - } /* end if */ + } - if (pass) { - ret = H5Dclose(dsid); + ret = H5Dclose(dsid); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_cpt_i: H5Dclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "vrfy_ds_cpt_i: H5Dclose() failed."; + return false; } - return; + return true; } /* vrfy_ds_cpt_i() */ @@ -2661,20 +2253,32 @@ vrfy_ds_cpt_i(hid_t fid, const char *dset_name, hbool_t write_data) { * Write data to the data set or not as indicated by the * write_data parameter. * - * If pass is false on entry, do nothing. - * - * If an error is detected, set pass to FALSE, and set + * If an error is detected, return false, and set * failure_mssg to point to an appropriate error message. * - * Return: void + * Return: true on success, false on failure. * * Programmer: John Mainzer * 9/14/15 * *------------------------------------------------------------------------- */ -void -ds_ctg_v(hid_t fid, const char *dset_name, hbool_t write_data) { +static bool +missing_ds_ctg_v(hid_t fid, const char *dset_name, + hbool_t H5_ATTR_UNUSED write_data) +{ + return file_has_no_path(fid, dset_name); +} + +static bool +rm_ds_ctg_v(hid_t fid, const char *dset_name, hbool_t H5_ATTR_UNUSED write_data) +{ + return remove_from_file_path(fid, dset_name); +} + +bool +ds_ctg_v(hid_t fid, const char *dset_name, hbool_t write_data) +{ hid_t dsid = -1; hid_t sid = -1; hid_t tid = -1; @@ -2683,124 +2287,101 @@ ds_ctg_v(hid_t fid, const char *dset_name, hbool_t write_data) { hvl_t *wdata = NULL; unsigned u; - if (pass) { - sid = H5Screate_simple(1, dims, NULL); + sid = H5Screate_simple(1, dims, NULL); - if (sid <= 0) { - pass = FALSE; - failure_mssg = "ds_ctg_v: H5Screate_simple"; - } - HDassert(sid > 0); + if (sid <= 0) { + failure_mssg = "ds_ctg_v: H5Screate_simple"; + return false; } - if (pass) { - tid = H5Tvlen_create(H5T_NATIVE_INT); + tid = H5Tvlen_create(H5T_NATIVE_INT); - if (tid <= 0) { - pass = FALSE; - failure_mssg = "ds_ctg_v: H5Tvlen_create() failed."; - } - HDassert(tid > 0); + if (tid <= 0) { + failure_mssg = "ds_ctg_v: H5Tvlen_create() failed."; + return false; } - if (pass) { - dsid = H5Dcreate2(fid, dset_name, tid, sid, H5P_DEFAULT, - H5P_DEFAULT, H5P_DEFAULT); + dsid = H5Dcreate2(fid, dset_name, tid, sid, H5P_DEFAULT, + H5P_DEFAULT, H5P_DEFAULT); - if (dsid <= 0) { - pass = FALSE; - failure_mssg = "ds_ctg_v: H5Dcreate2() failed."; - } - HDassert(dsid > 0); + if (dsid <= 0) { + failure_mssg = "ds_ctg_v: H5Dcreate2() failed."; + return false; } - if ((pass) && (write_data)) { - wdata = (hvl_t *) HDmalloc(sizeof(hvl_t) * DSET_SMALL_DIMS); + if (write_data) { + wdata = HDmalloc(sizeof(hvl_t) * DSET_SMALL_DIMS); if (!wdata) { - pass = FALSE; failure_mssg = "ds_ctg_v: HDmalloc of wdata failed."; + return false; } - HDassert(wdata); - } - if ((pass) && (write_data)) { for (u = 0; u < DSET_SMALL_DIMS; u++) { int *tdata; unsigned len; unsigned v; len = (u % 10) + 1; - tdata = (int *) HDmalloc(sizeof(int) * len); + tdata = HDmalloc(sizeof(int) * len); if (!tdata) { - pass = FALSE; failure_mssg = "ds_ctg_v: HDmalloc of tdata failed."; - break; + while (u > 0) + free(wdata[u--].p); + HDfree(wdata); + return false; } - HDassert(tdata); for (v = 0; v < len; v++) tdata[v] = (int) (u + v); wdata[u].len = len; wdata[u].p = tdata; - } /* end for */ - } + } - if ((pass) && (write_data)) { ret = H5Dwrite(dsid, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata); if (ret < 0) { - pass = FALSE; failure_mssg = "ds_ctg_v: H5Dwrite() failed."; + for (u = 0; u < DSET_SMALL_DIMS; u++) + free(wdata[u].p); + HDfree(wdata); + return false; } - HDassert(ret >= 0); - } - if ((pass) && (write_data)) { ret = H5Treclaim(tid, sid, H5P_DEFAULT, wdata); + HDfree(wdata); + if (ret < 0) { - pass = FALSE; failure_mssg = "ds_ctg_v: H5Treclaim() failed."; + return false; } - HDassert(ret >= 0); - - HDfree(wdata); - } /* end if */ + } - if (pass) { - ret = H5Sclose(sid); + ret = H5Sclose(sid); - if (sid < 0) { - pass = FALSE; - failure_mssg = "ds_ctg_v: H5Sclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "ds_ctg_v: H5Sclose() failed."; + return false; } - if (pass) { - ret = H5Tclose(tid); + ret = H5Tclose(tid); - if (tid < 0) { - pass = FALSE; - failure_mssg = "ds_ctg_v: H5Tclose() failed."; - } - HDassert(ret >= 0); + if (tid < 0) { + failure_mssg = "ds_ctg_v: H5Tclose() failed."; + return false; } - if (pass) { - ret = H5Dclose(dsid); + ret = H5Dclose(dsid); - if (ret < 0) { - pass = FALSE; - failure_mssg = "ds_ctg_v: H5Dclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "ds_ctg_v: H5Dclose() failed."; + return false; } - return; + return true; } /* ds_ctg_v() */ @@ -2810,20 +2391,19 @@ ds_ctg_v(hid_t fid, const char *dset_name, hbool_t write_data) { * Purpose: Validate a contiguous datasets w/variable-length datatypes. * Validate data if indicated via the write_data parameter. * - * If pass is false on entry, do nothing. - * - * If an error is detected, set pass to FALSE, and set + * If an error is detected, return false, and set * failure_mssg to point to an appropriate error message. * - * Return: void + * Return: true on success, false on failure. * * Programmer: John Mainzer * 9/14/15 * *------------------------------------------------------------------------- */ -void -vrfy_ds_ctg_v(hid_t fid, const char *dset_name, hbool_t write_data) { +bool +vrfy_ds_ctg_v(hid_t fid, const char *dset_name, hbool_t write_data) +{ hid_t dsid = -1; hid_t sid = -1; hid_t tid = -1; @@ -2838,170 +2418,120 @@ vrfy_ds_ctg_v(hid_t fid, const char *dset_name, hbool_t write_data) { unsigned u; herr_t ret; - if (pass) { - dsid = H5Dopen2(fid, dset_name, H5P_DEFAULT); + dsid = H5Dopen2(fid, dset_name, H5P_DEFAULT); - if (dsid <= 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_v: H5Dopen2() failed."; - } - HDassert(dsid > 0); + if (dsid <= 0) { + failure_mssg = "vrfy_ds_ctg_v: H5Dopen2() failed."; + return false; } - if (pass) { - sid = H5Dget_space(dsid); + sid = H5Dget_space(dsid); - if (sid <= 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_v: H5Dget_space() failed"; - } - HDassert(sid > 0); + if (sid <= 0) { + failure_mssg = "vrfy_ds_ctg_v: H5Dget_space() failed"; + return false; } - if (pass) { - ndims = H5Sget_simple_extent_ndims(sid); + ndims = H5Sget_simple_extent_ndims(sid); - if (1 != ndims) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_v: 1 != ndims"; - } - HDassert(1 == ndims); + if (1 != ndims) { + failure_mssg = "vrfy_ds_ctg_v: 1 != ndims"; + return false; } - if (pass) { - ret = H5Sget_simple_extent_dims(sid, dims, max_dims); + ret = H5Sget_simple_extent_dims(sid, dims, max_dims); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_v: H5Sget_simple_extent_dims() failed."; - } - else if ( DSET_SMALL_DIMS != dims[0]) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_v: DSET_SMALL_DIMS != dims[0]"; - } - else if ( DSET_SMALL_DIMS != max_dims[0]) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_v: DSET_SMALL_DIMS != max_dims[0]"; - } - HDassert(ret >= 0); - HDassert(DSET_SMALL_DIMS == dims[0]); - HDassert(DSET_SMALL_DIMS == max_dims[0]); + if (ret < 0) { + failure_mssg = "vrfy_ds_ctg_v: H5Sget_simple_extent_dims() failed."; + return false; + } else if ( DSET_SMALL_DIMS != dims[0]) { + failure_mssg = "vrfy_ds_ctg_v: DSET_SMALL_DIMS != dims[0]"; + return false; + } else if ( DSET_SMALL_DIMS != max_dims[0]) { + failure_mssg = "vrfy_ds_ctg_v: DSET_SMALL_DIMS != max_dims[0]"; + return false; } - if (pass) { - tid = H5Dget_type(dsid); + tid = H5Dget_type(dsid); - if (tid <= 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_v: H5Dget_type() failed."; - } - HDassert(tid > 0); + if (tid <= 0) { + failure_mssg = "vrfy_ds_ctg_v: H5Dget_type() failed."; + return false; } - if (pass) { - tmp_tid = H5Tvlen_create(H5T_NATIVE_INT); + tmp_tid = H5Tvlen_create(H5T_NATIVE_INT); - if (tmp_tid <= 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_v: H5Tvlen_create() failed."; - } - HDassert(tmp_tid > 0); + if (tmp_tid <= 0) { + failure_mssg = "vrfy_ds_ctg_v: H5Tvlen_create() failed."; + return false; } - if (pass) { - type_equal = H5Tequal(tid, tmp_tid); + type_equal = H5Tequal(tid, tmp_tid); - if (1 != type_equal) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_v: type != vlen H5T_NATIVE_INT"; - } - HDassert(1 == type_equal); + if (1 != type_equal) { + failure_mssg = "vrfy_ds_ctg_v: type != vlen H5T_NATIVE_INT"; + return false; } - if (pass) { - ret = H5Tclose(tmp_tid); + ret = H5Tclose(tmp_tid); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_v: H5Tclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "vrfy_ds_ctg_v: H5Tclose() failed."; + return false; } - if (pass) { - ret = H5Dget_space_status(dsid, &allocation); + ret = H5Dget_space_status(dsid, &allocation); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_v: H5Dget_space_status() failed"; - } - else if (write_data && (allocation != H5D_SPACE_STATUS_ALLOCATED)) { - pass = FALSE; - failure_mssg = - "vrfy_ds_ctg_v: write_data && allocation != H5D_SPACE_STATUS_ALLOCATED"; - } - else if (!write_data - && (allocation != H5D_SPACE_STATUS_NOT_ALLOCATED)) { - pass = FALSE; - failure_mssg = - "vrfy_ds_ctg_v: !write_data && allocation != H5D_SPACE_STATUS_NOT_ALLOCATED"; - } - HDassert(ret >= 0); - HDassert((write_data && allocation == H5D_SPACE_STATUS_ALLOCATED) - || (!write_data && allocation == H5D_SPACE_STATUS_NOT_ALLOCATED)); + if (ret < 0) { + failure_mssg = "vrfy_ds_ctg_v: H5Dget_space_status() failed"; + return false; + } else if (write_data && (allocation != H5D_SPACE_STATUS_ALLOCATED)) { + failure_mssg = + "vrfy_ds_ctg_v: write_data && allocation != H5D_SPACE_STATUS_ALLOCATED"; + return false; + } else if (!write_data + && (allocation != H5D_SPACE_STATUS_NOT_ALLOCATED)) { + failure_mssg = + "vrfy_ds_ctg_v: !write_data && allocation != H5D_SPACE_STATUS_NOT_ALLOCATED"; + return false; } - if (pass) { - dcpl = H5Dget_create_plist(dsid); + dcpl = H5Dget_create_plist(dsid); - if (dcpl <= 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_v: H5Dget_create_plist() failed."; - } - HDassert(dcpl > 0); + if (dcpl <= 0) { + failure_mssg = "vrfy_ds_ctg_v: H5Dget_create_plist() failed."; + return false; } - if (pass) { - layout = H5Pget_layout(dcpl); + layout = H5Pget_layout(dcpl); - if (H5D_CONTIGUOUS != layout) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_v: H5D_CONTIGUOUS != layout"; - } - HDassert(H5D_CONTIGUOUS == layout); + if (H5D_CONTIGUOUS != layout) { + failure_mssg = "vrfy_ds_ctg_v: H5D_CONTIGUOUS != layout"; + return false; } - if (pass) { - ret = H5Pclose(dcpl); + ret = H5Pclose(dcpl); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_v: H5Pclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "vrfy_ds_ctg_v: H5Pclose() failed."; + return false; } - if ((pass) && (write_data)) { - rdata = (hvl_t *) HDmalloc(sizeof(hvl_t) * DSET_SMALL_DIMS); + if (write_data) { + rdata = HDmalloc(sizeof(hvl_t) * DSET_SMALL_DIMS); if (!rdata) { - pass = FALSE; failure_mssg = "vrfy_ds_ctg_v: HDmalloc of rdata failed."; + return false; } - HDassert(rdata); - } - if ((pass) && (write_data)) { ret = H5Dread(dsid, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, rdata); if (ret < 0) { - pass = FALSE; failure_mssg = "vrfy_ds_ctg_v: H5Dread() failed."; + return false; } - HDassert(ret >= 0); - } - if ((pass) && (write_data)) { for (u = 0; u < DSET_SMALL_DIMS; u++) { unsigned len; unsigned v; @@ -3011,66 +2541,222 @@ vrfy_ds_ctg_v(hid_t fid, const char *dset_name, hbool_t write_data) { int *tdata = (int *) rdata[u].p; if (!tdata) { - pass = FALSE; failure_mssg = "vrfy_ds_ctg_v: !tdata"; - break; - } - else if ((int) (u + v) != tdata[v]) { - pass = FALSE; + return false; + } else if ((int) (u + v) != tdata[v]) { failure_mssg = "vrfy_ds_ctg_v: (int)(u + v) != tdata[v]"; - break; + return false; } - HDassert(tdata); - HDassert((int )(u + v) == tdata[v]); - } /* end for */ - } /* end for */ - } + } + } - if ((pass) && (write_data)) { ret = H5Treclaim(tid, sid, H5P_DEFAULT, rdata); + HDfree(rdata); + if (ret < 0) { - pass = FALSE; failure_mssg = "vrfy_ds_ctg_v: H5Treclaim() failed."; + return false; } - HDassert(ret >= 0); - - HDfree(rdata); - } /* end if */ + } - if (pass) { - ret = H5Sclose(sid); + ret = H5Sclose(sid); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_v: H5Sclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "vrfy_ds_ctg_v: H5Sclose() failed."; + return false; } - if (pass) { - ret = H5Tclose(tid); + ret = H5Tclose(tid); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_v: H5Tclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "vrfy_ds_ctg_v: H5Tclose() failed."; + return false; } - if (pass) { - ret = H5Dclose(dsid); + ret = H5Dclose(dsid); - if (ret < 0) { - pass = FALSE; - failure_mssg = "vrfy_ds_ctg_v: H5Dclose() failed."; - } - HDassert(ret >= 0); + if (ret < 0) { + failure_mssg = "vrfy_ds_ctg_v: H5Dclose() failed."; + return false; } - return; + return true; } /* vrfy_ds_ctg_v() */ +/* Create or, if `validate` is true, validate objects in file `fid` under + * group `full_path`. `proc_num` tells the processor number the test runs + * on. The set of objects to create/validate is chosen by `selector`. + * + * The valid selectors are consecutive and they start at 0. + * + * If the selected objects cannot be created/validated, `*okp` will be set + * to `false`, indicating that the selected objects could not be + * created/validated, and `failure_mmsg` set to a description of the error + * that occurred. If the objects can be created/validated, then `*okp` + * will be set to `true`. + * + * The program may also fail an assert()ion if the selected objects cannot + * be created/validated. + * + * Return `true` if the selector was valid, `false` if it was not. + */ + +static bool +create_or_validate_selection(hid_t fid, const char *full_path, + int selector, zoo_config_t config, phase_t phase, bool *okp) +{ + bool ok; + + switch (selector) { + case 0: /* Add & verify an empty "new style" group */ + ok = ns_grp_0_fntbl[phase](fid, full_path); + break; + case 1: /* Add & verify a compact "new style" group (3 link messages) */ + ok = ns_grp_c_fntbl[phase](fid, full_path, 3); + break; + case 2: + /* Add & verify a dense "new style" group (w/300 links, + * in v2 B-tree & fractal heap) + */ + ok = ns_grp_d_fntbl[phase](fid, full_path, 300); + break; + case 3: /* Add & verify an empty "old style" group to file */ + ok = os_grp_0_fntbl[phase](fid, full_path); + break; + case 4: + /* Add & verify an "old style" group (w/300 links, in + * v1 B-tree & local heap) to file + */ + ok = os_grp_n_fntbl[phase](fid, full_path, config.proc_num, 300); + break; + case 5: + /* Add & verify a contiguous dataset w/integer datatype (but no data) + * to file + */ + ok = ds_ctg_i_fntbl[phase](fid, full_path, false); + break; + case 6: + /* Add & verify a contiguous dataset w/integer datatype (with data) + * to file + */ + ok = ds_ctg_i_fntbl[phase](fid, full_path, true); + break; + case 7: + /* Add & verify a chunked dataset w/integer datatype (but no data) + * to file + */ + ok = ds_chk_i_fntbl[phase](fid, full_path, false); + break; + case 8: + /* Add & verify a chunked dataset w/integer datatype (and data) + * to file + */ + ok = ds_chk_i_fntbl[phase](fid, full_path, true); + break; + case 9: + /* Add & verify a compact dataset w/integer datatype (but no data) + * to file + */ + ok = config.skip_compact || + ds_cpt_i_fntbl[phase](fid, full_path, false); + break; + case 10: + /* Add & verify a compact dataset w/integer datatype (and data) + * to file + */ + ok = config.skip_compact || ds_cpt_i_fntbl[phase](fid, full_path, true); + break; + case 11: + /* Add & verify a contiguous dataset w/variable-length datatype + * (but no data) to file + */ + ok = config.skip_varlen || ds_ctg_v_fntbl[phase](fid, full_path, false); + break; + case 12: + /* Add & verify a contiguous dataset w/variable-length datatype + * (and data) to file + */ + ok = config.skip_varlen || ds_ctg_v_fntbl[phase](fid, full_path, true); + break; + default: + return false; + } + *okp = ok; + return true; +} + +/* Sleep for no more than `max_pause_msecs` milliseconds. */ +static void +random_pause(unsigned int max_pause_msecs) +{ + struct timespec delay; + const uint64_t nsecs_per_sec = 1000 * 1000 * 1000; + uint64_t nsecs_per_msec, nsecs; + + if (max_pause_msecs == 0) + return; + + nsecs_per_msec = 1 + (uint64_t)random() % (1000 * 1000); + nsecs = max_pause_msecs * nsecs_per_msec; + + delay.tv_sec = (time_t)(nsecs / nsecs_per_sec); + delay.tv_nsec = (long)(nsecs % nsecs_per_sec); + for (;;) { + if (nanosleep(&delay, &delay) == 0) + break; + if (errno == EINTR) + continue; + errx(EXIT_FAILURE, "%s: nanosleep", __func__); + } +} + +/* Create and validate objects or, if `only_validate` is true, only + * validate objects in file `fid` under group `base_path`. `config.proc_num` + * tells the processor number the test runs on. If `config.skip_varlen` is + * true, do NOT perform tests that use variable-length data. + * + * Return true if all tests pass, false if any test fails. + */ + +static bool +tend_zoo(hid_t fid, const char *base_path, zoo_config_t config, + const phase_t *phase, size_t nphases) +{ + char full_path[1024]; + int i, nwritten; + size_t j; + char *leafp; + bool ok = true; + + nwritten = snprintf(full_path, sizeof(full_path), "%s/*", base_path); + if (nwritten < 0 || (size_t)nwritten >= sizeof(full_path)) { + failure_mssg = "tend_zoo: snprintf failed"; + return false; + } + + if ((leafp = strrchr(full_path, '*')) == NULL) { + failure_mssg = "tend_zoo: strrchr failed"; + return false; + } + + for (i = 0; ok; i++) { + assert('A' + i <= 'Z'); + *leafp = (char)('A' + i); + for (j = 0; j < nphases; j++) { + if (!create_or_validate_selection(fid, full_path, i, config, + phase[j], &ok)) + goto out; + if (phase[j] == PHASE_CREATE || phase[j] == PHASE_DELETE) + zoo_create_hook(fid); + } + random_pause(config.max_pause_msecs); + } +out: + if (!ok) + warnx("%s: %s", __func__, failure_mssg); + return ok; +} /*------------------------------------------------------------------------- * Function: create_zoo @@ -3080,9 +2766,7 @@ vrfy_ds_ctg_v(hid_t fid, const char *dset_name, hbool_t write_data) { * include instances of all on disk data structures used * in the HDF5 library. * - * If pass is false on entry, do nothing. - * - * If an error is detected, set pass to FALSE, and set + * If an error is detected, return false, and set * failure_mssg to point to an appropriate error message. * * This function was initially created to assist in testing @@ -3092,214 +2776,16 @@ vrfy_ds_ctg_v(hid_t fid, const char *dset_name, hbool_t write_data) { * the superblock. * * Note the associated validate_zoo() function. - * - * Return: void - * - * Programmer: John Mainzer - * 9/14/15 - * *------------------------------------------------------------------------- */ -void -create_zoo(hid_t fid, const char *base_path, int proc_num) +bool +create_zoo(hid_t fid, const char *base_path, zoo_config_t config) { - char full_path[1024]; - - HDassert(base_path); - - /* Add & verify an empty "new style" group */ - if ( pass ) { - HDsprintf(full_path, "%s/A", base_path); - HDassert(HDstrlen(full_path) < 1024); - ns_grp_0(fid, full_path); - } - - if ( pass ) { - HDsprintf(full_path, "%s/A", base_path); - HDassert(HDstrlen(full_path) < 1024); - vrfy_ns_grp_0(fid, full_path); - } - - /* Add & verify a compact "new style" group (3 link messages) */ - if ( pass ) { - HDsprintf(full_path, "%s/B", base_path); - HDassert(HDstrlen(full_path) < 1024); - ns_grp_c(fid, full_path, 3); - } - - if ( pass ) { - HDsprintf(full_path, "%s/B", base_path); - HDassert(HDstrlen(full_path) < 1024); - vrfy_ns_grp_c(fid, full_path, 3); - } - - /* Add & verify a dense "new style" group (w/300 links, in v2 B-tree & - * fractal heap) - */ - if ( pass ) { - HDsprintf(full_path, "%s/C", base_path); - HDassert(HDstrlen(full_path) < 1024); - ns_grp_d(fid, full_path, 300); - } - - if ( pass ) { - HDsprintf(full_path, "%s/C", base_path); - HDassert(HDstrlen(full_path) < 1024); - vrfy_ns_grp_d(fid, full_path, 300); - } - - /* Add & verify an empty "old style" group to file */ - if ( pass ) { - HDsprintf(full_path, "%s/D", base_path); - HDassert(HDstrlen(full_path) < 1024); - os_grp_0(fid, full_path); - } - - if ( pass ) { - HDsprintf(full_path, "%s/D", base_path); - HDassert(HDstrlen(full_path) < 1024); - vrfy_os_grp_0(fid, full_path); - } - - /* Add & verify an "old style" group (w/300 links, in v1 B-tree & - * local heap) to file - */ - if ( pass ) { - HDsprintf(full_path, "%s/E", base_path); - HDassert(HDstrlen(full_path) < 1024); - os_grp_n(fid, full_path, proc_num, 300); - } - - if ( pass ) { - HDsprintf(full_path, "%s/E", base_path); - HDassert(HDstrlen(full_path) < 1024); - vrfy_os_grp_n(fid, full_path, proc_num, 300); - } - - /* Add & verify a contiguous dataset w/integer datatype (but no data) - * to file - */ - if ( pass ) { - HDsprintf(full_path, "%s/F", base_path); - HDassert(HDstrlen(full_path) < 1024); - ds_ctg_i(fid, full_path, FALSE); - } - - if ( pass ) { - HDsprintf(full_path, "%s/F", base_path); - HDassert(HDstrlen(full_path) < 1024); - vrfy_ds_ctg_i(fid, full_path, FALSE); - } - - /* Add & verify a contiguous dataset w/integer datatype (with data) - * to file - */ - if ( pass ) { - HDsprintf(full_path, "%s/G", base_path); - HDassert(HDstrlen(full_path) < 1024); - ds_ctg_i(fid, full_path, TRUE); - } - - if ( pass ) { - HDsprintf(full_path, "%s/G", base_path); - HDassert(HDstrlen(full_path) < 1024); - vrfy_ds_ctg_i(fid, full_path, TRUE); - } - - /* Add & verify a chunked dataset w/integer datatype (but no data) - * to file - */ - if ( pass ) { - HDsprintf(full_path, "%s/H", base_path); - HDassert(HDstrlen(full_path) < 1024); - ds_chk_i(fid, full_path, FALSE); - } - - if ( pass ) { - HDsprintf(full_path, "%s/H", base_path); - HDassert(HDstrlen(full_path) < 1024); - vrfy_ds_chk_i(fid, full_path, FALSE); - } - - /* Add & verify a chunked dataset w/integer datatype (and data) - * to file - */ - if ( pass ) { - HDsprintf(full_path, "%s/I", base_path); - HDassert(HDstrlen(full_path) < 1024); - ds_chk_i(fid, full_path, TRUE); - } - - if ( pass ) { - HDsprintf(full_path, "%s/I", base_path); - HDassert(HDstrlen(full_path) < 1024); - vrfy_ds_chk_i(fid, full_path, TRUE); - } - - /* Add & verify a compact dataset w/integer datatype (but no data) - * to file - */ - if ( pass ) { - HDsprintf(full_path, "%s/J", base_path); - HDassert(HDstrlen(full_path) < 1024); - ds_cpt_i(fid, full_path, FALSE); - } - - if ( pass ) { - HDsprintf(full_path, "%s/J", base_path); - HDassert(HDstrlen(full_path) < 1024); - vrfy_ds_cpt_i(fid, full_path, FALSE); - } - - /* Add & verify a compact dataset w/integer datatype (and data) - * to file - */ - if ( pass ) { - HDsprintf(full_path, "%s/K", base_path); - HDassert(HDstrlen(full_path) < 1024); - ds_cpt_i(fid, full_path, TRUE); - } - - if ( pass ) { - HDsprintf(full_path, "%s/K", base_path); - HDassert(HDstrlen(full_path) < 1024); - vrfy_ds_cpt_i(fid, full_path, TRUE); - } - - /* Add & verify a contiguous dataset w/variable-length datatype - * (but no data) to file - */ - if ( pass ) { - HDsprintf(full_path, "%s/L", base_path); - HDassert(HDstrlen(full_path) < 1024); - ds_ctg_v(fid, full_path, FALSE); - } - - if ( pass ) { - HDsprintf(full_path, "%s/L", base_path); - HDassert(HDstrlen(full_path) < 1024); - vrfy_ds_ctg_v(fid, full_path, FALSE); - } - - /* Add & verify a contiguous dataset w/variable-length datatype - * (and data) to file - */ - if ( pass ) { - HDsprintf(full_path, "%s/M", base_path); - HDassert(HDstrlen(full_path) < 1024); - ds_ctg_v(fid, full_path, TRUE); - } - - if ( pass ) { - HDsprintf(full_path, "%s/M", base_path); - HDassert(HDstrlen(full_path) < 1024); - vrfy_ds_ctg_v(fid, full_path, TRUE); - } - - return; -} /* create_zoo() */ + const phase_t phase[] = {PHASE_CREATE, PHASE_VALIDATE}; + return tend_zoo(fid, base_path, config, phase, NELMTS(phase)); +} /*------------------------------------------------------------------------- * Function: validate_zoo @@ -3307,9 +2793,7 @@ create_zoo(hid_t fid, const char *base_path, int proc_num) * Purpose: Given the path to a group in which a "zoo" has been * constructed, validate the objects in the "zoo". * - * If pass is false on entry, do nothing. - * - * If an error is detected, set pass to FALSE, and set + * If an error is detected, return false, and set * failure_mssg to point to an appropriate error message. * * This function was initially created to assist in testing @@ -3318,134 +2802,31 @@ create_zoo(hid_t fid, const char *base_path, int proc_num) * on disk structures that can occur with this version of * the superblock. * - * Note the associated validate_zoo() function. - * - * Return: void - * - * Programmer: John Mainzer - * 9/14/15 - * + * Note the associated create_zoo() function. *------------------------------------------------------------------------- */ -void -validate_zoo(hid_t fid, const char *base_path, int proc_num) +bool +validate_zoo(hid_t fid, const char *base_path, zoo_config_t config) { - char full_path[1024]; + const phase_t phase[] = {PHASE_VALIDATE}; + + return tend_zoo(fid, base_path, config, phase, NELMTS(phase)); +} + +bool +delete_zoo(hid_t fid, const char *base_path, zoo_config_t config) +{ + const phase_t phase[] = {PHASE_DELETE}; + + return tend_zoo(fid, base_path, config, phase, NELMTS(phase)); +} + +bool +validate_deleted_zoo(hid_t fid, const char *base_path, zoo_config_t config) +{ + const phase_t phase[] = {PHASE_VALIDATE_DELETION}; - HDassert(base_path); - - /* validate an empty "new style" group */ - if ( pass ) { - HDsprintf(full_path, "%s/A", base_path); - HDassert(HDstrlen(full_path) < 1024); - vrfy_ns_grp_0(fid, full_path); - } - - /* validate a compact "new style" group (3 link messages) */ - if ( pass ) { - HDsprintf(full_path, "%s/B", base_path); - HDassert(HDstrlen(full_path) < 1024); - vrfy_ns_grp_c(fid, full_path, 3); - } - - /* validate a dense "new style" group (w/300 links, in v2 B-tree & - * fractal heap) - */ - if ( pass ) { - HDsprintf(full_path, "%s/C", base_path); - HDassert(HDstrlen(full_path) < 1024); - vrfy_ns_grp_d(fid, full_path, 300); - } - - /* validate an empty "old style" group in file */ - if ( pass ) { - HDsprintf(full_path, "%s/D", base_path); - HDassert(HDstrlen(full_path) < 1024); - vrfy_os_grp_0(fid, full_path); - } - - /* validate an "old style" group (w/300 links, in v1 B-tree & - * local heap) - */ - if ( pass ) { - HDsprintf(full_path, "%s/E", base_path); - HDassert(HDstrlen(full_path) < 1024); - vrfy_os_grp_n(fid, full_path, proc_num, 300); - } - - /* validate a contiguous dataset w/integer datatype (but no data) - * in file. - */ - if ( pass ) { - HDsprintf(full_path, "%s/F", base_path); - HDassert(HDstrlen(full_path) < 1024); - vrfy_ds_ctg_i(fid, full_path, FALSE); - } - - /* validate a contiguous dataset w/integer datatype (with data) - * in file. - */ - if ( pass ) { - HDsprintf(full_path, "%s/G", base_path); - HDassert(HDstrlen(full_path) < 1024); - vrfy_ds_ctg_i(fid, full_path, TRUE); - } - - /* validate a chunked dataset w/integer datatype (but no data) - * in file - */ - if ( pass ) { - HDsprintf(full_path, "%s/H", base_path); - HDassert(HDstrlen(full_path) < 1024); - vrfy_ds_chk_i(fid, full_path, FALSE); - } - - /* validate a chunked dataset w/integer datatype (and data) - * in file - */ - if ( pass ) { - HDsprintf(full_path, "%s/I", base_path); - HDassert(HDstrlen(full_path) < 1024); - vrfy_ds_chk_i(fid, full_path, TRUE); - } - - /* Validate a compact dataset w/integer datatype (but no data) - * in file - */ - if ( pass ) { - HDsprintf(full_path, "%s/J", base_path); - HDassert(HDstrlen(full_path) < 1024); - vrfy_ds_cpt_i(fid, full_path, FALSE); - } - - /* validate a compact dataset w/integer datatype (and data) - * in file - */ - if ( pass ) { - HDsprintf(full_path, "%s/K", base_path); - HDassert(HDstrlen(full_path) < 1024); - vrfy_ds_cpt_i(fid, full_path, TRUE); - } - - /* validate a contiguous dataset w/variable-length datatype - * (but no data) to file - */ - if ( pass ) { - HDsprintf(full_path, "%s/L", base_path); - HDassert(HDstrlen(full_path) < 1024); - vrfy_ds_ctg_v(fid, full_path, FALSE); - } - - /* validate a contiguous dataset w/variable-length datatype - * (and data) to file - */ - if ( pass ) { - HDsprintf(full_path, "%s/M", base_path); - HDassert(HDstrlen(full_path) < 1024); - vrfy_ds_ctg_v(fid, full_path, TRUE); - } - - return; -} /* validate_zoo() */ + return tend_zoo(fid, base_path, config, phase, NELMTS(phase)); +} diff --git a/test/genall5.h b/test/genall5.h index 20141de..a03ccce 100644 --- a/test/genall5.h +++ b/test/genall5.h @@ -11,41 +11,54 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -/* Programmer: John Mainzer - * 9/4/15 - * - * This file contains declarations of all functions defined - * in genall5.c +/* + * This file contains declarations of all functions defined in genall5.c */ -void create_zoo(hid_t fid, const char *base_path, int proc_num); -void validate_zoo(hid_t fid, const char *base_path, int proc_num); +typedef struct _zoo_config { + int proc_num; + bool continue_on_failure; + bool skip_compact; + bool skip_varlen; + unsigned max_pause_msecs; +} zoo_config_t; -void ns_grp_0(hid_t fid, const char *group_name); -void vrfy_ns_grp_0(hid_t fid, const char *group_name); +bool create_zoo(hid_t, const char *, zoo_config_t); +bool validate_zoo(hid_t, const char *, zoo_config_t); +bool delete_zoo(hid_t, const char *, zoo_config_t); +bool validate_deleted_zoo(hid_t, const char *, zoo_config_t); -void ns_grp_c(hid_t fid, const char *group_name, unsigned nlinks); -void vrfy_ns_grp_c(hid_t fid, const char *group_name, unsigned nlinks); +bool ns_grp_0(hid_t fid, const char *group_name); +bool vrfy_ns_grp_0(hid_t fid, const char *group_name); -void ns_grp_d(hid_t fid, const char *group_name, unsigned nlinks); -void vrfy_ns_grp_d(hid_t fid, const char *group_name, unsigned nlinks); +bool ns_grp_c(hid_t fid, const char *group_name, unsigned nlinks); +bool vrfy_ns_grp_c(hid_t fid, const char *group_name, unsigned nlinks); -void os_grp_0(hid_t fid, const char *group_name); -void vrfy_os_grp_0(hid_t fid, const char *group_name); +bool ns_grp_d(hid_t fid, const char *group_name, unsigned nlinks); +bool vrfy_ns_grp_d(hid_t fid, const char *group_name, unsigned nlinks); -void os_grp_n(hid_t fid, const char *group_name, int proc_num, unsigned nlinks); -void vrfy_os_grp_n(hid_t fid, const char *group_name, int proc_num, +bool os_grp_0(hid_t fid, const char *group_name); +bool vrfy_os_grp_0(hid_t fid, const char *group_name); + +bool os_grp_n(hid_t fid, const char *group_name, int proc_num, unsigned nlinks); +bool vrfy_os_grp_n(hid_t fid, const char *group_name, int proc_num, unsigned nlinks); -void ds_ctg_i(hid_t fid, const char *dset_name, hbool_t write_data); -void vrfy_ds_ctg_i(hid_t fid, const char *dset_name, hbool_t write_data); +bool ds_ctg_i(hid_t fid, const char *dset_name, hbool_t write_data); +bool vrfy_ds_ctg_i(hid_t fid, const char *dset_name, hbool_t write_data); -void ds_chk_i(hid_t fid, const char *dset_name, hbool_t write_data); -void vrfy_ds_chk_i(hid_t fid, const char *dset_name, hbool_t write_data); +bool ds_chk_i(hid_t fid, const char *dset_name, hbool_t write_data); +bool vrfy_ds_chk_i(hid_t fid, const char *dset_name, hbool_t write_data); -void ds_cpt_i(hid_t fid, const char *dset_name, hbool_t write_data); -void vrfy_ds_cpt_i(hid_t fid, const char *dset_name, hbool_t write_data); +bool ds_cpt_i(hid_t fid, const char *dset_name, hbool_t write_data); +bool vrfy_ds_cpt_i(hid_t fid, const char *dset_name, hbool_t write_data); -void ds_ctg_v(hid_t fid, const char *dset_name, hbool_t write_data); -void vrfy_ds_ctg_v(hid_t fid, const char *dset_name, hbool_t write_data); +bool ds_ctg_v(hid_t fid, const char *dset_name, hbool_t write_data); +bool vrfy_ds_ctg_v(hid_t fid, const char *dset_name, hbool_t write_data); +/* Individual tests can override zoo_create_hook(), which is called + * after each step of create_zoo(). The `hid_t` argument identifies + * the file where the step was performed. The test library provides a + * default implementation of zoo_create_hook() that does nothing. + */ +void zoo_create_hook(hid_t); diff --git a/test/h5test.c b/test/h5test.c index 9cca100..0496ca4 100644 --- a/test/h5test.c +++ b/test/h5test.c @@ -150,7 +150,18 @@ h5_errors(hid_t estack, void H5_ATTR_UNUSED *client_data) return 0; } - +void +h5_testing(const char *fmt, ...) +{ + va_list ap; + char buf[62 + 1]; /* room for 62-char field + NUL */ + va_start(ap, fmt); + vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + printf("Testing %s", buf); + fflush(stdout); +} + /*------------------------------------------------------------------------- * Function: h5_clean_files * @@ -1941,6 +1952,7 @@ static const H5FD_class_t H5FD_dummy_g = { NULL, /* truncate */ NULL, /* lock */ NULL, /* unlock */ + NULL, /* dedup */ H5FD_FLMAP_DICHOTOMY /* fl_map */ }; diff --git a/test/h5test.h b/test/h5test.h index b1ddc58..1b51772 100644 --- a/test/h5test.h +++ b/test/h5test.h @@ -101,7 +101,7 @@ H5TEST_DLLVAR MPI_Info h5_io_info_g; /* MPI INFO object for IO */ * spaces. If the h5_errors() is used for automatic error handling then * the H5_FAILED() macro is invoked automatically when an API function fails. */ -#define TESTING(WHAT) {HDprintf("Testing %-62s",WHAT); HDfflush(stdout);} +#define TESTING(...) h5_testing(__VA_ARGS__); #define TESTING_2(WHAT) {HDprintf(" Testing %-60s",WHAT); HDfflush(stdout);} #define PASSED() do {HDputs(" PASSED");HDfflush(stdout);} while (0) #define H5_FAILED() {HDputs("*FAILED*");HDfflush(stdout);} @@ -113,6 +113,7 @@ H5TEST_DLLVAR MPI_Info h5_io_info_g; /* MPI INFO object for IO */ #define FAIL_STACK_ERROR {H5_FAILED(); AT(); H5Eprint2(H5E_DEFAULT, stdout); \ goto error;} #define FAIL_PUTS_ERROR(s) {H5_FAILED(); AT(); HDputs(s); goto error;} +#define FAIL_PRINTF_ERROR(fmt, ...) {H5_FAILED(); AT(); HDprintf(fmt, __VA_ARGS__); goto error;} /* * Alarm definitions to wait up (terminate) a test that runs too long. @@ -189,6 +190,7 @@ extern "C" { #endif /* Generally useful testing routines */ +H5TEST_DLL void h5_testing(const char *, ...) H5_ATTR_FORMAT(printf, 1, 2); H5TEST_DLL void h5_clean_files(const char *base_name[], hid_t fapl); H5TEST_DLL int h5_cleanup(const char *base_name[], hid_t fapl); H5TEST_DLL char *h5_fixname(const char *base_name, hid_t fapl, char *fullname, size_t size); diff --git a/test/hyperslab.c b/test/hyperslab.c index e702023..4c84868 100644 --- a/test/hyperslab.c +++ b/test/hyperslab.c @@ -1051,8 +1051,7 @@ test_array_fill(size_t lo, size_t hi) size_t u, v, w; /* Local index variables */ char s[256]; - HDsprintf(s, "array filling %4lu-%-4lu elements", (unsigned long)lo,(unsigned long)hi); - TESTING(s); + TESTING("array filling %4zu-%-4zu elements", lo, hi); /* Initialize */ if(NULL == (dst = (int *)HDcalloc(sizeof(int),ARRAY_FILL_SIZE * hi))) @@ -1116,8 +1115,7 @@ test_array_offset_n_calc(size_t n, size_t x, size_t y, size_t z) hsize_t new_coords[ARRAY_OFFSET_NDIMS]; /* X, Y & X coordinates of offset */ char s[256]; - HDsprintf(s, "array offset %4lux%4lux%4lu elements", (unsigned long)z,(unsigned long)y,(unsigned long)x); - TESTING(s); + TESTING("array offset %4zux%4zux%4zu elements", z, y, x); /* Initialize */ if(NULL == (a = (hsize_t *)HDmalloc(sizeof(hsize_t) * x * y *z))) diff --git a/test/istore.c b/test/istore.c index c8fe866..be6949e 100644 --- a/test/istore.c +++ b/test/istore.c @@ -280,8 +280,7 @@ test_extend(hid_t f, const char *prefix, (unsigned long) nx, (unsigned long) ny, (unsigned long) nz); } - HDsprintf(s, "istore extend: %s", dims); - TESTING(s); + TESTING("istore extend: %s", dims); buf = (uint8_t *)HDmalloc(nx * ny * nz); check = (uint8_t *)HDmalloc(nx * ny * nz); whole = (uint8_t *)HDcalloc((size_t)1, nx * ny * nz); @@ -493,8 +492,7 @@ test_sparse(hid_t f, const char *prefix, size_t nblocks, (unsigned long) nx, (unsigned long) ny, (unsigned long) nz); } - HDsprintf(s, "istore sparse: %s", dims); - TESTING(s); + TESTING("istore sparse: %s", dims); if(skip_test){ SKIPPED() return SUCCEED; diff --git a/test/page_buffer.c b/test/page_buffer.c index a508dc9..6b6de02 100644 --- a/test/page_buffer.c +++ b/test/page_buffer.c @@ -19,6 +19,9 @@ * *************************************************************/ +#include <err.h> +#include <libgen.h> + #include "h5test.h" #include "H5CXprivate.h" /* API Contexts */ @@ -39,30 +42,156 @@ #define FILENAME_LEN 1024 +#ifndef H5_HAVE_PARALLEL +#define NUM_DSETS 5 +#define NX 100 +#define NY 50 +#endif + +/* helper routines */ +#ifndef H5_HAVE_PARALLEL +static unsigned create_file(char *filename, hid_t fcpl, hid_t fapl); +static unsigned open_file(char *filename, hid_t fapl, hsize_t page_size, size_t page_buffer_size); +#endif /* H5_HAVE_PARALLEL */ + /* test routines */ #ifdef H5_HAVE_PARALLEL static unsigned verify_page_buffering_disabled(hid_t orig_fapl, const char *env_h5_drvr); #else -#define NUM_DSETS 5 -#define NX 100 -#define NY 50 - static unsigned test_args(hid_t fapl, const char *env_h5_drvr); -static unsigned test_raw_data_handling(hid_t orig_fapl, const char *env_h5_drvr); +static unsigned test_raw_data_handling(hid_t orig_fapl, const char *env_h5_drvr, + bool); static unsigned test_lru_processing(hid_t orig_fapl, const char *env_h5_drvr); static unsigned test_min_threshold(hid_t orig_fapl, const char *env_h5_drvr); static unsigned test_stats_collection(hid_t orig_fapl, const char *env_h5_drvr); - -/* helper routines */ -static unsigned create_file(char *filename, hid_t fcpl, hid_t fapl); -static unsigned open_file(char *filename, hid_t fapl, hsize_t page_size, size_t page_buffer_size); #endif /* H5_HAVE_PARALLEL */ -const char *FILENAME[] = { - "filepaged", - NULL -}; +#define FILENAME "filepaged" +static const char *namebases[] = {FILENAME, NULL}; +static const char *namebase = FILENAME; + +static hid_t +paging_fcpl_create(const hsize_t pgsz) +{ + hid_t fcpl = H5I_INVALID_HID; + + if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + goto error; + if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, 1) < 0) + goto error; + if (H5Pset_file_space_page_size(fcpl, pgsz) < 0) + goto error; + + return fcpl; +error: + if (fcpl != H5I_INVALID_HID) + H5Pclose(fcpl); + return H5I_INVALID_HID; +} + +static struct timespec +print_elapsed_time(const struct timespec H5_ATTR_UNUSED *lastp, + const char H5_ATTR_UNUSED *fn, int H5_ATTR_UNUSED ln) +{ +#if 0 + uint64_t elapsed_ns; + struct timespec diff, now, last; +#endif + struct timespec now; + + if (clock_gettime(CLOCK_MONOTONIC, &now) == -1) + err(EXIT_FAILURE, "%s: clock_gettime", __func__); + +#if 0 + last = (lastp == NULL) ? now : *lastp; + + timespecsub(&now, &last, &diff); + + elapsed_ns = (uint64_t)diff.tv_sec * (uint64_t)1000000000 + diff.tv_nsec; + + printf("%5" PRIu64 ".%03" PRIu64 " %s.%d\n", + elapsed_ns / 1000000000, (elapsed_ns / 1000000) % 1000, fn, ln); +#endif + + return now; +} + +static int +swmr_fapl_augment(hid_t fapl, const char *filename, uint32_t max_lag) +{ + H5F_vfd_swmr_config_t config = { + .version = H5F__CURR_VFD_SWMR_CONFIG_VERSION + , .tick_len = 4 + , .max_lag = max_lag + , .writer = true + , .md_pages_reserved = 128 + }; + const char *dname; + char *tname; + + if ((tname = strdup(filename)) == NULL) { + HDfprintf(stderr, "temporary string allocation failed\n"); + return -1; + } + dname = dirname(tname); + snprintf(config.md_file_path, sizeof(config.md_file_path), + "%s/my_md_file", dname); + free(tname); + + /* Enable VFD SWMR configuration */ + if(H5Pset_vfd_swmr_config(fapl, &config) < 0) { + HDfprintf(stderr, "H5Pset_vrd_swmr_config failed\n"); + return -1; + } + return 0; +} + +static bool +pgbuf_read_each_equals(H5F_t *f, H5FD_mem_t ty, haddr_t addr, size_t nelts, + int *data, int val) +{ + size_t i; + + /* Read all elements using the VFD. */ + if (H5F_block_read(f, ty, addr, sizeof(int) * nelts, data) < 0) + FAIL_STACK_ERROR; + + for (i = 0; i < nelts; i++) { + if (data[i] != val) { + printf("%s: read %d at data[%zu], expected %d\n", __func__, + data[i], i, val); + return false; + } + } + return true; +error: + return false; +} + +static bool +vfd_read_each_equals(H5F_t *f, H5FD_mem_t ty, haddr_t addr, size_t nelts, + int *data, int val) +{ + size_t i; + + /* Read all elements using the VFD. */ + if (H5FD_read(f->shared->lf, ty, addr, sizeof(int) * nelts, data) < 0) + FAIL_STACK_ERROR; + + for (i = 0; i < nelts; i++) { + if (data[i] != val) { +#if 0 + printf("%s: read %d at data[%d], expected %d\n", __func__, + data[i], i, val); +#endif + return false; + } + } + return true; +error: + return false; +} #ifndef H5_HAVE_PARALLEL @@ -204,7 +333,7 @@ error: HDfree(data); } H5E_END_TRY; return(1); -} /* create_file */ +} /*------------------------------------------------------------------------- @@ -248,11 +377,11 @@ open_file(char *filename, hid_t fapl, hsize_t page_size, if(NULL == (f = (H5F_t *)H5VL_object(file_id))) FAIL_STACK_ERROR; - if(f->shared->page_buf == NULL) + if(f->shared->pb_ptr == NULL) FAIL_STACK_ERROR; - if(f->shared->page_buf->page_size != page_size) + if(f->shared->pb_ptr->page_size != page_size) FAIL_STACK_ERROR; - if(f->shared->page_buf->max_size != page_buffer_size) + if(f->shared->pb_ptr->max_size != page_buffer_size) FAIL_STACK_ERROR; if((grp_id = H5Gopen2(file_id, "GROUP", H5P_DEFAULT)) < 0) @@ -342,7 +471,7 @@ set_multi_split(const char *env_h5_drvr, hid_t fapl, hsize_t pagesize) /* Set memb_addr aligned */ for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt++) memb_addr[mt] = ((memb_addr[mt] + pagesize - 1) / pagesize) * pagesize; - } /* end else */ + } /* Set multi driver with new FAPLs */ if(H5Pset_fapl_multi(fapl, memb_map, memb_fapl_arr, (const char * const *)memb_name, memb_addr, relax) < 0) @@ -352,14 +481,14 @@ set_multi_split(const char *env_h5_drvr, hid_t fapl, hsize_t pagesize) for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt++) free(memb_name[mt]); - } /* end if */ + } return 0; error: return 1; -} /* set_multi_split() */ +} #ifndef H5_HAVE_PARALLEL @@ -382,6 +511,10 @@ error: * Programmer: unknown * ?? / ?? / ?? * + * Changes: Minor updates to adapt to new implementation of page + * buffer. + * JRM -- 10//26/18 + * *------------------------------------------------------------------------- */ @@ -396,7 +529,7 @@ test_args(hid_t orig_fapl, const char *env_h5_drvr) TESTING("Settings for Page Buffering"); - h5_fixname(FILENAME[0], orig_fapl, filename, sizeof(filename)); + h5_fixname(namebase, orig_fapl, filename, sizeof(filename)); if((fapl = H5Pcopy(orig_fapl)) < 0) TEST_ERROR @@ -421,7 +554,7 @@ test_args(hid_t orig_fapl, const char *env_h5_drvr) /* Test setting a page buffer with a size smaller than a single * page size - should fail */ - if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, 1) < 0) TEST_ERROR; if(H5Pset_file_space_page_size(fcpl, 512) < 0) @@ -452,7 +585,7 @@ test_args(hid_t orig_fapl, const char *env_h5_drvr) TEST_ERROR; /* Test setting a page buffer with a size equal to a single page size */ - if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, 1) < 0) TEST_ERROR; if(H5Pset_file_space_page_size(fcpl, 512) < 0) @@ -471,7 +604,7 @@ test_args(hid_t orig_fapl, const char *env_h5_drvr) /* Test setting a page buffer with a size slightly larger than a * single page size */ - if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, 1) < 0) TEST_ERROR; if(H5Pset_file_space_page_size(fcpl, 512) < 0) @@ -491,7 +624,7 @@ test_args(hid_t orig_fapl, const char *env_h5_drvr) /* Test setting a large page buffer size and page size */ - if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, 1) < 0) TEST_ERROR; if(H5Pset_file_space_page_size(fcpl, 4194304) < 0) @@ -511,7 +644,7 @@ test_args(hid_t orig_fapl, const char *env_h5_drvr) /* Test setting a 512 byte page buffer size and page size */ - if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, 1) < 0) TEST_ERROR; if(H5Pset_file_space_page_size(fcpl, 512) < 0) @@ -541,20 +674,495 @@ error: } /* test_args */ -/*------------------------------------------------------------------------- +/* + * Function: test_mpmde_delay_basic() + * + * Purpose: Check that a multi-page metadata entry + * (MPMDE) is not written immediately to the HDF5 file in + * VFD SWMR mode, but it is buffered in the shadow file + * until max_lag + 1 ticks have elapsed. Furthermore, + * check that it appears *immediately* after max_lag + 1 + * ticks, since the LRU list does not hold onto MPMDEs. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: David Young + * 16 Sep 2019 + */ +static unsigned +test_mpmde_delay_basic(hid_t orig_fapl, const char *env_h5_drvr) +{ + char filename[FILENAME_LEN]; /* Filename to use */ + hid_t file_id = -1; /* File ID */ + hid_t fcpl = -1; + hid_t fapl = -1; + size_t i, num_elements = 2000; + int *data = NULL, *odata = NULL; + H5F_t *f; + const uint32_t max_lag = 5; + hsize_t pgsz = sizeof(int) * 200; + haddr_t addr; + + TESTING("Multipage Metadata Delay Handling"); + + h5_fixname(namebase, orig_fapl, filename, sizeof(filename)); + + if ((fapl = H5Pcopy(orig_fapl)) < 0) + TEST_ERROR + + if (set_multi_split(env_h5_drvr, fapl, pgsz) != 0) + TEST_ERROR; + + if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + TEST_ERROR; + if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, 1) < 0) + TEST_ERROR; + if (H5Pset_file_space_page_size(fcpl, pgsz) < 0) + TEST_ERROR; + if (H5Pset_page_buffer_size(fapl, 10 * pgsz, 0, 0) < 0) + TEST_ERROR; + + if (swmr_fapl_augment(fapl, filename, max_lag) < 0) + TEST_ERROR; + + if ((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR; + + /* Get a pointer to the internal file object */ + if (NULL == (f = (H5F_t *)H5VL_object(file_id))) + FAIL_STACK_ERROR; + + addr = H5MF_alloc(f, H5FD_MEM_BTREE, sizeof(int) * num_elements); + /* allocate space for 2000 elements */ + if (HADDR_UNDEF == addr) + FAIL_STACK_ERROR; + + if ((odata = (int *)HDcalloc(num_elements, sizeof(int))) == NULL) + TEST_ERROR; + + if ((data = (int *)HDcalloc(num_elements, sizeof(int))) == NULL) + TEST_ERROR; + + /* initialize all the elements to have a value of -1 */ + for(i = 0; i < num_elements; i++) + odata[i] = -1; + + if (H5F_block_write(f, H5FD_MEM_BTREE, addr, sizeof(int) * num_elements, + odata) < 0) + FAIL_STACK_ERROR; + + /* H5Fvfd_swmr_end_tick() processes delayed writes before it increases + * the tick number, so it takes `max_lag + 1` times through this loop + * for a multi-page metadata write to make it to the HDF5 file. + */ + for (i = 0; i < max_lag + 1; i++) { + /* All elements read using the VFD should be 0. */ + if (!vfd_read_each_equals(f, H5FD_MEM_BTREE, addr, num_elements, + data, 0)) + TEST_ERROR; + H5Fvfd_swmr_end_tick(file_id); + } + + /* It is not necessary to flush the page buffer because delayed + * multi-page metadata buffers are flushed *immediately* + * when their delay elapses. + * + * (If we were waiting for a single-page metadata buffer to + * appear at the VFD layer, then it may reside in the LRU queue + * for a while.) + */ +#if 0 + if (H5PB_flush(f) < 0) + FAIL_STACK_ERROR; +#endif + + /* All elements read using the VFD should be -1. */ + if (!vfd_read_each_equals(f, H5FD_MEM_BTREE, addr, num_elements, data, -1)) + TEST_ERROR; + + if (H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(fapl) < 0) + FAIL_STACK_ERROR; + HDfree(data); + HDfree(odata); + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + if (fapl != H5I_INVALID_HID) + H5Pclose(fapl); + if (fcpl != H5I_INVALID_HID) + H5Pclose(fcpl); + if (file_id != H5I_INVALID_HID) + H5Fclose(file_id); + if (data != NULL) + HDfree(data); + if (odata != NULL) + HDfree(odata); + } H5E_END_TRY; + return 1; +} + + +/* + * Function: test_spmde_lru_evict_basic() + * + * Purpose: Check that once a single-page metadata entry + * (SPMDE) is eligible to be written to the HDF5 file + * (because it has resided unchanged in the metadata file + * for max_lag + 1 ticks), filling the page buffer to + * capacity causes the entry to be flushed. + * + * Further check that the page was evicted by writing + * changes to the VFD layer ("under" the page buffer) + * and trying to read the changes back through the page + * buffer: stale page-buffer content should not shadow + * the changes. + * + * XXX + * XXX reduce duplication with test_spmde_delay_basic! + * XXX + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: David Young + * 16 Sep 2019 + */ +static unsigned +test_spmde_lru_evict_basic(hid_t orig_fapl, const char *env_h5_drvr) +{ + char filename[FILENAME_LEN]; /* Filename to use */ + hid_t file_id = -1; /* File ID */ + hid_t fcpl = -1; + hid_t fapl = -1; + size_t i, num_elements = 20; + int *data = NULL, *odata = NULL; + H5F_t *f; + const uint32_t max_lag = 5; + const hsize_t pgsz = sizeof(int) * 200; + const hsize_t pgbufsz = 10 * pgsz; + hsize_t ofs; + bool flushed; + struct timespec last; + haddr_t addr; + haddr_t pressure; + + TESTING("Single Page Metadata Flush & Eviction Handling"); + + h5_fixname(namebase, orig_fapl, filename, sizeof(filename)); + + last = print_elapsed_time(NULL, __func__, __LINE__); + if ((fapl = H5Pcopy(orig_fapl)) < 0) + TEST_ERROR + + if (set_multi_split(env_h5_drvr, fapl, pgsz) != 0) + TEST_ERROR; + + if ((fcpl = paging_fcpl_create(pgsz)) < 0) + FAIL_STACK_ERROR; + + if (H5Pset_page_buffer_size(fapl, pgbufsz, 0, 0) < 0) + FAIL_STACK_ERROR; + + if (swmr_fapl_augment(fapl, filename, max_lag) < 0) + FAIL_STACK_ERROR; + + if ((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR; + + /* Get a pointer to the internal file object */ + if (NULL == (f = (H5F_t *)H5VL_object(file_id))) + FAIL_STACK_ERROR; + + last = print_elapsed_time(&last, __func__, __LINE__); + + /* Allocate a region whose pages we can write to force eviction of + * least-recently used pages. + */ + pressure = H5MF_alloc(f, H5FD_MEM_BTREE, pgbufsz); + if (HADDR_UNDEF == pressure) + FAIL_STACK_ERROR; + + /* Allocate a whole page for our 20 elements so that they do + * not share a "hot" page with something else. + */ + addr = H5MF_alloc(f, H5FD_MEM_BTREE, pgsz); + if (HADDR_UNDEF == addr) + FAIL_STACK_ERROR; + + if ((odata = (int *)HDcalloc(num_elements, sizeof(int))) == NULL) + TEST_ERROR; + + if ((data = (int *)HDcalloc(num_elements, sizeof(int))) == NULL) + TEST_ERROR; + + /* initialize all the elements to have a value of -1 */ + for(i = 0; i < num_elements; i++) + odata[i] = -1; + + if (H5F_block_write(f, H5FD_MEM_BTREE, addr, sizeof(int) * num_elements, + odata) < 0) + FAIL_STACK_ERROR; + + last = print_elapsed_time(&last, __func__, __LINE__); + + /* H5Fvfd_swmr_end_tick() processes delayed writes before it increases + * the tick number, so only after `max_lag + 1` times through this loop + * is a metadata write eligible to be written to the HDF5 file. + */ + for (i = 0; i < max_lag + 1; i++) { + /* All elements read using the VFD should be 0. */ + if (!vfd_read_each_equals(f, H5FD_MEM_BTREE, addr, num_elements, + data, 0)) + TEST_ERROR; + H5Fvfd_swmr_end_tick(file_id); + } + + last = print_elapsed_time(&last, __func__, __LINE__); + + flushed = false; + /* We are waiting for a single-page metadata buffer to + * appear at the VFD layer, but it may reside in the LRU queue. + * Dirty new blocks to apply pressure on the page buffer so that + * it empties the LRU queue. Writing to N distinct pages, + * for N the number of pages in the page buffer, ought to do + * the trick. + */ + for (ofs = 0; ofs < pgbufsz - pgsz * 2; ofs += pgsz) { + int tmp = -1; + if (H5F_block_write(f, H5FD_MEM_BTREE, pressure + ofs, + sizeof(tmp), &tmp) < 0) + FAIL_STACK_ERROR; + if (!flushed && + vfd_read_each_equals(f, H5FD_MEM_BTREE, addr, num_elements, + data, -1)) { + flushed = true; +#if 0 + printf("Writing page %" PRIuHSIZE " flushed target page.\n", + ofs / pgsz); +#endif + } + } + + last = print_elapsed_time(&last, __func__, __LINE__); + + if (!vfd_read_each_equals(f, H5FD_MEM_BTREE, addr, num_elements, data, -1)) + TEST_ERROR; + + /* initialize all the elements to have a value of -2 */ + for(i = 0; i < num_elements; i++) + odata[i] = -2; + /* Write -2 to our target page using the VFD. */ + if (H5FD_write(f->shared->lf, H5FD_MEM_BTREE, addr, + sizeof(int) * num_elements, odata) < 0) + FAIL_STACK_ERROR; + + last = print_elapsed_time(&last, __func__, __LINE__); + + /* All elements read through the page buffer should be -2. That is, + * no page-buffer entry should shadow the page. + */ + if (!pgbuf_read_each_equals(f, H5FD_MEM_BTREE, addr, num_elements, data, + -2)) + TEST_ERROR; + + last = print_elapsed_time(&last, __func__, __LINE__); + + /* Force ticks to occur so that H5Fclose() doesn't pause waiting + * for them to elapse. + */ + for (i = 0; i < max_lag + 1; i++) + H5Fvfd_swmr_end_tick(file_id); + + if (H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(fapl) < 0) + FAIL_STACK_ERROR; + HDfree(data); + HDfree(odata); + + last = print_elapsed_time(&last, __func__, __LINE__); + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + if (fapl != H5I_INVALID_HID) + H5Pclose(fapl); + if (fcpl != H5I_INVALID_HID) + H5Pclose(fcpl); + if (file_id != H5I_INVALID_HID) + H5Fclose(file_id); + if (data != NULL) + HDfree(data); + if (odata != NULL) + HDfree(odata); + } H5E_END_TRY; + return 1; +} + + +/* + * Function: test_spmde_delay_basic() + * + * Purpose: Check that a single-page metadata entry + * (SPMDE) is not written immediately to the HDF5 file in + * VFD SWMR mode, but it is buffered in the shadow file + * until max_lag + 1 ticks have elapsed. + * + * The LRU list will hold onto SPMDEs, so it's necessary to + * flush the page buffer to make sure the buffer is flushed + * to the VFD layer. + * + * XXX + * XXX reduce duplication with test_mpmde_delay_basic! + * XXX + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: David Young + * 16 Sep 2019 + */ +static unsigned +test_spmde_delay_basic(hid_t orig_fapl, const char *env_h5_drvr) +{ + char filename[FILENAME_LEN]; /* Filename to use */ + hid_t file_id = -1; /* File ID */ + hid_t fcpl = -1; + hid_t fapl = -1; + size_t i, num_elements = 20; + int *data = NULL, *odata = NULL; + H5F_t *f; + const uint32_t max_lag = 5; + hsize_t pgsz = sizeof(int) * 200; + haddr_t addr; + + TESTING("Single Page Metadata Delay Handling"); + + h5_fixname(namebase, orig_fapl, filename, sizeof(filename)); + + if ((fapl = H5Pcopy(orig_fapl)) < 0) + TEST_ERROR + + if (set_multi_split(env_h5_drvr, fapl, pgsz) != 0) + TEST_ERROR; + + if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + TEST_ERROR; + if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, 1) < 0) + TEST_ERROR; + if (H5Pset_file_space_page_size(fcpl, pgsz) < 0) + TEST_ERROR; + if (H5Pset_page_buffer_size(fapl, 10 * pgsz, 0, 0) < 0) + TEST_ERROR; + + if (swmr_fapl_augment(fapl, filename, max_lag) < 0) + TEST_ERROR; + + if ((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR; + + /* Get a pointer to the internal file object */ + if (NULL == (f = (H5F_t *)H5VL_object(file_id))) + FAIL_STACK_ERROR; + + addr = H5MF_alloc(f, H5FD_MEM_BTREE, + sizeof(int) * num_elements); + /* allocate space for 2000 elements */ + if (HADDR_UNDEF == addr) + FAIL_STACK_ERROR; + + if ((odata = (int *)HDcalloc(num_elements, sizeof(int))) == NULL) + TEST_ERROR; + + if ((data = (int *)HDcalloc(num_elements, sizeof(int))) == NULL) + TEST_ERROR; + + /* initialize all the elements to have a value of -1 */ + for(i = 0; i < num_elements; i++) + odata[i] = -1; + + if (H5F_block_write(f, H5FD_MEM_BTREE, addr, sizeof(int) * num_elements, + odata) < 0) + FAIL_STACK_ERROR; + + /* H5Fvfd_swmr_end_tick() processes delayed writes before it increases + * the tick number, so only after `max_lag + 1` times through this loop + * is a metadata write eligible to be written to the HDF5 file. + */ + for (i = 0; i < max_lag + 1; i++) { + /* All elements read using the VFD should be 0. */ + if (!vfd_read_each_equals(f, H5FD_MEM_BTREE, addr, num_elements, + data, 0)) + TEST_ERROR; + H5Fvfd_swmr_end_tick(file_id); + } + + /* We are waiting for a single-page metadata buffer to + * appear at the VFD layer, but it may reside in the LRU queue + * for a while if we do not flush the page buffer. + */ + if (H5PB_flush(f->shared) < 0) + FAIL_STACK_ERROR; + + /* All elements read using the VFD should be -1. */ + if (!vfd_read_each_equals(f, H5FD_MEM_BTREE, addr, num_elements, data, -1)) + TEST_ERROR; + + if (H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(fapl) < 0) + FAIL_STACK_ERROR; + HDfree(data); + HDfree(odata); + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + if (fapl != H5I_INVALID_HID) + H5Pclose(fapl); + if (fcpl != H5I_INVALID_HID) + H5Pclose(fcpl); + if (file_id != H5I_INVALID_HID) + H5Fclose(file_id); + if (data != NULL) + HDfree(data); + if (odata != NULL) + HDfree(odata); + } H5E_END_TRY; + return 1; +} + + +/* * Function: test_raw_data_handling() * - * Purpose: The purpose of this function appears to be a smoke check - * of raw data reads and writes via the page buffer. + * Purpose: Check that raw data is written to the HDF5 + * file when expected whether in VFD SWMR mode or not. * - * Any data mis-matches or failures reported by the HDF5 - * library result in test failure. + * Any data mis-matches or failures reported by the HDF5 + * library result in test failure. * * Return: 0 if test is sucessful * 1 if test fails * - * Programmer: unknown - * ?? / ?? / ?? + * Programmer: David Young + * 16 Sep 2019 * * Changes: Added base_page_cnt field as supporting code. This allows * the test to adjust to the number of page buffer pages @@ -567,47 +1175,51 @@ error: * * JRM -- 2/23/17 * - *------------------------------------------------------------------------- + * Minor changes to adapt to re-implementation of the + * page buffer. + * + * JRM -- 10/26/18 */ /* Changes due to file space page size has a minimum size of 512 */ static unsigned -test_raw_data_handling(hid_t orig_fapl, const char *env_h5_drvr) +test_raw_data_handling(hid_t orig_fapl, const char *env_h5_drvr, + bool vfd_swmr_mode) { char filename[FILENAME_LEN]; /* Filename to use */ hid_t file_id = -1; /* File ID */ hid_t fcpl = -1; hid_t fapl = -1; - size_t base_page_cnt; - size_t page_count = 0; + int64_t base_page_cnt; + int64_t page_count = 0; int i, num_elements = 2000; haddr_t addr = HADDR_UNDEF; int *data = NULL; H5F_t *f = NULL; + const uint32_t max_lag = 5; - TESTING("Raw Data Handling"); + TESTING("%sRaw Data Handling", vfd_swmr_mode ? "VFD SWMR " : ""); - h5_fixname(FILENAME[0], orig_fapl, filename, sizeof(filename)); + h5_fixname(namebase, orig_fapl, filename, sizeof(filename)); - if((fapl = H5Pcopy(orig_fapl)) < 0) + if ((fapl = H5Pcopy(orig_fapl)) < 0) TEST_ERROR - if(set_multi_split(env_h5_drvr, fapl, sizeof(int)*200) != 0) + if (set_multi_split(env_h5_drvr, fapl, sizeof(int) * 200) != 0) TEST_ERROR; - if((data = (int *)HDcalloc((size_t)num_elements, sizeof(int))) == NULL) + if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) TEST_ERROR; - - if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, 1) < 0) TEST_ERROR; - if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + if (H5Pset_file_space_page_size(fcpl, sizeof(int) * 200) < 0) TEST_ERROR; - if(H5Pset_file_space_page_size(fcpl, sizeof(int)*200) < 0) - TEST_ERROR; - if(H5Pset_page_buffer_size(fapl, sizeof(int)*2000, 0, 0) < 0) + if (H5Pset_page_buffer_size(fapl, sizeof(int) * 2000, 0, 0) < 0) TEST_ERROR; - if((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + if (vfd_swmr_mode && swmr_fapl_augment(fapl, filename, max_lag) < 0) + TEST_ERROR; + if ((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) FAIL_STACK_ERROR; /* Get a pointer to the internal file object */ @@ -616,129 +1228,139 @@ test_raw_data_handling(hid_t orig_fapl, const char *env_h5_drvr) /* opening the file inserts one or more pages into the page buffer. * Get the number of pages inserted, and verify that it is the - * the expected value. + * expected value. */ - base_page_cnt = H5SL_count(f->shared->page_buf->slist_ptr); - if(base_page_cnt != 1) + base_page_cnt = f->shared->pb_ptr->curr_pages; + if (base_page_cnt != 2) TEST_ERROR; - /* allocate space for a 2000 elements */ - if(HADDR_UNDEF == (addr = H5MF_alloc(f, H5FD_MEM_DRAW, sizeof(int)*(size_t)num_elements))) + /* allocate space for 2000 elements */ + if (HADDR_UNDEF == (addr = H5MF_alloc(f, H5FD_MEM_DRAW, sizeof(int) * (size_t)num_elements))) FAIL_STACK_ERROR; + if ((data = (int *)HDcalloc((size_t)num_elements, sizeof(int))) == NULL) + TEST_ERROR; + /* initialize all the elements to have a value of -1 */ for(i=0 ; i<num_elements ; i++) data[i] = -1; - if(H5F_block_write(f, H5FD_MEM_DRAW, addr, sizeof(int)*(size_t)num_elements, data) < 0) + if (H5F_block_write(f, H5FD_MEM_DRAW, addr, sizeof(int) * (size_t)num_elements, data) < 0) FAIL_STACK_ERROR; - /* update the first 50 elements to have values 0-49 - this will be + /* update the first 100 elements to have values 0-99 - this will be a page buffer update with 1 page resulting in the page buffer. */ - /* Changes: 100 */ for(i=0 ; i<100 ; i++) data[i] = i; - if(H5F_block_write(f, H5FD_MEM_DRAW, addr, sizeof(int)*100, data) < 0) + if (H5F_block_write(f, H5FD_MEM_DRAW, addr, sizeof(int) * 100, data) < 0) FAIL_STACK_ERROR; page_count ++; - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + if (f->shared->pb_ptr->curr_pages != page_count + base_page_cnt) FAIL_STACK_ERROR; /* update elements 300 - 450, with values 300 - - this will bring two more pages into the page buffer. */ for(i=0 ; i<150 ; i++) data[i] = i+300; - if(H5F_block_write(f, H5FD_MEM_DRAW, addr+(sizeof(int)*300), sizeof(int)*150, data) < 0) + if (H5F_block_write(f, H5FD_MEM_DRAW, addr + (sizeof(int) * 300), sizeof(int) * 150, data) < 0) FAIL_STACK_ERROR; page_count += 2; - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + if (f->shared->pb_ptr->curr_pages != page_count + base_page_cnt) FAIL_STACK_ERROR; /* update elements 100 - 300, this will go to disk but also update existing pages in the page buffer. */ for(i=0 ; i<200 ; i++) data[i] = i+100; - if(H5F_block_write(f, H5FD_MEM_DRAW, addr+(sizeof(int)*100), sizeof(int)*200, data) < 0) + if (H5F_block_write(f, H5FD_MEM_DRAW, addr + (sizeof(int) * 100), sizeof(int) * 200, data) < 0) FAIL_STACK_ERROR; - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + if (f->shared->pb_ptr->curr_pages != page_count + base_page_cnt) FAIL_STACK_ERROR; /* Update elements 225-300 - this will update an existing page in the PB */ /* Changes: 450 - 600; 150 */ for(i=0 ; i<150 ; i++) data[i] = i+450; - if(H5F_block_write(f, H5FD_MEM_DRAW, addr+(sizeof(int)*450), sizeof(int)*150, data) < 0) + if (H5F_block_write(f, H5FD_MEM_DRAW, addr + (sizeof(int) * 450), sizeof(int) * 150, data) < 0) FAIL_STACK_ERROR; - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + if (f->shared->pb_ptr->curr_pages != page_count + base_page_cnt) FAIL_STACK_ERROR; /* Do a full page write to block 600-800 - should bypass the PB */ for(i=0 ; i<200 ; i++) data[i] = i+600; - if(H5F_block_write(f, H5FD_MEM_DRAW, addr+(sizeof(int)*600), sizeof(int)*200, data) < 0) + if (H5F_block_write(f, H5FD_MEM_DRAW, addr + (sizeof(int) * 600), sizeof(int) * 200, data) < 0) FAIL_STACK_ERROR; - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + if (f->shared->pb_ptr->curr_pages != page_count + base_page_cnt) FAIL_STACK_ERROR; /* read elements 800 - 1200, this should not affect the PB, and should read -1s */ - if(H5F_block_read(f, H5FD_MEM_DRAW, addr+(sizeof(int)*800), sizeof(int)*400, data) < 0) + if (H5F_block_read(f, H5FD_MEM_DRAW, addr + (sizeof(int) * 800), sizeof(int) * 400, data) < 0) FAIL_STACK_ERROR; for (i=0; i < 400; i++) { - if(data[i] != -1) { + if (data[i] != -1) { HDfprintf(stderr, "Read different values than written\n"); + HDfprintf(stderr, "data[%d] = %d, %d expected.\n", i, data[i], -1); FAIL_STACK_ERROR; } } - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + if (f->shared->pb_ptr->curr_pages != page_count + base_page_cnt) FAIL_STACK_ERROR; /* read elements 1200 - 1201, this should read -1 and bring in an * entire page of addr 1200 */ - if(H5F_block_read(f, H5FD_MEM_DRAW, addr+(sizeof(int)*1200), sizeof(int)*1, data) < 0) + if (H5F_block_read(f, H5FD_MEM_DRAW, addr + (sizeof(int) * 1200), sizeof(int) * 1, data) < 0) FAIL_STACK_ERROR; for (i=0; i < 1; i++) { - if(data[i] != -1) { + if (data[i] != -1) { HDfprintf(stderr, "Read different values than written\n"); + HDfprintf(stderr, "data[%d] = %d, %d expected.\n", i, data[i], -1); TEST_ERROR; } } page_count ++; - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + if (f->shared->pb_ptr->curr_pages != page_count + base_page_cnt) TEST_ERROR; /* read elements 175 - 225, this should use the PB existing pages */ /* Changes: 350 - 450 */ /* read elements 175 - 225, this should use the PB existing pages */ - if(H5F_block_read(f, H5FD_MEM_DRAW, addr+(sizeof(int)*350), sizeof(int)*100, data) < 0) + if (H5F_block_read(f, H5FD_MEM_DRAW, addr + (sizeof(int) * 350), sizeof(int) * 100, data) < 0) FAIL_STACK_ERROR; for (i=0; i < 100; i++) { - if(data[i] != i+350) { + if (data[i] != i + 350) { HDfprintf(stderr, "Read different values than written\n"); + HDfprintf(stderr, "data[%d] = %d, %d expected.\n", i, data[i], + i + 350); TEST_ERROR; } } - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + if (f->shared->pb_ptr->curr_pages != page_count + base_page_cnt) TEST_ERROR; /* read elements 0 - 800 using the VFD.. this should result in -1s except for the writes that went through the PB (100-300 & 600-800) */ - if(H5FD_read(f->shared->lf, H5FD_MEM_DRAW, addr, sizeof(int)*800, data) < 0) + if (H5FD_read(f->shared->lf, H5FD_MEM_DRAW, addr, sizeof(int) * 800, data) < 0) FAIL_STACK_ERROR; i = 0; while (i < 800) { - if((i>=100 && i<300) || (i>=600)) { - if(data[i] != i) { + if((i>=100 && i<300) || i >= 600) { + if (data[i] != i) { HDfprintf(stderr, "Read different values than written\n"); + HDfprintf(stderr, "data[%d] = %d, %d expected.\n", + i, data[i], i); TEST_ERROR; } } else { - if(data[i] != -1) { + if (data[i] != -1) { HDfprintf(stderr, "Read different values than written\n"); + HDfprintf(stderr, "data[%d] = %d, %d expected.\n", + i, data[i], -1); TEST_ERROR; } } @@ -748,13 +1370,15 @@ test_raw_data_handling(hid_t orig_fapl, const char *env_h5_drvr) /* read elements 0 - 800 using the PB.. this should result in all * what we have written so far and should get the updates from the PB */ - if(H5F_block_read(f, H5FD_MEM_DRAW, addr, sizeof(int)*800, data) < 0) + if (H5F_block_read(f, H5FD_MEM_DRAW, addr, sizeof(int) * 800, data) < 0) FAIL_STACK_ERROR; - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + if (f->shared->pb_ptr->curr_pages != page_count + base_page_cnt) TEST_ERROR; for (i=0; i < 800; i++) { - if(data[i] != i) { + if (data[i] != i) { HDfprintf(stderr, "Read different values than written\n"); + HDfprintf(stderr, "data[%d] = %d, %d expected.\n", + i, data[i], i); TEST_ERROR; } } @@ -765,41 +1389,45 @@ test_raw_data_handling(hid_t orig_fapl, const char *env_h5_drvr) */ for(i=0 ; i<1000 ; i++) data[i] = 0; - if(H5F_block_write(f, H5FD_MEM_DRAW, addr+(sizeof(int)*400), sizeof(int)*1000, data) < 0) + if (H5F_block_write(f, H5FD_MEM_DRAW, addr + (sizeof(int) * 400), sizeof(int) * 1000, data) < 0) FAIL_STACK_ERROR; page_count -= 2; - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + if (f->shared->pb_ptr->curr_pages != page_count + base_page_cnt) TEST_ERROR; /* read elements 0 - 1000.. this should go to disk then update the * buffer result 200-400 with existing pages */ - if(H5F_block_read(f, H5FD_MEM_DRAW, addr, sizeof(int)*1000, data) < 0) + if (H5F_block_read(f, H5FD_MEM_DRAW, addr, sizeof(int) * 1000, data) < 0) FAIL_STACK_ERROR; i=0; while (i < 1000) { if(i<400) { - if(data[i] != i) { + if (data[i] != i) { HDfprintf(stderr, "Read different values than written\n"); + HDfprintf(stderr, "data[%d] = %d, %d expected.\n", + i, data[i], i); TEST_ERROR; } } else { - if(data[i] != 0) { + if (data[i] != 0) { HDfprintf(stderr, "Read different values than written\n"); + HDfprintf(stderr, "data[%d] = %d, %d expected.\n", + i, data[i], 0); TEST_ERROR; } } i++; } - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + if (f->shared->pb_ptr->curr_pages != page_count + base_page_cnt) TEST_ERROR; - if(H5Fclose(file_id) < 0) + if (H5Fclose(file_id) < 0) FAIL_STACK_ERROR; - if(H5Pclose(fcpl) < 0) + if (H5Pclose(fcpl) < 0) FAIL_STACK_ERROR; - if(H5Pclose(fapl) < 0) + if (H5Pclose(fapl) < 0) FAIL_STACK_ERROR; HDfree(data); @@ -808,10 +1436,13 @@ test_raw_data_handling(hid_t orig_fapl, const char *env_h5_drvr) error: H5E_BEGIN_TRY { - H5Pclose(fapl); - H5Pclose(fcpl); - H5Fclose(file_id); - if(data) + if (fapl != H5I_INVALID_HID) + H5Pclose(fapl); + if (fcpl != H5I_INVALID_HID) + H5Pclose(fcpl); + if (file_id != H5I_INVALID_HID) + H5Fclose(file_id); + if (data != NULL) HDfree(data); } H5E_END_TRY; return 1; @@ -832,17 +1463,12 @@ error: * * Programmer: unknown * ?? / ?? / ?? + * + * Changes: Reworked for new implementation of page buffer. Major + * change was adaption to the new implementation's greater + * respect for max_pages. * - * Changes: Added base_page_cnt field as supporting code. This allows - * the test to adjust to the number of page buffer pages - * accessed during file open / create. - * - * The test for the value of base_page_cnt just after file - * open exists detect changes in library behavior. Assuming - * any such change is not indicative of other issues, these - * tests can be modified to reflect the change. - * - * JRM -- 2/23/17 + * JRM -- 10/26/18 * * *------------------------------------------------------------------------- @@ -852,11 +1478,10 @@ static unsigned test_lru_processing(hid_t orig_fapl, const char *env_h5_drvr) { char filename[FILENAME_LEN]; /* Filename to use */ + hbool_t page_exists; hid_t file_id = -1; /* File ID */ hid_t fcpl = -1; hid_t fapl = -1; - size_t base_page_cnt; - size_t page_count = 0; int i; int num_elements = 2000; haddr_t addr = HADDR_UNDEF; @@ -866,7 +1491,7 @@ test_lru_processing(hid_t orig_fapl, const char *env_h5_drvr) TESTING("LRU Processing"); - h5_fixname(FILENAME[0], orig_fapl, filename, sizeof(filename)); + h5_fixname(namebase, orig_fapl, filename, sizeof(filename)); if((fapl = H5Pcopy(orig_fapl)) < 0) FAIL_STACK_ERROR @@ -880,7 +1505,7 @@ test_lru_processing(hid_t orig_fapl, const char *env_h5_drvr) if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) FAIL_STACK_ERROR; - if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, 1) < 0) FAIL_STACK_ERROR; if(H5Pset_file_space_page_size(fcpl, sizeof(int)*200) < 0) @@ -897,28 +1522,25 @@ test_lru_processing(hid_t orig_fapl, const char *env_h5_drvr) if(NULL == (f = (H5F_t *)H5VL_object(file_id))) FAIL_STACK_ERROR; - /* opening the file inserts one or more pages into the page buffer. - * Get the number of pages inserted, and verify that it is the - * the expected value. - */ - base_page_cnt = H5SL_count(f->shared->page_buf->slist_ptr); - if(base_page_cnt != 1) - TEST_ERROR; - - /* allocate space for a 2000 elements */ - if(HADDR_UNDEF == (addr = H5MF_alloc(f, H5FD_MEM_DRAW, sizeof(int)*(size_t)num_elements))) + /* allocate space for 2000 elements */ + if(HADDR_UNDEF == (addr = H5MF_alloc(f, H5FD_MEM_DRAW, + sizeof(int)*(size_t)num_elements))) FAIL_STACK_ERROR; /* initialize all the elements to have a value of -1 */ for(i=0 ; i<num_elements ; i++) data[i] = -1; - if(H5F_block_write(f, H5FD_MEM_DRAW, addr, sizeof(int)*(size_t)num_elements, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, addr, + sizeof(int)*(size_t)num_elements, data) < 0) + FAIL_STACK_ERROR; + + /* there should be no raw data pages in the page buffer -- verify this */ + if (f->shared->pb_ptr->curr_rd_pages != 0) FAIL_STACK_ERROR; /* update the first 100 elements to have values 0-99 - this will be - * a page buffer update with 1 page resulting in the page - * buffer. + * a page buffer update that loads page addr + 0 into the page buffer. */ for(i=0 ; i<100 ; i++) data[i] = i; @@ -926,126 +1548,142 @@ test_lru_processing(hid_t orig_fapl, const char *env_h5_drvr) if(H5F_block_write(f, H5FD_MEM_DRAW, addr, sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - page_count ++; + /* verify addr + 0 is the only raw data page in the page buffer */ + search_addr = addr; + if((H5PB_page_exists(f->shared, search_addr, &page_exists) < 0) || (!page_exists)) + FAIL_STACK_ERROR; + if (f->shared->pb_ptr->curr_rd_pages != 1) + FAIL_STACK_ERROR; - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) - TEST_ERROR; /* update elements 300 - 450, with values 300 - 449 - this will - * bring two pages into the page buffer and evict 0. + * bring two pages (addr + 200 & addr + 400) into the page buffer and + * evict addr + 0. */ for(i=0 ; i<150 ; i++) data[i] = i+300; - if(H5F_block_write(f, H5FD_MEM_DRAW, addr+(sizeof(int)*300), sizeof(int)*150, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, addr+(sizeof(int)*300), + sizeof(int)*150, data) < 0) FAIL_STACK_ERROR; - page_count = 2; + /* verify that addr + 200 and addr + 400 are the only raw data pages in + * the page buffer. + */ + search_addr = addr + sizeof(int)*200; + if((H5PB_page_exists(f->shared, search_addr, &page_exists) < 0) || (!page_exists)) + FAIL_STACK_ERROR; + search_addr = addr + sizeof(int)*400; + if((H5PB_page_exists(f->shared, search_addr, &page_exists) < 0) || (!page_exists)) + FAIL_STACK_ERROR; + if (f->shared->pb_ptr->curr_rd_pages != 2) + FAIL_STACK_ERROR; - /* at this point, the page buffer entry created at file open should - * have been evicted -- thus no further need to consider base_page_cnt. + /* at this point, the page buffer entries created at file open should + * have been evicted. */ - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + if(f->shared->pb_ptr->curr_md_pages != 0) FAIL_STACK_ERROR; - /* The two pages should be the ones with address 100 and 200; 0 - should have been evicted */ - /* Changes: 200, 400 */ - search_addr = addr; - if(NULL != H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + + /* update elements 300-301, this will update page addr + 200 in + * page buffer and move it to the top of the LRU. + */ + for(i=0 ; i<1 ; i++) + data[i] = i+300; + if(H5F_block_write(f, H5FD_MEM_DRAW, addr+(sizeof(int)*300), + sizeof(int)*2, data) < 0) FAIL_STACK_ERROR; + + /* verify that addr + 200 and addr + 400 are the only raw data pages in + * the page buffer. + */ search_addr = addr + sizeof(int)*200; - if(NULL == H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + if((H5PB_page_exists(f->shared, search_addr, &page_exists) < 0) || (!page_exists)) FAIL_STACK_ERROR; search_addr = addr + sizeof(int)*400; - if(NULL == H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + if((H5PB_page_exists(f->shared, search_addr, &page_exists) < 0) || (!page_exists)) FAIL_STACK_ERROR; - - /* update elements 150-151, this will update existing pages in the - page buffer and move it to the top of the LRU. */ - /* Changes: 300 - 301 */ - for(i=0 ; i<1 ; i++) - data[i] = i+300; - if(H5F_block_write(f, H5FD_MEM_DRAW, addr+(sizeof(int)*300), sizeof(int)*1, data) < 0) + if (f->shared->pb_ptr->curr_rd_pages != 2) FAIL_STACK_ERROR; - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + if(f->shared->pb_ptr->curr_pages != 2) FAIL_STACK_ERROR; - /* read elements 600 - 601, this should read -1 and bring in an - entire page of addr 600, and evict page 200 */ - /* Changes: 1200 - 1201; 1200, 400 */ - if(H5F_block_read(f, H5FD_MEM_DRAW, addr+(sizeof(int)*1200), sizeof(int)*1, data) < 0) + /* read elements 1200 - 1201, this should read -1, bring in page + * addr + 1200, and evict page addr + 400 + */ + if(H5F_block_read(f, H5FD_MEM_DRAW, addr+(sizeof(int)*1200), + sizeof(int)*1, data) < 0) FAIL_STACK_ERROR; for (i=0; i < 1; i++) { if(data[i] != -1) { HDfprintf(stderr, "Read different values than written\n"); + HDfprintf(stderr, "data[%d] = %d, %d expected.\n", i, data[i], -1); TEST_ERROR; - } /* end if */ - } /* end for */ - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) - FAIL_STACK_ERROR; - - /* Changes: 400 */ - search_addr = addr + sizeof(int)*400; - if(NULL != H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) - FAIL_STACK_ERROR; + } + } - /* Changes: 200 */ + /* verify that addr + 200 and addr + 1200 are the only raw data pages in + * the page buffer. + */ search_addr = addr + sizeof(int)*200; - if(NULL == H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + if((H5PB_page_exists(f->shared, search_addr, &page_exists) < 0) || (!page_exists)) FAIL_STACK_ERROR; - - /* Changes: 1200 */ search_addr = addr + sizeof(int)*1200; - if(NULL == H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + if((H5PB_page_exists(f->shared, search_addr, &page_exists) < 0) || (!page_exists)) + FAIL_STACK_ERROR; + if (f->shared->pb_ptr->curr_rd_pages != 2) FAIL_STACK_ERROR; - /* read elements 175 - 225, this should move 100 to the top, evict 600 and bring in 200 */ - /* Changes: 350 - 450; 200, 1200, 400 */ - if(H5F_block_read(f, H5FD_MEM_DRAW, addr+(sizeof(int)*350), sizeof(int)*100, data) < 0) + if(f->shared->pb_ptr->curr_pages != 2) + FAIL_STACK_ERROR; + + /* read elements 350 - 450, this should load page addr + 400 and move + * it to the top of the LRU, and evict page addr + 1200. + */ + if(H5F_block_read(f, H5FD_MEM_DRAW, addr+(sizeof(int)*350), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; for (i=0; i < 100; i++) { if(data[i] != i+350) { HDfprintf(stderr, "Read different values than written\n"); + HDfprintf(stderr, "data[%d] = %d, %d expected.\n", i, data[i], + i + 350); TEST_ERROR; - } /* end if */ - } /* end for */ - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) - FAIL_STACK_ERROR; - - /* Changes: 1200 */ - search_addr = addr + sizeof(int)*1200; - if(NULL != H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) - FAIL_STACK_ERROR; + } + } - /* Changes: 200 */ + /* verify that addr + 200 and addr + 400 are the only raw data pages in + * the page buffer. + */ search_addr = addr + sizeof(int)*200; - if(NULL == H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + if((H5PB_page_exists(f->shared, search_addr, &page_exists) < 0) || (!page_exists)) FAIL_STACK_ERROR; - - /* Changes: 400 */ search_addr = addr + sizeof(int)*400; - if(NULL == H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + if((H5PB_page_exists(f->shared, search_addr, &page_exists) < 0) || (!page_exists)) + FAIL_STACK_ERROR; + if (f->shared->pb_ptr->curr_rd_pages != 2) + FAIL_STACK_ERROR; + if(f->shared->pb_ptr->curr_pages != 2) FAIL_STACK_ERROR; - /* update elements 200 - 700 to value 0, this will go to disk but - also discarding existing pages from the PB (page 200). */ - /* Changes: 400 - 1400; 400 */ + + /* update elements 400 - 1400 to value 0, this will overwrite and + * evict page addr + 400. + */ for(i=0 ; i<1000 ; i++) data[i] = 0; - if(H5F_block_write(f, H5FD_MEM_DRAW, addr+(sizeof(int)*400), sizeof(int)*1000, data) < 0) - FAIL_STACK_ERROR; - page_count -= 1; - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + if(H5F_block_write(f, H5FD_MEM_DRAW, addr+(sizeof(int)*400), + sizeof(int)*1000, data) < 0) FAIL_STACK_ERROR; - /* Changes: 200 */ + /* verify that addr + 200 is the only raw data page in the page buffer. + */ search_addr = addr + sizeof(int)*200; - if(NULL == H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + if((H5PB_page_exists(f->shared, search_addr, &page_exists) < 0) || (!page_exists)) FAIL_STACK_ERROR; - - /* Changes: 400 */ - search_addr = addr + sizeof(int)*400; - if(NULL != H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + if (f->shared->pb_ptr->curr_rd_pages != 1) + FAIL_STACK_ERROR; + if(f->shared->pb_ptr->curr_pages != 1) FAIL_STACK_ERROR; if(H5Fclose(file_id) < 0) @@ -1061,10 +1699,13 @@ test_lru_processing(hid_t orig_fapl, const char *env_h5_drvr) error: H5E_BEGIN_TRY { - H5Pclose(fapl); - H5Pclose(fcpl); - H5Fclose(file_id); - if(data) + if (fapl != H5I_INVALID_HID) + H5Pclose(fapl); + if (fcpl != H5I_INVALID_HID) + H5Pclose(fcpl); + if (file_id != H5I_INVALID_HID) + H5Fclose(file_id); + if (data != NULL) HDfree(data); } H5E_END_TRY; return 1; @@ -1099,6 +1740,14 @@ error: * * JRM -- 2/23/17 * + * Reworked test for new implementatin of the page buffer. + * The major change was adapting the test for the new + * page buffers refusal to buffer any raw data when + * min_md_pages == max_pages, or any metadata pages wwhen + * min_rd_pages == max_pages. + * + * JRM -- 10/27/18 + * *------------------------------------------------------------------------- */ @@ -1109,12 +1758,11 @@ test_min_threshold(hid_t orig_fapl, const char *env_h5_drvr) hid_t file_id = -1; /* File ID */ hid_t fcpl = -1; hid_t fapl = -1; - size_t base_raw_cnt = 0; - size_t base_meta_cnt = 0; - size_t page_count = 0; + int64_t base_raw_cnt = 0; + int64_t base_meta_cnt = 0; int i; int num_elements = 1000; - H5PB_t *page_buf; + H5PB_t *pb_ptr; haddr_t meta_addr = HADDR_UNDEF; haddr_t raw_addr = HADDR_UNDEF; int *data = NULL; @@ -1122,7 +1770,7 @@ test_min_threshold(hid_t orig_fapl, const char *env_h5_drvr) TESTING("Minimum Metadata threshold Processing"); HDprintf("\n"); - h5_fixname(FILENAME[0], orig_fapl, filename, sizeof(filename)); + h5_fixname(namebase, orig_fapl, filename, sizeof(filename)); if((fapl = H5Pcopy(orig_fapl)) < 0) TEST_ERROR @@ -1136,7 +1784,7 @@ test_min_threshold(hid_t orig_fapl, const char *env_h5_drvr) if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) FAIL_STACK_ERROR; - if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, 1) < 0) FAIL_STACK_ERROR; if(H5Pset_file_space_page_size(fcpl, sizeof(int)*200) < 0) @@ -1161,35 +1809,35 @@ test_min_threshold(hid_t orig_fapl, const char *env_h5_drvr) */ HDassert(f); HDassert(f->shared); - HDassert(f->shared->page_buf); + HDassert(f->shared->pb_ptr); - base_raw_cnt = f->shared->page_buf->raw_count; - base_meta_cnt = f->shared->page_buf->meta_count; + base_raw_cnt = f->shared->pb_ptr->curr_rd_pages; + base_meta_cnt = f->shared->pb_ptr->curr_md_pages; if(base_raw_cnt != 0) TEST_ERROR; - if(base_meta_cnt != 1) + if(base_meta_cnt != 2) TEST_ERROR; - page_buf = f->shared->page_buf; + pb_ptr = f->shared->pb_ptr; - if(page_buf->min_meta_count != 5) + if(pb_ptr->min_md_pages != 5) TEST_ERROR; - if(page_buf->min_raw_count != 0) + if(pb_ptr->min_rd_pages != 0) TEST_ERROR; - if(HADDR_UNDEF == (meta_addr = H5MF_alloc(f, H5FD_MEM_SUPER, sizeof(int)*(size_t)num_elements))) + if(HADDR_UNDEF == (meta_addr = H5MF_alloc(f, H5FD_MEM_SUPER, + sizeof(int)*(size_t)num_elements))) FAIL_STACK_ERROR; - if(HADDR_UNDEF == (raw_addr = H5MF_alloc(f, H5FD_MEM_DRAW, sizeof(int)*(size_t)num_elements))) + if(HADDR_UNDEF == (raw_addr = H5MF_alloc(f, H5FD_MEM_DRAW, + sizeof(int)*(size_t)num_elements))) FAIL_STACK_ERROR; - /* write all raw data, this would end up in page buffer since there - * is no metadata yet - * - * Not necessarily -- opening the file may may load a metadata page. + /* write all raw data. Since min_md_pages == max_pages, none of it + * should end up in the page buffer. */ for(i=0 ; i<100 ; i++) data[i] = i; @@ -1197,77 +1845,88 @@ test_min_threshold(hid_t orig_fapl, const char *env_h5_drvr) if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*200), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*200), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*400), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*400), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*600), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*600), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*800), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*800), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - page_count += 5; - - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + if(f->shared->pb_ptr->curr_pages != base_meta_cnt) FAIL_STACK_ERROR; - if(page_buf->raw_count != 5 - base_meta_cnt) + if(pb_ptr->curr_rd_pages != 0) TEST_ERROR; /* write all meta data, this would end up in page buffer */ - if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*200), sizeof(int)*50, data) < 0) + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*200), + sizeof(int)*50, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*400), sizeof(int)*50, data) < 0) + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*400), + sizeof(int)*50, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*600), sizeof(int)*50, data) < 0) + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*600), + sizeof(int)*50, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*800), sizeof(int)*50, data) < 0) + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*800), + sizeof(int)*50, data) < 0) FAIL_STACK_ERROR; - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + if(f->shared->pb_ptr->curr_pages != 5) FAIL_STACK_ERROR; - if(page_buf->meta_count != 5) + if(pb_ptr->curr_md_pages != 5) TEST_ERROR; - if(page_buf->raw_count != 0) + if(pb_ptr->curr_rd_pages != 0) TEST_ERROR; /* write and read more raw data and make sure that they don't end up in - * page buffer since the minimum metadata is actually the entire * page buffer */ - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*200), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*200), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*350), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*350), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*500), sizeof(int)*100, data) < 0) + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*500), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*750), sizeof(int)*100, data) < 0) + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*750), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*900), sizeof(int)*100, data) < 0) + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*900), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + if(f->shared->pb_ptr->curr_pages != 5) FAIL_STACK_ERROR; - if(page_buf->meta_count != 5) + if(pb_ptr->curr_md_pages != 5) TEST_ERROR; - if(page_buf->raw_count != 0) + if(pb_ptr->curr_rd_pages != 0) TEST_ERROR; if(H5Fclose(file_id) < 0) @@ -1276,10 +1935,7 @@ test_min_threshold(hid_t orig_fapl, const char *env_h5_drvr) HDprintf("\tMinimum raw data threshold = 100%%\n"); - page_count = 0; - /* keep 5 pages at max in the page buffer and 5 raw page minimum */ - /* Changes: 1000 */ if(H5Pset_page_buffer_size(fapl, sizeof(int)*1000, 0, 100) < 0) TEST_ERROR; @@ -1296,31 +1952,34 @@ test_min_threshold(hid_t orig_fapl, const char *env_h5_drvr) */ HDassert(f); HDassert(f->shared); - HDassert(f->shared->page_buf); + HDassert(f->shared->pb_ptr); - base_raw_cnt = f->shared->page_buf->raw_count; - base_meta_cnt = f->shared->page_buf->meta_count; + base_raw_cnt = f->shared->pb_ptr->curr_rd_pages; + base_meta_cnt = f->shared->pb_ptr->curr_md_pages; if(base_raw_cnt != 0) TEST_ERROR; - if(base_meta_cnt != 1) + if(base_meta_cnt != 0) TEST_ERROR; - page_buf = f->shared->page_buf; + pb_ptr = f->shared->pb_ptr; - if(page_buf->min_meta_count != 0) + if(pb_ptr->min_md_pages != 0) TEST_ERROR; - if(page_buf->min_raw_count != 5) + if(pb_ptr->min_rd_pages != 5) FAIL_STACK_ERROR; - if(HADDR_UNDEF == (meta_addr = H5MF_alloc(f, H5FD_MEM_SUPER, sizeof(int)*(size_t)num_elements))) + if(HADDR_UNDEF == (meta_addr = H5MF_alloc(f, H5FD_MEM_SUPER, + sizeof(int)*(size_t)num_elements))) FAIL_STACK_ERROR; - if(HADDR_UNDEF == (raw_addr = H5MF_alloc(f, H5FD_MEM_DRAW, sizeof(int)*(size_t)num_elements))) + if(HADDR_UNDEF == (raw_addr = H5MF_alloc(f, H5FD_MEM_DRAW, + sizeof(int)*(size_t)num_elements))) TEST_ERROR; - /* write all meta data, this would end up in page buffer since there + /* write all meta data, none of this should end up in the page buffer since + * min_rd_pages == max_pages * is no raw data yet */ for(i=0 ; i<100 ; i++) @@ -1329,86 +1988,97 @@ test_min_threshold(hid_t orig_fapl, const char *env_h5_drvr) if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*200), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*200), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*400), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*400), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*600), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*600), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*800), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*800), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - page_count += 5; - - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + if(f->shared->pb_ptr->curr_pages != 0) FAIL_STACK_ERROR; - if(page_buf->meta_count != 5 - base_raw_cnt) + if(pb_ptr->curr_md_pages != 0) TEST_ERROR; /* write/read all raw data, this would end up in page buffer */ if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*200), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*200), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*400), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*400), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*600), sizeof(int)*100, data) < 0) + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*600), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*800), sizeof(int)*100, data) < 0) + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*800), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + if(f->shared->pb_ptr->curr_pages != 5) FAIL_STACK_ERROR; - if(page_buf->raw_count != 5) + if(pb_ptr->curr_rd_pages != 5) TEST_ERROR; - if(page_buf->meta_count != 0) + if(pb_ptr->curr_md_pages != 0) TEST_ERROR; /* write and read more meta data and make sure that they don't end up in - * page buffer since the minimum metadata is actually the entire * page buffer */ - if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*100), sizeof(int)*50, data) < 0) + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*100), + sizeof(int)*50, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*350), sizeof(int)*50, data) < 0) + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*350), + sizeof(int)*50, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*500), sizeof(int)*50, data) < 0) + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*500), + sizeof(int)*50, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*750), sizeof(int)*50, data) < 0) + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*750), + sizeof(int)*50, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*900), sizeof(int)*50, data) < 0) + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*900), + sizeof(int)*50, data) < 0) FAIL_STACK_ERROR; - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + if(f->shared->pb_ptr->curr_pages != 5) FAIL_STACK_ERROR; - if(page_buf->raw_count != 5) + if(pb_ptr->curr_rd_pages != 5) TEST_ERROR; - if(page_buf->meta_count != 0) + if(pb_ptr->curr_md_pages != 0) TEST_ERROR; if(H5Fclose(file_id) < 0) FAIL_STACK_ERROR; - HDprintf("\tMinimum metadata threshold = 40%%, Minimum rawdata threshold = 40%%\n"); - page_count = 0; - /* keep 5 pages at max in the page buffer 2 meta pages, 2 raw pages - * minimum + HDprintf("\tMinimum metadata threshold = 40%%, "); + HDprintf("Minimum rawdata threshold = 40%%\n"); + + /* keep 5 pages at max in the page buffer 2 meta pages, 2 raw pages + * minimum */ if(H5Pset_page_buffer_size(fapl, sizeof(int)*1000, 40, 40) < 0) TEST_ERROR; @@ -1428,33 +2098,41 @@ test_min_threshold(hid_t orig_fapl, const char *env_h5_drvr) * metadata entry counts. */ + base_raw_cnt = f->shared->pb_ptr->curr_rd_pages; + base_meta_cnt = f->shared->pb_ptr->curr_md_pages; + if(base_raw_cnt != 0) TEST_ERROR; - if(base_meta_cnt != 1) + if(base_meta_cnt != 2) TEST_ERROR; - page_buf = f->shared->page_buf; - if(page_buf->min_meta_count != 2) + pb_ptr = f->shared->pb_ptr; + + if(pb_ptr->min_md_pages != 2) TEST_ERROR; - if(page_buf->min_raw_count != 2) + if(pb_ptr->min_rd_pages != 2) TEST_ERROR; - if(HADDR_UNDEF == (meta_addr = H5MF_alloc(f, H5FD_MEM_SUPER, sizeof(int)*(size_t)num_elements))) + if(HADDR_UNDEF == (meta_addr = H5MF_alloc(f, H5FD_MEM_SUPER, + sizeof(int)*(size_t)num_elements))) FAIL_STACK_ERROR; - if(HADDR_UNDEF == (raw_addr = H5MF_alloc(f, H5FD_MEM_DRAW, sizeof(int)*(size_t)num_elements))) + if(HADDR_UNDEF == (raw_addr = H5MF_alloc(f, H5FD_MEM_DRAW, + sizeof(int)*(size_t)num_elements))) FAIL_STACK_ERROR; /* initialize all the elements to have a value of -1 */ for(i=0 ; i<num_elements ; i++) data[i] = -1; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*(size_t)num_elements, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, + sizeof(int)*(size_t)num_elements, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*(size_t)num_elements, data) < 0) + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, + sizeof(int)*(size_t)num_elements, data) < 0) FAIL_STACK_ERROR; /* fill the page buffer with raw data */ @@ -1464,124 +2142,169 @@ test_min_threshold(hid_t orig_fapl, const char *env_h5_drvr) if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*200), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*200), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*400), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*400), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*600), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*600), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*800), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*800), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - page_count += 5; - - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + if(f->shared->pb_ptr->curr_pages != 5) TEST_ERROR; - if(f->shared->page_buf->raw_count != 5 - base_meta_cnt) + if(f->shared->pb_ptr->curr_rd_pages != 5 - base_meta_cnt) TEST_ERROR; - /* add 3 meta entries evicting 3 raw entries */ + /* add 3 meta entries evicting 1 raw entry */ if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*200), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*200), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*400), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*400), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + if(f->shared->pb_ptr->curr_pages != 5) FAIL_STACK_ERROR; - if(f->shared->page_buf->meta_count != 3) + if(f->shared->pb_ptr->curr_md_pages != 3) TEST_ERROR; - if(f->shared->page_buf->raw_count != 2) + if(f->shared->pb_ptr->curr_rd_pages != 2) TEST_ERROR; /* adding more meta entires should replace meta entries since raw data * is at its minimum */ - if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*600), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*600), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*800), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*800), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(f->shared->page_buf->meta_count != 3) + if(f->shared->pb_ptr->curr_md_pages != 3) TEST_ERROR; - if(f->shared->page_buf->raw_count != 2) + if(f->shared->pb_ptr->curr_rd_pages != 2) TEST_ERROR; /* bring existing raw entires up the LRU */ - if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*750), sizeof(int)*100, data) < 0) + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*750), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; /* adding 2 raw entries (even with 1 call) should only evict 1 meta * entry and another raw entry */ - if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*350), sizeof(int)*100, data) < 0) + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*350), + sizeof(int)*100, data) < 0) + FAIL_STACK_ERROR; + + if(f->shared->pb_ptr->curr_md_pages != 2) + TEST_ERROR; + + if(f->shared->pb_ptr->curr_rd_pages != 3) + TEST_ERROR; + + /* read a metadata entry to force the flush of the metadata entries + * in the page buffer, and then read some raw data so that the metadata + * pages are at the bottom of the LRU. + * + * When we are done, should still have 2 metadata pages and 3 raw data + * pages in the page buffer + */ + + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*442), + sizeof(int)*100, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*150), + sizeof(int)*100, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*550), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(f->shared->page_buf->meta_count != 2) + if(f->shared->pb_ptr->curr_md_pages != 2) TEST_ERROR; - if(f->shared->page_buf->raw_count != 3) + if(f->shared->pb_ptr->curr_rd_pages != 3) TEST_ERROR; - /* adding 2 meta entries should replace 2 entires at the bottom of the LRU */ - if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*98), sizeof(int)*100, data) < 0) + /* adding 2 meta entries should replace 2 entires at the bottom + * of the LRU + */ + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*98), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*242), sizeof(int)*100, data) < 0) + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*242), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(f->shared->page_buf->meta_count != 2) + if(f->shared->pb_ptr->curr_md_pages != 2) TEST_ERROR; - if(f->shared->page_buf->raw_count != 3) + if(f->shared->pb_ptr->curr_rd_pages != 3) TEST_ERROR; if(H5Fclose(file_id) < 0) FAIL_STACK_ERROR; HDprintf("\tMinimum metadata threshold = 20%%\n"); - page_count = 0; + /* keep 5 pages at max in the page buffer and 1 meta page minimum */ if(H5Pset_page_buffer_size(fapl, sizeof(int)*1000, 39, 0) < 0) TEST_ERROR; + /* create the file */ if((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) FAIL_STACK_ERROR; + /* Get a pointer to the internal file object */ if(NULL == (f = (H5F_t *)H5VL_object(file_id))) FAIL_STACK_ERROR; - page_buf = f->shared->page_buf; - if(page_buf->min_meta_count != 1) + pb_ptr = f->shared->pb_ptr; + + if(pb_ptr->min_md_pages != 1) TEST_ERROR; - if(page_buf->min_raw_count != 0) + if(pb_ptr->min_rd_pages != 0) TEST_ERROR; - if(HADDR_UNDEF == (meta_addr = H5MF_alloc(f, H5FD_MEM_SUPER, sizeof(int)*(size_t)num_elements))) + if(HADDR_UNDEF == (meta_addr = H5MF_alloc(f, H5FD_MEM_SUPER, + sizeof(int)*(size_t)num_elements))) FAIL_STACK_ERROR; - if(HADDR_UNDEF == (raw_addr = H5MF_alloc(f, H5FD_MEM_DRAW, sizeof(int)*(size_t)num_elements))) + if(HADDR_UNDEF == (raw_addr = H5MF_alloc(f, H5FD_MEM_DRAW, + sizeof(int)*(size_t)num_elements))) FAIL_STACK_ERROR; /* initialize all the elements to have a value of -1 */ for(i=0 ; i<num_elements ; i++) data[i] = -1; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*(size_t)num_elements, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, + sizeof(int)*(size_t)num_elements, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*(size_t)num_elements, data) < 0) + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, + sizeof(int)*(size_t)num_elements, data) < 0) FAIL_STACK_ERROR; /* fill the page buffer with raw data */ @@ -1591,90 +2314,105 @@ test_min_threshold(hid_t orig_fapl, const char *env_h5_drvr) if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*200), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*200), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*400), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*400), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*600), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*600), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*800), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*800), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - page_count += 5; - - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + if(f->shared->pb_ptr->curr_pages != 5) FAIL_STACK_ERROR; + if(f->shared->pb_ptr->curr_md_pages != 1) + TEST_ERROR; + + if(f->shared->pb_ptr->curr_rd_pages != 4) + TEST_ERROR; + /* add 2 meta entries evicting 2 raw entries */ if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*200), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*200), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + if(f->shared->pb_ptr->curr_pages != 5) FAIL_STACK_ERROR; - if(f->shared->page_buf->meta_count != 2) + if(f->shared->pb_ptr->curr_md_pages != 3) TEST_ERROR; - if(f->shared->page_buf->raw_count != 3) + if(f->shared->pb_ptr->curr_rd_pages != 2) TEST_ERROR; /* bring the rest of the raw entries up the LRU */ - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*500), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*500), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*700), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*700), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*900), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*900), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; /* write one more raw entry which replace one meta entry */ - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*100), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*100), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + if(f->shared->pb_ptr->curr_pages != 5) FAIL_STACK_ERROR; - if(f->shared->page_buf->meta_count != 1) + if(f->shared->pb_ptr->curr_md_pages != 1) TEST_ERROR; - if(f->shared->page_buf->raw_count != 4) + if(f->shared->pb_ptr->curr_rd_pages != 4) TEST_ERROR; /* write one more raw entry which should replace another raw entry * keeping min threshold of meta entries */ - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*300), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*300), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + if(f->shared->pb_ptr->curr_pages != 5) FAIL_STACK_ERROR; - if(f->shared->page_buf->meta_count != 1) + if(f->shared->pb_ptr->curr_md_pages != 1) TEST_ERROR; - if(f->shared->page_buf->raw_count != 4) + if(f->shared->pb_ptr->curr_rd_pages != 4) TEST_ERROR; /* write a metadata entry that should replace the metadata entry * at the bottom of the LRU */ - if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*500), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*500), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + if(f->shared->pb_ptr->curr_pages != 5) FAIL_STACK_ERROR; - if(f->shared->page_buf->meta_count != 1) + if(f->shared->pb_ptr->curr_md_pages != 1) TEST_ERROR; - if(f->shared->page_buf->raw_count != 4) + if(f->shared->pb_ptr->curr_rd_pages != 4) TEST_ERROR; if(H5Fclose(file_id) < 0) @@ -1693,15 +2431,16 @@ test_min_threshold(hid_t orig_fapl, const char *env_h5_drvr) return 0; error: - H5E_BEGIN_TRY { - H5Pclose(fapl); - H5Pclose(fcpl); - H5Fclose(file_id); - if(data) + if (fapl != H5I_INVALID_HID) + H5Pclose(fapl); + if (fcpl != H5I_INVALID_HID) + H5Pclose(fcpl); + if (file_id != H5I_INVALID_HID) + H5Fclose(file_id); + if (data != NULL) HDfree(data); } H5E_END_TRY; - return 1; } /* test_min_threshold */ @@ -1735,6 +2474,10 @@ error: * * JRM -- 2/23/17 * + * Reworked test for the new page buffer implementation. + * + * JRM -- 10/28/18 + * *------------------------------------------------------------------------- */ static unsigned @@ -1746,8 +2489,8 @@ test_stats_collection(hid_t orig_fapl, const char *env_h5_drvr) hid_t fapl = -1; int i; int num_elements = 1000; - size_t base_raw_cnt = 0; - size_t base_meta_cnt = 0; + int64_t base_raw_cnt = 0; + int64_t base_meta_cnt = 0; haddr_t meta_addr = HADDR_UNDEF; haddr_t raw_addr = HADDR_UNDEF; int *data = NULL; @@ -1755,7 +2498,7 @@ test_stats_collection(hid_t orig_fapl, const char *env_h5_drvr) TESTING("Statistics Collection"); - h5_fixname(FILENAME[0], orig_fapl, filename, sizeof(filename)); + h5_fixname(namebase, orig_fapl, filename, sizeof(filename)); if((fapl = H5Pcopy(orig_fapl)) < 0) TEST_ERROR @@ -1769,7 +2512,7 @@ test_stats_collection(hid_t orig_fapl, const char *env_h5_drvr) if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) TEST_ERROR; - if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, 1) < 0) TEST_ERROR; if(H5Pset_file_space_page_size(fcpl, sizeof(int)*200) < 0) @@ -1792,25 +2535,27 @@ test_stats_collection(hid_t orig_fapl, const char *env_h5_drvr) */ HDassert(f); HDassert(f->shared); - HDassert(f->shared->page_buf); + HDassert(f->shared->pb_ptr); - base_raw_cnt = f->shared->page_buf->raw_count; - base_meta_cnt = f->shared->page_buf->meta_count; + base_raw_cnt = f->shared->pb_ptr->curr_rd_pages; + base_meta_cnt = f->shared->pb_ptr->curr_md_pages; if(base_raw_cnt != 0) TEST_ERROR; - if(base_meta_cnt != 1) + if(base_meta_cnt != 2) TEST_ERROR; /* reset statistics before we begin the tests */ if(H5Freset_page_buffering_stats(file_id) < 0) FAIL_STACK_ERROR; - if(HADDR_UNDEF == (meta_addr = H5MF_alloc(f, H5FD_MEM_SUPER, sizeof(int)*(size_t)num_elements))) + if(HADDR_UNDEF == (meta_addr = H5MF_alloc(f, H5FD_MEM_SUPER, + sizeof(int)*(size_t)num_elements))) FAIL_STACK_ERROR; - if(HADDR_UNDEF == (raw_addr = H5MF_alloc(f, H5FD_MEM_DRAW, sizeof(int)*(size_t)num_elements))) + if(HADDR_UNDEF == (raw_addr = H5MF_alloc(f, H5FD_MEM_DRAW, + sizeof(int)*(size_t)num_elements))) FAIL_STACK_ERROR; @@ -1818,10 +2563,12 @@ test_stats_collection(hid_t orig_fapl, const char *env_h5_drvr) for(i=0 ; i<num_elements ; i++) data[i] = -1; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*(size_t)num_elements, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, + sizeof(int)*(size_t)num_elements, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*(size_t)num_elements, data) < 0) + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, + sizeof(int)*(size_t)num_elements, data) < 0) FAIL_STACK_ERROR; for(i=0 ; i<200 ; i++) @@ -1830,137 +2577,217 @@ test_stats_collection(hid_t orig_fapl, const char *env_h5_drvr) if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*200), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*200), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*400), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*400), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*200), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*200), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*600), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*600), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*800), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*800), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*600), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*600), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*500), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*500), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*700), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*700), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*900), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*900), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*400), sizeof(int)*200, data) < 0) + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*400), + sizeof(int)*200, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*100), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*100), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*300), sizeof(int)*100, data) < 0) + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*300), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*800), sizeof(int)*182, data) < 0) + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*800), + sizeof(int)*182, data) < 0) FAIL_STACK_ERROR; if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*200), sizeof(int)*100, data) < 0) + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*200), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*400), sizeof(int)*100, data) < 0) + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*400), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*200), sizeof(int)*100, data) < 0) + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*200), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*600), sizeof(int)*100, data) < 0) + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*600), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*800), sizeof(int)*100, data) < 0) + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*800), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*400), sizeof(int)*100, data) < 0) + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*400), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*600), sizeof(int)*200, data) < 0) + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*600), + sizeof(int)*200, data) < 0) FAIL_STACK_ERROR; - if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*800), sizeof(int)*100, data) < 0) + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*800), + sizeof(int)*100, data) < 0) FAIL_STACK_ERROR; - if(f->shared->page_buf->accesses[0] != 8) - TEST_ERROR; - if(f->shared->page_buf->accesses[1] != 16) - TEST_ERROR; + if ( ( f->shared->pb_ptr->accesses[0] != 9 ) || + ( f->shared->pb_ptr->accesses[1] != 16 ) || + ( f->shared->pb_ptr->accesses[2] != 0 ) ) { - if(f->shared->page_buf->bypasses[0] != 3) - TEST_ERROR; - if(f->shared->page_buf->bypasses[1] != 1) + HDfprintf(stderr, "accesses[] = {%d, %d, %d}. {9, 16, 0} expected\n", + f->shared->pb_ptr->accesses[0], + f->shared->pb_ptr->accesses[1], + f->shared->pb_ptr->accesses[2]); TEST_ERROR; + } - if(f->shared->page_buf->hits[0] != 0) - TEST_ERROR; - if(f->shared->page_buf->hits[1] != 4) - TEST_ERROR; + if ( ( f->shared->pb_ptr->bypasses[0] != 2 ) || + ( f->shared->pb_ptr->bypasses[1] != 1 ) || + ( f->shared->pb_ptr->bypasses[2] != 1 ) ) { - if(f->shared->page_buf->misses[0] != 8) + HDfprintf(stderr, "bypasses[] = {%d, %d, %d}. {2, 1, 1} expected\n", + f->shared->pb_ptr->bypasses[0], + f->shared->pb_ptr->bypasses[1], + f->shared->pb_ptr->bypasses[2]); TEST_ERROR; - if(f->shared->page_buf->misses[1] != 11) + } + + if ( ( f->shared->pb_ptr->hits[0] != 0 ) || + ( f->shared->pb_ptr->hits[1] != 4 ) || + ( f->shared->pb_ptr->hits[2] != 0 ) ) { + + HDfprintf(stderr, "hits[] = {%d, %d, %d}. {0, 4, 0} expected\n", + f->shared->pb_ptr->hits[0], + f->shared->pb_ptr->hits[1], + f->shared->pb_ptr->hits[2]); TEST_ERROR; + } + + if ( ( f->shared->pb_ptr->misses[0] != 9 ) || + ( f->shared->pb_ptr->misses[1] != 16 ) || + ( f->shared->pb_ptr->misses[2] != 0 ) ) { - if(f->shared->page_buf->evictions[0] != 5 + base_meta_cnt) + HDfprintf(stderr, "misses[] = {%d, %d, %d}. {9, 16, 0} expected\n", + f->shared->pb_ptr->misses[0], + f->shared->pb_ptr->misses[1], + f->shared->pb_ptr->misses[2]); TEST_ERROR; - if(f->shared->page_buf->evictions[1] != 9 + base_raw_cnt) + } + + if ( ( f->shared->pb_ptr->evictions[0] != 7) || + ( f->shared->pb_ptr->evictions[1] != 9) || + ( f->shared->pb_ptr->evictions[2] != 0 ) ) { + + HDfprintf(stderr, "evictions[] = {%d, %d, %d}. {%d, %d, 0} expected\n", + f->shared->pb_ptr->evictions[0], + f->shared->pb_ptr->evictions[1], + f->shared->pb_ptr->evictions[2], 7, 9); TEST_ERROR; + } { - unsigned accesses[2]; - unsigned hits[2]; - unsigned misses[2]; - unsigned evictions[2]; - unsigned bypasses[2]; - - if(H5Fget_page_buffering_stats(file_id, accesses, hits, misses, evictions, bypasses) < 0) + unsigned accesses[3]; + unsigned hits[3]; + unsigned misses[3]; + unsigned evictions[3]; + unsigned bypasses[3]; + + if(H5Fget_page_buffering_stats(file_id, accesses, hits, misses, + evictions, bypasses) < 0) FAIL_STACK_ERROR; - if(accesses[0] != 8) - TEST_ERROR; - if(accesses[1] != 16) - TEST_ERROR; - if(bypasses[0] != 3) - TEST_ERROR; - if(bypasses[1] != 1) - TEST_ERROR; - if(hits[0] != 0) - TEST_ERROR; - if(hits[1] != 4) + if ( ( accesses[0] != 9 ) || + ( accesses[1] != 16 ) || + ( accesses[2] != 0 ) ) { + + HDfprintf(stderr, + "accesses[] = {%d, %d, %d}. {9, 16, 0} expected\n", + accesses[0], accesses[1], accesses[2]); TEST_ERROR; - if(misses[0] != 8) + } + + if ( ( bypasses[0] != 2 ) || + ( bypasses[1] != 1 ) || + ( bypasses[2] != 1 ) ) { + + HDfprintf(stderr, "bypasses[] = {%d, %d, %d}. {2, 1, 1} expected\n", + bypasses[0], bypasses[1], bypasses[2]); TEST_ERROR; - if(misses[1] != 11) + } + + if ( ( hits[0] != 0 ) || + ( hits[1] != 4 ) || + ( hits[2] != 0 ) ) { + + HDfprintf(stderr, "hits[] = {%d, %d, %d}. {0, 4, 0} expected\n", + hits[0], hits[1], hits[2]); TEST_ERROR; - if(evictions[0] != 5 + base_meta_cnt) + } + + if ( ( misses[0] != 9 ) || + ( misses[1] != 16 ) || + ( misses[2] != 0 ) ) { + + HDfprintf(stderr, "misses[] = {%d, %d, %d}. {9, 16, 0} expected\n", + misses[0], misses[1], misses[2]); TEST_ERROR; - if(evictions[1] != 9 + base_raw_cnt) + } + + if ( ( evictions[0] != 7 ) || + ( evictions[1] != 9 ) || + ( evictions[2] != 0 ) ) { + + HDfprintf(stderr, + "evictions[] = {%d, %d, %d}. {%d, %d, 0} expected\n", + evictions[0], evictions[1], evictions[2], 7, 9); TEST_ERROR; + } if(H5Freset_page_buffering_stats(file_id) < 0) FAIL_STACK_ERROR; - if(H5Fget_page_buffering_stats(file_id, accesses, hits, misses, evictions, bypasses) < 0) + + if(H5Fget_page_buffering_stats(file_id, accesses, hits, misses, + evictions, bypasses) < 0) FAIL_STACK_ERROR; if(accesses[0] != 0) @@ -1983,7 +2810,7 @@ test_stats_collection(hid_t orig_fapl, const char *env_h5_drvr) TEST_ERROR; if(evictions[1] != 0) TEST_ERROR; - } /* end block */ + } if(H5Fclose(file_id) < 0) FAIL_STACK_ERROR; @@ -1999,10 +2826,13 @@ test_stats_collection(hid_t orig_fapl, const char *env_h5_drvr) error: H5E_BEGIN_TRY { - H5Pclose(fapl); - H5Pclose(fcpl); - H5Fclose(file_id); - if(data) + if (fapl != H5I_INVALID_HID) + H5Pclose(fapl); + if (fcpl != H5I_INVALID_HID) + H5Pclose(fcpl); + if (file_id != H5I_INVALID_HID) + H5Fclose(file_id); + if (data != NULL) HDfree(data); } H5E_END_TRY; @@ -2041,7 +2871,7 @@ verify_page_buffering_disabled(hid_t orig_fapl, const char *env_h5_drvr) hid_t fapl = -1; TESTING("Page Buffering Disabled"); - h5_fixname(FILENAME[0], orig_fapl, filename, sizeof(filename)); + h5_fixname(namebase, orig_fapl, filename, sizeof(filename)); /* first, try to create a file with page buffering enabled */ @@ -2055,7 +2885,7 @@ verify_page_buffering_disabled(hid_t orig_fapl, const char *env_h5_drvr) if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) FAIL_STACK_ERROR; - if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, 1) < 0) FAIL_STACK_ERROR; if(H5Pset_file_space_page_size(fcpl, 4096) < 0) @@ -2078,7 +2908,7 @@ verify_page_buffering_disabled(hid_t orig_fapl, const char *env_h5_drvr) if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) FAIL_STACK_ERROR; - if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, 1) < 0) FAIL_STACK_ERROR; if(H5Pset_file_space_page_size(fcpl, 4096) < 0) @@ -2115,14 +2945,17 @@ verify_page_buffering_disabled(hid_t orig_fapl, const char *env_h5_drvr) error: H5E_BEGIN_TRY { - H5Pclose(fapl); - H5Pclose(fcpl); - H5Fclose(file_id); + if (fapl != H5I_INVALID_HID) + H5Pclose(fapl); + if (fcpl != H5I_INVALID_HID) + H5Pclose(fcpl); + if (file_id != H5I_INVALID_HID) + H5Fclose(file_id); } H5E_END_TRY; return 1; -} /* verify_page_buffering_disabled() */ +} #endif /* H5_HAVE_PARALLEL */ @@ -2163,13 +2996,14 @@ main(void) SKIPPED() HDputs("Skip page buffering test because paged aggregation is disabled for multi/split drivers"); + HDputs("Furthermore, VFD SWMR is not (yet) expected to work with multi/split drivers"); HDexit(EXIT_SUCCESS); - } /* end if */ + } if((fapl = h5_fileaccess()) < 0) { nerrors++; PUTS_ERROR("Can't get VFD-dependent fapl") - } /* end if */ + } /* Push API context */ if(H5CX_push() < 0) FAIL_STACK_ERROR @@ -2183,14 +3017,18 @@ main(void) #else /* H5_HAVE_PARALLEL */ nerrors += test_args(fapl, env_h5_drvr); - nerrors += test_raw_data_handling(fapl, env_h5_drvr); + nerrors += test_raw_data_handling(fapl, env_h5_drvr, false); + nerrors += test_raw_data_handling(fapl, env_h5_drvr, true); + nerrors += test_spmde_delay_basic(fapl, env_h5_drvr); + nerrors += test_mpmde_delay_basic(fapl, env_h5_drvr); + nerrors += test_spmde_lru_evict_basic(fapl, env_h5_drvr); nerrors += test_lru_processing(fapl, env_h5_drvr); nerrors += test_min_threshold(fapl, env_h5_drvr); nerrors += test_stats_collection(fapl, env_h5_drvr); #endif /* H5_HAVE_PARALLEL */ - h5_clean_files(FILENAME, fapl); + h5_clean_files(namebases, fapl); if(nerrors) goto error; @@ -2214,5 +3052,4 @@ error: if(api_ctx_pushed) H5CX_pop(); HDexit(EXIT_FAILURE); -} /* main() */ - +} diff --git a/test/stubs.c b/test/stubs.c new file mode 100644 index 0000000..c04f1ba --- /dev/null +++ b/test/stubs.c @@ -0,0 +1,25 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "h5test.h" +#include "genall5.h" + +/* The default, do-nothing implementation of zoo_create_hook(), which + * is called after each create_zoo() step. + */ +void +zoo_create_hook(hid_t H5_ATTR_UNUSED fid) +{ + return; +} + diff --git a/test/supervise.subr b/test/supervise.subr new file mode 100644 index 0000000..d2b0395 --- /dev/null +++ b/test/supervise.subr @@ -0,0 +1,32 @@ +#!/bin/sh + +# +# catch_out_err_and_rc outbase command [arguments] +# +# Run `command` with any `arguments` provided. Redirect `command`'s +# stderr and stdout to the file `outbase.out`. Record the result code +# of `command` in `outbase.rc`. +# +catch_out_err_and_rc() +{ + if [ $# -lt 2 ]; then + echo "usage: catch_output_and_rc outbase command [arguments]" \ + 1>&2 + exit 1 + fi + outbase=$1 + shift + { + "$@" < ${STDIN_PATH:-/dev/stdin} > ${STDOUT_PATH:-/dev/stdout} & + echo $! > ${outbase}.pid + wait $(cat ${outbase}.pid) + echo $? > ${outbase}.rc + } 2>&1 | tee ${outbase}.out +} + +#catch_out_err_and_rc xxlsxx ls smiles & + +#wait +#echo result=$(cat xxlsxx.rc) + +#exit 0 diff --git a/test/swmr_addrem_writer.c b/test/swmr_addrem_writer.c index 71e4929..7c79de4 100644 --- a/test/swmr_addrem_writer.c +++ b/test/swmr_addrem_writer.c @@ -193,7 +193,7 @@ addrem_records(hid_t fid, unsigned verbose, unsigned long nops, unsigned long fl hid_t file_sid; /* Dataset's space ID */ /* Get a random dataset, according to the symbol distribution */ - symbol = choose_dataset(); + symbol = choose_dataset(NULL, NULL); /* Decide whether to shrink or expand, and by how much */ count[1] = (hsize_t)HDrandom() % (MAX_SIZE_CHANGE * 2) + 1; diff --git a/test/swmr_common.c b/test/swmr_common.c index 7ae1fad..0290fe5 100644 --- a/test/swmr_common.c +++ b/test/swmr_common.c @@ -109,8 +109,9 @@ char VDS_DSET_NAME[NAME_LEN] = "vds_dset"; *------------------------------------------------------------------------- */ symbol_info_t * -choose_dataset(void) +choose_dataset(unsigned *levelp, unsigned *offsetp) { + static unsigned ncalls = 0; unsigned level; /* The level of the dataset */ unsigned offset; /* The "offset" of the dataset at that level */ @@ -120,10 +121,19 @@ choose_dataset(void) /* Determine the offset of the level */ offset = (unsigned)(HDrandom() % (int)symbol_count[level]); + ++ncalls; + if ((ncalls % 1000) == 0) { + fprintf(stderr, "%s: call %u chose level %u offset %u\n", __func__, + ncalls, level, offset); + } + if (levelp != NULL) + *levelp = level; + if (offsetp != NULL) + *offsetp = offset; return &symbol_info[level][offset]; } /* end choose_dataset() */ - + /*------------------------------------------------------------------------- * Function: create_symbol_datatype * @@ -217,13 +227,12 @@ generate_symbols(void) unsigned u, v; /* Local index variables */ for(u = 0; u < NLEVELS; u++) { - symbol_info[u] = (symbol_info_t *)HDmalloc(symbol_count[u] * sizeof(symbol_info_t)); + symbol_info[u] = HDmalloc(symbol_count[u] * sizeof(symbol_info_t)); for(v = 0; v < symbol_count[u]; v++) { char name_buf[64]; generate_name(name_buf, u, v); - symbol_info[u][v].name = (char *)HDmalloc(HDstrlen(name_buf) + 1); - HDstrcpy(symbol_info[u][v].name, name_buf); + symbol_info[u][v].name = HDstrdup(name_buf); symbol_info[u][v].dsid = -1; symbol_info[u][v].nrecords = 0; } /* end for */ diff --git a/test/swmr_common.h b/test/swmr_common.h index a0bc581..47f96b7 100644 --- a/test/swmr_common.h +++ b/test/swmr_common.h @@ -64,7 +64,7 @@ H5TEST_DLLVAR unsigned symbol_count[NLEVELS]; extern "C" { #endif -H5TEST_DLL symbol_info_t * choose_dataset(void); +H5TEST_DLL symbol_info_t * choose_dataset(unsigned *, unsigned *); H5TEST_DLL hid_t create_symbol_datatype(void); H5TEST_DLL int generate_name(char *name_buf, unsigned level, unsigned count); H5TEST_DLL int generate_symbols(void); diff --git a/test/swmr_reader.c b/test/swmr_reader.c index ee263e3..28a6597 100644 --- a/test/swmr_reader.c +++ b/test/swmr_reader.c @@ -250,7 +250,7 @@ read_records(const char *filename, hbool_t verbose, FILE *verbose_file, symbol_info_t *sym; /* Symbol to use */ /* Determine the symbol, within all symbols */ - if(NULL == (sym = choose_dataset())) + if(NULL == (sym = choose_dataset(NULL, NULL))) return -1; sym_rand[v] = sym; diff --git a/test/swmr_remove_reader.c b/test/swmr_remove_reader.c index b02d16f..41f8437 100644 --- a/test/swmr_remove_reader.c +++ b/test/swmr_remove_reader.c @@ -247,7 +247,7 @@ read_records(const char *filename, unsigned verbose, unsigned long nseconds, symbol_info_t *sym; /* Symbol to use */ /* Determine the symbol, within all symbols */ - if(NULL == (sym = choose_dataset())) + if(NULL == (sym = choose_dataset(NULL, NULL))) return -1; sym_rand[v] = sym; diff --git a/test/swmr_remove_writer.c b/test/swmr_remove_writer.c index 2bebab9..2db9493 100644 --- a/test/swmr_remove_writer.c +++ b/test/swmr_remove_writer.c @@ -179,7 +179,7 @@ remove_records(hid_t fid, unsigned verbose, unsigned long nshrinks, unsigned lon hsize_t remove_size; /* Size to reduce dataset dimension by */ /* Get a random dataset, according to the symbol distribution */ - symbol = choose_dataset(); + symbol = choose_dataset(NULL, NULL); /* Shrink the dataset's dataspace */ remove_size = (hsize_t)HDrandom() % MAX_REMOVE_SIZE + 1; diff --git a/test/swmr_sparse_reader.c b/test/swmr_sparse_reader.c index 2d558df..cdf03ba 100644 --- a/test/swmr_sparse_reader.c +++ b/test/swmr_sparse_reader.c @@ -29,6 +29,7 @@ /* Headers */ /***********/ +#include <inttypes.h> /* for PRIu64 */ #include "h5test.h" #include "swmr_common.h" @@ -118,7 +119,7 @@ check_dataset(hid_t fid, unsigned verbose, const symbol_info_t *symbol, symbol_t HDfprintf(stderr, "Symbol = '%s', location = %" PRIuMAX ",%" PRIuMAX "\n", symbol->name, (uintmax_t)start[0], (uintmax_t)start[1]); /* Read record from dataset */ - record->rec_id = (uint64_t)ULLONG_MAX; + record->rec_id = UINT64_MAX; if(H5Dread(dsid, symbol_tid, rec_sid, file_sid, H5P_DEFAULT, record) < 0) return -1; @@ -231,7 +232,7 @@ read_records(const char *filename, unsigned verbose, unsigned long nrecords, unsigned long file_u; /* Attribute sequence number (writer's "u") */ /* Get a random dataset, according to the symbol distribution */ - symbol = choose_dataset(); + symbol = choose_dataset(NULL, NULL); /* Fill in "nrecords" field. Note that this depends on the writer * using the same algorithm and "nrecords" */ diff --git a/test/swmr_sparse_writer.c b/test/swmr_sparse_writer.c index 5173c71..e33ebd4 100644 --- a/test/swmr_sparse_writer.c +++ b/test/swmr_sparse_writer.c @@ -201,7 +201,7 @@ add_records(hid_t fid, unsigned verbose, unsigned long nrecords, unsigned long f hbool_t corked; /* Whether the dataset was corked */ /* Get a random dataset, according to the symbol distribution */ - symbol = choose_dataset(); + symbol = choose_dataset(NULL, NULL); /* If this is the first time the dataset has been opened, extend it and * add the sequence attribute */ diff --git a/test/swmr_start_write.c b/test/swmr_start_write.c index fc7e7a5..5522795 100644 --- a/test/swmr_start_write.c +++ b/test/swmr_start_write.c @@ -257,7 +257,7 @@ add_records(hid_t fid, hbool_t verbose, FILE *verbose_file, hid_t file_sid; /* Dataset's space ID */ /* Get a random dataset, according to the symbol distribution */ - symbol = choose_dataset(); + symbol = choose_dataset(NULL, NULL); /* Set the record's ID (equal to its position) */ record.rec_id = symbol->nrecords; diff --git a/test/swmr_writer.c b/test/swmr_writer.c index d4387aa..5db17ef 100644 --- a/test/swmr_writer.c +++ b/test/swmr_writer.c @@ -197,7 +197,7 @@ add_records(hid_t fid, hbool_t verbose, FILE *verbose_file, hid_t file_sid; /* Dataset's space ID */ /* Get a random dataset, according to the symbol distribution */ - symbol = choose_dataset(); + symbol = choose_dataset(NULL, NULL); /* Set the record's ID (equal to its position) */ record.rec_id = symbol->nrecords; diff --git a/test/testfiles/err_compat_1 b/test/testfiles/err_compat_1 index fc99f77..978c1f4 100644 --- a/test/testfiles/err_compat_1 +++ b/test/testfiles/err_compat_1 @@ -1,4 +1,6 @@ -Testing error API H5Eset/get_auto Testing error API based on data I/O All error API tests passed. +Testing error API H5Eset/get_auto +Testing error API based on data I/O +All error API tests passed. This program tests the Error API compatible with HDF5 version (number). There are supposed to be some error messages ********* Print error stack in HDF5 default way ********* HDF5-DIAG: Error detected in HDF5 (version (number)) thread (IDs): diff --git a/test/testfiles/plist_files/dcpl_32be b/test/testfiles/plist_files/dcpl_32be Binary files differindex 667c67f..c8241a6 100644 --- a/test/testfiles/plist_files/dcpl_32be +++ b/test/testfiles/plist_files/dcpl_32be diff --git a/test/testfiles/plist_files/dcpl_32le b/test/testfiles/plist_files/dcpl_32le Binary files differindex 667c67f..c8241a6 100644 --- a/test/testfiles/plist_files/dcpl_32le +++ b/test/testfiles/plist_files/dcpl_32le diff --git a/test/testfiles/plist_files/dcpl_64be b/test/testfiles/plist_files/dcpl_64be Binary files differindex 667c67f..c8241a6 100644 --- a/test/testfiles/plist_files/dcpl_64be +++ b/test/testfiles/plist_files/dcpl_64be diff --git a/test/testfiles/plist_files/dcpl_64le b/test/testfiles/plist_files/dcpl_64le Binary files differindex 667c67f..c8241a6 100644 --- a/test/testfiles/plist_files/dcpl_64le +++ b/test/testfiles/plist_files/dcpl_64le diff --git a/test/testfiles/plist_files/def_dcpl_32be b/test/testfiles/plist_files/def_dcpl_32be Binary files differindex 01b233c..0909391 100644 --- a/test/testfiles/plist_files/def_dcpl_32be +++ b/test/testfiles/plist_files/def_dcpl_32be diff --git a/test/testfiles/plist_files/def_dcpl_32le b/test/testfiles/plist_files/def_dcpl_32le Binary files differindex 01b233c..0909391 100644 --- a/test/testfiles/plist_files/def_dcpl_32le +++ b/test/testfiles/plist_files/def_dcpl_32le diff --git a/test/testfiles/plist_files/def_dcpl_64be b/test/testfiles/plist_files/def_dcpl_64be Binary files differindex 01b233c..0909391 100644 --- a/test/testfiles/plist_files/def_dcpl_64be +++ b/test/testfiles/plist_files/def_dcpl_64be diff --git a/test/testfiles/plist_files/def_dcpl_64le b/test/testfiles/plist_files/def_dcpl_64le Binary files differindex 01b233c..0909391 100644 --- a/test/testfiles/plist_files/def_dcpl_64le +++ b/test/testfiles/plist_files/def_dcpl_64le diff --git a/test/testfiles/plist_files/def_dxpl_32be b/test/testfiles/plist_files/def_dxpl_32be Binary files differindex b13f456..da29037 100644 --- a/test/testfiles/plist_files/def_dxpl_32be +++ b/test/testfiles/plist_files/def_dxpl_32be diff --git a/test/testfiles/plist_files/def_dxpl_32le b/test/testfiles/plist_files/def_dxpl_32le Binary files differindex b13f456..da29037 100644 --- a/test/testfiles/plist_files/def_dxpl_32le +++ b/test/testfiles/plist_files/def_dxpl_32le diff --git a/test/testfiles/plist_files/def_dxpl_64be b/test/testfiles/plist_files/def_dxpl_64be Binary files differindex b13f456..da29037 100644 --- a/test/testfiles/plist_files/def_dxpl_64be +++ b/test/testfiles/plist_files/def_dxpl_64be diff --git a/test/testfiles/plist_files/def_dxpl_64le b/test/testfiles/plist_files/def_dxpl_64le Binary files differindex b13f456..da29037 100644 --- a/test/testfiles/plist_files/def_dxpl_64le +++ b/test/testfiles/plist_files/def_dxpl_64le diff --git a/test/testfiles/plist_files/def_fapl_32be b/test/testfiles/plist_files/def_fapl_32be Binary files differindex 53ef572..6738cda 100644 --- a/test/testfiles/plist_files/def_fapl_32be +++ b/test/testfiles/plist_files/def_fapl_32be diff --git a/test/testfiles/plist_files/def_fapl_32le b/test/testfiles/plist_files/def_fapl_32le Binary files differindex 53ef572..6738cda 100644 --- a/test/testfiles/plist_files/def_fapl_32le +++ b/test/testfiles/plist_files/def_fapl_32le diff --git a/test/testfiles/plist_files/def_fapl_64be b/test/testfiles/plist_files/def_fapl_64be Binary files differindex 53ef572..6738cda 100644 --- a/test/testfiles/plist_files/def_fapl_64be +++ b/test/testfiles/plist_files/def_fapl_64be diff --git a/test/testfiles/plist_files/def_fapl_64le b/test/testfiles/plist_files/def_fapl_64le Binary files differindex 53ef572..6738cda 100644 --- a/test/testfiles/plist_files/def_fapl_64le +++ b/test/testfiles/plist_files/def_fapl_64le diff --git a/test/testfiles/plist_files/dxpl_32be b/test/testfiles/plist_files/dxpl_32be Binary files differindex 5ff2ea0..cac7e50 100644 --- a/test/testfiles/plist_files/dxpl_32be +++ b/test/testfiles/plist_files/dxpl_32be diff --git a/test/testfiles/plist_files/dxpl_32le b/test/testfiles/plist_files/dxpl_32le Binary files differindex 5ff2ea0..cac7e50 100644 --- a/test/testfiles/plist_files/dxpl_32le +++ b/test/testfiles/plist_files/dxpl_32le diff --git a/test/testfiles/plist_files/dxpl_64be b/test/testfiles/plist_files/dxpl_64be Binary files differindex 5ff2ea0..cac7e50 100644 --- a/test/testfiles/plist_files/dxpl_64be +++ b/test/testfiles/plist_files/dxpl_64be diff --git a/test/testfiles/plist_files/dxpl_64le b/test/testfiles/plist_files/dxpl_64le Binary files differindex 5ff2ea0..cac7e50 100644 --- a/test/testfiles/plist_files/dxpl_64le +++ b/test/testfiles/plist_files/dxpl_64le diff --git a/test/testfiles/plist_files/fapl_32be b/test/testfiles/plist_files/fapl_32be Binary files differindex d89a44c..807c04a 100644 --- a/test/testfiles/plist_files/fapl_32be +++ b/test/testfiles/plist_files/fapl_32be diff --git a/test/testfiles/plist_files/fapl_32le b/test/testfiles/plist_files/fapl_32le Binary files differindex d89a44c..807c04a 100644 --- a/test/testfiles/plist_files/fapl_32le +++ b/test/testfiles/plist_files/fapl_32le diff --git a/test/testfiles/plist_files/fapl_64be b/test/testfiles/plist_files/fapl_64be Binary files differindex d89a44c..807c04a 100644 --- a/test/testfiles/plist_files/fapl_64be +++ b/test/testfiles/plist_files/fapl_64be diff --git a/test/testfiles/plist_files/fapl_64le b/test/testfiles/plist_files/fapl_64le Binary files differindex d89a44c..807c04a 100644 --- a/test/testfiles/plist_files/fapl_64le +++ b/test/testfiles/plist_files/fapl_64le diff --git a/test/testfiles/plist_files/lapl_32be b/test/testfiles/plist_files/lapl_32be Binary files differindex eee238e..3db7163 100644 --- a/test/testfiles/plist_files/lapl_32be +++ b/test/testfiles/plist_files/lapl_32be diff --git a/test/testfiles/plist_files/lapl_32le b/test/testfiles/plist_files/lapl_32le Binary files differindex eee238e..3db7163 100644 --- a/test/testfiles/plist_files/lapl_32le +++ b/test/testfiles/plist_files/lapl_32le diff --git a/test/testfiles/plist_files/lapl_64be b/test/testfiles/plist_files/lapl_64be Binary files differindex eee238e..3db7163 100644 --- a/test/testfiles/plist_files/lapl_64be +++ b/test/testfiles/plist_files/lapl_64be diff --git a/test/testfiles/plist_files/lapl_64le b/test/testfiles/plist_files/lapl_64le Binary files differindex eee238e..3db7163 100644 --- a/test/testfiles/plist_files/lapl_64le +++ b/test/testfiles/plist_files/lapl_64le diff --git a/test/testvfdswmr.sh.in b/test/testvfdswmr.sh.in new file mode 100644 index 0000000..a355245 --- /dev/null +++ b/test/testvfdswmr.sh.in @@ -0,0 +1,746 @@ +#! /bin/bash +# +# Copyright by The HDF Group. +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# Tests for the vfd swmr feature. +# +############################################################################### +# VFD SWMR concurrent tests which are modified from existing swmr concurrent tests. +# This is copied and modified from testswmr.sh.in +# +############################################################################### + +srcdir=@srcdir@ + +. ${srcdir}/supervise.subr + +############################################################################### +## test parameters +############################################################################### + +Nreaders=5 # number of readers to launch +Nrdrs_spa=3 # number of sparse readers to launch +Nrecords=400000 # number of records to write +Nrecs_rem=40000 # number of times to shrink +Nrecs_spa=20000 # number of records to write in the sparse test +Nsecs_add=5 # number of seconds per read interval +Nsecs_rem=3 # number of seconds per read interval +Nsecs_addrem=8 # number of seconds per read interval +nerrors=0 +nsofterrors=0 # soft errors are expected to occur some of the time + # on a couple of nondeterministic tests. + +############################################################################### +## definitions for message file to coordinate test runs +############################################################################### +WRITER_MESSAGE=VFD_SWMR_WRITER_MESSAGE # The message file created by writer that the open is complete + # This should be the same as the define in "./swmr_common.h" +MESSAGE_TIMEOUT=300 # Message timeout length in secs + # This should be the same as the define in "./h5test.h" + +############################################################################### +## short hands and function definitions +############################################################################### +DPRINT=: # Set to "echo Debug:" for debugging printing, + # else ":" for noop. +IFDEBUG=: # Set to null to turn on debugging, else ":" for noop. + +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Testing". +# +TESTING() { + SPACES=" " + echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' +} + +# To wait for the writer message file or till the maximum # of seconds is reached +# $1 is the message file to wait for +# This performs similar function as the routine h5_wait_message() in test/h5test.c +WAIT_MESSAGE() { + message=$1 # Get the name of the message file to wait for + t0=`date +%s` # Get current time in seconds + difft=0 # Initialize the time difference + mexist=0 # Indicate whether the message file is found + while [ $difft -lt $MESSAGE_TIMEOUT ] ; # Loop till message times out + do + t1=`date +%s` # Get current time in seconds + difft=`expr $t1 - $t0` # Calculate the time difference + if [ -e $message ]; then # If message file is found: + mexist=1 # indicate the message file is found + rm $message # remove the message file + break # get out of the while loop + fi + done; + if test $mexist -eq 0; then + # Issue warning that the writer message file is not found, continue with launching the reader(s) + echo warning: $WRITER_MESSAGE is not found after waiting $MESSAGE_TIMEOUT seconds + else + echo $WRITER_MESSAGE is found + fi +} + +############################################################################### +## Main +## +## Modifications: +## Vailin Choi; July 2013 +## Add waiting of message file before launching the reader(s). +## Due to the implementation of file locking, coordination +## is needed in file opening for the writer/reader tests +## to proceed as expected. +## +############################################################################### +# The build (current) directory might be different than the source directory. +if test -z "$srcdir"; then + srcdir=. +fi + +# Check to see if the VFD specified by the HDF5_DRIVER environment variable +# supports SWMR. ??? DO I NEED TO MODIFY THIS ???? +./swmr_check_compat_vfd +rc=$? +if [ $rc -ne 0 ] ; then + echo + echo "The VFD specified by the HDF5_DRIVER environment variable" + echo "does not support VFD SWMR." + echo + echo "VFD SWMR acceptance tests skipped" + echo + exit 0 +fi + +all_tests="generator expand shrink expand_shrink sparse vlstr_null vlstr_oob zoo groups" +all_tests="${all_tests} few_big many_small" +tests=${all_tests} + +if [ $# -gt 0 ]; then + tests= +fi + +for t; do + if ! echo $all_tests | grep -q "\<${t}\>"; then + echo "$t: Unknown test, ${t}" + exit 1 + fi + tests="${tests} ${t}" +done + +echo tests=${tests} +for t in ${tests}; do + eval do_${t}=yes +done + +# HDF5 has several tests that create and delete signal files to communicate +# between processes, and it seems that even though the names of the files are +# different, occasionally the wrong file is deleted, interrupting the flow of +# the test. Running each of these tests in its own directory should eliminate +# the problem. +rm -rf vfd_swmr_test +mkdir vfd_swmr_test + +## With the --disable-shared option, swmr program files are built in the test +## directory, otherwise they are in test/.libs with a corresponding wrapper +## script in the test directory. The programs or wrapper scripts in test should +## always be copied, swmr files in .libs should be copied only if they exists. +#if [ -f .libs/vfd_swmr ]; then +# mkdir vfd_swmr_test/.libs +# for FILE in .libs/vfd_swmr*; do +# case "$FILE" in +# *.o) continue ;; ## don't copy the .o files +# esac +# cp $FILE vfd_swmr_test/.libs +# done +#fi + +cd vfd_swmr_test + +# Loop over index types +for index_type in "-i ea" "-i b2" +do + # Try without compression, only; uncomment "-c 5" to try with compression. + + for compress in "" #"-c 5" + do + echo + echo "** Loop testing parameters: $index_type $compress" + echo + if [ ${do_generator:-no} = yes ]; then + echo + echo "## Generator test" + # Launch the Generator without VFD SWMR write + echo launch the vfd_swmr_generator + ../vfd_swmr_generator $compress $index_type + if test $? -ne 0; then + echo generator had error + nerrors=`expr $nerrors + 1` + fi + + # Launch the Generator with VFD SWMR write + echo launch the vfd_swmr_generator with VFD SWMR write + ../vfd_swmr_generator -s $compress $index_type + if test $? -ne 0; then + echo generator had error + nerrors=`expr $nerrors + 1` + fi + fi + + if [ ${do_expand:-no} = yes ]; then + echo + echo "## Writer test - test expanding the dataset" + + # Launch the Generator + echo launch the vfd_swmr_generator with VFD SWMR write + ../vfd_swmr_generator -s $compress $index_type + if test $? -ne 0; then + echo generator had error + nerrors=`expr $nerrors + 1` + fi + + # Remove any possible writer message file before launching writer + rm -f $WRITER_MESSAGE + # + # Launch the Writer + echo launch the vfd_swmr_writer + seed="" # Put -r <random seed> command here + catch_out_err_and_rc vfd_swmr_writer \ + ../vfd_swmr_writer -o $Nrecords $seed & + pid_writer=$! + $DPRINT pid_writer=$pid_writer + + # Wait for message from writer process before starting reader(s) + WAIT_MESSAGE $WRITER_MESSAGE + # + # Launch the Readers + #declare -a seeds=(<seed1> <seed2> <seed3> ... ) + echo launch $Nreaders vfd_swmr_readers + pid_readers="" + n=0 + while [ $n -lt $Nreaders ]; do + #seed="-r ${seeds[$n]}" + seed="" + catch_out_err_and_rc vfd_swmr_reader.$n \ + ../vfd_swmr_reader $Nsecs_add $seed & + pid_readers="$pid_readers $!" + n=`expr $n + 1` + done + $DPRINT pid_readers=$pid_readers + $IFDEBUG ps + + # Wait for the readers to finish before signalling the + # writer to quit: the writer holds the file open so that the + # readers will find the shadow file when they reopen + # the .h5 file. + wait $pid_readers + kill -USR1 $(cat vfd_swmr_writer.pid) + wait $pid_writer + + # Collect exit codes of the readers + n=0 + while [ $n -lt $Nreaders ]; do + if [ $(cat vfd_swmr_reader.$n.rc) -ne 0 ]; then + echo reader had error + nerrors=$((nerrors + 1)) + fi + n=$((n + 1)) + done + + # Collect exit code of the writer + $DPRINT checked writer $pid_writer + if [ $(cat vfd_swmr_writer.rc) -ne 0 ]; then + echo writer had error + nerrors=$((nerrors + 1)) + fi + + # Clean up output files + rm -f vfd_swmr_writer.{out,rc} + rm -f vfd_swmr_reader.*.{out,rc} + fi + + if [ ${do_shrink:-no} = yes ]; then + if [ ${do_expand:-no} != yes ]; then + echo "Cancelling the 'shrink' test: it depends on the .h5 file left behind by the 'expand' test." 1>&2 + exit 1 + fi + echo + echo "## Remove test - test shrinking the dataset" + + # Remove any possible writer message file before launching writer + rm -f $WRITER_MESSAGE + # Launch the Remove Writer + echo launch the vfd_swmr_remove_writer + seed="" # Put -r <random seed> command here + catch_out_err_and_rc vfd_swmr_writer \ + ../vfd_swmr_remove_writer -o $Nrecs_rem $seed & + pid_writer=$! + $DPRINT pid_writer=$pid_writer + + # Wait for message from writer process before starting reader(s) + WAIT_MESSAGE $WRITER_MESSAGE + # + # Launch the Remove Readers + #declare -a seeds=(<seed1> <seed2> <seed3> ... ) + n=0 + pid_readers="" + echo launch $Nreaders swmr_remove_readers + while [ $n -lt $Nreaders ]; do + #seed="-r ${seeds[$n]}" + seed="" + catch_out_err_and_rc vfd_swmr_reader.$n \ + ../vfd_swmr_remove_reader $Nsecs_rem $seed & + pid_readers="$pid_readers $!" + n=`expr $n + 1` + done + $DPRINT pid_readers=$pid_readers + $IFDEBUG ps + + # Wait for the readers to finish before signalling the + # writer to quit: the writer holds the file open so that the + # readers will find the shadow file when they reopen + # the .h5 file. + wait $pid_readers + kill -USR1 $(cat vfd_swmr_writer.pid) + wait $pid_writer + + # Collect exit codes of the readers + n=0 + while [ $n -lt $Nreaders ]; do + if [ $(cat vfd_swmr_reader.$n.rc) -ne 0 ]; then + echo reader had error + nerrors=$((nerrors + 1)) + fi + n=$((n + 1)) + done + + # Collect exit code of the writer + $DPRINT checked writer $pid_writer + if [ $(cat vfd_swmr_writer.rc) -ne 0 ]; then + echo writer had error + nerrors=$((nerrors + 1)) + fi + + # Clean up output files + rm -f vfd_swmr_writer.{out,rc} + rm -f vfd_swmr_reader.*.{out,rc} + fi + + if [ ${do_expand_shrink:-no} = yes ]; then + echo + echo "## Expand/shrink test - randomly grow or shrink the dataset" + + # Launch the Generator + echo launch the vfd_swmr_generator with VFD SWMR write + ../vfd_swmr_generator -s $compress $index_type + if test $? -ne 0; then + echo generator had error + nerrors=`expr $nerrors + 1` + fi + + # Launch the Writer (not in parallel - just to rebuild the datasets) + echo launch the vfd_swmr_writer + seed="" # Put -r <random seed> command here + ../vfd_swmr_writer -W $Nrecords $seed + if test $? -ne 0; then + echo writer had error + nerrors=`expr $nerrors + 1` + fi + + # Remove any possible writer message file before launching writer + rm -f $WRITER_MESSAGE + # + # Launch the Add/Remove Writer + echo launch the vfd_swmr_addrem_writer + seed="" # Put -r <random seed> command here + catch_out_err_and_rc vfd_swmr_writer \ + ../vfd_swmr_addrem_writer $Nrecords $seed & + pid_writer=$! + $DPRINT pid_writer=$pid_writer + + # Wait for message from writer process before starting reader(s) + WAIT_MESSAGE $WRITER_MESSAGE + # + # Launch the Add/Remove Readers + #declare -a seeds=(<seed1> <seed2> <seed3> ... ) + n=0 + pid_readers="" + echo launch $Nreaders vfd_swmr_remove_readers + while [ $n -lt $Nreaders ]; do + #seed="-r ${seeds[$n]}" + seed="" + catch_out_err_and_rc vfd_swmr_reader.$n \ + ../vfd_swmr_remove_reader $Nsecs_addrem $seed & + pid_readers="$pid_readers $!" + n=`expr $n + 1` + done + $DPRINT pid_readers=$pid_readers + $IFDEBUG ps + + # Wait for the readers to finish before signalling the + # writer to quit: the writer holds the file open so that the + # readers will find the shadow file when they reopen + # the .h5 file. + wait $pid_readers + kill -USR1 $(cat vfd_swmr_writer.pid) + wait $pid_writer + + # Collect exit codes of the readers + n=0 + while [ $n -lt $Nreaders ]; do + if [ $(cat vfd_swmr_reader.$n.rc) -ne 0 ]; then + echo reader had error + nerrors=$((nerrors + 1)) + fi + n=$((n + 1)) + done + + # Collect exit code of the writer + $DPRINT checked writer $pid_writer + if [ ! -e vfd_swmr_writer.rc ] || + [ $(cat vfd_swmr_writer.rc) -ne 0 ]; then + echo writer had error + nerrors=$((nerrors + 1)) + fi + + # Clean up output files + rm -f vfd_swmr_writer.{out,rc} + rm -f vfd_swmr_reader.*.{out,rc} + fi + + if [ ${do_sparse:-no} = yes ]; then + echo + echo "## Sparse writer test - write random dataset locations" + + # Launch the Generator + # NOTE: Random seed is shared between readers and writers and is + # created by the generator. + echo launch the vfd_swmr_generator with VFD SWMR write + seed="" # Put -r <random seed> command here + ../vfd_swmr_generator -s $compress $index_type $seed + if test $? -ne 0; then + echo generator had error + nerrors=`expr $nerrors + 1` + fi + + # Remove any possible writer message file before launching writer + rm -f $WRITER_MESSAGE + # Launch the Sparse writer + echo launch the vfd_swmr_sparse_writer + catch_out_err_and_rc vfd_swmr_writer nice -n 20 \ + ../vfd_swmr_sparse_writer $Nrecs_spa & + pid_writer=$! + $DPRINT pid_writer=$pid_writer + + # Wait for message from writer process before starting reader(s) + WAIT_MESSAGE $WRITER_MESSAGE + # + # Launch the Sparse readers + n=0 + pid_readers="" + echo launch $Nrdrs_spa vfd_swmr_sparse_readers + while [ $n -lt $Nrdrs_spa ]; do + # The sparse reader spits out a LOT of data so it's set to 'quiet' + catch_out_err_and_rc vfd_swmr_reader.$n \ + ../vfd_swmr_sparse_reader -q $Nrecs_spa & + pid_readers="$pid_readers $!" + n=`expr $n + 1` + done + $DPRINT pid_readers=$pid_readers + $IFDEBUG ps + + # Wait for the readers and the writer to finish. + echo "pid_readers=$pid_readers" + echo "pid_writer=$pid_writer" + + # Wait for the readers to finish before signalling the + # writer to quit: the writer holds the file open so that the + # readers will find the shadow file when they reopen + # the .h5 file. + wait $pid_readers + kill -USR1 $(cat vfd_swmr_writer.pid) + wait $pid_writer + + # Collect exit codes of the readers + n=0 + while [ $n -lt $Nrdrs_spa ]; do + if [ $(cat vfd_swmr_reader.$n.rc) -ne 0 ]; then + echo reader had error + nerrors=$((nerrors + 1)) + fi + n=$((n + 1)) + done + + # Collect exit code of the writer + $DPRINT checked writer $pid_writer + if [ $(cat vfd_swmr_writer.rc) -ne 0 ]; then + echo writer had error + nerrors=$((nerrors + 1)) + fi + + # Clean up output files + rm -f vfd_swmr_writer.{out,rc} + rm -f vfd_swmr_reader.*.{out,rc} + fi + done +done + +# +# Test variable-length strings, expecting errors. +# +for ty in null oob; do + + if [ ${ty} = null ]; then + [ ${do_vlstr_null:-no} = no ] && continue + echo + echo "## VL string 1 - expect to read NULL" + else + [ ${do_vlstr_oob:-no} = no ] && continue + echo + echo "## VL string 2 - expect out-of-bounds access" + fi + + echo launch vfd_swmr_vlstr_writer + catch_out_err_and_rc vfd_swmr_vlstr_writer \ + ../vfd_swmr_vlstr_writer -n 500 -q -t ${ty} & + pid_writer=$! + + # pause? + + catch_out_err_and_rc vfd_swmr_vlstr_reader \ + ../vfd_swmr_vlstr_reader -n 500 -q -t ${ty} & + pid_reader=$! + + # Wait for the reader to finish before signalling the + # writer to quit: the writer holds the file open so that the + # reader will find the shadow file when it opens + # the .h5 file. + wait $pid_reader + kill -USR1 $(cat vfd_swmr_vlstr_writer.pid) + wait $pid_writer + + # Collect exit code of the reader + if [ $(cat vfd_swmr_vlstr_reader.rc) -ne 0 ]; then + echo reader had error + nsofterrors=$((nsofterrors + 1)) + fi + + # Collect exit code of the writer + if [ $(cat vfd_swmr_vlstr_writer.rc) -ne 0 ]; then + echo writer had error + nerrors=$((nerrors + 1)) + fi + + # Clean up output files + rm -f vfd_swmr_vlstr_writer.{out,rc} + rm -f vfd_swmr_vlstr_reader.*.{out,rc} +done + +# +# Make sure that a "zoo"---the variety of HDF5 object types---can be +# read and written by VFD SWMR. +# +if [ ${do_zoo:-no} = yes ]; then + [ -e ./fifo ] && rm -f ./fifo + mkfifo -m 0600 ./fifo + rm -f ./shared_tick_num + echo launch vfd_swmr_zoo_writer + STDIN_PATH="./fifo" catch_out_err_and_rc vfd_swmr_zoo_writer \ + ../vfd_swmr_zoo_writer -m 1000 -q & + pid_writer=$! + + STDOUT_PATH="./fifo" catch_out_err_and_rc vfd_swmr_zoo_reader \ + ../vfd_swmr_zoo_reader -q -W & + pid_reader=$! + + # Wait for the reader to finish before signalling the + # writer to quit: the writer holds the file open so that the + # reader will find the shadow file when it opens + # the .h5 file. + wait $pid_reader + kill -USR1 $(cat vfd_swmr_zoo_writer.pid) + wait $pid_writer + + # Collect exit code of the reader + if [ $(cat vfd_swmr_zoo_reader.rc) -ne 0 ]; then + echo reader had error + nerrors=$((nerrors + 1)) + fi + + # Collect exit code of the writer + if [ $(cat vfd_swmr_zoo_writer.rc) -ne 0 ]; then + echo writer had error + nerrors=$((nerrors + 1)) + fi + + # Clean up output files + rm -f ./fifo + rm -f vfd_swmr_zoo_writer.{out,rc} + rm -f vfd_swmr_zoo_reader.*.{out,rc} +fi + +# +# Make sure that we can create 10000 groups while a reader waits +# for each to appear. +# +if [ ${do_groups:-no} = yes ]; then + echo launch vfd_swmr_group_writer + catch_out_err_and_rc vfd_swmr_group_writer \ + ../vfd_swmr_group_writer -q -u 10 -n 10000 & + pid_writer=$! + + catch_out_err_and_rc vfd_swmr_group_reader \ + ../vfd_swmr_group_reader -q -u 10 -n 10000 -W & + pid_reader=$! + + # Wait for the reader to finish before signalling the + # writer to quit: the writer holds the file open so that the + # reader will find the shadow file when it opens + # the .h5 file. + wait $pid_reader + kill -USR1 $(cat vfd_swmr_group_writer.pid) + wait $pid_writer + + # Collect exit code of the reader + if [ $(cat vfd_swmr_group_reader.rc) -ne 0 ]; then + echo reader had error + nerrors=$((nerrors + 1)) + fi + + # Collect exit code of the writer + if [ $(cat vfd_swmr_group_writer.rc) -ne 0 ]; then + echo writer had error + nerrors=$((nerrors + 1)) + fi + + # Clean up output files + rm -f vfd_swmr_group_writer.{out,rc} + rm -f vfd_swmr_group_reader.*.{out,rc} +fi + +for options in "-d 1" "-d 2" "-d 1 -V" "-d 1 -M"; do + if [ ${do_many_small:-no} = no ]; then + continue + fi + # + # Test many small datasets of one and two dimensions. + # + # Perform 50 iterations on 1000 extensible datasets configured with + # 16x16 chunks of 32-bit unsigned integer elements, + # expanding each dataset by a chunk in one dimension (up to 50x1 + # 16x16 chunks) on each iteration. + # + # Perform the test again, extending each dataset + # in *two* dimensions (up to 50x50 16x16 chunks). + # + echo launch vfd_swmr_bigset_writer many small, options $options + catch_out_err_and_rc vfd_swmr_bigset_writer \ + ../vfd_swmr_bigset_writer -n 50 $options -s 1000 -r 16 -c 16 -q & + pid_writer=$! + + catch_out_err_and_rc vfd_swmr_bigset_reader \ + ../vfd_swmr_bigset_reader -n 50 $options -s 1000 -r 16 -c 16 -q -W & + pid_reader=$! + + # Wait for the reader to finish before signalling the + # writer to quit: the writer holds the file open so that the + # reader will find the shadow file when it opens + # the .h5 file. + wait $pid_reader + kill -USR1 $(cat vfd_swmr_bigset_writer.pid) + wait $pid_writer + + # Collect exit code of the reader + if [ $(cat vfd_swmr_bigset_reader.rc) -ne 0 ]; then + echo reader had error + nerrors=$((nerrors + 1)) + fi + + # Collect exit code of the writer + if [ $(cat vfd_swmr_bigset_writer.rc) -ne 0 ]; then + echo writer had error + nerrors=$((nerrors + 1)) + fi + + # Clean up output files + rm -f vfd_swmr_bigset_writer.{out,rc} + rm -f vfd_swmr_bigset_reader.*.{out,rc} +done + +for options in "-d 1" "-d 2" "-d 1 -V" "-d 1 -M"; do + # + # Test a few big datasets of one and two dimensions. + # + # Perform 50 iterations on 5 extensible datasets configured with + # 256x256 chunks of 32-bit unsigned integer elements, + # expanding each dataset by a chunk in one dimension (up to 50x1 + # 256x256 chunks) on each iteration. + # + # Perform the test again, extending each dataset + # in *two* dimensions (up to 50x50 256x256 chunks). + # + if [ ${do_few_big:-no} = no ]; then + continue + fi + echo launch vfd_swmr_bigset_writer few big, options $options + catch_out_err_and_rc vfd_swmr_bigset_writer \ + ../vfd_swmr_bigset_writer -n 50 $options -s 40 -r 256 -c 256 -q & + pid_writer=$! + + catch_out_err_and_rc vfd_swmr_bigset_reader \ + ../vfd_swmr_bigset_reader -n 50 $options -s 40 -r 256 -c 256 -q -W & + pid_reader=$! + + # Wait for the reader to finish before signalling the + # writer to quit: the writer holds the file open so that the + # reader will find the shadow file when it opens + # the .h5 file. + wait $pid_reader + kill -USR1 $(cat vfd_swmr_bigset_writer.pid) + wait $pid_writer + + # Collect exit code of the reader + if [ $(cat vfd_swmr_bigset_reader.rc) -ne 0 ]; then + echo reader had error + nerrors=$((nerrors + 1)) + fi + + # Collect exit code of the writer + if [ $(cat vfd_swmr_bigset_writer.rc) -ne 0 ]; then + echo writer had error + nerrors=$((nerrors + 1)) + fi + + # Clean up output files + rm -f vfd_swmr_bigset_writer.{out,rc} + rm -f vfd_swmr_bigset_reader.*.{out,rc} +done + +############################################################################### +## Report and exit +############################################################################### +cd .. +$DPRINT nerrors $nerrors nsofterrors $nsofterrors +if test $nerrors -eq 0 ; then + echo "VFD SWMR tests passed." + if test $nsofterrors -ne 0 ; then + echo + echo "${nsofterrors} soft errors occurred. That's safe to ignore." + fi + if test -z "$HDF5_NOCLEANUP"; then + # delete the test directory + rm -rf vfd_swmr_test + fi + exit 0 +else + echo -n "VFD SWMR tests failed with $nerrors hard errors " + echo "and $nsofterrors soft errors." + exit 1 +fi + diff --git a/test/tmisc.c b/test/tmisc.c index 6eb6872..f35daac 100644 --- a/test/tmisc.c +++ b/test/tmisc.c @@ -135,6 +135,7 @@ typedef struct #ifndef H5_HAVE_PARALLEL #define MISC8_DSETNAME2 "Dataset2" #define MISC8_DSETNAME3 "Dataset3" +#define MISC8_DSETNAME4 "Dataset4" #define MISC8_DSETNAME6 "Dataset6" #define MISC8_DSETNAME7 "Dataset7" #define MISC8_DSETNAME9 "Dataset9" diff --git a/test/unlink.c b/test/unlink.c index 48dd79d..16d5d0c 100644 --- a/test/unlink.c +++ b/test/unlink.c @@ -1259,7 +1259,7 @@ test_create_unlink(const char *msg, hid_t fapl) char groupname[1024]; char filename[1024]; - TESTING(msg); + TESTING("%s", msg); /* Create file */ h5_fixname(FILENAME[3], fapl, filename, sizeof filename); diff --git a/test/vfd_swmr.c b/test/vfd_swmr.c new file mode 100644 index 0000000..d49629f --- /dev/null +++ b/test/vfd_swmr.c @@ -0,0 +1,3474 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*********************************************************** +* +* Test program: +* +* Tests the VFD SWMR Feature. +* +*************************************************************/ + +#include "H5queue.h" +#include "h5test.h" +#include "vfd_swmr_common.h" + +/* + * This file needs to access private information from the H5F package. + */ + +#define H5F_FRIEND /*suppress error about including H5Fpkg */ +#define H5FD_FRIEND /*suppress error about including H5FDpkg */ +#define H5F_TESTING +#define H5FD_TESTING +#include "H5FDprivate.h" +#include "H5Fpkg.h" +#include "H5FDpkg.h" +#include "H5Iprivate.h" + +#define H5FD_FRIEND /*suppress error about including H5FDpkg */ +#include "H5FDpkg.h" + +#define FS_PAGE_SIZE 512 +#define FILENAME "vfd_swmr_file.h5" +#define MD_FILENAME "vfd_swmr_metadata_file" + +#define FILENAME2 "vfd_swmr_file2.h5" +#define MD_FILENAME2 "vfd_swmr_metadata_file2" + +#define FILENAME3 "vfd_swmr_file3.h5" +#define MD_FILENAME3 "vfd_swmr_metadata_file3" + +#define FNAME "non_vfd_swmr_file.h5" + +/* test routines for VFD SWMR */ +static unsigned test_fapl(void); +static unsigned test_file_end_tick(void); +static unsigned test_file_fapl(void); +static unsigned test_writer_md(void); + +/* helper routines */ +static hid_t +init_vfd_swmr_config_fapl(H5F_vfd_swmr_config_t *config, uint32_t tick_len, uint32_t max_lag, + hbool_t is_writer, uint32_t md_pages_reserved, const char *md_file_path, size_t pbuf_size); + +/*------------------------------------------------------------------------- + * Function: init_vfd_swmr_config_fapl + * + * Purpose: Helper routine to initialize the fields for VFD SWMR configuration + * + * Return: void + * + *------------------------------------------------------------------------- + */ +static hid_t +init_vfd_swmr_config_fapl(H5F_vfd_swmr_config_t *config, uint32_t tick_len, uint32_t max_lag, + hbool_t is_writer, uint32_t md_pages_reserved, const char *md_file_path, size_t pbuf_size) +{ + hid_t fapl; + + HDmemset(config, 0, sizeof(*config)); + + config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION; + config->tick_len = tick_len; + config->max_lag = max_lag; + config->writer = is_writer; + config->md_pages_reserved = md_pages_reserved; + HDstrcpy(config->md_file_path, md_file_path); + + /* Create a copy of the file access property list */ + if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + return H5I_INVALID_HID; + + if(H5Pset_vfd_swmr_config(fapl, config) < 0) { + (void)H5Pclose(fapl); + return H5I_INVALID_HID; + } + + /* Enable page buffering */ + if(pbuf_size != 0 && H5Pset_page_buffer_size(fapl, pbuf_size, 0, 0) < 0) { + (void)H5Pclose(fapl); + return H5I_INVALID_HID; + } + + return fapl; +} /* init_vfd_swmr_config_fapl() */ + + +/*------------------------------------------------------------------------- + * Function: test_fapl() + * + * Purpose: A) Verify that invalid info set in the fapl fails + * as expected (see the RFC for VFD SWMR): + * --version: should be a known version + * --tick_len: should be >= 0 + * --max_lag: should be >= 3 + * --md_pages_reserved: should be >= 2 + * --md_file_path: should contain the metadata file path (POSIX) + * B) Verify that info set in the fapl is retrieved correctly. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: Vailin Choi; July 2018 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_fapl(void) +{ + hid_t fapl = -1; /* File access property list */ + H5F_vfd_swmr_config_t *my_config = NULL; /* Configuration for VFD SWMR */ + herr_t ret; /* Return value */ + + TESTING("Configure VFD SWMR with fapl"); + + /* Allocate memory for the configuration structure */ + if((my_config = HDmalloc(sizeof(*my_config))) == NULL) + FAIL_STACK_ERROR; + HDmemset(my_config, 0, sizeof(*my_config)); + + /* Get a copy of the file access property list */ + if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + + /* Should get invalid VFD SWMR config info */ + if(H5Pget_vfd_swmr_config(fapl, my_config) < 0) + TEST_ERROR; + + /* Verify that the version is incorrect */ + if(my_config->version >= H5F__CURR_VFD_SWMR_CONFIG_VERSION) + TEST_ERROR; + + /* Should fail: version is 0 */ + H5E_BEGIN_TRY { + ret = H5Pset_vfd_swmr_config(fapl, my_config); + } H5E_END_TRY; + if(ret >= 0) + TEST_ERROR; + + /* Set valid version */ + my_config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION; + + /* Set valid tick_len */ + my_config->tick_len = 3; + /* Should fail: max_lag is 2 */ + my_config->max_lag = 2; + H5E_BEGIN_TRY { + ret = H5Pset_vfd_swmr_config(fapl, my_config); + } H5E_END_TRY; + if(ret >= 0) + TEST_ERROR; + + /* Set valid max_lag */ + my_config->max_lag = 3; + /* Should fail: md_pages_reserved is 0 */ + H5E_BEGIN_TRY { + ret = H5Pset_vfd_swmr_config(fapl, my_config); + } H5E_END_TRY; + if(ret >= 0) + TEST_ERROR; + + /* Set valid md_pages_reserved */ + my_config->md_pages_reserved = 2; + /* Should fail: empty md_file_path */ + H5E_BEGIN_TRY { + ret = H5Pset_vfd_swmr_config(fapl, my_config); + } H5E_END_TRY; + if(ret >= 0) + TEST_ERROR; + + /* Set md_file_path */ + HDstrcpy(my_config->md_file_path, MD_FILENAME); + my_config->writer = TRUE; + + /* Should succeed in setting the configuration info */ + if(H5Pset_vfd_swmr_config(fapl, my_config) < 0) + TEST_ERROR; + + /* Clear the configuration structure */ + HDmemset(my_config, 0, sizeof(H5F_vfd_swmr_config_t)); + + /* Retrieve the configuration info just set */ + if(H5Pget_vfd_swmr_config(fapl, my_config) < 0) + TEST_ERROR; + + /* Verify the configuration info */ + if(my_config->version < H5F__CURR_VFD_SWMR_CONFIG_VERSION) + TEST_ERROR; + if(my_config->md_pages_reserved != 2) + TEST_ERROR; + if(HDstrcmp(my_config->md_file_path, MD_FILENAME) != 0) + TEST_ERROR; + + /* Close the file access property list */ + if(H5Pclose(fapl) < 0) + FAIL_STACK_ERROR; + + if(my_config) + HDfree(my_config); + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(fapl); + } H5E_END_TRY; + if(my_config) + HDfree(my_config); + return 1; +} /* test_fapl() */ + + +/*------------------------------------------------------------------------- + * Function: test_file_fapl() + * + * Purpose: A) Verify that page buffering and paged aggregation + * have to be enabled for a file to be configured + * with VFD SWMR. + * B) Verify that the "writer" setting in the fapl's VFD + * SWMR configuration should be consistent with the + * file access flags. + * C) Verify the VFD SWMR configuration set in fapl + * used to create/open the file is the same as the + * configuration retrieved from the file's fapl. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: Vailin Choi; July 2018 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_file_fapl(void) +{ + hid_t fid = -1; /* File ID */ + hid_t fid2 = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list ID */ + hid_t fapl1 = -1; /* File access property list ID */ + hid_t fapl2 = -1; /* File access property list ID */ + hid_t file_fapl = -1; /* File access property list ID associated with the file */ + H5F_vfd_swmr_config_t *config1 = NULL; /* Configuration for VFD SWMR */ + H5F_vfd_swmr_config_t *config2 = NULL; /* Configuration for VFD SWMR */ + H5F_vfd_swmr_config_t *file_config = NULL; /* Configuration for VFD SWMR */ + + TESTING("VFD SWMR configuration for the file and fapl"); + + /* Should succeed without VFD SWMR configured */ + if((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Close the file */ + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + + /* Allocate memory for the configuration structure */ + if((config1 = (H5F_vfd_swmr_config_t *)HDmalloc(sizeof(H5F_vfd_swmr_config_t))) == NULL) + FAIL_STACK_ERROR; + if((config2 = (H5F_vfd_swmr_config_t *)HDmalloc(sizeof(H5F_vfd_swmr_config_t))) == NULL) + FAIL_STACK_ERROR; + if((file_config = (H5F_vfd_swmr_config_t *)HDmalloc(sizeof(H5F_vfd_swmr_config_t))) == NULL) + FAIL_STACK_ERROR; + + /* Configured as VFD SWMR reader + no page buffering */ + fapl1 = init_vfd_swmr_config_fapl(config1, 4, 6, FALSE, 2, MD_FILENAME, 0); + if(fapl1 == H5I_INVALID_HID) + TEST_ERROR + + /* Should fail to create: file access is writer but VFD SWMR config is reader */ + H5E_BEGIN_TRY { + fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, fapl1); + } H5E_END_TRY; + if(fid >= 0) + TEST_ERROR; + + if(H5Pclose(fapl1) < 0) + FAIL_STACK_ERROR + + /* Configured as VFD SWMR writer + no page buffering */ + fapl1 = init_vfd_swmr_config_fapl(config1, 4, 6, TRUE, 2, MD_FILENAME, 0); + if(fapl1 == H5I_INVALID_HID) + TEST_ERROR + + /* Should fail to create: page buffering and paged aggregation not enabled */ + H5E_BEGIN_TRY { + fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, fapl1); + } H5E_END_TRY; + if(fid >= 0) + TEST_ERROR; + + /* Create a copy of the file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set file space strategy */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0) + FAIL_STACK_ERROR; + + /* Should fail to create: no page buffering */ + H5E_BEGIN_TRY { + fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl1); + } H5E_END_TRY; + if(fid >= 0) + TEST_ERROR; + + if(H5Pclose(fapl1) < 0) + FAIL_STACK_ERROR + + /* Configured as VFD SWMR writer + page buffering */ + fapl1 = init_vfd_swmr_config_fapl(config1, 4, 6, TRUE, 2, MD_FILENAME, 4096); + if(fapl1 == H5I_INVALID_HID) + TEST_ERROR + + /* Should succeed to create the file: paged aggregation and page buffering enabled */ + if((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl1)) < 0) + TEST_ERROR; + + /* Get the file's file access property list */ + if((file_fapl = H5Fget_access_plist(fid)) < 0) + FAIL_STACK_ERROR; + + /* Retrieve the VFD SWMR configuration from file_fapl */ + if(H5Pget_vfd_swmr_config(file_fapl, file_config) < 0) + TEST_ERROR; + + /* Verify the retrieved info is the same as config1 */ + if(HDmemcmp(config1, file_config, sizeof(H5F_vfd_swmr_config_t)) != 0) + TEST_ERROR; + + /* Closing */ + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(file_fapl) < 0) + FAIL_STACK_ERROR; + + /* Should fail to open: file access is reader but VFD SWMR config is writer */ + H5E_BEGIN_TRY { + fid = H5Fopen(FILENAME, H5F_ACC_RDONLY, fapl1); + } H5E_END_TRY; + if(fid >= 0) + TEST_ERROR; + + /* Should succeed to open: file access and VFD SWMR config are consistent */ + if((fid = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl1)) < 0) + TEST_ERROR; + + /* Get the file's file access property list */ + if((file_fapl = H5Fget_access_plist(fid)) < 0) + FAIL_STACK_ERROR; + + /* Clear info in file_config */ + HDmemset(file_config, 0, sizeof(H5F_vfd_swmr_config_t)); + + /* Retrieve the VFD SWMR configuration from file_fapl */ + if(H5Pget_vfd_swmr_config(file_fapl, file_config) < 0) + TEST_ERROR; + + /* Verify the retrieved info is the same as config1 */ + if(HDmemcmp(config1, file_config, sizeof(H5F_vfd_swmr_config_t)) != 0) + TEST_ERROR; + + /* Closing */ + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(file_fapl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fapl1) < 0) + FAIL_STACK_ERROR; + + /* Set up different VFD SWMR configuration + page_buffering */ + fapl2 = init_vfd_swmr_config_fapl(config2, 4, 10, TRUE, 2, MD_FILENAME, 4096); + if(fapl2 == H5I_INVALID_HID) + TEST_ERROR + + /* Should succeed to open the file as VFD SWMR writer */ + if((fid = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl2)) < 0) + TEST_ERROR; + + /* Get the file's file access property list */ + if((file_fapl = H5Fget_access_plist(fid)) < 0) + FAIL_STACK_ERROR; + + /* Clear info in file_config */ + HDmemset(file_config, 0, sizeof(H5F_vfd_swmr_config_t)); + + /* Retrieve the VFD SWMR configuration from file_fapl */ + if(H5Pget_vfd_swmr_config(file_fapl, file_config) < 0) + TEST_ERROR; + + /* Verify the retrieved info is NOT the same as config1 */ + if(HDmemcmp(config1, file_config, sizeof(H5F_vfd_swmr_config_t)) == 0) + TEST_ERROR; + + /* Verify the retrieved info is the same as config2 */ + if(HDmemcmp(config2, file_config, sizeof(H5F_vfd_swmr_config_t)) != 0) + TEST_ERROR; + + /* The file previously opened as VDF SWMR writer is still open */ + /* with VFD SWMR configuration in config2 */ + + /* Set up as VFD SWMR writer in config1 but different from config2 */ + fapl1 = init_vfd_swmr_config_fapl(config1, 3, 8, TRUE, 3, MD_FILENAME, 4096); + if(fapl1 == H5I_INVALID_HID) + TEST_ERROR; + + /* Re-open the same file with config1 */ + /* Should fail to open since config1 is different from config2 setting */ + H5E_BEGIN_TRY { + fid2 = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl1); + } H5E_END_TRY; + if(fid2 >= 0) + TEST_ERROR; + + /* Close fapl1 */ + if(H5Pclose(fapl1) < 0) + FAIL_STACK_ERROR; + + /* Set up as VFD SWMR reader in config1 which is same as config2 */ + fapl1 = init_vfd_swmr_config_fapl(config1, 4, 10, TRUE, 2, MD_FILENAME, 4096); + if(fapl1 == H5I_INVALID_HID) + TEST_ERROR; + + /* Re-open the same file as VFD SWMR writer */ + /* Should succeed since config1 is same as the setting in config2 */ + if((fid2 = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl1)) < 0) + TEST_ERROR + + /* Close fapl1 */ + if(H5Pclose(fapl1) < 0) + FAIL_STACK_ERROR; + + HDmemset(file_config, 0, sizeof(H5F_vfd_swmr_config_t)); + + /* Get the file's file access property list */ + if((file_fapl = H5Fget_access_plist(fid)) < 0) + FAIL_STACK_ERROR; + + /* Retrieve the VFD SWMR configuration from file_fapl */ + if(H5Pget_vfd_swmr_config(file_fapl, file_config) < 0) + TEST_ERROR; + + /* Should be the same as config1 */ + if(HDmemcmp(config1, file_config, sizeof(H5F_vfd_swmr_config_t)) != 0) + TEST_ERROR; + + /* Should be the the same as config2 */ + if(HDmemcmp(config2, file_config, sizeof(H5F_vfd_swmr_config_t)) != 0) + TEST_ERROR; + + /* Closing */ + if(H5Fclose(fid2) < 0) + FAIL_STACK_ERROR; + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + + if(H5Pclose(fapl2) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(file_fapl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + + /* Free buffers */ + if(config1) + HDfree(config1); + if(config2) + HDfree(config2); + if(file_config) + HDfree(file_config); + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(fapl1); + H5Pclose(fapl2); + H5Pclose(file_fapl); + H5Pclose(fcpl); + H5Fclose(fid); + H5Fclose(fid2); + } H5E_END_TRY; + if(config1) + HDfree(config1); + if(config2) + HDfree(config2); + if(file_config) + HDfree(file_config); + return 1; +} /* test_file_fapl() */ + + +/*------------------------------------------------------------------------- + * Function: test_file_end_tick() + * + * Purpose: Verify the public routine H5Fvfd_swmr_end_tick() works + * as described in the RFC for VFD SWMR. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: Vailin Choi; June 2020 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_file_end_tick(void) +{ + hid_t fid1 = H5I_INVALID_HID; /* File ID */ + hid_t fid2 = H5I_INVALID_HID; /* File ID */ + hid_t fid3 = H5I_INVALID_HID; /* File ID */ + hid_t fcpl = H5I_INVALID_HID; /* File creation property list ID */ + hid_t fapl1 = H5I_INVALID_HID; /* File access property list ID */ + hid_t fapl2 = H5I_INVALID_HID; /* File access property list ID */ + hid_t fapl3 = H5I_INVALID_HID; /* File access property list ID */ + H5F_vfd_swmr_config_t *config1 = NULL; /* Configuration for VFD SWMR */ + H5F_vfd_swmr_config_t *config2 = NULL; /* Configuration for VFD SWMR */ + H5F_vfd_swmr_config_t *config3 = NULL; /* Configuration for VFD SWMR */ + H5F_t *f1, *f2, *f3; /* File pointer */ + uint64_t s1 = 0; /* Saved tick_num */ + uint64_t s2 = 0; /* Saved tick_num */ + uint64_t s3 = 0; /* Saved tick_num */ + int ret; /* Return status */ + + TESTING("H5Fvfd_swmr_end_tick()"); + + /* Create a file without VFD SWMR configured */ + if((fid1 = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Should fail to trigger EOT */ + H5E_BEGIN_TRY { + ret = H5Fvfd_swmr_end_tick(fid1); + } H5E_END_TRY; + if(ret >= 0) + TEST_ERROR; + + /* Close the file */ + if(H5Fclose(fid1) < 0) + FAIL_STACK_ERROR; + + /* Allocate memory for the configuration structure */ + if((config1 = HDmalloc(sizeof(*config1))) == NULL) + FAIL_STACK_ERROR; + if((config2 = HDmalloc(sizeof(*config2))) == NULL) + FAIL_STACK_ERROR; + if((config3 = HDmalloc(sizeof(*config3))) == NULL) + FAIL_STACK_ERROR; + + /* Configured file 1 as VFD SWMR writer + page buffering */ + fapl1 = init_vfd_swmr_config_fapl(config1, 10, 15, TRUE, 2, MD_FILENAME, 4096); + if(fapl1 == H5I_INVALID_HID) + FAIL_STACK_ERROR; + + /* Configured file 2 as VFD SWMR writer + page buffering */ + fapl2 = init_vfd_swmr_config_fapl(config2, 5, 6, TRUE, 2, MD_FILENAME2, 4096); + if(fapl2 == H5I_INVALID_HID) + FAIL_STACK_ERROR; + + /* Configured file 3 as VFD SWMR writer + page buffering */ + fapl3 = init_vfd_swmr_config_fapl(config3, 3, 6, TRUE, 2, MD_FILENAME3, 4096); + if(fapl3 == H5I_INVALID_HID) + FAIL_STACK_ERROR; + + /* Create a copy of the file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set file space strategy */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0) + FAIL_STACK_ERROR; + + /* Create file 1 with VFD SWMR writer */ + if((fid1 = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl1)) < 0) + TEST_ERROR; + /* Keep file 1 opened */ + + /* Create file 2 with VFD SWMR writer */ + if((fid2 = H5Fcreate(FILENAME2, H5F_ACC_TRUNC, fcpl, fapl2)) < 0) + TEST_ERROR; + if(H5Fclose(fid2) < 0) + FAIL_STACK_ERROR; + + /* Create file 3 with VFD SWMR writer */ + if((fid3 = H5Fcreate(FILENAME3, H5F_ACC_TRUNC, fcpl, fapl3)) < 0) + TEST_ERROR; + if(H5Fclose(fid3) < 0) + FAIL_STACK_ERROR; + + /* Open file 2 as VFD SWMR writer */ + if((fid2 = H5Fopen(FILENAME2, H5F_ACC_RDWR, fapl2)) < 0) + TEST_ERROR; + + /* Open file 3 as VFD SWMR writer */ + if((fid3 = H5Fopen(FILENAME3, H5F_ACC_RDWR, fapl3)) < 0) + TEST_ERROR; + + /* Get file pointer for the 3 files */ + f1 = H5VL_object(fid1); + f2 = H5VL_object(fid2); + f3 = H5VL_object(fid3); + + /* Saved tick_num for the 3 files */ + s1 = f1->shared->tick_num; + s2 = f2->shared->tick_num; + s3 = f3->shared->tick_num; + + /* Trigger EOT for file 2 */ + if(H5Fvfd_swmr_end_tick(fid2) < 0) + TEST_ERROR; + + /* file 2: tick_num should increase or at least same as previous tick_num */ + if(f2->shared->tick_num < s2) + TEST_ERROR; + + /* Disable EOT for file 2 */ + if(H5Fvfd_swmr_disable_end_of_tick(fid2) < 0) + TEST_ERROR; + + /* Should fail to trigger end of tick processing for file 2 */ + H5E_BEGIN_TRY { + ret = H5Fvfd_swmr_end_tick(fid2); + } H5E_END_TRY; + if(ret >= 0) + TEST_ERROR; + + if(H5Fclose(fid2) < 0) + FAIL_STACK_ERROR; + + /* Trigger EOT for file 1 */ + if(H5Fvfd_swmr_end_tick(fid1) < 0) + TEST_ERROR; + + /* file 1: tick_num should increase or at least same as previous tick_num */ + if(f1->shared->tick_num < s1) + TEST_ERROR; + + if(H5Fclose(fid1) < 0) + FAIL_STACK_ERROR; + + + /* Trigger EOT for file 3 */ + if(H5Fvfd_swmr_end_tick(fid3) < 0) + TEST_ERROR; + + /* file 3: tick_num should increase or at least same as previous tick_num */ + if(f3->shared->tick_num < s3) + TEST_ERROR; + + if(H5Fclose(fid3) < 0) + FAIL_STACK_ERROR; + + + /* Closing */ + if(H5Pclose(fapl1) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fapl2) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fapl3) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + + /* Free buffers */ + if(config1) + HDfree(config1); + if(config2) + HDfree(config2); + if(config3) + HDfree(config3); + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(fapl1); + H5Pclose(fapl2); + H5Pclose(fapl3); + H5Pclose(fcpl); + H5Fclose(fid1); + H5Fclose(fid2); + H5Fclose(fid3); + } H5E_END_TRY; + + if(config1) + HDfree(config1); + if(config2) + HDfree(config2); + if(config3) + HDfree(config3); + + return 1; +} /* test_file_end_tick() */ + + +/*------------------------------------------------------------------------- + * Function: test_writer_create_open_flush() + * + * Purpose: Verify info in the metadata file when: + * --creating the HDF5 file + * --flushing the HDF5 file + * --opening an existing HDF5 file + * It will call the internal testing routine + * H5F__vfd_swmr_writer_create_open_flush_test() to do the following: + * --Open the metadata file + * --Verify the file size is as expected (md_pages_reserved) + * --For file create: + * --No header magic is found + * --For file open or file flush: + * --Read and decode the header and index in the metadata file + * --Verify info in the header and index read from + * the metadata file is as expected (empty index) + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: Vailin Choi; October 2018 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_writer_create_open_flush(void) +{ + hid_t fid = -1; /* File ID */ + hid_t fapl = -1; /* File access property list */ + hid_t fcpl = -1; /* File creation property list */ + H5F_vfd_swmr_config_t *my_config = NULL; /* Configuration for VFD SWMR */ + + TESTING("Create/Open/Flush an HDF5 file for VFD SWMR"); + + /* Allocate memory for the configuration structure */ + if((my_config = HDmalloc(sizeof(H5F_vfd_swmr_config_t))) == NULL) + FAIL_STACK_ERROR; + + /* Set up the VFD SWMR configuration + page buffering */ + fapl = init_vfd_swmr_config_fapl(my_config, 1, 3, TRUE, 2, MD_FILENAME, 4096); + if(fapl == H5I_INVALID_HID) + FAIL_STACK_ERROR; + + /* Create a copy of the file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set file space strategy */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0) + FAIL_STACK_ERROR; + + /* Create an HDF5 file with VFD SWMR configured */ + if((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR; + + /* Verify info in metadata file when creating the HDF5 file */ + if(H5F__vfd_swmr_writer_create_open_flush_test(fid, TRUE) < 0) + FAIL_STACK_ERROR; + +#ifdef LATER /* Will activate the test when flush is implemented */ + /* Flush the HDF5 file */ + if(H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0) + FAIL_STACK_ERROR + + /* Verify info in metadata file when flushing the HDF5 file */ + if(H5F__vfd_swmr_writer_create_open_flush_test(fid, FALSE) < 0) + FAIL_STACK_ERROR +#endif + + /* Close the file */ + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + + /* Re-open the file as VFD SWMR writer */ + if((fid = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl)) < 0) + FAIL_STACK_ERROR; + + /* Verify info in metadata file when reopening the HDF5 file */ + if(H5F__vfd_swmr_writer_create_open_flush_test(fid, FALSE) < 0) + FAIL_STACK_ERROR; + + /* Closing */ + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fapl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + + if(my_config) + HDfree(my_config); + + PASSED(); + return 0; + +error: + if(my_config) + HDfree(my_config); + + H5E_BEGIN_TRY { + H5Pclose(fapl); + H5Pclose(fcpl); + H5Fclose(fid); + } H5E_END_TRY; + + return 1; +} /* test_writer_create_open_flush() */ + +/* Sleep for `tenths` tenths of a second. + * + * This routine may quietly perform a too-short sleep if an error occurs + * in nanosleep(2). + */ +static void +decisleep(uint32_t tenths) +{ + struct timespec delay = {.tv_sec = tenths / 10, + .tv_nsec = tenths * 100 * 1000 * 1000}; + + while (nanosleep(&delay, &delay) == -1 && errno == EINTR) + ; // do nothing +} + + +/*------------------------------------------------------------------------- + * Function: test_writer_md() + * + * Purpose: Verify info in the metadata file after updating with the + * constructed index: (A), (B), (C), (D) + * It will call the internal testing routine + * H5F__vfd_swmr_writer_md_test() to do the following: + * --Update the metadata file with the input index via the + * internal library routine H5F_update_vfd_swmr_metadata_file() + * --Verify the entries in the delayed list is as expected: + * --num_dl_entries + * --Open the metadata file, read and decode the header and index + * --Verify header and index info just read from the metadata + * file is as expected: + * --num_entries and index + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: Vailin Choi; October 2018 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_writer_md(void) +{ + hid_t fid = -1; /* File ID */ + hid_t fapl = -1; /* File access property list */ + hid_t fcpl = -1; /* File creation property list */ + const unsigned num_entries = 10; /* index size */ + unsigned i = 0; /* Local index variables */ + uint8_t *buf = NULL; /* Data page from the page buffer */ + hid_t dcpl = -1; /* Dataset creation property list */ + hid_t sid = -1; /* Dataspace ID */ + hid_t did = -1; /* Dataset ID */ + int *rwbuf = NULL; /* Data buffer for writing */ + H5O_info_t oinfo; /* Object metadata information */ + char dname[100]; /* Name of dataset */ + hsize_t dims[2] = {50, 20}; /* Dataset dimension sizes */ + hsize_t max_dims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; /* Dataset maximum dimension sizes */ + hsize_t chunk_dims[2] = {2, 5}; /* Dataset chunked dimension sizes */ + H5FD_vfd_swmr_idx_entry_t *index = NULL; /* Pointer to the index entries */ + H5F_vfd_swmr_config_t *my_config = NULL; /* Configuration for VFD SWMR */ + + TESTING("Verify the metadata file for VFD SWMR writer"); + + /* Allocate memory for the configuration structure */ + if((my_config = HDmalloc(sizeof(H5F_vfd_swmr_config_t))) == NULL) + FAIL_STACK_ERROR; + + /* Set up the VFD SWMR configuration + page buffering */ + fapl = init_vfd_swmr_config_fapl(my_config, 1, 3, TRUE, 256, MD_FILENAME, FS_PAGE_SIZE); + if(fapl == H5I_INVALID_HID) + FAIL_STACK_ERROR + + /* Create a copy of the file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set file space strategy */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0) + FAIL_STACK_ERROR; + if(H5Pset_file_space_page_size(fcpl, FS_PAGE_SIZE) < 0) + FAIL_STACK_ERROR; + + /* Create an HDF5 file with VFD SWMR configured */ + if((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR; + + /* Allocate num_entries for the data buffer */ + if((buf = HDcalloc(num_entries, FS_PAGE_SIZE)) == NULL) + FAIL_STACK_ERROR; + + /* Allocate memory for num_entries index */ + index = HDcalloc(num_entries, sizeof(H5FD_vfd_swmr_idx_entry_t)); + if(NULL == index) + FAIL_STACK_ERROR; + + /* (A) Construct index for updating the metadata file */ + for(i = 0; i < num_entries; i++) { + index[i].hdf5_page_offset = 3 + 7 * i; + index[i].md_file_page_offset = 1 + (num_entries - i) * 5; + index[i].length = (uint32_t)FS_PAGE_SIZE; + index[i].entry_ptr = &buf[i * FS_PAGE_SIZE]; + } + + /* Update with index and verify info in the metadata file */ + /* Also verify that 0 entries will be on the delayed list */ + if(H5F__vfd_swmr_writer_md_test(fid, num_entries, index, 0) < 0) + TEST_ERROR + + /* Create dataset creation property list */ + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set to use chunked dataset */ + if(H5Pset_chunk(dcpl, 2, chunk_dims) < 0) + FAIL_STACK_ERROR + + /* Create dataspace */ + if((sid = H5Screate_simple(2, dims, max_dims)) < 0) + FAIL_STACK_ERROR + + /* Perform activities to ensure that max_lag ticks elapse */ + for(i = 0; i < my_config->max_lag + 1; i++) { + decisleep(my_config->tick_len); + + /* Create a chunked dataset */ + sprintf(dname, "dset %d", i); + if((did = H5Dcreate2(fid, dname, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR + + /* Get dataset object header address */ + if(H5Oget_info(did, &oinfo, H5O_INFO_BASIC) < 0) + FAIL_STACK_ERROR + + /* Close the dataset */ + if(H5Dclose(did) < 0) + FAIL_STACK_ERROR + } + + /* (B) Update every other entry in the index */ + for(i = 0; i < num_entries; i+= 2) + index[i].entry_ptr = &buf[i * FS_PAGE_SIZE]; + + /* Update with index and verify info in the metadata file */ + /* Also verify that 5 entries will be on the delayed list */ + if(H5F__vfd_swmr_writer_md_test(fid, num_entries, index, 5) < 0) + TEST_ERROR + + /* Allocate memory for the read/write buffer */ + if((rwbuf = HDmalloc(sizeof(*rwbuf) * (50 * 20))) == NULL) + FAIL_STACK_ERROR; + for(i = 0; i < (50 * 20); i++) + rwbuf[i] = (int)i; + + /* Perform activities to ensure that max_lag ticks elapse */ + for(i = 0; i < my_config->max_lag + 1; i++) { + decisleep(my_config->tick_len); + + /* Open the dataset */ + sprintf(dname, "dset %d", i); + if((did = H5Dopen2(fid, dname, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR + + /* Write to the dataset */ + if(H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rwbuf) < 0) + FAIL_STACK_ERROR + + /* Get dataset object info */ + if(H5Oget_info(did, &oinfo, H5O_INFO_BASIC) < 0) + FAIL_STACK_ERROR + + /* Close the dataset */ + if(H5Dclose(did) < 0) + FAIL_STACK_ERROR + } + + /* (C) Update every 3 entry in the index */ + for(i = 0; i < num_entries; i+= 3) + index[i].entry_ptr = &buf[i * FS_PAGE_SIZE]; + + /* Update with index and verify info in the metadata file */ + /* Also verify that 4 entries will be on the delayed list */ + if(H5F__vfd_swmr_writer_md_test(fid, num_entries, index, 4) < 0) + TEST_ERROR + + /* Clear the read/write buffer */ + HDmemset(rwbuf, 0, sizeof(sizeof(int) * (50 * 20))); + + /* Perform activities to ensure that max_lag ticks elapse */ + for(i = 0; i < my_config->max_lag + 1; i++) { + decisleep(my_config->tick_len); + + /* Open the dataset */ + sprintf(dname, "dset %d", i); + if((did = H5Dopen2(fid, dname, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR + + /* Read from the dataset */ + if(H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rwbuf) < 0) + FAIL_STACK_ERROR + + /* Get dataset object info */ + if(H5Oget_info(did, &oinfo, H5O_INFO_BASIC) < 0) + FAIL_STACK_ERROR + + /* Close the dataset */ + if(H5Dclose(did) < 0) + FAIL_STACK_ERROR + } + + /* (D) Update two entries in the index */ + index[1].entry_ptr = &buf[1 * FS_PAGE_SIZE]; + index[5].entry_ptr = &buf[5 * FS_PAGE_SIZE]; + + /* Update with index and verify info in the metadata file */ + /* Also verify that 2 entries will be on the delayed list */ + if(H5F__vfd_swmr_writer_md_test(fid, num_entries, index, 2) < 0) + TEST_ERROR + + /* Closing */ + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + if(H5Sclose(sid) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fapl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + + /* Free resources */ + if(my_config) + HDfree(my_config); + if(buf) + HDfree(buf); + if(rwbuf) + HDfree(rwbuf); + if(index) + HDfree(index); + + PASSED(); + return 0; + +error: + if(my_config) + HDfree(my_config); + if(buf) + HDfree(buf); + if(rwbuf) + HDfree(rwbuf); + if(index) + HDfree(index); + + H5E_BEGIN_TRY { + H5Dclose(did); + H5Sclose(sid); + H5Pclose(dcpl); + H5Pclose(fapl); + H5Pclose(fcpl); + H5Fclose(fid); + } H5E_END_TRY; + + return 1; +} /* test_writer__md() */ + + +#if !(defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID) && defined(H5_HAVE_FLOCK)) + +static unsigned +test_reader_md_concur(void) +{ + /* Output message about test being performed */ + TESTING("Verify the metadata file for VFD SWMR reader"); + SKIPPED(); + HDputs(" Test skipped due to fork, waitpid, or flock not defined."); + return 0; + +} /* test_reader_md_concur() */ + +static unsigned +test_multiple_file_opens_concur(void) +{ + /* Output message about test being performed */ + TESTING("EOT queue entries when opening files concurrently with VFD SWMR"); + SKIPPED(); + HDputs(" Test skipped due to fork, waitpid, or flock not defined."); + return 0; + +} /* test_multiple_file_opens_concur() */ + +static unsigned +test_disable_enable_eot_concur(void) + + /* Output message about test being performed */ + TESTING("Verify concurrent H5Fvfd_swmr_enable/disable_end_of_tick()"); + SKIPPED(); + HDputs(" Test skipped due to fork, waitpid, or flock not defined."); + return 0; + +} /* test_disable_enble_eot_concur() */ + +#else /* defined(H5_HAVE_FORK && defined(H5_HAVE_WAITPID) && defined(H5_HAVE_FLOCK) */ + +/*------------------------------------------------------------------------- + * Function: test_reader_md_concur() + * + * Purpose: Verify metadata file info updated by the writer is + * what the reader obtained from the metadata file: + * --Cases (A), (B), (C), (D), (E) + * NOTE: Changes for page buffering/cache are not in place yet. + * Index entries are constructed at the front end by the + * writer and verified at the back end by the reader. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: Vailin Choi; October 2018 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_reader_md_concur(void) +{ + unsigned i = 0; /* Local index variables */ + uint8_t *buf = NULL; /* Data page from the page buffer */ + hid_t dcpl = -1; /* Dataset creation property list */ + hid_t sid = -1; /* Dataspace ID */ + hid_t did = -1; /* Dataset ID */ + int *rwbuf = NULL; /* Data buffer for writing */ + H5O_info_t oinfo; /* Object metadata information */ + char dname[100]; /* Name of dataset */ + hsize_t dims[2] = {50, 20}; /* Dataset dimension sizes */ + hsize_t max_dims[2] = /* Dataset maximum dimension sizes */ + {H5S_UNLIMITED, H5S_UNLIMITED}; + hsize_t chunk_dims[2] = {2, 5}; /* Dataset chunked dimension sizes */ + unsigned num_entries = 0 ; /* Number of entries in the index */ + H5FD_vfd_swmr_idx_entry_t *index = NULL; /* Pointer to the index entries */ + + hid_t fcpl = -1; /* File creation property list */ + hid_t fid_writer = -1; /* File ID for writer */ + hid_t fapl_writer = -1; /* File access property list for writer */ + H5F_vfd_swmr_config_t *config_writer = NULL; /* VFD SWMR Configuration for writer */ + pid_t tmppid; /* Child process ID returned by waitpid */ + pid_t childpid = 0; /* Child process ID */ + int child_status; /* Status passed to waitpid */ + int child_wait_option=0; /* Options passed to waitpid */ + int child_exit_val; /* Exit status of the child */ + + int parent_pfd[2]; /* Pipe for parent process as writer */ + int child_pfd[2]; /* Pipe for child process as reader */ + int notify = 0; /* Notification between parent and child */ + H5F_t *file_writer; /* File pointer for writer */ + + TESTING("Verify the metadata file for VFD SWMR reader"); + + /* Allocate memory for the configuration structure */ + if((config_writer = HDmalloc(sizeof(*config_writer))) == NULL) + FAIL_STACK_ERROR; + + /* Set up the VFD SWMR configuration + page buffering */ + fapl_writer = init_vfd_swmr_config_fapl(config_writer, 1, 3, TRUE, 256, MD_FILENAME, FS_PAGE_SIZE); + if(fapl_writer == H5I_INVALID_HID) + FAIL_STACK_ERROR; + + /* Create a copy of the file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set file space strategy */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0) + FAIL_STACK_ERROR; + + if(H5Pset_file_space_page_size(fcpl, FS_PAGE_SIZE) < 0) + FAIL_STACK_ERROR; + + /* Create an HDF5 file with VFD SWMR configured */ + if((fid_writer = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl_writer)) < 0) + FAIL_STACK_ERROR; + + /* Close the file */ + if(H5Fclose(fid_writer) < 0) + FAIL_STACK_ERROR; + + /* Create 2 pipes */ + if(HDpipe(parent_pfd) < 0) + FAIL_STACK_ERROR + + if(HDpipe(child_pfd) < 0) + FAIL_STACK_ERROR + + /* Fork child process */ + if((childpid = HDfork()) < 0) + FAIL_STACK_ERROR + + /* + * Child process as reader + */ + if(childpid == 0) { + int child_notify = 0; /* Notification between child and parent */ + hid_t fid_reader = -1; /* File ID for reader */ + hid_t fapl_reader = -1; /* File access property list for reader */ + H5F_t *file_reader; /* File pointer for reader */ + H5F_vfd_swmr_config_t *config_reader = NULL; /* VFD SWMR configuration for reader */ + unsigned child_num_entries = 0; /* Number of entries passed to reader */ + H5FD_vfd_swmr_idx_entry_t *child_index = NULL; /* Index passed to reader */ + + /* Close unused write end for writer pipe */ + if(HDclose(parent_pfd[1]) < 0) + HDexit(EXIT_FAILURE); + + /* Close unused read end for reader pipe */ + if(HDclose(child_pfd[0]) < 0) + HDexit(EXIT_FAILURE); + + /* + * Case A: reader + * --verify an empty index + */ + + /* Wait for notification 1 from parent to start verification */ + while(child_notify != 1) { + if(HDread(parent_pfd[0], &child_notify, sizeof(int)) < 0) + HDexit(EXIT_FAILURE); + } + + /* Allocate memory for the configuration structure */ + if((config_reader = HDmalloc(sizeof(*config_reader))) == NULL) + HDexit(EXIT_FAILURE); + + /* Set up the VFD SWMR configuration as reader + page buffering */ + fapl_reader = init_vfd_swmr_config_fapl(config_reader, 1, 3, FALSE, 256, MD_FILENAME, FS_PAGE_SIZE); + if(fapl_reader == H5I_INVALID_HID) + HDexit(EXIT_FAILURE); + + /* Open the test file as reader */ + if((fid_reader = H5Fopen(FILENAME, H5F_ACC_RDONLY, fapl_reader)) < 0) + HDexit(EXIT_FAILURE); + + /* Get file pointer */ + file_reader = H5VL_object(fid_reader); + + /* Read and verify header and an empty index in the metadata file */ + if(H5FD__vfd_swmr_reader_md_test(file_reader->shared->lf, 0, NULL) < 0) + HDexit(EXIT_FAILURE); + + /* Send notification 2 to parent that the verification is complete */ + child_notify = 2; + if(HDwrite(child_pfd[1], &child_notify, sizeof(int)) < 0) + HDexit(EXIT_FAILURE); + + /* + * Case B: reader + * --verify index as sent from writer + */ + + /* Wait for notification 3 from parent to start verification */ + while(child_notify != 3) { + if(HDread(parent_pfd[0], &child_notify, sizeof(int)) < 0) + HDexit(EXIT_FAILURE); + } + + /* Read num_entries from writer pipe */ + if(HDread(parent_pfd[0], &child_num_entries, sizeof(int)) < 0) + HDexit(EXIT_FAILURE); + + /* Free previous index */ + if(child_index) + HDfree(child_index); + + if(child_num_entries) { + + /* Allocate memory for num_entries index */ + if((child_index = HDcalloc(child_num_entries, + sizeof(*child_index))) == NULL) + HDexit(EXIT_FAILURE); + + /* Read index from writer pipe */ + if(HDread(parent_pfd[0], child_index, + child_num_entries * sizeof(*child_index)) < 0) + HDexit(EXIT_FAILURE); + } + + /* Read and verify the expected header and index info in the + * metadata file + */ + if(H5FD__vfd_swmr_reader_md_test(file_reader->shared->lf, + child_num_entries, child_index) < 0) + HDexit(EXIT_FAILURE); + + /* Send notification 4 to parent that the verification is complete */ + child_notify = 4; + if(HDwrite(child_pfd[1], &child_notify, sizeof(int)) < 0) + HDexit(EXIT_FAILURE); + + /* + * Case C: reader + * --verify index as sent from writer + */ + + /* Wait for notification 5 from parent to start verification */ + while(child_notify != 5) { + if(HDread(parent_pfd[0], &child_notify, sizeof(int)) < 0) + HDexit(EXIT_FAILURE); + } + + /* Read num_entries from writer pipe */ + if(HDread(parent_pfd[0], &child_num_entries, sizeof(int)) < 0) + HDexit(EXIT_FAILURE); + + /* Free previous index */ + if(child_index) + HDfree(child_index); + + if(child_num_entries) { + /* Allocate memory for num_entries index */ + if((child_index = (H5FD_vfd_swmr_idx_entry_t *) + HDcalloc(child_num_entries, + sizeof(H5FD_vfd_swmr_idx_entry_t))) == NULL) + HDexit(EXIT_FAILURE); + + /* Read index from writer pipe */ + if(HDread(parent_pfd[0], child_index, + child_num_entries * sizeof(H5FD_vfd_swmr_idx_entry_t)) < 0) + HDexit(EXIT_FAILURE); + } + + /* Read and verify the expected header and index info in the + * metadata file + */ + if(H5FD__vfd_swmr_reader_md_test(file_reader->shared->lf, + child_num_entries, child_index) < 0) + HDexit(EXIT_FAILURE); + + /* Send notification 6 to parent that the verification is complete */ + child_notify = 6; + if(HDwrite(child_pfd[1], &child_notify, sizeof(int)) < 0) + HDexit(EXIT_FAILURE); + + /* + * Case D: reader + * --verify index as sent from writer + */ + + /* Wait for notification 7 from parent to start verification */ + while(child_notify != 7) { + + if(HDread(parent_pfd[0], &child_notify, sizeof(int)) < 0) + + HDexit(EXIT_FAILURE); + } + + /* Read num_entries from writer pipe */ + if(HDread(parent_pfd[0], &child_num_entries, sizeof(int)) < 0) + HDexit(EXIT_FAILURE); + + /* Free previous index */ + if(child_index) + HDfree(child_index); + + if(child_num_entries) { + /* Allocate memory for num_entries index */ + if((child_index = (H5FD_vfd_swmr_idx_entry_t *) + HDcalloc(child_num_entries, + sizeof(H5FD_vfd_swmr_idx_entry_t))) == NULL) + HDexit(EXIT_FAILURE); + + /* Read index from writer pipe */ + if(HDread(parent_pfd[0], child_index, + child_num_entries * sizeof(H5FD_vfd_swmr_idx_entry_t)) < 0) + HDexit(EXIT_FAILURE); + } + + /* Read and verify the expected header and index info in the + * metadata file + */ + if(H5FD__vfd_swmr_reader_md_test(file_reader->shared->lf, + child_num_entries, child_index) < 0) + HDexit(EXIT_FAILURE); + + /* Send notification 8 to parent that the verification is complete */ + child_notify = 8; + if(HDwrite(child_pfd[1], &child_notify, sizeof(int)) < 0) + HDexit(EXIT_FAILURE); + + /* + * Case E: reader + * --verify an empty index + */ + + /* Wait for notification 9 from parent to start verification */ + while(child_notify != 9) { + if(HDread(parent_pfd[0], &child_notify, sizeof(int)) < 0) + HDexit(EXIT_FAILURE); + } + + /* Read and verify header and an empty index in the metadata file */ + if(H5FD__vfd_swmr_reader_md_test(file_reader->shared->lf, 0, NULL) < 0) + HDexit(EXIT_FAILURE); + + /* Free resources */ + if(child_index) + HDfree(child_index); + if(config_reader) + HDfree(config_reader); + + /* Closing */ + if(H5Fclose(fid_reader) < 0) + HDexit(EXIT_FAILURE); + if(H5Pclose(fapl_reader) < 0) + HDexit(EXIT_FAILURE); + + /* Close the pipes */ + if(HDclose(parent_pfd[0]) < 0) + HDexit(EXIT_FAILURE); + if(HDclose(child_pfd[1]) < 0) + HDexit(EXIT_FAILURE); + + HDexit(EXIT_SUCCESS); + } /* end child process */ + + /* + * Parent process as writer + */ + + /* Close unused read end for writer pipe */ + if(HDclose(parent_pfd[0]) < 0) + FAIL_STACK_ERROR + + /* Close unused write end for reader pipe */ + if(HDclose(child_pfd[1]) < 0) + FAIL_STACK_ERROR + + /* + * Case A: writer + * --open the file as VFD SWMR writer + */ + + /* Open as VFD SWMR writer */ + if((fid_writer = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl_writer)) < 0) + FAIL_STACK_ERROR; + + /* Send notification 1 to reader to start verfication */ + notify = 1; + if(HDwrite(parent_pfd[1], ¬ify, sizeof(int)) < 0) + FAIL_STACK_ERROR; + + /* + * Case B: writer + * --create datasets to ensure ticks elapse + * --construct 12 entries in the index + * --update the metadata file with the index + */ + + /* Wait for notification 2 from reader that the verifcation is complete */ + while(notify != 2) { + if(HDread(child_pfd[0], ¬ify, sizeof(int)) < 0) + FAIL_STACK_ERROR; + } + + /* Create dataset creation property list */ + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set to use chunked dataset */ + if(H5Pset_chunk(dcpl, 2, chunk_dims) < 0) + FAIL_STACK_ERROR + + /* Create dataspace */ + if((sid = H5Screate_simple(2, dims, max_dims)) < 0) + FAIL_STACK_ERROR + + /* Perform activities to ensure that ticks elapse */ + for(i = 0; i < config_writer->max_lag + 1; i++) { + decisleep(config_writer->tick_len); + + /* Create a chunked dataset */ + sprintf(dname, "dset %d", i); + if((did = H5Dcreate2(fid_writer, dname, H5T_NATIVE_INT, sid, + H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR + + /* Get dataset object header address */ + if(H5Oget_info(did, &oinfo, H5O_INFO_BASIC) < 0) + FAIL_STACK_ERROR + + /* Close the dataset */ + if(H5Dclose(did) < 0) + FAIL_STACK_ERROR + } + + num_entries = 12; + + /* Allocate num_entries for the data buffer */ + if((buf = HDcalloc(num_entries, FS_PAGE_SIZE)) == NULL) + FAIL_STACK_ERROR; + + /* Allocate memory for num_entries index */ + index = HDcalloc(num_entries, sizeof(H5FD_vfd_swmr_idx_entry_t)); + if(NULL == index) + FAIL_STACK_ERROR; + + /* Construct index for updating the metadata file */ + for(i = 0; i < num_entries; i++) { + index[i].hdf5_page_offset = 3 + 7 * i; + index[i].md_file_page_offset = 1 + (num_entries - i) * 5; + index[i].length = (uint32_t)FS_PAGE_SIZE; + index[i].entry_ptr = &buf[i * FS_PAGE_SIZE]; + } + + /* Get the file pointer */ + file_writer = H5VL_object(fid_writer); + + /* Update the metadata file with the index */ + if(H5F_update_vfd_swmr_metadata_file(file_writer, num_entries, index) < 0) + TEST_ERROR; + + /* Send notification 3 to child to start verification */ + notify = 3; + if(HDwrite(parent_pfd[1], ¬ify, sizeof(int)) < 0) + FAIL_STACK_ERROR; + + /* Send num_entries to the reader */ + if(HDwrite(parent_pfd[1], &num_entries, sizeof(int)) < 0) + FAIL_STACK_ERROR; + + /* Send index to the reader */ + if(HDwrite(parent_pfd[1], index, + num_entries * sizeof(H5FD_vfd_swmr_idx_entry_t)) < 0) + FAIL_STACK_ERROR; + + /* + * Case C: writer + * --write to the datasets to ensure ticks elapse + * --update 3 entries in the index + * --update the metadata file with the index + */ + + /* Wait for notification 4 from reader that the verifcation is complete */ + while(notify != 4) { + if(HDread(child_pfd[0], ¬ify, sizeof(int)) < 0) + FAIL_STACK_ERROR; + } + + /* Allocate memory for the read/write buffer */ + if((rwbuf = HDmalloc(sizeof(*rwbuf) * (50 * 20))) == NULL) + FAIL_STACK_ERROR; + for(i = 0; i < (50 * 20); i++) + rwbuf[i] = (int)i; + + /* Perform activities to ensure that max_lag ticks elapse */ + for(i = 0; i < config_writer->max_lag + 1; i++) { + decisleep(config_writer->tick_len); + + /* Open the dataset */ + sprintf(dname, "dset %d", i); + if((did = H5Dopen2(fid_writer, dname, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR + + /* Write to the dataset */ + if(H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, + H5P_DEFAULT, rwbuf) < 0) + FAIL_STACK_ERROR + + /* Close the dataset */ + if(H5Dclose(did) < 0) + FAIL_STACK_ERROR + } + + /* Update 3 entries in the index */ + num_entries = 3; + for(i = 0; i < num_entries; i++) + index[i].entry_ptr = &buf[i * FS_PAGE_SIZE]; + + /* Update the metadata file with the index */ + if(H5F_update_vfd_swmr_metadata_file(file_writer, num_entries, index) < 0) + TEST_ERROR; + + /* Send notification 5 to reader to start verification */ + notify = 5; + if(HDwrite(parent_pfd[1], ¬ify, sizeof(int)) < 0) + FAIL_STACK_ERROR; + + /* Send num_entries to the reader */ + if(HDwrite(parent_pfd[1], &num_entries, sizeof(int)) < 0) + FAIL_STACK_ERROR; + + /* Send index to the reader */ + if(HDwrite(parent_pfd[1], index, + num_entries * sizeof(H5FD_vfd_swmr_idx_entry_t)) < 0) + FAIL_STACK_ERROR; + + /* + * Case D: writer + * --read from the datasets to ensure ticks elapse + * --update 5 entries in the index + * --update the metadata file with the index + */ + + /* Wait for notification 6 from reader that the verifcation is complete */ + while(notify != 6) { + if(HDread(child_pfd[0], ¬ify, sizeof(int)) < 0) + FAIL_STACK_ERROR; + } + + /* Perform activities to ensure that max_lag ticks elapse */ + for(i = 0; i < config_writer->max_lag + 1; i++) { + decisleep(config_writer->tick_len); + + /* Open the dataset */ + sprintf(dname, "dset %d", i); + if((did = H5Dopen2(fid_writer, dname, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR + + /* Read from the dataset */ + if(H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, + H5P_DEFAULT, rwbuf) < 0) + FAIL_STACK_ERROR + + /* Close the dataset */ + if(H5Dclose(did) < 0) + FAIL_STACK_ERROR + } + + /* Update 5 entries in the index */ + num_entries = 5; + for(i = 0; i < num_entries; i++) + index[i].entry_ptr = &buf[i * FS_PAGE_SIZE]; + + /* Update the metadata file with the index */ + if(H5F_update_vfd_swmr_metadata_file(file_writer, num_entries, index) < 0) + TEST_ERROR; + + /* Send notification 7 to reader to start verification */ + notify = 7; + if(HDwrite(parent_pfd[1], ¬ify, sizeof(int)) < 0) + FAIL_STACK_ERROR; + + /* Send num_entries to the reader */ + if(HDwrite(parent_pfd[1], &num_entries, sizeof(int)) < 0) + FAIL_STACK_ERROR; + + /* Send index to the reader */ + if(HDwrite(parent_pfd[1], index, + num_entries * sizeof(H5FD_vfd_swmr_idx_entry_t)) < 0) + FAIL_STACK_ERROR; + + /* + * Case E: writer + * --write to the datasets again to ensure ticks elapse + * --update the metadata file with an empty index + */ + + /* Wait for notification 8 from reader that the verifcation is complete */ + while(notify != 8) { + if(HDread(child_pfd[0], ¬ify, sizeof(int)) < 0) + FAIL_STACK_ERROR; + } + + /* Perform activities to ensure that ticks elapse */ + for(i = 0; i < config_writer->max_lag + 1; i++) { + decisleep(config_writer->tick_len); + + /* Open the dataset */ + sprintf(dname, "dset %d", i); + if((did = H5Dopen2(fid_writer, dname, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR + + /* Write to the dataset */ + if(H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, + H5P_DEFAULT, rwbuf) < 0) + FAIL_STACK_ERROR + + /* Close the dataset */ + if(H5Dclose(did) < 0) + FAIL_STACK_ERROR + } + + /* Update the metadata file with 0 entries and NULL index */ + if(H5F_update_vfd_swmr_metadata_file(file_writer, 0, NULL) < 0) + TEST_ERROR; + + /* Send notification 8 to reader to start verification */ + notify = 9; + if(HDwrite(parent_pfd[1], ¬ify, sizeof(int)) < 0) + FAIL_STACK_ERROR; + + /* + * Done + */ + + /* Close the pipes */ + if(HDclose(parent_pfd[1]) < 0) + FAIL_STACK_ERROR; + if(HDclose(child_pfd[0]) < 0) + FAIL_STACK_ERROR; + + /* Wait for child process to complete */ + if((tmppid = HDwaitpid(childpid, &child_status, child_wait_option)) < 0) + FAIL_STACK_ERROR + + /* Check exit status of child process */ + if(WIFEXITED(child_status)) { + if((child_exit_val = WEXITSTATUS(child_status)) != 0) + TEST_ERROR + } else { /* child process terminated abnormally */ + TEST_ERROR + } + + /* Closing */ + if(H5Fclose(fid_writer) < 0) + FAIL_STACK_ERROR + if(H5Pclose(fapl_writer) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + + /* Free resources */ + if(config_writer) + HDfree(config_writer); + if(buf) + HDfree(buf); + if(rwbuf) + HDfree(rwbuf); + if(index) + HDfree(index); + + PASSED(); + return 0; + +error: + if(config_writer) + HDfree(config_writer); + if(buf) + HDfree(buf); + if(rwbuf) + HDfree(rwbuf); + if(index) + HDfree(index); + + H5E_BEGIN_TRY { + H5Pclose(fapl_writer); + H5Fclose(fid_writer); + H5Pclose(fcpl); + } H5E_END_TRY; + + return 1; +} /* test_reader_md_concur() */ + + +/*------------------------------------------------------------------------- + * Function: test_multiple_file_opens_concur() + * + * Purpose: Verify the entries on the EOT queue when opening files + * with and without VFD SWMR configured. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: Vailin Choi; 11/18/2019 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_multiple_file_opens_concur(void) +{ + hid_t fcpl = H5I_INVALID_HID; + pid_t tmppid; /* Child process ID returned by waitpid */ + pid_t childpid = 0; /* Child process ID */ + int child_status; /* Status passed to waitpid */ + int child_wait_option=0; /* Options passed to waitpid */ + int child_exit_val; /* Exit status of the child */ + int parent_pfd[2]; /* Pipe for parent process as writer */ + int child_pfd[2]; /* Pipe for child process as reader */ + int notify = 0; /* Notification between parent and child */ + hid_t fid1 = H5I_INVALID_HID, fid2 = H5I_INVALID_HID; + hid_t fapl1 = H5I_INVALID_HID, fapl2 = H5I_INVALID_HID; + H5F_vfd_swmr_config_t *config1 = NULL; /* VFD SWMR configuration */ + H5F_vfd_swmr_config_t *config2 = NULL; /* VFD SWMR configuration */ + H5F_t *f1, *f2; /* File pointer */ + eot_queue_entry_t *curr; + + TESTING("EOT queue entries when opening files concurrently with VFD SWMR"); + + /* Create a copy of the file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set file space strategy */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0) + FAIL_STACK_ERROR; + + if(H5Pset_file_space_page_size(fcpl, FS_PAGE_SIZE) < 0) + FAIL_STACK_ERROR; + + /* Create file A */ + if((fid1 = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* Close the file */ + if(H5Fclose(fid1) < 0) + FAIL_STACK_ERROR; + + /* Create file B */ + if((fid2 = H5Fcreate(FILENAME2, H5F_ACC_TRUNC, fcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* Close the file */ + if(H5Fclose(fid2) < 0) + FAIL_STACK_ERROR; + + /* Create 2 pipes */ + if(HDpipe(parent_pfd) < 0) + FAIL_STACK_ERROR + + if(HDpipe(child_pfd) < 0) + FAIL_STACK_ERROR + + /* Fork child process */ + if((childpid = HDfork()) < 0) + FAIL_STACK_ERROR + + /* + * Child process + */ + if(childpid == 0) { + int child_notify = 0; /* Notification between child and parent */ + hid_t fid_writer = -1; /* File ID for writer */ + hid_t fapl_writer = -1; /* File access property list for writer */ + H5F_vfd_swmr_config_t *config_writer = NULL; /* VFD SWMR configuration for reader */ + + /* Close unused write end for writer pipe */ + if(HDclose(parent_pfd[1]) < 0) + HDexit(EXIT_FAILURE); + + /* Close unused read end for reader pipe */ + if(HDclose(child_pfd[0]) < 0) + HDexit(EXIT_FAILURE); + + /* + * Set up and open file B as VFD SWMR writer + */ + + /* Wait for notification 1 from parent before opening file B */ + while(child_notify != 1) { + if(HDread(parent_pfd[0], &child_notify, sizeof(int)) < 0) + HDexit(EXIT_FAILURE); + } + + /* Allocate memory for VFD SMWR configuration */ + if((config_writer = HDmalloc(sizeof(*config_writer))) == NULL) + HDexit(EXIT_FAILURE); + + + /* Set the VFD SWMR configuration in fapl_writer + page buffering */ + fapl_writer = init_vfd_swmr_config_fapl(config_writer, 1, 3, TRUE, 256, MD_FILENAME2, FS_PAGE_SIZE); + if(fapl_writer == H5I_INVALID_HID) + HDexit(EXIT_FAILURE); + + /* Open file B as VFD SWMR writer */ + if((fid_writer = H5Fopen(FILENAME2, H5F_ACC_RDWR, fapl_writer)) < 0) + HDexit(EXIT_FAILURE); + + /* Send notification 2 to parent that file B is open */ + child_notify = 2; + if(HDwrite(child_pfd[1], &child_notify, sizeof(int)) < 0) + HDexit(EXIT_FAILURE); + + /* Wait for notification 3 from parent before closing file B */ + while(child_notify != 3) { + if(HDread(parent_pfd[0], &child_notify, sizeof(int)) < 0) + HDexit(EXIT_FAILURE); + } + + if(config_writer ) + HDfree(config_writer); + + /* Close the file */ + if(H5Fclose(fid_writer) < 0) + HDexit(EXIT_FAILURE); + if(H5Pclose(fapl_writer) < 0) + HDexit(EXIT_FAILURE); + + /* Send notification 4 to parent that file B is closed */ + child_notify = 4; + if(HDwrite(child_pfd[1], &child_notify, sizeof(int)) < 0) + HDexit(EXIT_FAILURE); + + /* Close the pipes */ + if(HDclose(parent_pfd[0]) < 0) + HDexit(EXIT_FAILURE); + if(HDclose(child_pfd[1]) < 0) + HDexit(EXIT_FAILURE); + + HDexit(EXIT_SUCCESS); + } /* end child process */ + + /* + * Parent process + */ + + /* Close unused read end for writer pipe */ + if(HDclose(parent_pfd[0]) < 0) + FAIL_STACK_ERROR + + /* Close unused write end for reader pipe */ + if(HDclose(child_pfd[1]) < 0) + FAIL_STACK_ERROR + + /* + * Set up and open file A as VFD SWMR writer + */ + + /* Allocate memory for VFD SWMR configuration */ + if((config1 = HDmalloc(sizeof(*config1))) == NULL) + FAIL_STACK_ERROR + + /* Set the VFD SWMR configuration in fapl1 + page buffering */ + fapl1 = init_vfd_swmr_config_fapl(config1, 7, 10, TRUE, 256, MD_FILENAME, FS_PAGE_SIZE); + if(fapl1 == H5I_INVALID_HID) + FAIL_STACK_ERROR + + /* Open file A as VFD SWMR writer */ + if((fid1 = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl1)) < 0) + FAIL_STACK_ERROR; + + /* Get a pointer to the internal file object */ + if(NULL == (f1 = H5VL_object(fid1))) + FAIL_STACK_ERROR + + /* Head of EOT queue should be a writer */ + if((curr = TAILQ_FIRST(&eot_queue_g)) == NULL || !curr->vfd_swmr_writer) + TEST_ERROR; + + /* The EOT queue's first entry should be f1 */ + if((curr = TAILQ_FIRST(&eot_queue_g)) == NULL || curr->vfd_swmr_file != f1) + TEST_ERROR; + + + /* Send notification 1 to child to open file B */ + notify = 1; + if(HDwrite(parent_pfd[1], ¬ify, sizeof(int)) < 0) + FAIL_STACK_ERROR; + + /* Wait for notification 2 from child that file B is open */ + while(notify != 2) { + if(HDread(child_pfd[0], ¬ify, sizeof(int)) < 0) + FAIL_STACK_ERROR; + } + + /* Open file B as VFD SWMR reader */ + + /* Allocate memory for VFD SWMR configuration */ + if((config2 = HDmalloc(sizeof(*config2))) == NULL) + FAIL_STACK_ERROR + + /* Set the VFD SWMR configuration in fapl2 + page buffering */ + fapl2 = init_vfd_swmr_config_fapl(config2, 1, 3, FALSE, 256, MD_FILENAME2, FS_PAGE_SIZE); + if(fapl2 == H5I_INVALID_HID) + FAIL_STACK_ERROR + + /* Open file B as VFD SWMR reader */ + if((fid2 = H5Fopen(FILENAME2, H5F_ACC_RDONLY, fapl2)) < 0) + FAIL_STACK_ERROR; + + /* Get a pointer to the internal file object */ + if(NULL == (f2 = H5VL_object(fid2))) + FAIL_STACK_ERROR + + /* Head of EOT queue should NOT be a writer */ + if((curr = TAILQ_FIRST(&eot_queue_g)) != NULL && curr->vfd_swmr_writer) + TEST_ERROR; + + /* The EOT queue's first entry should be f2 */ + if((curr = TAILQ_FIRST(&eot_queue_g)) == NULL || curr->vfd_swmr_file != f2) + TEST_ERROR; + + /* Send notification 3 to child to close file B */ + notify = 3; + if(HDwrite(parent_pfd[1], ¬ify, sizeof(int)) < 0) + FAIL_STACK_ERROR; + + /* Wait for notification 4 from child that file B is closed */ + while(notify != 4) { + if(HDread(child_pfd[0], ¬ify, sizeof(int)) < 0) + FAIL_STACK_ERROR; + } + + /* + * Done + */ + + /* Close the pipes */ + if(HDclose(parent_pfd[1]) < 0) + FAIL_STACK_ERROR; + if(HDclose(child_pfd[0]) < 0) + FAIL_STACK_ERROR; + + /* Wait for child process to complete */ + if((tmppid = HDwaitpid(childpid, &child_status, child_wait_option)) < 0) + FAIL_STACK_ERROR + + /* Check exit status of child process */ + if(WIFEXITED(child_status)) { + if((child_exit_val = WEXITSTATUS(child_status)) != 0) + TEST_ERROR + } else { /* child process terminated abnormally */ + TEST_ERROR + } + + /* Closing */ + if(H5Fclose(fid1) < 0) + FAIL_STACK_ERROR + if(H5Fclose(fid2) < 0) + FAIL_STACK_ERROR + if(H5Pclose(fapl1) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fapl2) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + + /* Free resources */ + if(config1) + HDfree(config1); + if(config2) + HDfree(config2); + + PASSED(); + return 0; + +error: + if(config1) + HDfree(config1); + if(config2) + HDfree(config2); + + H5E_BEGIN_TRY { + H5Pclose(fapl1); + H5Pclose(fapl2); + H5Fclose(fid1); + H5Fclose(fid2); + H5Pclose(fcpl); + } H5E_END_TRY; + + return 1; +} /* test_multiple_file_opens_concur() */ + + +/*------------------------------------------------------------------------- + * Function: test_enable_disable_eot_concur() + * + * Purpose: Verify the public routines: + * H5Fvfd_swmr_disable_end_of_tick() + * H5Fvfd_swmr_enable_end_of_tick() + * enables/disables EOT when the files are opened + * concurrently. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: Vailin Choi; June 2020 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_disable_enable_eot_concur(void) +{ + hid_t fcpl = -1; /* File creation property list */ + hid_t fid_writer = -1; /* File ID for writer */ + hid_t fapl_writer = -1; /* File access property list for writer */ + H5F_vfd_swmr_config_t *config_writer = NULL; /* VFD SWMR Configuration for writer */ + pid_t tmppid; /* Child process ID returned by waitpid */ + pid_t childpid = 0; /* Child process ID */ + int child_status; /* Status passed to waitpid */ + int child_wait_option=0; /* Options passed to waitpid */ + int child_exit_val; /* Exit status of the child */ + + int parent_pfd[2]; /* Pipe for parent process as writer */ + int child_pfd[2]; /* Pipe for child process as reader */ + int notify = 0; /* Notification between parent and child */ + + TESTING("Verify concurrent H5Fvfd_swmr_enable/disable_end_of_tick()"); + + /* Allocate memory for the configuration structure */ + if((config_writer = HDmalloc(sizeof(*config_writer))) == NULL) + FAIL_STACK_ERROR; + + /* Set up the VFD SWMR configuration + page buffering */ + fapl_writer = init_vfd_swmr_config_fapl(config_writer, 1, 3, TRUE, 256, MD_FILENAME, FS_PAGE_SIZE); + if(fapl_writer == H5I_INVALID_HID) + FAIL_STACK_ERROR; + + /* Create a copy of the file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set file space strategy */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0) + FAIL_STACK_ERROR; + + if(H5Pset_file_space_page_size(fcpl, FS_PAGE_SIZE) < 0) + FAIL_STACK_ERROR; + + /* Create an HDF5 file with VFD SWMR configured */ + if((fid_writer = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl_writer)) < 0) + FAIL_STACK_ERROR; + + /* Close the file */ + if(H5Fclose(fid_writer) < 0) + FAIL_STACK_ERROR; + + /* Create 2 pipes */ + if(HDpipe(parent_pfd) < 0) + FAIL_STACK_ERROR + + if(HDpipe(child_pfd) < 0) + FAIL_STACK_ERROR + + /* Fork child process */ + if((childpid = HDfork()) < 0) + FAIL_STACK_ERROR + + /* + * Child process as reader + */ + if(childpid == 0) { + int child_notify = 0; /* Notification between child and parent */ + hid_t fid_reader = H5I_INVALID_HID; /* File ID for reader */ + hid_t fid_reader2 = H5I_INVALID_HID; /* File ID for reader */ + hid_t fid_reader3 = H5I_INVALID_HID; /* File ID for reader */ + hid_t fapl_reader = H5I_INVALID_HID; /* File access property list for reader */ + H5F_vfd_swmr_config_t *config_reader = NULL; /* VFD SWMR configuration */ + H5F_t *file_reader; /* File pointer */ + eot_queue_entry_t *curr; /* Pointer to an entry on the EOT queue */ + unsigned count = 0; /* Counter */ + + /* Close unused write end for writer pipe */ + if(HDclose(parent_pfd[1]) < 0) + HDexit(EXIT_FAILURE); + + /* Close unused read end for reader pipe */ + if(HDclose(child_pfd[0]) < 0) + HDexit(EXIT_FAILURE); + + /* + * Open the file 3 times as VFD SWMR reader + * Enable and disable EOT for a file + * Verify the state of the EOT queue + */ + + /* Wait for notification 1 from parent to start verification */ + while(child_notify != 1) { + if(HDread(parent_pfd[0], &child_notify, sizeof(int)) < 0) + HDexit(EXIT_FAILURE); + } + + /* Allocate memory for the configuration structure */ + if((config_reader = HDmalloc(sizeof(*config_reader))) == NULL) + HDexit(EXIT_FAILURE); + + /* Set up the VFD SWMR configuration as reader + page buffering */ + fapl_reader = init_vfd_swmr_config_fapl(config_reader, 1, 3, FALSE, 256, MD_FILENAME, FS_PAGE_SIZE); + if(fapl_reader == H5I_INVALID_HID) + FAIL_STACK_ERROR; + + /* Open the test file as reader */ + if((fid_reader = H5Fopen(FILENAME, H5F_ACC_RDONLY, fapl_reader)) < 0) + HDexit(EXIT_FAILURE); + + /* Open the same test file as reader (a second time) */ + if((fid_reader2 = H5Fopen(FILENAME, H5F_ACC_RDONLY, fapl_reader)) < 0) + HDexit(EXIT_FAILURE); + + /* Open the same test file as reader (a third time) */ + if((fid_reader3 = H5Fopen(FILENAME, H5F_ACC_RDONLY, fapl_reader)) < 0) + HDexit(EXIT_FAILURE); + + /* Verify the # of files on the EOT queue is 3 */ + count = 0; + TAILQ_FOREACH(curr, &eot_queue_g, link) + count++; + if(count != 3) + HDexit(EXIT_FAILURE); + + /* Disable EOT for the second opened file */ + if(H5Fvfd_swmr_disable_end_of_tick(fid_reader2) < 0) + HDexit(EXIT_FAILURE); + + /* Verify the # of files on the EOT queue is 2 */ + count = 0; + TAILQ_FOREACH(curr, &eot_queue_g, link) + count++; + if(count != 2) + HDexit(EXIT_FAILURE); + + /* Get file pointer */ + file_reader = H5VL_object(fid_reader2); + + /* Should not find the second opened file on the EOT queue */ + TAILQ_FOREACH(curr, &eot_queue_g, link) { + if(curr->vfd_swmr_file == file_reader) + break; + } + if(curr != NULL && curr->vfd_swmr_file == file_reader) + HDexit(EXIT_FAILURE); + + /* Enable EOT for the second opened file again */ + if(H5Fvfd_swmr_enable_end_of_tick(fid_reader2) < 0) + HDexit(EXIT_FAILURE); + + /* Verify the # of files on the EOT queue is 3 */ + count = 0; + TAILQ_FOREACH(curr, &eot_queue_g, link) + count++; + if(count != 3) + HDexit(EXIT_FAILURE); + + /* Should find the second opened file on the EOT queue */ + TAILQ_FOREACH(curr, &eot_queue_g, link) { + if(curr->vfd_swmr_file == file_reader) + break; + } + if(curr == NULL || curr->vfd_swmr_file != file_reader) + HDexit(EXIT_FAILURE); + + /* Closing */ + if(H5Fclose(fid_reader) < 0) + HDexit(EXIT_FAILURE); + if(H5Fclose(fid_reader2) < 0) + HDexit(EXIT_FAILURE); + if(H5Fclose(fid_reader3) < 0) + HDexit(EXIT_FAILURE); + if(H5Pclose(fapl_reader) < 0) + HDexit(EXIT_FAILURE); + if(config_reader) + HDfree(config_reader); + + /* Close the pipes */ + if(HDclose(parent_pfd[0]) < 0) + HDexit(EXIT_FAILURE); + if(HDclose(child_pfd[1]) < 0) + HDexit(EXIT_FAILURE); + + HDexit(EXIT_SUCCESS); + } /* end child process */ + + /* + * Parent process as writer + */ + + /* Close unused read end for writer pipe */ + if(HDclose(parent_pfd[0]) < 0) + FAIL_STACK_ERROR + + /* Close unused write end for reader pipe */ + if(HDclose(child_pfd[1]) < 0) + FAIL_STACK_ERROR + + /* + * Open the file as VFD SWMR writer + */ + + /* Open as VFD SWMR writer */ + if((fid_writer = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl_writer)) < 0) + FAIL_STACK_ERROR; + + /* Send notification 1 to reader to start verfication */ + notify = 1; + if(HDwrite(parent_pfd[1], ¬ify, sizeof(int)) < 0) + FAIL_STACK_ERROR; + + /* + * Done + */ + + /* Close the pipes */ + if(HDclose(parent_pfd[1]) < 0) + FAIL_STACK_ERROR; + if(HDclose(child_pfd[0]) < 0) + FAIL_STACK_ERROR; + + /* Wait for child process to complete */ + if((tmppid = HDwaitpid(childpid, &child_status, child_wait_option)) < 0) + FAIL_STACK_ERROR + + /* Check exit status of child process */ + if(WIFEXITED(child_status)) { + if((child_exit_val = WEXITSTATUS(child_status)) != 0) + TEST_ERROR + } else { /* child process terminated abnormally */ + TEST_ERROR + } + + /* Closing */ + if(H5Fclose(fid_writer) < 0) + FAIL_STACK_ERROR + if(H5Pclose(fapl_writer) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + + /* Free resources */ + if(config_writer) + HDfree(config_writer); + + PASSED(); + return 0; + +error: + if(config_writer) + HDfree(config_writer); + + H5E_BEGIN_TRY { + H5Pclose(fapl_writer); + H5Fclose(fid_writer); + H5Pclose(fcpl); + } H5E_END_TRY; + + return 1; +} /* test_disable_enable_eot_concur() */ + + +/*------------------------------------------------------------------------- + * Function: test_file_end_tick_concur() + * + * Purpose: Verify the public routine H5Fvfd_swmr_end_tick() + * triggers end of tick processing when the files + * are opened concurrently. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: Vailin Choi; June 2020 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_file_end_tick_concur(void) +{ + hid_t fcpl = -1; /* File creation property list */ + hid_t fid_writer = -1; /* File ID for writer */ + hid_t fapl_writer = -1; /* File access property list for writer */ + H5F_vfd_swmr_config_t *config_writer = NULL; /* VFD SWMR Configuration for writer */ + pid_t tmppid; /* Child process ID returned by waitpid */ + pid_t childpid = 0; /* Child process ID */ + int child_status; /* Status passed to waitpid */ + int child_wait_option=0; /* Options passed to waitpid */ + int child_exit_val; /* Exit status of the child */ + + int parent_pfd[2]; /* Pipe for parent process as writer */ + int child_pfd[2]; /* Pipe for child process as reader */ + int notify = 0; /* Notification between parent and child */ + + TESTING("Verify concurrent H5Fvfd_swmr_end_tick()"); + + /* Allocate memory for the configuration structure */ + if((config_writer = HDmalloc(sizeof(*config_writer))) == NULL) + FAIL_STACK_ERROR; + + /* Set up the VFD SWMR configuration + page buffering */ + fapl_writer = init_vfd_swmr_config_fapl(config_writer, 1, 3, TRUE, 256, MD_FILENAME, FS_PAGE_SIZE); + if(fapl_writer == H5I_INVALID_HID) + FAIL_STACK_ERROR; + + /* Create a copy of the file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set file space strategy */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0) + FAIL_STACK_ERROR; + + if(H5Pset_file_space_page_size(fcpl, FS_PAGE_SIZE) < 0) + FAIL_STACK_ERROR; + + /* Create an HDF5 file with VFD SWMR configured */ + if((fid_writer = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl_writer)) < 0) + FAIL_STACK_ERROR; + + /* Close the file */ + if(H5Fclose(fid_writer) < 0) + FAIL_STACK_ERROR; + + /* Create 2 pipes */ + if(HDpipe(parent_pfd) < 0) + FAIL_STACK_ERROR + + if(HDpipe(child_pfd) < 0) + FAIL_STACK_ERROR + + /* Fork child process */ + if((childpid = HDfork()) < 0) + FAIL_STACK_ERROR + + /* + * Child process as reader + */ + if(childpid == 0) { + int child_notify = 0; /* Notification between child and parent */ + hid_t fid_reader1 = H5I_INVALID_HID; /* File ID for reader */ + hid_t fid_reader2 = H5I_INVALID_HID; /* File ID for reader */ + hid_t fid_reader3 = H5I_INVALID_HID; /* File ID for reader */ + hid_t fapl_reader = H5I_INVALID_HID; /* File access property list for reader */ + H5F_vfd_swmr_config_t *config_reader = NULL; /* VFD SWMR configuration */ + H5F_t *f1, *f2, *f3; /* File pointer */ + uint64_t s1 = 0; /* Saved tick_num */ + uint64_t s2 = 0; /* Saved tick_num */ + uint64_t s3 = 0; /* Saved tick_num */ + + /* Close unused write end for writer pipe */ + if(HDclose(parent_pfd[1]) < 0) + HDexit(EXIT_FAILURE); + + /* Close unused read end for reader pipe */ + if(HDclose(child_pfd[0]) < 0) + HDexit(EXIT_FAILURE); + + /* + * Open the file 3 times as VFD SWMR reader + * Trigger EOT for the files + */ + + /* Wait for notification 1 from parent to start verification */ + while(child_notify != 1) { + if(HDread(parent_pfd[0], &child_notify, sizeof(int)) < 0) + HDexit(EXIT_FAILURE); + } + + /* Allocate memory for the configuration structure */ + if((config_reader = HDmalloc(sizeof(*config_reader))) == NULL) + HDexit(EXIT_FAILURE); + + /* Set up the VFD SWMR configuration as reader + page buffering */ + fapl_reader = init_vfd_swmr_config_fapl(config_reader, 1, 3, FALSE, 256, MD_FILENAME, FS_PAGE_SIZE); + if(fapl_reader == H5I_INVALID_HID) + FAIL_STACK_ERROR; + + /* Open the test file as reader */ + if((fid_reader1 = H5Fopen(FILENAME, H5F_ACC_RDONLY, fapl_reader)) < 0) + HDexit(EXIT_FAILURE); + + /* Open the same test file as reader (a second time) */ + if((fid_reader2 = H5Fopen(FILENAME, H5F_ACC_RDONLY, fapl_reader)) < 0) + HDexit(EXIT_FAILURE); + + /* Open the same test file as reader (a third time) */ + if((fid_reader3 = H5Fopen(FILENAME, H5F_ACC_RDONLY, fapl_reader)) < 0) + HDexit(EXIT_FAILURE); + + /* Get file pointer */ + f1 = H5VL_object(fid_reader1); + f2 = H5VL_object(fid_reader2); + f3 = H5VL_object(fid_reader3); + + /* Saved tick_num for the 3 files */ + s1 = f1->shared->tick_num; + s2 = f2->shared->tick_num; + s3 = f3->shared->tick_num; + + /* Trigger EOT for the second opened file */ + if(H5Fvfd_swmr_end_tick(fid_reader2) < 0) + HDexit(EXIT_FAILURE); + + /* Verify tick_num should not be less than the previous tick_num */ + if(f2->shared->tick_num < s2) + HDexit(EXIT_FAILURE); + + if(H5Fclose(fid_reader2) < 0) + HDexit(EXIT_FAILURE); + + /* Trigger EOT for the first opened file */ + if(H5Fvfd_swmr_end_tick(fid_reader1) < 0) + HDexit(EXIT_FAILURE); + + /* Verify tick_num should not be less than the previous tick_num */ + if(f1->shared->tick_num < s1) + HDexit(EXIT_FAILURE); + + if(H5Fclose(fid_reader1) < 0) + HDexit(EXIT_FAILURE); + + /* Trigger end tick processing for the third opened file */ + if(H5Fvfd_swmr_end_tick(fid_reader3) < 0) + HDexit(EXIT_FAILURE); + + /* Verify tick_num should not be less than the previous tick_num */ + if(f3->shared->tick_num < s3) + HDexit(EXIT_FAILURE); + + if(H5Fclose(fid_reader3) < 0) + HDexit(EXIT_FAILURE); + + if(H5Pclose(fapl_reader) < 0) + HDexit(EXIT_FAILURE); + if(config_reader) + HDfree(config_reader); + + /* Close the pipes */ + if(HDclose(parent_pfd[0]) < 0) + HDexit(EXIT_FAILURE); + if(HDclose(child_pfd[1]) < 0) + HDexit(EXIT_FAILURE); + + HDexit(EXIT_SUCCESS); + } /* end child process */ + + /* + * Parent process as writer + */ + + /* Close unused read end for writer pipe */ + if(HDclose(parent_pfd[0]) < 0) + FAIL_STACK_ERROR + + /* Close unused write end for reader pipe */ + if(HDclose(child_pfd[1]) < 0) + FAIL_STACK_ERROR + + /* + * Open the file as VFD SWMR writer + */ + + /* Open as VFD SWMR writer */ + if((fid_writer = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl_writer)) < 0) + FAIL_STACK_ERROR; + + /* Send notification 1 to reader to start verfication */ + notify = 1; + if(HDwrite(parent_pfd[1], ¬ify, sizeof(int)) < 0) + FAIL_STACK_ERROR; + + /* + * Done + */ + + /* Close the pipes */ + if(HDclose(parent_pfd[1]) < 0) + FAIL_STACK_ERROR; + if(HDclose(child_pfd[0]) < 0) + FAIL_STACK_ERROR; + + /* Wait for child process to complete */ + if((tmppid = HDwaitpid(childpid, &child_status, child_wait_option)) < 0) + FAIL_STACK_ERROR + + /* Check exit status of child process */ + if(WIFEXITED(child_status)) { + if((child_exit_val = WEXITSTATUS(child_status)) != 0) + TEST_ERROR + } else { /* child process terminated abnormally */ + TEST_ERROR + } + + /* Closing */ + if(H5Fclose(fid_writer) < 0) + FAIL_STACK_ERROR + if(H5Pclose(fapl_writer) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + + /* Free resources */ + if(config_writer) + HDfree(config_writer); + + PASSED(); + return 0; + +error: + if(config_writer) + HDfree(config_writer); + + H5E_BEGIN_TRY { + H5Pclose(fapl_writer); + H5Fclose(fid_writer); + H5Pclose(fcpl); + } H5E_END_TRY; + + return 1; +} /* test_file_end_tick_concur() */ + +#endif /* !(defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID) && defined(H5_HAVE_FLOCK)) */ + + +/*------------------------------------------------------------------------- + * Function: test_multiple_file_opens() + * + * Purpose: Verify the entries on the EOT queue when opening files + * with and without VFD SWMR configured. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: Vailin Choi; 11/18/2019 + * + *------------------------------------------------------------------------- + */ + +static unsigned +test_multiple_file_opens(void) +{ + hid_t fid1 = -1; /* File ID */ + hid_t fid2 = -1; /* File ID */ + hid_t fid = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list ID */ + hid_t fapl1 = -1; /* File access property list ID */ + hid_t fapl2 = -1; /* File access property list ID */ + H5F_t *f1, *f2, *f; /* File pointer */ + H5F_vfd_swmr_config_t *config1 = NULL; /* Configuration for VFD SWMR */ + H5F_vfd_swmr_config_t *config2 = NULL; /* Configuration for VFD SWMR */ + eot_queue_entry_t *curr; + + TESTING("EOT queue entries when opening files with/without VFD SWMR"); + + /* Allocate memory for the configuration structure */ + if((config1 = HDmalloc(sizeof(*config1))) == NULL) + FAIL_STACK_ERROR; + if((config2 = HDmalloc(sizeof(*config2))) == NULL) + FAIL_STACK_ERROR; + + /* Configured as VFD SWMR writer + page buffering */ + fapl1 = init_vfd_swmr_config_fapl(config1, 4, 6, TRUE, 2, MD_FILENAME, 4096); + if(fapl1 == H5I_INVALID_HID) + FAIL_STACK_ERROR; + + /* Configured as VFD SWMR writer + page buffering */ + fapl2 = init_vfd_swmr_config_fapl(config2, 4, 6, TRUE, 2, MD_FILENAME2, 4096); + if(fapl2 == H5I_INVALID_HID) + FAIL_STACK_ERROR; + + /* Create a copy of the file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set file space strategy */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0) + FAIL_STACK_ERROR; + + /* Create a file without VFD SWMR */ + if((fid = H5Fcreate(FNAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Get a pointer to the internal file object */ + if(NULL == (f = H5VL_object(fid))) + FAIL_STACK_ERROR + + /* Verify the global vfd_swmr_writer_g is not set */ + if((curr = TAILQ_FIRST(&eot_queue_g)) != NULL && curr->vfd_swmr_writer) + TEST_ERROR; + /* The EOT queue should be empty */ + if(!TAILQ_EMPTY(&eot_queue_g)) + TEST_ERROR; + + /* Create a file with VFD SWMR writer */ + if((fid1 = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl1)) < 0) + TEST_ERROR; + + /* Get a pointer to the internal file object */ + if(NULL == (f1 = H5VL_object(fid1))) + FAIL_STACK_ERROR + + /* Head of EOT queue should be a writer */ + if((curr = TAILQ_FIRST(&eot_queue_g)) == NULL || !curr->vfd_swmr_writer) + TEST_ERROR; + /* The EOT queue should be initialized with the first entry equals to f1 */ + if((curr = TAILQ_FIRST(&eot_queue_g)) == NULL || curr->vfd_swmr_file != f1) + TEST_ERROR; + + /* Create another file with VFD SWMR writer */ + if((fid2 = H5Fcreate(FILENAME2, H5F_ACC_TRUNC, fcpl, fapl2)) < 0) + TEST_ERROR; + + /* Get a pointer to the internal file object */ + if(NULL == (f2 = H5VL_object(fid2))) + FAIL_STACK_ERROR + + /* Head of EOT queue should be a writer */ + if((curr = TAILQ_FIRST(&eot_queue_g)) == NULL || !curr->vfd_swmr_writer) + TEST_ERROR; + /* The EOT queue's first entry should be f1 */ + if((curr = TAILQ_FIRST(&eot_queue_g)) == NULL || curr->vfd_swmr_file != f1) + TEST_ERROR; + + /* The file without VFD SWMR should not exist on the EOT queue */ + TAILQ_FOREACH(curr, &eot_queue_g, link) { + if(curr->vfd_swmr_file == f) + TEST_ERROR + } + + /* Close the first file with VFD SWMR */ + if(H5Fclose(fid1) < 0) + FAIL_STACK_ERROR; + + /* Head of EOT queue should be a writer */ + if((curr = TAILQ_FIRST(&eot_queue_g)) == NULL || !curr->vfd_swmr_writer) + TEST_ERROR; + /* The EOT queue's first entry should be f2 */ + if((curr = TAILQ_FIRST(&eot_queue_g)) == NULL || curr->vfd_swmr_file != f2) + TEST_ERROR; + + /* Close the second file with VFD SWMR */ + if(H5Fclose(fid2) < 0) + FAIL_STACK_ERROR; + + /* Head of EOT queue should not be a writer */ + if((curr = TAILQ_FIRST(&eot_queue_g)) != NULL && curr->vfd_swmr_writer) + TEST_ERROR; + /* The EOT queue should be empty */ + if(!TAILQ_EMPTY(&eot_queue_g)) + TEST_ERROR; + + /* Closing */ + if(H5Pclose(fapl1) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fapl2) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + + /* Free buffers */ + if(config1) + HDfree(config1); + if(config2) + HDfree(config2); + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(fapl1); + H5Pclose(fapl2); + H5Pclose(fcpl); + H5Fclose(fid); + H5Fclose(fid1); + H5Fclose(fid2); + } H5E_END_TRY; + if(config1) + HDfree(config1); + if(config2) + HDfree(config2); + + return 1; +} /* test_multiple_file_opens() */ + + + +/*------------------------------------------------------------------------- + * Function: test_same_file_opens() + * + * Purpose: Verify multiple opens of the same file as listed below: + * + * #1st open# + * #2nd open# VW VR W R + * ------------------ + * VW | s f f f | + * VR | f f f f | + * W | f f s f | + * R | f f s s | + * ------------------ + * + * Notations: + * W: H5F_ACC_RDWR + * R: H5F_ACC_RDONLY + * VW: VFD SWMR writer + * VR: VFD SWMR reader + * + * f: the open fails + * s: the open succeeds + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: Vailin Choi; October 2019 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_same_file_opens(void) +{ + hid_t fid = -1; /* File ID */ + hid_t fid2 = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list ID */ + hid_t fapl1 = -1; /* File access property list ID */ + hid_t fapl2 = -1; /* File access property list ID */ + H5F_vfd_swmr_config_t *config1 = NULL; /* Configuration for VFD SWMR */ + H5F_vfd_swmr_config_t *config2 = NULL; /* Configuration for VFD SWMR */ + + TESTING("Multiple opens of the same file with VFD SWMR configuration"); + + /* Should succeed without VFD SWMR configured */ + if((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Close the file */ + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + + /* Allocate memory for the configuration structure */ + if((config1 = HDmalloc(sizeof(*config1))) == NULL) + FAIL_STACK_ERROR; + if((config2 = HDmalloc(sizeof(*config2))) == NULL) + FAIL_STACK_ERROR; + + /* Create a copy of the file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set file space strategy */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0) + FAIL_STACK_ERROR; + + /* + * Tests for first column + */ + + /* Create the test file */ + if((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Close the file */ + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + + + /* Set the VFD SWMR configuration in fapl1 + page buffering */ + fapl1 = init_vfd_swmr_config_fapl(config1, 4, 10, TRUE, 2, MD_FILENAME, 4096); + if(fapl1 == H5I_INVALID_HID) + FAIL_STACK_ERROR; + + /* Open the file as VFD SWMR writer */ + /* Keep the file open */ + if((fid = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl1)) < 0) + TEST_ERROR; + + /* Open the same file again as VFD SWMR writer */ + /* Should succeed: 1st open--VFD SWMR writer, 2nd open--VFD SWMR writer */ + if((fid2 = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl1)) < 0) + TEST_ERROR; + + /* Close the second file open */ + if(H5Fclose(fid2) < 0) + FAIL_STACK_ERROR; + + /* Set the VFD SWMR configuration in fapl2 + page buffering */ + fapl2 = init_vfd_swmr_config_fapl(config2, 3, 8, FALSE, 3, MD_FILENAME, 4096); + if(fapl2 == H5I_INVALID_HID) + FAIL_STACK_ERROR; + + /* Open the same file again as VFD SWMR reader */ + /* Should fail: 1st open--VFD SWMR writer, 2nd open--VFD SWMR reader */ + H5E_BEGIN_TRY { + fid2 = H5Fopen(FILENAME, H5F_ACC_RDONLY, fapl2); + } H5E_END_TRY; + if(fid2 >= 0) + TEST_ERROR; + + if(H5Pclose(fapl2) < 0) + FAIL_STACK_ERROR; + + /* Open the same file again as regular writer */ + /* Should fail: 1st open--VFD SWMR writer, 2nd open--regular writer */ + H5E_BEGIN_TRY { + fid2 = H5Fopen(FILENAME, H5F_ACC_RDWR, H5P_DEFAULT); + } H5E_END_TRY; + if(fid2 >= 0) + TEST_ERROR; + + /* Open the same file again as regular reader */ + /* Should fail: 1st open--VFD SWMR writer, 2nd open--regular reader */ + H5E_BEGIN_TRY { + fid2 = H5Fopen(FILENAME, H5F_ACC_RDONLY, H5P_DEFAULT); + } H5E_END_TRY; + if(fid2 >= 0) + TEST_ERROR; + + /* Close the 1st open file */ + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + + + /* + * Tests for second column + */ + + /* Set up as VFD SWMR reader + page buffering */ + fapl1 = init_vfd_swmr_config_fapl(config1, 4, 10, FALSE, 2, MD_FILENAME, 4096); + if(fapl1 == H5I_INVALID_HID) + FAIL_STACK_ERROR; + + /* Open the file as VFD SWMR reader */ + /* Should fail because there is no metadata file */ + /* Take a while to complete due to retries */ + H5E_BEGIN_TRY { + fid = H5Fopen(FILENAME, H5F_ACC_RDONLY, fapl1); + } H5E_END_TRY; + if(fid >= 0) + TEST_ERROR; + + if(H5Pclose(fapl1) < 0) + FAIL_STACK_ERROR; + + /* + * Tests for third column + */ + + /* Open the file as regular writer */ + /* Keep the file open */ + if((fid = H5Fopen(FILENAME, H5F_ACC_RDWR, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Set up as VFD SWMR writer + page buffering */ + fapl1 = init_vfd_swmr_config_fapl(config1, 4, 10, TRUE, 2, MD_FILENAME, 4096); + if(fapl1 == H5I_INVALID_HID) + FAIL_STACK_ERROR; + + /* Open the same file again as VFD SWMR writer */ + /* Should fail: 1st open--regular writer, 2nd open--VFD SWMR writer */ + H5E_BEGIN_TRY { + fid2 = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl1); + } H5E_END_TRY; + if(fid2 >= 0) + TEST_ERROR; + + if(H5Pclose(fapl1) < 0) + FAIL_STACK_ERROR; + + + /* Open the same file again as regular writer */ + /* Should succeed: 1st open--regular writer, 2nd open--regular writer */ + if((fid2 = H5Fopen(FILENAME, H5F_ACC_RDWR, H5P_DEFAULT)) < 0) + TEST_ERROR; + + if(H5Fclose(fid2) < 0) + FAIL_STACK_ERROR; + + + /* Open the same file again as regular reader */ + /* Should succeed: 1st open is writer, 2nd open is the same file as reader */ + if((fid2 = H5Fopen(FILENAME, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) + TEST_ERROR; + + if(H5Fclose(fid2) < 0) + FAIL_STACK_ERROR; + + /* Close the 1st open file */ + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + + /* + * Tests for fourth column + */ + + /* Open the file as regular reader */ + /* keep the file open */ + if((fid = H5Fopen(FILENAME, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Set up as VFD SWMR writer + page buffering */ + fapl1 = init_vfd_swmr_config_fapl(config1, 4, 10, TRUE, 2, MD_FILENAME, 4096); + if(fapl1 == H5I_INVALID_HID) + FAIL_STACK_ERROR; + + /* Open the same file again as VFD SMWR writer */ + /* Should fail: 1st open--regular reader, 2nd open--VFD SWMR writer */ + H5E_BEGIN_TRY { + fid2 = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl1); + } H5E_END_TRY; + if(fid2 >= 0) + TEST_ERROR; + + if(H5Pclose(fapl1) < 0) + FAIL_STACK_ERROR; + + /* Open the same file again as regular reader */ + /* Should succeed: 1st open--regular reader, 2nd open--regular reader */ + if((fid2 = H5Fopen(FILENAME, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) + TEST_ERROR; + + if(H5Fclose(fid2) < 0) + FAIL_STACK_ERROR; + + /* Open the same file again as regular writer */ + /* Should fail: 1st open--regular reader, 2nd open--regular writer */ + H5E_BEGIN_TRY { + fid2 = H5Fopen(FILENAME, H5F_ACC_RDWR, H5P_DEFAULT); + } H5E_END_TRY; + if(fid2 >= 0) + TEST_ERROR; + + /* Close the 1st open file */ + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + + /* Free buffers */ + if(config1) + HDfree(config1); + if(config2) + HDfree(config2); + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(fapl1); + H5Pclose(fapl2); + H5Pclose(fcpl); + H5Fclose(fid); + H5Fclose(fid2); + } H5E_END_TRY; + if(config1) + HDfree(config1); + if(config2) + HDfree(config2); + return 1; +} /* test_same_file_opens() */ + +#ifndef _arraycount +#define _arraycount(_a) (sizeof(_a)/sizeof(_a[0])) +#endif + +static unsigned +test_shadow_index_lookup(void) +{ + unsigned nerrors = 0; + H5FD_vfd_swmr_idx_entry_t *idx; + uint32_t size[] = {0, 1, 2, 3, 4, 0}; + char vector[8]; + unsigned seed = 1; + unsigned i, j, failj = UINT_MAX; + bool have_failj = false; + unsigned long tmpl; + char *ostate; + const char *seedvar = "H5_SHADOW_INDEX_SEED"; + const char *failvar = "H5_SHADOW_INDEX_FAIL"; + + TESTING("Shadow-index lookups"); + + /* get seed from environment or else from time(3) */ + switch (fetch_env_ulong(seedvar, UINT_MAX, &tmpl)) { + case -1: + nerrors = 1; + goto out; + case 0: + seed = (unsigned int)time(NULL); + break; + default: + seed = (unsigned int)tmpl; + break; + } + + /* get forced-fail index from environment */ + switch (fetch_env_ulong(failvar, UINT_MAX, &tmpl)) { + case -1: + nerrors = 1; + goto out; + case 0: + break; + default: + failj = (unsigned int)tmpl; + have_failj = true; + break; + } + + ostate = initstate(seed, vector, _arraycount(vector)); + + size[5] = (uint32_t)(1024 + random() % (16 * 1024 * 1024 - 1024)); + + for (i = 0; i < _arraycount(size); i++) { + uint32_t cursize = size[i]; + const uint64_t modulus = UINT64_MAX / MAX(1, cursize); + uint64_t pageno; + + assert(modulus > 1); // so that modulus - 1 > 0, below + + idx = (cursize == 0) ? NULL : calloc(cursize, sizeof(*idx)); + if (idx == NULL && cursize != 0) { + fprintf(stderr, "couldn't allocate %" PRIu32 " indices\n", + cursize); + exit(EXIT_FAILURE); + } + for (pageno = (uint64_t)random() % modulus, j = 0; + j < cursize; + j++, pageno += 1 + (uint64_t)random() % (modulus - 1)) { + idx[j].hdf5_page_offset = pageno; + } + for (j = 0; j < cursize; j++) { + H5FD_vfd_swmr_idx_entry_t *found; + + found = vfd_swmr_pageno_to_mdf_idx_entry(idx, cursize, + idx[j].hdf5_page_offset, false); + if ((have_failj && failj == j) || found != &idx[j]) + break; + } + if (j < cursize) { + printf("\nshadow-index entry %d lookup, pageno %" PRIu64 + ", index size %" PRIu32 ", seed %u", j, + idx[j].hdf5_page_offset, cursize, seed); + nerrors++; + } + if (idx != NULL) + free(idx); + } + (void)setstate(ostate); +out: + if (nerrors == 0) + PASSED(); + else + printf(" FAILED\n"); + return nerrors; +} + +/*------------------------------------------------------------------------- + * Function: test_enable_disable_eot() + * + * Purpose: Verify the public routines: + * H5Fvfd_swmr_enable_end_of_tick() + * H5Fvfd_swmr_disable_end_of_tick() + * enables/disables EOT for the specified file + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: Vailin Choi; June 2020 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_enable_disable_eot(void) +{ + hid_t fid = H5I_INVALID_HID; /* File ID */ + hid_t fid1 = H5I_INVALID_HID; /* File ID */ + hid_t fid2 = H5I_INVALID_HID; /* File ID */ + hid_t fid3 = H5I_INVALID_HID; /* File ID */ + hid_t fcpl = H5I_INVALID_HID; /* File creation property list ID */ + hid_t fapl1 = H5I_INVALID_HID; /* File access property list ID */ + hid_t fapl2 = H5I_INVALID_HID; /* File access property list ID */ + hid_t fapl3 = H5I_INVALID_HID; /* File access property list ID */ + H5F_t *f1, *f2, *f3; /* File pointer */ + H5F_vfd_swmr_config_t *config1 = NULL; /* Configuration for VFD SWMR */ + H5F_vfd_swmr_config_t *config2 = NULL; /* Configuration for VFD SWMR */ + H5F_vfd_swmr_config_t *config3 = NULL; /* Configuration for VFD SWMR */ + eot_queue_entry_t *curr; /* Pointer to an entry on the EOT queue */ + unsigned count = 0; /* Counter */ + herr_t ret; /* Return value */ + + TESTING("H5Fvfd_swmr_enable/disable_end_of_tick()"); + + /* Allocate memory for the configuration structure */ + if((config1 = HDmalloc(sizeof(*config1))) == NULL) + FAIL_STACK_ERROR; + if((config2 = HDmalloc(sizeof(*config2))) == NULL) + FAIL_STACK_ERROR; + if((config3 = HDmalloc(sizeof(*config3))) == NULL) + FAIL_STACK_ERROR; + + /* Configured first file as VFD SWMR writer + page buffering */ + fapl1 = init_vfd_swmr_config_fapl(config1, 4, 6, TRUE, 2, MD_FILENAME, 4096); + if(fapl1 == H5I_INVALID_HID) + FAIL_STACK_ERROR; + + /* Configured second file as VFD SWMR writer + page buffering */ + fapl2 = init_vfd_swmr_config_fapl(config2, 4, 6, TRUE, 2, MD_FILENAME2, 4096); + if(fapl2 == H5I_INVALID_HID) + FAIL_STACK_ERROR; + + /* Configured third file as VFD SWMR writer + page buffering */ + fapl3 = init_vfd_swmr_config_fapl(config3, 4, 6, TRUE, 2, MD_FILENAME3, 4096); + if(fapl3 == H5I_INVALID_HID) + FAIL_STACK_ERROR; + + /* Create a copy of the file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set file space strategy */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0) + FAIL_STACK_ERROR; + + /* Create a file without VFD SWMR */ + if((fid = H5Fcreate(FNAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Should fail to disable the file because VFD SWMR is not configured */ + H5E_BEGIN_TRY { + ret = H5Fvfd_swmr_disable_end_of_tick(fid); + } H5E_END_TRY; + if(ret >=0) + TEST_ERROR + + if(H5Fclose(fid) < 0) + TEST_ERROR + + /* Create file 1 with VFD SWMR writer */ + if((fid1 = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl1)) < 0) + TEST_ERROR; + + /* Create file 2 with VFD SWMR writer */ + if((fid2 = H5Fcreate(FILENAME2, H5F_ACC_TRUNC, fcpl, fapl2)) < 0) + TEST_ERROR; + + /* Create file 3 with VFD SWMR writer */ + if((fid3 = H5Fcreate(FILENAME3, H5F_ACC_TRUNC, fcpl, fapl3)) < 0) + TEST_ERROR; + + /* Should have 3 files on the EOT queue */ + TAILQ_FOREACH(curr, &eot_queue_g, link) + count++; + if(count != 3) + TEST_ERROR; + + /* Disable EOT for file 1 */ + if(H5Fvfd_swmr_disable_end_of_tick(fid1) < 0) + TEST_ERROR + + /* Disable file 1 again should fail because the file has just been disabled */ + H5E_BEGIN_TRY { + ret = H5Fvfd_swmr_disable_end_of_tick(fid1); + } H5E_END_TRY; + if(ret >=0) + TEST_ERROR + + /* Should have 2 files on the EOT queue */ + count = 0; + TAILQ_FOREACH(curr, &eot_queue_g, link) + count++; + if(count != 2) + TEST_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f1 = H5VL_object(fid1))) + FAIL_STACK_ERROR + + /* Should not find file 1 on the EOT queue */ + TAILQ_FOREACH(curr, &eot_queue_g, link) { + if(curr->vfd_swmr_file == f1) + break; + } + if(curr != NULL && curr->vfd_swmr_file == f1) + TEST_ERROR + + /* Enable EOT for file 2 should fail because the file has not been disabled */ + H5E_BEGIN_TRY { + ret = H5Fvfd_swmr_enable_end_of_tick(fid2); + } H5E_END_TRY; + if(ret >=0) + TEST_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f2 = H5VL_object(fid2))) + FAIL_STACK_ERROR + + /* File 2 should be on the EOT queue */ + TAILQ_FOREACH(curr, &eot_queue_g, link) { + if(curr->vfd_swmr_file == f2) + break; + } + if(curr == NULL || curr->vfd_swmr_file != f2) + TEST_ERROR + + /* Close file 3 */ + if(H5Fclose(fid3) < 0) + TEST_ERROR + + /* Open file 3 again without VFD SWMR writer */ + if((fid3 = H5Fopen(FILENAME3, H5F_ACC_RDWR, H5P_DEFAULT)) < 0) + TEST_ERROR + + /* Get a pointer to the internal file object for file 3 */ + if(NULL == (f3 = H5VL_object(fid3))) + FAIL_STACK_ERROR + + /* File 3 should not exist on the EOT queue */ + TAILQ_FOREACH(curr, &eot_queue_g, link) { + if(curr->vfd_swmr_file == f3) + break; + } + if(curr != NULL && curr->vfd_swmr_file == f3) + TEST_ERROR + + /* Should have 2 files on the EOT queue */ + count = 0; + TAILQ_FOREACH(curr, &eot_queue_g, link) + count++; + if(count != 1) + TEST_ERROR; + + /* Should fail to enable file 3 */ + H5E_BEGIN_TRY { + ret = H5Fvfd_swmr_enable_end_of_tick(fid3); + } H5E_END_TRY; + if(ret >= 0) + TEST_ERROR + + /* Should fail to disable file 3 */ + H5E_BEGIN_TRY { + ret = H5Fvfd_swmr_disable_end_of_tick(fid3); + } H5E_END_TRY; + if(ret >= 0) + TEST_ERROR + + /* Closing */ + if(H5Fclose(fid1) < 0) + FAIL_STACK_ERROR; + if(H5Fclose(fid2) < 0) + FAIL_STACK_ERROR; + if(H5Fclose(fid3) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fapl1) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fapl2) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fapl3) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + + /* Free buffers */ + if(config1) + HDfree(config1); + if(config2) + HDfree(config2); + if(config3) + HDfree(config3); + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(fapl1); + H5Pclose(fapl2); + H5Pclose(fapl3); + H5Pclose(fcpl); + H5Fclose(fid); + H5Fclose(fid1); + H5Fclose(fid2); + H5Fclose(fid3); + } H5E_END_TRY; + if(config1) + HDfree(config1); + if(config2) + HDfree(config2); + if(config3) + HDfree(config3); + + return 1; +} /* test_enable_disable_eot() */ + + +/*------------------------------------------------------------------------- + * Function: main() + * + * Purpose: Main function for VFD SWMR tests. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + *------------------------------------------------------------------------- + */ +int +main(void) +{ + hid_t fapl = -1; /* File access property list for */ + /* data files */ + unsigned nerrors = 0; /* Cumulative error count */ + char *lock_env_var = NULL; /* File locking env var pointer */ + const char *env_h5_drvr = NULL; /* File Driver value from environment */ + hbool_t use_file_locking; /* Read from env var */ + + /* Check the environment variable that determines if we care + * about file locking. File locking should be used unless explicitly + * disabled. + */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if(lock_env_var && !HDstrcmp(lock_env_var, "FALSE")) + use_file_locking = FALSE; + else + use_file_locking = TRUE; + + /* Get the VFD to use */ + env_h5_drvr = HDgetenv("HDF5_DRIVER"); + if(env_h5_drvr == NULL) + env_h5_drvr = "nomatch"; + + /* Temporary skip testing with multi/split drivers: + * Page buffering depends on paged aggregation which is + * currently disabled for multi/split drivers. + */ + if((0 == HDstrcmp(env_h5_drvr, "multi")) || + (0 == HDstrcmp(env_h5_drvr, "split"))) { + + SKIPPED() + HDputs("Skip VFD SWMR test because paged aggregation is disabled for multi/split drivers"); + HDexit(EXIT_SUCCESS); + } /* end if */ + + /* Set up */ + h5_reset(); + + if((fapl = h5_fileaccess()) < 0) { + nerrors++; + PUTS_ERROR("Can't get VFD-dependent fapl") + } /* end if */ + + nerrors += test_fapl(); + + if(use_file_locking) { + nerrors += test_shadow_index_lookup(); + + nerrors += test_file_fapl(); + nerrors += test_writer_create_open_flush(); + nerrors += test_writer_md(); + nerrors += test_reader_md_concur(); + + nerrors += test_multiple_file_opens(); + nerrors += test_multiple_file_opens_concur(); + nerrors += test_same_file_opens(); + + nerrors += test_enable_disable_eot(); + nerrors += test_disable_enable_eot_concur(); + + nerrors += test_file_end_tick(); + nerrors += test_file_end_tick_concur(); + } + + if(nerrors) + goto error; + + HDputs("All VFD SWMR tests passed."); + + HDexit(EXIT_SUCCESS); + +error: + HDprintf("***** %d VFD SWMR TEST%s FAILED! *****\n", + nerrors, nerrors > 1 ? "S" : ""); + + H5E_BEGIN_TRY { + H5Pclose(fapl); + } H5E_END_TRY; + + HDexit(EXIT_FAILURE); +} diff --git a/test/vfd_swmr_addrem_writer.c b/test/vfd_swmr_addrem_writer.c new file mode 100644 index 0000000..a8aefa9 --- /dev/null +++ b/test/vfd_swmr_addrem_writer.c @@ -0,0 +1,502 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: vfd_swmr_addrem_writer.c + * (copied and modified from swmr_addrem_writer.c) + * + * Purpose: Adds and removes data to a randomly selected subset of the + * datasets in the VFD SWMR test file. + * + * This program is intended to run concurrently with the + * vfd_swmr_reader program. It is also run AFTER a sequential + * (not concurrent!) invoking of vfd_swmr_writer so the writer + * can dump a bunch of data into the datasets. Otherwise, + * there wouldn't be much to shrink :) + * + *------------------------------------------------------------------------- + */ + +/***********/ +/* Headers */ +/***********/ + +#include <err.h> /* errx(3) */ +#include <stdlib.h> /* EXIT_FAILURE */ + +#include "h5test.h" +#include "vfd_swmr_common.h" + +/****************/ +/* Local Macros */ +/****************/ + +/********************/ +/* Local Prototypes */ +/********************/ + +static hid_t open_skeleton(const char *filename, unsigned verbose); +static int addrem_records(hid_t fid, unsigned verbose, unsigned long nops, + unsigned long flush_count); +static void usage(void); + + +/*------------------------------------------------------------------------- + * Function: open_skeleton + * + * Purpose: Opens the SWMR HDF5 file and datasets. + * + * Parameters: const char *filename + * The filename of the SWMR HDF5 file to open + * + * unsigned verbose + * Whether or not to emit verbose console messages + * + * Return: Success: The file ID of the opened SWMR file + * The dataset IDs are stored in a global array + * + * Failure: -1 + * + *------------------------------------------------------------------------- + */ +static hid_t +open_skeleton(const char *filename, unsigned verbose) +{ + hid_t dapl = H5I_INVALID_HID; + hid_t fid = -1; /* File ID for new HDF5 file */ + hid_t fapl = -1; /* File access property list */ + hid_t sid = -1; /* Dataspace ID */ + hsize_t dim[2]; /* Dataspace dimension */ + unsigned u, v; /* Local index variable */ + H5F_vfd_swmr_config_t *config = NULL; /* Configuration for VFD SWMR */ + + HDassert(filename); + + /* Create file access property list */ + if((fapl = h5_fileaccess()) < 0) + goto error; + + if ((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + errx(EXIT_FAILURE, "%s.%d: H5Pcreate failed", __func__, __LINE__); + + if (H5Pset_chunk_cache(dapl, H5D_CHUNK_CACHE_NSLOTS_DEFAULT, 0, + H5D_CHUNK_CACHE_W0_DEFAULT) < 0) + errx(EXIT_FAILURE, "H5Pset_chunk_cache failed"); + + /* Set to use the latest library format */ + if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + goto error; + + /* + * Set up to open the file with VFD SWMR configured. + */ + + /* Enable page buffering */ + if(H5Pset_page_buffer_size(fapl, 4096, 0, 0) < 0) + goto error; + + /* Allocate memory for the configuration structure */ + if((config = (H5F_vfd_swmr_config_t *)calloc(1, sizeof(H5F_vfd_swmr_config_t))) == NULL) + goto error; + + config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION; + config->tick_len = 4; + config->max_lag = 5; + config->writer = TRUE; + config->md_pages_reserved = 128; + HDstrcpy(config->md_file_path, "./my_md_file"); + + /* Enable VFD SWMR configuration */ + if(H5Pset_vfd_swmr_config(fapl, config) < 0) + goto error; + + /* Open the file */ + if((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) + goto error; + + /* Close file access property list */ + if(H5Pclose(fapl) < 0) + goto error; + + if(config) + HDfree(config); + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "WRITER: Opening datasets\n"); + + /* Open the datasets */ + for(u = 0; u < NLEVELS; u++) + for(v = 0; v < symbol_count[u]; v++) { + hid_t dsid; + + if((dsid = H5Dopen2(fid, symbol_info[u][v].name, dapl)) < 0) + goto error; + + symbol_info[u][v].dsid = dsid; + + if((sid = H5Dget_space(symbol_info[u][v].dsid)) < 0) + goto error; + if(2 != H5Sget_simple_extent_ndims(sid)) + goto error; + if(H5Sget_simple_extent_dims(sid, dim, NULL) < 0) + goto error; + symbol_info[u][v].nrecords = dim[1]; + + if(H5Sclose(sid) < 0) + goto error; + } /* end for */ + + return fid; + +error: + if(config) + HDfree(config); + + H5E_BEGIN_TRY { + for(u = 0; u < NLEVELS; u++) + for(v = 0; v < symbol_count[u]; v++) + H5Dclose(symbol_info[u][v].dsid); + H5Sclose(sid); + H5Pclose(fapl); + H5Fclose(fid); + H5Pclose(dapl); + } H5E_END_TRY; + + return -1; + +} /* open_skeleton() */ + + +/*------------------------------------------------------------------------- + * Function: addrem_records + * + * Purpose: Adds/removes a specified number of records to random datasets + * to the SWMR test file. + * + * Parameters: hid_t fid + * The file ID of the SWMR HDF5 file + * + * unsigned verbose + * Whether or not to emit verbose console messages + * + * unsigned long nops + * # of records to read/write in the datasets + * + * unsigned long flush_count + * # of records to write before flushing the file to disk + * + * Return: Success: 0 + * Failure: -1 + * + *------------------------------------------------------------------------- + */ +static int +addrem_records(hid_t fid, unsigned verbose, unsigned long nops, unsigned long flush_count) +{ + hid_t tid = -1; /* Datatype ID for records */ + hid_t mem_sid = -1; /* Memory dataspace ID */ + hid_t file_sid = -1; /* Dataset's space ID */ + hsize_t start[2] = {0, 0}, count[2] = {1, 1}; /* Hyperslab selection values */ + hsize_t dim[2] = {1, 0}; /* Dataspace dimensions */ + symbol_t buf[MAX_SIZE_CHANGE]; /* Write buffer */ + unsigned long op_to_flush; /* # of operations before flush */ + unsigned long u, v; /* Local index variables */ + + HDassert(fid > 0); + + /* Reset the buffer */ + HDmemset(&buf, 0, sizeof(buf)); + + /* Create a dataspace for the record to add */ + if((mem_sid = H5Screate_simple(2, count, NULL)) < 0) + goto error; + + /* Create datatype for appending records */ + if((tid = create_symbol_datatype()) < 0) + goto error; + + /* Add and remove records to random datasets, according to frequency + * distribution */ + op_to_flush = flush_count; + for(u=0; u<nops; u++) { + symbol_info_t *symbol; /* Symbol to write record to */ + + /* Get a random dataset, according to the symbol distribution */ + symbol = choose_dataset(NULL, NULL); + + /* Decide whether to shrink or expand, and by how much */ + count[1] = (hsize_t)HDrandom() % (MAX_SIZE_CHANGE * 2) + 1; + + if(count[1] > MAX_SIZE_CHANGE) { + /* Add records */ + count[1] -= MAX_SIZE_CHANGE; + + /* Set the buffer's IDs (equal to its position) */ + for(v=0; v<count[1]; v++) + buf[v].rec_id = (uint64_t)symbol->nrecords + (uint64_t)v; + + /* Set the memory space to the correct size */ + if(H5Sset_extent_simple(mem_sid, 2, count, NULL) < 0) + goto error; + + /* Get the coordinates to write */ + start[1] = symbol->nrecords; + + /* Extend the dataset's dataspace to hold the new record */ + symbol->nrecords+= count[1]; + dim[1] = symbol->nrecords; + if(H5Dset_extent(symbol->dsid, dim) < 0) + goto error; + + /* Get the dataset's dataspace */ + if((file_sid = H5Dget_space(symbol->dsid)) < 0) + goto error; + + /* Choose the last record in the dataset */ + if(H5Sselect_hyperslab(file_sid, H5S_SELECT_SET, start, NULL, count, NULL) < 0) + goto error; + + /* Write record to the dataset */ + if(H5Dwrite(symbol->dsid, tid, mem_sid, file_sid, H5P_DEFAULT, &buf) < 0) + goto error; + + if(H5Dflush(symbol->dsid) < 0) + goto error; + + /* Close the dataset's dataspace */ + if(H5Sclose(file_sid) < 0) + goto error; + } /* end if */ + else { + /* Shrink the dataset's dataspace */ + if(count[1] > symbol->nrecords) + symbol->nrecords = 0; + else + symbol->nrecords -= count[1]; + dim[1] = symbol->nrecords; + if(H5Dset_extent(symbol->dsid, dim) < 0) + goto error; + } /* end else */ + + /* Check for flushing file */ + if(flush_count > 0) { + /* Decrement count of records to write before flushing */ + op_to_flush--; + + /* Check for counter being reached */ + if(0 == op_to_flush) { +#ifdef TEMP_OUT + /* Flush contents of file */ + if(H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0) + return -1; +#endif /* TEMP_OUT */ + + /* Reset flush counter */ + op_to_flush = flush_count; + } /* end if */ + } /* end if */ + } /* end for */ + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "WRITER: Closing datasets\n"); + + /* Close the datasets */ + for(u = 0; u < NLEVELS; u++) + for(v = 0; v < symbol_count[u]; v++) + if(H5Dclose(symbol_info[u][v].dsid) < 0) + return -1; + + return 0; + +error: + H5E_BEGIN_TRY { + H5Sclose(mem_sid); + H5Sclose(file_sid); + H5Tclose(tid); + + for(u = 0; u < NLEVELS; u++) + for(v = 0; v < symbol_count[u]; v++) + H5Dclose(symbol_info[u][v].dsid); + + } H5E_END_TRY; + + return -1; + +} /* addrem_records() */ + +static void +usage(void) +{ + printf("\n"); + printf("Usage error!\n"); + printf("\n"); + printf("Usage: vfd_swmr_addrem_writer [-q] [-f <# of operations between flushing\n"); + printf(" file contents>] [-r <random seed>] <# of operations>\n"); + printf("\n"); + printf("<# of operations between flushing file contents> should be 0 (for\n"); + printf("no flushing) or between 1 and (<# of operations> - 1).\n"); + printf("\n"); + printf("<# of operations> must be specified.\n"); + printf("\n"); + printf("Defaults to verbose (no '-q' given), flushing every 1000 operations\n"); + printf("('-f 1000'), and will generate a random seed (no -r given).\n"); + printf("\n"); + HDexit(1); +} /* usage() */ + +int main(int argc, const char *argv[]) +{ + sigset_t oldset; + hid_t fid; /* File ID for file opened */ + long nops = 0; /* # of times to grow or shrink the dataset */ + long flush_count = 1000; /* # of records to write between flushing file */ + unsigned verbose = 1; /* Whether to emit some informational messages */ + unsigned use_seed = 0; /* Set to 1 if a seed was set on the command line */ + unsigned random_seed = 0; /* Random # seed */ + unsigned u; /* Local index variable */ + int temp; + + block_signals(&oldset); + + /* Parse command line options */ + if(argc < 2) + usage(); + if(argc > 1) { + u = 1; + while(u < (unsigned)argc) { + if(argv[u][0] == '-') { + switch(argv[u][1]) { + /* # of records to write between flushing file */ + case 'f': + flush_count = HDatol(argv[u + 1]); + if(flush_count < 0) + usage(); + u += 2; + break; + + /* Be quiet */ + case 'q': + verbose = 0; + u++; + break; + + /* Random # seed */ + case 'r': + use_seed = 1; + temp = HDatoi(argv[u + 1]); + if(temp < 0) + usage(); + else + random_seed = (unsigned)temp; + u += 2; + break; + + default: + usage(); + break; + } /* end switch */ + } /* end if */ + else { + /* Get the number of records to append */ + nops = HDatol(argv[u]); + if(nops <= 0) + usage(); + + u++; + } /* end else */ + } /* end while */ + } /* end if */ + if(nops <= 0) + usage(); + if(flush_count >= nops) + usage(); + + /* Emit informational message */ + if(verbose) { + HDfprintf(stderr, "WRITER: Parameters:\n"); + HDfprintf(stderr, "\t# of operations between flushes = %ld\n", flush_count); + HDfprintf(stderr, "\t# of operations = %ld\n", nops); + } /* end if */ + + /* Set the random seed */ + if(0 == use_seed) { + struct timeval t; + HDgettimeofday(&t, NULL); + random_seed = (unsigned)(t.tv_usec); + } /* end if */ + HDsrandom(random_seed); + /* ALWAYS emit the random seed for possible debugging */ + HDfprintf(stderr, "WRITER: Using writer random seed: %u\n", random_seed); + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "WRITER: Generating symbol names\n"); + + /* Generate dataset names */ + if(generate_symbols() < 0) + return -1; + + /* Emit informational message */ + if(verbose) { + HDfprintf(stderr, "WRITER: Opening skeleton file: %s\n", + COMMON_FILENAME); + } + + /* Open file skeleton */ + if((fid = open_skeleton(COMMON_FILENAME, verbose)) < 0) { + HDfprintf(stderr, "WRITER: Error opening skeleton file!\n"); + HDexit(1); + } /* end if */ + + /* Send a message to indicate "H5Fopen" is complete--releasing the file lock */ + h5_send_message(WRITER_MESSAGE, NULL, NULL); + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "WRITER: Adding and removing records\n"); + + /* Grow and shrink datasets */ + if(addrem_records(fid, verbose, (unsigned long)nops, (unsigned long)flush_count) < 0) { + HDfprintf(stderr, "WRITER: Error adding and removing records from datasets!\n"); + HDexit(1); + } /* end if */ + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "WRITER: Releasing symbols\n"); + + /* Clean up the symbols */ + if(shutdown_symbols() < 0) { + HDfprintf(stderr, "WRITER: Error releasing symbols!\n"); + HDexit(1); + } /* end if */ + + await_signal(fid); + + restore_signals(&oldset); + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "WRITER: Closing objects\n"); + + /* Close objects opened */ + if(H5Fclose(fid) < 0) { + HDfprintf(stderr, "WRITER: Error closing file!\n"); + HDexit(1); + } /* end if */ + + return 0; +} /* main() */ diff --git a/test/vfd_swmr_bigset_writer.c b/test/vfd_swmr_bigset_writer.c new file mode 100644 index 0000000..6d65c8b --- /dev/null +++ b/test/vfd_swmr_bigset_writer.c @@ -0,0 +1,1104 @@ +/* + * Copyright by The HDF Group. + * Copyright by the Board of Trustees of the University of Illinois. + * All rights reserved. + * + * This file is part of HDF5. The full HDF5 copyright notice, including + * terms governing use, modification, and redistribution, is contained in + * the COPYING file, which can be found at the root of the source code + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. + * If you do not have access to either file, you may request a copy from + * help@hdfgroup.org. + */ + +#include <err.h> +#include <libgen.h> +#include <time.h> /* nanosleep(2) */ +#include <unistd.h> /* getopt(3) */ + +#define H5C_FRIEND /*suppress error about including H5Cpkg */ +#define H5F_FRIEND /*suppress error about including H5Fpkg */ + +#include "hdf5.h" + +#include "H5Cpkg.h" +#include "H5Fpkg.h" +// #include "H5Iprivate.h" +#include "H5HGprivate.h" +#include "H5VLprivate.h" + +#include "testhdf5.h" +#include "vfd_swmr_common.h" + +#define ROWS 256 +#define COLS 512 +#define RANK 2 + +static const unsigned int hang_back = 3; + +typedef struct _base { + hsize_t row, col; +} base_t; + +typedef struct _mat { + unsigned rows, cols; + uint32_t elt[1]; +} mat_t; + +typedef struct _quadrant { + hsize_t start[RANK]; + hsize_t stride[RANK]; + hsize_t block[RANK]; + hsize_t count[RANK]; + hid_t space, src_space; +} quadrant_t; + +typedef struct _sources { + hid_t ul, ur, bl, br; +} sources_t; + +#define MANY_FILES 4 + +typedef struct { + hid_t *dataset; + sources_t *sources; + hid_t file[MANY_FILES]; + hid_t dapl, filetype, memspace, one_by_one_sid, quadrant_dcpl; + unsigned ndatasets; + const char *filename[MANY_FILES]; + char progname[PATH_MAX]; + struct timespec update_interval; + struct { + quadrant_t ul, ur, bl, br, src; + } quadrants; + unsigned int cols, rows; + unsigned int asteps; + unsigned int nsteps; + bool two_dee; + bool wait_for_signal; + enum {vds_off, vds_single, vds_multi} vds; + bool use_vfd_swmr; + bool writer; + hsize_t chunk_dims[RANK]; + hsize_t one_dee_max_dims[RANK]; +} state_t; + +static inline state_t +state_initializer(void) +{ + return (state_t){ + .memspace = H5I_INVALID_HID + , .dapl = H5I_INVALID_HID + , .file = {H5I_INVALID_HID, H5I_INVALID_HID, + H5I_INVALID_HID, H5I_INVALID_HID} + , .filetype = H5T_NATIVE_UINT32 + , .one_by_one_sid = H5I_INVALID_HID + , .quadrant_dcpl = H5I_INVALID_HID + , .rows = ROWS + , .cols = COLS + , .ndatasets = 5 + , .asteps = 10 + , .nsteps = 100 + , .filename = {"", "", "", ""} + , .two_dee = false + , .wait_for_signal = true + , .vds = vds_off + , .use_vfd_swmr = true + , .writer = true + , .one_dee_max_dims = {ROWS, H5S_UNLIMITED} + , .chunk_dims = {ROWS, COLS} + , .update_interval = (struct timespec){ + .tv_sec = 0 + , .tv_nsec = 1000000000UL / 30 /* 1/30 second */}}; +} + +static void state_init(state_t *, int, char **); + +static const hid_t badhid = H5I_INVALID_HID; + +static const hsize_t two_dee_max_dims[RANK] = {H5S_UNLIMITED, H5S_UNLIMITED}; + +static uint32_t +matget(const mat_t *mat, unsigned i, unsigned j) +{ + assert(i < mat->rows && j < mat->cols); + + return mat->elt[i * mat->cols + j]; +} + +static void +matset(mat_t *mat, unsigned i, unsigned j, uint32_t v) +{ + assert(i < mat->rows && j < mat->cols); + + mat->elt[i * mat->cols + j] = v; +} + +static mat_t * +newmat(unsigned rows, unsigned cols) +{ + mat_t *mat; + + mat = malloc(sizeof(*mat) + (rows * cols - 1) * sizeof(mat->elt[0])); + + if (mat == NULL) + err(EXIT_FAILURE, "%s: malloc", __func__); + + mat->rows = rows; + mat->cols = cols; + + return mat; +} + +static void +usage(const char *progname) +{ + fprintf(stderr, "usage: %s [-S] [-W] [-a steps] [-b] [-c cols]\n" + " [-d dims]\n" + " [-n iterations] [-r rows] [-s datasets]\n" + " [-u milliseconds]\n" + "\n" + "-M: use virtual datasets and many source\n" + " files\n" + "-S: do not use VFD SWMR\n" + "-V: use virtual datasets and a single\n" + " source file\n" + "-W: do not wait for a signal before\n" + " exiting\n" + "-a steps: `steps` between adding attributes\n" + "-b: write data in big-endian byte order\n" + "-c cols: `cols` columns per chunk\n" + "-d 1|one|2|two|both: select dataset expansion in one or\n" + " both dimensions\n" + "-n iterations: how many times to expand each dataset\n" + "-r rows: `rows` rows per chunk\n" + "-s datasets: number of datasets to create\n" + "-u ms: milliseconds interval between updates\n" + " to %s.h5\n" + "\n", + progname, progname); + exit(EXIT_FAILURE); +} + +static void +make_quadrant_dataspace(state_t *s, quadrant_t *q) +{ + q->space = H5Screate_simple(NELMTS(s->chunk_dims), s->chunk_dims, + s->two_dee ? two_dee_max_dims : s->one_dee_max_dims); + + if (q->space < 0) { + errx(EXIT_FAILURE, "%s.%d: H5Screate_simple failed", + __func__, __LINE__); + } + + if (H5Sselect_hyperslab(q->space, H5S_SELECT_SET, q->start, q->stride, + q->count, q->block) < 0) + errx(EXIT_FAILURE, "%s: H5Sselect_hyperslab failed", __func__); +} + +static void +state_init(state_t *s, int argc, char **argv) +{ + unsigned long tmp; + int ch; + unsigned i; + const hsize_t dims = 1; + char tfile[PATH_MAX]; + char *end; + unsigned long millis; + quadrant_t * const ul = &s->quadrants.ul, + * const ur = &s->quadrants.ur, + * const bl = &s->quadrants.bl, + * const br = &s->quadrants.br, + * const src = &s->quadrants.src; + const char *personality; + + *s = state_initializer(); + esnprintf(tfile, sizeof(tfile), "%s", argv[0]); + esnprintf(s->progname, sizeof(s->progname), "%s", basename(tfile)); + + while ((ch = getopt(argc, argv, "MSVWa:bc:d:n:qr:s:u:")) != -1) { + switch (ch) { + case 'M': + s->vds = vds_multi; + break; + case 'S': + s->use_vfd_swmr = false; + break; + case 'V': + s->vds = vds_single; + break; + case 'W': + s->wait_for_signal = false; + break; + case 'd': + if (strcmp(optarg, "1") == 0 || + strcmp(optarg, "one") == 0) + s->two_dee = false; + else if (strcmp(optarg, "2") == 0 || + strcmp(optarg, "two") == 0 || + strcmp(optarg, "both") == 0) + s->two_dee = true; + else { + errx(EXIT_FAILURE, + "bad -d argument \"%s\"", optarg); + } + break; + case 'a': + case 'c': + case 'n': + case 'r': + case 's': + errno = 0; + tmp = strtoul(optarg, &end, 0); + if (end == optarg || *end != '\0') { + errx(EXIT_FAILURE, "couldn't parse `-%c` argument `%s`", ch, + optarg); + } else if (errno != 0) { + err(EXIT_FAILURE, "couldn't parse `-%c` argument `%s`", ch, + optarg); + } else if (tmp > UINT_MAX) + errx(EXIT_FAILURE, "`-%c` argument `%lu` too large", ch, tmp); + + if ((ch == 'c' || ch == 'r') && tmp == 0) { + errx(EXIT_FAILURE, "`-%c` argument `%lu` must be >= 1", ch, + tmp); + } + + if (ch == 'a') + s->asteps = (unsigned)tmp; + else if (ch == 'c') + s->cols = (unsigned)tmp; + else if (ch == 'n') + s->nsteps = (unsigned)tmp; + else if (ch == 'r') + s->rows = (unsigned)tmp; + else + s->ndatasets = (unsigned)tmp; + break; + case 'b': + s->filetype = H5T_STD_U32BE; + break; + case 'q': + verbosity = 0; + break; + case 'u': + errno = 0; + millis = strtoul(optarg, &end, 0); + if (millis == ULONG_MAX && errno == ERANGE) { + err(EXIT_FAILURE, + "option -p argument \"%s\"", optarg); + } else if (*end != '\0') { + errx(EXIT_FAILURE, + "garbage after -p argument \"%s\"", optarg); + } + s->update_interval.tv_sec = (time_t)(millis / 1000UL); + s->update_interval.tv_nsec = + (long)((millis * 1000000UL) % 1000000000UL); + break; + case '?': + default: + usage(s->progname); + break; + } + } + argc -= optind; + argv += optind; + + if (argc > 0) + errx(EXIT_FAILURE, "unexpected command-line arguments"); + + if (s->vds != vds_off && s->two_dee) { + errx(EXIT_FAILURE, + "virtual datasets and 2D datasets are mutually exclusive"); + } + + s->chunk_dims[0] = s->rows; + s->chunk_dims[1] = s->cols; + s->one_dee_max_dims[0] = s->rows; + s->one_dee_max_dims[1] = H5S_UNLIMITED; + + if (s->vds != vds_off) { + const hsize_t half_chunk_dims[RANK] = {s->rows / 2, s->cols / 2}; + const hsize_t half_max_dims[RANK] = {s->rows / 2, H5S_UNLIMITED}; + + if ((s->quadrant_dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) { + errx(EXIT_FAILURE, "%s.%d: H5Pcreate failed", + __func__, __LINE__); + } + + if (H5Pset_chunk(s->quadrant_dcpl, RANK, half_chunk_dims) < 0) + errx(EXIT_FAILURE, "H5Pset_chunk failed"); + + *ul = (quadrant_t){ + .start = {0, 0} + , .stride = {s->rows, s->cols} + , .block = {s->rows / 2, s->cols / 2} + , .count = {1, H5S_UNLIMITED}}; + + *ur = (quadrant_t){ + .start = {s->rows / 2, 0} + , .stride = {s->rows, s->cols} + , .block = {s->rows / 2, s->cols / 2} + , .count = {1, H5S_UNLIMITED}}; + + *bl = (quadrant_t){ + .start = {0, s->cols / 2} + , .stride = {s->rows, s->cols} + , .block = {s->rows / 2, s->cols / 2} + , .count = {1, H5S_UNLIMITED}}; + + *br = (quadrant_t){ + .start = {s->rows / 2, s->cols / 2} + , .stride = {s->rows, s->cols} + , .block = {s->rows / 2, s->cols / 2} + , .count = {1, H5S_UNLIMITED}}; + + make_quadrant_dataspace(s, ul); + make_quadrant_dataspace(s, ur); + make_quadrant_dataspace(s, bl); + make_quadrant_dataspace(s, br); + + *src = (quadrant_t){ + .start = {0, 0} + , .stride = {s->rows / 2, s->cols / 2} + , .block = {s->rows / 2, s->cols / 2} + , .count = {1, H5S_UNLIMITED}}; + + src->space = H5Screate_simple(RANK, half_chunk_dims, + half_max_dims); + + if (src->space < 0) { + errx(EXIT_FAILURE, "%s.%d: H5Screate_simple failed", + __func__, __LINE__); + } + + if (H5Sselect_hyperslab(src->space, H5S_SELECT_SET, src->start, + src->stride, src->count, src->block) < 0) + errx(EXIT_FAILURE, "%s: H5Sselect_hyperslab failed", __func__); + + ul->src_space = H5Screate_simple(RANK, half_chunk_dims, half_max_dims); + + if (ul->src_space < 0) { + errx(EXIT_FAILURE, "%s.%d: H5Screate_simple failed", + __func__, __LINE__); + } + + ur->src_space = H5Screate_simple(RANK, half_chunk_dims, half_max_dims); + + if (ur->src_space < 0) { + errx(EXIT_FAILURE, "%s.%d: H5Screate_simple failed", + __func__, __LINE__); + } + + bl->src_space = H5Screate_simple(RANK, half_chunk_dims, half_max_dims); + + if (bl->src_space < 0) { + errx(EXIT_FAILURE, "%s.%d: H5Screate_simple failed", + __func__, __LINE__); + } + + br->src_space = H5Screate_simple(RANK, half_chunk_dims, half_max_dims); + + if (br->src_space < 0) { + errx(EXIT_FAILURE, "%s.%d: H5Screate_simple failed", + __func__, __LINE__); + } + } + + /* space for attributes */ + if ((s->one_by_one_sid = H5Screate_simple(1, &dims, &dims)) < 0) + errx(EXIT_FAILURE, "H5Screate_simple failed"); + + s->dataset = malloc(sizeof(*s->dataset) * s->ndatasets); + if (s->dataset == NULL) + err(EXIT_FAILURE, "could not allocate dataset handles"); + + s->sources = malloc(sizeof(*s->sources) * s->ndatasets); + if (s->sources == NULL) + err(EXIT_FAILURE, "could not allocate quadrant dataset handles"); + + for (i = 0; i < s->ndatasets; i++) { + s->dataset[i] = badhid; + s->sources[i].ul = s->sources[i].ur = s->sources[i].bl = + s->sources[i].br = badhid; + } + + s->memspace = H5Screate_simple(RANK, s->chunk_dims, NULL); + + if (s->memspace < 0) { + errx(EXIT_FAILURE, "%s.%d: H5Screate_simple failed", + __func__, __LINE__); + } + + s->filename[0] = "vfd_swmr_bigset.h5"; + if (s->vds == vds_multi) { + s->filename[1] = "vfd_swmr_bigset-ur.h5"; + s->filename[2] = "vfd_swmr_bigset-bl.h5"; + s->filename[3] = "vfd_swmr_bigset-br.h5"; + } else { + s->filename[1] = s->filename[0]; + s->filename[2] = s->filename[0]; + s->filename[3] = s->filename[0]; + } + + personality = strstr(s->progname, "vfd_swmr_bigset_"); + + if (personality != NULL && + strcmp(personality, "vfd_swmr_bigset_writer") == 0) + s->writer = true; + else if (personality != NULL && + strcmp(personality, "vfd_swmr_bigset_reader") == 0) + s->writer = false; + else { + errx(EXIT_FAILURE, + "unknown personality, expected vfd_swmr_bigset_{reader,writer}"); + } + + if ((s->dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + errx(EXIT_FAILURE, "%s.%d: H5Pcreate failed", __func__, __LINE__); + + if (H5Pset_chunk_cache(s->dapl, 0, 0, + H5D_CHUNK_CACHE_W0_DEFAULT) < 0) + errx(EXIT_FAILURE, "H5Pset_chunk_cache failed"); + + if (s->vds != vds_off && + H5Pset_virtual_view(s->dapl, H5D_VDS_FIRST_MISSING) < 0) + errx(EXIT_FAILURE, "H5Pset_virtual_view failed"); +} + +static void +state_destroy(state_t *s) +{ + size_t i; + + if (H5Pclose(s->dapl) < 0) + errx(EXIT_FAILURE, "H5Pclose(fapl)"); + + s->dapl = badhid; + + if (s->vds != vds_off) { + quadrant_t * const ul = &s->quadrants.ul, + * const ur = &s->quadrants.ur, + * const bl = &s->quadrants.bl, + * const br = &s->quadrants.br; + + if (H5Sclose(ul->src_space) < 0 || + H5Sclose(ur->src_space) < 0 || + H5Sclose(bl->src_space) < 0 || + H5Sclose(br->src_space) < 0) + errx(EXIT_FAILURE, "H5Sclose failed"); + + ul->src_space = ur->src_space = bl->src_space = br->src_space = badhid; + + if (H5Pclose(s->quadrant_dcpl) < 0) + errx(EXIT_FAILURE, "H5Pclose(dcpl)"); + + s->quadrant_dcpl = badhid; + + /* TBD destroy spaces belonging to quadrants */ + } + + for (i = 0; i < NELMTS(s->file); i++) { + hid_t fid = s->file[i]; + + s->file[i] = badhid; + + if (s->vds != vds_multi && i > 0) + continue; + + if (H5Fclose(fid) < 0) + errx(EXIT_FAILURE, "H5Fclose"); + } +} + +static void +create_extensible_dset(state_t *s, unsigned int which) +{ + quadrant_t * const ul = &s->quadrants.ul, + * const ur = &s->quadrants.ur, + * const bl = &s->quadrants.bl, + * const br = &s->quadrants.br, + * const src = &s->quadrants.src; + char dname[sizeof("/dataset-9999999999")]; + char ul_dname[sizeof("/ul-dataset-9999999999")], + ur_dname[sizeof("/ur-dataset-9999999999")], + bl_dname[sizeof("/bl-dataset-9999999999")], + br_dname[sizeof("/br-dataset-9999999999")]; + hid_t dcpl, ds, filespace; + + assert(which < s->ndatasets); + assert(s->dataset[which] == badhid); + + esnprintf(dname, sizeof(dname), "/dataset-%d", which); + + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) { + errx(EXIT_FAILURE, "%s.%d: H5Pcreate failed", + __func__, __LINE__); + } + + if (H5Pset_chunk(dcpl, RANK, s->chunk_dims) < 0) + errx(EXIT_FAILURE, "H5Pset_chunk failed"); + + if (s->vds != vds_off) { + sources_t * const srcs = &s->sources[which]; + + esnprintf(ul_dname, sizeof(ul_dname), "/ul-dataset-%d", which); + esnprintf(ur_dname, sizeof(ur_dname), "/ur-dataset-%d", which); + esnprintf(bl_dname, sizeof(bl_dname), "/bl-dataset-%d", which); + esnprintf(br_dname, sizeof(br_dname), "/br-dataset-%d", which); + + srcs->ul = H5Dcreate2(s->file[0], ul_dname, s->filetype, ul->src_space, + H5P_DEFAULT, s->quadrant_dcpl, s->dapl); + + if (srcs->ul < 0) + errx(EXIT_FAILURE, "H5Dcreate(, \"%s\", ) failed", ul_dname); + + srcs->ur = H5Dcreate2(s->file[1], ur_dname, s->filetype, ur->src_space, + H5P_DEFAULT, s->quadrant_dcpl, s->dapl); + + if (srcs->ur < 0) + errx(EXIT_FAILURE, "H5Dcreate(, \"%s\", ) failed", ur_dname); + + srcs->bl = H5Dcreate2(s->file[2], bl_dname, s->filetype, bl->src_space, + H5P_DEFAULT, s->quadrant_dcpl, s->dapl); + + if (srcs->bl < 0) + errx(EXIT_FAILURE, "H5Dcreate(, \"%s\", ) failed", bl_dname); + + srcs->br = H5Dcreate2(s->file[3], br_dname, s->filetype, br->src_space, + H5P_DEFAULT, s->quadrant_dcpl, s->dapl); + + if (srcs->br < 0) + errx(EXIT_FAILURE, "H5Dcreate(, \"%s\", ) failed", br_dname); + + if (H5Pset_virtual(dcpl, ul->space, s->filename[0], ul_dname, + src->space) < 0) + errx(EXIT_FAILURE, "%s: H5Pset_virtual failed", __func__); + + if (H5Pset_virtual(dcpl, ur->space, s->filename[1], ur_dname, + src->space) < 0) + errx(EXIT_FAILURE, "%s: H5Pset_virtual failed", __func__); + + if (H5Pset_virtual(dcpl, bl->space, s->filename[2], bl_dname, + src->space) < 0) + errx(EXIT_FAILURE, "%s: H5Pset_virtual failed", __func__); + + if (H5Pset_virtual(dcpl, br->space, s->filename[3], br_dname, + src->space) < 0) + errx(EXIT_FAILURE, "%s: H5Pset_virtual failed", __func__); + } + + filespace = H5Screate_simple(NELMTS(s->chunk_dims), s->chunk_dims, + s->two_dee ? two_dee_max_dims : s->one_dee_max_dims); + + if (filespace < 0) { + errx(EXIT_FAILURE, "%s.%d: H5Screate_simple failed", + __func__, __LINE__); + } + + ds = H5Dcreate2(s->file[0], dname, s->filetype, filespace, + H5P_DEFAULT, dcpl, s->dapl); + + if (ds < 0) + errx(EXIT_FAILURE, "H5Dcreate(, \"%s\", ) failed", dname); + + if (H5Sclose(filespace) < 0) + errx(EXIT_FAILURE, "%s: H5Sclose failed", __func__); + + if (H5Pclose(dcpl) < 0) + errx(EXIT_FAILURE, "%s: H5Pclose failed", __func__); + + s->dataset[which] = ds; +} + +static void +close_extensible_dset(state_t *s, unsigned int which) +{ + char dname[sizeof("/dataset-9999999999")]; + hid_t ds; + + assert(which < s->ndatasets); + + esnprintf(dname, sizeof(dname), "/dataset-%d", which); + + ds = s->dataset[which]; + + if (H5Dclose(ds) < 0) + errx(EXIT_FAILURE, "H5Dclose failed for \"%s\"", dname); + + s->dataset[which] = badhid; + + if (s->vds != vds_off && s->writer) { + sources_t * const srcs = &s->sources[which]; + + if (H5Dclose(srcs->ul) < 0 || H5Dclose(srcs->ur) < 0 || + H5Dclose(srcs->bl) < 0 || H5Dclose(srcs->br) < 0) + errx(EXIT_FAILURE, "H5Dclose failed"); + + srcs->ul = srcs->ur = srcs->bl = srcs->br = badhid; + } +} + +static void +open_extensible_dset(state_t *s, unsigned int which) +{ + hsize_t dims[RANK], maxdims[RANK]; + char dname[sizeof("/dataset-9999999999")]; + hid_t ds, filespace, ty; + + assert(which < s->ndatasets); + assert(s->dataset[which] == badhid); + + esnprintf(dname, sizeof(dname), "/dataset-%d", which); + + ds = H5Dopen(s->file[0], dname, s->dapl); + + if (ds < 0) + errx(EXIT_FAILURE, "H5Dopen(, \"%s\", ) failed", dname); + + if ((ty = H5Dget_type(ds)) < 0) + errx(EXIT_FAILURE, "H5Dget_type failed"); + + if (H5Tequal(ty, s->filetype) <= 0) + errx(EXIT_FAILURE, "Unexpected data type"); + + if ((filespace = H5Dget_space(ds)) < 0) + errx(EXIT_FAILURE, "H5Dget_space failed"); + + if (H5Sget_simple_extent_ndims(filespace) != RANK) + errx(EXIT_FAILURE, "Unexpected rank"); + + if (H5Sget_simple_extent_dims(filespace, dims, maxdims) < 0) + errx(EXIT_FAILURE, "H5Sget_simple_extent_dims failed"); + + if (H5Sclose(filespace) < 0) + errx(EXIT_FAILURE, "H5Sclose failed"); + + filespace = badhid; + + if (s->two_dee) { + if (maxdims[0] != two_dee_max_dims[0] || + maxdims[1] != two_dee_max_dims[1] || + maxdims[0] != maxdims[1]) { + errx(EXIT_FAILURE, "Unexpected maximum dimensions %" + PRIuHSIZE " x %" PRIuHSIZE, maxdims[0], maxdims[1]); + } + } else if (maxdims[0] != s->one_dee_max_dims[0] || + maxdims[1] != s->one_dee_max_dims[1] || + dims[0] != s->chunk_dims[0]) { + errx(EXIT_FAILURE, "Unexpected maximum dimensions %" + PRIuHSIZE " x %" PRIuHSIZE " or columns %" PRIuHSIZE, + maxdims[0], maxdims[1], dims[1]); + } + + s->dataset[which] = ds; +} + +static void +set_or_verify_matrix(mat_t *mat, unsigned int which, base_t base, bool do_set) +{ + unsigned row, col; + + for (row = 0; row < mat->rows; row++) { + for (col = 0; col < mat->cols; col++) { + uint32_t v; + hsize_t i = base.row + row, + j = base.col + col, + u; + + if (j <= i) + u = (i + 1) * (i + 1) - 1 - j; + else + u = j * j + i; + + assert(UINT32_MAX - u >= which); + v = (uint32_t)(u + which); + if (do_set) + matset(mat, row, col, v); + else if (matget(mat, row, col) != v) { + errx(EXIT_FAILURE, "matrix mismatch " + "at %" PRIuHSIZE ", %" PRIuHSIZE " (%u, %u), " + "read %" PRIu32 " expecting %" PRIu32, + i, j, row, col, matget(mat, row, col), v); + } + } + } +} + +static void +init_matrix(mat_t *mat, unsigned int which, base_t base) +{ + set_or_verify_matrix(mat, which, base, true); +} + +static void +verify_matrix(mat_t *mat, unsigned int which, base_t base) +{ + set_or_verify_matrix(mat, which, base, false); +} + +static void +verify_chunk(state_t *s, hid_t filespace, + mat_t *mat, unsigned which, base_t base) +{ + hsize_t offset[RANK] = {base.row, base.col}; + herr_t status; + hid_t ds; + + assert(which < s->ndatasets); + + dbgf(1, "verifying chunk %" PRIuHSIZE ", %" PRIuHSIZE "\n", + base.row, base.col); + + ds = s->dataset[which]; + + status = H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, + NULL, s->chunk_dims, NULL); + + if (status < 0) + errx(EXIT_FAILURE, "H5Sselect_hyperslab failed"); + + status = H5Dread(ds, H5T_NATIVE_UINT32, s->memspace, filespace, + H5P_DEFAULT, mat->elt); + + if (status < 0) + errx(EXIT_FAILURE, "H5Dread failed"); + + verify_matrix(mat, which, base); +} + +static void +init_and_write_chunk(state_t *s, hid_t filespace, + mat_t *mat, unsigned which, base_t base) +{ + hsize_t offset[RANK] = {base.row, base.col}; + herr_t status; + hid_t ds; + + assert(which < s->ndatasets); + + ds = s->dataset[which]; + + init_matrix(mat, which, base); + + status = H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, + NULL, s->chunk_dims, NULL); + + if (status < 0) + errx(EXIT_FAILURE, "H5Sselect_hyperslab failed"); + + status = H5Dwrite(ds, H5T_NATIVE_UINT32, s->memspace, filespace, + H5P_DEFAULT, mat->elt); + + if (status < 0) + errx(EXIT_FAILURE, "H5Dwrite failed"); +} + +static void +verify_dset_attribute(hid_t ds, unsigned int which, unsigned int step) +{ + unsigned int read_step; + hid_t aid; + char name[sizeof("attr-9999999999")]; + + esnprintf(name, sizeof(name), "attr-%u", step); + + dbgf(1, "verifying attribute %s on dataset %u equals %u\n", name, which, + step); + + if ((aid = H5Aopen(ds, name, H5P_DEFAULT)) < 0) + errx(EXIT_FAILURE, "H5Aopen failed"); + + if (H5Aread(aid, H5T_NATIVE_UINT, &read_step) < 0) + errx(EXIT_FAILURE, "H5Aread failed"); + + if (H5Aclose(aid) < 0) + errx(EXIT_FAILURE, "H5Aclose failed"); + + if (read_step != step) + errx(EXIT_FAILURE, "expected %u read %u", step, read_step); +} + +static void +verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, + unsigned *stepp) +{ + hid_t ds, filespace; + hsize_t size[RANK]; + base_t base, last; + unsigned int ncols, last_step, step; + + assert(which < s->ndatasets); + + ds = s->dataset[which]; + + if (H5Drefresh(ds) < 0) + errx(EXIT_FAILURE, "H5Drefresh failed"); + + filespace = H5Dget_space(ds); + + if (filespace == badhid) + errx(EXIT_FAILURE, "H5Dget_space failed"); + + if (H5Sget_simple_extent_dims(filespace, size, NULL) < 0) + errx(EXIT_FAILURE, "H5Sget_simple_extent_dims failed"); + + ncols = (unsigned)(size[1] / s->chunk_dims[1]); + if (ncols < hang_back) + goto out; + + last_step = ncols - hang_back; + + for (step = *stepp; step <= last_step; step++) { + const unsigned ofs = step % 2; + + dbgf(1, "%s: which %u step %u\n", __func__, which, step); + + if (s->two_dee) { + size[0] = s->chunk_dims[0] * (1 + step); + size[1] = s->chunk_dims[1] * (1 + step); + last.row = s->chunk_dims[0] * step + ofs; + last.col = s->chunk_dims[1] * step + ofs; + } else { + size[0] = s->chunk_dims[0]; + size[1] = s->chunk_dims[1] * (1 + step); + last.row = 0; + last.col = s->chunk_dims[1] * step + ofs; + } + + dbgf(1, "new size %" PRIuHSIZE ", %" PRIuHSIZE "\n", size[0], size[1]); + dbgf(1, "last row %" PRIuHSIZE " col %" PRIuHSIZE "\n", last.row, + last.col); + + if (s->two_dee) { + + /* Down the right side, intersecting the bottom row. */ + base.col = last.col; + for (base.row = ofs; base.row <= last.row; + base.row += s->chunk_dims[0]) { + verify_chunk(s, filespace, mat, which, base); + } + + /* Across the bottom, stopping before the last column to + * avoid re-writing the bottom-right chunk. + */ + base.row = last.row; + for (base.col = ofs; base.col < last.col; + base.col += s->chunk_dims[1]) { + verify_chunk(s, filespace, mat, which, base); + } + } else { + verify_chunk(s, filespace, mat, which, last); + } + if (s->asteps != 0 && step % s->asteps == 0) + verify_dset_attribute(ds, which, step); + } + + *stepp = last_step; + +out: + if (H5Sclose(filespace) < 0) + errx(EXIT_FAILURE, "H5Sclose failed"); +} + +static void +add_dset_attribute(const state_t *s, hid_t ds, hid_t sid, unsigned int which, + unsigned int step) +{ + hid_t aid; + char name[sizeof("attr-9999999999")]; + + esnprintf(name, sizeof(name), "attr-%u", step); + + dbgf(1, "setting attribute %s on dataset %u to %u\n", name, which, step); + + if ((aid = H5Acreate2(ds, name, s->filetype, sid, H5P_DEFAULT, + H5P_DEFAULT)) < 0) + errx(EXIT_FAILURE, "H5Acreate2 failed"); + + if (H5Awrite(aid, H5T_NATIVE_UINT, &step) < 0) + errx(EXIT_FAILURE, "H5Awrite failed"); + if (H5Aclose(aid) < 0) + errx(EXIT_FAILURE, "H5Aclose failed"); +} + +static void +write_extensible_dset(state_t *s, unsigned int which, unsigned int step, + mat_t *mat) +{ + hid_t ds, filespace; + hsize_t size[RANK]; + base_t base, last; + + dbgf(1, "%s: which %u step %u\n", __func__, which, step); + + assert(which < s->ndatasets); + + ds = s->dataset[which]; + + if (s->asteps != 0 && step % s->asteps == 0) + add_dset_attribute(s, ds, s->one_by_one_sid, which, step); + + if (s->two_dee) { + size[0] = s->chunk_dims[0] * (1 + step); + size[1] = s->chunk_dims[1] * (1 + step); + last.row = s->chunk_dims[0] * step; + last.col = s->chunk_dims[1] * step; + } else { + size[0] = s->chunk_dims[0]; + size[1] = s->chunk_dims[1] * (1 + step); + last.row = 0; + last.col = s->chunk_dims[1] * step; + } + + dbgf(1, "new size %" PRIuHSIZE ", %" PRIuHSIZE "\n", size[0], size[1]); + + if (s->vds != vds_off) { + const hsize_t half_size[RANK] = {size[0] / 2, size[1] / 2}; + sources_t * const srcs = &s->sources[which]; + + if (H5Dset_extent(srcs->ul, half_size) < 0) { + errx(EXIT_FAILURE, "%s.%d: H5Dset_extent failed", + __func__, __LINE__); + } + if (H5Dset_extent(srcs->ur, half_size) < 0) { + errx(EXIT_FAILURE, "%s.%d: H5Dset_extent failed", + __func__, __LINE__); + } + if (H5Dset_extent(srcs->bl, half_size) < 0) { + errx(EXIT_FAILURE, "%s.%d: H5Dset_extent failed", + __func__, __LINE__); + } + if (H5Dset_extent(srcs->br, half_size) < 0) { + errx(EXIT_FAILURE, "%s.%d: H5Dset_extent failed", + __func__, __LINE__); + } + + } else if (H5Dset_extent(ds, size) < 0) + errx(EXIT_FAILURE, "H5Dset_extent failed"); + + filespace = H5Dget_space(ds); + + if (filespace == badhid) + errx(EXIT_FAILURE, "H5Dget_space failed"); + + if (s->two_dee) { + base.col = last.col; + for (base.row = 0; base.row <= last.row; base.row += s->chunk_dims[0]) { + dbgf(1, "writing chunk %" PRIuHSIZE ", %" PRIuHSIZE "\n", + base.row, base.col); + init_and_write_chunk(s, filespace, mat, which, base); + } + + base.row = last.row; + for (base.col = 0; base.col < last.col; base.col += s->chunk_dims[1]) { + dbgf(1, "writing chunk %" PRIuHSIZE ", %" PRIuHSIZE "\n", + base.row, base.col); + init_and_write_chunk(s, filespace, mat, which, base); + } + } else { + init_and_write_chunk(s, filespace, mat, which, last); + } + + if (H5Sclose(filespace) < 0) + errx(EXIT_FAILURE, "H5Sclose failed"); +} + +int +main(int argc, char **argv) +{ + mat_t *mat; + hid_t fcpl; + sigset_t oldsigs; + herr_t ret; + unsigned step, which; + state_t s; + size_t i; + + state_init(&s, argc, argv); + + if ((mat = newmat(s.rows, s.cols)) == NULL) + err(EXIT_FAILURE, "%s: could not allocate matrix", __func__); + + if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + errx(EXIT_FAILURE, "H5Pcreate"); + + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, false, 1); + if (ret < 0) + errx(EXIT_FAILURE, "H5Pset_file_space_strategy"); + + for (i = 0; i < NELMTS(s.file); i++) { + hid_t fapl; + + if (s.vds != vds_multi && i > 0) { + s.file[i] = s.file[0]; + continue; + } + + fapl = vfd_swmr_create_fapl(s.writer, true, s.use_vfd_swmr, + "./metadata-%zu", i); + + if (fapl < 0) + errx(EXIT_FAILURE, "vfd_swmr_create_fapl"); + + s.file[i] = + s.writer ? H5Fcreate(s.filename[i], H5F_ACC_TRUNC, fcpl, fapl) + : H5Fopen(s.filename[i], H5F_ACC_RDONLY, fapl); + + if (s.file[i] == badhid) + errx(EXIT_FAILURE, s.writer ? "H5Fcreate" : "H5Fopen"); + + if (H5Pclose(fapl) < 0) + errx(EXIT_FAILURE, "H5Pclose(fapl)"); + } + + block_signals(&oldsigs); + + if (s.writer) { + for (which = 0; which < s.ndatasets; which++) + create_extensible_dset(&s, which); + + for (step = 0; step < s.nsteps; step++) { + for (which = 0; which < s.ndatasets; which++) { + dbgf(2, "step %d which %d\n", step, which); + write_extensible_dset(&s, which, step, mat); + if (s.ndatasets <= s.nsteps) + nanosleep(&s.update_interval, NULL); + } + if (s.ndatasets > s.nsteps) + nanosleep(&s.update_interval, NULL); + } + } else { + for (which = 0; which < s.ndatasets; which++) + open_extensible_dset(&s, which); + + for (step = 0; hang_back + step < s.nsteps;) { + for (which = s.ndatasets; which-- > 0; ) { + dbgf(2, "step %d which %d\n", step, which); + verify_extensible_dset(&s, which, mat, &step); + if (s.ndatasets <= s.nsteps) + nanosleep(&s.update_interval, NULL); + } + if (s.ndatasets > s.nsteps) + nanosleep(&s.update_interval, NULL); + } + } + + for (which = 0; which < s.ndatasets; which++) + close_extensible_dset(&s, which); + + if (s.use_vfd_swmr && s.wait_for_signal) + await_signal(s.file[0]); + + restore_signals(&oldsigs); + + if (H5Pclose(fcpl) < 0) + errx(EXIT_FAILURE, "H5Pclose(fcpl)"); + + state_destroy(&s); + + free(mat); + + return EXIT_SUCCESS; +} diff --git a/test/vfd_swmr_common.c b/test/vfd_swmr_common.c new file mode 100644 index 0000000..e4f4889 --- /dev/null +++ b/test/vfd_swmr_common.c @@ -0,0 +1,262 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: swmr_common.c + * + * Purpose: Utility functions for the SWMR test code. + * + *------------------------------------------------------------------------- + */ + +/***********/ +/* Headers */ +/***********/ + +#include <err.h> /* for err(3) */ + +#include "h5test.h" +#include "vfd_swmr_common.h" + +static const hid_t badhid = H5I_INVALID_HID; + +int verbosity = 2; + +void +evsnprintf(char *buf, size_t bufsz, const char *fmt, va_list ap) +{ + int rc; + + rc = vsnprintf(buf, bufsz, fmt, ap); + + if (rc < 0) + err(EXIT_FAILURE, "%s: vsnprintf", __func__); + else if ((size_t)rc >= bufsz) + errx(EXIT_FAILURE, "%s: buffer too small", __func__); +} + +void +esnprintf(char *buf, size_t bufsz, const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + evsnprintf(buf, bufsz, fmt, ap); + va_end(ap); +} + +void +dbgf(int level, const char *fmt, ...) +{ + va_list ap; + + if (verbosity < level) + return; + + va_start(ap, fmt); + (void)vfprintf(stderr, fmt, ap); + va_end(ap); +} + +estack_state_t +disable_estack(void) +{ + estack_state_t es = estack_get_state(); + + (void)H5Eset_auto(H5E_DEFAULT, NULL, NULL); + + return es; +} + +estack_state_t +estack_get_state(void) +{ + estack_state_t es; + + (void)H5Eget_auto(H5E_DEFAULT, &es.efunc, &es.edata); + + return es; +} + +void +restore_estack(estack_state_t es) +{ + (void)H5Eset_auto(H5E_DEFAULT, es.efunc, es.edata); +} + +void +block_signals(sigset_t *oldset) +{ + sigset_t fullset; + + if (sigfillset(&fullset) == -1) { + err(EXIT_FAILURE, "%s.%d: could not initialize signal masks", + __func__, __LINE__); + } + + if (sigprocmask(SIG_BLOCK, &fullset, oldset) == -1) + err(EXIT_FAILURE, "%s.%d: sigprocmask", __func__, __LINE__); +} + +void +restore_signals(sigset_t *oldset) +{ + if (sigprocmask(SIG_SETMASK, oldset, NULL) == -1) + err(EXIT_FAILURE, "%s.%d: sigprocmask", __func__, __LINE__); +} + +#if 0 +static const char * +strsignal(int signum) +{ + switch (signum) { + case SIGUSR1: + return "SIGUSR1"; + case SIGINT: + return "SIGINT"; + case SIGPIPE: + return "SIGPIPE"; + default: + return "<unknown>"; + } +} +#endif + +void +await_signal(hid_t fid) +{ + sigset_t sleepset; + struct timespec tick = {.tv_sec = 0, .tv_nsec = 1000000000 / 100}; + + if (sigfillset(&sleepset) == -1) { + err(EXIT_FAILURE, "%s.%d: could not initialize signal mask", + __func__, __LINE__); + } + + /* Avoid deadlock: flush the file before waiting for the reader's + * message. + */ + if (H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0) + errx(EXIT_FAILURE, "%s: H5Fflush failed", __func__); + + dbgf(1, "waiting for signal\n"); + + for (;;) { + const int rc = sigtimedwait(&sleepset, NULL, &tick); + + if (rc != -1) { + fprintf(stderr, "Received %s, wrapping things up.\n", + strsignal(rc)); + break; + } else if (rc == -1 && errno == EAGAIN) { + estack_state_t es; + + /* Avoid deadlock with peer: periodically enter the API so that + * tick processing occurs and data is flushed so that the peer + * can see it. + * + * The call we make will fail, but that's ok, + * so squelch errors. + */ + es = disable_estack(); + (void)H5Aexists_by_name(fid, "nonexistent", "nonexistent", + H5P_DEFAULT); + restore_estack(es); + } else if (rc == -1) + err(EXIT_FAILURE, "%s: sigtimedwait", __func__); + } +} + +hid_t +vfd_swmr_create_fapl(bool writer, bool only_meta_pages, bool use_vfd_swmr, + const char *mdfile_fmtstr, ...) +{ + H5F_vfd_swmr_config_t config; + hid_t fapl; + va_list ap; + + /* Create file access property list */ + if((fapl = h5_fileaccess()) < 0) { + warnx("h5_fileaccess"); + return badhid; + } + + /* FOR NOW: set to use latest format, the "old" parameter is not used */ + if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) { + warnx("H5Pset_libver_bounds"); + return badhid; + } + + /* + * Set up to open the file with VFD SWMR configured. + */ + + /* Enable page buffering */ + if(H5Pset_page_buffer_size(fapl, 4096, only_meta_pages ? 100 : 0, 0) < 0) { + warnx("H5Pset_page_buffer_size"); + return badhid; + } + + memset(&config, 0, sizeof(config)); + + config.version = H5F__CURR_VFD_SWMR_CONFIG_VERSION; + config.tick_len = 4; + config.max_lag = 7; + config.writer = writer; + config.md_pages_reserved = 128; + va_start(ap, mdfile_fmtstr); + evsnprintf(config.md_file_path, sizeof(config.md_file_path), + mdfile_fmtstr, ap); + va_end(ap); + + /* Enable VFD SWMR configuration */ + if(use_vfd_swmr && H5Pset_vfd_swmr_config(fapl, &config) < 0) { + warnx("H5Pset_vfd_swmr_config"); + return badhid; + } + return fapl; +} + +/* Fetch a variable from the environment and parse it for unsigned long + * content. Return 0 if the variable is not present, -1 if it is present + * but it does not parse and compare less than `limit`, 1 if it's present, + * parses, and is in-bounds. + */ +int +fetch_env_ulong(const char *varname, unsigned long limit, unsigned long *valp) +{ + char *end; + unsigned long ul; + char *tmp; + + if ((tmp = getenv(varname)) == NULL) + return 0; + + errno = 0; + ul = strtoul(tmp, &end, 0); + if (ul == ULONG_MAX && errno != 0) { + fprintf(stderr, "could not parse %s: %s\n", varname, strerror(errno)); + return -1; + } + if (end == tmp || *end != '\0') { + fprintf(stderr, "could not parse %s\n", varname); + return -1; + } + if (ul > limit) { + fprintf(stderr, "%s (%lu) out of range\n", varname, ul); + return -1; + } + *valp = ul; + return 1; +} diff --git a/test/vfd_swmr_common.h b/test/vfd_swmr_common.h new file mode 100644 index 0000000..8e1f877 --- /dev/null +++ b/test/vfd_swmr_common.h @@ -0,0 +1,114 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef _VFD_SWMR_COMMON_H +#define _VFD_SWMR_COMMON_H + +/***********/ +/* Headers */ +/***********/ + +#include <stdarg.h> +#include "h5test.h" + +/**********/ +/* Macros */ +/**********/ + +/* The maximum # of records to add/remove from the dataset in one step, + * used by vfd_swmr_addrem_writer and vfd_swmr_remove_reader. + */ +#define MAX_SIZE_CHANGE 10 + +#define NLEVELS 5 /* # of datasets in the SWMR test file */ + +#define NMAPPING 9 + +#define COMMON_FILENAME "vfd_swmr_data.h5" /* SWMR test file name */ +#define DTYPE_SIZE 150 /* Data size in opaque type */ + +/* The message sent by writer that the file open is done--releasing the file lock */ +#define WRITER_MESSAGE "VFD_SWMR_WRITER_MESSAGE" + +/************/ +/* Typedefs */ +/************/ + +typedef struct _estack_state { + H5E_auto_t efunc; + void *edata; +} estack_state_t; + +/* Information about a symbol/dataset */ +typedef struct { + char *name; /* Dataset name for symbol */ + hid_t dsid; /* Dataset ID for symbol */ + hsize_t nrecords; /* Number of records for the symbol */ +} symbol_info_t; + +/* A symbol's record */ +typedef struct { + uint64_t rec_id; /* ID for this record (unique in symbol) */ + uint8_t info[DTYPE_SIZE]; /* "Other" information for this record */ +} symbol_t; + +typedef enum _testsel { + TEST_NONE = 0 +, TEST_NULL +, TEST_OOB +} testsel_t; + +/********************/ +/* Global Variables */ +/********************/ +H5TEST_DLLVAR symbol_info_t *symbol_info[NLEVELS]; +H5TEST_DLLVAR unsigned symbol_count[NLEVELS]; + +/**************/ +/* Prototypes */ +/**************/ +#ifdef __cplusplus +extern "C" { +#endif + +H5TEST_DLL estack_state_t estack_get_state(void); +H5TEST_DLL estack_state_t disable_estack(void); +H5TEST_DLL void restore_estack(estack_state_t); + +H5TEST_DLL symbol_info_t * choose_dataset(unsigned *, unsigned *); +H5TEST_DLL hid_t create_symbol_datatype(void); +H5TEST_DLL int generate_name(char *name_buf, unsigned level, unsigned count); +H5TEST_DLL int generate_symbols(void); +H5TEST_DLL int shutdown_symbols(void); +H5TEST_DLL int print_metadata_retries_info(hid_t fid); + +H5TEST_DLL void block_signals(sigset_t *); +H5TEST_DLL void restore_signals(sigset_t *); +H5TEST_DLL void await_signal(hid_t); +H5TEST_DLL hid_t vfd_swmr_create_fapl(bool, bool, bool, const char *, ...) + H5_ATTR_FORMAT(printf, 4, 5); + +H5TEST_DLL void dbgf(int, const char *, ...) H5_ATTR_FORMAT(printf, 2, 3); +H5TEST_DLL void evsnprintf(char *, size_t, const char *, va_list); +H5TEST_DLL void esnprintf(char *, size_t, const char *, ...) + H5_ATTR_FORMAT(printf, 3, 4); + +H5TEST_DLL int fetch_env_ulong(const char *, unsigned long, unsigned long *); + +#ifdef __cplusplus +} +#endif + +extern int verbosity; + +#endif /* _SWMR_COMMON_H */ diff --git a/test/vfd_swmr_generator.c b/test/vfd_swmr_generator.c new file mode 100644 index 0000000..13fc941 --- /dev/null +++ b/test/vfd_swmr_generator.c @@ -0,0 +1,399 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: vfd_swmr_generator.c + * (copied and modified from swmr_generator.c) + * + * Purpose: Functions for building and setting up the VFD SWMR test file + * and datasets. + * + *------------------------------------------------------------------------- + */ + +/***********/ +/* Headers */ +/***********/ + +#include "h5test.h" +#include "vfd_swmr_common.h" + +/* + * This file needs to access testing codefrom the H5O package. + */ +#define H5O_FRIEND /*suppress error about including H5Opkg */ +#define H5O_TESTING +#include "H5Opkg.h" /* Object headers */ + + +/****************/ +/* Local Macros */ +/****************/ + +#define CHUNK_SIZE 50 /* Chunk size for created datasets */ + +/********************/ +/* Local Prototypes */ +/********************/ + +static int gen_skeleton(const char *filename, hbool_t verbose, + hbool_t vfd_swmr_write, int comp_level, const char *index_type, + unsigned random_seed); +static void usage(void); + + +/*------------------------------------------------------------------------- + * Function: gen_skeleton + * + * Purpose: Creates the HDF5 file and datasets which will be used in + * the SWMR testing. + * + * Parameters: const char *filename + * The SWMR test file's name. + * + * hbool_t verbose + * Whether verbose console output is desired. + * + * hbool_t vfd_swmr_write + * Whether to create the file with VFD SWMR writing enabled + * + * int comp_level + * The zlib compression level to use. -1 = no compression. + * + * const char *index_type + * The chunk index type (b1 | b2 | ea | fa) + * + * unsigned random_seed + * The random seed to store in the file. The sparse tests use + * this value. + * + * Return: Success: 0 + * Failure: Can't fail + * + *------------------------------------------------------------------------- + */ +static int +gen_skeleton(const char *filename, hbool_t verbose, hbool_t vfd_swmr_write, + int comp_level, const char *index_type, unsigned random_seed) +{ + hid_t fid; /* File ID for new HDF5 file */ + hid_t fcpl; /* File creation property list */ + hid_t fapl; /* File access property list */ + hid_t dcpl; /* Dataset creation property list */ + hid_t tid; /* Datatype for dataset elements */ + hid_t sid; /* Dataspace ID */ + hid_t aid; /* Attribute ID */ + hsize_t dims[2] = {1, 0}; /* Dataset starting dimensions */ + hsize_t max_dims[2] = {1, H5S_UNLIMITED}; /* Dataset maximum dimensions */ + hsize_t chunk_dims[2] = {1, CHUNK_SIZE}; /* Chunk dimensions */ +#ifdef FILLVAL_WORKS + symbol_t fillval; /* Dataset fill value */ +#endif /* FILLVAL_WORKS */ + unsigned u, v; /* Local index variable */ + H5F_vfd_swmr_config_t *config = NULL; /* Configuration for VFD SWMR */ + + HDassert(filename); + HDassert(index_type); + + /* Create file access property list */ + if((fapl = h5_fileaccess()) < 0) + return -1; + + /* Set to use the latest format with the latest chunk indexing */ + /* FOR NOW: the parameter vfd_swmr_write is not used here as in swmr_generator.c */ + if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + return -1; + + /* There are two chunk indexes tested here. + * With one unlimited dimension, we get the extensible array index + * type, with two unlimited dimensions, we get a v2 B-tree. + */ + if(!HDstrcmp(index_type, "b2")) + max_dims[0] = H5S_UNLIMITED; + + /* Create file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + return -1; + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "Creating file\n"); + + /* + * Set up to create the file with VFD SWMR write configured. + */ + + if(vfd_swmr_write) { + /* Set file space strategy to paged aggregation in fcpl */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0) + return -1; + + /* Enable page buffering in fapl */ + if(H5Pset_page_buffer_size(fapl, 4096, 0, 0) < 0) + return -1; + + /* Allocate memory for the VFD SWMR configuration structure */ + if((config = (H5F_vfd_swmr_config_t *)HDcalloc(1, sizeof(H5F_vfd_swmr_config_t))) == NULL) + return -1; + + config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION; + config->tick_len = 4; + config->max_lag = 10; + config->writer = TRUE; + config->md_pages_reserved = 128; + HDstrcpy(config->md_file_path, "my_md_file"); + + /* Enable VFD SWMR configuration in fapl */ + if(H5Pset_vfd_swmr_config(fapl, config) < 0) + return -1; + } + + /* Create the file with VFD SWMR write configured */ + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + return -1; + + /* Close file creation property list */ + if(H5Pclose(fcpl) < 0) + return -1; + + /* Close file access property list */ + if(H5Pclose(fapl) < 0) + return -1; + + /* Create attribute with (shared) random number seed - for sparse test */ + if((sid = H5Screate(H5S_SCALAR)) < 0) + return -1; + if((aid = H5Acreate2(fid, "seed", H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0) + return -1; + if(H5Awrite(aid, H5T_NATIVE_UINT, &random_seed) < 0) + return -1; + if(H5Aclose(aid) < 0) + return -1; + if(H5Sclose(sid) < 0) + return -1; + + /* Create datatype for creating datasets */ + if((tid = create_symbol_datatype()) < 0) + return -1; + + /* Create dataspace for creating datasets */ + if((sid = H5Screate_simple(2, dims, max_dims)) < 0) + return -1; + + /* Create dataset creation property list */ + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + return -1; + if(H5Pset_chunk(dcpl, 2, chunk_dims) < 0) + return -1; + if(comp_level >= 0) { + if(H5Pset_deflate(dcpl, (unsigned)comp_level) < 0) + return -1; + } /* end if */ +#ifdef FILLVAL_WORKS + /* Currently fill values do not work because they can bump the dataspace + * message to the second object header chunk. We should enable the fillval + * here when this is fixed. -NAF 8/11/11 */ + HDmemset(&fillval, 0, sizeof(fillval)); + fillval.rec_id = (uint64_t)ULLONG_MAX; + if(H5Pset_fill_value(dcpl, tid, &fillval) < 0) + return -1; +#endif /* FILLVAL_WORKS */ + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "Creating datasets\n"); + +#if 0 /* delete this once the race condiditon bug is fixed */ /* JRM */ + sleep(1); +#endif /* JRM */ + + /* Create the datasets */ + for(u = 0; u < NLEVELS; u++) + for(v = 0; v < symbol_count[u]; v++) { + hid_t dsid; /* Dataset ID */ + char name_buf[64]; + hbool_t move_dataspace_message = FALSE; /* Whether to move the dataspace message out of object header chunk #0 */ + + generate_name(name_buf, u, v); + if((dsid = H5Dcreate2(fid, name_buf, tid, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + return -1; + + /* Determine if the dataspace message for this dataset should be + * moved out of chunk #0 of the object header + * (Set to TRUE for every fourth dataset) + */ + move_dataspace_message = !(HDrandom() % 4); + if(move_dataspace_message) { + unsigned chunk_num; /* Object header chunk # for dataspace message */ + + /* Move the dataspace message to a new object header chunk */ + if(H5O__msg_move_to_new_chunk_test(dsid, H5O_SDSPACE_ID) < 0) + return -1; + + /* Retrieve the chunk # for the dataspace message */ + chunk_num = UINT_MAX; + if(H5O__msg_get_chunkno_test(dsid, H5O_SDSPACE_ID, &chunk_num) < 0) + return -1; + /* Should not be in chunk #0 for now */ + if(0 == chunk_num) + return -1; + } /* end if */ + + if(H5Dclose(dsid) < 0) + return -1; + } /* end for */ + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "Closing objects\n"); + + /* Close everythign */ + if(H5Pclose(dcpl) < 0) + return -1; + if(H5Sclose(sid) < 0) + return -1; + if(H5Tclose(tid) < 0) + return -1; + + if(verbose) + HDfprintf(stderr, "Closing file\n"); + + if(H5Fclose(fid) < 0) + return -1; + + /* Free the config structure */ + if(config) + HDfree(config); + return 0; +} /* end gen_skeleton() */ + +static void +usage(void) +{ + printf("\n"); + printf("Usage error!\n"); + printf("\n"); + printf("Usage: swmr_generator [-q] [-s] [-c <deflate compression level>]\n"); + printf(" [-i <index type>] [-r <random seed>]\n"); + printf("\n"); + printf("NOTE: The random seed option is only used by the sparse test. Other\n"); + printf(" tests specify the random seed as a reader/writer option.\n"); + printf("\n"); + printf("<deflate compression level> should be -1 (for no compression) or 0-9\n"); + printf("\n"); + printf("<index type> should be b2 or ea\n"); + printf("\n"); + printf("Defaults to verbose (no '-q' given), no VFD_SWMR_WRITE mode (no '-s' given) no\n"); + printf("compression ('-c -1'), v1 b-tree indexing (-i b1), and will generate a random\n"); + printf("seed (no -r given).\n"); + printf("\n"); + HDexit(1); +} /* end usage() */ + +int main(int argc, const char *argv[]) +{ + int comp_level = -1; /* Compression level (-1 is no compression) */ + hbool_t verbose = TRUE; /* Whether to emit some informational messages */ + hbool_t vfd_swmr_write = FALSE; /* Whether to create file with VFD SWMR access */ + const char *index_type = "b1"; /* Chunk index type */ + hbool_t use_seed = FALSE; /* Set to TRUE if a seed was set on the command line */ + unsigned random_seed = 0; /* Random # seed */ + unsigned u; /* Local index variables */ + int temp; + + /* Parse command line options */ + if(argc > 1) { + u = 1; + while(u < (unsigned)argc) { + if(argv[u][0] == '-') { + switch(argv[u][1]) { + /* Compress dataset chunks */ + case 'c': + comp_level = HDatoi(argv[u + 1]); + if(comp_level < -1 || comp_level > 9) + usage(); + u += 2; + break; + + /* Chunk index type */ + case 'i': + index_type = argv[u + 1]; + if(HDstrcmp(index_type, "ea") + && HDstrcmp(index_type, "b2")) + usage(); + u += 2; + break; + + /* Random # seed */ + case 'r': + use_seed = TRUE; + temp = HDatoi(argv[u + 1]); + if(temp < 0) + usage(); + else + random_seed = (unsigned)temp; + u += 2; + break; + + /* Be quiet */ + case 'q': + verbose = FALSE; + u++; + break; + + /* Run with SWMR_WRITE */ + case 's': + vfd_swmr_write = TRUE; + u++; + break; + + default: + usage(); + break; + } /* end switch */ + } /* end if */ + } /* end while */ + } /* end if */ + + /* Emit informational message */ + if(verbose) { + HDfprintf(stderr, "Parameters:\n"); + HDfprintf(stderr, "\tswmr writes %s\n", vfd_swmr_write ? "on" : "off"); + HDfprintf(stderr, "\tcompression level = %d\n", comp_level); + HDfprintf(stderr, "\tindex type = %s\n", index_type); + } /* end if */ + + /* Set the random seed */ + if(!use_seed) { + struct timeval t; + + HDgettimeofday(&t, NULL); + random_seed = (unsigned)(t.tv_usec); + } /* end if */ + HDsrandom(random_seed); + /* ALWAYS emit the random seed for possible debugging */ + HDfprintf(stderr, "Using generator random seed (used in sparse test only): %u\n", random_seed); + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "Generating skeleton file: %s\n", COMMON_FILENAME); + + /* Generate file skeleton */ + if(gen_skeleton(COMMON_FILENAME, verbose, vfd_swmr_write, comp_level, index_type, random_seed) < 0) { + HDfprintf(stderr, "Error generating skeleton file!\n"); + HDexit(1); + } /* end if */ + + return 0; +} /* main() */ diff --git a/test/vfd_swmr_group_writer.c b/test/vfd_swmr_group_writer.c new file mode 100644 index 0000000..a1bea61 --- /dev/null +++ b/test/vfd_swmr_group_writer.c @@ -0,0 +1,343 @@ +/* + * Copyright by The HDF Group. + * Copyright by the Board of Trustees of the University of Illinois. + * All rights reserved. + * + * This file is part of HDF5. The full HDF5 copyright notice, including + * terms governing use, modification, and redistribution, is contained in + * the COPYING file, which can be found at the root of the source code + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. + * If you do not have access to either file, you may request a copy from + * help@hdfgroup.org. + */ + +#include <err.h> +#include <libgen.h> +#include <time.h> /* nanosleep(2) */ +#include <unistd.h> /* getopt(3) */ + +#define H5F_FRIEND /*suppress error about including H5Fpkg */ + +#include "hdf5.h" + +#include "H5Fpkg.h" +// #include "H5Iprivate.h" +#include "H5HGprivate.h" +#include "H5VLprivate.h" + +#include "testhdf5.h" +#include "vfd_swmr_common.h" + +typedef struct { + hid_t file, filetype, one_by_one_sid; + char filename[PATH_MAX]; + char progname[PATH_MAX]; + struct timespec update_interval; + unsigned int asteps; + unsigned int nsteps; + bool wait_for_signal; + bool use_vfd_swmr; +} state_t; + +#define ALL_HID_INITIALIZER (state_t){ \ + .file = H5I_INVALID_HID \ + , .one_by_one_sid = H5I_INVALID_HID \ + , .filename = "" \ + , .filetype = H5T_NATIVE_UINT32 \ + , .asteps = 10 \ + , .nsteps = 100 \ + , .wait_for_signal = true \ + , .use_vfd_swmr = true \ + , .update_interval = (struct timespec){ \ + .tv_sec = 0 \ + , .tv_nsec = 1000000000UL / 30 /* 1/30 second */}} + +static void state_init(state_t *, int, char **); + +static const hid_t badhid = H5I_INVALID_HID; + +static void +usage(const char *progname) +{ + fprintf(stderr, "usage: %s [-S] [-W] [-a steps] [-b]\n" + " [-n iterations] [-u milliseconds]\n" + "\n" + "-S: do not use VFD SWMR\n" + "-W: do not wait for a signal before\n" + " exiting\n" + "-a steps: `steps` between adding attributes\n" + "-b: write data in big-endian byte order\n" + "-n iterations: how many times to expand each dataset\n" + "-u ms: milliseconds interval between updates\n" + " to %s.h5\n" + "\n", + progname, progname); + exit(EXIT_FAILURE); +} + +static void +state_init(state_t *s, int argc, char **argv) +{ + unsigned long tmp; + int ch; + const hsize_t dims = 1; + char tfile[PATH_MAX]; + char *end; + unsigned long millis; + + *s = ALL_HID_INITIALIZER; + esnprintf(tfile, sizeof(tfile), "%s", argv[0]); + esnprintf(s->progname, sizeof(s->progname), "%s", basename(tfile)); + + while ((ch = getopt(argc, argv, "SWa:bn:qu:")) != -1) { + switch (ch) { + case 'S': + s->use_vfd_swmr = false; + break; + case 'W': + s->wait_for_signal = false; + break; + case 'a': + case 'n': + errno = 0; + tmp = strtoul(optarg, &end, 0); + if (end == optarg || *end != '\0') { + errx(EXIT_FAILURE, "couldn't parse `-%c` argument `%s`", ch, + optarg); + } else if (errno != 0) { + err(EXIT_FAILURE, "couldn't parse `-%c` argument `%s`", ch, + optarg); + } else if (tmp > UINT_MAX) + errx(EXIT_FAILURE, "`-%c` argument `%lu` too large", ch, tmp); + + if (ch == 'a') + s->asteps = (unsigned)tmp; + else + s->nsteps = (unsigned)tmp; + break; + case 'b': + s->filetype = H5T_STD_U32BE; + break; + case 'q': + verbosity = 0; + break; + case 'u': + errno = 0; + millis = strtoul(optarg, &end, 0); + if (millis == ULONG_MAX && errno == ERANGE) { + err(EXIT_FAILURE, + "option -p argument \"%s\"", optarg); + } else if (*end != '\0') { + errx(EXIT_FAILURE, + "garbage after -p argument \"%s\"", optarg); + } + s->update_interval.tv_sec = (time_t)(millis / 1000UL); + s->update_interval.tv_nsec = + (long)((millis * 1000000UL) % 1000000000UL); + dbgf(1, "%lu milliseconds between updates\n", millis); + break; + case '?': + default: + usage(s->progname); + break; + } + } + argc -= optind; + argv += optind; + + /* space for attributes */ + if ((s->one_by_one_sid = H5Screate_simple(1, &dims, &dims)) < 0) + errx(EXIT_FAILURE, "H5Screate_simple failed"); + + if (argc > 0) + errx(EXIT_FAILURE, "unexpected command-line arguments"); + + esnprintf(s->filename, sizeof(s->filename), "vfd_swmr_group.h5"); +} + +static void +add_group_attribute(const state_t *s, hid_t g, hid_t sid, unsigned int which) +{ + hid_t aid; + char name[sizeof("attr-9999999999")]; + + esnprintf(name, sizeof(name), "attr-%u", which); + + dbgf(1, "setting attribute %s on group %u to %u\n", name, which, which); + + if ((aid = H5Acreate2(g, name, s->filetype, sid, H5P_DEFAULT, + H5P_DEFAULT)) < 0) + errx(EXIT_FAILURE, "H5Acreate2 failed"); + + if (H5Awrite(aid, H5T_NATIVE_UINT, &which) < 0) + errx(EXIT_FAILURE, "H5Awrite failed"); + if (H5Aclose(aid) < 0) + errx(EXIT_FAILURE, "H5Aclose failed"); +} + + +static void +write_group(state_t *s, unsigned int which) +{ + char name[sizeof("/group-9999999999")]; + hid_t g; + + assert(which < s->nsteps); + + esnprintf(name, sizeof(name), "/group-%d", which); + + g = H5Gcreate2(s->file, name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + + if (g < 0) + errx(EXIT_FAILURE, "H5Gcreate(, \"%s\", ) failed", name); + + if (s->asteps != 0 && which % s->asteps == 0) + add_group_attribute(s, g, s->one_by_one_sid, which); + + if (H5Gclose(g) < 0) + errx(EXIT_FAILURE, "H5Gclose failed"); +} + +static bool +verify_group_attribute(hid_t g, unsigned int which) +{ + estack_state_t es; + unsigned int read_which; + hid_t aid; + char name[sizeof("attr-9999999999")]; + + esnprintf(name, sizeof(name), "attr-%u", which); + + dbgf(1, "verifying attribute %s on group %u equals %u\n", name, which, + which); + + es = disable_estack(); + if ((aid = H5Aopen(g, name, H5P_DEFAULT)) < 0) { + restore_estack(es); + return false; + } + + if (H5Aread(aid, H5T_NATIVE_UINT, &read_which) < 0) { + restore_estack(es); + if (H5Aclose(aid) < 0) + errx(EXIT_FAILURE, "H5Aclose failed"); + return false; + } + + restore_estack(es); + + if (H5Aclose(aid) < 0) + errx(EXIT_FAILURE, "H5Aclose failed"); + + return read_which == which; +} + +static bool +verify_group(state_t *s, unsigned int which) +{ + char name[sizeof("/group-9999999999")]; + hid_t g; + estack_state_t es; + bool result; + + assert(which < s->nsteps); + + esnprintf(name, sizeof(name), "/group-%d", which); + + es = disable_estack(); + g = H5Gopen(s->file, name, H5P_DEFAULT); + restore_estack(es); + + if (g < 0) + return false; + + if (s->asteps != 0 && which % s->asteps == 0) + result = verify_group_attribute(g, which); + else + result = true; + + if (H5Gclose(g) < 0) + errx(EXIT_FAILURE, "H5Gclose failed"); + + return result; +} + +int +main(int argc, char **argv) +{ + hid_t fapl, fcpl; + sigset_t oldsigs; + herr_t ret; + unsigned step; + bool writer; + state_t s; + const char *personality; + + state_init(&s, argc, argv); + + personality = strstr(s.progname, "vfd_swmr_group_"); + + if (personality != NULL && + strcmp(personality, "vfd_swmr_group_writer") == 0) + writer = true; + else if (personality != NULL && + strcmp(personality, "vfd_swmr_group_reader") == 0) + writer = false; + else { + errx(EXIT_FAILURE, + "unknown personality, expected vfd_swmr_group_{reader,writer}"); + } + + fapl = vfd_swmr_create_fapl(writer, true, s.use_vfd_swmr, "./shadow"); + + if (fapl < 0) + errx(EXIT_FAILURE, "vfd_swmr_create_fapl"); + + if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + errx(EXIT_FAILURE, "H5Pcreate"); + + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, false, 1); + if (ret < 0) + errx(EXIT_FAILURE, "H5Pset_file_space_strategy"); + + if (writer) + s.file = H5Fcreate(s.filename, H5F_ACC_TRUNC, fcpl, fapl); + else + s.file = H5Fopen(s.filename, H5F_ACC_RDONLY, fapl); + + if (s.file == badhid) + errx(EXIT_FAILURE, writer ? "H5Fcreate" : "H5Fopen"); + + block_signals(&oldsigs); + + if (writer) { + for (step = 0; step < s.nsteps; step++) { + dbgf(2, "step %d\n", step); + write_group(&s, step); + nanosleep(&s.update_interval, NULL); + } + } else { + for (step = 0; step < s.nsteps;) { + dbgf(2, "step %d\n", step); + if (verify_group(&s, step)) + step++; + nanosleep(&s.update_interval, NULL); + } + } + + if (s.use_vfd_swmr && s.wait_for_signal) + await_signal(s.file); + + restore_signals(&oldsigs); + + if (H5Pclose(fapl) < 0) + errx(EXIT_FAILURE, "H5Pclose(fapl)"); + + if (H5Pclose(fcpl) < 0) + errx(EXIT_FAILURE, "H5Pclose(fcpl)"); + + if (H5Fclose(s.file) < 0) + errx(EXIT_FAILURE, "H5Fclose"); + + return EXIT_SUCCESS; +} diff --git a/test/vfd_swmr_reader.c b/test/vfd_swmr_reader.c new file mode 100644 index 0000000..706c894 --- /dev/null +++ b/test/vfd_swmr_reader.c @@ -0,0 +1,677 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: vfd_swmr_reader.c + * (copied and modified from swmr_reader.c) + * + * Purpose: Reads data from a randomly selected subset of the datasets + * in the VFD SWMR test file. + * + * This program is intended to run concurrently with the + * vfd_swmr_writer program. + * + *------------------------------------------------------------------------- + */ + +/***********/ +/* Headers */ +/***********/ + +#include "h5test.h" +#include "vfd_swmr_common.h" + +/********************/ +/* Local Prototypes */ +/********************/ + +static int check_dataset(hid_t fid, hbool_t verbose, FILE *verbose_file, + const char *sym_name, symbol_t *record, hid_t rec_sid); +static int read_records(const char *filename, hbool_t verbose, FILE *verbose_file, + unsigned random_seed, unsigned long nseconds, unsigned poll_time, + unsigned ncommon, unsigned nrandom); + +/*******************/ +/* Local Variables */ +/*******************/ + +static hid_t symbol_tid = -1; /* The type ID for the SWMR datasets */ + + +/*------------------------------------------------------------------------- + * Function: check_dataset + * + * Purpose: For a given dataset, checks to make sure that the stated + * and actual sizes are the same. If they are not, then + * we have an inconsistent dataset due to a SWMR error. + * + * Parameters: hid_t fid + * The SWMR test file's ID. + * + * hbool_t verbose + * Whether verbose console output is desired. + * + * FILE *verbose_file + * File handle for verbose output + * + * const char *sym_name + * The name of the dataset from which to read. + * + * symbol_t *record + * Memory for the record. Must be pre-allocated. + * + * hid_t rec_sid + * The memory dataspace for access. It's always the same so + * there is no need to re-create it every time this function + * is called. + * + * Return: Success: 0 + * Failure: -1 + * + *------------------------------------------------------------------------- + */ +static int +check_dataset(hid_t fid, hbool_t verbose, FILE *verbose_file, + const char *sym_name, symbol_t *record, hid_t rec_sid) +{ + int fill_count = 0; /* # of times fill value (0) was read + * instead of the expected value. + */ + hid_t dsid; /* Dataset ID */ + hid_t file_sid; /* Dataset's space ID */ + hssize_t snpoints; /* Number of elements in dataset */ + hsize_t start[2] = {0, 0}, count[2] = {1, 1}; /* Hyperslab selection values */ + + HDassert(fid >= 0); + HDassert(sym_name); + HDassert(record); + HDassert(rec_sid >= 0); + + /* Open dataset for symbol */ + if((dsid = H5Dopen2(fid, sym_name, H5P_DEFAULT)) < 0) { + fprintf(stderr, "%s.%d: H5Dopen2 failed\n", __func__, __LINE__); + goto error; + } + + /* Get the dataset's dataspace */ + if((file_sid = H5Dget_space(dsid)) < 0) { + fprintf(stderr, "%s.%d: H5Dget_space failed\n", __func__, __LINE__); + goto error; + } + + /* Get the number of elements (= records, for 1-D datasets) */ + if((snpoints = H5Sget_simple_extent_npoints(file_sid)) < 0) { + fprintf(stderr, "%s.%d: H5Sget_simple_extent_npoints failed\n", + __func__, __LINE__); + goto error; + } + + /* Emit informational message */ + if(verbose) + HDfprintf(verbose_file, "READER: Symbol = '%s', # of records = %lld\n", sym_name, (long long)snpoints); + + /* Check if there are records for symbol */ + if(snpoints > 0) { + /* Choose the last record in the dataset */ + start[1] = (hsize_t)(snpoints - 1); + if(H5Sselect_hyperslab(file_sid, H5S_SELECT_SET, start, NULL, count, NULL) < 0) { + fprintf(stderr, "%s.%d: H5Sselect_hyperslab failed\n", + __func__, __LINE__); + goto error; + } + + /* Read record from dataset */ + record->rec_id = UINT64_MAX; + if(H5Dread(dsid, symbol_tid, rec_sid, file_sid, H5P_DEFAULT, record) < 0) { + fprintf(stderr, "%s.%d: H5Dread failed\n", __func__, __LINE__); + goto error; + } + + /* Verify record value */ + if (record->rec_id != start[1] && record->rec_id == 0) + fill_count++; + else if (record->rec_id != start[1]) { + struct timeval tv; + + HDgettimeofday(&tv, NULL); + + if(verbose) { + HDfprintf(verbose_file, "*** READER ERROR ***\n"); + HDfprintf(verbose_file, "Incorrect record value!\n"); + HDfprintf(verbose_file, + "Time = %jd.%06jd, Symbol = '%s'" + ", # of records = %" PRIdHSIZE + ", record->rec_id = %" PRIu64 "\n", + (intmax_t)tv.tv_sec, (intmax_t)tv.tv_usec, + sym_name, snpoints, record->rec_id); + } /* end if */ + fprintf(stderr, + "%s.%d: record value %" PRIu64 " != %" PRIuHSIZE "\n", + __func__, __LINE__, record->rec_id, start[1]); + goto error; + } /* end if */ + } /* end if */ + + /* Close the dataset's dataspace */ + if(H5Sclose(file_sid) < 0) { + fprintf(stderr, "%s.%d: H5Sclose failed\n", __func__, __LINE__); + goto error; + } + + /* Close dataset for symbol */ + if(H5Dclose(dsid) < 0) { + fprintf(stderr, "%s.%d: H5Dclose failed\n", __func__, __LINE__); + goto error; + } + + return fill_count; + +error: + H5E_BEGIN_TRY { + H5Sclose(file_sid); + H5Dclose(dsid); + } H5E_END_TRY; + + return -1; +} /* end check_dataset() */ + + +/*------------------------------------------------------------------------- + * Function: read_records + * + * Purpose: For a given dataset, checks to make sure that the stated + * and actual sizes are the same. If they are not, then + * we have an inconsistent dataset due to a SWMR error. + * + * The "common" datasets are a random selection from among + * the level 0 datasets. The "random" datasets are a random + * selection from among all the file's datasets. This scheme + * ensures that the level 0 datasets are interrogated vigorously. + * + * Parameters: const char *filename + * The SWMR test file's name. + * + * hbool_t verbose + * Whether verbose console output is desired. + * + * FILE *verbose_file + * File handle for verbose output + * + * unsigned random_seed + * Random seed for the file (used for verbose logging) + * + * unsigned long nseconds + * The amount of time to read records (ns). + * + * unsigned poll_time + * The amount of time to sleep (s). + * + * unsigned ncommon + * The number of common/non-random datasets that will be opened. + * + * unsigned nrandom + * The number of random datasets that will be opened. + * + * Return: Success: 0 + * Failure: -1 + * + *------------------------------------------------------------------------- + */ +static int +read_records(const char *filename, hbool_t verbose, FILE *verbose_file, + unsigned random_seed, unsigned long nseconds, unsigned poll_time, + unsigned ncommon, unsigned nrandom) +{ + time_t start_time; /* Starting time */ + time_t curr_time; /* Current time */ + symbol_info_t **sym_com = NULL; /* Pointers to array of common dataset IDs */ + symbol_info_t **sym_rand = NULL; /* Pointers to array of random dataset IDs */ + hid_t mem_sid; /* Memory dataspace ID */ + hid_t fid; /* SWMR test file ID */ + hid_t fapl; /* file access property list */ + symbol_t record; /* The record to read from the dataset */ + unsigned v; /* Local index variable */ + hbool_t use_log_vfd = FALSE; /* Use the log VFD (set this manually) */ + H5F_vfd_swmr_config_t *config = NULL; /* Configuration for VFD SWMR */ + + HDassert(filename); + HDassert(nseconds != 0); + HDassert(poll_time != 0); + + /* Reset the record */ + /* (record's 'info' field might need to change for each record read, also) */ + HDmemset(&record, 0, sizeof(record)); + + /* Emit informational message */ + if(verbose) + HDfprintf(verbose_file, "READER: Choosing datasets\n"); + + /* Allocate space for 'common' datasets, if any */ + if(ncommon > 0) { + /* Allocate array to hold pointers to symbols for common datasets */ + if(NULL == (sym_com = (symbol_info_t **)HDmalloc(sizeof(symbol_info_t *) * ncommon))) { + fprintf(stderr, "%s.%d: malloc failed\n", __func__, __LINE__); + goto error; + } + + /* Open the common datasets */ + for(v = 0; v < ncommon; v++) { + unsigned offset; /* Offset of symbol to use */ + + /* Determine the offset of the symbol, within level 0 symbols */ + /* (level 0 symbols are the most common symbols) */ + offset = (unsigned)((unsigned)HDrandom() % symbol_count[0]); + sym_com[v] = &symbol_info[0][offset]; + + /* Emit informational message */ + if(verbose) + HDfprintf(verbose_file, "READER: Common symbol #%u = '%s'\n", v, symbol_info[0][offset].name); + } /* end for */ + } /* end if */ + + /* Allocate space for 'random' datasets, if any */ + if(nrandom > 0) { + /* Allocate array to hold pointers to symbols for random datasets */ + if(NULL == (sym_rand = (symbol_info_t **)HDmalloc(sizeof(symbol_info_t *) * nrandom))) { + fprintf(stderr, "%s.%d: malloc failed\n", __func__, __LINE__); + goto error; + } + + /* Determine the random datasets */ + for(v = 0; v < nrandom; v++) { + symbol_info_t *sym; /* Symbol to use */ + + /* Determine the symbol, within all symbols */ + if(NULL == (sym = choose_dataset(NULL, NULL))) + return -1; + sym_rand[v] = sym; + + /* Emit informational message */ + if(verbose) + HDfprintf(verbose_file, "READER: Random symbol #%u = '%s'\n", v, sym->name); + } /* end for */ + } /* end if */ + + /* Create a dataspace for the record to read */ + if((mem_sid = H5Screate(H5S_SCALAR)) < 0) { + fprintf(stderr, "%s.%d: H5Screate failed\n", __func__, __LINE__); + goto error; + } + + /* Emit informational message */ + if(verbose) + HDfprintf(verbose_file, "READER: Going to open file and read records\n"); + + /* Get the starting time */ + start_time = HDtime(NULL); + curr_time = start_time; + + /* Create file access property list */ + if((fapl = h5_fileaccess()) < 0) { + fprintf(stderr, "%s.%d: h5_fileaccess failed\n", __func__, __LINE__); + goto error; + } + + /* Log I/O when verbose output it enbabled */ + if(use_log_vfd) { + char verbose_name[1024]; + + HDsnprintf(verbose_name, sizeof(verbose_name), "vfd_swmr_reader.log.%u", random_seed); + + H5Pset_fapl_log(fapl, verbose_name, H5FD_LOG_ALL, (size_t)(512 * 1024 * 1024)); + } /* end if */ + + /* + * Set up to open the file with VFD SWMR configured. + */ + /* Enable page buffering */ + if(H5Pset_page_buffer_size(fapl, 4096, 0, 0) < 0) { + fprintf(stderr, "%s.%d: H5Pset_page_buffer_size failed\n", + __func__, __LINE__); + goto error; + } + + /* Allocate memory for the configuration structure */ + if((config = (H5F_vfd_swmr_config_t *)HDcalloc(1, sizeof(H5F_vfd_swmr_config_t))) == NULL) { + fprintf(stderr, "%s.%d: malloc failed\n", __func__, __LINE__); + goto error; + } + + config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION; + config->tick_len = 4; + config->max_lag = 5; + config->writer = FALSE; + config->md_pages_reserved = 128; + HDstrcpy(config->md_file_path, "./my_md_file"); + + /* Enable VFD SWMR configuration */ + if(H5Pset_vfd_swmr_config(fapl, config) < 0) { + fprintf(stderr, "%s.%d: H5Pset_vfd_swmr_config failed\n", + __func__, __LINE__); + goto error; + } + + /* Loop over reading records until [at least] the correct # of seconds have passed */ + while(curr_time < (time_t)(start_time + (time_t)nseconds)) { + + /* Emit informational message */ + if(verbose) + HDfprintf(verbose_file, "READER: Opening file: %s\n", filename); + + /* Open the file with VFD SWMR configured */ + /* Remove H5E_BEGIN_TRY/END_TRY if you want to see the error stack */ + H5E_BEGIN_TRY { + fid = H5Fopen(filename, H5F_ACC_RDONLY, fapl); + } H5E_END_TRY; + if(fid < 0) { + HDfprintf(stderr, "READER: Error in opening the file: %s\n", filename); + goto error; + } + + /* Check 'common' datasets, if any */ + if(ncommon > 0) { + /* Emit informational message */ + if(verbose) + HDfprintf(verbose_file, "READER: Checking common symbols after FILE OPEN\n"); + + /* Iterate over common datasets */ + for(v = 0; v < ncommon; v++) { + /* Check common dataset */ + const int fill_count = + check_dataset(fid, verbose, verbose_file, sym_com[v]->name, + &record, mem_sid); + if(fill_count < 0) { + fprintf(stderr, "%s.%d: check_dataset failed\n", + __func__, __LINE__); + goto error; + } + HDmemset(&record, 0, sizeof(record)); + if (fill_count > 0) { + fprintf(stderr, "common dataset: read fill at %d records\n", + fill_count); + } + } /* end for */ + } /* end if */ + + /* Check 'random' datasets, if any */ + if(nrandom > 0) { + /* Emit informational message */ + if(verbose) + HDfprintf(verbose_file, "READER: Checking random symbols after FILE OPEN\n"); + + /* Iterate over random datasets */ + for(v = 0; v < nrandom; v++) { + /* Check random dataset */ + const int fill_count = check_dataset(fid, verbose, verbose_file, + sym_rand[v]->name, &record, mem_sid); + if(fill_count < 0) { + fprintf(stderr, "%s.%d: check_dataset failed\n", + __func__, __LINE__); + goto error; + } + HDmemset(&record, 0, sizeof(record)); + if (fill_count > 0) { + fprintf(stderr, "random dataset: read fill at %d records\n", + fill_count); + } + } /* end for */ + } /* end if */ + + /* Emit informational message */ + if(verbose) + HDfprintf(verbose_file, "READER: Closing file\n"); + + /* Close the file */ + if(H5Fclose(fid) < 0) { + fprintf(stderr, "%s.%d: H5Fclose failed\n", __func__, __LINE__); + goto error; + } + + /* Sleep for the appropriate # of seconds */ + HDsleep(poll_time); + + /* Retrieve the current time */ + curr_time = HDtime(NULL); + } /* end while */ + + /* Close the memory dataspace */ + if(H5Sclose(mem_sid) < 0) { + fprintf(stderr, "%s.%d: H5Sclose failed\n", __func__, __LINE__); + goto error; + } + + /* Close the fapl */ + if(H5Pclose(fapl) < 0) { + fprintf(stderr, "%s.%d: H5Pclose failed\n", __func__, __LINE__); + goto error; + } + + if(config) + HDfree(config); + + /* Emit informational message */ + if(verbose) + HDfprintf(verbose_file, "READER: Closing datasets\n"); + + /* Close 'random' datasets, if any */ + if(nrandom > 0) { + /* Release array holding dataset ID's for random datasets */ + HDfree(sym_rand); + } /* end if */ + + /* Close 'common' datasets, if any */ + if(ncommon > 0) { + /* Release array holding dataset ID's for common datasets */ + HDfree(sym_com); + } /* end if */ + + return 0; + +error: + if(config) + HDfree(config); + + if(sym_rand) + HDfree(sym_rand); + + if(sym_com) + HDfree(sym_com); + + H5E_BEGIN_TRY { + H5Sclose(mem_sid); + H5Fclose(fid); + H5Pclose(fapl); + } H5E_END_TRY; + + return -1; +} /* end read_records() */ + +static void +usage(void) +{ + printf("\n"); + printf("Usage error!\n"); + printf("\n"); + printf("Usage: vfd_swmr_reader [-q] [-s <# of seconds to sleep between polling>]\n"); + printf(" [-h <# of common symbols to poll>] [-l <# of random symbols to poll>]\n"); + printf(" [-r <random seed>] <# of seconds to test>\n"); + printf("\n"); + printf("<# of seconds to test> must be specified.\n"); + printf("\n"); + printf("Defaults to verbose (no '-q' given), 1 second between polling ('-s 1'),\n"); + printf("5 common symbols to poll ('-h 5'), 10 random symbols to poll ('-l 10'),\n"); + printf("and will generate a random seed (no -r given).\n"); + printf("\n"); + HDexit(1); +} + +int main(int argc, const char *argv[]) +{ + long nseconds = 0; /* # of seconds to test */ + int poll_time = 1; /* # of seconds between polling */ + int ncommon = 5; /* # of common symbols to poll */ + int nrandom = 10; /* # of random symbols to poll */ + hbool_t verbose = TRUE; /* Whether to emit some informational messages */ + FILE *verbose_file = NULL; /* File handle for verbose output */ + hbool_t use_seed = FALSE; /* Set to 1 if a seed was set on the command line */ + unsigned random_seed = 0; /* Random # seed */ + unsigned u; /* Local index variables */ + int temp; + + /* Parse command line options */ + if(argc < 2) + usage(); + if(argc > 1) { + u = 1; + while(u < (unsigned)argc) { + if(argv[u][0] == '-') { + switch(argv[u][1]) { + /* # of common symbols to poll */ + case 'h': + ncommon = HDatoi(argv[u + 1]); + if(ncommon < 0) + usage(); + u += 2; + break; + + /* # of random symbols to poll */ + case 'l': + nrandom = HDatoi(argv[u + 1]); + if(nrandom < 0) + usage(); + u += 2; + break; + + /* Be quiet */ + case 'q': + verbose = FALSE; + u++; + break; + + /* Random # seed */ + case 'r': + use_seed = TRUE; + temp = HDatoi(argv[u + 1]); + if(temp < 0) + usage(); + else + random_seed = (unsigned)temp; + u += 2; + break; + + /* # of seconds between polling */ + case 's': + poll_time = HDatoi(argv[u + 1]); + if(poll_time < 0) + usage(); + u += 2; + break; + + default: + usage(); + break; + } /* end switch */ + } /* end if */ + else { + /* Get the number of records to append */ + nseconds = HDatol(argv[u]); + if(nseconds <= 0) + usage(); + + u++; + } /* end else */ + } /* end while */ + } /* end if */ + if(nseconds <= 0) + usage(); + if(poll_time >= nseconds) + usage(); + + /* Set the random seed */ + if(!use_seed) { + struct timeval t; + + HDgettimeofday(&t, NULL); + random_seed = (unsigned)(t.tv_usec); + } /* end if */ + HDsrandom(random_seed); + + /* Open output file */ + if(verbose) { + char verbose_name[1024]; + + HDsnprintf(verbose_name, sizeof(verbose_name), "vfd_swmr_reader.out.%u", random_seed); + if(NULL == (verbose_file = HDfopen(verbose_name, "w"))) { + HDfprintf(stderr, "READER: Can't open verbose output file!\n"); + HDexit(1); + } + } /* end if */ + + /* Emit informational message */ + if(verbose) { + HDfprintf(verbose_file, "READER: Parameters:\n"); + HDfprintf(verbose_file, "\t# of seconds between polling = %d\n", poll_time); + HDfprintf(verbose_file, "\t# of common symbols to poll = %d\n", ncommon); + HDfprintf(verbose_file, "\t# of random symbols to poll = %d\n", nrandom); + HDfprintf(verbose_file, "\t# of seconds to test = %ld\n", nseconds); + } /* end if */ + + /* ALWAYS emit the random seed for possible debugging */ + HDfprintf(stdout, "READER: Using reader random seed: %u\n", random_seed); + + /* Emit informational message */ + if(verbose) + HDfprintf(verbose_file, "READER: Generating symbol names\n"); + + /* Generate dataset names */ + if(generate_symbols() < 0) { + HDfprintf(stderr, "READER: Error generating symbol names!\n"); + HDexit(1); + } /* end if */ + + /* Create datatype for creating datasets */ + if((symbol_tid = create_symbol_datatype()) < 0) { + HDfprintf(stderr, "READER: Error creating symbol datatype!\n"); + HDexit(1); + } + + /* Reading records from datasets */ + if(read_records(COMMON_FILENAME, verbose, verbose_file, random_seed, (unsigned long)nseconds, (unsigned)poll_time, (unsigned)ncommon, (unsigned)nrandom) < 0) { + HDfprintf(stderr, "READER: Error reading records from datasets (random_seed = %u)!\n", random_seed); + HDexit(1); + } /* end if */ + + /* Emit informational message */ + if(verbose) + HDfprintf(verbose_file, "READER: Releasing symbols\n"); + + /* Clean up the symbols */ + if(shutdown_symbols() < 0) { + HDfprintf(stderr, "READER: Error releasing symbols!\n"); + HDexit(1); + } /* end if */ + + /* Emit informational message */ + if(verbose) + HDfprintf(verbose_file, "READER: Closing objects\n"); + + /* Close objects created */ + if(H5Tclose(symbol_tid) < 0) { + HDfprintf(stderr, "READER: Error closing symbol datatype!\n"); + HDexit(1); + } /* end if */ + + return 0; +} + diff --git a/test/vfd_swmr_remove_reader.c b/test/vfd_swmr_remove_reader.c new file mode 100644 index 0000000..3515987 --- /dev/null +++ b/test/vfd_swmr_remove_reader.c @@ -0,0 +1,596 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: vfd_swmr_remove_reader.c + * (copied and modified from swmr_remove_reader.c) + * + * Purpose: Reads data from a randomly selected subset of the datasets + * in the VFD SWMR test file. Unlike the regular reader, these + * datasets will be shrinking. + * + *------------------------------------------------------------------------- + */ + +/***********/ +/* Headers */ +/***********/ + +#include <err.h> /* errx(3) */ +#include <stdlib.h> /* EXIT_FAILURE */ + +#include "h5test.h" +#include "vfd_swmr_common.h" + +/*******************/ +/* Local Variables */ +/*******************/ + +static hid_t symbol_tid = -1; + +/********************/ +/* Local Prototypes */ +/********************/ + +static int check_dataset(hid_t, hid_t, unsigned, const char *, + symbol_t *, hid_t); +static int read_records(const char *, unsigned, unsigned long, + unsigned, unsigned, unsigned); +static void usage(void); + + +/*------------------------------------------------------------------------- + * Function: check_dataset + * + * Purpose: For a given dataset, checks to make sure that the stated + * and actual sizes are the same. If they are not, then + * we have an inconsistent dataset due to a SWMR error. + * + * Parameters: hid_t fid + * The SWMR test file's ID. + * + * unsigned verbose + * Whether verbose console output is desired. + * + * const char *sym_name + * The name of the dataset from which to read. + * + * symbol_t *record + * Memory for the record. Must be pre-allocated. + * + * hid_t rec_sid + * The memory dataspace for access. It's always the same so + * there is no need to re-create it every time this function + * is called. + * + * Return: Success: 0 + * Failure: -1 + * + *------------------------------------------------------------------------- + */ +static int +check_dataset(hid_t fid, hid_t dapl, unsigned verbose, const char *sym_name, symbol_t *record, + hid_t rec_sid) +{ + hid_t dsid; /* Dataset ID */ + hid_t file_sid; /* Dataset's space ID */ + hssize_t snpoints; /* Number of elements in dataset */ + hsize_t start[2] = {0, 0}, count[2] = {1, 1}; /* Hyperslab selection values */ + + HDassert(fid >= 0); + HDassert(sym_name); + HDassert(record); + HDassert(rec_sid >= 0); + + /* Open dataset for symbol */ + if((dsid = H5Dopen2(fid, sym_name, dapl)) < 0) + goto error; + + /* Get the dataset's dataspace */ + if((file_sid = H5Dget_space(dsid)) < 0) + goto error; + + /* Get the number of elements (= records, for 1-D datasets) */ + if((snpoints = H5Sget_simple_extent_npoints(file_sid)) < 0) + goto error; + + /* Back off by one: it's possible that the metadata indicating + * `snpoints` available is new, but the data is stale, because + * a tick occurred on the writer between H5Dset_extent() and H5Dwrite(). + */ + snpoints -= MAX_SIZE_CHANGE; + + /* Emit informational message */ + if(verbose) { + HDfprintf(stderr, "READER: Symbol = '%s'" + ", # of records = %" PRIdHSIZE "\n", sym_name, snpoints); + } + + /* Check if there are records for symbol */ + if(snpoints > 0) { + /* Choose a random record in the dataset, choosing the last record half + * the time */ + start[1] = (hsize_t)(HDrandom() % (snpoints * 2)); + if(start[1] > (hsize_t)(snpoints - 1)) + start[1] = (hsize_t)(snpoints - 1); + if(H5Sselect_hyperslab(file_sid, H5S_SELECT_SET, start, NULL, count, NULL) < 0) + goto error; + + /* Read record from dataset */ +#ifdef FILLVAL_WORKS + /* When shrinking the dataset, we cannot guarantee that the buffer will + * even be touched, unless there is a fill value. Since fill values do + * not work with SWMR currently (see note in swmr_generator.c), we + * simply initialize rec_id to 0. */ + record->rec_id = (uint64_t)ULLONG_MAX - 1; +#else /* FILLVAL_WORKS */ + record->rec_id = (uint64_t)0; +#endif /* FILLVAL_WORKS */ + if(H5Dread(dsid, symbol_tid, rec_sid, file_sid, H5P_DEFAULT, record) < 0) + goto error; + + /* Verify record value - note that it may be the fill value, because the + * chunk may be deleted before the object header has the updated + * dimensions */ + if(record->rec_id != start[1] && record->rec_id != 0) { + HDfprintf(stderr, "*** READER: ERROR ***\n"); + HDfprintf(stderr, "Incorrect record value!\n"); + HDfprintf(stderr, "Symbol = '%s', # of records = %" PRIdHSIZE + ", record->rec_id = %" PRIx64 ", expected %" PRIxHSIZE "\n", + sym_name, snpoints, record->rec_id, start[1]); + return -1; + } /* end if */ + } /* end if */ + + /* Close the dataset's dataspace */ + if(H5Sclose(file_sid) < 0) + goto error; + + /* Close dataset for symbol */ + if(H5Dclose(dsid) < 0) + goto error; + + return 0; + +error: + H5E_BEGIN_TRY { + H5Sclose(file_sid); + H5Dclose(dsid); + } H5E_END_TRY; + + return -1; +} /* end check_dataset() */ + + +/*------------------------------------------------------------------------- + * Function: read_records + * + * Purpose: For a given dataset, checks to make sure that the stated + * and actual sizes are the same. If they are not, then + * we have an inconsistent dataset due to a SWMR error. + * + * The "common" datasets are a random selection from among + * the level 0 datasets. The "random" datasets are a random + * selection from among all the file's datasets. This scheme + * ensures that the level 0 datasets are interrogated vigorously. + * + * Parameters: const char *filename + * The SWMR test file's name. + * + * unsigned verbose + * Whether verbose console output is desired. + * + * unsigned long nseconds + * The amount of time to read records (ns). + * + * unsigned poll_time + * The amount of time to sleep (s). + * + * unsigned ncommon + * The number of common/non-random datasets that will be opened. + * + * unsigned nrandom + * The number of random datasets that will be opened. + * + * Return: Success: 0 + * Failure: -1 + * + *------------------------------------------------------------------------- + */ +static int +read_records(const char *filename, unsigned verbose, unsigned long nseconds, + unsigned poll_time, unsigned ncommon, unsigned nrandom) +{ + time_t start_time; /* Starting time */ + time_t curr_time; /* Current time */ + symbol_info_t **sym_com = NULL; /* Pointers to array of common dataset IDs */ + symbol_info_t **sym_rand = NULL; /* Pointers to array of random dataset IDs */ + hid_t dapl; + hid_t mem_sid; /* Memory dataspace ID */ + hid_t fid; /* SWMR test file ID */ + hid_t fapl; /* File access property list */ + symbol_t record; /* The record to add to the dataset */ + unsigned v; /* Local index variable */ + H5F_vfd_swmr_config_t *config = NULL; /* Configuration for VFD SWMR */ + + HDassert(filename); + HDassert(nseconds != 0); + HDassert(poll_time != 0); + + /* Reset the record */ + /* (record's 'info' field might need to change for each record written, also) */ + HDmemset(&record, 0, sizeof(record)); + + if ((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + errx(EXIT_FAILURE, "%s.%d: H5Pcreate failed", __func__, __LINE__); + + if (H5Pset_chunk_cache(dapl, H5D_CHUNK_CACHE_NSLOTS_DEFAULT, 0, + H5D_CHUNK_CACHE_W0_DEFAULT) < 0) + errx(EXIT_FAILURE, "H5Pset_chunk_cache failed"); + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "READER: Choosing datasets\n"); + + /* Allocate space for 'common' datasets, if any */ + if(ncommon > 0) { + /* Allocate array to hold pointers to symbols for common datasets */ + if(NULL == (sym_com = (symbol_info_t **)HDmalloc(sizeof(symbol_info_t *) * ncommon))) + goto error; + + /* Open the common datasets */ + for(v = 0; v < ncommon; v++) { + unsigned offset; /* Offset of symbol to use */ + + /* Determine the offset of the symbol, within level 0 symbols */ + /* (level 0 symbols are the most common symbols) */ + offset = (unsigned)HDrandom() % symbol_count[0]; + sym_com[v] = &symbol_info[0][offset]; + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "READER: Common symbol #%u = '%s'\n", v, symbol_info[0][offset].name); + } /* end for */ + } /* end if */ + + /* Allocate space for 'random' datasets, if any */ + if(nrandom > 0) { + /* Allocate array to hold pointers to symbols for random datasets */ + if(NULL == (sym_rand = (symbol_info_t **)HDmalloc(sizeof(symbol_info_t *) * nrandom))) + goto error; + + /* Determine the random datasets */ + for(v = 0; v < nrandom; v++) { + symbol_info_t *sym; /* Symbol to use */ + + /* Determine the symbol, within all symbols */ + if(NULL == (sym = choose_dataset(NULL, NULL))) + goto error; + sym_rand[v] = sym; + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "READER: Random symbol #%u = '%s'\n", v, sym->name); + } /* end for */ + } /* end if */ + + /* Create a dataspace for the record to read */ + if((mem_sid = H5Screate(H5S_SCALAR)) < 0) + goto error; + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "READER: Reading records\n"); + + /* Get the starting time */ + start_time = HDtime(NULL); + curr_time = start_time; + + /* Create file access property list */ + if((fapl = h5_fileaccess()) < 0) + goto error; + + /* + * Set up to open the file with VFD SWMR configured. + */ + /* Enable page buffering */ + if(H5Pset_page_buffer_size(fapl, 4096, 0, 0) < 0) + goto error; + + /* Allocate memory for the configuration structure */ + if((config = (H5F_vfd_swmr_config_t *)HDcalloc(1, sizeof(H5F_vfd_swmr_config_t))) == NULL) + goto error; + + config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION; + config->tick_len = 4; + config->max_lag = 5; + config->writer = FALSE; + config->md_pages_reserved = 128; + HDstrcpy(config->md_file_path, "./my_md_file"); + + /* Enable VFD SWMR configuration */ + if(H5Pset_vfd_swmr_config(fapl, config) < 0) + goto error; + + /* Loop over reading records until [at least] the correct # of seconds have passed */ + while(curr_time < (time_t)(start_time + (time_t)nseconds)) { + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "READER: Opening file: %s\n", filename); + + /* Open the file */ + /* Remove H5E_BEGIN_TRY/END_TRY if you want to see the error stack */ + H5E_BEGIN_TRY { + fid = H5Fopen(filename, H5F_ACC_RDONLY, fapl); + } H5E_END_TRY; + if(fid < 0) { + HDfprintf(stderr, "READER: Error in opening the file: %s\n", filename); + goto error; + } + + /* Check 'common' datasets, if any */ + if(ncommon > 0) { + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "READER: Checking common symbols\n"); + + /* Iterate over common datasets */ + for(v = 0; v < ncommon; v++) { + /* Check common dataset */ + if(check_dataset(fid, dapl, verbose, sym_com[v]->name, &record, mem_sid) < 0) + goto error; + HDmemset(&record, 0, sizeof(record)); + } /* end for */ + } /* end if */ + + /* Check 'random' datasets, if any */ + if(nrandom > 0) { + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "READER: Checking random symbols\n"); + + /* Iterate over random datasets */ + for(v = 0; v < nrandom; v++) { + /* Check random dataset */ + if(check_dataset(fid, dapl, verbose, sym_rand[v]->name, &record, mem_sid) < 0) + goto error; + HDmemset(&record, 0, sizeof(record)); + } /* end for */ + } /* end if */ + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "READER: Closing file\n"); + + /* Close the file */ + if(H5Fclose(fid) < 0) + goto error; + + /* Sleep for the appropriate # of seconds */ + HDsleep(poll_time); + + /* Retrieve the current time */ + curr_time = HDtime(NULL); + } /* end while */ + + /* Close the fapl */ + if(H5Pclose(fapl) < 0) + goto error; + + if(config) + HDfree(config); + + /* Close the memory dataspace */ + if(H5Sclose(mem_sid) < 0) + goto error; + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "READER: Closing datasets\n"); + + /* Close 'random' datasets, if any */ + if(nrandom > 0) { + /* Release array holding dataset ID's for random datasets */ + HDfree(sym_rand); + } /* end if */ + + /* Close 'common' datasets, if any */ + if(ncommon > 0) { + /* Release array holding dataset ID's for common datasets */ + HDfree(sym_com); + } /* end if */ + + return 0; + +error: + if(config) + HDfree(config); + + if(sym_rand) + HDfree(sym_rand); + + if(sym_com) + HDfree(sym_com); + + H5E_BEGIN_TRY { + H5Sclose(mem_sid); + H5Fclose(fid); + H5Pclose(fapl); + H5Pclose(dapl); + } H5E_END_TRY; + + return -1; + +} /* end read_records() */ + +static void +usage(void) +{ + printf("\n"); + printf("Usage error!\n"); + printf("\n"); + printf("Usage: vfd_swmr_remove_reader [-q] [-s <# of seconds to sleep between\n"); + printf(" polling>] [-h <# of common symbols to poll>] [-l <# of random symbols\n"); + printf(" to poll>] [-r <random seed>] <# of seconds to test>\n"); + printf("\n"); + printf("Defaults to verbose (no '-q' given), 1 second between polling ('-s 1'),\n"); + printf("5 common symbols to poll ('-h 5'), 10 random symbols to poll ('-l 10'),\n"); + printf("and will generate a random seed (no -r given).\n"); + printf("\n"); + HDexit(1); +} + +int main(int argc, const char *argv[]) +{ + long nseconds = 0; /* # of seconds to test */ + int poll_time = 1; /* # of seconds between polling */ + int ncommon = 5; /* # of common symbols to poll */ + int nrandom = 10; /* # of random symbols to poll */ + unsigned verbose = 1; /* Whether to emit some informational messages */ + unsigned use_seed = 0; /* Set to 1 if a seed was set on the command line */ + unsigned random_seed = 0; /* Random # seed */ + unsigned u; /* Local index variables */ + int temp; + + /* Parse command line options */ + if(argc < 2) + usage(); + if(argc > 1) { + u = 1; + while(u < (unsigned)argc) { + if(argv[u][0] == '-') { + switch(argv[u][1]) { + /* # of common symbols to poll */ + case 'h': + ncommon = HDatoi(argv[u + 1]); + if(ncommon < 0) + usage(); + u += 2; + break; + + /* # of random symbols to poll */ + case 'l': + nrandom = HDatoi(argv[u + 1]); + if(nrandom < 0) + usage(); + u += 2; + break; + + /* Be quiet */ + case 'q': + verbose = 0; + u++; + break; + + /* Random # seed */ + case 'r': + use_seed = 1; + temp = HDatoi(argv[u + 1]); + if(temp < 0) + usage(); + else + random_seed = (unsigned)temp; + u += 2; + break; + + /* # of seconds between polling */ + case 's': + poll_time = HDatoi(argv[u + 1]); + if(poll_time < 0) + usage(); + u += 2; + break; + + default: + usage(); + break; + } /* end switch */ + } /* end if */ + else { + /* Get the number of records to append */ + nseconds = HDatol(argv[u]); + if(nseconds <= 0) + usage(); + + u++; + } /* end else */ + } /* end while */ + } /* end if */ + if(nseconds <= 0) + usage(); + if(poll_time >= nseconds) + usage(); + + /* Emit informational message */ + if(verbose) { + HDfprintf(stderr, "READER: Parameters:\n"); + HDfprintf(stderr, "\t# of seconds between polling = %d\n", poll_time); + HDfprintf(stderr, "\t# of common symbols to poll = %d\n", ncommon); + HDfprintf(stderr, "\t# of random symbols to poll = %d\n", nrandom); + HDfprintf(stderr, "\t# of seconds to test = %ld\n", nseconds); + } /* end if */ + + /* Set the random seed */ + if(0 == use_seed) { + struct timeval t; + HDgettimeofday(&t, NULL); + random_seed = (unsigned)(t.tv_usec); + } /* end if */ + HDsrandom(random_seed); + /* ALWAYS emit the random seed for possible debugging */ + HDfprintf(stderr, "READER: Using reader random seed: %u\n", random_seed); + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "READER: Generating symbol names\n"); + + /* Generate dataset names */ + if(generate_symbols() < 0) { + HDfprintf(stderr, "READER: Error generating symbol names!\n"); + HDexit(1); + } /* end if */ + + /* Create datatype for creating datasets */ + if((symbol_tid = create_symbol_datatype()) < 0) + return -1; + + /* Reading records from datasets */ + if(read_records(COMMON_FILENAME, verbose, (unsigned long)nseconds, (unsigned)poll_time, (unsigned)ncommon, (unsigned)nrandom) < 0) { + HDfprintf(stderr, "READER: Error reading records from datasets!\n"); + HDexit(1); + } /* end if */ + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "READER: Releasing symbols\n"); + + /* Clean up the symbols */ + if(shutdown_symbols() < 0) { + HDfprintf(stderr, "READER: Error releasing symbols!\n"); + HDexit(1); + } /* end if */ + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "READER: Closing objects\n"); + + /* Close objects created */ + if(H5Tclose(symbol_tid) < 0) { + HDfprintf(stderr, "READER: Error closing symbol datatype!\n"); + HDexit(1); + } /* end if */ + + return 0; +} diff --git a/test/vfd_swmr_remove_writer.c b/test/vfd_swmr_remove_writer.c new file mode 100644 index 0000000..2ebe96f --- /dev/null +++ b/test/vfd_swmr_remove_writer.c @@ -0,0 +1,426 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: vfd_swmr_remove_writer.c + * (copied and modified from swmr_remove_writer.c) + * + * Purpose: Removes data from a randomly selected subset of the datasets + * in the VFD SWMR test file. + * + * This program is intended to run concurrently with the + * vfd_swmr_remove_reader program. It is also run AFTER a sequential + * (not concurrent!) invoking of vfd_swmr_writer so the writer + * can dump a bunch of data into the datasets. Otherwise, + * there wouldn't be much to shrink :) + * + *------------------------------------------------------------------------- + */ + +/***********/ +/* Headers */ +/***********/ + +#include "h5test.h" +#include "vfd_swmr_common.h" + +/****************/ +/* Local Macros */ +/****************/ + +/* The maximum number of records to remove in one step */ +#define MAX_REMOVE_SIZE 10 + +/********************/ +/* Local Prototypes */ +/********************/ + +static hid_t open_skeleton(const char *filename, unsigned verbose, unsigned old); +static int remove_records(hid_t fid, unsigned verbose, unsigned long nshrinks, + unsigned long flush_count); +static void usage(void); + + +/*------------------------------------------------------------------------- + * Function: open_skeleton + * + * Purpose: Opens the SWMR HDF5 file and datasets. + * + * Parameters: const char *filename + * The filename of the SWMR HDF5 file to open + * + * unsigned verbose + * Whether or not to emit verbose console messages + * + * Return: Success: The file ID of the opened SWMR file + * The dataset IDs are stored in a global array + * + * Failure: -1 + * + *------------------------------------------------------------------------- + */ +static hid_t +open_skeleton(const char *filename, unsigned verbose, + unsigned old H5_ATTR_UNUSED) +{ + hid_t fid = -1; /* File ID for new HDF5 file */ + hid_t fapl = -1; /* File access property list */ + hid_t sid = -1; /* Dataspace ID */ + hsize_t dim[2]; /* Dataspace dimensions */ + unsigned u, v; /* Local index variable */ + H5F_vfd_swmr_config_t *config = NULL; /* Configuration for VFD SWMR */ + + HDassert(filename); + + /* Create file access property list */ + if((fapl = h5_fileaccess()) < 0) + goto error; + + /* FOR NOW: set to use latest format, the "old" parameter is not used */ + if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + goto error; + + /* + * Set up to open the file with VFD SWMR configured. + */ + + /* Enable page buffering */ + if(H5Pset_page_buffer_size(fapl, 4096, 0, 0) < 0) + goto error; + + /* Allocate memory for the configuration structure */ + if((config = (H5F_vfd_swmr_config_t *)HDcalloc(1, sizeof(H5F_vfd_swmr_config_t))) == NULL) + goto error; + + config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION; + config->tick_len = 4; + config->max_lag = 5; + config->writer = TRUE; + config->md_pages_reserved = 128; + HDstrcpy(config->md_file_path, "./my_md_file"); + + /* Enable VFD SWMR configuration */ + if(H5Pset_vfd_swmr_config(fapl, config) < 0) + goto error; + + /* Open the file */ + if((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) + goto error; + + /* Close file access property list */ + if(H5Pclose(fapl) < 0) + goto error; + + if(config) + HDfree(config); + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "WRITER: Opening datasets\n"); + + /* Open the datasets */ + for(u = 0; u < NLEVELS; u++) + for(v = 0; v < symbol_count[u]; v++) { + if((symbol_info[u][v].dsid = H5Dopen2(fid, symbol_info[u][v].name, H5P_DEFAULT)) < 0) + goto error; + if((sid = H5Dget_space(symbol_info[u][v].dsid)) < 0) + goto error; + if(2 != H5Sget_simple_extent_ndims(sid)) + goto error; + if(H5Sget_simple_extent_dims(sid, dim, NULL) < 0) + goto error; + symbol_info[u][v].nrecords = dim[1]; + + if(H5Sclose(sid) < 0) + goto error; + } /* end for */ + + return fid; + +error: + if(config) + HDfree(config); + + H5E_BEGIN_TRY { + for(u = 0; u < NLEVELS; u++) + for(v = 0; v < symbol_count[u]; v++) + H5Dclose(symbol_info[u][v].dsid); + H5Sclose(sid); + H5Pclose(fapl); + H5Fclose(fid); + } H5E_END_TRY; + + return -1; + +} /* open_skeleton() */ + + +/*------------------------------------------------------------------------- + * Function: remove_records + * + * Purpose: Removes a specified number of records from random datasets in + * the SWMR test file. + * + * Parameters: hid_t fid + * The file ID of the SWMR HDF5 file + * + * unsigned verbose + * Whether or not to emit verbose console messages + * + * unsigned long nshrinks + * # of records to remove from the datasets + * + * unsigned long flush_count + * # of records to write before flushing the file to disk + * + * Return: Success: 0 + * Failure: -1 + * + *------------------------------------------------------------------------- + */ +static int +remove_records(hid_t fid, unsigned verbose, unsigned long nshrinks, unsigned long flush_count) +{ + unsigned long shrink_to_flush; /* # of removals before flush */ + hsize_t dim[2] = {1,0}; /* Dataspace dimensions */ + unsigned long u, v; /* Local index variables */ + + HDassert(fid >= 0); + + /* Remove records from random datasets, according to frequency distribution */ + shrink_to_flush = flush_count; + for(u = 0; u < nshrinks; u++) { + symbol_info_t *symbol; /* Symbol to remove record from */ + hsize_t remove_size; /* Size to reduce dataset dimension by */ + + /* Get a random dataset, according to the symbol distribution */ + symbol = choose_dataset(NULL, NULL); + + /* Shrink the dataset's dataspace */ + remove_size = (hsize_t)HDrandom() % MAX_REMOVE_SIZE + 1; + if(remove_size > symbol->nrecords) + symbol->nrecords = 0; + else + symbol->nrecords -= remove_size; + dim[1] = symbol->nrecords; + if(H5Dset_extent(symbol->dsid, dim) < 0) + goto error; + + /* Check for flushing file */ + if(flush_count > 0) { + /* Decrement count of records to write before flushing */ + shrink_to_flush--; + + /* Check for counter being reached */ + if(0 == shrink_to_flush) { +#ifdef TEMP_OUT + /* Flush contents of file */ + if(H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0) + return -1; +#endif + + /* Reset flush counter */ + shrink_to_flush = flush_count; + } /* end if */ + } /* end if */ + } /* end for */ + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "WRITER: Closing datasets\n"); + + /* Close the datasets */ + for(u = 0; u < NLEVELS; u++) + for(v = 0; v < symbol_count[u]; v++) + if(H5Dclose(symbol_info[u][v].dsid) < 0) + goto error; + + return 0; + +error: + H5E_BEGIN_TRY { + for(u = 0; u < NLEVELS; u++) + for(v = 0; v < symbol_count[u]; v++) + H5Dclose(symbol_info[u][v].dsid); + + } H5E_END_TRY; + + return -1; +} /* remove_records() */ + +static void +usage(void) +{ + printf("\n"); + printf("Usage error!\n"); + printf("\n"); + printf("Usage: swmr_remove_writer [-q] [-o] [-f <# of shrinks between flushing\n"); + printf(" file contents>] [-r <random seed>] <# of shrinks>\n"); + printf("\n"); + printf("<# of shrinks between flushing file contents> should be 0 (for no\n"); + printf("flushing) or between 1 and (<# of shrinks> - 1)\n"); + printf("\n"); + printf("Defaults to verbose (no '-q' given), latest format when opening file (no '-o' given),\n"); + printf("flushing every 1000 shrinks ('-f 1000'), and will generate a random seed (no -r given).\n"); + printf("\n"); + HDexit(1); +} /* usage() */ + +int main(int argc, const char *argv[]) +{ + sigset_t oldset; + hid_t fid; /* File ID for file opened */ + long nshrinks = 0; /* # of times to shrink the dataset */ + long flush_count = 1000; /* # of records to write between flushing file */ + unsigned verbose = 1; /* Whether to emit some informational messages */ + unsigned old = 0; /* Whether to use non-latest-format when opening file */ + unsigned use_seed = 0; /* Set to 1 if a seed was set on the command line */ + unsigned random_seed = 0; /* Random # seed */ + unsigned u; /* Local index variable */ + int temp; + + block_signals(&oldset); + + /* Parse command line options */ + if(argc < 2) + usage(); + if(argc > 1) { + u = 1; + while(u < (unsigned)argc) { + if(argv[u][0] == '-') { + switch(argv[u][1]) { + /* # of records to write between flushing file */ + case 'f': + flush_count = HDatol(argv[u + 1]); + if(flush_count < 0) + usage(); + u += 2; + break; + + /* Be quiet */ + case 'q': + verbose = 0; + u++; + break; + + /* Random # seed */ + case 'r': + use_seed = 1; + temp = HDatoi(argv[u + 1]); + random_seed = (unsigned)temp; + u += 2; + break; + + /* Use non-latest-format when opening file */ + case 'o': + old = 1; + u++; + break; + + default: + usage(); + break; + } /* end switch */ + } /* end if */ + else { + /* Get the number of records to append */ + nshrinks = HDatol(argv[u]); + if(nshrinks <= 0) + usage(); + + u++; + } /* end else */ + } /* end while */ + } /* end if */ + if(nshrinks <= 0) + usage(); + if(flush_count >= nshrinks) + usage(); + + /* Emit informational message */ + if(verbose) { + HDfprintf(stderr, "WRITER: Parameters:\n"); + HDfprintf(stderr, "\t# of shrinks between flushes = %ld\n", flush_count); + HDfprintf(stderr, "\t# of shrinks = %ld\n", nshrinks); + } /* end if */ + + /* Set the random seed */ + if(0 == use_seed) { + struct timeval t; + HDgettimeofday(&t, NULL); + random_seed = (unsigned)(t.tv_usec); + } /* end if */ + HDsrandom(random_seed); + /* ALWAYS emit the random seed for possible debugging */ + HDfprintf(stderr, "WRITER: Using writer random seed: %u\n", random_seed); + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "WRITER: Generating symbol names\n"); + + /* Generate dataset names */ + if(generate_symbols() < 0) + return -1; + + /* Emit informational message */ + if(verbose) { + HDfprintf(stderr, "WRITER: Opening skeleton file: %s\n", + COMMON_FILENAME); + } + + /* Open file skeleton */ + if((fid = open_skeleton(COMMON_FILENAME, verbose, old)) < 0) { + HDfprintf(stderr, "WRITER: Error opening skeleton file!\n"); + HDexit(1); + } /* end if */ + + /* Send a message to indicate "H5Fopen" is complete--releasing the file lock */ + h5_send_message(WRITER_MESSAGE, NULL, NULL); + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "WRITER: Removing records\n"); + + /* Remove records from datasets */ + if(remove_records(fid, verbose, (unsigned long)nshrinks, (unsigned long)flush_count) < 0) { + HDfprintf(stderr, "WRITER: Error removing records from datasets!\n"); + HDexit(1); + } /* end if */ + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "WRITER: Releasing symbols\n"); + + /* Clean up the symbols */ + if(shutdown_symbols() < 0) { + HDfprintf(stderr, "WRITER: Error releasing symbols!\n"); + HDexit(1); + } /* end if */ + + await_signal(fid); + + restore_signals(&oldset); + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "WRITER: Closing objects\n"); + + /* Close objects opened */ + if(H5Fclose(fid) < 0) { + HDfprintf(stderr, "WRITER: Error closing file!\n"); + HDexit(1); + } /* end if */ + + return 0; +} /* main() */ diff --git a/test/vfd_swmr_sparse_reader.c b/test/vfd_swmr_sparse_reader.c new file mode 100644 index 0000000..83fb886 --- /dev/null +++ b/test/vfd_swmr_sparse_reader.c @@ -0,0 +1,535 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: vfd_swmr_sparse_reader.c + * (copied and modified from swmr_sparse_reader.c) + * + * Purpose: Reads data from a randomly selected subset of the datasets + * in the VFD SWMR test file. Unlike the regular reader, these + * datasets will be shrinking. + * + * This program is intended to run concurrently with the + * vfd_swmr_sparse_writer program. + * + *------------------------------------------------------------------------- + */ + +/***********/ +/* Headers */ +/***********/ + +#include <inttypes.h> /* for PRIu64 */ + +#include "h5test.h" +#include "vfd_swmr_common.h" + +/****************/ +/* Local Macros */ +/****************/ + +#define TIMEOUT 30 + +/*******************/ +/* Local Variables */ +/*******************/ + +static hid_t symbol_tid = (-1); + +/********************/ +/* Local Prototypes */ +/********************/ + +static int check_dataset(hid_t fid, unsigned verbose, const symbol_info_t *symbol, + symbol_t *record, hid_t rec_sid); +static int read_records(const char *filename, unsigned verbose, unsigned long nrecords, + unsigned poll_time, unsigned reopen_count); +static void usage(void); + + +/*------------------------------------------------------------------------- + * Function: check_dataset + * + * Purpose: For a given dataset, checks to make sure that the stated + * and actual sizes are the same. If they are not, then + * we have an inconsistent dataset due to a SWMR error. + * + * Parameters: hid_t fid + * The SWMR test file's ID. + * + * unsigned verbose + * Whether verbose console output is desired. + * + * const symbol_info_t *symbol + * The dataset from which to read (the ID is in the struct). + * Must be pre-allocated. + * + * symbol_t *record + * Memory for the record. Must be pre-allocated. + * + * hid_t rec_sid + * The memory dataspace for access. It's always the same so + * there is no need to re-create it every time this function + * is called. + * + * Return: Success: 0 + * Failure: -1 + * + *------------------------------------------------------------------------- + */ +static int +check_dataset(hid_t fid, unsigned verbose, const symbol_info_t *symbol, symbol_t *record, + hid_t rec_sid) +{ + hid_t dsid = -1; /* Dataset ID */ + hid_t file_sid = -1; /* Dataset's space ID */ + hsize_t start[2] = {0, 0}; /* Hyperslab selection values */ + hsize_t count[2] = {1, 1}; /* Hyperslab selection values */ + + HDassert(fid >= 0); + HDassert(symbol); + HDassert(record); + HDassert(rec_sid >= 0); + + /* Open dataset for symbol */ + if((dsid = H5Dopen2(fid, symbol->name, H5P_DEFAULT)) < 0) + goto error; + + /* Get the dataset's dataspace */ + if((file_sid = H5Dget_space(dsid)) < 0) + goto error; + + /* Choose the random record in the dataset (will be the same as chosen by + * the writer) */ + start[1] = (hsize_t)HDrandom() % symbol->nrecords; + if(H5Sselect_hyperslab(file_sid, H5S_SELECT_SET, start, NULL, count, NULL) < 0) + goto error; + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "READER: Symbol = '%s', nrecords = %Hu, name = %s, location = %Hu, %Hu\n", \ + symbol->name, symbol->nrecords, symbol->name, start[0], start[1]); + + /* Read record from dataset */ + record->rec_id = UINT64_MAX; + if(H5Dread(dsid, symbol_tid, rec_sid, file_sid, H5P_DEFAULT, record) < 0) + goto error; + + /* Verify record value */ + if(record->rec_id != start[1]) { + HDfprintf(stderr, "*** READER: ERROR ***\n"); + HDfprintf(stderr, "Incorrect record value!\n"); + HDfprintf(stderr, "Symbol = '%s', location = %Hu, %Hu, record->rec_id = %" PRIu64 "\n", symbol->name, start[0], start[1], record->rec_id); + goto error; + } /* end if */ + + /* Close the dataset's dataspace */ + if(H5Sclose(file_sid) < 0) + goto error; + + /* Close dataset for symbol */ + if(H5Dclose(dsid) < 0) + goto error; + + return 0; + +error: + + H5E_BEGIN_TRY { + H5Sclose(file_sid); + H5Dclose(dsid); + } H5E_END_TRY; + + return -1; +} /* end check_dataset() */ + + +/*------------------------------------------------------------------------- + * Function: read_records + * + * Purpose: For a given dataset, checks to make sure that the stated + * and actual sizes are the same. If they are not, then + * we have an inconsistent dataset due to a SWMR error. + * + * Parameters: const char *filename + * The SWMR test file's name. + * + * unsigned verbose + * Whether verbose console output is desired. + * + * unsigned long nrecords + * The total number of records to read. + * + * unsigned poll_time + * The amount of time to sleep (s). + * + * unsigned reopen_count + * + * + * Return: Success: 0 + * Failure: -1 + * + *------------------------------------------------------------------------- + */ +static int +read_records(const char *filename, unsigned verbose, unsigned long nrecords, + unsigned poll_time, unsigned reopen_count) +{ + hid_t fid = H5I_INVALID_HID; + hid_t aid = H5I_INVALID_HID; + time_t start_time; /* Starting time */ + hid_t mem_sid = H5I_INVALID_HID; + symbol_t record; /* The record to add to the dataset */ + unsigned seed; /* Seed for random number generator */ + unsigned iter_to_reopen = reopen_count; /* # of iterations until reopen */ + unsigned long u; /* Local index variable */ + hid_t fapl = H5I_INVALID_HID; + H5F_vfd_swmr_config_t *config = NULL; /* Configuration for VFD SWMR */ + + HDassert(filename); + HDassert(poll_time != 0); + + /* Create file access property list */ + if((fapl = h5_fileaccess()) < 0) + goto error; + + if(H5Pset_fclose_degree(fapl, H5F_CLOSE_SEMI) < 0) + goto error; + + /* + * Set up to open the file with VFD SWMR configured. + */ + /* Enable page buffering */ + if(H5Pset_page_buffer_size(fapl, 4096, 0, 0) < 0) + goto error; + + /* Allocate memory for the configuration structure */ + if((config = HDcalloc(1, sizeof(H5F_vfd_swmr_config_t))) == NULL) + goto error; + + config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION; + config->tick_len = 4; + config->max_lag = 5; + config->writer = FALSE; + config->md_pages_reserved = 128; + HDstrcpy(config->md_file_path, "./my_md_file"); + + /* Enable VFD SWMR configuration */ + if(H5Pset_vfd_swmr_config(fapl, config) < 0) + goto error; + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "READER: Opening file: %s\n", filename); + + /* Open the file */ + /* Remove H5E_BEGIN_TRY/END_TRY to see the error stack if error */ + H5E_BEGIN_TRY { + fid = H5Fopen(filename, H5F_ACC_RDONLY, fapl); + } H5E_END_TRY; + if(fid < 0) { + HDfprintf(stderr, "READER: Error in opening the file: %s\n", filename); + goto error; + } else + HDfprintf(stderr, "READER: SUCCESS in opening the file: %s\n", filename); + + /* Seed the random number generator with the attribute in the file */ + if((aid = H5Aopen(fid, "seed", H5P_DEFAULT)) < 0) + goto error; + if(H5Aread(aid, H5T_NATIVE_UINT, &seed) < 0) + goto error; + if(H5Aclose(aid) < 0) + goto error; + HDsrandom(seed); + + /* Reset the record */ + /* (record's 'info' field might need to change for each record written, also) */ + HDmemset(&record, 0, sizeof(record)); + + /* Create a dataspace for the record to read */ + if((mem_sid = H5Screate(H5S_SCALAR)) < 0) + goto error; + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "READER: Reading records\n"); + + /* Read records */ + for(u = 0; u < nrecords; u++) { + unsigned level, offset; + symbol_info_t *symbol = NULL; /* Symbol (dataset) */ + htri_t attr_exists; /* Whether the sequence number attribute exists */ + unsigned long file_u; /* Attribute sequence number (writer's "u") */ + + /* Get a random dataset, according to the symbol distribution */ + symbol = choose_dataset(&level, &offset); + + /* Fill in "nrecords" field. Note that this depends on the writer + * using the same algorithm and "nrecords" */ + symbol->nrecords = nrecords / 5; + + /* Get the starting time */ + if ((start_time = HDtime(NULL)) == (time_t)-1) { + fprintf(stderr, "READER: could not read time.\n"); + goto error; + } + + /* Wait until we can read the dataset */ + for (;;) { + if((attr_exists = H5Aexists_by_name(fid, symbol->name, "seq", H5P_DEFAULT)) < 0) + goto error; + + if(attr_exists) { + if((aid = H5Aopen_by_name(fid, symbol->name, "seq", + H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + if(H5Aread(aid, H5T_NATIVE_ULONG, &file_u) < 0) + goto error; + if(H5Aclose(aid) < 0) + goto error; + + if(file_u >= u) + break; + } + + if(HDtime(NULL) >= (time_t)(start_time + (time_t)TIMEOUT)) { + HDfprintf(stderr, + "READER: Reader timed at record %lu level %u offset %u", + u, level, offset); + if (attr_exists) { + HDfprintf(stderr, ", read sequence %lu\n", file_u); + } else { + HDfprintf(stderr, ", read no sequence\n"); + HDfprintf(stderr, ", read no sequence\n"); + } + goto error; + } + + HDsleep(poll_time); + + if(verbose) + HDfprintf(stderr, "READER: Reopening file (do while loop): %s\n", filename); + + if(print_metadata_retries_info(fid) < 0) + HDfprintf(stderr, "READER: Warning: could not obtain metadata retries info\n"); + + if(H5Fclose(fid) < 0) + goto error; + + H5E_BEGIN_TRY { + fid = H5Fopen(filename, H5F_ACC_RDONLY, fapl); + } H5E_END_TRY; + if(fid < 0) { + HDfprintf(stderr, "READER: Error in reopening the file (do while loop): %s\n", filename); + goto error; + } + iter_to_reopen = reopen_count; + } + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "READER: Checking dataset %lu\n", u); + + /* Check dataset */ + if(check_dataset(fid, verbose, symbol, &record, mem_sid) < 0) + goto error; + HDmemset(&record, 0, sizeof(record)); + + /* Check for reopen */ + iter_to_reopen--; + if(iter_to_reopen == 0) { + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "READER: Reopening file (iter_to_reopen): %s\n", filename); + + /* Retrieve and print the collection of metadata read retries */ + if(print_metadata_retries_info(fid) < 0) + HDfprintf(stderr, "READER: Warning: could not obtain metadata retries info\n"); + + /* Reopen the file */ + if(H5Fclose(fid) < 0) + goto error; + + /* Remove H5E_BEGIN_TRY/END_TRY to see the error stack if error */ +// H5E_BEGIN_TRY { + fid = H5Fopen(filename, H5F_ACC_RDONLY, fapl); + // } H5E_END_TRY; + if(fid < 0) { + HDfprintf(stderr, "READER: Error in reopening the file (iter_to_reopen): %s\n", filename); + goto error; + } + + iter_to_reopen = reopen_count; + } /* end if */ + } /* end while */ + + /* Retrieve and print the collection of metadata read retries */ + if(print_metadata_retries_info(fid) < 0) + HDfprintf(stderr, "READER: Warning: could not obtain metadata retries info\n"); + + /* Close file */ + if(H5Fclose(fid) < 0) + goto error; + + /* Close the memory dataspace */ + if(H5Sclose(mem_sid) < 0) + goto error; + + /* Close the file access property list */ + if(H5Pclose(fapl) < 0) + goto error; + + if(config) + HDfree(config); + + return 0; + +error: + if(config) + HDfree(config); + + H5E_BEGIN_TRY { + H5Aclose(aid); + H5Sclose(mem_sid); + H5Fclose(fid); + H5Pclose(fapl); + } H5E_END_TRY; + + return -1; +} /* end read_records() */ + +static void +usage(void) +{ + printf("\n"); + printf("Usage error!\n"); + printf("\n"); + printf("Usage: vfd_swmr_sparse_reader [-q] [-s <# of seconds to wait for writer>]\n"); + printf(" [-n <# of reads between reopens>] <# of records>\n"); + printf("\n"); + printf("Defaults to verbose (no '-q' given), 1 second wait ('-s 1') and 1 read\n"); + printf("between reopens ('-r 1')\n"); + printf("\n"); + printf("Note that the # of records *must* be the same as that supplied to\n"); + printf("vfd_swmr_sparse_writer\n"); + printf("\n"); + HDexit(1); +} /* end usage() */ + +int main(int argc, const char *argv[]) +{ + long nrecords = 0; /* # of records to read */ + int poll_time = 1; /* # of seconds to sleep when waiting for writer */ + int reopen_count = 1; /* # of reads between reopens */ + unsigned verbose = 1; /* Whether to emit some informational messages */ + unsigned u; /* Local index variables */ + + /* Parse command line options */ + if(argc < 2) + usage(); + if(argc > 1) { + u = 1; + while(u < (unsigned)argc) { + if(argv[u][0] == '-') { + switch(argv[u][1]) { + /* # of reads between reopens */ + case 'n': + reopen_count = HDatoi(argv[u + 1]); + if(reopen_count < 0) + usage(); + u += 2; + break; + + /* Be quiet */ + case 'q': + verbose = 0; + u++; + break; + + /* # of seconds between polling */ + case 's': + poll_time = HDatoi(argv[u + 1]); + if(poll_time < 0) + usage(); + u += 2; + break; + + default: + usage(); + break; + } /* end switch */ + } /* end if */ + else { + /* Get the number of records to read */ + nrecords = HDatol(argv[u]); + if(nrecords <= 0) + usage(); + + u++; + } /* end else */ + } /* end while */ + } /* end if */ + + /* Emit informational message */ + if(verbose) { + HDfprintf(stderr, "READER: Parameters:\n"); + HDfprintf(stderr, "\t# of seconds between polling = %d\n", poll_time); + HDfprintf(stderr, "\t# of reads between reopens = %d\n", reopen_count); + HDfprintf(stderr, "\t# of records to read = %ld\n", nrecords); + } /* end if */ + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "READER: Generating symbol names\n"); + + /* Generate dataset names */ + if(generate_symbols() < 0) { + HDfprintf(stderr, "READER: Error generating symbol names!\n"); + HDexit(1); + } /* end if */ + + /* Create datatype for creating datasets */ + if((symbol_tid = create_symbol_datatype()) < 0) { + HDfprintf(stderr, "READER: Error creating symbol datatype!\n"); + HDexit(1); + } + + /* Reading records from datasets */ + if(read_records(COMMON_FILENAME, verbose, (unsigned long) nrecords, (unsigned)poll_time, (unsigned)reopen_count) < 0) { + HDfprintf(stderr, "READER: Error reading records from datasets!\n"); + HDexit(1); + } /* end if */ + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "READER: Releasing symbols\n"); + + /* Clean up the symbols */ + if(shutdown_symbols() < 0) { + HDfprintf(stderr, "READER: Error releasing symbols!\n"); + HDexit(1); + } /* end if */ + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "READER: Closing objects\n"); + + /* Close objects created */ + if(H5Tclose(symbol_tid) < 0) { + HDfprintf(stderr, "READER: Error closing symbol datatype!\n"); + HDexit(1); + } /* end if */ + + return 0; +} /* main() */ diff --git a/test/vfd_swmr_sparse_writer.c b/test/vfd_swmr_sparse_writer.c new file mode 100644 index 0000000..5722558 --- /dev/null +++ b/test/vfd_swmr_sparse_writer.c @@ -0,0 +1,488 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + + /*------------------------------------------------------------------------- + * + * Created: vfd_swmr_sparse_writer.c + * (copied and modified from swmr_sparse_writer.c) + * + * Purpose: Writes data to a randomly selected subset of the datasets + * in the VFD_SWMR test file. + * + * This program is intended to run concurrently with the + * vfd_swmr_sparse_reader program. + * + *------------------------------------------------------------------------- + */ + +/***********/ +/* Headers */ +/***********/ +#include <err.h> +#include <signal.h> + +#include "h5test.h" +#include "vfd_swmr_common.h" + +/****************/ +/* Local Macros */ +/****************/ + +#ifdef OUT +#define BUSY_WAIT 100000 +#endif /* OUT */ + +/********************/ +/* Local Prototypes */ +/********************/ + +static hid_t open_skeleton(const char *filename, unsigned verbose); +static int add_records(hid_t fid, unsigned verbose, unsigned long nrecords, + unsigned long flush_count); +static void usage(void); + + + +/*------------------------------------------------------------------------- + * Function: open_skeleton + * + * Purpose: Opens the SWMR HDF5 file and datasets. + * + * Parameters: const char *filename + * The filename of the SWMR HDF5 file to open + * + * unsigned verbose + * Whether or not to emit verbose console messages + * + * Return: Success: The file ID of the opened SWMR file + * The dataset IDs are stored in a global array + * + * Failure: -1 + * + *------------------------------------------------------------------------- + */ +static hid_t +open_skeleton(const char *filename, unsigned verbose) +{ + hid_t fid = -1; /* File ID for new HDF5 file */ + hid_t fapl = -1; /* File access property list */ + hid_t aid = -1; /* Attribute ID */ + unsigned seed; /* Seed for random number generator */ + unsigned u, v; /* Local index variable */ + H5F_vfd_swmr_config_t *config = NULL; /* Configuration for VFD SWMR */ + + HDassert(filename); + + /* Create file access property list */ + if((fapl = h5_fileaccess()) < 0) + goto error; + + /* Set to use the latest library format */ + if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + goto error; + + /* + * Set up to open the file with VFD SWMR configured. + */ + + /* Enable page buffering */ + if(H5Pset_page_buffer_size(fapl, 4096, 0, 0) < 0) + goto error; + + /* Allocate memory for the configuration structure */ + if((config = (H5F_vfd_swmr_config_t *)HDcalloc(1, sizeof(H5F_vfd_swmr_config_t))) == NULL) + goto error; + + config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION; + config->tick_len = 4; + config->max_lag = 5; + config->writer = TRUE; + config->md_pages_reserved = 128; + HDstrcpy(config->md_file_path, "./my_md_file"); + + /* Enable VFD SWMR configuration */ + if(H5Pset_vfd_swmr_config(fapl, config) < 0) + goto error; + + /* Open the file */ + if((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) + goto error; + + /* Close file access property list */ + if(H5Pclose(fapl) < 0) + goto error; + + if(config) + HDfree(config); + + /* Emit informational message */ + if(verbose) + fprintf(stderr, "WRITER: Opening datasets\n"); + + /* Seed the random number generator with the attribute in the file */ + if((aid = H5Aopen(fid, "seed", H5P_DEFAULT)) < 0) + goto error; + if(H5Aread(aid, H5T_NATIVE_UINT, &seed) < 0) + goto error; + if(H5Aclose(aid) < 0) + goto error; + HDsrandom(seed); + + /* Open the datasets */ + for(u = 0; u < NLEVELS; u++) + for(v = 0; v < symbol_count[u]; v++) { + if((symbol_info[u][v].dsid = H5Dopen2(fid, symbol_info[u][v].name, H5P_DEFAULT)) < 0) + return(-1); + symbol_info[u][v].nrecords = 0; + } /* end for */ + + return fid; + +error: + if(config) + HDfree(config); + + H5E_BEGIN_TRY { + for(u = 0; u < NLEVELS; u++) + for(v = 0; v < symbol_count[u]; v++) + H5Dclose(symbol_info[u][v].dsid); + H5Aclose(aid); + H5Pclose(fapl); + H5Fclose(fid); + } H5E_END_TRY; + + return -1; + +} /* open_skeleton() */ + + +/*------------------------------------------------------------------------- + * Function: add_records + * + * Purpose: Writes a specified number of records to random datasets in + * the SWMR test file. + * + * Parameters: hid_t fid + * The file ID of the SWMR HDF5 file + * + * unsigned verbose + * Whether or not to emit verbose console messages + * + * unsigned long nrecords + * # of records to write to the datasets + * + * unsigned long flush_count + * # of records to write before flushing the file to disk + * + * Return: Success: 0 + * Failure: -1 + * + *------------------------------------------------------------------------- + */ +static int +add_records(hid_t fid, unsigned verbose, unsigned long nrecords, unsigned long flush_count) +{ + hid_t tid = -1; /* Datatype ID for records */ + hid_t mem_sid = -1; /* Memory dataspace ID */ + hid_t file_sid = -1; /* Dataset's space ID */ + hid_t aid = -1; /* Attribute ID */ + hsize_t start[2] = {0, 0}; /* Hyperslab selection values */ + hsize_t count[2] = {1, 1}; /* Hyperslab selection values */ + symbol_t record; /* The record to add to the dataset */ + unsigned long rec_to_flush; /* # of records left to write before flush */ +#ifdef OUT + volatile int dummy; /* Dummy varialbe for busy sleep */ +#endif /* OUT */ + hsize_t dim[2] = {1,0}; /* Dataspace dimensions */ + unsigned long u, v; /* Local index variables */ + + HDassert(fid >= 0); + + /* Reset the record */ + /* (record's 'info' field might need to change for each record written, also) */ + HDmemset(&record, 0, sizeof(record)); + + /* Create a dataspace for the record to add */ + if((mem_sid = H5Screate(H5S_SCALAR)) < 0) + goto error; + + /* Create datatype for appending records */ + if((tid = create_symbol_datatype()) < 0) + goto error; + + /* Add records to random datasets, according to frequency distribution */ + rec_to_flush = flush_count; + for(u = 0; u < nrecords; u++) { + symbol_info_t *symbol; /* Symbol to write record to */ + + /* Get a random dataset, according to the symbol distribution */ + symbol = choose_dataset(NULL, NULL); + + /* If this is the first time the dataset has been opened, extend it and + * add the sequence attribute */ + if(symbol->nrecords == 0) { + symbol->nrecords = nrecords / 5; + dim[1] = symbol->nrecords; + + if(H5Dset_extent(symbol->dsid, dim) < 0) + goto error; + + if((file_sid = H5Screate(H5S_SCALAR)) < 0) + goto error; + if((aid = H5Acreate2(symbol->dsid, "seq", H5T_NATIVE_ULONG, file_sid, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + if(H5Sclose(file_sid) < 0) + goto error; + } else if ((aid = H5Aopen(symbol->dsid, "seq", H5P_DEFAULT)) < 0) + goto error; + + /* Get the coordinate to write */ + start[1] = (hsize_t)HDrandom() % symbol->nrecords; + + /* Set the record's ID (equal to its position) */ + record.rec_id = start[1]; + + /* Get the dataset's dataspace */ + if((file_sid = H5Dget_space(symbol->dsid)) < 0) + goto error; + + /* Choose a random record in the dataset */ + if(H5Sselect_hyperslab(file_sid, H5S_SELECT_SET, start, NULL, count, NULL) < 0) + goto error; + + /* Write record to the dataset */ + if(H5Dwrite(symbol->dsid, tid, mem_sid, file_sid, H5P_DEFAULT, &record) < 0) + goto error; + + /* Write the sequence number attribute. Since we synchronize the random + * number seed, the readers will always generate the same sequence of + * randomly chosen datasets and offsets. Therefore, and because of the + * flush dependencies on the object header, the reader will be + * guaranteed to see the written data if the sequence attribute is >=u. + */ + if(H5Awrite(aid, H5T_NATIVE_ULONG, &u) < 0) + goto error; + + /* Close the attribute */ + if(H5Aclose(aid) < 0) + goto error; + + /* Close the dataset's dataspace */ + if(H5Sclose(file_sid) < 0) + goto error; + + /* Check for flushing file */ + if(flush_count > 0) { + /* Decrement count of records to write before flushing */ + rec_to_flush--; + + /* Check for counter being reached */ + if(0 == rec_to_flush) { +#ifdef TEMP_OUT + /* Flush contents of file */ + if(H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0) + return -1; +#endif + + /* Reset flush counter */ + rec_to_flush = flush_count; + } /* end if */ + } /* end if */ + +#ifdef OUT + /* Busy wait, to let readers catch up */ + /* If this is removed, also remove the BUSY_WAIT symbol + * at the top of the file. + */ + dummy = 0; + for(v=0; v<BUSY_WAIT; v++) + dummy++; + if((unsigned long)dummy != v) + return -1; +#endif /* OUT */ + + } /* end for */ + + /* Close the memory dataspace */ + if(H5Sclose(mem_sid) < 0) + goto error; + + /* Close the datatype */ + if(H5Tclose(tid) < 0) + goto error; + + /* Emit informational message */ + if(verbose) + fprintf(stderr, "WRITER: Closing datasets\n"); + + /* Close the datasets */ + for(u = 0; u < NLEVELS; u++) + for(v = 0; v < symbol_count[u]; v++) + if(H5Dclose(symbol_info[u][v].dsid) < 0) + goto error; + + return 0; + +error: + H5E_BEGIN_TRY { + H5Sclose(mem_sid); + H5Sclose(file_sid); + H5Tclose(tid); + H5Aclose(aid); + + for(u = 0; u < NLEVELS; u++) + for(v = 0; v < symbol_count[u]; v++) + H5Dclose(symbol_info[u][v].dsid); + + } H5E_END_TRY; + + return -1; +} /* add_records() */ + +static void +usage(void) +{ + printf("\n"); + printf("Usage error!\n"); + printf("\n"); + printf("Usage: vfd_swmr_sparse_writer [-q] [-f <# of records to write between\n"); + printf(" flushing file contents>] <# of records>\n"); + printf("\n"); + printf("<# of records to write between flushing file contents> should be 0\n"); + printf("(for no flushing) or between 1 and (<# of records> - 1)\n"); + printf("\n"); + printf("Defaults to verbose (no '-q' given) and flushing every 1000 records\n"); + printf("('-f 1000')\n"); + printf("\n"); + HDexit(1); +} /* usage() */ + +int main(int argc, const char *argv[]) +{ + sigset_t oldset; + hid_t fid; /* File ID for file opened */ + long nrecords = 0; /* # of records to append */ + long flush_count = 1000; /* # of records to write between flushing file */ + unsigned verbose = 1; /* Whether to emit some informational messages */ + unsigned u; /* Local index variable */ + + block_signals(&oldset); + + /* Parse command line options */ + if(argc < 2) + usage(); + if(argc > 1) { + u = 1; + while(u < (unsigned)argc) { + if(argv[u][0] == '-') { + switch(argv[u][1]) { + /* # of records to write between flushing file */ + case 'f': + flush_count = HDatol(argv[u + 1]); + if(flush_count < 0) + usage(); + u += 2; + break; + + /* Be quiet */ + case 'q': + verbose = 0; + u++; + break; + + default: + usage(); + break; + } /* end switch */ + } /* end if */ + else { + /* Get the number of records to append */ + nrecords = HDatol(argv[u]); + if(nrecords <= 0) + usage(); + + u++; + } /* end else */ + } /* end while */ + } /* end if */ + if(nrecords <= 0) + usage(); + if(flush_count >= nrecords) + usage(); + + /* Emit informational message */ + if(verbose) { + HDfprintf(stderr, "WRITER: Parameters:\n"); + HDfprintf(stderr, "\t# of records between flushes = %ld\n", flush_count); + HDfprintf(stderr, "\t# of records to write = %ld\n", nrecords); + } /* end if */ + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "WRITER: Generating symbol names\n"); + + /* Generate dataset names */ + if(generate_symbols() < 0) + return -1; + + /* Emit informational message */ + if(verbose) { + HDfprintf(stderr, "WRITER: Opening skeleton file: %s\n", + COMMON_FILENAME); + } + + /* Open file skeleton */ + if((fid = open_skeleton(COMMON_FILENAME, verbose)) < 0) { + HDfprintf(stderr, "WRITER: Error opening skeleton file!\n"); + HDexit(1); + } /* end if */ + + /* Send a message to indicate "H5Fopen" is complete--releasing the file lock */ + h5_send_message(WRITER_MESSAGE, NULL, NULL); + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "WRITER: Adding records\n"); + + /* Append records to datasets */ + if(add_records(fid, verbose, (unsigned long)nrecords, (unsigned long)flush_count) < 0) { + HDfprintf(stderr, "WRITER: Error appending records to datasets!\n"); + HDexit(1); + } /* end if */ + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "WRITER: Releasing symbols\n"); + + /* Clean up the symbols */ + if(shutdown_symbols() < 0) { + HDfprintf(stderr, "WRITER: Error releasing symbols!\n"); + HDexit(1); + } /* end if */ + + await_signal(fid); + + restore_signals(&oldset); + + /* Emit informational message */ + if(verbose) + HDfprintf(stderr, "WRITER: Closing objects/file\n"); + + /* Close objects opened */ + if(H5Fclose(fid) < 0) { + HDfprintf(stderr, "WRITER: Error closing file!\n"); + HDexit(1); + } /* end if */ + + return 0; +} /* main() */ diff --git a/test/vfd_swmr_vlstr_reader.c b/test/vfd_swmr_vlstr_reader.c new file mode 100644 index 0000000..df9037729 --- /dev/null +++ b/test/vfd_swmr_vlstr_reader.c @@ -0,0 +1,232 @@ +/* + * Copyright by The HDF Group. + * Copyright by the Board of Trustees of the University of Illinois. + * All rights reserved. + * + * This file is part of HDF5. The full HDF5 copyright notice, including + * terms governing use, modification, and redistribution, is contained in + * the COPYING file, which can be found at the root of the source code + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. + * If you do not have access to either file, you may request a copy from + * help@hdfgroup.org. + */ + +#include <err.h> +#include <time.h> /* nanosleep(2) */ +#include <unistd.h> /* getopt(3) */ + +#define H5C_FRIEND /*suppress error about including H5Cpkg */ +#define H5F_FRIEND /*suppress error about including H5Fpkg */ + +#include "hdf5.h" + +#include "H5Cpkg.h" +#include "H5Fpkg.h" +// #include "H5Iprivate.h" +#include "H5HGprivate.h" +#include "H5VLprivate.h" + +#include "testhdf5.h" +#include "vfd_swmr_common.h" + +typedef enum _step { + CREATE = 0 +, LENGTHEN +, SHORTEN +, DELETE +, NSTEPS +} step_t; + +static const hid_t badhid = H5I_INVALID_HID; // abbreviate +static bool caught_out_of_bounds = false; +static bool read_null = false; + +static bool +read_vl_dset(hid_t dset, hid_t type, char **data) +{ + bool success; + estack_state_t es; + + es = disable_estack(); + success = H5Dread(dset, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) >= 0; + if (*data == NULL) { + read_null = true; + return false; + } + restore_estack(es); + + return success; +} + +static void +usage(const char *progname) +{ + fprintf(stderr, "usage: %s [-W] [-V] [-t (oob|null)] \n", progname); + fprintf(stderr, "\n -S: do not use VFD SWMR\n"); + fprintf(stderr, " -n: number of test steps to perform\n"); + fprintf(stderr, " -q: be quiet: few/no progress messages\n"); + fprintf(stderr, " -t (oob|null): select out-of-bounds or NULL test\n"); + exit(EXIT_FAILURE); +} + +bool +H5HG_trap(const char *reason) +{ + if (strcmp(reason, "out of bounds") == 0) { + caught_out_of_bounds = true; + return true; + } + return false; +} + +int +main(int argc, char **argv) +{ + hid_t fapl, fid, space, type; + hid_t dset[2]; + char *content[2]; + char name[2][96]; + int ch, i, ntimes = 100; + unsigned long tmp; + bool use_vfd_swmr = true; + char *end; + const long millisec_in_nanosecs = 1000 * 1000; + const struct timespec delay = + {.tv_sec = 0, .tv_nsec = millisec_in_nanosecs * 11 / 10}; + testsel_t sel = TEST_NONE; + + assert(H5T_C_S1 != badhid); + + while ((ch = getopt(argc, argv, "Sn:qt:")) != -1) { + switch(ch) { + case 'S': + use_vfd_swmr = false; + break; + case 'n': + errno = 0; + tmp = strtoul(optarg, &end, 0); + if (end == optarg || *end != '\0') + errx(EXIT_FAILURE, "couldn't parse `-n` argument `%s`", optarg); + else if (errno != 0) + err(EXIT_FAILURE, "couldn't parse `-n` argument `%s`", optarg); + else if (tmp > INT_MAX) + errx(EXIT_FAILURE, "`-n` argument `%lu` too large", tmp); + ntimes = (int)tmp; + break; + case 'q': + verbosity = 1; + break; + case 't': + if (strcmp(optarg, "oob") == 0) + sel = TEST_OOB; + else if (strcmp(optarg, "null") == 0) + sel = TEST_NULL; + else + usage(argv[0]); + break; + default: + usage(argv[0]); + break; + } + } + argv += optind; + argc -= optind; + + if (argc > 0) + errx(EXIT_FAILURE, "unexpected command-line arguments"); + + fapl = vfd_swmr_create_fapl(false, sel == TEST_OOB, use_vfd_swmr, + "./shadow"); + if (fapl < 0) + errx(EXIT_FAILURE, "vfd_swmr_create_fapl"); + + fid = H5Fopen("vfd_swmr_vlstr.h5", H5F_ACC_RDONLY, fapl); + + /* Create the VL string datatype and a scalar dataspace, or a + * fixed-length string datatype and a simple dataspace. + */ + if ((type = H5Tcopy(H5T_C_S1)) == badhid) + errx(EXIT_FAILURE, "H5Tcopy"); + + /* Create the VL string datatype and a scalar dataspace */ + if ((type = H5Tcopy(H5T_C_S1)) == badhid) + errx(EXIT_FAILURE, "H5Tcopy"); + + if (H5Tset_size(type, H5T_VARIABLE) < 0) + errx(EXIT_FAILURE, "H5Tset_size"); + space = H5Screate(H5S_SCALAR); + + if (space == badhid) + errx(EXIT_FAILURE, "H5Screate"); + + if (fid == badhid) + errx(EXIT_FAILURE, "H5Fcreate"); + + /* content 1 seq 1 short + * content 1 seq 1 long long long long long long long long + * content 1 seq 1 medium medium medium + */ + for (i = 0; + !caught_out_of_bounds && i < ntimes; + (i % 2 == 0) ? nanosleep(&delay, NULL) : 0, i++) { + estack_state_t es; + const int ndsets = 2; + const int which = i % ndsets; + int nconverted; + struct { + int which; + int seq; + char tail[96]; + } scanned_content; + + dbgf(2, "iteration %d which %d", i, which); + (void)snprintf(name[which], sizeof(name[which]), + "dset-%d", which); + es = disable_estack(); + dset[which] = H5Dopen(fid, name[which], H5P_DEFAULT); + restore_estack(es); + if (caught_out_of_bounds || dset[which] == badhid) { + dbgf(2, ": couldn't open\n"); + continue; + } + if (!read_vl_dset(dset[which], type, &content[which])) { + H5Dclose(dset[which]); + dbgf(2, ": couldn't read\n"); + continue; + } + nconverted = sscanf(content[which], "content %d seq %d %96s", + &scanned_content.which, &scanned_content.seq, scanned_content.tail); + if (nconverted != 3) { + dbgf(2, ": couldn't scan\n"); + continue; + } + dbgf(2, ": read which %d seq %d tail %s\n", + scanned_content.which, scanned_content.seq, scanned_content.tail); + H5Dclose(dset[which]); + } + + if (caught_out_of_bounds) + fprintf(stderr, "caught out of bounds\n"); + + if (read_null) + fprintf(stderr, "read NULL\n"); + + if (H5Pclose(fapl) < 0) + errx(EXIT_FAILURE, "H5Pclose(fapl)"); + + if (H5Tclose(type) < 0) + errx(EXIT_FAILURE, "H5Tclose"); + + if (H5Sclose(space) < 0) + errx(EXIT_FAILURE, "H5Sclose"); + + if (H5Fclose(fid) < 0) + errx(EXIT_FAILURE, "H5Fclose"); + + if (sel == TEST_OOB) + return caught_out_of_bounds ? EXIT_SUCCESS : EXIT_FAILURE; + else if (sel == TEST_NULL) + return read_null ? EXIT_SUCCESS : EXIT_FAILURE; + + return EXIT_SUCCESS; +} diff --git a/test/vfd_swmr_vlstr_writer.c b/test/vfd_swmr_vlstr_writer.c new file mode 100644 index 0000000..193e03c --- /dev/null +++ b/test/vfd_swmr_vlstr_writer.c @@ -0,0 +1,322 @@ +/* + * Copyright by The HDF Group. + * Copyright by the Board of Trustees of the University of Illinois. + * All rights reserved. + * + * This file is part of HDF5. The full HDF5 copyright notice, including + * terms governing use, modification, and redistribution, is contained in + * the COPYING file, which can be found at the root of the source code + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. + * If you do not have access to either file, you may request a copy from + * help@hdfgroup.org. + */ + +#include <err.h> +#include <time.h> /* nanosleep(2) */ +#include <unistd.h> /* getopt(3) */ + +#define H5C_FRIEND /*suppress error about including H5Cpkg */ +#define H5F_FRIEND /*suppress error about including H5Fpkg */ + +#include "hdf5.h" + +#include "H5Cpkg.h" +#include "H5Fpkg.h" +// #include "H5Iprivate.h" +#include "H5HGprivate.h" +#include "H5VLprivate.h" + +#include "testhdf5.h" +#include "vfd_swmr_common.h" + +enum _step { + CREATE = 0 +, LENGTHEN +, SHORTEN +, DELETE +, NSTEPS +} step_t; + +static const hid_t badhid = H5I_INVALID_HID; // abbreviate +static bool caught_out_of_bounds = false; + +static void +write_vl_dset(hid_t dset, hid_t type, hid_t space, char *data) +{ + if (H5Dwrite(dset, type, space, space, H5P_DEFAULT, &data) < 0) + errx(EXIT_FAILURE, "%s: H5Dwrite", __func__); + if (H5Dflush(dset) < 0) + errx(EXIT_FAILURE, "%s: H5Dflush", __func__); +} + +#if 0 +static hid_t +initialize_dset(hid_t file, hid_t type, hid_t space, const char *name, + void *data) +{ + hid_t dset; + + dset = H5Dcreate2(file, name, type, space, H5P_DEFAULT, H5P_DEFAULT, + H5P_DEFAULT); + + if (dset == badhid) + errx(EXIT_FAILURE, "H5Dcreate2"); + + if (H5Dwrite(dset, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0) + errx(EXIT_FAILURE, "H5Dwrite"); + + if (H5Dflush(dset) < 0) + errx(EXIT_FAILURE, "%s: H5Dflush", __func__); + + return dset; +} + +static void +rewrite_dset(hid_t dset, hid_t type, char *data) +{ + if (H5Dwrite(dset, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0) + errx(EXIT_FAILURE, "%s: H5Dwrite", __func__); + if (H5Dflush(dset) < 0) + errx(EXIT_FAILURE, "%s: H5Dflush", __func__); +} +#endif + +static hid_t +create_vl_dset(hid_t file, hid_t type, hid_t space, const char *name) +{ + hid_t dset; + + dset = H5Dcreate2(file, name, type, space, H5P_DEFAULT, H5P_DEFAULT, + H5P_DEFAULT); + + if (dset == badhid) + errx(EXIT_FAILURE, "H5Dcreate2"); + + return dset; +} + +static void +print_cache_hits(H5C_t *cache) +{ + int i; + + for (i = 0; i < H5AC_NTYPES; i++) { + dbgf(3, "type-%d cache hits %" PRId64 "%s\n", + i, cache->hits[i], (i == H5AC_GHEAP_ID) ? " *" : ""); + } + dbgf(3, "\n"); +} + +static void +usage(const char *progname) +{ + fprintf(stderr, "usage: %s [-W] [-V]\n", progname); + fprintf(stderr, "\n -W: do not wait for SIGINT or SIGUSR1\n"); + fprintf(stderr, "\n -S: do not use VFD SWMR\n"); + fprintf(stderr, " -f: use fixed-length string\n"); + fprintf(stderr, " (default: variable-length string)\n"); + fprintf(stderr, " -n: number of test steps to perform\n"); + fprintf(stderr, " -q: be quiet: few/no progress messages\n"); + fprintf(stderr, " -t (oob|null): select out-of-bounds or NULL test\n"); + exit(EXIT_FAILURE); +} + +bool +H5HG_trap(const char *reason) +{ + if (strcmp(reason, "out of bounds") == 0) { + caught_out_of_bounds = true; + return false; + } + return true; +} + +int +main(int argc, char **argv) +{ + hid_t fapl, fcpl, fid, space, type; + hid_t dset[2]; + char content[2][96]; + char name[2][96]; + H5F_t *f; + H5C_t *cache; + sigset_t oldsigs; + herr_t ret; + bool variable = true, wait_for_signal = true; + const hsize_t dims = 1; + int ch, i, ntimes = 100; + unsigned long tmp; + char *end; + bool use_vfd_swmr = true; + const struct timespec delay = + {.tv_sec = 0, .tv_nsec = 1000 * 1000 * 1000 / 10}; + testsel_t sel = TEST_NONE; + + assert(H5T_C_S1 != badhid); + + while ((ch = getopt(argc, argv, "SWfn:qt:")) != -1) { + switch(ch) { + case 'S': + use_vfd_swmr = false; + break; + case 'W': + wait_for_signal = false; + break; + case 'f': + variable = false; + break; + case 'n': + errno = 0; + tmp = strtoul(optarg, &end, 0); + if (end == optarg || *end != '\0') + errx(EXIT_FAILURE, "couldn't parse `-n` argument `%s`", optarg); + else if (errno != 0) + err(EXIT_FAILURE, "couldn't parse `-n` argument `%s`", optarg); + else if (tmp > INT_MAX) + errx(EXIT_FAILURE, "`-n` argument `%lu` too large", tmp); + ntimes = (int)tmp; + break; + case 'q': + verbosity = 1; + break; + case 't': + if (strcmp(optarg, "oob") == 0) + sel = TEST_OOB; + else if (strcmp(optarg, "null") == 0) + sel = TEST_NULL; + else + usage(argv[0]); + break; + default: + usage(argv[0]); + break; + } + } + argv += optind; + argc -= optind; + + if (argc > 0) + errx(EXIT_FAILURE, "unexpected command-line arguments"); + + fapl = vfd_swmr_create_fapl(true, sel == TEST_OOB, use_vfd_swmr, + "./shadow"); + if (fapl < 0) + errx(EXIT_FAILURE, "vfd_swmr_create_fapl"); + + if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + errx(EXIT_FAILURE, "H5Pcreate"); + + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, false, 1); + if (ret < 0) + errx(EXIT_FAILURE, "H5Pset_file_space_strategy"); + + fid = H5Fcreate("vfd_swmr_vlstr.h5", H5F_ACC_TRUNC, fcpl, fapl); + + /* Create the VL string datatype and a scalar dataspace, or a + * fixed-length string datatype and a simple dataspace. + */ + if ((type = H5Tcopy(H5T_C_S1)) == badhid) + errx(EXIT_FAILURE, "H5Tcopy"); + + /* Create the VL string datatype and a scalar dataspace */ + if ((type = H5Tcopy(H5T_C_S1)) == badhid) + errx(EXIT_FAILURE, "H5Tcopy"); + + if (!variable) { + if (H5Tset_size(type, 32) < 0) + errx(EXIT_FAILURE, "H5Tset_size"); + space = H5Screate_simple(1, &dims, NULL); + } else { + if (H5Tset_size(type, H5T_VARIABLE) < 0) + errx(EXIT_FAILURE, "H5Tset_size"); + space = H5Screate(H5S_SCALAR); + } + + if (space == badhid) + errx(EXIT_FAILURE, "H5Screate"); + + if ((f = H5VL_object_verify(fid, H5I_FILE)) == NULL) + errx(EXIT_FAILURE, "H5VL_object_verify"); + + cache = f->shared->cache; + + if (fid == badhid) + errx(EXIT_FAILURE, "H5Fcreate"); + + block_signals(&oldsigs); + + print_cache_hits(cache); + + /* content 1 seq 1 short + * content 1 seq 1 long long long long long long long long + * content 1 seq 1 medium medium medium + */ + for (i = 0; i < ntimes; i++) { + const int ndsets = 2; + const int step = i % NSTEPS; + const int which = (i / NSTEPS) % ndsets; + const int seq = i / (ndsets * NSTEPS); + dbgf(2, "iteration %d which %d step %d seq %d\n", + i, which, step, seq); + switch (step) { + case CREATE: + (void)snprintf(name[which], sizeof(name[which]), + "dset-%d", which); + (void)snprintf(content[which], sizeof(content[which]), + "content %d seq %d short", which, seq); + dset[which] = + create_vl_dset(fid, type, space, name[which]); + write_vl_dset(dset[which], type, space, content[which]); + break; + case LENGTHEN: + (void)snprintf(content[which], sizeof(content[which]), + "content %d seq %d long long long long long long long long", + which, seq); + write_vl_dset(dset[which], type, space, content[which]); + break; + case SHORTEN: + (void)snprintf(content[which], sizeof(content[which]), + "content %d seq %d medium medium medium", + which, seq); + write_vl_dset(dset[which], type, space, content[which]); + break; + case DELETE: + if (H5Dclose(dset[which]) < 0) + errx(EXIT_FAILURE, "H5Dclose"); + if (H5Ldelete(fid, name[which], H5P_DEFAULT) < 0) { + errx(EXIT_FAILURE, "%s: H5Ldelete(, \"%s\", ) failed", + __func__, name[which]); + } + break; + default: + errx(EXIT_FAILURE, "%s: unknown step %d", __func__, step); + } + if (caught_out_of_bounds) { + fprintf(stderr, "caught out of bounds\n"); + break; + } + nanosleep(&delay, NULL); + } + + if (use_vfd_swmr && wait_for_signal) + await_signal(fid); + + restore_signals(&oldsigs); + + if (H5Pclose(fapl) < 0) + errx(EXIT_FAILURE, "H5Pclose(fapl)"); + + if (H5Pclose(fcpl) < 0) + errx(EXIT_FAILURE, "H5Pclose(fcpl)"); + + if (H5Tclose(type) < 0) + errx(EXIT_FAILURE, "H5Tclose"); + + if (H5Sclose(space) < 0) + errx(EXIT_FAILURE, "H5Sclose"); + + if (H5Fclose(fid) < 0) + errx(EXIT_FAILURE, "H5Fclose"); + + return EXIT_SUCCESS; +} diff --git a/test/vfd_swmr_writer.c b/test/vfd_swmr_writer.c new file mode 100644 index 0000000..1d73dd6 --- /dev/null +++ b/test/vfd_swmr_writer.c @@ -0,0 +1,454 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: vfd_swmr_writer.c + * (copied and modified from swmr_writer.c) + * + * Purpose: Writes data to a randomly selected subset of the datasets + * in the VFD SWMR test file. + * + * This program is intended to run concurrently with the + * vfd_swmr_reader program. + * + *------------------------------------------------------------------------- + */ + +/***********/ +/* Headers */ +/***********/ + +#include <unistd.h> /* getopt(3) */ + +#include "h5test.h" +#include "vfd_swmr_common.h" + +/********************/ +/* Local Prototypes */ +/********************/ + +static hid_t open_skeleton(const char *filename, hbool_t verbose, FILE *verbose_file, + unsigned random_seed, hbool_t old); +static int add_records(hid_t fid, hbool_t verbose, FILE *verbose_file, + unsigned long nrecords, unsigned long flush_count); +static void usage(void); + + +/*------------------------------------------------------------------------- + * Function: open_skeleton + * + * Purpose: Opens the SWMR HDF5 file and datasets. + * + * Parameters: const char *filename + * The filename of the SWMR HDF5 file to open + * + * hbool_t verbose + * Whether or not to emit verbose console messages + * + * FILE *verbose_file + * File handle for verbose output + * + * unsigned random_seed + * Random seed for the file (used for verbose logging) + * + * hbool_t old + * Whether to write in "old" file format + * + * Return: Success: The file ID of the opened SWMR file + * The dataset IDs are stored in a global array + * + * Failure: -1 + * + *------------------------------------------------------------------------- + */ +static hid_t +open_skeleton(const char *filename, hbool_t verbose, FILE *verbose_file, + unsigned random_seed, hbool_t old H5_ATTR_UNUSED) +{ + hid_t fid; /* File ID for new HDF5 file */ + hid_t fapl; /* File access property list */ + unsigned u, v; /* Local index variable */ + hbool_t use_log_vfd = FALSE; /* Use the log VFD (set this manually) */ + H5F_vfd_swmr_config_t *config = NULL; /* Configuration for VFD SWMR */ + + HDassert(filename); + + /* Create file access property list */ + if((fapl = h5_fileaccess()) < 0) + return -1; + + /* FOR NOW: set to use latest format, the "old" parameter is not used */ + if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + return -1; + + if(use_log_vfd) { + char verbose_name[1024]; + + HDsnprintf(verbose_name, sizeof(verbose_name), "vfd_swmr_writer.log.%u", random_seed); + + H5Pset_fapl_log(fapl, verbose_name, H5FD_LOG_ALL, (size_t)(512 * 1024 * 1024)); + } /* end if */ + + /* + * Set up to open the file with VFD SWMR configured. + */ + + /* Enable page buffering */ + if(H5Pset_page_buffer_size(fapl, 4096, 0, 0) < 0) + return -1; + + /* Allocate memory for the configuration structure */ + if((config = (H5F_vfd_swmr_config_t *)HDcalloc(1, sizeof(H5F_vfd_swmr_config_t))) == NULL) + return -1; + + config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION; + config->tick_len = 4; + config->max_lag = 5; + config->writer = TRUE; + config->md_pages_reserved = 128; + HDstrcpy(config->md_file_path, "./my_md_file"); + + /* Enable VFD SWMR configuration */ + if(H5Pset_vfd_swmr_config(fapl, config) < 0) + return -1; + + /* Open the file with VFD SWMR configured */ + if((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) + return -1; + + /* Close file access property list */ + if(H5Pclose(fapl) < 0) + return -1; + + if(config) + HDfree(config); + + /* Emit informational message */ + if(verbose) + HDfprintf(verbose_file, "WRITER: Opening datasets\n"); + + /* Open the datasets */ + for(u = 0; u < NLEVELS; u++) + for(v = 0; v < symbol_count[u]; v++) { + if((symbol_info[u][v].dsid = H5Dopen2(fid, symbol_info[u][v].name, H5P_DEFAULT)) < 0) + return -1; + symbol_info[u][v].nrecords = 0; + } /* end for */ + + return fid; +} /* open_skeleton() */ + + +/*------------------------------------------------------------------------- + * Function: add_records + * + * Purpose: Writes a specified number of records to random datasets in + * the SWMR test file. + * + * Parameters: hid_t fid + * The file ID of the SWMR HDF5 file + * + * hbool_t verbose + * Whether or not to emit verbose console messages + * + * FILE *verbose_file + * File handle for verbose output + * + * unsigned long nrecords + * # of records to write to the datasets + * + * unsigned long flush_count + * # of records to write before flushing the file to disk + * + * Return: Success: 0 + * Failure: -1 + * + *------------------------------------------------------------------------- + */ +static int +add_records(hid_t fid, hbool_t verbose, FILE *verbose_file, + unsigned long nrecords, unsigned long flush_count) +{ + hid_t tid; /* Datatype ID for records */ + hid_t mem_sid; /* Memory dataspace ID */ + hsize_t start[2] = {0, 0}, count[2] = {1, 1}; /* Hyperslab selection values */ + hsize_t dim[2] = {1, 0}; /* Dataspace dimensions */ + symbol_t record; /* The record to add to the dataset */ + unsigned long rec_to_flush; /* # of records left to write before flush */ + unsigned long u, v; /* Local index variables */ + + HDassert(fid >= 0); + + /* Reset the record */ + /* (record's 'info' field might need to change for each record written, also) */ + HDmemset(&record, 0, sizeof(record)); + + /* Create a dataspace for the record to add */ + if((mem_sid = H5Screate(H5S_SCALAR)) < 0) + return -1; + + /* Create datatype for appending records */ + if((tid = create_symbol_datatype()) < 0) + return -1; + + /* Add records to random datasets, according to frequency distribution */ + rec_to_flush = flush_count; + for(u = 0; u < nrecords; u++) { + symbol_info_t *symbol; /* Symbol to write record to */ + hid_t file_sid; /* Dataset's space ID */ + + /* Get a random dataset, according to the symbol distribution */ + symbol = choose_dataset(NULL, NULL); + + /* Set the record's ID (equal to its position) */ + record.rec_id = symbol->nrecords; + + /* Get the coordinate to write */ + start[1] = symbol->nrecords; + + /* Extend the dataset's dataspace to hold the new record */ + symbol->nrecords++; + dim[1] = symbol->nrecords; + if(H5Dset_extent(symbol->dsid, dim) < 0) + return -1; + + /* Get the dataset's dataspace */ + if((file_sid = H5Dget_space(symbol->dsid)) < 0) + return -1; + + /* Choose the last record in the dataset */ + if(H5Sselect_hyperslab(file_sid, H5S_SELECT_SET, start, NULL, count, NULL) < 0) + return -1; + + /* Write record to the dataset */ + if(H5Dwrite(symbol->dsid, tid, mem_sid, file_sid, H5P_DEFAULT, &record) < 0) + return -1; + + /* Close the dataset's dataspace */ + if(H5Sclose(file_sid) < 0) + return -1; + + /* Check for flushing file */ + if(flush_count > 0) { + /* Decrement count of records to write before flushing */ + rec_to_flush--; + + /* Check for counter being reached */ + if(0 == rec_to_flush) { +#ifdef TEMP_OUT + /* Flush contents of file */ + if(H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0) + return -1; +#endif /* TEMP_OUT */ + + /* Reset flush counter */ + rec_to_flush = flush_count; + } /* end if */ + } /* end if */ + } /* end for */ + + /* Close the memory dataspace */ + if(H5Sclose(mem_sid) < 0) + return -1; + + /* Close the datatype */ + if(H5Tclose(tid) < 0) + return -1; + + /* Emit informational message */ + if(verbose) + HDfprintf(verbose_file, "WRITER: Closing datasets\n"); + + /* Close the datasets */ + for(u = 0; u < NLEVELS; u++) + for(v = 0; v < symbol_count[u]; v++) + if(H5Dclose(symbol_info[u][v].dsid) < 0) + return -1; + + return 0; +} + +static void +usage(void) +{ + printf("\n"); + printf("Usage error!\n"); + printf("\n"); + printf("Usage: vfd_swmr_writer [-q] [-o] [-f <# of records to write between flushing\n"); + printf(" file contents>] [-r <random seed>] <# of records>\n"); + printf("\n"); + printf("<# of records to write between flushing file contents> should be 0\n"); + printf("(for no flushing) or between 1 and (<# of records> - 1).\n"); + printf("\n"); + printf("<# of records> must be specified.\n"); + printf("\n"); + printf("Defaults to verbose (no '-q' given), latest format when opening file (no '-o' given),\n"); + printf("flushing every 10000 records ('-f 10000'), and will generate a random seed (no -r given).\n"); + printf("\n"); + HDexit(1); +} + +int +main(int argc, char * const *argv) +{ + sigset_t oldset; + hid_t fid; /* File ID for file opened */ + long nrecords = 0; /* # of records to append */ + long flush_count = 10000; /* # of records to write between flushing file */ + hbool_t verbose = TRUE; /* Whether to emit some informational messages */ + FILE *verbose_file = NULL; /* File handle for verbose output */ + hbool_t old = FALSE; /* Whether to use non-latest-format when opening file */ + hbool_t use_seed = FALSE; /* Set to TRUE if a seed was set on the command line */ + hbool_t wait_for_signal = TRUE; + unsigned random_seed = 0; /* Random # seed */ + int ch, temp; + + block_signals(&oldset); + + while ((ch = getopt(argc, argv, "Wf:qr:o")) != -1) { + switch(ch) { + /* # of records to write between flushing file */ + case 'f': + flush_count = HDatol(optarg); + if(flush_count < 0) + usage(); + break; + + /* Be quiet */ + case 'q': + verbose = FALSE; + break; + + /* Random # seed */ + case 'r': + use_seed = TRUE; + temp = HDatoi(optarg); + random_seed = (unsigned)temp; + break; + + case 'W': + wait_for_signal = FALSE; + break; + + /* Use non-latest-format when opening file */ + case 'o': + old = TRUE; + break; + + default: + usage(); + break; + } + } + argv += optind; + argc -= optind; + /* Parse command line options */ + if(argc < 1) + usage(); + /* Get the number of records to append */ + nrecords = HDatol(argv[0]); + if(nrecords <= 0 || flush_count >= nrecords) + usage(); + + /* Set the random seed */ + if(!use_seed) { + struct timeval t; + + HDgettimeofday(&t, NULL); + random_seed = (unsigned)(t.tv_usec); + } /* end if */ + HDsrandom(random_seed); + + /* Open output file */ + if(verbose) { + char verbose_name[1024]; + + HDsnprintf(verbose_name, sizeof(verbose_name), "vfd_swmr_writer.out.%u", random_seed); + if(NULL == (verbose_file = HDfopen(verbose_name, "w"))) { + HDfprintf(stderr, "WRITER: Can't open verbose output file!\n"); + HDexit(1); + } + } /* end if */ + + /* Emit informational message */ + if(verbose) { + HDfprintf(verbose_file, "WRITER: Parameters:\n"); + HDfprintf(verbose_file, "\t# of records between flushes = %ld\n", flush_count); + HDfprintf(verbose_file, "\t# of records to write = %ld\n", nrecords); + } /* end if */ + + /* ALWAYS emit the random seed for possible debugging */ + HDfprintf(stdout, "Using writer random seed: %u\n", random_seed); + + /* Emit informational message */ + if(verbose) + HDfprintf(verbose_file, "WRITER: Generating symbol names\n"); + + /* Generate dataset names */ + if(generate_symbols() < 0) + return -1; + + /* Emit informational message */ + if(verbose) { + HDfprintf(verbose_file, "WRITER: Opening skeleton file: %s\n", + COMMON_FILENAME); + } + + /* Open file skeleton */ + if((fid = open_skeleton(COMMON_FILENAME, verbose, verbose_file, random_seed, old)) < 0) { + HDfprintf(stderr, "WRITER: Error opening skeleton file!\n"); + HDexit(1); + } /* end if */ + + /* Send a message to indicate "H5Fopen" is complete--releasing the file lock */ + h5_send_message(WRITER_MESSAGE, NULL, NULL); + + /* Emit informational message */ + if(verbose) + HDfprintf(verbose_file, "WRITER: Adding records\n"); + + /* Append records to datasets */ + if(add_records(fid, verbose, verbose_file, (unsigned long)nrecords, (unsigned long)flush_count) < 0) { + HDfprintf(stderr, "WRITER: Error appending records to datasets!\n"); + HDexit(1); + } /* end if */ + + /* Emit informational message */ + if(verbose) + HDfprintf(verbose_file, "WRITER: Releasing symbols\n"); + + /* Clean up the symbols */ + if(shutdown_symbols() < 0) { + HDfprintf(stderr, "WRITER: Error releasing symbols!\n"); + HDexit(1); + } /* end if */ + + if (wait_for_signal) + await_signal(fid); + + restore_signals(&oldset); + + /* Emit informational message */ + if(verbose) + HDfprintf(verbose_file, "WRITER: Closing objects/file\n"); + + /* Close objects opened */ + if(H5Fclose(fid) < 0) { + HDfprintf(stderr, "WRITER: Error closing file!\n"); + HDexit(1); + } /* end if */ + + return 0; +} + diff --git a/test/vfd_swmr_zoo_writer.c b/test/vfd_swmr_zoo_writer.c new file mode 100644 index 0000000..e2892ec --- /dev/null +++ b/test/vfd_swmr_zoo_writer.c @@ -0,0 +1,412 @@ +/* + * Copyright by The HDF Group. + * Copyright by the Board of Trustees of the University of Illinois. + * All rights reserved. + * + * This file is part of HDF5. The full HDF5 copyright notice, including + * terms governing use, modification, and redistribution, is contained in + * the COPYING file, which can be found at the root of the source code + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. + * If you do not have access to either file, you may request a copy from + * help@hdfgroup.org. + */ + +#include <err.h> +#include <libgen.h> /* basename(3) */ +#include <time.h> /* nanosleep(2) */ +#include <unistd.h> /* getopt(3) */ + +#define H5C_FRIEND /* suppress error about including H5Cpkg */ +#define H5F_FRIEND /* suppress error about including H5Fpkg */ + +#include "hdf5.h" + +#include "H5private.h" +#include "H5retry_private.h" +#include "H5Cpkg.h" +#include "H5Fpkg.h" +// #include "H5Iprivate.h" +#include "H5HGprivate.h" +#include "H5VLprivate.h" + +#include "testhdf5.h" +#include "genall5.h" +#include "vfd_swmr_common.h" + +#ifndef _arraycount +#define _arraycount(_a) (sizeof(_a)/sizeof(_a[0])) +#endif + +typedef struct _shared_ticks { + uint64_t reader_tick; +} shared_ticks_t; + +typedef struct _tick_stats { + uint64_t writer_tried_increase; + uint64_t writer_aborted_increase; + uint64_t writer_read_shared_file; + uint64_t reader_tick_was_zero; // writer read reader tick equal to 0 + uint64_t reader_tick_lead_writer; // writer read reader tick greater than + // proposed writer tick + uint64_t writer_lead_reader_by[1]; // proposed writer tick lead reader + // tick by `lead` ticks + // `writer_lead_reader_by[lead]` + // times, for `0 <= lead <= max_lag - 1` +} tick_stats_t; + +static H5F_vfd_swmr_config_t swmr_config; +static tick_stats_t *tick_stats = NULL; +static const hid_t badhid = H5I_INVALID_HID; +static bool writer; + +static void +print_cache_hits(H5C_t *cache) +{ + int i; + + for (i = 0; i < H5AC_NTYPES; i++) { + dbgf(3, "type-%d cache hits %" PRId64 "%s\n", + i, cache->hits[i], (i == H5AC_GHEAP_ID) ? " *" : ""); + } + dbgf(3, "\n"); +} + +void +zoo_create_hook(hid_t fid) +{ + dbgf(3, "%s: enter\n", __func__); + if (writer) + H5Fvfd_swmr_end_tick(fid); +} + +static void +usage(const char *progname) +{ + fprintf(stderr, "usage: %s [-C] [-S] [-W] [-a] [-e] [-m] [-q] [-v]\n", + progname); + fprintf(stderr, "\n -C: skip compact dataset tests\n"); + fprintf(stderr, " -S: do not use VFD SWMR\n"); + fprintf(stderr, " -W: do not wait for SIGINT or SIGUSR1\n"); + fprintf(stderr, " -a: run all tests, including variable-length data\n"); + fprintf(stderr, " -e: print error stacks\n"); + fprintf(stderr, " -m ms: maximum `ms` milliseconds pause between\n"); + fprintf(stderr, " each create/delete step\n"); + fprintf(stderr, " -q: be quiet: few/no progress messages\n"); + fprintf(stderr, " -v: be verbose: most progress messages\n"); + exit(EXIT_FAILURE); +} + +bool +vfd_swmr_writer_may_increase_tick_to(uint64_t new_tick, bool wait_for_reader) +{ + static int fd = -1; + shared_ticks_t shared; + ssize_t nread; + h5_retry_t retry; + bool do_try; + + dbgf(3, "%s: enter\n", __func__); + + if (fd == -1) { + fd = open("./shared_tick_num", O_RDONLY); + if (fd == -1) { + warn("%s: open", __func__); // TBD ratelimit/silence this warning + return true; + } + assert(tick_stats == NULL); + tick_stats = calloc(1, sizeof(*tick_stats) + + (swmr_config.max_lag - 1) * + sizeof(tick_stats->writer_lead_reader_by[0])); + if (tick_stats == NULL) + err(EXIT_FAILURE, "%s: calloc", __func__); + } + + tick_stats->writer_tried_increase++; + + for (do_try = h5_retry_init(&retry, 14, 10 * 1000 * 1000, + 100 * 1000 * 1000); + do_try; + do_try = wait_for_reader && h5_retry_next(&retry)) { + + tick_stats->writer_read_shared_file++; + + if ((nread = pread(fd, &shared, sizeof(shared), 0)) == -1) + err(EXIT_FAILURE, "%s: pread", __func__); + + if (nread != sizeof(shared)) + errx(EXIT_FAILURE, "%s: pread", __func__); + + // TBD convert endianness + + if (shared.reader_tick == 0) { + tick_stats->reader_tick_was_zero++; + return true; + } + + if (new_tick < shared.reader_tick) { + tick_stats->reader_tick_lead_writer++; + return true; + } + if (new_tick <= shared.reader_tick + swmr_config.max_lag - 1) { + uint64_t lead = new_tick - shared.reader_tick; + assert(lead <= swmr_config.max_lag - 1); + tick_stats->writer_lead_reader_by[lead]++; + return true; + } + } + if (wait_for_reader && !do_try) + errx(EXIT_FAILURE, "%s: timed out waiting for reader", __func__); + + tick_stats->writer_aborted_increase++; + + return false; +} + +void +vfd_swmr_reader_did_increase_tick_to(uint64_t new_tick) +{ + static int fd = -1; + shared_ticks_t shared; + ssize_t nwritten; + + dbgf(3, "%s: enter\n", __func__); + + if (fd == -1) { + // TBD create a temporary file, here, and move it to its final path + // after writing it. + fd = open("./shared_tick_num", O_RDWR|O_CREAT, 0600); + if (fd == -1) + err(EXIT_FAILURE, "%s: open", __func__); + } + + shared.reader_tick = new_tick; + + // TBD convert endianness + + if ((nwritten = pwrite(fd, &shared, sizeof(shared), 0)) == -1) + errx(EXIT_FAILURE, "%s: pwrite", __func__); + + if (nwritten != sizeof(shared)) + errx(EXIT_FAILURE, "%s: pwrite", __func__); + + if (new_tick == 0) { + if (unlink("./shared_tick_num") == -1) + warn("%s: unlink", __func__); + if (close(fd) == -1) + err(EXIT_FAILURE, "%s: close", __func__); + fd = -1; + } +} + +int +main(int argc, char **argv) +{ + hid_t fapl, fcpl, fid; + H5F_t *f; + H5C_t *cache; + sigset_t oldsigs; + herr_t ret; + zoo_config_t config = { + .proc_num = 0 + , .skip_compact = false + , .skip_varlen = true + , .max_pause_msecs = 0 + }; + bool wait_for_signal; + int ch; + char vector[8]; + unsigned seed; + unsigned long tmpl; + char *end, *ostate; + const char *seedvar = "H5_ZOO_STEP_SEED"; + bool use_vfd_swmr = true; + bool print_estack = false; + const char *progname = basename(argv[0]); + const char *personality = strstr(progname, "vfd_swmr_zoo_"); + estack_state_t es; + char step = 'b'; + + if (personality != NULL && strcmp(personality, "vfd_swmr_zoo_writer") == 0) + writer = wait_for_signal = true; + else if (personality != NULL && + strcmp(personality, "vfd_swmr_zoo_reader") == 0) + writer = false; + else { + errx(EXIT_FAILURE, + "unknown personality, expected vfd_swmr_zoo_{reader,writer}"); + } + + if (writer) + config.max_pause_msecs = 50; + + while ((ch = getopt(argc, argv, "CSWaem:qv")) != -1) { + switch(ch) { + case 'C': + config.skip_compact = true; + break; + case 'S': + use_vfd_swmr = false; + break; + case 'W': + wait_for_signal = false; + break; + case 'a': + config.skip_varlen = false; + break; + case 'e': + print_estack = true; + break; + case 'm': + errno = 0; + tmpl = strtoul(optarg, &end, 0); + if (end == optarg || *end != '\0') + errx(EXIT_FAILURE, "couldn't parse `-m` argument `%s`", optarg); + else if (errno != 0) + err(EXIT_FAILURE, "couldn't parse `-m` argument `%s`", optarg); + else if (tmpl > UINT_MAX) + errx(EXIT_FAILURE, "`-m` argument `%lu` too large", tmpl); + config.max_pause_msecs = (unsigned)tmpl; + break; + case 'q': + verbosity = 1; + break; + case 'v': + verbosity = 3; + break; + default: + usage(argv[0]); + break; + } + } + argv += optind; + argc -= optind; + + if (argc > 0) + errx(EXIT_FAILURE, "unexpected command-line arguments"); + + fapl = vfd_swmr_create_fapl(writer, true, use_vfd_swmr, "./shadow"); + + if (use_vfd_swmr && H5Pget_vfd_swmr_config(fapl, &swmr_config) < 0) + errx(EXIT_FAILURE, "H5Pget_vfd_swmr_config"); + + if (fapl < 0) + errx(EXIT_FAILURE, "vfd_swmr_create_fapl"); + + if (H5Pset_libver_bounds(fapl, H5F_LIBVER_EARLIEST, H5F_LIBVER_LATEST) < 0){ + errx(EXIT_FAILURE, "%s.%d: H5Pset_libver_bounds", __func__, __LINE__); + } + + if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + errx(EXIT_FAILURE, "H5Pcreate"); + + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, false, 1); + if (ret < 0) + errx(EXIT_FAILURE, "H5Pset_file_space_strategy"); + + if (writer) + fid = H5Fcreate("vfd_swmr_zoo.h5", H5F_ACC_TRUNC, fcpl, fapl); + else + fid = H5Fopen("vfd_swmr_zoo.h5", H5F_ACC_RDONLY, fapl); + + if (fid == badhid) + errx(EXIT_FAILURE, writer ? "H5Fcreate" : "H5Fopen"); + + if ((f = H5VL_object_verify(fid, H5I_FILE)) == NULL) + errx(EXIT_FAILURE, "H5VL_object_verify"); + + cache = f->shared->cache; + + if (wait_for_signal) + block_signals(&oldsigs); + + print_cache_hits(cache); + + es = print_estack ? estack_get_state() : disable_estack(); + if (writer) { + + dbgf(2, "Writing zoo...\n"); + + /* get seed from environment or else from time(3) */ + switch (fetch_env_ulong(seedvar, UINT_MAX, &tmpl)) { + case -1: + errx(EXIT_FAILURE, "%s: fetch_env_ulong", __func__); + case 0: + seed = (unsigned int)time(NULL); + break; + default: + seed = (unsigned int)tmpl; + break; + } + + dbgf(1, "To reproduce, set seed (%s) to %u.\n", seedvar, seed); + + ostate = initstate(seed, vector, _arraycount(vector)); + + if (!create_zoo(fid, ".", config)) + errx(EXIT_FAILURE, "create_zoo didn't pass self-check"); + + /* Avoid deadlock: flush the file before waiting for the reader's + * message. + */ + if (H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0) + errx(EXIT_FAILURE, "%s: H5Fflush failed", __func__); + + if (read(STDIN_FILENO, &step, sizeof(step)) == -1) + err(EXIT_FAILURE, "read"); + + if (step != 'b') + errx(EXIT_FAILURE, "expected 'b' read '%c'", step); + + if (!delete_zoo(fid, ".", config)) + errx(EXIT_FAILURE, "delete_zoo failed"); + (void)setstate(ostate); + } else { + dbgf(2, "Reading zoo...\n"); + + while (!validate_zoo(fid, ".", config)) + ; + + if (write(STDOUT_FILENO, &step, sizeof(step)) == -1) + err(EXIT_FAILURE, "write"); + while (!validate_deleted_zoo(fid, ".", config)) + ; + } + restore_estack(es); + + if (use_vfd_swmr && wait_for_signal) + await_signal(fid); + + if (writer && tick_stats != NULL) { + uint64_t lead; + + dbgf(2, "writer tried tick increase %" PRIu64 "\n", + tick_stats->writer_tried_increase); + dbgf(2, "writer aborted tick increase %" PRIu64 "\n", + tick_stats->writer_aborted_increase); + dbgf(2, "writer read shared file %" PRIu64 "\n", + tick_stats->writer_read_shared_file); + dbgf(2, "writer read reader tick equal to 0 %" PRIu64 "\n", + tick_stats->reader_tick_was_zero); + dbgf(2, "writer read reader tick leading writer %" PRIu64 "\n", + tick_stats->reader_tick_lead_writer); + + for (lead = 0; lead < swmr_config.max_lag; lead++) { + dbgf(2, "writer tick lead writer by %" PRIu64 " %" PRIu64 "\n", + lead, tick_stats->writer_lead_reader_by[lead]); + } + } + + if (H5Pclose(fapl) < 0) + errx(EXIT_FAILURE, "H5Pclose(fapl)"); + + if (H5Pclose(fcpl) < 0) + errx(EXIT_FAILURE, "H5Pclose(fcpl)"); + + if (H5Fclose(fid) < 0) + errx(EXIT_FAILURE, "H5Fclose"); + + if (wait_for_signal) + restore_signals(&oldsigs); + + return EXIT_SUCCESS; +} |