diff options
| -rw-r--r-- | MANIFEST | 1 | ||||
| -rw-r--r-- | src/H5Faccum.c | 11 | ||||
| -rw-r--r-- | src/H5Ocache.c | 196 | ||||
| -rw-r--r-- | test/Makefile.am | 4 | ||||
| -rw-r--r-- | test/Makefile.in | 61 | ||||
| -rw-r--r-- | test/accum.c | 168 | ||||
| -rw-r--r-- | test/accum_swmr_reader.c | 80 | ||||
| -rw-r--r-- | test/ohdr.c | 132 |
8 files changed, 556 insertions, 97 deletions
@@ -953,6 +953,7 @@ ./test/Makefile.in ./test/SWMR_UseCase_UG.txt ./test/accum.c +./test/accum_swmr_reader.c ./test/app_ref.c ./test/atomic_reader.c ./test/atomic_writer.c diff --git a/src/H5Faccum.c b/src/H5Faccum.c index 0e549fa..529384b 100644 --- a/src/H5Faccum.c +++ b/src/H5Faccum.c @@ -413,6 +413,11 @@ done: * koziol@hdfgroup.org * Jan 10 2008 * + * Modifications: + * Vailin Choi; June 2013 + * This is a fix for SWMR: + * For a large write that is >= than H5F_ACCUM_MAX_SIZE, + * flush the metadata in the accumulator first before the write. *------------------------------------------------------------------------- */ herr_t @@ -726,6 +731,12 @@ HDmemset(f->shared->accum.buf + size, 0, (f->shared->accum.alloc_size - size)); } /* end else */ } /* end if */ else { + if((f->intent & H5F_ACC_SWMR_WRITE) > 0) { + /* Flush if dirty and reset accumulator */ + if(H5F_accum_reset(f, dxpl_id, TRUE) < 0) + HGOTO_ERROR(H5E_IO, H5E_CANTRESET, FAIL, "can't reset accumulator") + } + /* Write the data */ if(H5FD_write(f->shared->lf, dxpl_id, map_type, addr, size, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") diff --git a/src/H5Ocache.c b/src/H5Ocache.c index 856a5bc..80f5c05 100644 --- a/src/H5Ocache.c +++ b/src/H5Ocache.c @@ -93,6 +93,8 @@ static herr_t H5O_chunk_serialize(const H5F_t *f, H5O_t *oh, unsigned chunkno); static herr_t H5O_add_cont_msg(H5O_cont_msgs_t *cont_msg_info, const H5O_cont_t *cont); +static herr_t H5O_decode_prefix(H5F_t *f, H5O_t *oh, const uint8_t *buf, void *_udata); + /*********************/ /* Package Variables */ @@ -139,76 +141,34 @@ H5FL_SEQ_DEFINE(H5O_cont_t); /* Local Variables */ /*******************/ - /*------------------------------------------------------------------------- - * Function: H5O_load + * Function: H5O_decode_prefix * - * Purpose: Loads an object header from disk. + * Purpose: To decode the object header prefix. + * The coding is extracted fromt H5O_load() to this routine. * - * Return: Success: Pointer to the new object header. + * Return: Non-negative on success/Negative on failure * - * Failure: NULL - * - * Programmer: Robb Matzke - * matzke@llnl.gov - * Aug 5 1997 + * Programmer: Vailin Choi; June 2013 * *------------------------------------------------------------------------- */ -static H5O_t * -H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) +static herr_t +H5O_decode_prefix(H5F_t *f, H5O_t *oh, const uint8_t *buf, void *_udata) { - H5O_t *oh = NULL; /* Object header read in */ H5O_cache_ud_t *udata = (H5O_cache_ud_t *)_udata; /* User data for callback */ - H5WB_t *wb = NULL; /* Wrapped buffer for prefix data */ - uint8_t read_buf[H5O_SPEC_READ_SIZE]; /* Buffer for speculative read */ - const uint8_t *p; /* Pointer into buffer to decode */ - uint8_t *buf; /* Buffer to decode */ - size_t spec_read_size; /* Size of buffer to speculatively read in */ - size_t prefix_size; /* Size of object header prefix */ - size_t buf_size; /* Size of prefix+chunk #0 buffer */ - haddr_t eoa; /* Relative end of file address */ - H5O_t *ret_value; /* Return value */ + const uint8_t *p = buf; /* Pointer into buffer to decode */ + size_t prefix_size; /* Size of object header prefix */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT /* Check arguments */ HDassert(f); - HDassert(H5F_addr_defined(addr)); + HDassert(oh); + HDassert(buf); HDassert(udata); - HDassert(udata->common.f); - HDassert(udata->common.cont_msg_info); - - /* Make certain we don't speculatively read off the end of the file */ - if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, H5FD_MEM_OHDR))) - HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, NULL, "unable to determine file size") - - /* Compute the size of the speculative object header buffer */ - H5_ASSIGN_OVERFLOW(spec_read_size, MIN(eoa - addr, H5O_SPEC_READ_SIZE), /* From: */ hsize_t, /* To: */ size_t); - - /* Attempt to speculatively read both object header prefix and first chunk */ - if(H5F_block_read(f, H5FD_MEM_OHDR, addr, spec_read_size, dxpl_id, read_buf) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_READERROR, NULL, "unable to read object header") - p = read_buf; - - /* Allocate space for the object header data structure */ - if(NULL == (oh = H5FL_CALLOC(H5O_t))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - - /* File-specific, non-stored information */ - oh->sizeof_size = H5F_SIZEOF_SIZE(udata->common.f); - oh->sizeof_addr = H5F_SIZEOF_ADDR(udata->common.f); - oh->swmr_write = !!(H5F_INTENT(udata->common.f) & H5F_ACC_SWMR_WRITE); - - /* Create object header proxy if doing SWMR writes */ - HDassert(!oh->proxy_present); - if(H5F_INTENT(f) & H5F_ACC_SWMR_WRITE) { - if(H5O_proxy_create(f, dxpl_id, oh) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTCREATE, NULL, "can't create object header proxy") - } /* end if */ - else - oh->proxy_addr = HADDR_UNDEF; /* Check for presence of magic number */ /* (indicates version 2 or later) */ @@ -219,12 +179,12 @@ H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) /* Version */ oh->version = *p++; if(H5O_VERSION_2 != oh->version) - HGOTO_ERROR(H5E_OHDR, H5E_VERSION, NULL, "bad object header version number") + HGOTO_ERROR(H5E_OHDR, H5E_VERSION, FAIL, "bad object header version number") /* Flags */ oh->flags = *p++; if(oh->flags & ~H5O_HDR_ALL_FLAGS) - HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "unknown object header status flag(s)") + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "unknown object header status flag(s)") /* Number of links to object (unless overridden by refcount message) */ oh->nlink = 1; @@ -250,7 +210,7 @@ H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) UINT16DECODE(p, oh->max_compact); UINT16DECODE(p, oh->min_dense); if(oh->max_compact < oh->min_dense) - HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "bad object header attribute phase change values") + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad object header attribute phase change values") } /* end if */ else { oh->max_compact = H5O_CRT_ATTR_MAX_COMPACT_DEF; @@ -276,16 +236,16 @@ H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) break; default: - HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "bad size for chunk 0") + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad size for chunk 0") } /* end switch */ if(oh->chunk0_size > 0 && oh->chunk0_size < H5O_SIZEOF_MSGHDR_OH(oh)) - HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "bad object header chunk size") + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad object header chunk size") } /* end if */ else { /* Version */ oh->version = *p++; if(H5O_VERSION_1 != oh->version) - HGOTO_ERROR(H5E_OHDR, H5E_VERSION, NULL, "bad object header version number") + HGOTO_ERROR(H5E_OHDR, H5E_VERSION, FAIL, "bad object header version number") /* Flags */ oh->flags = H5O_CRT_OHDR_FLAGS_DEF; @@ -310,16 +270,100 @@ H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) UINT32DECODE(p, oh->chunk0_size); if((udata->v1_pfx_nmesgs > 0 && oh->chunk0_size < H5O_SIZEOF_MSGHDR_OH(oh)) || (udata->v1_pfx_nmesgs == 0 && oh->chunk0_size > 0)) - HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "bad object header chunk size") + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad object header chunk size") /* Reserved, in version 1 (for 8-byte alignment padding) */ p += 4; } /* end else */ /* Determine object header prefix length */ - prefix_size = (size_t)(p - (const uint8_t *)read_buf); + prefix_size = (size_t)(p - buf); HDassert((size_t)prefix_size == (size_t)(H5O_SIZEOF_HDR(oh) - H5O_SIZEOF_CHKSUM_OH(oh))); +done: + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5O_decode_prefix() */ + + +/*------------------------------------------------------------------------- + * Function: H5O_load + * + * Purpose: Loads an object header from disk. + * + * Return: Success: Pointer to the new object header. + * + * Failure: NULL + * + * Programmer: Robb Matzke + * matzke@llnl.gov + * Aug 5 1997 + * + * Modifications: + * Vailin Choi; June 2013 + * This is a fix for SWMR: + * When reading a block that is > spec_read_size, read the whole block in + * again and possibly decode the header. + * Move the coding for decoding the object header prefix to another routine + * H5O_decode_prefix(). + *------------------------------------------------------------------------- + */ +static H5O_t * +H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) +{ + H5O_t *oh = NULL; /* Object header read in */ + H5O_cache_ud_t *udata = (H5O_cache_ud_t *)_udata; /* User data for callback */ + H5WB_t *wb = NULL; /* Wrapped buffer for prefix data */ + uint8_t read_buf[H5O_SPEC_READ_SIZE]; /* Buffer for speculative read */ + const uint8_t *p; /* Pointer into buffer to decode */ + uint8_t *buf; /* Buffer to decode */ + size_t spec_read_size; /* Size of buffer to speculatively read in */ + size_t buf_size; /* Size of prefix+chunk #0 buffer */ + haddr_t eoa; /* Relative end of file address */ + H5O_t *ret_value; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Check arguments */ + HDassert(f); + HDassert(H5F_addr_defined(addr)); + HDassert(udata); + HDassert(udata->common.f); + HDassert(udata->common.cont_msg_info); + + /* Make certain we don't speculatively read off the end of the file */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, H5FD_MEM_OHDR))) + HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, NULL, "unable to determine file size") + + /* Compute the size of the speculative object header buffer */ + H5_ASSIGN_OVERFLOW(spec_read_size, MIN(eoa - addr, H5O_SPEC_READ_SIZE), /* From: */ hsize_t, /* To: */ size_t); + + /* Attempt to speculatively read both object header prefix and first chunk */ + if(H5F_block_read(f, H5FD_MEM_OHDR, addr, spec_read_size, dxpl_id, read_buf) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_READERROR, NULL, "unable to read object header") + p = read_buf; + + /* Allocate space for the object header data structure */ + if(NULL == (oh = H5FL_CALLOC(H5O_t))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") + + /* File-specific, non-stored information */ + oh->sizeof_size = H5F_SIZEOF_SIZE(udata->common.f); + oh->sizeof_addr = H5F_SIZEOF_ADDR(udata->common.f); + oh->swmr_write = !!(H5F_INTENT(udata->common.f) & H5F_ACC_SWMR_WRITE); + + /* Create object header proxy if doing SWMR writes */ + HDassert(!oh->proxy_present); + if(H5F_INTENT(f) & H5F_ACC_SWMR_WRITE) { + if(H5O_proxy_create(f, dxpl_id, oh) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTCREATE, NULL, "can't create object header proxy") + } /* end if */ + else + oh->proxy_addr = HADDR_UNDEF; + + if(H5O_decode_prefix(f, oh, read_buf, udata) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "can't decode object header prefix") + /* Compute the size of the buffer used */ buf_size = oh->chunk0_size + (size_t)H5O_SIZEOF_HDR(oh); @@ -333,14 +377,26 @@ H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) if(NULL == (buf = (uint8_t *)H5WB_actual(wb, buf_size))) HGOTO_ERROR(H5E_OHDR, H5E_NOSPACE, NULL, "can't get actual buffer") - /* Copy existing raw data into new buffer */ - HDmemcpy(buf, read_buf, spec_read_size); - - /* Read rest of the raw data */ - if(H5F_block_read(f, H5FD_MEM_OHDR, (addr + spec_read_size), (buf_size - spec_read_size), dxpl_id, (buf + spec_read_size)) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_READERROR, NULL, "unable to read object header data") - } /* end if */ - else + /* SWMR access */ + if(H5F_INTENT(f) & H5F_ACC_SWMR_READ || H5F_INTENT(f) & H5F_ACC_SWMR_WRITE) { + /* Read the chunk that is bigger than the speculative read size */ + if(H5F_block_read(f, H5FD_MEM_OHDR, addr, buf_size, dxpl_id, buf) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_READERROR, NULL, "unable to read object header") + /* See if the data read now is the same as what is read initially */ + if(HDmemcmp(buf, read_buf, spec_read_size)) { + HDmemset(oh, 0, sizeof(oh)); + if(H5O_decode_prefix(f, oh, buf, udata) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "can't decode object header prefix") + } + } else { + /* Copy existing raw data into new buffer */ + HDmemcpy(buf, read_buf, spec_read_size); + + /* Read rest of the raw data */ + if(H5F_block_read(f, H5FD_MEM_OHDR, (addr + spec_read_size), (buf_size - spec_read_size), dxpl_id, (buf + spec_read_size)) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_READERROR, NULL, "unable to read object header data") + } + } else buf = read_buf; /* Parse the first chunk */ diff --git a/test/Makefile.am b/test/Makefile.am index 50060eb..0c77646 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -51,7 +51,7 @@ TEST_PROG= testhdf5 lheap ohdr stab gheap cache cache_api cache_tagging \ bin_PROGRAMS=swmr_generator swmr_reader swmr_writer swmr_remove_reader \ swmr_remove_writer swmr_addrem_writer swmr_sparse_reader swmr_sparse_writer \ - use_append_chunk use_append_mchunks atomic_writer atomic_reader + use_append_chunk use_append_mchunks atomic_writer atomic_reader accum_swmr_reader # List programs to be built when testing here. error_test and err_compat are # built at the same time as the other tests, but executed by testerror.sh. @@ -162,7 +162,7 @@ CHECK_CLEANFILES+=accum.h5 cmpd_dset.h5 compact_dataset.h5 dataset.h5 \ swmr_data.h5 use_append_chunk.h5 use_append_mchunks.h5 \ flushrefresh.h5 flushrefresh_VERIFICATION_START \ flushrefresh_VERIFICATION_CHECKPOINT1 flushrefresh_VERIFICATION_CHECKPOINT2 \ - flushrefresh_VERIFICATION_DONE atomic_data + flushrefresh_VERIFICATION_DONE atomic_data accum_swmr_big.h5 ohdr_swmr.h5 # Sources for testhdf5 executable testhdf5_SOURCES=testhdf5.c tarray.c tattr.c tchecksum.c tconfig.c tfile.c \ diff --git a/test/Makefile.in b/test/Makefile.in index bbe07d5..faed943 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -84,7 +84,8 @@ bin_PROGRAMS = swmr_generator$(EXEEXT) swmr_reader$(EXEEXT) \ swmr_remove_writer$(EXEEXT) swmr_addrem_writer$(EXEEXT) \ swmr_sparse_reader$(EXEEXT) swmr_sparse_writer$(EXEEXT) \ use_append_chunk$(EXEEXT) use_append_mchunks$(EXEEXT) \ - atomic_writer$(EXEEXT) atomic_reader$(EXEEXT) + atomic_writer$(EXEEXT) atomic_reader$(EXEEXT) \ + accum_swmr_reader$(EXEEXT) check_PROGRAMS = $(am__EXEEXT_1) error_test$(EXEEXT) \ err_compat$(EXEEXT) tcheck_version$(EXEEXT) testmeta$(EXEEXT) \ links_env$(EXEEXT) flushrefresh$(EXEEXT) $(am__EXEEXT_2) @@ -191,6 +192,10 @@ accum_SOURCES = accum.c accum_OBJECTS = accum.$(OBJEXT) accum_LDADD = $(LDADD) accum_DEPENDENCIES = libh5test.la $(LIBHDF5) +accum_swmr_reader_SOURCES = accum_swmr_reader.c +accum_swmr_reader_OBJECTS = accum_swmr_reader.$(OBJEXT) +accum_swmr_reader_LDADD = $(LDADD) +accum_swmr_reader_DEPENDENCIES = libh5test.la $(LIBHDF5) app_ref_SOURCES = app_ref.c app_ref_OBJECTS = app_ref.$(OBJEXT) app_ref_LDADD = $(LDADD) @@ -592,17 +597,18 @@ am__v_CCLD_0 = @echo " CCLD " $@; am__v_CCLD_1 = SOURCES = $(libdynlib1_la_SOURCES) $(libdynlib2_la_SOURCES) \ $(libdynlib3_la_SOURCES) $(libh5test_la_SOURCES) accum.c \ - app_ref.c atomic_reader.c atomic_writer.c big.c bittests.c \ - btree2.c cache.c cache_api.c cache_tagging.c cmpd_dset.c \ - cross_read.c dangle.c dsets.c dt_arith.c dtransform.c dtypes.c \ - earray.c efc.c enc_dec_plist.c enc_dec_plist_with_endianess.c \ - enum.c err_compat.c error_test.c extend.c external.c farray.c \ - fheap.c file_image.c fillval.c filter_fail.c flush1.c flush2.c \ - flushrefresh.c freespace.c gen_bad_ohdr.c gen_bogus.c \ - gen_cross.c gen_deflate.c gen_file_image.c gen_filespace.c \ - gen_filters.c gen_idx.c gen_new_array.c gen_new_fill.c \ - gen_new_group.c gen_new_mtime.c gen_new_super.c \ - gen_noencoder.c gen_nullspace.c gen_plist.c gen_sizes_lheap.c \ + accum_swmr_reader.c app_ref.c atomic_reader.c atomic_writer.c \ + big.c bittests.c btree2.c cache.c cache_api.c cache_tagging.c \ + cmpd_dset.c cross_read.c dangle.c dsets.c dt_arith.c \ + dtransform.c dtypes.c earray.c efc.c enc_dec_plist.c \ + enc_dec_plist_with_endianess.c enum.c err_compat.c \ + error_test.c extend.c external.c farray.c fheap.c file_image.c \ + fillval.c filter_fail.c flush1.c flush2.c flushrefresh.c \ + freespace.c gen_bad_ohdr.c gen_bogus.c gen_cross.c \ + gen_deflate.c gen_file_image.c gen_filespace.c gen_filters.c \ + gen_idx.c gen_new_array.c gen_new_fill.c gen_new_group.c \ + gen_new_mtime.c gen_new_super.c gen_noencoder.c \ + gen_nullspace.c gen_plist.c gen_sizes_lheap.c \ gen_specmetaread.c gen_udlinks.c getname.c gheap.c hyperslab.c \ istore.c lheap.c links.c links_env.c mf.c mount.c mtime.c \ ntypes.c objcopy.c ohdr.c plugin.c pool.c reserved.c \ @@ -615,18 +621,18 @@ SOURCES = $(libdynlib1_la_SOURCES) $(libdynlib2_la_SOURCES) \ DIST_SOURCES = $(am__libdynlib1_la_SOURCES_DIST) \ $(am__libdynlib2_la_SOURCES_DIST) \ $(am__libdynlib3_la_SOURCES_DIST) $(libh5test_la_SOURCES) \ - accum.c app_ref.c atomic_reader.c atomic_writer.c big.c \ - bittests.c btree2.c cache.c cache_api.c cache_tagging.c \ - cmpd_dset.c cross_read.c dangle.c dsets.c dt_arith.c \ - dtransform.c dtypes.c earray.c efc.c enc_dec_plist.c \ - enc_dec_plist_with_endianess.c enum.c err_compat.c \ - error_test.c extend.c external.c farray.c fheap.c file_image.c \ - fillval.c filter_fail.c flush1.c flush2.c flushrefresh.c \ - freespace.c gen_bad_ohdr.c gen_bogus.c gen_cross.c \ - gen_deflate.c gen_file_image.c gen_filespace.c gen_filters.c \ - gen_idx.c gen_new_array.c gen_new_fill.c gen_new_group.c \ - gen_new_mtime.c gen_new_super.c gen_noencoder.c \ - gen_nullspace.c gen_plist.c gen_sizes_lheap.c \ + accum.c accum_swmr_reader.c app_ref.c atomic_reader.c \ + atomic_writer.c big.c bittests.c btree2.c cache.c cache_api.c \ + cache_tagging.c cmpd_dset.c cross_read.c dangle.c dsets.c \ + dt_arith.c dtransform.c dtypes.c earray.c efc.c \ + enc_dec_plist.c enc_dec_plist_with_endianess.c enum.c \ + err_compat.c error_test.c extend.c external.c farray.c fheap.c \ + file_image.c fillval.c filter_fail.c flush1.c flush2.c \ + flushrefresh.c freespace.c gen_bad_ohdr.c gen_bogus.c \ + gen_cross.c gen_deflate.c gen_file_image.c gen_filespace.c \ + gen_filters.c gen_idx.c gen_new_array.c gen_new_fill.c \ + gen_new_group.c gen_new_mtime.c gen_new_super.c \ + gen_noencoder.c gen_nullspace.c gen_plist.c gen_sizes_lheap.c \ gen_specmetaread.c gen_udlinks.c getname.c gheap.c hyperslab.c \ istore.c lheap.c links.c links_env.c mf.c mount.c mtime.c \ ntypes.c objcopy.c ohdr.c plugin.c pool.c reserved.c \ @@ -961,7 +967,8 @@ CHECK_CLEANFILES = *.chkexe *.chklog *.clog accum.h5 cmpd_dset.h5 \ flushrefresh.h5 flushrefresh_VERIFICATION_START \ flushrefresh_VERIFICATION_CHECKPOINT1 \ flushrefresh_VERIFICATION_CHECKPOINT2 \ - flushrefresh_VERIFICATION_DONE atomic_data + flushrefresh_VERIFICATION_DONE atomic_data accum_swmr_big.h5 \ + ohdr_swmr.h5 INCLUDES = -I$(top_srcdir)/src -I$(top_builddir)/src # Test script for error_test and err_compat @@ -1223,6 +1230,9 @@ clean-noinstPROGRAMS: accum$(EXEEXT): $(accum_OBJECTS) $(accum_DEPENDENCIES) $(EXTRA_accum_DEPENDENCIES) @rm -f accum$(EXEEXT) $(AM_V_CCLD)$(LINK) $(accum_OBJECTS) $(accum_LDADD) $(LIBS) +accum_swmr_reader$(EXEEXT): $(accum_swmr_reader_OBJECTS) $(accum_swmr_reader_DEPENDENCIES) $(EXTRA_accum_swmr_reader_DEPENDENCIES) + @rm -f accum_swmr_reader$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(accum_swmr_reader_OBJECTS) $(accum_swmr_reader_LDADD) $(LIBS) app_ref$(EXEEXT): $(app_ref_OBJECTS) $(app_ref_DEPENDENCIES) $(EXTRA_app_ref_DEPENDENCIES) @rm -f app_ref$(EXEEXT) $(AM_V_CCLD)$(LINK) $(app_ref_OBJECTS) $(app_ref_LDADD) $(LIBS) @@ -1495,6 +1505,7 @@ distclean-compile: -rm -f *.tab.c @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/accum.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/accum_swmr_reader.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/app_ref.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/atomic_reader.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/atomic_writer.Po@am__quote@ diff --git a/test/accum.c b/test/accum.c index 488b866..6c89765 100644 --- a/test/accum.c +++ b/test/accum.c @@ -25,6 +25,11 @@ /* Filename */ #define FILENAME "accum.h5" +/* The file name is the same as the define in accum_swmr_reader.c */ +#define SWMR_FILENAME "accum_swmr_big.h5" +/* The reader forked by test_swmr_write_big() */ +#define SWMR_READER "accum_swmr_reader" + /* "big" I/O test values */ #define BIG_BUF_SIZE (6 * 1024 * 1024) @@ -50,6 +55,7 @@ unsigned test_read_after(void); unsigned test_free(void); unsigned test_big(void); unsigned test_random_write(void); +unsigned test_swmr_write_big(void); /* Helper Function Prototypes */ void accum_printf(void); @@ -119,6 +125,9 @@ main(void) if(H5Fclose(fid) < 0) TEST_ERROR HDremove(FILENAME); + /* This test uses a different file */ + nerrors += test_swmr_write_big(); + if(nerrors) goto error; puts("All metadata accumulator tests passed."); @@ -1772,6 +1781,165 @@ error: return 1; } /* end test_random_write() */ +/*------------------------------------------------------------------------- + * Function: test_swmr_write_big + * + * Purpose: A SWMR test: verifies that writing "large" metadata to a file + * opened with SWMR_WRITE will flush the existing metadata in the + * accumulator to disk first before writing the "large" metadata + * to disk. + * This test will fork and exec a reader "accum_swmr_reader" which + * opens the same file with SWMR_READ and verifies that the correct + * metadata is read from disk. + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Vailin Choi; April 2013 + * + *------------------------------------------------------------------------- + */ +unsigned +test_swmr_write_big(void) +{ + hid_t fid = -1; /* File ID */ + hid_t fapl = -1; /* File access property list */ + H5F_t *rf = NULL; /* File pointer */ + uint8_t *wbuf2, *rbuf; /* Buffers for reading & writing */ + uint8_t wbuf[1024]; /* Buffer for reading & writing */ + unsigned u; /* Local index variable */ + pid_t pid; /* Process ID */ + int status; /* Status returned from child process */ + + TESTING("SWMR write of large metadata"); + + /* File access property list */ + if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + FAIL_STACK_ERROR + + /* Set to use latest library format */ + if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + FAIL_STACK_ERROR + + /* Create a test file with latest format (ensure version 2 object header for SWMR) */ + if((fid = H5Fcreate(SWMR_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + FAIL_STACK_ERROR + + /* Close the file */ + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR + + /* Open the file with SWMR_WRITE */ + if((fid = H5Fopen(SWMR_FILENAME, H5F_ACC_RDWR | H5F_ACC_SWMR_WRITE, fapl)) < 0) + FAIL_STACK_ERROR + + /* Get H5F_t * to internal file structure */ + if(NULL == (rf = (H5F_t *)H5I_object(fid))) FAIL_STACK_ERROR + + /* We'll be writing lots of garbage data, so extend the + file a ways. 10MB should do. */ + if(H5FD_set_eoa(rf->shared->lf, H5FD_MEM_DEFAULT, (haddr_t)(1024*1024*10)) < 0) FAIL_STACK_ERROR + + /* Reset metadata accumulator for the file */ + if(H5F_accum_reset(rf, H5P_DATASET_XFER_DEFAULT, TRUE) < 0) + FAIL_STACK_ERROR; + + /* Allocate space for the write & read buffers */ + if((wbuf2 = (uint8_t *)HDmalloc(BIG_BUF_SIZE)) == NULL) + FAIL_STACK_ERROR; + if((rbuf = (uint8_t *)HDmalloc(BIG_BUF_SIZE)) == NULL) + FAIL_STACK_ERROR; + + /* Initialize wbuf with "0, 1, 2...1024"*/ + for(u = 0; u < 1024; u++) + wbuf[u] = (uint8_t)u; + + /* Write [1024, 1024] bytes with wbuf */ + if(H5F_block_write(rf, H5FD_MEM_DEFAULT, (haddr_t)1024, (size_t)1024, H5P_DATASET_XFER_DEFAULT, wbuf) < 0) + FAIL_STACK_ERROR; + /* Read the data */ + if(H5F_block_read(rf, H5FD_MEM_DEFAULT, (haddr_t)1024, (size_t)1024, H5P_DATASET_XFER_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + /* Verify the data read is correct */ + if(HDmemcmp(wbuf, rbuf, 1024) != 0) TEST_ERROR; + /* Flush the data to disk */ + if(H5F_accum_reset(rf, H5P_DATASET_XFER_DEFAULT, TRUE) < 0) + FAIL_STACK_ERROR; + + /* Initialize wbuf with all 1s */ + for(u = 0; u < 1024; u++) + wbuf[u] = (uint8_t)1; + + /* Initialize wbuf2 */ + for(u = 0; u < BIG_BUF_SIZE; u++) + wbuf2[u] = (uint8_t)(u + 1); + + /* Write [1024,1024] with wbuf--all 1s */ + if(H5F_block_write(rf, H5FD_MEM_DEFAULT, (haddr_t)1024, (size_t)1024, H5P_DATASET_XFER_DEFAULT, wbuf) < 0) + FAIL_STACK_ERROR; + /* Read the data */ + if(H5F_block_read(rf, H5FD_MEM_DEFAULT, (haddr_t)1024, (size_t)1024, H5P_DATASET_XFER_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + /* Verify the data read is correct */ + if(HDmemcmp(wbuf, rbuf, 1024) != 0) TEST_ERROR; + /* The data stays in the accumulator */ + + /* Write a large piece of metadata [2048, BIG_BUF_SIZE] with wbuf2 */ + if(H5F_block_write(rf, H5FD_MEM_DEFAULT, (haddr_t)2048, (size_t)BIG_BUF_SIZE, H5P_DATASET_XFER_DEFAULT, wbuf2) < 0) + FAIL_STACK_ERROR; + /* Read the data */ + if(H5F_block_read(rf, H5FD_MEM_DEFAULT, (haddr_t)2048, (size_t)BIG_BUF_SIZE, H5P_DATASET_XFER_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + /* Verify the data read is correct */ + if(HDmemcmp(wbuf2, rbuf, BIG_BUF_SIZE) != 0) TEST_ERROR; + + /* Fork child process to verify that the data at [1024, 2014] does get written to disk */ + if((pid = HDfork()) < 0) { + HDperror("fork"); + FAIL_STACK_ERROR; + } else if(0 == pid) { /* Child process */ + /* Run the reader */ + status = HDexecve(SWMR_READER, NULL, NULL); + FAIL_STACK_ERROR; + } + + /* Parent process -- wait for the child process to complete */ + while(pid != HDwaitpid(pid, &status, 0)) + /*void*/; + + /* Check if child process terminates normally and its return value */ + if(WIFEXITED(status) && !WEXITSTATUS(status)) { + /* Flush the accumulator */ + if(H5F_accum_reset(rf, H5P_DATASET_XFER_DEFAULT, TRUE) < 0) + FAIL_STACK_ERROR; + /* Close the property list */ + if(H5Pclose(fapl) < 0) + FAIL_STACK_ERROR; + + /* Close and remove the file */ + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + HDremove(SWMR_FILENAME); + + /* Release memory */ + if(wbuf2) HDfree(wbuf2); + if(rbuf) HDfree(rbuf); + PASSED(); + return 0; + } + +error: + /* Closing and remove the file */ + H5Pclose(fapl); + H5Fclose(fid); + HDremove(SWMR_FILENAME); + /* Release memory */ + if(wbuf2) HDfree(wbuf2); + if(rbuf) HDfree(rbuf); + + return 1; +} /* end test_swmr_write_big() */ + /*------------------------------------------------------------------------- * Function: accum_printf diff --git a/test/accum_swmr_reader.c b/test/accum_swmr_reader.c new file mode 100644 index 0000000..b9bf98d --- /dev/null +++ b/test/accum_swmr_reader.c @@ -0,0 +1,80 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +#include "h5test.h" + +#define H5F_PACKAGE +#include "H5Fpkg.h" +#include "H5FDprivate.h" +#include "H5Iprivate.h" + +/* Filename: this is the same as the define in accum.c used by test_swmr_write_big() */ +#define SWMR_FILENAME "accum_swmr_big.h5" + + +/*------------------------------------------------------------------------- + * Function: main + * + * Purpose: This is the reader forked/execved by "test_swmr_write_big()" + * test in accum.c. The reader reads at address 1024 from the file + * and verifies that the metadata in the accumulator at address + * 1024 does get written to disk. + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Vailin Choi; June 2013 + * + *------------------------------------------------------------------------- + */ +int +main(void) +{ + hid_t fid = -1; /* File ID */ + H5F_t *f = NULL; /* File pointer */ + unsigned u; /* Local index variable */ + uint8_t rbuf[1024]; /* Buffer for reading */ + uint8_t buf[1024]; /* Buffer for holding the expected data */ + + /* Initialize buffers */ + for(u = 0; u < 1024; u++) { + rbuf[u] = 0; /* The buffer for reading */ + buf[u] = 1; /* The expected data should be all 1s */ + } + + /* Open the file with SWMR_READ */ + if((fid = H5Fopen(SWMR_FILENAME, H5F_ACC_RDONLY | H5F_ACC_SWMR_READ, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR + + /* Get H5F_t * to internal file structure */ + if(NULL == (f = (H5F_t *)H5I_object(fid))) + FAIL_STACK_ERROR + + /* Should read in [1024, 2024] with buf data */ + if(H5F_block_read(f, H5FD_MEM_DEFAULT, (haddr_t)1024, (size_t)1024, H5P_DATASET_XFER_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify the data read is correct */ + if(HDmemcmp(buf, rbuf, 1024) != 0) + TEST_ERROR; + + /* CLose the file */ + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + + return 0; + +error: + H5Fclose(fid); + return 1; +} /* end main() */ diff --git a/test/ohdr.c b/test/ohdr.c index 502a8b1..fa00716 100644 --- a/test/ohdr.c +++ b/test/ohdr.c @@ -45,6 +45,9 @@ const char *FILENAME[] = { */ #define FILE_BOGUS "tbogus.h5" +/* */ +#define FILE_OHDR_SWMR "ohdr_swmr.h5" +#define DSET_NAME "COMPACT_DSET" /* * Verify that messages are moved forward into a "continuation message": * Create an object header with several continuation chunks @@ -295,6 +298,131 @@ error: return -1; } /* test_ohdr_cache() */ +/* + * To exercise the coding for the re-read of the object header for SWMR access. + * When the object header is read in H5O_load() of H5Ocache.c, the library initially reads + * 512 bytes for decoding, then reads the remaining bytes later if the object header is + * greater than 512 bytes. For SWMR access, the read should be done all at one time. + */ +static herr_t +test_ohdr_swmr(void) +{ + hid_t fid = -1; /* File ID */ + hid_t fapl = -1; /* File access property list */ + hid_t did = -1; /* Dataset ID */ + hid_t sid = -1; /* Dataspace ID */ + hid_t plist = -1; /* Dataset creation property list */ + size_t compact_size = 1024; /* The size of compact dataset */ + int wbuf[1024]; /* Buffer for writing */ + hsize_t dims[1]; /* Dimension sizes */ + unsigned int n = 0, u; /* Locatl index variable */ + H5O_info_t obj_info; /* Information for the object */ + + TESTING("exercise the coding for the re-read of the object header for SWMR access"); + + /* File access property list */ + if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + FAIL_STACK_ERROR + + /* Set to use latest library format */ + if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + FAIL_STACK_ERROR + + /* Initialize data */ + for(u = 0; u < compact_size; u++) + wbuf[u] = n++; + + /* Create the file with the latest format (ensure version 2 object header for SWMR) */ + if((fid = H5Fcreate(FILE_OHDR_SWMR, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + TEST_ERROR + + /* Create a small data space for compact dataset */ + dims[0] = (hsize_t)compact_size; + if((sid = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR + + /* Create property list for compact dataset creation */ + if((plist = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set the layout for the compact dataset */ + if(H5Pset_layout(plist, H5D_COMPACT) < 0) + FAIL_STACK_ERROR + + /* Create a compact dataset */ + if((did = H5Dcreate2(fid, DSET_NAME, H5T_NATIVE_INT, sid, H5P_DEFAULT, plist, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR + + /* Write to the compact dataset */ + if(H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf) < 0) + FAIL_STACK_ERROR + + /* Close the dataset */ + if(H5Dclose(did) < 0) + FAIL_STACK_ERROR + + /* Close the file */ + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR + + /* Open the file for SWMR write and latest format */ + if((fid = H5Fopen(FILE_OHDR_SWMR, H5F_ACC_RDWR|H5F_ACC_SWMR_WRITE, fapl)) < 0) + FAIL_STACK_ERROR + + /* Open the compact dataset */ + if((did = H5Dopen2(fid, DSET_NAME, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR + + /* Get the object information */ + if(H5Oget_info(did, &obj_info) < 0) + FAIL_STACK_ERROR + + /* The size of object header should be greater than the speculative read size of 512 */ + /* This will exercise the coding for the re-read of the object header for SWMR access */ + if(obj_info.hdr.space.total < 512) + TEST_ERROR; + + /* Close the dataset */ + if(H5Dclose(did) < 0) + FAIL_STACK_ERROR + + /* Close the file */ + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR + + /* Close the dataspace */ + if(H5Sclose(sid) < 0) + FAIL_STACK_ERROR + + /* Close the dataset creation property list */ + if(H5Pclose(plist) < 0) + FAIL_STACK_ERROR + + /* Close the file access property list */ + if(H5Pclose(fapl) < 0) + FAIL_STACK_ERROR + + /* Remove the test file */ + if(HDremove(FILE_OHDR_SWMR) < 0) + FAIL_STACK_ERROR + + PASSED(); + + return 0; + +error: + H5E_BEGIN_TRY { + H5Fclose(fid); + H5Dclose(did); + H5Sclose(sid); + H5Pclose(plist); + H5Pclose(fapl); + HDremove(FILE_OHDR_SWMR); + } H5E_END_TRY; + + return -1; +} /* test_ohdr_swmr() */ + /*------------------------------------------------------------------------- * Function: main @@ -786,6 +914,7 @@ main(void) PASSED(); } + /* Close the file we created */ if(H5Fclose(file) < 0) TEST_ERROR @@ -798,6 +927,9 @@ main(void) /* Verify symbol table messages are cached */ if(h5_verify_cached_stabs(FILENAME, fapl) < 0) TEST_ERROR + /* A test to exercise the re-read of the object header for SWMR access */ + if(test_ohdr_swmr() < 0) TEST_ERROR + puts("All object header tests passed."); h5_cleanup(FILENAME, fapl); return(0); |