summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorVailin Choi <vchoi@jam.ad.hdfgroup.org>2018-09-28 16:29:02 (GMT)
committerVailin Choi <vchoi@jam.ad.hdfgroup.org>2018-09-28 16:29:02 (GMT)
commit5c7ca8afff0e635f12652c35dbe9818235ac34c4 (patch)
tree1245d78ca4b0aaeb40ecab6b2615ddad886042c3
parent06fa8a94558939400dc43e99c08a784e26dfa01d (diff)
downloadhdf5-5c7ca8afff0e635f12652c35dbe9818235ac34c4.zip
hdf5-5c7ca8afff0e635f12652c35dbe9818235ac34c4.tar.gz
hdf5-5c7ca8afff0e635f12652c35dbe9818235ac34c4.tar.bz2
Third batch of checkin:
1) Free space manager for the metadata file 2) Delayed free space release linked list 3) H5F_update_vfd_swmr_metadata_file() 3) VFD SWMR driver: read callback 4) Flushing for VFD SWMR 5) Port one concurrent test from swmr test set 6) Bug fixes and refactoring
Diffstat
-rw-r--r--MANIFEST15
-rw-r--r--configure.ac1
-rw-r--r--src/H5.c7
-rw-r--r--src/H5F.c1
-rw-r--r--src/H5FDprivate.h66
-rw-r--r--src/H5FDvfd_swmr.c315
-rw-r--r--src/H5FDvfd_swmr.h1
-rw-r--r--src/H5FSprivate.h1
-rw-r--r--src/H5Fint.c579
-rw-r--r--src/H5Fpkg.h46
-rw-r--r--src/H5Fprivate.h20
-rw-r--r--src/H5Fquery.c24
-rw-r--r--src/H5Fspace.c160
-rw-r--r--src/H5MV.c518
-rw-r--r--src/H5MVmodule.h33
-rw-r--r--src/H5MVpkg.h83
-rw-r--r--src/H5MVprivate.h58
-rw-r--r--src/H5MVsection.c361
-rw-r--r--src/H5Pfapl.c4
-rw-r--r--src/H5private.h27
-rw-r--r--src/Makefile.am1
-rw-r--r--test/Makefile.am5
-rw-r--r--test/testvfdswmr.sh.in292
-rw-r--r--test/vfd_swmr_common.c288
-rw-r--r--test/vfd_swmr_common.h78
-rw-r--r--test/vfd_swmr_generator.c415
-rw-r--r--test/vfd_swmr_reader.c575
-rw-r--r--test/vfd_swmr_writer.c473
28 files changed, 4134 insertions, 313 deletions
diff --git a/MANIFEST b/MANIFEST
index 117c531..c9a6706 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -636,6 +636,8 @@
./src/H5FDstdio.c
./src/H5FDstdio.h
./src/H5FDtest.c
+./src/H5FDvfd_swmr.c
+./src/H5FDvfd_swmr.h
./src/H5FDwindows.c
./src/H5FDwindows.h
./src/H5FL.c
@@ -743,6 +745,11 @@
./src/H5MPpkg.h
./src/H5MPprivate.h
./src/H5MPtest.c
+./src/H5MV.c
+./src/H5MVmodule.h
+./src/H5MVsection.c
+./src/H5MVpkg.h
+./src/H5MVprivate.h
./src/H5O.c
./src/H5Oainfo.c
./src/H5Oalloc.c
@@ -1090,6 +1097,7 @@
./test/testmeta.c
./test/testswmr.sh.in
./test/testvdsswmr.sh.in
+./test/testvfdswmr.sh.in
./test/tfile.c
./test/tgenprop.c
./test/th5o.c
@@ -1138,7 +1146,12 @@
./test/vds_swmr_gen.c
./test/vds_swmr_reader.c
./test/vds_swmr_writer.c
-
+./test/vfd_swmr.c
+./test/vfd_swmr_common.c
+./test/vfd_swmr_common.h
+./test/vfd_swmr_generator.c
+./test/vfd_swmr_reader.c
+./test/vfd_swmr_writer.c
./test/testfiles/err_compat_1
./test/testfiles/err_compat_2
./test/testfiles/error_test_1
diff --git a/configure.ac b/configure.ac
index c06359d..279c443 100644
--- a/configure.ac
+++ b/configure.ac
@@ -3439,6 +3439,7 @@ AC_CONFIG_FILES([src/libhdf5.settings
test/testlibinfo.sh
test/testlinks_env.sh
test/testswmr.sh
+ test/testvfdswmr.sh
test/testvdsswmr.sh
test/test_filenotclosed.sh
test/test_filter_plugin.sh
diff --git a/src/H5.c b/src/H5.c
index 20038fe..c401667 100644
--- a/src/H5.c
+++ b/src/H5.c
@@ -85,13 +85,6 @@ char H5_lib_vers_info_g[] = H5_VERS_INFO;
static hbool_t H5_dont_atexit_g = FALSE;
H5_debug_t H5_debug_g; /* debugging info */
-hbool_t vfd_swmr_g = FALSE;
-hbool_t vfd_swmr_writer_g = FALSE;
-uint64_t tick_num_g = 0;
-struct timespec end_of_tick_g;
-H5F_file_t *vfd_swmr_file_g;
-
-
/*******************/
/* Local Variables */
/*******************/
diff --git a/src/H5F.c b/src/H5F.c
index c6db168..e168343 100644
--- a/src/H5F.c
+++ b/src/H5F.c
@@ -2016,6 +2016,7 @@ H5Fvfd_swmr_end_tick(hid_t file_id)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "must have VFD SWMR configured for this public routine")
/* ??Trigger end of tick processing later */
+ H5F_vfd_swmr_writer_end_of_tick();
done:
FUNC_LEAVE_API(ret_value)
diff --git a/src/H5FDprivate.h b/src/H5FDprivate.h
index 57be23a..16bb2b7 100644
--- a/src/H5FDprivate.h
+++ b/src/H5FDprivate.h
@@ -71,23 +71,15 @@
/* Size of the metadata file index */
#define H5FD_MD_INDEX_SIZE(N) /* N is number of entries in index */ \
( \
- H5_SIZEOF_MAGIC /* Signature */ \
+ H5_SIZEOF_MAGIC /* Signature */ \
+ 8 /* Tick num */ \
+ 4 /* Number of entries */ \
+ (N * H5FD_MD_INDEX_ENTRY_SIZE) /* Index entries */ \
+ H5FD_SIZEOF_CHKSUM /* Metadata header checksum */ \
)
-#define H5FD_MD_EMPTY_INDEX_SIZE \
- ( \
- H5_SIZEOF_MAGIC /* Signature */ \
- + 8 /* Tick num */ \
- + 4 /* Number of entries */ \
- + H5FD_SIZEOF_CHKSUM /* Metadata header checksum */ \
- )
-
/* Retries for metadata file */
-#define H5FD_VFD_SWMR_MD_FILE_RETRY_MAX 50 /* Maximum retries when opening the MD file */
+#define H5FD_VFD_SWMR_MD_FILE_RETRY_MAX 50 /* Maximum retries when opening the MD file */
#define H5FD_VFD_SWMR_MD_LOAD_RETRY_MAX 20 /* Maximum retries when trying to load the MD file header and index */
#define H5FD_VFD_SWMR_MD_HEADER_RETRY_MAX 40 /* Maximum retries when deserializing the MD file header */
#define H5FD_VFD_SWMR_MD_INDEX_RETRY_MAX 5 /* Maximum retries when deserializing the MD file index */
@@ -115,16 +107,48 @@ typedef struct H5FD_vfd_swmr_md_header {
* metadata page, or multi page metadata entry in the HDF5
* file IN PAGES.
* To obtain byte offset, multiply this value by the page size.
+ *
* md_file_page_offset: Unsigned 64-bit value containing the base address of the
* metadata page, or multi page metadata entry in the metadata
* file IN PAGES.
* To obtain byte offset, multiply this value by the page size.
- * length: The length of the metadata page or multi page metadata entry
+ *
+ * length: The length of the metadata page or multi- page metadata entry
* in BYTES.
- * chksum: Checksum for the metadata page or multi page metadata entry
+ *
+ * chksum: Checksum for the metadata page or multi-page metadata entry.
+ * For the VFD SWMR writer, this value is undefined until the
+ * referenced entry has been written to the metadata file.
+ *
+ * entry_ptr: Used by the VFD SWMR writer only.
+ * For the VFD SWMR reader, this field should always be NULL.
+ * If the referenced metadata page or multi-page metadata
+ * entry was modified in the current tick, this field points to
+ * a buffer in the page buffer containing its value.
+ * This field is used by the metadata file creation/update code
+ * to access the metadata pages or multi-page metadata entries
+ * so that their current values can be copied into the metadata
+ * file. After this copy, this field should be set to NULL.
+ *
* tick_of_last_change: Number of the last tick in which this index entry was changed.
- * This field is only used by the VFD SWMR writer.
- * For readers, it will always be set to 0.
+ * Used by the VFD SWMR writer only.
+ * For the VFD SWMR reader, this field will always be set to 0.
+ *
+ * clean: Used by the VFD SWMR writer only.
+ * Set to TRUE whenever the referenced metadata page or multi-page
+ * metadata entry is written to the HDF5 file.
+ * Set to FALSE whenever it is marked dirty in the page buffer.
+ *
+ * tick_of_last_flush: Number of the tick in which this entry was last written to the
+ * HDF5 file or zero if it has never been flusehd.
+ * Used by the VFD SWMR writer only.
+ * For the VFD SWMR reader, this field should always be 0.
+ *
+ * delayed_flush: If the flush of the referenced metadata page or multi-page
+ * metadata entry must be delayed, the earliest tick in which
+ * it may be flushed, or zero if there is no such constraint.
+ * Used by the VFD SWMR writer only.
+ *
* is_moved_to_hdf5_file: Set to TRUE iff the entry referenced is in the HDF5 file and
* is therefore about to be removed from the metadata file
*/
@@ -133,8 +157,12 @@ typedef struct H5FD_vfd_swmr_idx_entry_t {
uint64_t md_file_page_offset;
uint32_t length;
uint32_t chksum;
+ void *entry_ptr;
uint64_t tick_of_last_change;
- hbool_t is_moved_to_hdf5_file;
+ hbool_t clean;
+ uint64_t tick_of_last_flush;
+ uint64_t delayed_flush;
+ hbool_t moved_to_hdf5_file;
} H5FD_vfd_swmr_idx_entry_t;
/*
@@ -145,9 +173,9 @@ typedef struct H5FD_vfd_swmr_idx_entry_t {
* entries: The array of index entries
*/
typedef struct H5FD_vfd_swmr_md_index {
- uint64_t tick_num;
- uint32_t num_entries;
- H5FD_vfd_swmr_idx_entry_t *entries;
+ uint64_t tick_num;
+ uint32_t num_entries;
+ H5FD_vfd_swmr_idx_entry_t *entries;
} H5FD_vfd_swmr_md_index;
@@ -270,8 +298,6 @@ H5_DLL haddr_t H5FD_get_base_addr(const H5FD_t *file);
H5_DLL herr_t H5FD_set_paged_aggr(H5FD_t *file, hbool_t paged);
/* Function prototypes for VFD SWMR */
-H5_DLL herr_t H5FD_vfd_swmr_writer_end_of_tick();
-H5_DLL herr_t H5FD_vfd_swmr_reader_end_of_tick();
H5_DLL herr_t H5FD_vfd_swmr_get_tick_and_idx(H5FD_t *_file, hbool_t read_index,
uint64_t *tick_ptr, uint32_t *num_entries_ptr, H5FD_vfd_swmr_idx_entry_t index[]);
diff --git a/src/H5FDvfd_swmr.c b/src/H5FDvfd_swmr.c
index e4df4f1..9dcf8ef 100644
--- a/src/H5FDvfd_swmr.c
+++ b/src/H5FDvfd_swmr.c
@@ -12,7 +12,7 @@
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
- * Purpose:
+ * Purpose: VFD SWMR driver
*/
#include "H5FDdrvr_module.h" /* This source code file is part of the H5FD driver module */
@@ -251,8 +251,8 @@ H5FD_vfd_swmr_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxa
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list")
/* Allocate buffer for reading the VFD SWMR configuration */
- if(NULL == (vfd_swmr_config = (uint8_t *)H5MM_malloc(sizeof(H5F_vfd_swmr_config_t))))
- HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "memory allocation failed for H5F_vfd_swmr_config_t")
+ if(NULL == (vfd_swmr_config = (H5F_vfd_swmr_config_t *)H5MM_malloc(sizeof(H5F_vfd_swmr_config_t))))
+ HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "memory allocation failed for H5F_vfd_swmr_config_t")
/* Get VFD SWMR configuration */
if(H5P_get(plist, H5F_ACS_VFD_SWMR_CONFIG_NAME, vfd_swmr_config) < 0)
@@ -290,7 +290,7 @@ H5FD_vfd_swmr_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxa
HGOTO_ERROR(H5E_VFL, H5E_OPENERROR, NULL, "unable to open the metadata file after all retry attempts")
/* Retry on loading and decoding the header and index in the metadata file */
- if(H5FD_vfd_swmr_load_hdr_and_idx(file, TRUE) < 0)
+ if(H5FD_vfd_swmr_load_hdr_and_idx((H5FD_t *)file, TRUE) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "unable to load/decode the md file header/index")
/* Hard-wired to open the underlying HDF5 file with SEC2 */
@@ -303,12 +303,12 @@ H5FD_vfd_swmr_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxa
done:
/* Free the buffer */
if(vfd_swmr_config)
- vfd_swmr_config = (uint8_t *)H5MM_xfree(vfd_swmr_config);
+ vfd_swmr_config = (H5F_vfd_swmr_config_t *)H5MM_xfree(vfd_swmr_config);
/* Handle closing if error */
if(NULL == ret_value && file) {
- if(H5FD_vfd_swmr_close(file) < 0)
- HDONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "error from closing")
+ if(H5FD_vfd_swmr_close((H5FD_t *)file) < 0)
+ HDONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, NULL, "error from closing")
} /* end if */
FUNC_LEAVE_NOAPI(ret_value)
@@ -370,9 +370,6 @@ H5FD_vfd_swmr_close(H5FD_t *_file)
* Failure: never fails (arguments were checked by the
* caller).
*
- * Programmer: Robb Matzke
- * Thursday, July 29, 1999
- *
*-------------------------------------------------------------------------
*/
static int
@@ -398,13 +395,10 @@ H5FD_vfd_swmr_cmp(const H5FD_t *_f1, const H5FD_t *_f2)
*
* Return: SUCCEED (Can't fail)
*
- * Programmer: Quincey Koziol
- * Friday, August 25, 2000
- *
*-------------------------------------------------------------------------
*/
static herr_t
-H5FD_vfd_swmr_query(const H5FD_t *_file, unsigned long *flags /* out */)
+H5FD_vfd_swmr_query(const H5FD_t H5_ATTR_UNUSED *_file, unsigned long *flags /* out */)
{
FUNC_ENTER_NOAPI_NOINIT_NOERR
@@ -434,9 +428,6 @@ H5FD_vfd_swmr_query(const H5FD_t *_file, unsigned long *flags /* out */)
*
* Return: The end-of-address marker.
*
- * Programmer: Robb Matzke
- * Monday, August 2, 1999
- *
*-------------------------------------------------------------------------
*/
static haddr_t
@@ -448,7 +439,7 @@ H5FD_vfd_swmr_get_eoa(const H5FD_t *_file, H5FD_mem_t type)
FUNC_ENTER_NOAPI_NOINIT
if((ret_value = H5FD_get_eoa(file->hdf5_file_lf, type)) == HADDR_UNDEF)
- HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to get HDF5 file eoa")
+ HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, HADDR_UNDEF, "unable to get HDF5 file eoa")
done:
FUNC_LEAVE_NOAPI(ret_value)
@@ -464,9 +455,6 @@ done:
*
* Return: SUCCEED (Can't fail)
*
- * Programmer: Robb Matzke
- * Thursday, July 29, 1999
- *
*-------------------------------------------------------------------------
*/
static herr_t
@@ -507,7 +495,7 @@ H5FD_vfd_swmr_get_eof(const H5FD_t *_file, H5FD_mem_t type)
/* LATER: need to determine the metadata file or underlying HDF5 file ? */
if((ret_value = H5FD_get_eof(file->hdf5_file_lf, type)) == HADDR_UNDEF)
- HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to set file eoa")
+ HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, HADDR_UNDEF, "unable to set file eoa")
done:
FUNC_LEAVE_NOAPI(ret_value)
@@ -559,44 +547,112 @@ static herr_t
H5FD_vfd_swmr_read(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id,
haddr_t addr, size_t size, void *buf /*out*/)
{
- H5FD_vfd_swmr_t *file = (H5FD_vfd_swmr_t *)_file;
-H5FD_vfd_swmr_idx_entry_t *index;
- herr_t ret_value = SUCCEED; /* Return value */
+ H5FD_vfd_swmr_t *file = (H5FD_vfd_swmr_t *)_file; /* VFD SWMR file struct */
+ H5FD_vfd_swmr_idx_entry_t *index = NULL; /* Metadata file index */
+ unsigned entry_retries = H5FD_VFD_SWMR_MD_INDEX_RETRY_MAX; /* # of retries */
+ uint64_t nanosec = 1; /* # of nanoseconds to sleep between retries */
+ uint32_t num_entries = 0; /* Number of entries in index */
+ uint32_t fs_page_size; /* Page size */
+ unsigned lo = 0, hi; /* Low & high index values */
+ unsigned my_idx = 0; /* Final index value */
+ int cmp; /* Return from comparison */
+ uint32_t computed_chksum; /* Computed checksum */
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI_NOINIT
HDassert(file && file->pub.cls);
HDassert(buf);
-#ifdef TEMP
-tick_num
-num_entries
-H5FD_vfd_swmr_idx_entry_t *index;
- if(H5FD_vfd_swmr_get_tick_and_idx(file, TRUE, &tick_num, &num_entries, NULL) < 0)
- HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FALSE, "unable to retrieve tick and entries from the md file")
+ /* Try loading and decoding the header and index in the metadata file */
+ if(H5FD_vfd_swmr_load_hdr_and_idx(_file, FALSE) < 0)
+ HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "unable to load/decode the md file header/index")
+ fs_page_size = file->md_header.fs_page_size;
+ num_entries = file->md_index.num_entries;
if(num_entries) {
-
/* Allocate memory for index entries */
if(NULL == (index = H5FL_SEQ_MALLOC(H5FD_vfd_swmr_idx_entry_t, num_entries)))
- HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FALSE, "memory allocation failed for index entries")
-
- /* Get the entries for the index */
- if(H5FD_vfd_swmr_get_tick_and_idx(file, FALSE, FALSE, &tick_num, &num_entries, index) < 0)
- HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "unable to load/decode the md file header/index")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "memory allocation failed for index entries")
+ HDmemcpy(index, file->md_index.entries, num_entries * sizeof(H5FD_vfd_swmr_idx_entry_t));
}
-search for addr from the index
-if found, read the page into buf based on the offset/size found in the index and return
-otherwise read from the underlying HDF5 file
-#endif
+ /* Try finding the addr from the index */
+ cmp = -1;
+ lo = 0;
+ hi = num_entries;
+ while(lo < hi && cmp) {
+ my_idx = (lo + hi) / 2;
+ cmp = H5F_addr_cmp(index[my_idx].hdf5_page_offset * fs_page_size, addr);
+ if(cmp < 0)
+ hi = my_idx;
+ else
+ lo = my_idx + 1;
+ } /* end while */
+
+ /* Found in index, read from the metadata file */
+ if(cmp == 0) {
+ HDassert(size == index[my_idx].length);
+
+ do {
+ if(HDlseek(file->md_fd, (HDoff_t)index[my_idx].md_file_page_offset * fs_page_size, SEEK_SET) < 0)
+ HGOTO_ERROR(H5E_VFL, H5E_SEEKERROR, FAIL, "unable to seek in metadata file")
+
+ /* Coding borrowed from sec2 read */
+ while(size > 0) {
+
+ h5_posix_io_t bytes_in = 0; /* # of bytes to read */
+ h5_posix_io_ret_t bytes_read = -1; /* # of bytes actually read */
+
+ /* Trying to read more bytes than the return type can handle is
+ * undefined behavior in POSIX.
+ */
+ if(size > H5_POSIX_MAX_IO_BYTES)
+ bytes_in = H5_POSIX_MAX_IO_BYTES;
+ else
+ bytes_in = (h5_posix_io_t)size;
+
+ do {
+ bytes_read = HDread(file->md_fd, buf, bytes_in);
+ } while(-1 == bytes_read && EINTR == errno);
+
+ if(-1 == bytes_read) /* error */
+ HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "error reading the page/multi-page entry from the md file")
+
+ HDassert(bytes_read >= 0);
+ HDassert((size_t)bytes_read <= size);
+
+ size -= (size_t)bytes_read;
+ addr += (haddr_t)bytes_read;
+ buf = (char *)buf + bytes_read;
+ } /* end while size */
+
+ /* Verify stored and computed checksums are equal */
+ computed_chksum = H5_checksum_metadata(buf, index[my_idx].length, 0);
+ if(index[my_idx].chksum == computed_chksum)
+ break;
+
+ /* Double the sleep time next time */
+ H5_nanosleep(nanosec);
+ nanosec *= 2;
- if(H5FD_read(file->hdf5_file_lf, type, addr, size, buf) < 0)
- HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "file read request failed")
+ } while(--entry_retries);
+
+ /* Exhaust all retries for reading the page/multi-page entry */
+ if(entry_retries == 0)
+ HGOTO_ERROR(H5E_VFL, H5E_CANTLOAD, FAIL, "error in reading the page/multi-page entry")
+
+ } else { /* Cannot find addr in index, read from the underlying hdf5 file */
+ if(H5FD_read(file->hdf5_file_lf, type, addr, size, buf) < 0)
+ HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "file read request failed")
+ }
done:
if(ret_value < 0) {
- /* What needs to be cleaned up */
+ if(index) {
+ HDassert(num_entries);
+ index = H5FL_SEQ_FREE(H5FD_vfd_swmr_idx_entry_t, num_entries);
+ }
} /* end if */
FUNC_LEAVE_NOAPI(ret_value)
@@ -628,16 +684,12 @@ H5FD_vfd_swmr_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id
HDassert(file && file->pub.cls);
HDassert(buf);
-
+
+ /* SHOULDN'T come here ?? */
if(H5FD_write(file->hdf5_file_lf, type, addr, size, buf) < 0)
HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "file write request failed")
done:
- if(ret_value < 0) {
- /* Reset last file I/O information */
- /* What needs to be cleaned up */
- } /* end if */
-
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_vfd_swmr_write() */
@@ -653,7 +705,7 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
-H5FD_vfd_swmr_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing)
+H5FD_vfd_swmr_truncate(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, hbool_t closing)
{
H5FD_vfd_swmr_t *file = (H5FD_vfd_swmr_t *)_file; /* VFD SWMR file struct */
herr_t ret_value = SUCCEED; /* Return value */
@@ -683,7 +735,6 @@ static herr_t
H5FD_vfd_swmr_lock(H5FD_t *_file, hbool_t rw)
{
H5FD_vfd_swmr_t *file = (H5FD_vfd_swmr_t *)_file; /* VFD SWMR file struct */
- int lock_flags; /* file locking flags */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI_NOINIT
@@ -724,49 +775,6 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_vfd_swmr_unlock() */
-
-/*-------------------------------------------------------------------------
- * Function: H5FD_vfd_swmr_get_tick_and_idx()
- *
- * Purpose: Retrieve tick_num, num_entries and index from the metadata file
- * --If the parameter "reload_hdr_and_index" is true, load and decode the header
- * and index via H5FD_vfd_swmr_load_hdr_and_idx(), which may replace the VFD's
- * local copies of header and index with the latest info read.
- * --Return tick_num, num_entries and index from the VFD's local copies.
- *
- * Return: Success: SUCCEED
- * Failure: FAIL
- *
- *-------------------------------------------------------------------------
- */
-herr_t
-H5FD_vfd_swmr_get_tick_and_idx(H5FD_t *_file, hbool_t reload_hdr_and_index,
- uint64_t *tick_ptr, uint32_t *num_entries_ptr, H5FD_vfd_swmr_idx_entry_t index[])
-{
- H5FD_vfd_swmr_t *file = (H5FD_vfd_swmr_t *)_file; /* VFD SWMR file struct */
- H5FD_t *ret_value = FALSE; /* Return value */
-
- FUNC_ENTER_NOAPI_NOINIT
-
- if(reload_hdr_and_index) {
- if(H5FD_vfd_swmr_load_hdr_and_idx(file, FALSE) < 0)
- HGOTO_ERROR(H5E_VFL, H5E_CANTLOAD, FAIL, "unable to load/decode md header and index")
- }
-
- /* Return tick_num */
- if(tick_ptr != NULL)
- *tick_ptr = file->md_header.tick_num;
-
- if(num_entries_ptr != NULL) {
- if(*num_entries_ptr >= file->md_index.num_entries && index != NULL)
- HDmemcpy(index, file->md_index.entries, (file->md_index.num_entries * sizeof(H5FD_vfd_swmr_idx_entry_t)));
-
- *num_entries_ptr = file->md_index.num_entries;
- }
-
-done:
- FUNC_LEAVE_NOAPI(ret_value)
-} /* H5FD_vfd_swmr_get_tick_and_idx() */
/*-------------------------------------------------------------------------
@@ -802,7 +810,7 @@ H5FD_vfd_swmr_load_hdr_and_idx(H5FD_t *_file, hbool_t open)
uint64_t nanosec = 1; /* # of nanoseconds to sleep between retries */
H5FD_vfd_swmr_md_header md_header; /* Metadata file header */
H5FD_vfd_swmr_md_index md_index; /* Metadata file index */
- H5FD_t *ret_value = FALSE; /* Return value */
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI_NOINIT
@@ -811,11 +819,11 @@ H5FD_vfd_swmr_load_hdr_and_idx(H5FD_t *_file, hbool_t open)
HDmemset(&md_index, 0, sizeof(H5FD_vfd_swmr_md_index));
/* Load and decode the header */
- if(H5FD_vfd_swmr_header_deserialize(file, &md_header) >= 0) {
+ if(H5FD_vfd_swmr_header_deserialize(_file, &md_header) >= 0) {
/* Error if header + index fit does not within md_pages_reserved */
if((H5FD_MD_HEADER_SIZE + md_header.index_length) >
- (file->md_pages_reserved * md_header.fs_page_size))
+ (uint64_t)((hsize_t)file->md_pages_reserved * md_header.fs_page_size))
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "header + index does not fit within md_pages_reserved")
if(!open) {
@@ -828,7 +836,7 @@ H5FD_vfd_swmr_load_hdr_and_idx(H5FD_t *_file, hbool_t open)
HDassert(md_header.tick_num > file->md_header.tick_num || open);
/* Load and decode the index */
- if(H5FD_vfd_swmr_index_deserialize(file, &md_index, &md_header) >= 0) {
+ if(H5FD_vfd_swmr_index_deserialize(_file, &md_index, &md_header) >= 0) {
/* tick_num is the same in both header and index */
if(md_header.tick_num == md_index.tick_num) {
@@ -891,7 +899,7 @@ H5FD_vfd_swmr_header_deserialize(H5FD_t *_file, H5FD_vfd_swmr_md_header *md_head
do {
/* Retrieve the metadata file size */
if(HDfstat(file->md_fd, &stat_buf))
- HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "unable to fstat the md file")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "unable to fstat the md file")
/* Verify file size is at least header size */
if(stat_buf.st_size >= H5FD_MD_HEADER_SIZE)
@@ -904,7 +912,7 @@ H5FD_vfd_swmr_header_deserialize(H5FD_t *_file, H5FD_vfd_swmr_md_header *md_head
/* Exhaust all retries for "stat" the md file */
if(file_retries == 0)
- HGOTO_ERROR(H5E_VFL, H5E_OPENERROR, NULL, "unable to the metadata file after all retry attempts")
+ HGOTO_ERROR(H5E_VFL, H5E_OPENERROR, FAIL, "unable to the metadata file after all retry attempts")
/* Try to get valid magic and checksum for header */
p = image;
@@ -991,10 +999,10 @@ H5FD_vfd_swmr_index_deserialize(H5FD_t *_file, H5FD_vfd_swmr_md_index *md_index,
do {
/* Retrieve the metadata file size */
if(HDfstat(file->md_fd, &stat_buf))
- HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "unable to fstat the md file")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "unable to fstat the md file")
/* Verify file size is at least header size */
- if(stat_buf.st_size >= (H5FD_MD_HEADER_SIZE + md_header->index_length))
+ if((uint64_t)stat_buf.st_size >= (H5FD_MD_HEADER_SIZE + md_header->index_length))
break;
/* Sleep and double the sleep time next time */
@@ -1011,7 +1019,7 @@ H5FD_vfd_swmr_index_deserialize(H5FD_t *_file, H5FD_vfd_swmr_md_index *md_index,
do {
if(HDlseek(file->md_fd, (HDoff_t)md_header->index_offset, SEEK_SET) < 0)
HGOTO_ERROR(H5E_VFL, H5E_SEEKERROR, FAIL, "unable to seek in metadata file")
- if(HDread(file->md_fd, image, md_header->index_length) < md_header->index_length)
+ if(HDread(file->md_fd, image, md_header->index_length) < (int64_t)md_header->index_length)
HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "error in reading the header in metadata file")
/* Verify valid magic for index */
@@ -1022,8 +1030,9 @@ H5FD_vfd_swmr_index_deserialize(H5FD_t *_file, H5FD_vfd_swmr_md_index *md_index,
if(stored_chksum == computed_chksum)
break;
}
+ /* Double the sleep time next time */
H5_nanosleep(nanosec);
- nanosec *= 2; /* Double the sleep time next time */
+ nanosec *= 2;
} while(--index_retries);
/* Exhaust all retries for loading the index */
@@ -1041,7 +1050,7 @@ H5FD_vfd_swmr_index_deserialize(H5FD_t *_file, H5FD_vfd_swmr_md_index *md_index,
if(md_index->num_entries) {
/* Allocate memory for index entries */
if(NULL == (md_index->entries = H5FL_SEQ_MALLOC(H5FD_vfd_swmr_idx_entry_t, md_index->num_entries)))
- HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "memory allocation failed for index entries")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "memory allocation failed for index entries")
/* Decode index entries */
for(i = 0; i < md_index->num_entries; i++) {
@@ -1051,8 +1060,6 @@ H5FD_vfd_swmr_index_deserialize(H5FD_t *_file, H5FD_vfd_swmr_md_index *md_index,
UINT32DECODE(p, md_index->entries[i].chksum);
} /* end for */
- /* MAY BE SORTED ALREADY: Sort entries in increasing hdf5_page_offset */
- //HDqsort(file->md_index.entries, num_entries, sizeof(H5FD_vfd_swmr_idx_entry_t), H5FD__idx_entry_cmp);
} /* end if */
/* Checksum is already valid */
@@ -1070,70 +1077,46 @@ done:
/*-------------------------------------------------------------------------
-* Function: H5FD_vfd_swmr_writer_end_of_tick
-*
-* Purpose: TBD
-*
-* Return: SUCCEED/FAIL
-*
-* Programmer:
-*
-*-------------------------------------------------------------------------
-*/
-herr_t
-H5FD_vfd_swmr_writer_end_of_tick(void)
-{
- herr_t ret_value = SUCCEED; /* Return value */
-
- FUNC_ENTER_NOAPI_NOINIT_NOERR
-
- FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5FD_vfd_swmr_writer_end_of_tick() */
-
-
-/*-------------------------------------------------------------------------
-* Function: H5FD_vfd_swmr_reader_end_of_tick
-*
-* Purpose: TBD
-*
-* Return: SUCCEED/FAIL
-*
-*-------------------------------------------------------------------------
-*/
+ * Function: H5FD_vfd_swmr_get_tick_and_idx()
+ *
+ * Purpose: Retrieve tick_num, num_entries and index from the metadata file
+ * --If the parameter "reload_hdr_and_index" is true, load and decode
+ * the header and index via H5FD_vfd_swmr_load_hdr_and_idx(), which
+ * may replace the VFD's local copies of header and index with the
+ * latest info read.
+ * --Return tick_num, num_entries and index from the VFD's local copies.
+ *
+ * Return: Success: SUCCEED
+ * Failure: FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
herr_t
-H5FD_vfd_swmr_reader_end_of_tick(void)
+H5FD_vfd_swmr_get_tick_and_idx(H5FD_t *_file, hbool_t reload_hdr_and_index,
+ uint64_t *tick_ptr, uint32_t *num_entries_ptr, H5FD_vfd_swmr_idx_entry_t index[])
{
- herr_t ret_value = SUCCEED; /* Return value */
-
- FUNC_ENTER_NOAPI_NOINIT_NOERR
-
- FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5FD_vfd_swmr_reader_end_of_tick() */
-
+ H5FD_vfd_swmr_t *file = (H5FD_vfd_swmr_t *)_file; /* VFD SWMR file struct */
+ herr_t ret_value = SUCCEED; /* Return value */
-/* Used by HDqsort: keep this for now */
-#ifdef TEMP
-static int
-H5FD__idx_entry_cmp(const void *_entry1, const void *_entry2)
-{
- const H5FD_vfd_swmr_idx_entry_t *entry1 = (const H5FD_vfd_swmr_idx_entry_t *)_entry1;
- const H5FD_vfd_swmr_idx_entry_t *entry2 = (const H5FD_vfd_swmr_idx_entry_t *)_entry2;
+ FUNC_ENTER_NOAPI_NOINIT
- int ret_value = 0; /* Return value */
+ /* Load and decode the header and index as indicated */
+ if(reload_hdr_and_index) {
+ if(H5FD_vfd_swmr_load_hdr_and_idx(_file, FALSE) < 0)
+ HGOTO_ERROR(H5E_VFL, H5E_CANTLOAD, FAIL, "unable to load/decode md header and index")
+ }
- FUNC_ENTER_STATIC_NOERR
+ /* Return tick_num */
+ if(tick_ptr != NULL)
+ *tick_ptr = file->md_header.tick_num;
- /* Sanity checks */
- HDassert(entry1);
- HDassert(entry2);
+ if(num_entries_ptr != NULL) {
+ if(*num_entries_ptr >= file->md_index.num_entries && index != NULL)
+ HDmemcpy(index, file->md_index.entries, (file->md_index.num_entries * sizeof(H5FD_vfd_swmr_idx_entry_t)));
- if(entry1->hdf5_page_offset < entry2->hdf5_page_offset)
- ret_value = -1;
- else if(entry1->hdf5_page_offset > entry2->hdf5_page_offset)
- ret_value = 1;
- else
- ret_value = 0;
+ *num_entries_ptr = file->md_index.num_entries;
+ }
+done:
FUNC_LEAVE_NOAPI(ret_value)
-} /* H5FD__idx_entry_cmp() */
-#endif
+} /* H5FD_vfd_swmr_get_tick_and_idx() */
diff --git a/src/H5FDvfd_swmr.h b/src/H5FDvfd_swmr.h
index d317a18..3e4a1ce 100644
--- a/src/H5FDvfd_swmr.h
+++ b/src/H5FDvfd_swmr.h
@@ -27,6 +27,7 @@ extern "C" {
#endif
H5_DLL hid_t H5FD_vfd_swmr_init(void);
+H5_DLL herr_t H5Pset_fapl_vfd_swmr(hid_t fapl_id);
#ifdef __cplusplus
}
diff --git a/src/H5FSprivate.h b/src/H5FSprivate.h
index de1b4c1..374576e 100644
--- a/src/H5FSprivate.h
+++ b/src/H5FSprivate.h
@@ -133,6 +133,7 @@ struct H5FS_section_info_t {
typedef enum H5FS_client_t {
H5FS_CLIENT_FHEAP_ID = 0, /* Free space is used by fractal heap */
H5FS_CLIENT_FILE_ID, /* Free space is used by file */
+ H5FS_CLIENT_MD_VFD_ID, /* Free space is used by the metadata file for VFD SWMR */
H5FS_NUM_CLIENT_ID /* Number of free space client IDs (must be last) */
} H5FS_client_t;
diff --git a/src/H5Fint.c b/src/H5Fint.c
index 17c494f..248d7a4 100644
--- a/src/H5Fint.c
+++ b/src/H5Fint.c
@@ -33,6 +33,7 @@
#include "H5Iprivate.h" /* IDs */
#include "H5Lprivate.h" /* Links */
#include "H5MFprivate.h" /* File memory management */
+#include "H5MVprivate.h" /* File memory management for VFD SWMR */
#include "H5MMprivate.h" /* Memory management */
#include "H5Pprivate.h" /* Property lists */
#include "H5SMprivate.h" /* Shared Object Header Messages */
@@ -43,6 +44,41 @@
/* Local Macros */
/****************/
+/* VFD SWMR */
+/* Append entry to the delayed free spaced release linked list */
+#define H5F_DC_APPEND(entry_ptr, head_ptr, tail_ptr, len) \
+{ \
+ if((head_ptr) == NULL) { \
+ (head_ptr) = (entry_ptr); \
+ (tail_ptr) = (entry_ptr); \
+ } else { \
+ (tail_ptr)->next = (entry_ptr); \
+ (entry_ptr)->prev = (tail_ptr); \
+ (tail_ptr) = (entry_ptr); \
+ } \
+ (len)++; \
+} /* H5F_DC_APPEND() */
+
+/* Remove entry from delayed free spaced release linked list */
+#define H5F_DC_REMOVE(entry_ptr, head_ptr, tail_ptr, len) \
+{ \
+ if((head_ptr) == (entry_ptr)) { \
+ (head_ptr) = (entry_ptr)->next; \
+ if((head_ptr) != NULL ) \
+ (head_ptr)->prev = NULL; \
+ } else \
+ (entry_ptr)->prev->next = (entry_ptr)->next; \
+ if((tail_ptr) == (entry_ptr)) { \
+ (tail_ptr) = (entry_ptr)->prev; \
+ if((tail_ptr) != NULL) \
+ (tail_ptr)->next = NULL; \
+ } else \
+ (entry_ptr)->next->prev = (entry_ptr)->prev; \
+ entry_ptr->next = NULL; \
+ entry_ptr->prev = NULL; \
+ (len)--; \
+} /* H5F_DC_REMOVE() */
+
/******************/
/* Local Typedefs */
/******************/
@@ -82,15 +118,25 @@ static herr_t H5F__flush_phase1(H5F_t *f);
static herr_t H5F__flush_phase2(H5F_t *f, hbool_t closing);
/* VFD SWMR */
-static herr_t H5F__vfd_swmr_init_info(H5F_t *f);
-static herr_t H5F__vfd_swmr_init_md(H5F_t *f);
-static herr_t H5F__vfd_swmr_close_md(H5F_t *f);
+static herr_t H5F__vfd_swmr_init(H5F_t *f, hbool_t file_create);
+static herr_t H5F__vfd_swmr_close_or_flush(H5F_t *f, hbool_t closing);
+static herr_t H5F__vfd_swmr_update_end_of_tick(H5F_t *f);
+static herr_t H5F__vfd_swmr_construct_write_md_hdr(H5F_t *f, uint32_t index_len);
+static herr_t H5F__vfd_swmr_construct_write_md_idx(H5F_t *f, uint32_t index_len, struct H5FD_vfd_swmr_idx_entry_t index[]);
+static herr_t H5F__idx_entry_cmp(const void *_entry1, const void *_entry2);
/*********************/
/* Package Variables */
/*********************/
+/* VFD SWMR globals */
+H5F_t *vfd_swmr_file_g; /* Points to the file struct */
+hbool_t vfd_swmr_g = FALSE; /* Is this a VFD SWMR configured file */
+hbool_t vfd_swmr_writer_g = FALSE; /* Is this the VFD SWMR writer */
+uint64_t tick_num_g = 0; /* The current tick_num */
+struct timespec end_of_tick_g; /* The current end_of_tick */
+
/*****************************/
/* Library Private Variables */
@@ -107,6 +153,10 @@ H5FL_DEFINE(H5F_t);
/* Declare a free list to manage the H5F_file_t struct */
H5FL_DEFINE(H5F_file_t);
+/* Declare a free list to manage the H5F_vfd_swmr_dl_entry_t struct */
+H5FL_DEFINE(H5F_vfd_swmr_dl_entry_t);
+
+
/*-------------------------------------------------------------------------
@@ -1046,7 +1096,13 @@ H5F__new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_
/* Get VFD SWMR configuration */
if(H5P_get(plist, H5F_ACS_VFD_SWMR_CONFIG_NAME, &(f->shared->vfd_swmr_config)) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get VFD SWMR config info")
+
+ /* Initialization for VFD SWMR */
f->shared->vfd_swmr_md_fd = -1;
+ f->shared->fs_man_md = NULL;
+ f->shared->dl_head_ptr = NULL;
+ f->shared->dl_tail_ptr = NULL;
+ f->shared->dl_len = 0;
/* Create a metadata cache with the specified number of elements.
* The cache might be created with a different number of elements and
@@ -1308,9 +1364,9 @@ H5F__dest(H5F_t *f, hbool_t flush)
/* Push error, but keep going*/
HDONE_ERROR(H5E_FILE, H5E_CANTDEC, FAIL, "can't close property list")
- /* Handle closing for the metadata file (VFD SWMR writer) */
+ /* VFD SWMR: closing down */
if(H5F_ACC_RDWR & H5F_INTENT(f) && f->shared->vfd_swmr_md_fd >= 0) {
- if(H5F__vfd_swmr_close_md(f) < 0)
+ if(H5F__vfd_swmr_close_or_flush(f, TRUE) < 0)
HDONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "unable to close the metadata file")
}
@@ -1664,30 +1720,16 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id)
} /* end if */
+ /* Checked if configured for VFD SWMR */
if(H5F_VFD_SWMR_CONFIG(file)) {
/* Page buffering and page allocation strategy have to be enabled */
if(!page_buf_size || !H5F_PAGED_AGGR(file))
HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "file open fail with VFD SWMR writer")
- if(1 == shared->nrefs) {
- /* Initialize globals for VFD SWMR writer and reader */
- if(H5F__vfd_swmr_init_info(file) < 0)
+ /* Initialization for VFD SWMR writer and reader */
+ if(1 == shared->nrefs) {
+ if(H5F__vfd_swmr_init(file, file_create) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTSET, NULL, "file open fail with initialization for VFD SWMR")
-
- /* For VFD SWMR writer */
- if(H5F_INTENT(file) & H5F_ACC_RDWR) {
- HDassert(file->shared->vfd_swmr_config.vfd_swmr_writer);
-
- /* Create the metadata file */
- if(((file->shared->vfd_swmr_md_fd = HDopen(file->shared->vfd_swmr_config.md_file_path, O_CREAT|O_RDWR, H5_POSIX_CREATE_MODE_RW))) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to create the metadata file")
-
- /* Create header and empty index in the metadata file */
- if(!file_create) {
- if(H5F__vfd_swmr_init_md(file) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_CANTSET, NULL, "fail to initialize md for VFD SWMR writer")
- }
- }
}
}
@@ -1755,7 +1797,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id)
} /* version 3 superblock */
file->shared->sblock->status_flags |= H5F_SUPER_WRITE_ACCESS;
- if(H5F_INTENT(file) & H5F_ACC_SWMR_WRITE)
+ if(H5F_INTENT(file) & H5F_ACC_SWMR_WRITE || H5F_USE_VFD_SWMR(file))
file->shared->sblock->status_flags |= H5F_SUPER_SWMR_WRITE_ACCESS;
/* Flush the superblock & superblock extension */
@@ -1767,7 +1809,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id)
HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, NULL, "unable to flush superblock extension")
/* Remove the file lock for SWMR_WRITE */
- if(use_file_locking && (H5F_INTENT(file) & H5F_ACC_SWMR_WRITE)) {
+ if(use_file_locking && ((H5F_INTENT(file) & H5F_ACC_SWMR_WRITE) || H5F_USE_VFD_SWMR(file))) {
if(H5FD_unlock(file->shared->lf) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to unlock the file")
} /* end if */
@@ -1775,7 +1817,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id)
else { /* H5F_ACC_RDONLY: check consistency of status_flags */
/* Skip check of status_flags for file with < superblock version 3 */
if(file->shared->sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_3) {
- if(H5F_INTENT(file) & H5F_ACC_SWMR_READ) {
+ if(H5F_INTENT(file) & H5F_ACC_SWMR_READ || H5F_USE_VFD_SWMR(file)) {
if((file->shared->sblock->status_flags & H5F_SUPER_WRITE_ACCESS &&
!(file->shared->sblock->status_flags & H5F_SUPER_SWMR_WRITE_ACCESS))
||
@@ -1945,6 +1987,14 @@ H5F__flush(H5F_t *f)
/* Push error, but keep going*/
HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush file data")
+ /* VFD SWMR when flushing the HDF5 file */
+ if(f->shared->nrefs == 1 && f->shared->vfd_swmr_writer && f->shared->vfd_swmr_md_fd >= 0) {
+ HDassert(H5F_ACC_RDWR & H5F_INTENT(f));
+
+ if(H5F__vfd_swmr_close_or_flush(f, FALSE) < 0)
+ HDONE_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "unable to encode and write to the metadata file")
+ }
+
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5F__flush() */
@@ -3524,23 +3574,27 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5F__format_convert() */
+/*
+ * VFD SWMR
+ */
+
/*-------------------------------------------------------------------------
- * Function: H5F__vfd_swmr_init_info
+ * Function: H5F__vfd_swmr_init
*
* Purpose: Initialize globals and the corresponding fields in file pointer.
- * For VFD SWMR writer:
+ * For both:
* --set vfd_swmr_g to TRUE
+ * --set vfd_swmr_file_g to f
+ * --set end_of_tick to the current time + tick length
+ * For VFD SWMR writer:
* --set vfd_swmr_writer_g to TRUE
* --set tick_num_g to 0
- * --set end_of_tick_g to the current time + tick length
- * --set vfd_swmr_file_g to f->shared
+ * --create the metadata file
+ * --when opening an existing HDF5 file, write header and empty index in the metadata file
* For VFD SWMR reader:
- * --set vfd_swmr_g to TRUE
* --set vfd_swmr_writer_g to FALSE
* --set tick_num_g to the current tick read from the metadata file
- * --set end_of_tick_g to the current time + tick length
- * --set vfd_swmr_file_g to f->shared
*
* Return: Success: SUCCEED
* Failure: FAIL
@@ -3548,17 +3602,46 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
-H5F__vfd_swmr_init_info(H5F_t *f)
+H5F__vfd_swmr_init(H5F_t *f, hbool_t file_create)
{
- struct timespec tmp_end_of_tick;
+ hsize_t md_size; /* Size of the metadata file */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_STATIC
- if(f->shared->vfd_swmr_config.vfd_swmr_writer) {
+ HDassert(H5F_VFD_SWMR_CONFIG(f));
+
+ vfd_swmr_g = f->shared->vfd_swmr = TRUE;
+ vfd_swmr_file_g = f;
+
+ if(H5F_INTENT(f) & H5F_ACC_RDWR) {
+ HDassert(f->shared->vfd_swmr_config.vfd_swmr_writer);
+
vfd_swmr_writer_g = f->shared->vfd_swmr_writer = TRUE;
tick_num_g = f->shared->tick_num = 0;
- } else {
+
+ /* Create the metadata file */
+ if(((f->shared->vfd_swmr_md_fd = HDopen(f->shared->vfd_swmr_config.md_file_path, O_CREAT|O_RDWR, H5_POSIX_CREATE_MODE_RW))) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "unable to create the metadata file")
+
+ md_size = (hsize_t)f->shared->vfd_swmr_config.md_pages_reserved * f->shared->fs_page_size;
+
+ /* Set the metadata file size to md_pages_reserved */
+ if(-1 == HDftruncate(f->shared->vfd_swmr_md_fd, (HDoff_t)md_size))
+ HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "truncate fail for the metadata file")
+
+ /* Set eof for metadata file to md_pages_reserved */
+ f->shared->vfd_swmr_md_eoa = (haddr_t)md_size;
+
+ /* When opening an existing HDF5 file, create header and empty index in the metadata file */
+ if(!file_create) {
+ if(H5F__vfd_swmr_construct_write_md_hdr(f, 0) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "fail to create header in md")
+ if(H5F__vfd_swmr_construct_write_md_idx(f, 0, NULL) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "fail to create index in md")
+ }
+
+ } else { /* VFD SWMR reader */
HDassert(!f->shared->vfd_swmr_config.vfd_swmr_writer);
vfd_swmr_writer_g = f->shared->vfd_swmr_writer = FALSE;
@@ -3568,32 +3651,23 @@ H5F__vfd_swmr_init_info(H5F_t *f)
f->shared->tick_num = tick_num_g;
}
- /* Get current time */
- if(HDclock_gettime(CLOCK_MONOTONIC, &tmp_end_of_tick) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get time via clock_gettime")
-
- /* Increment by tick length */
- tmp_end_of_tick.tv_nsec += f->shared->vfd_swmr_config.tick_len * 100000000;
- tmp_end_of_tick.tv_sec += tmp_end_of_tick.tv_nsec / 1000000000;
- tmp_end_of_tick.tv_nsec = tmp_end_of_tick.tv_nsec % 1000000000;
- HDmemcpy(&end_of_tick_g, &tmp_end_of_tick, sizeof(struct timespec));
- HDmemcpy(&f->shared->end_of_tick, &tmp_end_of_tick, sizeof(struct timespec));
-
- vfd_swmr_g = f->shared->vfd_swmr = TRUE;
- vfd_swmr_file_g = f->shared;
+ /* Update end_of_tick */
+ if(H5F__vfd_swmr_update_end_of_tick(f) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to update end of tick")
done:
FUNC_LEAVE_NOAPI(ret_value)
-} /* H5F__vfd_swmr_init_info() */
+} /* H5F__vfd_swmr_init() */
-
/*-------------------------------------------------------------------------
- * Function: H5F__vfd_swmr_init_md
+ * Function: H5F__vfd_swmr_construct_write_md_hdr
*
- * Purpose: Encode the header and an empty index to the metadata file.
- * This is used by the VFD SWMR writer when:
- * --opening an existing HDF5 file
- * --the HDF5 file is flushed and about to close
+ * Purpose: Encode and write header to the metadata file.
+ * This is used by the VFD SWMR writer:
+ * --when opening an existing HDF5 file
+ * --when closing the HDF5 file
+ * --after flushing an HDF5 file
+ * --when updating the metadata file
*
* Return: Success: SUCCEED
* Failure: FAIL
@@ -3601,21 +3675,20 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
-H5F__vfd_swmr_init_md(H5F_t *f)
+H5F__vfd_swmr_construct_write_md_hdr(H5F_t *f, uint32_t index_len)
{
- uint8_t *p = NULL; /* Pointer to buffer */
- uint8_t *pp = NULL; /* Pointer to buffer for index */
- uint32_t metadata_chksum; /* Computed metadata checksum value */
- uint8_t image[H5FD_MD_HEADER_SIZE + H5FD_MD_EMPTY_INDEX_SIZE]; /* Buffer for header and empty index */
- unsigned hdr_idx_size = H5FD_MD_HEADER_SIZE + H5FD_MD_EMPTY_INDEX_SIZE; /* Size of header and index */
- herr_t ret_value = SUCCEED; /* Return value */
+ uint8_t image[H5FD_MD_HEADER_SIZE]; /* Buffer for header */
+ uint8_t *p = NULL; /* Pointer to buffer */
+ uint32_t metadata_chksum; /* Computed metadata checksum value */
+ unsigned hdr_size = H5FD_MD_HEADER_SIZE; /* Size of header and index */
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_STATIC
/*
* Encode metadata file header
*/
- p = image;
+ p = image;
/* Encode magic for header */
HDmemcpy(p, H5FD_MD_HEADER_MAGIC, (size_t)H5_SIZEOF_MAGIC);
@@ -3624,8 +3697,8 @@ H5F__vfd_swmr_init_md(H5F_t *f)
/* Encode page size, tick number, index offset, index length */
UINT32ENCODE(p, f->shared->fs_page_size);
UINT64ENCODE(p, f->shared->tick_num);
- UINT64ENCODE(p, H5FD_MD_HEADER_SIZE);
- UINT64ENCODE(p, H5FD_MD_INDEX_SIZE(0));
+ UINT64ENCODE(p, hdr_size);
+ UINT64ENCODE(p, H5FD_MD_INDEX_SIZE(index_len));
/* Calculate checksum for header */
metadata_chksum = H5_checksum_metadata(image, (size_t)(p - image), 0);
@@ -3634,12 +3707,57 @@ H5F__vfd_swmr_init_md(H5F_t *f)
UINT32ENCODE(p, metadata_chksum);
/* Sanity checks on header */
- HDassert((size_t)(p - image == H5FD_MD_HEADER_SIZE));
+ HDassert((size_t)(p - image == hdr_size));
+
+ /* Set to beginning of the file */
+ if(HDlseek(f->shared->vfd_swmr_md_fd, (HDoff_t)0, SEEK_SET) < 0)
+ HGOTO_ERROR(H5E_VFL, H5E_SEEKERROR, FAIL, "unable to seek in metadata file")
+
+ /* Write header to the metadata file */
+ if(HDwrite(f->shared->vfd_swmr_md_fd, image, hdr_size) != hdr_size)
+ HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in writing header to metadata file")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5F__vfd_swmr_construct_write_md_hdr() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5F__vfd_swmr_construct_write_md_idx
+ *
+ * Purpose: Encode and write index to the metadata file.
+ * This is used by the VFD SWMR writer:
+ * --when opening an existing HDF5 file
+ * --when closing the HDF5 file
+ * --after flushing an HDF5 file
+ * --when updating the metadata file
+ *
+ * Return: Success: SUCCEED
+ * Failure: FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5F__vfd_swmr_construct_write_md_idx(H5F_t *f, uint32_t index_len, struct H5FD_vfd_swmr_idx_entry_t index[])
+{
+ uint8_t *image = NULL; /* Pointer to buffer */
+ uint8_t *p = NULL; /* Pointer to buffer */
+ uint32_t metadata_chksum; /* Computed metadata checksum value */
+ unsigned idx_size = H5FD_MD_INDEX_SIZE(index_len); /* Size of index */
+ unsigned i; /* Local index variable */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_STATIC
+
+ HDassert((index_len != 0 && index != NULL) || (index_len == 0 && index == NULL));
+
+ /* Allocate space for the buffer to hold the index */
+ if((image = (uint8_t *)HDmalloc(idx_size)) == NULL)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for md index")
/*
* Encode metadata file index
*/
- pp = p;
+ p = image;
/* Encode magic for index */
HDmemcpy(p, H5FD_MD_INDEX_MAGIC, (size_t)H5_SIZEOF_MAGIC);
@@ -3648,42 +3766,96 @@ H5F__vfd_swmr_init_md(H5F_t *f)
/* Encode tick number */
UINT64ENCODE(p, f->shared->tick_num);
- /* Encode zero number of entries in index */
- UINT32ENCODE(p, 0);
+ /* Encode number of entries in index */
+ UINT32ENCODE(p, index_len);
+
+ /* Encode the index entries */
+ for(i = 0; i < index_len; i++) {
+ UINT32ENCODE(p, index[i].hdf5_page_offset);
+ UINT32ENCODE(p, index[i].md_file_page_offset);
+ UINT32ENCODE(p, index[i].length);
+ UINT32ENCODE(p, index[i].chksum);
+ }
/* Calculate checksum for index */
- metadata_chksum = H5_checksum_metadata(pp, (size_t)(p - pp), 0);
+ metadata_chksum = H5_checksum_metadata(image, (size_t)(p - image), 0);
/* Encode checksum for index */
UINT32ENCODE(p, metadata_chksum);
/* Sanity checks on index */
- HDassert((size_t)(p - pp == H5FD_MD_EMPTY_INDEX_SIZE));
+ HDassert((size_t)(p - image == idx_size));
/* Verify the md file descriptor exists */
HDassert(f->shared->vfd_swmr_md_fd >= 0);
- /* Set to beginning of the file */
- if(HDlseek(f->shared->vfd_swmr_md_fd, (HDoff_t)0, SEEK_SET) < 0)
+ /* Set to right after the header */
+ if(HDlseek(f->shared->vfd_swmr_md_fd, (HDoff_t)H5FD_MD_HEADER_SIZE, SEEK_SET) < 0)
HGOTO_ERROR(H5E_VFL, H5E_SEEKERROR, FAIL, "unable to seek in metadata file")
- /* Write header and empty index to the metadata file */
- if(HDwrite(f->shared->vfd_swmr_md_fd, image, hdr_idx_size) != hdr_idx_size)
- HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in writing header and index to metadata file")
+ /* Write index to the metadata file */
+ if(HDwrite(f->shared->vfd_swmr_md_fd, image, idx_size) != idx_size)
+ HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in writing index to metadata file")
done:
+ if(image)
+ HDfree(image);
FUNC_LEAVE_NOAPI(ret_value)
-} /* H5F__vfd_swmr_init_md() */
+} /* H5F__vfd_swmr_construct_write_idx() */
+
/*-------------------------------------------------------------------------
- * Function: H5F__vfd_swmr_close_md
+ * Function: H5F__vfd_swmr_update_end_of_tick
+ *
+ * Purpose: Set end_of_tick to the current time + tick length:
+ * --end_of_tick_g
+ * --f->shared->end_of_tick
+ *
+ * Return: Success: SUCCEED
+ * Failure: FAIL
*
- * Purpose: This is used by the VFD SWMR writer when the HDF5 file is
- * flushed and about to close:
- * --increment the tick
- * --write header and an empty index to the metadata file
- * --close and unlink the metadata file
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5F__vfd_swmr_update_end_of_tick(H5F_t *f)
+{
+ struct timespec tmp_end_of_tick; /* end_of_tick */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_STATIC
+
+ /* Get current time */
+ if(HDclock_gettime(CLOCK_MONOTONIC, &tmp_end_of_tick) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get time via clock_gettime")
+
+ /* Increment by tick length */
+ tmp_end_of_tick.tv_nsec += f->shared->vfd_swmr_config.tick_len * 100000000;
+ tmp_end_of_tick.tv_sec += tmp_end_of_tick.tv_nsec / 1000000000;
+ tmp_end_of_tick.tv_nsec = tmp_end_of_tick.tv_nsec % 1000000000;
+ HDmemcpy(&end_of_tick_g, &tmp_end_of_tick, sizeof(struct timespec));
+ HDmemcpy(&f->shared->end_of_tick, &tmp_end_of_tick, sizeof(struct timespec));
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5F__vfd_swmr_update_end_of_tick() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5F__vfd_swmr_close_or_flush
+ *
+ * Purpose: Used by the VFD SWMR writer when the HDF5 file is closed or flushed:
+ * 1) For file close:
+ * --write header and an empty index to the metadata file
+ * --increment tick_num
+ * --close the metadata file
+ * --unlink the metadata file
+ * --close the free-space manager for the metadata file
+ * 2) For file flush:
+ * --write header and an empty index to the metadata file
+ * --increment tick_num
+ * --start a new tick (??check with JM for sure)
+ * ??update end_of_tick
*
* Return: Success: SUCCEED
* Failure: FAIL
@@ -3691,30 +3863,243 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
-H5F__vfd_swmr_close_md(H5F_t *f)
+H5F__vfd_swmr_close_or_flush(H5F_t *f, hbool_t closing)
{
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_STATIC
+ HDassert(f->shared->vfd_swmr_writer);
HDassert(f->shared->vfd_swmr_md_fd >= 0);
+ /* Write empty index to the md file */
+ if(H5F__vfd_swmr_construct_write_md_idx(f, 0, NULL) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "fail to create index in md")
+ /* Write header to the md file */
+ if(H5F__vfd_swmr_construct_write_md_hdr(f, 0) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "fail to create header in md")
+
/* Increment tick_num */
tick_num_g = ++f->shared->tick_num;
- /* Write an empty header and index to the md file */
- if(H5F__vfd_swmr_init_md(f) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "unable to encode and write to the metadata file")
+ if(closing) { /* For file close */
+ /* Close the md file */
+ if(HDclose(f->shared->vfd_swmr_md_fd) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "unable to close the metadata file")
+ f->shared->vfd_swmr_md_fd = -1;
+
+ /* Unlink the md file */
+ if(HDunlink(f->shared->vfd_swmr_config.md_file_path) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTREMOVE, FAIL, "unable to unlink the metadata file")
+
+ /* Close the free-space manager for the metadata file */
+ if(H5MV_close(f) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "unable to close the free-space manager for the metadata file")
+
+ vfd_swmr_file_g = NULL;
+
+ } else { /* For file flush */
+ /* Update end_of_tick */
+ if(H5F__vfd_swmr_update_end_of_tick(f) < 0)
+ HDONE_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to update end of tick")
+ }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5F__vfd_swmr_close_or_flush() */
+
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5F__idx_entry_cmp()
+ *
+ * Purpose: Callback used by HDqsort to sort entries in the index
+ *
+ * Return: 0 if the entries are the same
+ * -1 if entry1's offset is less than that of entry2
+ * 1 if entry1's offset is greater than that of entry2
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5F__idx_entry_cmp(const void *_entry1, const void *_entry2)
+{
+ const H5FD_vfd_swmr_idx_entry_t *entry1 = (const H5FD_vfd_swmr_idx_entry_t *)_entry1;
+ const H5FD_vfd_swmr_idx_entry_t *entry2 = (const H5FD_vfd_swmr_idx_entry_t *)_entry2;
+
+ int ret_value = 0; /* Return value */
+
+ FUNC_ENTER_STATIC_NOERR
+
+ /* Sanity checks */
+ HDassert(entry1);
+ HDassert(entry2);
+
+ if(entry1->hdf5_page_offset < entry2->hdf5_page_offset)
+ ret_value = -1;
+ else if(entry1->hdf5_page_offset > entry2->hdf5_page_offset)
+ ret_value = 1;
- /* Close the md file */
- if(HDclose(f->shared->vfd_swmr_md_fd) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "unable to close the metadata file")
- f->shared->vfd_swmr_md_fd = -1;
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5F__idx_entry_cmp() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5F_update_vfd_swmr_metadata_file()
+ *
+ * Purpose: Updating the metadata file
+ * --Sort index
+ * --For each non-null entry_ptr in the index:
+ * --Insert previous image of the entry onto the delayed list
+ * --Allocate space for the entry in the metadata file
+ * --Compute checksum
+ * --Update index
+ * --Write the entry to the metadata file
+ * --Set entry_ptr to NULL
+ * --Construct on disk image of the index and write index to the metadata file
+ * --Construct on disk image of the header and write header to the metadata file
+ * --Release time out space from the delayed list to the free-space manager
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5F_update_vfd_swmr_metadata_file(H5F_t *f, uint32_t index_len, struct H5FD_vfd_swmr_idx_entry_t index[])
+{
+ H5F_vfd_swmr_dl_entry_t *prev; /* Points to the delayed list */
+ H5F_vfd_swmr_dl_entry_t *dl_entry;
+ haddr_t md_addr;
+ unsigned i; /* Local index variable */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+
+ /* Sort index by increasing offset in the HDF5 file */
+ HDqsort(index, index_len, sizeof(H5FD_vfd_swmr_idx_entry_t), H5F__idx_entry_cmp);
- /* Unlink the md file */
- if(HDunlink(f->shared->vfd_swmr_config.md_file_path) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_CANTREMOVE, FAIL, "unable to unlink the metadata file")
+ /* For each non-null entry_ptr in the index:
+ * --If exists, insert previous image of the entry to the delayed list
+ * --Allocate space for the entry in the metadata file
+ * --Compute checksum, update index, write entry to the metadata file
+ * --Set entry_ptr to NULL
+ */
+ for(i = 0; i < index_len; i++) {
+ if(index[i].entry_ptr != NULL) {
+ /* Append previous image of the entry to the delayed list */
+ if(index[i].md_file_page_offset) {
+ if(NULL == (dl_entry = H5FL_CALLOC(H5F_vfd_swmr_dl_entry_t)))
+ HGOTO_ERROR(H5E_FILE, H5E_CANTALLOC, FAIL, "unable to allocate the delayed entry")
+ dl_entry->hdf5_page_offset = index[i].hdf5_page_offset;
+ dl_entry->md_file_page_offset = index[i].md_file_page_offset;
+ dl_entry->length = index[i].length;
+ dl_entry->tick_num = f->shared->tick_num;
+ H5F_DC_APPEND(dl_entry, f->shared->dl_head_ptr, f->shared->dl_tail_ptr, f->shared->dl_len);
+ }
+
+ /* Allocate space for the entry in the metadata file */
+ if((md_addr = H5MV_alloc(f, index[i].length)) == HADDR_UNDEF)
+ HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in allocating space from the metadata file")
+ /* Compute checksum and update the index */
+ index[i].md_file_page_offset = md_addr/f->shared->fs_page_size;
+ index[i].chksum = H5_checksum_metadata(index[i].entry_ptr, (size_t)(index[i].length), 0);
+
+ /* Seek and write the entry to the metadata file */
+ if(HDlseek(f->shared->vfd_swmr_md_fd, (HDoff_t)md_addr, SEEK_SET) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_SEEKERROR, FAIL, "unable to seek in the metadata file")
+ if(HDwrite(f->shared->vfd_swmr_md_fd, index[i].entry_ptr, index[i].length) != index[i].length)
+ HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in writing the page/multi-page entry to metadata file")
+
+ /* Set entry_ptr to NULL */
+ index[i].entry_ptr = NULL;
+ } /* end if */
+
+ } /* end for */
+
+ /* Construct and write index to the md file */
+ if(H5F__vfd_swmr_construct_write_md_idx(f, index_len, index) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "fail to construct & write index in md")
+
+ /* Construct and write header to the md file */
+ if(H5F__vfd_swmr_construct_write_md_hdr(f, index_len) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "fail to construct & write header in md")
+
+ /*
+ * Release time out entry from the delayed list:
+ * Scan the delayed list from the bottom up:
+ * --release to the metadata file free space manager all entries that have
+ * resided on the list for more than max_lag ticks
+ * --remove the associated entries from the list
+ */
+ dl_entry = f->shared->dl_tail_ptr;
+ while(dl_entry != NULL) {
+
+ prev = dl_entry->prev;
+ /* max_lag is at least 3 */
+ if((int)dl_entry->tick_num <= ((int)f->shared->tick_num - f->shared->vfd_swmr_config.max_lag)) {
+ if(H5MV_free(f, dl_entry->md_file_page_offset * f->shared->fs_page_size, dl_entry->length) < 0)
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush clean entry")
+
+ /* Remove the entry from the delayed list */
+ H5F_DC_REMOVE(dl_entry, f->shared->dl_head_ptr, f->shared->dl_tail_ptr, f->shared->dl_len)
+
+ /* Free the delayed entry struct */
+ H5FL_FREE(H5F_vfd_swmr_dl_entry_t, dl_entry);
+ }
+ /* DO I break from it once it is false ?? */
+ dl_entry = prev;
+ } /* end while */
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5F_update_vfd_swmr_metadata_file() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5F_vfd_swmr_writer_end_of_tick
+ *
+ * Purpose: Dummy right now
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5F_vfd_swmr_writer_end_of_tick(void)
+{
+ H5FD_vfd_swmr_idx_entry_t *index;
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+
+ /* construct */
+ if(vfd_swmr_file_g) {
+ ;
+ }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5F_vfd_swmr_writer_end_of_tick() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5F_vfd_swmr_reader_end_of_tick
+ *
+ * Purpose: Dummy right now
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5F_vfd_swmr_reader_end_of_tick(void)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+
+ /* construct */
+ if(vfd_swmr_file_g) {
+ ;
+ }
done:
FUNC_LEAVE_NOAPI(ret_value)
-} /* H5F__vfd_swmr_close_md() */
+} /* end H5F_vfd_swmr_reader_end_of_tick() */
diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h
index 7e04f35..2a5847e 100644
--- a/src/H5Fpkg.h
+++ b/src/H5Fpkg.h
@@ -215,6 +215,28 @@ typedef struct H5F_mtab_t {
H5F_mount_t *child; /* An array of mount records */
} H5F_mtab_t;
+/*
+ * VFD SWMR: Entry for the delayed free space release doubly linked list
+ *
+ * md_file_page_offset: Unsigned 64-bit value containing the base address of the
+ * metadata page, or multi page metadata entry in the metadata
+ * file IN PAGES.
+ * To obtain byte offset, multiply this value by the page size.
+ * length: The length of the metadata page or multi page metadata entry
+ * in BYTES.
+ * tick_num: Sequence # of the current tick
+ * next: Pointer to the next entry in the the list
+ * prev: Pointer to the previous entry in the list
+ */
+typedef struct H5F_vfd_swmr_dl_entry_t {
+ uint64_t hdf5_page_offset;
+ uint64_t md_file_page_offset;
+ uint32_t length;
+ uint64_t tick_num;
+ struct H5F_vfd_swmr_dl_entry_t *next;
+ struct H5F_vfd_swmr_dl_entry_t *prev;
+} H5F_vfd_swmr_dl_entry_t;
+
/* Structure specifically to store superblock. This was originally
* maintained entirely within H5F_file_t, but is now extracted
* here because the superblock is now handled by the cache */
@@ -354,16 +376,32 @@ struct H5F_file_t {
/* Object flush info */
H5F_object_flush_t object_flush; /* Information for object flush callback */
- /* VFD SWMR configuration info */
+ /* VFD SWMR */
+
+ /* Configuration info */
H5F_vfd_swmr_config_t vfd_swmr_config; /* Copy of the VFD SWMR configuration from the
FAPL used to open the file */
hbool_t vfd_swmr; /* The file is opened with VFD SWMR configured or not*/
hbool_t vfd_swmr_writer; /* This is the VFD SWMR writer or not */
uint64_t tick_num; /* Number of the current tick */
struct timespec end_of_tick; /* End time of the current tick */
+
+ /* Metadata file */
int vfd_swmr_md_fd; /* POSIX: file descriptor of the metadata file */
+ haddr_t vfd_swmr_md_eoa; /* POSIX: eoa for the metadata file */
+
+ /* Free space manager for the metadata file */
+ H5FS_t *fs_man_md; /* Free-space manager */
+ H5F_fs_state_t fs_state_md; /* State of the free space manager */
+
+ /* Delayed free space release doubly linked list */
+ uint32_t dl_len; /* # of entries in the list */
+ H5F_vfd_swmr_dl_entry_t *dl_head_ptr; /* Points to the beginning of the list */
+ H5F_vfd_swmr_dl_entry_t *dl_tail_ptr; /* Points to the end of the list */
+
};
+
/*
* This is the top-level file descriptor. One of these structures is
* allocated every time H5Fopen() is called although they may contain pointers
@@ -471,6 +509,12 @@ H5_DLL haddr_t H5F__alloc(H5F_t *f, H5F_mem_t type, hsize_t size, haddr_t *frag_
H5_DLL herr_t H5F__free(H5F_t *f, H5F_mem_t type, haddr_t addr, hsize_t size);
H5_DLL htri_t H5F__try_extend(H5F_t *f, H5FD_mem_t type, haddr_t blk_end, hsize_t extra_requested);
+/* VFD SWMR */
+/* Space allocation routines */
+H5_DLL haddr_t H5F__alloc_md(H5F_t *f, hsize_t size);
+H5_DLL herr_t H5F__free_md(H5F_t *f, haddr_t addr, hsize_t size);
+H5_DLL htri_t H5F__try_extend_md(H5F_t *f, haddr_t blk_end, hsize_t extra_requested);
+
/* Functions that get/retrieve values from VFD layer */
H5_DLL herr_t H5F__set_eoa(const H5F_t *f, H5F_mem_t type, haddr_t addr);
H5_DLL herr_t H5F__set_base_addr(const H5F_t *f, haddr_t addr);
diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h
index 356a606..dc407c8 100644
--- a/src/H5Fprivate.h
+++ b/src/H5Fprivate.h
@@ -331,7 +331,8 @@ typedef struct H5F_t H5F_t;
#define H5F_POINT_OF_NO_RETURN(F) ((F)->shared->fs.point_of_no_return)
#define H5F_FIRST_ALLOC_DEALLOC(F) ((F)->shared->first_alloc_dealloc)
#define H5F_EOA_PRE_FSM_FSALLOC(F) ((F)->shared->eoa_pre_fsm_fsalloc)
-#define H5F_USE_VFD_SWMR(F) ((F)->shared->vfd_swmr)
+#define H5F_USE_VFD_SWMR(F) ((F)->shared->vfd_swmr)
+#define H5F_VFD_SWMR_MD_EOA(F) ((F)->shared->vfd_swmr_md_eoa)
#else /* H5F_MODULE */
#define H5F_LOW_BOUND(F) (H5F_get_low_bound(F))
#define H5F_HIGH_BOUND(F) (H5F_get_high_bound(F))
@@ -390,6 +391,7 @@ typedef struct H5F_t H5F_t;
#define H5F_FIRST_ALLOC_DEALLOC(F) (H5F_get_first_alloc_dealloc(F))
#define H5F_EOA_PRE_FSM_FSALLOC(F) (H5F_get_eoa_pre_fsm_fsalloc(F))
#define H5F_USE_VFD_SWMR(F) (H5F_use_vfd_swmr(F))
+#define H5F_VFD_SWMR_MD_EOA(F) (H5F_get_vfd_swmr_md_eoa(F))
#endif /* H5F_MODULE */
@@ -511,7 +513,7 @@ typedef struct H5F_t H5F_t;
#define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME "page_buffer_min_raw_perc" /* the min raw data percentage for the page buffer cache */
/* Default configuration for VFD SWMR: not configured */
-#define H5F_ACS_VFD_SWMR_CONFIG_NAME "vfd_swmr_config" /* VFD SWMR configuration */
+#define H5F_ACS_VFD_SWMR_CONFIG_NAME "vfd_swmr_config"
#define H5F__DEFAULT_VFD_SWMR_CONFIG \
{ \
/* int32_t version = */ 0, \
@@ -665,13 +667,15 @@ struct H5HG_heap_t;
struct H5VL_class_t;
struct H5P_genplist_t;
+/* VFD SWMR */
+/* Forward declaration */
+struct H5FD_vfd_swmr_idx_entry_t;
+
/* Forward declarations for anonymous H5F objects */
/* Main file structures */
typedef struct H5F_file_t H5F_file_t;
-extern H5F_file_t *vfd_swmr_file_g;
-
/* Block aggregation structure */
typedef struct H5F_blk_aggr_t H5F_blk_aggr_t;
@@ -761,7 +765,6 @@ H5_DLL hsize_t H5F_get_pgend_meta_thres(const H5F_t *f);
H5_DLL hbool_t H5F_get_point_of_no_return(const H5F_t *f);
H5_DLL hbool_t H5F_get_first_alloc_dealloc(const H5F_t *f);
H5_DLL haddr_t H5F_get_eoa_pre_fsm_fsalloc(const H5F_t *f);
-H5_DLL hbool_t H5F_use_vfd_swmr(const H5F_t *f);
/* Functions than retrieve values set/cached from the superblock/FCPL */
H5_DLL haddr_t H5F_get_base_addr(const H5F_t *f);
@@ -875,5 +878,12 @@ H5_DLL herr_t H5F_cwfs_remove_heap(H5F_file_t *shared, struct H5HG_heap_t *heap)
/* Debugging functions */
H5_DLL herr_t H5F_debug(H5F_t *f, FILE * stream, int indent, int fwidth);
+/* VFD SWMR */
+H5_DLL herr_t H5F_vfd_swmr_writer_end_of_tick(void);
+H5_DLL herr_t H5F_vfd_swmr_reader_end_of_tick(void);
+H5_DLL herr_t H5F_update_vfd_swmr_metadata_file(H5F_t *f, uint32_t index_len, struct H5FD_vfd_swmr_idx_entry_t *index);
+H5_DLL hbool_t H5F_use_vfd_swmr(const H5F_t *f);
+H5_DLL haddr_t H5F_get_vfd_swmr_md_eoa(const H5F_t *f);
+
#endif /* _H5Fprivate_H */
diff --git a/src/H5Fquery.c b/src/H5Fquery.c
index 93b50d7..1dadd15 100644
--- a/src/H5Fquery.c
+++ b/src/H5Fquery.c
@@ -1238,3 +1238,27 @@ H5F_use_vfd_swmr(const H5F_t *f)
FUNC_LEAVE_NOAPI(f->shared->vfd_swmr)
} /* end H5F_use_vfd_swmr() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5F_get_vfd_swmr_md_eoa
+ *
+ * Purpose: Quick and dirty routine to retrieve the EOA for the metadata file
+ * (Mainly added to stop non-file routines from poking about in the
+ * H5F_t data structure)
+ *
+ * Return: The EOA for the metadata file
+ *-------------------------------------------------------------------------
+ */
+haddr_t
+H5F_get_vfd_swmr_md_eoa(const H5F_t *f)
+{
+ /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ HDassert(f);
+ HDassert(f->shared);
+ HDassert(f->shared->vfd_swmr);
+
+ FUNC_LEAVE_NOAPI(f->shared->vfd_swmr_md_eoa)
+} /* end H5F_get_vfd_swmr_md_eof() */
diff --git a/src/H5Fspace.c b/src/H5Fspace.c
index 6baf163..a35fdff 100644
--- a/src/H5Fspace.c
+++ b/src/H5Fspace.c
@@ -42,6 +42,10 @@
/* Local Macros */
/****************/
+/* Define this to display debugging information for VFD SWMR */
+/* #define H5F_VFD_SWMR_DEBUG */
+
+
/******************/
/* Local Typedefs */
@@ -56,6 +60,7 @@
/********************/
/* Local Prototypes */
/********************/
+static haddr_t H5F__extend_md(H5F_t *f, hsize_t size);
/*********************/
@@ -222,3 +227,158 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5F__try_extend() */
+
+/*
+ * VFD SWMR
+ */
+
+/*-------------------------------------------------------------------------
+ * Function: H5F__free_md
+ *
+ * Purpose: Release space at the end of the metadata file's allocated space
+ *
+ * Return: Success: Non-negative
+ * Failure: Negative
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5F__free_md(H5F_t *f, haddr_t addr, hsize_t size)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE
+
+ /* Check args */
+ HDassert(f);
+ HDassert(f->shared);
+ HDassert(size > 0);
+
+ /* Sanity checking */
+ if(!H5F_addr_defined(addr))
+ HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "invalid file offset")
+
+ if(addr > f->shared->maxaddr || H5F_addr_overflow(addr, size) || (addr + size) > f->shared->maxaddr)
+ HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "invalid file free space region to free")
+
+ /* Check if this free block is at the end of file allocated space.
+ * Truncate it if this is true.
+ */
+ if(f->shared->vfd_swmr_md_eoa == (addr + size))
+ f->shared->vfd_swmr_md_eoa = addr;
+ else {
+ /* leak memory */
+#ifdef H5F_VFD_SWMR_DEBUG
+HDfprintf(stderr, "%s: LEAKED MEMORY!!! addr = %a, size = %Hu\n", FUNC, addr, size);
+#endif
+ } /* end else */
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5F__free_md() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5F__alloc_md
+ *
+ * Purpose: Allocate space at the end of the metadata file
+ *
+ * Return: Success: Non-negative
+ * Failure: Negative
+ *
+ *-------------------------------------------------------------------------
+ */
+haddr_t
+H5F__alloc_md(H5F_t *f, hsize_t size)
+{
+ haddr_t ret_value = HADDR_UNDEF; /* Return value */
+
+ FUNC_ENTER_PACKAGE
+
+ /* check args */
+ HDassert(f);
+ HDassert(f->shared);
+ HDassert(size > 0);
+
+ /* Extend the EOA space of the metadata file */
+ ret_value = H5F__extend_md(f, size);
+
+ if(!H5F_addr_defined(ret_value))
+ HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, HADDR_UNDEF, "driver eoa update request failed")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5F__alloc_md() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5F__try_extend_md
+ *
+ * Purpose: Try to extend a block at the end of the metadata file, if possible.
+ *
+ * Return: Success: Non-negative
+ * Failure: Negative
+ *
+ *-------------------------------------------------------------------------
+ */
+htri_t
+H5F__try_extend_md(H5F_t *f, haddr_t blk_end, hsize_t extra_requested)
+{
+ htri_t ret_value = FALSE; /* Return value */
+
+ FUNC_ENTER_PACKAGE
+
+ /* check args */
+ HDassert(f);
+ HDassert(f->shared);
+ HDassert(extra_requested > 0);
+
+ /* Check if the block is exactly at the end of the file */
+ if(H5F_addr_eq(blk_end, f->shared->vfd_swmr_md_eoa)) {
+
+ /* Extend the EOA space of the metadata file */
+ if(HADDR_UNDEF == H5F__extend_md(f, extra_requested))
+ HGOTO_ERROR(H5E_FILE, H5E_CANTEXTEND, FAIL, "driver extend request failed")
+
+ /* Indicate success */
+ ret_value = TRUE;
+ }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5F__try_extend_md() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5F__extend_md
+ *
+ * Purpose: Extend the EOA space of the metadata file.
+ *
+ * Return: Success: Non-negative
+ * Failure: Negative
+ *
+ *-------------------------------------------------------------------------
+ */
+static haddr_t
+H5F__extend_md(H5F_t *f, hsize_t size)
+{
+ haddr_t eoa;
+ haddr_t ret_value = HADDR_UNDEF; /* Return value */
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ /* Get current end-of-allocated space address */
+ eoa = f->shared->vfd_swmr_md_eoa;
+
+ /* Check for overflow when extending */
+ if(H5F_addr_overflow(eoa, size) || (eoa + size) > f->shared->maxaddr)
+ HGOTO_ERROR(H5E_FILE, H5E_NOSPACE, HADDR_UNDEF, "file allocation request failed")
+
+ /* Set the address to return */
+ ret_value = eoa;
+
+ /* Extend the end-of-allocated space address */
+ f->shared->vfd_swmr_md_eoa += size;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+}
diff --git a/src/H5MV.c b/src/H5MV.c
new file mode 100644
index 0000000..d0cca9f
--- /dev/null
+++ b/src/H5MV.c
@@ -0,0 +1,518 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * Copyright by the Board of Trustees of the University of Illinois. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*-------------------------------------------------------------------------
+ *
+ * Created: H5MV.c
+ *
+ * Purpose: Free-space manager for VFD SWMR's metadata file
+ *
+ *-------------------------------------------------------------------------
+ */
+
+/****************/
+/* Module Setup */
+/****************/
+
+#define H5F_FRIEND /*suppress error about including H5Fpkg */
+#define H5FS_FRIEND /*suppress error about including H5Fpkg */
+#include "H5MVmodule.h" /* This source code file is part of the H5MV module */
+
+
+/***********/
+/* Headers */
+/***********/
+#include "H5private.h" /* Generic Functions */
+#include "H5Eprivate.h" /* Error handling */
+#include "H5Fpkg.h" /* File access */
+#include "H5FSpkg.h" /* File free space */
+#include "H5Iprivate.h" /* IDs */
+#include "H5MVpkg.h" /* File memory management */
+#include "H5VMprivate.h" /* Vectors and arrays */
+
+
+/****************/
+/* Local Macros */
+/****************/
+
+#define H5MV_FSPACE_SHRINK 80 /* Percent of "normal" size to shrink serialized free space size */
+#define H5MV_FSPACE_EXPAND 120 /* Percent of "normal" size to expand serialized free space size */
+#define H5MV_FSPACE_THRHD_DEF 1 /* Default: no alignment threshold */
+#define H5MV_FSPACE_ALIGN_DEF 1 /* Default: no alignment */
+
+/******************/
+/* Local Typedefs */
+/******************/
+
+/* User data for section info iterator callback for iterating over free space sections */
+typedef struct {
+ H5F_sect_info_t *sects; /* section info to be retrieved */
+ size_t sect_count; /* # of sections requested */
+ size_t sect_idx; /* the current count of sections */
+} H5MV_sect_iter_ud_t;
+
+
+/********************/
+/* Package Typedefs */
+/********************/
+
+
+/********************/
+/* Local Prototypes */
+/********************/
+
+
+/*********************/
+/* Package Variables */
+/*********************/
+
+/* Package initialization variable */
+hbool_t H5_PKG_INIT_VAR = FALSE;
+
+
+/*****************************/
+/* Library Private Variables */
+/*****************************/
+
+
+/*******************/
+/* Local Variables */
+/*******************/
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MV__create()
+ *
+ * Purpose: Create free space manager for the metadata file by creating
+ * a free-space structure
+ *
+ * Return: Success: non-negative
+ * Failure: negative
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5MV__create(H5F_t *f)
+{
+ /* Free space section classes implemented for file */
+ const H5FS_section_class_t *classes[] = { H5MV_FSPACE_SECT_CLS_SIMPLE };
+ H5FS_create_t fs_create; /* Free space creation parameters */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE
+
+ /*
+ * Check arguments.
+ */
+ HDassert(f);
+ HDassert(f->shared);
+ HDassert(f->shared->fs_state_md == H5F_FS_STATE_CLOSED);
+
+ /* Set the free space creation parameters */
+ fs_create.client = H5FS_CLIENT_MD_VFD_ID;
+ fs_create.shrink_percent = H5MV_FSPACE_SHRINK;
+ fs_create.expand_percent = H5MV_FSPACE_EXPAND;
+ fs_create.max_sect_addr = 1 + H5VM_log2_gen((uint64_t)f->shared->maxaddr);
+ fs_create.max_sect_size = f->shared->maxaddr;
+
+ if(NULL == (f->shared->fs_man_md = H5FS_create(f, NULL, &fs_create, NELMTS(classes), classes, f, H5MV_FSPACE_ALIGN_DEF, H5MV_FSPACE_THRHD_DEF)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space info")
+
+ /* Set the state for the free space manager to "open", if it is now */
+ if(f->shared->fs_man_md)
+ f->shared->fs_state_md = H5F_FS_STATE_OPEN;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5MV__create() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MV_close
+ *
+ * Purpose: Close the free space manager for the metadata file
+ *
+ * Return: Success: non-negative
+ * Failure: negative
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5MV_close(H5F_t *f)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+
+ /*
+ * Check arguments.
+ */
+ HDassert(f);
+ HDassert(f->shared);
+#ifdef H5MV_VFD_SWMR_DEBUG
+HDfprintf(stderr, "%s: Trying to close free space manager\n", FUNC);
+#endif
+
+ /* Close an existing free space structure for the file */
+ if(f->shared->fs_man_md) {
+#ifdef H5MV_VFD_SWMR_DEBUG
+HDfprintf(stderr, "%s: Going to close free space manager\n", FUNC);
+#endif
+ HDassert(f->shared->fs_state_md != H5F_FS_STATE_CLOSED);
+
+ if(H5FS_close(f, f->shared->fs_man_md) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free space info")
+ }
+
+ f->shared->fs_man_md = NULL;
+ f->shared->fs_state_md = H5F_FS_STATE_CLOSED;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5MV_close() */
+
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MV__find_sect
+ *
+ * Purpose: To find a section from the specified free-space manager
+ * to fulfill the request.
+ * If found, re-add the left-over space back to the manager.
+ *
+ * Return: TRUE if a section is found to fulfill the request
+ * FALSE if not
+ *
+ *-------------------------------------------------------------------------
+ */
+htri_t
+H5MV__find_sect(H5F_t *f, hsize_t size, H5FS_t *fspace, haddr_t *addr)
+{
+ H5MV_free_section_t *node; /* Free space section pointer */
+ htri_t ret_value = FAIL; /* Whether an existing free list node was found */
+
+ FUNC_ENTER_PACKAGE
+
+ HDassert(f);
+ HDassert(fspace);
+
+ /* Try to get a section from the free space manager */
+ if((ret_value = H5FS_sect_find(f, fspace, size, (H5FS_section_info_t **)&node)) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "error locating free space in file")
+
+#ifdef H5MV_VFD_SWMR_DEBUG
+HDfprintf(stderr, "%s: section found = %t\n", FUNC, ret_value);
+#endif
+
+ /* Check for actually finding section */
+ if(ret_value) {
+ /* Sanity check */
+ HDassert(node);
+
+ /* Retrieve return value */
+ if(addr)
+ *addr = node->sect_info.addr;
+
+ /* Check for eliminating the section */
+ if(node->sect_info.size == size) {
+#ifdef H5MV_VFD_SWMR_DEBUG
+HDfprintf(stderr, "%s: freeing node\n", FUNC);
+#endif
+
+ /* Free section node */
+ if(H5MV__sect_free((H5FS_section_info_t *)node) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node")
+ } /* end if */
+ else {
+ /* Adjust information for section */
+ node->sect_info.addr += size;
+ node->sect_info.size -= size;
+
+#ifdef H5MV_VFD_SWMR_DEBUG
+HDfprintf(stderr, "%s: adding node, node->sect_info.addr = %a, node->sect_info.size = %Hu\n", FUNC, node->sect_info.addr, node->sect_info.size);
+#endif
+
+ /* Re-add the section to the free-space manager */
+ if(H5FS_sect_add(f, fspace, (H5FS_section_info_t *)node, H5FS_ADD_RETURNED_SPACE, f) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, FAIL, "can't re-add section to file free space")
+ } /* end else */
+ } /* end if */
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5MV__find_sect() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MV_alloc
+ *
+ * Purpose: Allocate SIZE bytes of file memory and return the relative
+ * address where that contiguous chunk of file memory exists.
+ *
+ * Return: Success: The file address of new chunk.
+ * Failure: HADDR_UNDEF
+ *
+ *-------------------------------------------------------------------------
+ */
+haddr_t
+H5MV_alloc(H5F_t *f, hsize_t size)
+{
+ haddr_t ret_value = HADDR_UNDEF; /* Return value */
+
+ FUNC_ENTER_NOAPI(HADDR_UNDEF)
+#ifdef H5MV_VFD_SWMR_DEBUG
+HDfprintf(stderr, "%s: size = %Hu\n", FUNC, size);
+#endif
+
+ /* check arguments */
+ HDassert(f);
+ HDassert(f->shared);
+ HDassert(f->shared->vfd_swmr_md_fd >= 0);
+ HDassert(size > 0);
+
+ /* Search for large enough space in the free space manager */
+ if(f->shared->fs_man_md != NULL) {
+ if(H5MV__find_sect(f, size, f->shared->fs_man_md, &ret_value) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "error locating a node")
+ }
+
+ /* If no space is found from the free-space manager or no free-space manager, extend md's EOF */
+ if(!H5F_addr_defined(ret_value)) {
+ if(HADDR_UNDEF == (ret_value = H5F__alloc_md(f, size)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "allocation failed")
+ } /* end if */
+ HDassert(H5F_addr_defined(ret_value));
+
+done:
+#ifdef H5MV_VFD_SWMR_DEBUG
+HDfprintf(stderr, "%s: Leaving: ret_value = %a, size = %Hu\n", FUNC, ret_value, size);
+#endif
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5MV_alloc() */
+
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MV_free
+ *
+ * Purpose: Frees part of a file, making that part of the file
+ * available for reuse.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5MV_free(H5F_t *f, haddr_t addr, hsize_t size)
+{
+ H5MV_free_section_t *node = NULL; /* Free space section pointer */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+#ifdef H5MV_VFD_SWMR_DEBUG
+HDfprintf(stderr, "%s: Entering - addr = %a, size = %Hu\n", FUNC, addr, size);
+#endif
+
+ /* check arguments */
+ HDassert(f);
+ if(!H5F_addr_defined(addr) || 0 == size)
+ HGOTO_DONE(SUCCEED)
+ HDassert(addr != 0);
+
+
+ /* Check if the free space manager for the file has been initialized */
+ if(f->shared->fs_man_md == NULL) {
+ /* If there's no free space manager for objects of this type,
+ * see if we can avoid creating one by checking if the freed
+ * space is at the end of the file
+ */
+#ifdef H5MV_VFD_SWMR_DEBUG
+HDfprintf(stderr, "%s: fs_addr = %a\n", FUNC, f->shared->fs_man_md);
+#endif
+ htri_t status; /* "can absorb" status for section into */
+
+#ifdef H5MV_VFD_SWMR_DEBUG
+HDfprintf(stderr, "%s: Trying to avoid starting up free space manager\n", FUNC);
+#endif
+ /* Try to shrink the file */
+ if((status = H5MV_try_shrink(f, addr, size)) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTMERGE, FAIL, "can't check for absorbing block")
+ else if(status > 0)
+ /* Indicate success */
+ HGOTO_DONE(SUCCEED)
+
+ /* If we are deleting the free space manager, leave now, to avoid
+ * [re-]starting it: dropping free space section on the floor.
+ */
+ if(f->shared->fs_state_md == H5F_FS_STATE_DELETING) {
+#ifdef H5MV_VFD_SWMR_DEBUG
+HDfprintf(stderr, "%s: dropping addr = %a, size = %Hu, on the floor!\n", FUNC, addr, size);
+#endif
+ HGOTO_DONE(SUCCEED)
+ } /* end if */
+
+ /* There's either already a free space manager, or the freed
+ * space isn't at the end of the file, so start up (or create)
+ * the file space manager
+ */
+ if(H5MV__create(f) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space manager")
+ } /* end if */
+
+ /* Create the free-space section for the freed section */
+ if(NULL == (node = H5MV__sect_new(addr, size)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space section")
+
+ HDassert(f->shared->fs_man_md);
+
+#ifdef H5MV_VFD_SWMR_DEBUG
+HDfprintf(stderr, "%s: Before H5FS_sect_add()\n", FUNC);
+#endif
+
+ /* Add the section */
+ if(H5FS_sect_add(f, f->shared->fs_man_md, (H5FS_section_info_t *)node, H5FS_ADD_RETURNED_SPACE, f) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, FAIL, "can't re-add section to file free space")
+
+ node = NULL;
+
+#ifdef H5MV_VFD_SWMR_DEBUG
+HDfprintf(stderr, "%s: After H5FS_sect_add()\n", FUNC);
+#endif
+
+done:
+ /* Release section node, if allocated and not added to section list or merged */
+ if(node)
+ if(H5MV__sect_free((H5FS_section_info_t *)node) < 0)
+ HDONE_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node")
+
+#ifdef H5MV_VFD_SWMR_DEBUG
+HDfprintf(stderr, "%s: Leaving, ret_value = %d\n", FUNC, ret_value);
+#endif
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5MV_free() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MV_try_extend
+ *
+ * Purpose: Extend a block at EOA in the file if possible.
+ *
+ * Return: Success: TRUE(1) - Block was extended
+ * FALSE(0) - Block could not be extended
+ * Failure: FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+htri_t
+H5MV_try_extend(H5F_t *f, haddr_t addr, hsize_t size, hsize_t extra_requested)
+{
+ haddr_t end; /* End of block to extend */
+ htri_t ret_value = FALSE; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+#ifdef H5MV_VFD_SWMR_DEBUG
+HDfprintf(stderr, "%s: Entering: addr = %a, size = %Hu, extra_requested = %Hu\n", FUNC, addr, size, extra_requested);
+#endif
+
+ /* Sanity check */
+ HDassert(f);
+ HDassert(H5F_INTENT(f) & H5F_ACC_RDWR);
+
+ /* Compute end of block to extend */
+ end = addr + size;
+
+ /* Try extending the block at EOA */
+ if((ret_value = H5F__try_extend_md(f, end, extra_requested)) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending file")
+#ifdef H5MV_VFD_SWMR_DEBUG
+HDfprintf(stderr, "%s: extended = %t\n", FUNC, ret_value);
+#endif
+
+ /* If no extension so far, try to extend into a free-space section */
+ if(ret_value == FALSE) {
+
+ /* Try to extend the block into a free-space section */
+ if(f->shared->fs_man_md) {
+ if((ret_value = H5FS_sect_try_extend(f, f->shared->fs_man_md, addr, size, extra_requested, H5FS_ADD_RETURNED_SPACE, NULL)) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending block in free space manager")
+#ifdef H5MV_VFD_SWMR_DEBUG
+HDfprintf(stderr, "%s: Try to H5FS_sect_try_extend = %t\n", FUNC, ret_value);
+#endif
+ } /* end if */
+
+ } /* end if */
+
+done:
+#ifdef H5MV_VFD_SWMR_DEBUG
+HDfprintf(stderr, "%s: Leaving: ret_value = %t\n", FUNC, ret_value);
+#endif
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5MV_try_extend() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MV_try_shrink
+ *
+ * Purpose: Try to shrink the size of a file with a block
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+htri_t
+H5MV_try_shrink(H5F_t *f, haddr_t addr, hsize_t size)
+{
+ H5MV_free_section_t *node = NULL; /* Free space section pointer */
+ htri_t ret_value = FALSE; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+#ifdef H5MV_VFD_SWMR_DEBUG
+HDfprintf(stderr, "%s: Entering - addr = %a, size = %Hu\n", FUNC, addr, size);
+#endif
+
+ /* check arguments */
+ HDassert(f);
+ HDassert(f->shared);
+ HDassert(f->shared->lf);
+ HDassert(H5F_addr_defined(addr));
+ HDassert(size > 0);
+
+ /* Create free-space section for block */
+ if(NULL == (node = H5MV__sect_new(addr, size)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space section")
+
+ /* Check if the block can shrink the container */
+ if((ret_value = H5MV__sect_can_shrink((const H5FS_section_info_t *)node, f)) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTMERGE, FAIL, "can't check if section can shrink container")
+ else if(ret_value > 0) {
+ /* Shrink or absorb the section */
+ if(H5MV__sect_shrink((H5FS_section_info_t **)&node, NULL) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink container")
+ } /* end if */
+
+done:
+ /* Free section node allocated */
+ if(node && H5MV__sect_free((H5FS_section_info_t *)node) < 0)
+ HDONE_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node")
+
+#ifdef H5MV_VFD_SWMR_DEBUG
+HDfprintf(stderr, "%s: Leaving, ret_value = %d\n", FUNC, ret_value);
+#endif
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5MV_try_shrink() */
+
+/*
+ * If need to find out the amount of free-space in the file, port routines in H5MF.c
+ */
diff --git a/src/H5MVmodule.h b/src/H5MVmodule.h
new file mode 100644
index 0000000..5a95767
--- /dev/null
+++ b/src/H5MVmodule.h
@@ -0,0 +1,33 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Programmer: Quincey Koziol <koziol@hdfgroup.org>
+ * Saturday, September 12, 2015
+ *
+ * Purpose: This file contains declarations which define macros for the
+ * H5MV package. Including this header means that the source file
+ * is part of the H5MV package.
+ */
+#ifndef _H5MVmodule_H
+#define _H5MVmodule_H
+
+/* Define the proper control macros for the generic FUNC_ENTER/LEAVE and error
+ * reporting macros.
+ */
+#define H5MV_MODULE
+#define H5_MY_PKG H5MV
+#define H5_MY_PKG_ERR H5E_RESOURCE
+#define H5_MY_PKG_INIT NO
+
+#endif /* _H5MVmodule_H */
+
diff --git a/src/H5MVpkg.h b/src/H5MVpkg.h
new file mode 100644
index 0000000..32b87ac
--- /dev/null
+++ b/src/H5MVpkg.h
@@ -0,0 +1,83 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * Copyright by the Board of Trustees of the University of Illinois. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Programmer: Quincey Koziol <koziol@hdfgroup.org>
+ * Tuesday, January 8, 2008
+ *
+ * Purpose: This file contains declarations which are visible only within
+ * the H5MV package. Source files outside the H5MV package should
+ * include H5MVprivate.h instead.
+ */
+#if !(defined H5MV_FRIEND || defined H5MV_MODULE)
+#error "Do not include this file outside the H5MV package!"
+#endif
+
+#ifndef _H5MVpkg_H
+#define _H5MVpkg_H
+
+/* Get package's private header */
+#include "H5MVprivate.h"
+
+/* Other private headers needed by this file */
+#include "H5FSprivate.h" /* File free space */
+
+
+/**************************/
+/* Package Private Macros */
+/**************************/
+
+/* Define this to display information about file allocations */
+/* #define H5MV_VFD_SWMR_DEBUG */
+
+/* Free-space section types for file */
+/* (values stored in free space data structures in file) */
+#define H5MV_FSPACE_SECT_SIMPLE 0 /* For non-paged aggregation: section is a range of actual bytes in file */
+
+/****************************/
+/* Package Private Typedefs */
+/****************************/
+
+/* File free space section info */
+typedef struct H5MV_free_section_t {
+ H5FS_section_info_t sect_info; /* Free space section information (must be first in struct) */
+} H5MV_free_section_t;
+
+/*****************************/
+/* Package Private Variables */
+/*****************************/
+
+/* H5MF single section inherits serializable properties from H5FS_section_class_t */
+H5_DLLVAR H5FS_section_class_t H5MV_FSPACE_SECT_CLS_SIMPLE[1];
+
+
+/******************************/
+/* Package Private Prototypes */
+/******************************/
+
+H5_DLL htri_t H5MV__find_sect(H5F_t *f, hsize_t size, H5FS_t *fspace, haddr_t *addr);
+H5_DLL herr_t H5MV__create(H5F_t *f);
+
+/* free-space section routines */
+H5_DLL H5MV_free_section_t *H5MV__sect_new(haddr_t sect_off, hsize_t sect_size);
+H5_DLL herr_t H5MV__sect_free(H5FS_section_info_t *sect);
+H5_DLL htri_t H5MV__sect_can_shrink(const H5FS_section_info_t *_sect, void *udata);
+H5_DLL herr_t H5MV__sect_shrink(H5FS_section_info_t **_sect, void *udata);
+
+
+/* Testing routines */
+#ifdef H5MF_TESTING
+#endif /* H5MF_TESTING */
+
+#endif /* _H5MFpkg_H */
+
diff --git a/src/H5MVprivate.h b/src/H5MVprivate.h
new file mode 100644
index 0000000..2c0e95a
--- /dev/null
+++ b/src/H5MVprivate.h
@@ -0,0 +1,58 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * Copyright by the Board of Trustees of the University of Illinois. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*-------------------------------------------------------------------------
+ *
+ * Created: H5MVprivate.h
+ *
+ * Purpose: Private header file for file memory management.
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef _H5MVprivate_H
+#define _H5MVprivate_H
+
+/* Private headers needed by this file */
+#include "H5Fprivate.h" /* File access */
+
+/**************************/
+/* Library Private Macros */
+/**************************/
+
+
+/****************************/
+/* Library Private Typedefs */
+/****************************/
+
+
+/*****************************/
+/* Library-private Variables */
+/*****************************/
+
+
+/***************************************/
+/* Library-private Function Prototypes */
+/***************************************/
+
+/* File space manager routines */
+
+H5_DLL herr_t H5MV_close(H5F_t *f);
+
+/* File space allocation routines */
+H5_DLL haddr_t H5MV_alloc(H5F_t *f, hsize_t size);
+H5_DLL herr_t H5MV_free(H5F_t *f, haddr_t addr, hsize_t size);
+H5_DLL herr_t H5MV_try_extend(H5F_t *f, haddr_t addr, hsize_t size, hsize_t extra_requested);
+H5_DLL htri_t H5MV_try_shrink(H5F_t *f, haddr_t addr, hsize_t size);
+
+#endif /* end _H5MVprivate_H */
+
diff --git a/src/H5MVsection.c b/src/H5MVsection.c
new file mode 100644
index 0000000..e2e2007
--- /dev/null
+++ b/src/H5MVsection.c
@@ -0,0 +1,361 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * Copyright by the Board of Trustees of the University of Illinois. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Programmer: Quincey Koziol <koziol@hdfgroup.org>
+ * Tuesday, January 8, 2008
+ *
+ * Purpose: Free space section callbacks for file.
+ *
+ */
+
+/****************/
+/* Module Setup */
+/****************/
+
+#define H5F_FRIEND /*suppress error about including H5Fpkg */
+#include "H5MVmodule.h" /* This source code file is part of the H5MF module */
+
+
+/***********/
+/* Headers */
+/***********/
+#include "H5private.h" /* Generic Functions */
+#include "H5Eprivate.h" /* Error handling */
+#include "H5Fpkg.h" /* File access */
+#include "H5MVpkg.h" /* File memory management */
+
+
+/****************/
+/* Local Macros */
+/****************/
+
+
+/******************/
+/* Local Typedefs */
+/******************/
+
+
+/********************/
+/* Package Typedefs */
+/********************/
+
+
+/********************/
+/* Local Prototypes */
+/********************/
+
+/* 'simple' section callbacks */
+static htri_t H5MV__sect_can_merge(const H5FS_section_info_t *sect1,
+ const H5FS_section_info_t *sect2, void *udata);
+static herr_t H5MV__sect_merge(H5FS_section_info_t **sect1,
+ H5FS_section_info_t *sect2, void *udata);
+static herr_t H5MV__sect_valid(const H5FS_section_class_t *cls,
+ const H5FS_section_info_t *sect);
+
+
+
+/*********************/
+/* Package Variables */
+/*********************/
+
+/* Class info for "simple" free space sections */
+H5FS_section_class_t H5MV_FSPACE_SECT_CLS_SIMPLE[1] = {{
+ /* Class variables */
+ H5MV_FSPACE_SECT_SIMPLE, /* Section type */
+ 0, /* Extra serialized size */
+ H5FS_CLS_MERGE_SYM | H5FS_CLS_ADJUST_OK | H5FS_CLS_GHOST_OBJ, /* Class flags */
+ NULL, /* Class private info */
+
+ /* Class methods */
+ NULL, /* Initialize section class */
+ NULL, /* Terminate section class */
+
+ /* Object methods */
+ NULL, /* Add section */
+ NULL, /* Serialize section */
+ NULL, /* Deserialize section */
+ H5MV__sect_can_merge, /* Can sections merge? */
+ H5MV__sect_merge, /* Merge sections */
+ H5MV__sect_can_shrink, /* Can section shrink container?*/
+ H5MV__sect_shrink, /* Shrink container w/section */
+ H5MV__sect_free, /* Free section */
+ H5MV__sect_valid, /* Check validity of section */
+ NULL, /* Split section node for alignment */
+ NULL, /* Dump debugging for section */
+}};
+
+
+/*****************************/
+/* Library Private Variables */
+/*****************************/
+
+
+/*******************/
+/* Local Variables */
+/*******************/
+
+/* Declare a free list to manage the H5MF_free_section_t struct */
+H5FL_DEFINE(H5MV_free_section_t);
+
+/*
+ * "simple" section callbacks
+ */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MV__sect_new
+ *
+ * Purpose: Create a new section and return it to the caller
+ *
+ * Return: Pointer to new section on success/NULL on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+H5MV_free_section_t *
+H5MV__sect_new(haddr_t sect_off, hsize_t sect_size)
+{
+ H5MV_free_section_t *sect; /* 'Simple' free space section to add */
+ H5MV_free_section_t *ret_value = NULL; /* Return value */
+
+ FUNC_ENTER_PACKAGE
+
+ /* Check arguments. */
+ HDassert(sect_size);
+
+ /* Create free space section node */
+ if(NULL == (sect = H5FL_MALLOC(H5MV_free_section_t)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for free section node")
+
+ /* Set the information passed in */
+ sect->sect_info.addr = sect_off;
+ sect->sect_info.size = sect_size;
+
+ /* Set the section's class & state */
+ sect->sect_info.type = H5MV_FSPACE_SECT_SIMPLE;
+ sect->sect_info.state = H5FS_SECT_LIVE;
+
+ /* Set return value */
+ ret_value = sect;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5MV__sect_new() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MV__sect_free
+ *
+ * Purpose: Free a 'simple' section node
+ *
+ * Return: Success: non-negative
+ * Failure: negative
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5MV__sect_free(H5FS_section_info_t *_sect)
+{
+ H5MV_free_section_t *sect = (H5MV_free_section_t *)_sect; /* File free section */
+
+ FUNC_ENTER_PACKAGE_NOERR
+
+ /* Check arguments. */
+ HDassert(sect);
+
+ /* Release the section */
+ sect = H5FL_FREE(H5MV_free_section_t, sect);
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* H5MV__sect_free() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MV__sect_valid
+ *
+ * Purpose: Check the validity of a section
+ *
+ * Return: Success: non-negative
+ * Failure: negative
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5MV__sect_valid(const H5FS_section_class_t H5_ATTR_UNUSED *cls, const H5FS_section_info_t *_sect)
+{
+ const H5MV_free_section_t *sect = (const H5MV_free_section_t *)_sect; /* File free section */
+
+ FUNC_ENTER_STATIC_NOERR
+
+ /* Check arguments. */
+ HDassert(sect);
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* H5MV__sect_valid() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MV__sect_can_merge
+ *
+ * Purpose: Can two sections of this type merge?
+ *
+ * Note: Second section must be "after" first section
+ *
+ * Return: Success: non-negative (TRUE/FALSE)
+ * Failure: negative
+ *
+ *-------------------------------------------------------------------------
+ */
+static htri_t
+H5MV__sect_can_merge(const H5FS_section_info_t *_sect1,
+ const H5FS_section_info_t *_sect2, void H5_ATTR_UNUSED *_udata)
+{
+ const H5MV_free_section_t *sect1 = (const H5MV_free_section_t *)_sect1; /* File free section */
+ const H5MV_free_section_t *sect2 = (const H5MV_free_section_t *)_sect2; /* File free section */
+ htri_t ret_value = FAIL; /* Return value */
+
+ FUNC_ENTER_STATIC_NOERR
+
+ /* Check arguments. */
+ HDassert(sect1);
+ HDassert(sect2);
+ HDassert(sect1->sect_info.type == sect2->sect_info.type); /* Checks "MERGE_SYM" flag */
+ HDassert(H5F_addr_lt(sect1->sect_info.addr, sect2->sect_info.addr));
+
+ /* Check if second section adjoins first section */
+ ret_value = H5F_addr_eq(sect1->sect_info.addr + sect1->sect_info.size, sect2->sect_info.addr);
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5MV__sect_can_merge() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MV__sect_merge
+ *
+ * Purpose: Merge two sections of this type
+ *
+ * Note: Second section always merges into first node
+ *
+ * Return: Success: non-negative
+ * Failure: negative
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5MV__sect_merge(H5FS_section_info_t **_sect1, H5FS_section_info_t *_sect2,
+ void H5_ATTR_UNUSED *_udata)
+{
+ H5MV_free_section_t **sect1 = (H5MV_free_section_t **)_sect1; /* File free section */
+ H5MV_free_section_t *sect2 = (H5MV_free_section_t *)_sect2; /* File free section */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_STATIC
+
+ /* Check arguments. */
+ HDassert(sect1);
+ HDassert((*sect1)->sect_info.type == H5MV_FSPACE_SECT_SIMPLE);
+ HDassert(sect2);
+ HDassert(sect2->sect_info.type == H5MV_FSPACE_SECT_SIMPLE);
+ HDassert(H5F_addr_eq((*sect1)->sect_info.addr + (*sect1)->sect_info.size, sect2->sect_info.addr));
+
+ /* Add second section's size to first section */
+ (*sect1)->sect_info.size += sect2->sect_info.size;
+
+ /* Get rid of second section */
+ if(H5MV__sect_free((H5FS_section_info_t *)sect2) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free section node")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5MV__sect_merge() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MV__sect_can_shrink
+ *
+ * Purpose: Can this section shrink the container?
+ *
+ * Return: Success: non-negative (TRUE/FALSE)
+ * Failure: negative
+ *
+ *-------------------------------------------------------------------------
+ */
+htri_t
+H5MV__sect_can_shrink(const H5FS_section_info_t *_sect, void *_udata)
+{
+ const H5MV_free_section_t *sect = (const H5MV_free_section_t *)_sect; /* File free section */
+ H5F_t *f = (H5F_t *)_udata;
+ haddr_t eoa; /* End of address space in the file */
+ haddr_t end; /* End of section to extend */
+ htri_t ret_value = FALSE; /* Return value */
+
+ FUNC_ENTER_STATIC
+
+ /* Check arguments. */
+ HDassert(sect);
+ HDassert(f);
+
+ /* Retrieve the end oa the file's address space */
+ if(HADDR_UNDEF == (eoa = H5F_VFD_SWMR_MD_EOA(f)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "get_eoa request for VFD SWMR metadata file failed")
+
+ /* Compute address of end of section to check */
+ end = sect->sect_info.addr + sect->sect_info.size;
+
+ /* Check if the section is exactly at the end of the allocated space in the file */
+ if(H5F_addr_eq(end, eoa))
+ /* Indicate shrinking can occur */
+ ret_value = TRUE;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5MV__sect_can_shrink() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MV__sect_shrink
+ *
+ * Purpose: Shrink container with section
+ *
+ * Return: Success: non-negative
+ * Failure: negative
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5MV__sect_shrink(H5FS_section_info_t **_sect, void *_udata)
+{
+ H5F_t *f = (H5F_t *)_udata;
+ H5MV_free_section_t **sect = (H5MV_free_section_t **)_sect; /* File free section */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_STATIC
+
+ /* Check arguments. */
+ HDassert(sect);
+ HDassert(H5F_INTENT(f) & H5F_ACC_RDWR);
+
+ /* Release section's space at EOA */
+ if(H5F__free_md(f, (*sect)->sect_info.addr, (*sect)->sect_info.size) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "free request for VFD SWMR metadata file failed")
+
+ /* Free the section */
+ if(H5MV__sect_free((H5FS_section_info_t *)*sect) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node")
+
+ /* Mark section as freed, for free space manager */
+ *sect = NULL;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5MV__sect_shrink() */
diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c
index f2e9a75..fbac84d 100644
--- a/src/H5Pfapl.c
+++ b/src/H5Pfapl.c
@@ -406,7 +406,7 @@ static const size_t H5F_def_page_buf_size_g = H5F_ACS_PAGE_BUFFER_SIZE_DEF;
static const unsigned H5F_def_page_buf_min_meta_perc_g = H5F_ACS_PAGE_BUFFER_MIN_META_PERC_DEF; /* Default page buffer minimum metadata size */
static const unsigned H5F_def_page_buf_min_raw_perc_g = H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEF; /* Default page buffer mininum raw data size */
-static const H5F_vfd_swmr_config_t *H5F_def_vfd_swmr_config_g = H5F_ACS_VFD_SWMR_CONFIG_DEF; /* Default vfd swmr configuration */
+static const H5F_vfd_swmr_config_t H5F_def_vfd_swmr_config_g = H5F_ACS_VFD_SWMR_CONFIG_DEF; /* Default vfd swmr configuration */
/*-------------------------------------------------------------------------
@@ -655,7 +655,7 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass)
HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class")
/* Register the default VFD SWMR configuration */
- if(H5P_register_real(pclass, H5F_ACS_VFD_SWMR_CONFIG_NAME, H5F_ACS_VFD_SWMR_CONFIG_SIZE, &H5F_def_vfd_swmr_config_g,
+ if(H5P__register_real(pclass, H5F_ACS_VFD_SWMR_CONFIG_NAME, H5F_ACS_VFD_SWMR_CONFIG_SIZE, &H5F_def_vfd_swmr_config_g,
NULL, NULL, NULL, H5F_ACS_VFD_SWMR_CONFIG_ENC, H5F_ACS_VFD_SWMR_CONFIG_DEC,
NULL, NULL, NULL, NULL) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class")
diff --git a/src/H5private.h b/src/H5private.h
index 45b0340..842ba87 100644
--- a/src/H5private.h
+++ b/src/H5private.h
@@ -1945,7 +1945,7 @@ extern hbool_t H5_libterm_g; /* Is the library being shutdown? */
#define H5_INIT_GLOBAL (H5_libinit_g)
#define H5_TERM_GLOBAL (H5_libterm_g)
-/* Temporary Gobals for VFD SWMR */
+/* Temporary globals for VFD SWMR */
extern hbool_t vfd_swmr_g;
extern hbool_t vfd_swmr_writer_g;
extern uint64_t tick_num_g;
@@ -2075,11 +2075,11 @@ H5_DLL herr_t H5CX_pop(void);
if( (curr_time.tv_sec >= end_of_tick_g.tv_sec) && \
(curr_time.tv_nsec >= end_of_tick_g.tv_nsec) ) { \
if(vfd_swmr_writer_g) { \
- if(H5FD_vfd_swmr_writer_end_of_tick() < 0) \
+ if(H5F_vfd_swmr_writer_end_of_tick() < 0) \
HGOTO_ERROR(H5E_FUNC, H5E_CANTSET, err, "end of tick error for VFD SWMR writer") \
} \
else if(!swmr_reader_exit) { \
- if(H5FD_vfd_swmr_reader_end_of_tick() < 0) \
+ if(H5F_vfd_swmr_reader_end_of_tick() < 0) \
HGOTO_ERROR(H5E_FUNC, H5E_CANTSET, err, "end of tick error for VFD SWMR reader") \
} \
} \
@@ -2089,9 +2089,9 @@ H5_DLL herr_t H5CX_pop(void);
#define FUNC_ENTER_API(err) {{ \
FUNC_ENTER_API_COMMON \
FUNC_ENTER_API_INIT(err); \
+ VFD_SWMR_TEST_FOR_END_OF_TICK(FALSE, err); \
/* Clear thread error stack entering public functions */ \
H5E_clear_stack(NULL); \
- VFD_SWMR_TEST_FOR_END_OF_TICK(FALSE, err); \
{
/*
@@ -2291,15 +2291,16 @@ H5_DLL herr_t H5CX_pop(void);
FINISH_MPE_LOG \
H5TRACE_RETURN(ret_value);
-#define FUNC_LEAVE_API(ret_value) \
- VFD_SWMR_TEST_FOR_END_OF_TICK(!vfd_swmr_writer_g, ret_value); \
- FUNC_LEAVE_API_COMMON(ret_value); \
- (void)H5CX_pop(); \
- H5_POP_FUNC \
- if(err_occurred) \
- (void)H5E_dump_api_stack(TRUE); \
- FUNC_LEAVE_API_THREADSAFE \
- return(ret_value); \
+#define FUNC_LEAVE_API(ret_value) \
+ if(!err_occurred) \
+ VFD_SWMR_TEST_FOR_END_OF_TICK(!vfd_swmr_writer_g, ret_value); \
+ FUNC_LEAVE_API_COMMON(ret_value); \
+ (void)H5CX_pop(); \
+ H5_POP_FUNC \
+ if(err_occurred) \
+ (void)H5E_dump_api_stack(TRUE); \
+ FUNC_LEAVE_API_THREADSAFE \
+ return(ret_value); \
}} /*end scope from beginning of FUNC_ENTER*/
/* Use this macro to match the FUNC_ENTER_API_NOINIT macro */
diff --git a/src/Makefile.am b/src/Makefile.am
index bc89990..ef4a880 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -79,6 +79,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \
H5HL.c H5HLcache.c H5HLdbg.c H5HLint.c H5HLprfx.c H5HLdblk.c\
H5HP.c H5I.c H5Itest.c H5L.c H5Lexternal.c H5lib_settings.c \
H5MF.c H5MFaggr.c H5MFdbg.c H5MFsection.c \
+ H5MV.c H5MVsection.c \
H5MM.c H5MP.c H5MPtest.c \
H5O.c H5Odeprec.c H5Oainfo.c H5Oalloc.c H5Oattr.c \
H5Oattribute.c H5Obogus.c H5Obtreek.c H5Ocache.c H5Ocache_image.c \
diff --git a/test/Makefile.am b/test/Makefile.am
index bcb50ec..9558e0f 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -33,10 +33,12 @@ AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_builddir)/src
# test_filter_plugin.sh: filter_plugin.c
# test_usecases.sh: use_append_chunk, use_append_mchunks, use_disable_mdc_flushes
TEST_SCRIPT = testerror.sh testlibinfo.sh testcheck_version.sh testlinks_env.sh test_filenotclosed.sh\
- testswmr.sh testvdsswmr.sh testflushrefresh.sh test_usecases.sh
+ testswmr.sh testvdsswmr.sh testflushrefresh.sh test_usecases.sh \
+ testvfdswmr.sh
SCRIPT_DEPEND = error_test$(EXEEXT) err_compat$(EXEEXT) links_env$(EXEEXT) filenotclosed$(EXEEXT) \
flushrefresh$(EXEEXT) use_append_chunk$(EXEEXT) use_append_mchunks$(EXEEXT) use_disable_mdc_flushes$(EXEEXT) \
swmr_generator$(EXEEXT) swmr_reader$(EXEEXT) swmr_writer$(EXEEXT) \
+ vfd_swmr_generator$(EXEEXT) vfd_swmr_writer$(EXEEXT) vfd_swmr_reader$(EXEEXT) \
swmr_remove_reader$(EXEEXT) swmr_remove_writer$(EXEEXT) swmr_addrem_writer$(EXEEXT) \
swmr_sparse_reader$(EXEEXT) swmr_sparse_writer$(EXEEXT) swmr_start_write$(EXEEXT) \
vds_swmr_gen$(EXEEXT) vds_swmr_reader$(EXEEXT) vds_swmr_writer$(EXEEXT)
@@ -80,6 +82,7 @@ check_PROGRAMS=$(TEST_PROG) error_test err_compat tcheck_version \
testmeta accum_swmr_reader atomic_writer atomic_reader \
links_env filenotclosed flushrefresh use_append_chunk use_append_mchunks use_disable_mdc_flushes \
swmr_generator swmr_start_write swmr_reader swmr_writer swmr_remove_reader \
+ vfd_swmr_generator vfd_swmr_reader vfd_swmr_writer \
swmr_remove_writer swmr_addrem_writer swmr_sparse_reader swmr_sparse_writer \
swmr_check_compat_vfd vds_swmr_gen vds_swmr_reader vds_swmr_writer
if HAVE_SHARED_CONDITIONAL
diff --git a/test/testvfdswmr.sh.in b/test/testvfdswmr.sh.in
new file mode 100644
index 0000000..a0b5aee
--- /dev/null
+++ b/test/testvfdswmr.sh.in
@@ -0,0 +1,292 @@
+#! /bin/bash
+#
+# Copyright by The HDF Group.
+# Copyright by the Board of Trustees of the University of Illinois.
+# All rights reserved.
+#
+# This file is part of HDF5. The full HDF5 copyright notice, including
+# terms governing use, modification, and redistribution, is contained in
+# the COPYING file, which can be found at the root of the source code
+# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases.
+# If you do not have access to either file, you may request a copy from
+# help@hdfgroup.org.
+#
+# Tests for the vfd swmr feature.
+#
+###############################################################################
+# Copied current SWMR concurrent tests and renamed to:
+# testswmr.sh.in-->test_vfd_swmr.sh.in
+# swmr_generator.c-->vfd_swmr_generator.c
+# swmr_reader.c-->vfd_swmr_reader.c
+# swmr_writer.c-->vfd_swmr_writer.c
+#
+# FOR NOW: only one test is copied
+###############################################################################
+
+srcdir=@srcdir@
+
+###############################################################################
+## test parameters
+###############################################################################
+
+Nreaders=1 # number of readers to launch
+Nrdrs_spa=3 # number of sparse readers to launch
+Nrecords=200000 # number of records to write
+Nrecs_rem=40000 # number of times to shrink
+Nrecs_spa=20000 # number of records to write in the sparse test
+Nsecs_add=5 # number of seconds per read interval
+Nsecs_rem=3 # number of seconds per read interval
+Nsecs_addrem=8 # number of seconds per read interval
+nerrors=0
+
+###############################################################################
+## definitions for message file to coordinate test runs
+###############################################################################
+WRITER_MESSAGE=SWMR_WRITER_MESSAGE # The message file created by writer that the open is complete
+ # This should be the same as the define in "./swmr_common.h"
+MESSAGE_TIMEOUT=300 # Message timeout length in secs
+ # This should be the same as the define in "./h5test.h"
+
+###############################################################################
+## short hands and function definitions
+###############################################################################
+##DPRINT=: # Set to "echo Debug:" for debugging printing,
+DPRINT="echo Debug:" # Set to "echo Debug:" for debugging printing,
+ # else ":" for noop.
+##IFDEBUG=: # Set to null to turn on debugging, else ":" for noop.
+IFDEBUG= # Set to null to turn on debugging, else ":" for noop.
+
+# Print a line-line message left justified in a field of 70 characters
+# beginning with the word "Testing".
+#
+TESTING() {
+ SPACES=" "
+ echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012'
+}
+
+# To wait for the writer message file or till the maximum # of seconds is reached
+# $1 is the message file to wait for
+# This performs similar function as the routine h5_wait_message() in test/h5test.c
+WAIT_MESSAGE() {
+ message=$1 # Get the name of the message file to wait for
+ t0=`date +%s` # Get current time in seconds
+ difft=0 # Initialize the time difference
+ mexist=0 # Indicate whether the message file is found
+ while [ $difft -lt $MESSAGE_TIMEOUT ] ; # Loop till message times out
+ do
+ t1=`date +%s` # Get current time in seconds
+ difft=`expr $t1 - $t0` # Calculate the time difference
+ if [ -e $message ]; then # If message file is found:
+ mexist=1 # indicate the message file is found
+ rm $message # remove the message file
+ break # get out of the while loop
+ fi
+ done;
+ if test $mexist -eq 0; then
+ # Issue warning that the writer message file is not found, continue with launching the reader(s)
+ echo warning: $WRITER_MESSAGE is not found after waiting $MESSAGE_TIMEOUT seconds
+ else
+ echo $WRITER_MESSAGE is found
+ fi
+}
+
+###############################################################################
+## Main
+##
+## Modifications:
+## Vailin Choi; July 2013
+## Add waiting of message file before launching the reader(s).
+## Due to the implementation of file locking, coordination
+## is needed in file opening for the writer/reader tests
+## to proceed as expected.
+##
+###############################################################################
+# The build (current) directory might be different than the source directory.
+if test -z "$srcdir"; then
+ srcdir=.
+fi
+
+# Check to see if the VFD specified by the HDF5_DRIVER environment variable
+# supports SWMR.
+./swmr_check_compat_vfd
+rc=$?
+if [ $rc -ne 0 ] ; then
+ echo
+ echo "The VFD specified by the HDF5_DRIVER environment variable"
+ echo "does not support SWMR."
+ echo
+ echo "SWMR acceptance tests skipped"
+ echo
+ exit 0
+fi
+
+# Parse options (none accepted at this time)
+while [ $# -gt 0 ]; do
+ case "$1" in
+ *) # unknown option
+ echo "$0: Unknown option ($1)"
+ exit 1
+ ;;
+ esac
+done
+
+# HDF5 has several tests that create and delete signal files to communicate
+# between processes, and it seems that even though the names of the files are
+# different, occasionally the wrong file is deleted, interrupting the flow of
+# the test. Running each of these tests in its own directory should eliminate
+# the problem.
+mkdir vfd_swmr_test
+for FILE in vfd_swmr*; do
+ case "$FILE" in
+ *.o) continue ;; ## don't copy the .o files
+ esac
+ cp $FILE vfd_swmr_test
+done
+cp vfd_swmr* vfd_swmr_test
+
+# With the --disable-shared option, swmr program files are built in the test
+# directory, otherwise they are in test/.libs with a corresponding wrapper
+# script in the test directory. The programs or wrapper scripts in test should
+# always be copied, swmr files in .libs should be copied only if they exists.
+if [ -f .libs/swmr ]; then
+ mkdir vfd_swmr_test/.libs
+ for FILE in .libs/vfd_swmr*; do
+ case "$FILE" in
+ *.o) continue ;; ## don't copy the .o files
+ esac
+ cp $FILE vfd_swmr_test/.libs
+ done
+fi
+
+cd vfd_swmr_test
+
+# Loop over index types
+for index_type in "-i ea" "-i b2"
+do
+ # Try with and without compression
+ for compress in "" "-c 5"
+ do
+ echo
+ echo "*******************************************************************************"
+ echo "** Loop testing parameters: $index_type $compress"
+ echo "*******************************************************************************"
+ echo
+ echo
+ echo "###############################################################################"
+ echo "## Generator test"
+ echo "###############################################################################"
+ # Launch the Generator without SWMR_WRITE
+ echo launch the vfd_swmr_generator
+ ./vfd_swmr_generator $compress $index_type
+ if test $? -ne 0; then
+ echo generator had error
+ nerrors=`expr $nerrors + 1`
+ fi
+
+ # Launch the Generator with SWMR_WRITE
+ echo launch the vfd_swmr_generator with SWMR_WRITE
+ ./vfd_swmr_generator -s $compress $index_type
+ if test $? -ne 0; then
+ echo generator had error
+ nerrors=`expr $nerrors + 1`
+ fi
+
+ # Check for error and exit if one occured
+ $DPRINT nerrors=$nerrors
+ if test $nerrors -ne 0 ; then
+ echo "SWMR tests failed with $nerrors errors."
+ exit 1
+ fi
+
+ echo
+ echo "###############################################################################"
+ echo "## Writer test - test expanding the dataset"
+ echo "###############################################################################"
+
+ # Launch the Generator
+ echo launch the vfd_swmr_generator
+ ./vfd_swmr_generator -s $compress $index_type
+ if test $? -ne 0; then
+ echo generator had error
+ nerrors=`expr $nerrors + 1`
+ fi
+
+ # Remove any possible writer message file before launching writer
+ rm -f $WRITER_MESSAGE
+ #
+ # Launch the Writer
+ echo launch the vfd_swmr_writer
+ seed="" # Put -r <random seed> command here
+ ./vfd_swmr_writer -o $Nrecords $seed 2>&1 |tee vfd_swmr_writer.out &
+ pid_writer=$!
+ $DPRINT pid_writer=$pid_writer
+
+ # Wait for message from writer process before starting reader(s)
+ WAIT_MESSAGE $WRITER_MESSAGE
+ #
+ # Launch the Readers
+ #declare -a seeds=(<seed1> <seed2> <seed3> ... )
+ echo launch $Nreaders vfd_swmr_readers
+ pid_readers=""
+ n=0
+ while [ $n -lt $Nreaders ]; do
+ #seed="-r ${seeds[$n]}"
+ seed=""
+ ./vfd_swmr_reader $Nsecs_add $seed 2>&1 |tee vfd_swmr_reader.out.$n &
+ pid_readers="$pid_readers $!"
+ n=`expr $n + 1`
+ done
+ $DPRINT pid_readers=$pid_readers
+ $IFDEBUG ps
+
+ # Collect exit code of the readers first because they usually finish
+ # before the writer.
+ for xpid in $pid_readers; do
+ $DPRINT checked reader $xpid
+ wait $xpid
+ if test $? -ne 0; then
+ echo reader had error
+ nerrors=`expr $nerrors + 1`
+ fi
+ done
+
+ # Collect exit code of the writer
+ $DPRINT checked writer $pid_writer
+ wait $pid_writer
+ if test $? -ne 0; then
+ echo writer had error
+ nerrors=`expr $nerrors + 1`
+ fi
+
+ # Check for error and exit if one occured
+ $DPRINT nerrors=$nerrors
+ if test $nerrors -ne 0 ; then
+ echo "VFD SWMR tests failed with $nerrors errors."
+ echo "(Writer and reader output preserved)"
+ exit 1
+ fi
+
+ # Clean up output files
+ ##rm -f vfd_swmr_writer.out
+ ##rm -f vfd_swmr_reader.out.*
+
+ done
+done
+
+###############################################################################
+## Report and exit
+###############################################################################
+cd ..
+$DPRINT nerrors=$nerrors
+if test $nerrors -eq 0 ; then
+ echo "VFD SWMR tests passed."
+ if test -z "$HDF5_NOCLEANUP"; then
+ # delete the test directory
+ rm -rf vfd_swmr_test
+ fi
+ exit 0
+else
+ echo "VFD SWMR tests failed with $nerrors errors."
+ exit 1
+fi
+
diff --git a/test/vfd_swmr_common.c b/test/vfd_swmr_common.c
new file mode 100644
index 0000000..a0d79e3
--- /dev/null
+++ b/test/vfd_swmr_common.c
@@ -0,0 +1,288 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * Copyright by the Board of Trustees of the University of Illinois. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*-------------------------------------------------------------------------
+ *
+ * Created: swmr_common.c
+ *
+ * Purpose: Utility functions for the SWMR test code.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+/***********/
+/* Headers */
+/***********/
+
+#include "h5test.h"
+#include "swmr_common.h"
+
+/*******************/
+/* Local Variables */
+/*******************/
+
+/* The SWMR data arrays:
+ *
+ * The code uses a 2-D jagged array of datasets. The first dimension is called
+ * the 'level' and there are five of them.
+ *
+ * #define NLEVELS 5
+ *
+ * The second dimension is the 'count' and there are quite a few datasets per
+ * 'level'.
+ *
+ * unsigned symbol_count[NLEVELS] = {100, 200, 400, 800, 1600};
+ *
+ * These datasets are created when the skeleton is generated and are initially
+ * empty. Each dataset has no upper bound on size (H5S_UNLIMITED). They
+ * are of compound type, with two members: an integer ID and an opaque
+ * 'data part'. The data part is not used by the SWMR testing.
+ *
+ * The SWMR testing will then randomly add and/or remove entries
+ * from these datasets. The selection of the level is skewed by a mapping
+ * table which preferentially hammers on the lower levels with their smaller
+ * number of datasets.
+ *
+ * static unsigned symbol_mapping[NMAPPING] = {0, 0, 0, 0, 1, 1, 2, 3, 4};
+ *
+ * The information about each dataset (name, hid_t, etc.) is stored in a
+ * separate array.
+ *
+ * symbol_info_t *symbol_info[NLEVELS];
+ */
+
+/* An array of dataset levels, used to select the level for a SWMR operation
+ * Note that this preferentially selects the lower levels with their smaller
+ * number of datasets.
+ */
+static unsigned symbol_mapping[NMAPPING] = {0, 0, 0, 0, 1, 1, 2, 3, 4};
+
+/* The number of datasets at each level */
+unsigned symbol_count[NLEVELS] = {100, 200, 400, 800, 1600};
+
+/* Array of dataset information entries (1 per dataset) */
+symbol_info_t *symbol_info[NLEVELS];
+
+
+/*-------------------------------------------------------------------------
+ * Function: choose_dataset
+ *
+ * Purpose: Selects a random dataset in the SWMR file
+ *
+ * Parameters: N/A
+ *
+ * Return: Success: A pointer to information about a dataset.
+ * Failure: Can't fail
+ *
+ *-------------------------------------------------------------------------
+ */
+symbol_info_t *
+choose_dataset(void)
+{
+ unsigned level; /* The level of the dataset */
+ unsigned offset; /* The "offset" of the dataset at that level */
+
+ /* Determine level of dataset */
+ level = symbol_mapping[HDrandom() % NMAPPING];
+
+ /* Determine the offset of the level */
+ offset = (unsigned)(HDrandom() % (int)symbol_count[level]);
+
+ return &symbol_info[level][offset];
+} /* end choose_dataset() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: create_symbol_datatype
+ *
+ * Purpose: Create's the HDF5 datatype used for elements in the SWMR
+ * testing datasets.
+ *
+ * Parameters: N/A
+ *
+ * Return: Success: An HDF5 type ID
+ * Failure: -1
+ *
+ *-------------------------------------------------------------------------
+ */
+hid_t
+create_symbol_datatype(void)
+{
+ hid_t sym_type_id; /* Datatype ID for symbol */
+ hid_t opaq_type_id; /* Datatype ID for opaque part of record */
+
+ /* Create opaque datatype to represent other information for this record */
+ if((opaq_type_id = H5Tcreate(H5T_OPAQUE, (size_t)DTYPE_SIZE)) < 0)
+ return -1;
+
+ /* Create compound datatype for symbol */
+ if((sym_type_id = H5Tcreate(H5T_COMPOUND, sizeof(symbol_t))) < 0)
+ return -1;
+
+ /* Insert fields in symbol datatype */
+ if(H5Tinsert(sym_type_id, "rec_id", HOFFSET(symbol_t, rec_id), H5T_NATIVE_UINT64) < 0)
+ return -1;
+ if(H5Tinsert(sym_type_id, "info", HOFFSET(symbol_t, info), opaq_type_id) < 0)
+ return -1;
+
+ /* Close opaque datatype */
+ if(H5Tclose(opaq_type_id) < 0)
+ return -1;
+
+ return sym_type_id;
+} /* end create_symbol_datatype() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: generate_name
+ *
+ * Purpose: Generates a SWMR testing dataset name given a level and
+ * count.
+ * The name is in the format <name>-<level> (%u-%04u).
+ *
+ * Parameters: char *name_buf
+ * Buffer for the created name. Must be pre-allocated.
+ * Since the name is formulaic, this isn't considered an issue.
+ *
+ * unsigned level
+ * The dataset's level
+ *
+ * unsigned count
+ * The dataset's count
+ *
+ * Return: Success: 0
+ *
+ * Failure: Can't fail
+ *
+ *-------------------------------------------------------------------------
+ */
+int
+generate_name(char *name_buf, unsigned level, unsigned count)
+{
+ HDassert(name_buf);
+
+ sprintf(name_buf, "%u-%04u", level, count);
+
+ return 0;
+} /* end generate_name() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: generate_symbols
+ *
+ * Purpose: Initializes the global dataset infomration arrays.
+ *
+ * Parameters: N/A
+ *
+ * Return: Success: 0
+ * Failure: Can't fail
+ *
+ *-------------------------------------------------------------------------
+ */
+int
+generate_symbols(void)
+{
+ unsigned u, v; /* Local index variables */
+
+ for(u = 0; u < NLEVELS; u++) {
+ symbol_info[u] = (symbol_info_t *)HDmalloc(symbol_count[u] * sizeof(symbol_info_t));
+ for(v = 0; v < symbol_count[u]; v++) {
+ char name_buf[64];
+
+ generate_name(name_buf, u, v);
+ symbol_info[u][v].name = (char *)HDmalloc(HDstrlen(name_buf) + 1);
+ HDstrcpy(symbol_info[u][v].name, name_buf);
+ symbol_info[u][v].dsid = -1;
+ symbol_info[u][v].nrecords = 0;
+ } /* end for */
+ } /* end for */
+
+ return 0;
+} /* end generate_symbols() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: shutdown_symbols
+ *
+ * Purpose: Cleans up the global dataset information arrays.
+ *
+ * Parameters: N/A
+ *
+ * Return: Success: 0
+ * Failure: Can't fail
+ *
+ *-------------------------------------------------------------------------
+ */
+int
+shutdown_symbols(void)
+{
+ unsigned u, v; /* Local index variables */
+
+ /* Clean up the symbols */
+ for(u = 0; u < NLEVELS; u++) {
+ for(v = 0; v < symbol_count[u]; v++)
+ HDfree(symbol_info[u][v].name);
+ HDfree(symbol_info[u]);
+ } /* end for */
+
+ return 0;
+} /* end shutdown_symbols() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: print_metadata_retries_info
+ *
+ * Purpose: To retrieve and print the collection of metadata retries for the file.
+ *
+ * Parameters: fid: the currently opened file identifier
+ *
+ * Return: Success: 0
+ * Failure: negative
+ *
+ *-------------------------------------------------------------------------
+ */
+int
+print_metadata_retries_info(hid_t fid)
+{
+ H5F_retry_info_t info;
+ unsigned i;
+
+ /* Retrieve the collection of retries */
+ if(H5Fget_metadata_read_retry_info(fid, &info) < 0)
+ return (-1);
+
+ /* Print information for each non-NULL retries[i] */
+ for(i = 0; i < H5F_NUM_METADATA_READ_RETRY_TYPES; i++) {
+ unsigned power;
+ unsigned j;
+
+ if(NULL == info.retries[i])
+ continue;
+
+ HDfprintf(stderr, "Metadata read retries for item %u:\n", i);
+ power = 1;
+ for(j = 0; j < info.nbins; j++) {
+ if(info.retries[i][j])
+ HDfprintf(stderr, "\t# of retries for %u - %u retries: %u\n",
+ power, (power * 10) - 1, info.retries[i][j]);
+ power *= 10;
+ } /* end for */
+ } /* end for */
+
+ /* Free memory for each non-NULL retries[i] */
+ for(i = 0; i < H5F_NUM_METADATA_READ_RETRY_TYPES; i++)
+ if(info.retries[i] != NULL)
+ H5free_memory(info.retries[i]);
+
+ return 0;
+} /* print_metadata_retries_info() */
diff --git a/test/vfd_swmr_common.h b/test/vfd_swmr_common.h
new file mode 100644
index 0000000..e77a5ff
--- /dev/null
+++ b/test/vfd_swmr_common.h
@@ -0,0 +1,78 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * Copyright by the Board of Trustees of the University of Illinois. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+#ifndef _SWMR_COMMON_H
+#define _SWMR_COMMON_H
+
+/***********/
+/* Headers */
+/***********/
+
+#include "h5test.h"
+
+/**********/
+/* Macros */
+/**********/
+
+#define NLEVELS 5 /* # of datasets in the SWMR test file */
+
+#define NMAPPING 9
+
+#define FILENAME "vfd_swmr_data.h5" /* SWMR test file name */
+#define DTYPE_SIZE 150 /* Data size in opaque type */
+
+/* The message sent by writer that the file open is done--releasing the file lock */
+#define WRITER_MESSAGE "SWMR_WRITER_MESSAGE"
+
+/************/
+/* Typedefs */
+/************/
+
+/* Information about a symbol/dataset */
+typedef struct {
+ char *name; /* Dataset name for symbol */
+ hid_t dsid; /* Dataset ID for symbol */
+ hsize_t nrecords; /* Number of records for the symbol */
+} symbol_info_t;
+
+/* A symbol's record */
+typedef struct {
+ uint64_t rec_id; /* ID for this record (unique in symbol) */
+ uint8_t info[DTYPE_SIZE]; /* "Other" information for this record */
+} symbol_t;
+
+/********************/
+/* Global Variables */
+/********************/
+H5TEST_DLLVAR symbol_info_t *symbol_info[NLEVELS];
+H5TEST_DLLVAR unsigned symbol_count[NLEVELS];
+
+/**************/
+/* Prototypes */
+/**************/
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+H5TEST_DLL symbol_info_t * choose_dataset(void);
+H5TEST_DLL hid_t create_symbol_datatype(void);
+H5TEST_DLL int generate_name(char *name_buf, unsigned level, unsigned count);
+H5TEST_DLL int generate_symbols(void);
+H5TEST_DLL int shutdown_symbols(void);
+H5TEST_DLL int print_metadata_retries_info(hid_t fid);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _SWMR_COMMON_H */
diff --git a/test/vfd_swmr_generator.c b/test/vfd_swmr_generator.c
new file mode 100644
index 0000000..3e20d68
--- /dev/null
+++ b/test/vfd_swmr_generator.c
@@ -0,0 +1,415 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * Copyright by the Board of Trustees of the University of Illinois. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*-------------------------------------------------------------------------
+ *
+ * Created: vfd_swmr_generator.c
+ *
+ * Purpose: Functions for building and setting up the SWMR test file
+ * and datasets.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+/***********/
+/* Headers */
+/***********/
+
+#include "h5test.h"
+#include "vfd_swmr_common.h"
+
+/*
+ * This file needs to access testing codefrom the H5O package.
+ */
+#define H5O_FRIEND /*suppress error about including H5Opkg */
+#define H5O_TESTING
+#include "H5Opkg.h" /* Object headers */
+
+
+/****************/
+/* Local Macros */
+/****************/
+
+#define CHUNK_SIZE 50 /* Chunk size for created datasets */
+
+/********************/
+/* Local Prototypes */
+/********************/
+
+static int gen_skeleton(const char *filename, hbool_t verbose,
+ hbool_t swmr_write, int comp_level, const char *index_type,
+ unsigned random_seed);
+static void usage(void);
+
+
+/*-------------------------------------------------------------------------
+ * Function: gen_skeleton
+ *
+ * Purpose: Creates the HDF5 file and datasets which will be used in
+ * the SWMR testing.
+ *
+ * Parameters: const char *filename
+ * The SWMR test file's name.
+ *
+ * hbool_t verbose
+ * Whether verbose console output is desired.
+ *
+ * hbool_t swmr_write
+ * Whether to create the file with SWMR writing enabled
+ *
+ * int comp_level
+ * The zlib compression level to use. -1 = no compression.
+ *
+ * const char *index_type
+ * The chunk index type (b1 | b2 | ea | fa)
+ *
+ * unsigned random_seed
+ * The random seed to store in the file. The sparse tests use
+ * this value.
+ *
+ * Return: Success: 0
+ * Failure: Can't fail
+ *
+ *-------------------------------------------------------------------------
+ */
+static int
+gen_skeleton(const char *filename, hbool_t verbose, hbool_t swmr_write,
+ int comp_level, const char *index_type, unsigned random_seed)
+{
+ hid_t fid; /* File ID for new HDF5 file */
+ hid_t fcpl; /* File creation property list */
+ hid_t fapl; /* File access property list */
+ hid_t dcpl; /* Dataset creation property list */
+ hid_t tid; /* Datatype for dataset elements */
+ hid_t sid; /* Dataspace ID */
+ hid_t aid; /* Attribute ID */
+ hsize_t dims[2] = {1, 0}; /* Dataset starting dimensions */
+ hsize_t max_dims[2] = {1, H5S_UNLIMITED}; /* Dataset maximum dimensions */
+ hsize_t chunk_dims[2] = {1, CHUNK_SIZE}; /* Chunk dimensions */
+#ifdef FILLVAL_WORKS
+ symbol_t fillval; /* Dataset fill value */
+#endif /* FILLVAL_WORKS */
+ unsigned u, v; /* Local index variable */
+ H5F_vfd_swmr_config_t *config = NULL; /* Configuration for VFD SWMR */
+
+ HDassert(filename);
+ HDassert(index_type);
+
+ /* Create file access property list */
+ if((fapl = h5_fileaccess()) < 0)
+ return -1;
+
+ /* FOR NOW: create the file with the latest format so that the latest chunk indexing will be used */
+ if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
+ return -1;
+
+ /* There are two chunk indexes tested here.
+ * With one unlimited dimension, we get the extensible array index
+ * type, with two unlimited dimensions, we get a v2 B-tree.
+ */
+ if(!HDstrcmp(index_type, "b2"))
+ max_dims[0] = H5S_UNLIMITED;
+
+#ifdef QAK
+ /* Increase the initial size of the metadata cache */
+ {
+ H5AC_cache_config_t mdc_config;
+
+ mdc_config.version = H5AC__CURR_CACHE_CONFIG_VERSION;
+ H5Pget_mdc_config(fapl, &mdc_config);
+ HDfprintf(stderr, "mdc_config.initial_size = %lu\n", (unsigned long)mdc_config.initial_size);
+ HDfprintf(stderr, "mdc_config.epoch_length = %lu\n", (unsigned long)mdc_config.epoch_length);
+ mdc_config.set_initial_size = 1;
+ mdc_config.initial_size = 16 * 1024 * 1024;
+ /* mdc_config.epoch_length = 5000; */
+ H5Pset_mdc_config(fapl, &mdc_config);
+ }
+#endif /* QAK */
+
+#ifdef QAK
+ H5Pset_small_data_block_size(fapl, (hsize_t)(50 * CHUNK_SIZE * DTYPE_SIZE));
+#endif /* QAK */
+
+#ifdef QAK
+ H5Pset_fapl_log(fapl, "append.log", H5FD_LOG_ALL, (size_t)(512 * 1024 * 1024));
+#endif /* QAK */
+
+ /* Create file creation property list */
+ if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0)
+ return -1;
+
+#ifdef QAK
+ H5Pset_link_phase_change(fcpl, 0, 0);
+#endif /* QAK */
+
+ /* Emit informational message */
+ if(verbose)
+ HDfprintf(stderr, "Creating file\n");
+
+ /*
+ * Set up to create the file with VFD SWMR configured.
+ */
+
+ if(swmr_write) {
+ /* Set file space strategy to paged aggregation */
+ if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)1) < 0)
+ return -1;
+
+ /* Enable page buffering */
+ if(H5Pset_page_buffer_size(fapl, 4096, 0, 0) < 0)
+ return -1;
+
+ /* Allocate memory for the configuration structure */
+ if((config = (H5F_vfd_swmr_config_t *)HDmalloc(sizeof(H5F_vfd_swmr_config_t))) == NULL)
+ return -1;
+
+ config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION;
+ config->tick_len = 4;
+ config->max_lag = 6;
+ config->vfd_swmr_writer = TRUE;
+ config->md_pages_reserved = 2;
+ HDstrcpy(config->md_file_path, "my_md_file");
+
+ /* Enable VFD SWMR configuration */
+ if(H5Pset_vfd_swmr_config(fapl, config) < 0)
+ return -1;
+ }
+
+ /* Create the file with VFD SWMR configured */
+ if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0)
+ return -1;
+
+ /* Close file creation property list */
+ if(H5Pclose(fcpl) < 0)
+ return -1;
+
+ /* Close file access property list */
+ if(H5Pclose(fapl) < 0)
+ return -1;
+
+ /* Create attribute with (shared) random number seed - for sparse test */
+ if((sid = H5Screate(H5S_SCALAR)) < 0)
+ return -1;
+ if((aid = H5Acreate2(fid, "seed", H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+ return -1;
+ if(H5Awrite(aid, H5T_NATIVE_UINT, &random_seed) < 0)
+ return -1;
+ if(H5Sclose(sid) < 0)
+ return -1;
+
+ /* Create datatype for creating datasets */
+ if((tid = create_symbol_datatype()) < 0)
+ return -1;
+
+ /* Create dataspace for creating datasets */
+ if((sid = H5Screate_simple(2, dims, max_dims)) < 0)
+ return -1;
+
+ /* Create dataset creation property list */
+ if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
+ return -1;
+ if(H5Pset_chunk(dcpl, 2, chunk_dims) < 0)
+ return -1;
+ if(comp_level >= 0) {
+ if(H5Pset_deflate(dcpl, (unsigned)comp_level) < 0)
+ return -1;
+ } /* end if */
+#ifdef FILLVAL_WORKS
+ /* Currently fill values do not work because they can bump the dataspace
+ * message to the second object header chunk. We should enable the fillval
+ * here when this is fixed. -NAF 8/11/11 */
+ HDmemset(&fillval, 0, sizeof(fillval));
+ fillval.rec_id = (uint64_t)ULLONG_MAX;
+ if(H5Pset_fill_value(dcpl, tid, &fillval) < 0)
+ return -1;
+#endif /* FILLVAL_WORKS */
+
+ /* Emit informational message */
+ if(verbose)
+ HDfprintf(stderr, "Creating datasets\n");
+
+ /* Create the datasets */
+ for(u = 0; u < NLEVELS; u++)
+ for(v = 0; v < symbol_count[u]; v++) {
+ hid_t dsid; /* Dataset ID */
+ char name_buf[64];
+ hbool_t move_dataspace_message = FALSE; /* Whether to move the dataspace message out of object header chunk #0 */
+
+ generate_name(name_buf, u, v);
+ if((dsid = H5Dcreate2(fid, name_buf, tid, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
+ return -1;
+
+ /* Determine if the dataspace message for this dataset should be
+ * moved out of chunk #0 of the object header
+ * (Set to TRUE for every fourth dataset)
+ */
+ move_dataspace_message = !(HDrandom() % 4);
+ if(move_dataspace_message) {
+ unsigned chunk_num; /* Object header chunk # for dataspace message */
+
+ /* Move the dataspace message to a new object header chunk */
+ if(H5O__msg_move_to_new_chunk_test(dsid, H5O_SDSPACE_ID) < 0)
+ return -1;
+
+ /* Retrieve the chunk # for the dataspace message */
+ chunk_num = UINT_MAX;
+ if(H5O__msg_get_chunkno_test(dsid, H5O_SDSPACE_ID, &chunk_num) < 0)
+ return -1;
+ /* Should not be in chunk #0 for now */
+ if(0 == chunk_num)
+ return -1;
+ } /* end if */
+
+ if(H5Dclose(dsid) < 0)
+ return -1;
+ } /* end for */
+
+ /* Emit informational message */
+ if(verbose)
+ HDfprintf(stderr, "Closing objects\n");
+
+ /* Close everythign */
+ if(H5Pclose(dcpl) < 0)
+ return -1;
+ if(H5Sclose(sid) < 0)
+ return -1;
+ if(H5Tclose(tid) < 0)
+ return -1;
+ if(H5Fclose(fid) < 0)
+ return -1;
+
+ /* Free the config structure */
+ if(config)
+ HDfree(config);
+ return 0;
+} /* end gen_skeleton() */
+
+static void
+usage(void)
+{
+ printf("\n");
+ printf("Usage error!\n");
+ printf("\n");
+ printf("Usage: swmr_generator [-q] [-s] [-c <deflate compression level>]\n");
+ printf(" [-i <index type>] [-r <random seed>]\n");
+ printf("\n");
+ printf("NOTE: The random seed option is only used by the sparse test. Other\n");
+ printf(" tests specify the random seed as a reader/writer option.\n");
+ printf("\n");
+ printf("<deflate compression level> should be -1 (for no compression) or 0-9\n");
+ printf("\n");
+ printf("<index type> should be b2 or ea\n");
+ printf("\n");
+ printf("Defaults to verbose (no '-q' given), no SWMR_WRITE mode (no '-s' given) no\n");
+ printf("compression ('-c -1'), v1 b-tree indexing (-i b1), and will generate a random\n");
+ printf("seed (no -r given).\n");
+ printf("\n");
+ HDexit(1);
+} /* end usage() */
+
+int main(int argc, const char *argv[])
+{
+ int comp_level = -1; /* Compression level (-1 is no compression) */
+ hbool_t verbose = TRUE; /* Whether to emit some informational messages */
+ hbool_t swmr_write = FALSE; /* Whether to create file with SWMR_WRITE access */
+ const char *index_type = "b1"; /* Chunk index type */
+ hbool_t use_seed = FALSE; /* Set to TRUE if a seed was set on the command line */
+ unsigned random_seed = 0; /* Random # seed */
+ unsigned u; /* Local index variables */
+ int temp;
+
+ /* Parse command line options */
+ if(argc > 1) {
+ u = 1;
+ while(u < (unsigned)argc) {
+ if(argv[u][0] == '-') {
+ switch(argv[u][1]) {
+ /* Compress dataset chunks */
+ case 'c':
+ comp_level = HDatoi(argv[u + 1]);
+ if(comp_level < -1 || comp_level > 9)
+ usage();
+ u += 2;
+ break;
+
+ /* Chunk index type */
+ case 'i':
+ index_type = argv[u + 1];
+ if(HDstrcmp(index_type, "ea")
+ && HDstrcmp(index_type, "b2"))
+ usage();
+ u += 2;
+ break;
+
+ /* Random # seed */
+ case 'r':
+ use_seed = TRUE;
+ temp = HDatoi(argv[u + 1]);
+ if(temp < 0)
+ usage();
+ else
+ random_seed = (unsigned)temp;
+ u += 2;
+ break;
+
+ /* Be quiet */
+ case 'q':
+ verbose = FALSE;
+ u++;
+ break;
+
+ /* Run with SWMR_WRITE */
+ case 's':
+ swmr_write = TRUE;
+ u++;
+ break;
+
+ default:
+ usage();
+ break;
+ } /* end switch */
+ } /* end if */
+ } /* end while */
+ } /* end if */
+
+ /* Emit informational message */
+ if(verbose) {
+ HDfprintf(stderr, "Parameters:\n");
+ HDfprintf(stderr, "\tswmr writes %s\n", swmr_write ? "on" : "off");
+ HDfprintf(stderr, "\tcompression level = %d\n", comp_level);
+ HDfprintf(stderr, "\tindex type = %s\n", index_type);
+ } /* end if */
+
+ /* Set the random seed */
+ if(!use_seed) {
+ struct timeval t;
+
+ HDgettimeofday(&t, NULL);
+ random_seed = (unsigned)(t.tv_usec);
+ } /* end if */
+ HDsrandom(random_seed);
+ /* ALWAYS emit the random seed for possible debugging */
+ HDfprintf(stderr, "Using generator random seed (used in sparse test only): %u\n", random_seed);
+
+ /* Emit informational message */
+ if(verbose)
+ HDfprintf(stderr, "Generating skeleton file: %s\n", FILENAME);
+
+ /* Generate file skeleton */
+ if(gen_skeleton(FILENAME, verbose, swmr_write, comp_level, index_type, random_seed) < 0) {
+ HDfprintf(stderr, "Error generating skeleton file!\n");
+ HDexit(1);
+ } /* end if */
+
+ return 0;
+}
diff --git a/test/vfd_swmr_reader.c b/test/vfd_swmr_reader.c
new file mode 100644
index 0000000..d074771
--- /dev/null
+++ b/test/vfd_swmr_reader.c
@@ -0,0 +1,575 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * Copyright by the Board of Trustees of the University of Illinois. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*-------------------------------------------------------------------------
+ *
+ * Created: vfd_swmr_reader.c
+ *
+ * Purpose: Reads data from a randomly selected subset of the datasets
+ * in the SWMR test file.
+ *
+ * This program is intended to run concurrently with the
+ * swmr_writer program.
+ *
+ * This is copied from swmr_reader.c and modified for testing
+ * with VFD SWMR.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+/***********/
+/* Headers */
+/***********/
+
+#include "h5test.h"
+#include "vfd_swmr_common.h"
+
+/********************/
+/* Local Prototypes */
+/********************/
+
+static int check_dataset(hid_t fid, hbool_t verbose, FILE *verbose_file,
+ const char *sym_name, symbol_t *record, hid_t rec_sid);
+static int read_records(const char *filename, hbool_t verbose, FILE *verbose_file,
+ unsigned random_seed, unsigned long nseconds, unsigned poll_time,
+ unsigned ncommon, unsigned nrandom);
+
+/*******************/
+/* Local Variables */
+/*******************/
+
+static hid_t symbol_tid = -1; /* The type ID for the SWMR datasets */
+
+
+/*-------------------------------------------------------------------------
+ * Function: check_dataset
+ *
+ * Purpose: For a given dataset, checks to make sure that the stated
+ * and actual sizes are the same. If they are not, then
+ * we have an inconsistent dataset due to a SWMR error.
+ *
+ * Parameters: hid_t fid
+ * The SWMR test file's ID.
+ *
+ * hbool_t verbose
+ * Whether verbose console output is desired.
+ *
+ * FILE *verbose_file
+ * File handle for verbose output
+ *
+ * const char *sym_name
+ * The name of the dataset from which to read.
+ *
+ * symbol_t *record
+ * Memory for the record. Must be pre-allocated.
+ *
+ * hid_t rec_sid
+ * The memory dataspace for access. It's always the same so
+ * there is no need to re-create it every time this function
+ * is called.
+ *
+ * Return: Success: 0
+ * Failure: -1
+ *
+ *-------------------------------------------------------------------------
+ */
+static int
+check_dataset(hid_t fid, hbool_t verbose, FILE *verbose_file,
+ const char *sym_name, symbol_t *record, hid_t rec_sid)
+{
+ hid_t dsid; /* Dataset ID */
+ hid_t file_sid; /* Dataset's space ID */
+ hssize_t snpoints; /* Number of elements in dataset */
+ hsize_t start[2] = {0, 0}, count[2] = {1, 1}; /* Hyperslab selection values */
+
+ HDassert(fid >= 0);
+ HDassert(sym_name);
+ HDassert(record);
+ HDassert(rec_sid >= 0);
+
+ /* Open dataset for symbol */
+ if((dsid = H5Dopen2(fid, sym_name, H5P_DEFAULT)) < 0)
+ return -1;
+
+ /* Get the dataset's dataspace */
+ if((file_sid = H5Dget_space(dsid)) < 0)
+ return -1;
+
+ /* Get the number of elements (= records, for 1-D datasets) */
+ if((snpoints = H5Sget_simple_extent_npoints(file_sid)) < 0)
+ return -1;
+
+ /* Emit informational message */
+ if(verbose)
+ HDfprintf(verbose_file, "Symbol = '%s', # of records = %lld\n", sym_name, (long long)snpoints);
+
+ /* Check if there are records for symbol */
+ if(snpoints > 0) {
+ /* Choose the last record in the dataset */
+ start[1] = (hsize_t)(snpoints - 1);
+ if(H5Sselect_hyperslab(file_sid, H5S_SELECT_SET, start, NULL, count, NULL) < 0)
+ return -1;
+
+ /* Read record from dataset */
+ record->rec_id = (uint64_t)ULLONG_MAX;
+ if(H5Dread(dsid, symbol_tid, rec_sid, file_sid, H5P_DEFAULT, record) < 0)
+ return -1;
+
+ /* Verify record value */
+ if(record->rec_id != start[1]) {
+ struct timeval tv;
+
+ HDgettimeofday(&tv, NULL);
+
+ if(verbose) {
+ HDfprintf(verbose_file, "*** ERROR ***\n");
+ HDfprintf(verbose_file, "Incorrect record value!\n");
+ HDfprintf(verbose_file, "Time = %llu.%llu, Symbol = '%s', # of records = %lld, record->rec_id = %llu\n", (unsigned long long)tv.tv_sec, (unsigned long long)tv.tv_usec, sym_name, (long long)snpoints, (unsigned long long)record->rec_id);
+ } /* end if */
+ return -1;
+ } /* end if */
+ } /* end if */
+
+ /* Close the dataset's dataspace */
+ if(H5Sclose(file_sid) < 0)
+ return -1;
+
+ /* Close dataset for symbol */
+ if(H5Dclose(dsid) < 0)
+ return -1;
+
+ return 0;
+} /* end check_dataset() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: read_records
+ *
+ * Purpose: For a given dataset, checks to make sure that the stated
+ * and actual sizes are the same. If they are not, then
+ * we have an inconsistent dataset due to a SWMR error.
+ *
+ * The "common" datasets are a random selection from among
+ * the level 0 datasets. The "random" datasets are a random
+ * selection from among all the file's datasets. This scheme
+ * ensures that the level 0 datasets are interrogated vigorously.
+ *
+ * Parameters: const char *filename
+ * The SWMR test file's name.
+ *
+ * hbool_t verbose
+ * Whether verbose console output is desired.
+ *
+ * FILE *verbose_file
+ * File handle for verbose output
+ *
+ * unsigned random_seed
+ * Random seed for the file (used for verbose logging)
+ *
+ * unsigned long nseconds
+ * The amount of time to read records (ns).
+ *
+ * unsigned poll_time
+ * The amount of time to sleep (s).
+ *
+ * unsigned ncommon
+ * The number of common/non-random datasets that will be opened.
+ *
+ * unsigned nrandom
+ * The number of random datasets that will be opened.
+ *
+ * Return: Success: 0
+ * Failure: -1
+ *
+ *-------------------------------------------------------------------------
+ */
+static int
+read_records(const char *filename, hbool_t verbose, FILE *verbose_file,
+ unsigned random_seed, unsigned long nseconds, unsigned poll_time,
+ unsigned ncommon, unsigned nrandom)
+{
+ time_t start_time; /* Starting time */
+ time_t curr_time; /* Current time */
+ symbol_info_t **sym_com = NULL; /* Pointers to array of common dataset IDs */
+ symbol_info_t **sym_rand = NULL; /* Pointers to array of random dataset IDs */
+ hid_t mem_sid; /* Memory dataspace ID */
+ hid_t fid; /* SWMR test file ID */
+ hid_t fapl; /* file access property list */
+ symbol_t record; /* The record to read from the dataset */
+ unsigned v; /* Local index variable */
+ hbool_t use_log_vfd = FALSE; /* Use the log VFD (set this manually) */
+H5F_vfd_swmr_config_t *config = NULL; /* Configuration for VFD SWMR */
+
+ HDassert(filename);
+ HDassert(nseconds != 0);
+ HDassert(poll_time != 0);
+
+ /* Reset the record */
+ /* (record's 'info' field might need to change for each record read, also) */
+ HDmemset(&record, 0, sizeof(record));
+
+ /* Emit informational message */
+ if(verbose)
+ HDfprintf(verbose_file, "Choosing datasets\n");
+
+ /* Allocate space for 'common' datasets, if any */
+ if(ncommon > 0) {
+ /* Allocate array to hold pointers to symbols for common datasets */
+ if(NULL == (sym_com = (symbol_info_t **)HDmalloc(sizeof(symbol_info_t *) * ncommon)))
+ return -1;
+
+ /* Open the common datasets */
+ for(v = 0; v < ncommon; v++) {
+ unsigned offset; /* Offset of symbol to use */
+
+ /* Determine the offset of the symbol, within level 0 symbols */
+ /* (level 0 symbols are the most common symbols) */
+ offset = (unsigned)((unsigned)HDrandom() % symbol_count[0]);
+ sym_com[v] = &symbol_info[0][offset];
+
+ /* Emit informational message */
+ if(verbose)
+ HDfprintf(verbose_file, "Common symbol #%u = '%s'\n", v, symbol_info[0][offset].name);
+ } /* end for */
+ } /* end if */
+
+ /* Allocate space for 'random' datasets, if any */
+ if(nrandom > 0) {
+ /* Allocate array to hold pointers to symbols for random datasets */
+ if(NULL == (sym_rand = (symbol_info_t **)HDmalloc(sizeof(symbol_info_t *) * nrandom)))
+ return -1;
+
+ /* Determine the random datasets */
+ for(v = 0; v < nrandom; v++) {
+ symbol_info_t *sym; /* Symbol to use */
+
+ /* Determine the symbol, within all symbols */
+ if(NULL == (sym = choose_dataset()))
+ return -1;
+ sym_rand[v] = sym;
+
+ /* Emit informational message */
+ if(verbose)
+ HDfprintf(verbose_file, "Random symbol #%u = '%s'\n", v, sym->name);
+ } /* end for */
+ } /* end if */
+
+ /* Create a dataspace for the record to read */
+ if((mem_sid = H5Screate(H5S_SCALAR)) < 0)
+ return -1;
+
+ /* Emit informational message */
+ if(verbose)
+ HDfprintf(verbose_file, "Reading records\n");
+
+ /* Get the starting time */
+ start_time = HDtime(NULL);
+ curr_time = start_time;
+
+ /* Create file access property list */
+ if((fapl = h5_fileaccess()) < 0)
+ return -1;
+
+ /* Log I/O when verbose output it enbabled */
+ if(use_log_vfd) {
+ char verbose_name[1024];
+
+ HDsnprintf(verbose_name, sizeof(verbose_name), "swmr_reader.log.%u", random_seed);
+
+ H5Pset_fapl_log(fapl, verbose_name, H5FD_LOG_ALL, (size_t)(512 * 1024 * 1024));
+ } /* end if */
+
+ /* Loop over reading records until [at least] the correct # of seconds have passed */
+ while(curr_time < (time_t)(start_time + (time_t)nseconds)) {
+
+ /* Emit informational message */
+ if(verbose)
+ HDfprintf(verbose_file, "Opening file: %s\n", filename);
+
+ /*
+ * Set up to open the file with VFD SWMR configured.
+ */
+ /* Enable page buffering */
+ if(H5Pset_page_buffer_size(fapl, 4096, 0, 0) < 0)
+ return -1;
+
+ /* Allocate memory for the configuration structure */
+ if((config = (H5F_vfd_swmr_config_t *)HDmalloc(sizeof(H5F_vfd_swmr_config_t))) == NULL)
+ return -1;
+
+ config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION;
+ config->tick_len = 4;
+ config->max_lag = 6;
+ config->vfd_swmr_writer = FALSE;
+ config->md_pages_reserved = 2;
+ HDstrcpy(config->md_file_path, "./my_md_file");
+
+ /* Enable VFD SWMR configuration */
+ if(H5Pset_vfd_swmr_config(fapl, config) < 0)
+ return -1;
+
+ /* Open the file with VFD SWMR configured */
+ if((fid = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0)
+ return -1;
+
+ /* Check 'common' datasets, if any */
+ if(ncommon > 0) {
+ /* Emit informational message */
+ if(verbose)
+ HDfprintf(verbose_file, "Checking common symbols\n");
+
+ /* Iterate over common datasets */
+ for(v = 0; v < ncommon; v++) {
+ /* Check common dataset */
+ if(check_dataset(fid, verbose, verbose_file, sym_com[v]->name, &record, mem_sid) < 0)
+ return -1;
+ HDmemset(&record, 0, sizeof(record));
+ } /* end for */
+ } /* end if */
+
+ /* Check 'random' datasets, if any */
+ if(nrandom > 0) {
+ /* Emit informational message */
+ if(verbose)
+ HDfprintf(verbose_file, "Checking random symbols\n");
+
+ /* Iterate over random datasets */
+ for(v = 0; v < nrandom; v++) {
+ /* Check random dataset */
+ if(check_dataset(fid, verbose, verbose_file, sym_rand[v]->name, &record, mem_sid) < 0)
+ return -1;
+ HDmemset(&record, 0, sizeof(record));
+ } /* end for */
+ } /* end if */
+
+ /* Emit informational message */
+ if(verbose)
+ HDfprintf(verbose_file, "Closing file\n");
+
+ /* Close the file */
+ if(H5Fclose(fid) < 0)
+ return -1;
+
+ /* Sleep for the appropriate # of seconds */
+ HDsleep(poll_time);
+
+ /* Retrieve the current time */
+ curr_time = HDtime(NULL);
+ } /* end while */
+
+ /* Close the memory dataspace */
+ if(H5Sclose(mem_sid) < 0)
+ return -1;
+
+ /* Close the fapl */
+ if(H5Pclose(fapl) < 0)
+ return -1;
+
+ /* Emit informational message */
+ if(verbose)
+ HDfprintf(verbose_file, "Closing datasets\n");
+
+ /* Close 'random' datasets, if any */
+ if(nrandom > 0) {
+ /* Release array holding dataset ID's for random datasets */
+ HDfree(sym_rand);
+ } /* end if */
+
+ /* Close 'common' datasets, if any */
+ if(ncommon > 0) {
+ /* Release array holding dataset ID's for common datasets */
+ HDfree(sym_com);
+ } /* end if */
+
+ /* Free the config structure */
+ if(config)
+ HDfree(config);
+ return 0;
+} /* end read_records() */
+
+static void
+usage(void)
+{
+ printf("\n");
+ printf("Usage error!\n");
+ printf("\n");
+ printf("Usage: swmr_reader [-q] [-s <# of seconds to sleep between polling>]\n");
+ printf(" [-h <# of common symbols to poll>] [-l <# of random symbols to poll>]\n");
+ printf(" [-r <random seed>] <# of seconds to test>\n");
+ printf("\n");
+ printf("<# of seconds to test> must be specified.\n");
+ printf("\n");
+ printf("Defaults to verbose (no '-q' given), 1 second between polling ('-s 1'),\n");
+ printf("5 common symbols to poll ('-h 5'), 10 random symbols to poll ('-l 10'),\n");
+ printf("and will generate a random seed (no -r given).\n");
+ printf("\n");
+ HDexit(1);
+}
+
+int main(int argc, const char *argv[])
+{
+ long nseconds = 0; /* # of seconds to test */
+ int poll_time = 1; /* # of seconds between polling */
+ int ncommon = 5; /* # of common symbols to poll */
+ int nrandom = 10; /* # of random symbols to poll */
+ hbool_t verbose = TRUE; /* Whether to emit some informational messages */
+ FILE *verbose_file = NULL; /* File handle for verbose output */
+ hbool_t use_seed = FALSE; /* Set to 1 if a seed was set on the command line */
+ unsigned random_seed = 0; /* Random # seed */
+ unsigned u; /* Local index variables */
+ int temp;
+
+ /* Parse command line options */
+ if(argc < 2)
+ usage();
+ if(argc > 1) {
+ u = 1;
+ while(u < (unsigned)argc) {
+ if(argv[u][0] == '-') {
+ switch(argv[u][1]) {
+ /* # of common symbols to poll */
+ case 'h':
+ ncommon = HDatoi(argv[u + 1]);
+ if(ncommon < 0)
+ usage();
+ u += 2;
+ break;
+
+ /* # of random symbols to poll */
+ case 'l':
+ nrandom = HDatoi(argv[u + 1]);
+ if(nrandom < 0)
+ usage();
+ u += 2;
+ break;
+
+ /* Be quiet */
+ case 'q':
+ verbose = FALSE;
+ u++;
+ break;
+
+ /* Random # seed */
+ case 'r':
+ use_seed = TRUE;
+ temp = HDatoi(argv[u + 1]);
+ if(temp < 0)
+ usage();
+ else
+ random_seed = (unsigned)temp;
+ u += 2;
+ break;
+
+ /* # of seconds between polling */
+ case 's':
+ poll_time = HDatoi(argv[u + 1]);
+ if(poll_time < 0)
+ usage();
+ u += 2;
+ break;
+
+ default:
+ usage();
+ break;
+ } /* end switch */
+ } /* end if */
+ else {
+ /* Get the number of records to append */
+ nseconds = HDatol(argv[u]);
+ if(nseconds <= 0)
+ usage();
+
+ u++;
+ } /* end else */
+ } /* end while */
+ } /* end if */
+ if(nseconds <= 0)
+ usage();
+ if(poll_time >= nseconds)
+ usage();
+
+ /* Set the random seed */
+ if(!use_seed) {
+ struct timeval t;
+
+ HDgettimeofday(&t, NULL);
+ random_seed = (unsigned)(t.tv_usec);
+ } /* end if */
+ HDsrandom(random_seed);
+
+ /* Open output file */
+ if(verbose) {
+ char verbose_name[1024];
+
+ HDsnprintf(verbose_name, sizeof(verbose_name), "swmr_reader.out.%u", random_seed);
+ if(NULL == (verbose_file = HDfopen(verbose_name, "w"))) {
+ HDfprintf(stderr, "Can't open verbose output file!\n");
+ HDexit(1);
+ }
+ } /* end if */
+
+ /* Emit informational message */
+ if(verbose) {
+ HDfprintf(verbose_file, "Parameters:\n");
+ HDfprintf(verbose_file, "\t# of seconds between polling = %d\n", poll_time);
+ HDfprintf(verbose_file, "\t# of common symbols to poll = %d\n", ncommon);
+ HDfprintf(verbose_file, "\t# of random symbols to poll = %d\n", nrandom);
+ HDfprintf(verbose_file, "\t# of seconds to test = %ld\n", nseconds);
+ } /* end if */
+
+ /* ALWAYS emit the random seed for possible debugging */
+ HDfprintf(stdout, "Using reader random seed: %u\n", random_seed);
+
+ /* Emit informational message */
+ if(verbose)
+ HDfprintf(verbose_file, "Generating symbol names\n");
+
+ /* Generate dataset names */
+ if(generate_symbols() < 0) {
+ HDfprintf(stderr, "Error generating symbol names!\n");
+ HDexit(1);
+ } /* end if */
+
+ /* Create datatype for creating datasets */
+ if((symbol_tid = create_symbol_datatype()) < 0)
+ return -1;
+
+ /* Reading records from datasets */
+ if(read_records(FILENAME, verbose, verbose_file, random_seed, (unsigned long)nseconds, (unsigned)poll_time, (unsigned)ncommon, (unsigned)nrandom) < 0) {
+ HDfprintf(stderr, "Error reading records from datasets (random_seed = %u)!\n", random_seed);
+ HDexit(1);
+ } /* end if */
+
+ /* Emit informational message */
+ if(verbose)
+ HDfprintf(verbose_file, "Releasing symbols\n");
+
+ /* Clean up the symbols */
+ if(shutdown_symbols() < 0) {
+ HDfprintf(stderr, "Error releasing symbols!\n");
+ HDexit(1);
+ } /* end if */
+
+ /* Emit informational message */
+ if(verbose)
+ HDfprintf(verbose_file, "Closing objects\n");
+
+ /* Close objects created */
+ if(H5Tclose(symbol_tid) < 0) {
+ HDfprintf(stderr, "Error closing symbol datatype!\n");
+ HDexit(1);
+ } /* end if */
+
+ return 0;
+}
+
diff --git a/test/vfd_swmr_writer.c b/test/vfd_swmr_writer.c
new file mode 100644
index 0000000..bf4acb7
--- /dev/null
+++ b/test/vfd_swmr_writer.c
@@ -0,0 +1,473 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * Copyright by the Board of Trustees of the University of Illinois. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*-------------------------------------------------------------------------
+ *
+ * Created: swmr_writer.c
+ *
+ * Purpose: Writes data to a randomly selected subset of the datasets
+ * in the SWMR test file.
+ *
+ * This program is intended to run concurrently with the
+ * swmr_reader program.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+/***********/
+/* Headers */
+/***********/
+
+#include "h5test.h"
+#include "vfd_swmr_common.h"
+
+/********************/
+/* Local Prototypes */
+/********************/
+
+static hid_t open_skeleton(const char *filename, hbool_t verbose, FILE *verbose_file,
+ unsigned random_seed, hbool_t old);
+static int add_records(hid_t fid, hbool_t verbose, FILE *verbose_file,
+ unsigned long nrecords, unsigned long flush_count);
+static void usage(void);
+
+
+/*-------------------------------------------------------------------------
+ * Function: open_skeleton
+ *
+ * Purpose: Opens the SWMR HDF5 file and datasets.
+ *
+ * Parameters: const char *filename
+ * The filename of the SWMR HDF5 file to open
+ *
+ * hbool_t verbose
+ * Whether or not to emit verbose console messages
+ *
+ * FILE *verbose_file
+ * File handle for verbose output
+ *
+ * unsigned random_seed
+ * Random seed for the file (used for verbose logging)
+ *
+ * hbool_t old
+ * Whether to write in "old" file format
+ *
+ * Return: Success: The file ID of the opened SWMR file
+ * The dataset IDs are stored in a global array
+ *
+ * Failure: -1
+ *
+ *-------------------------------------------------------------------------
+ */
+static hid_t
+open_skeleton(const char *filename, hbool_t verbose, FILE *verbose_file,
+ unsigned random_seed, hbool_t old)
+{
+ hid_t fid; /* File ID for new HDF5 file */
+ hid_t fapl; /* File access property list */
+ unsigned u, v; /* Local index variable */
+ hbool_t use_log_vfd = FALSE; /* Use the log VFD (set this manually) */
+H5F_vfd_swmr_config_t *config = NULL; /* Configuration for VFD SWMR */
+
+ HDassert(filename);
+
+ /* Create file access property list */
+ if((fapl = h5_fileaccess()) < 0)
+ return -1;
+
+ /* FOR NOW: set to use latest format, the "old"parameter is not used */
+ if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
+ return -1;
+
+#ifdef QAK
+ /* Increase the initial size of the metadata cache */
+ {
+ H5AC_cache_config_t mdc_config;
+
+ mdc_config.version = H5AC__CURR_CACHE_CONFIG_VERSION;
+ H5Pget_mdc_config(fapl, &mdc_config);
+ HDfprintf(stderr, "mdc_config.initial_size = %lu\n", (unsigned long)mdc_config.initial_size);
+ HDfprintf(stderr, "mdc_config.epoch_length = %lu\n", (unsigned long)mdc_config.epoch_length);
+ mdc_config.set_initial_size = 1;
+ mdc_config.initial_size = 16 * 1024 * 1024;
+ /* mdc_config.epoch_length = 5000; */
+ H5Pset_mdc_config(fapl, &mdc_config);
+ }
+#endif /* QAK */
+
+ if(use_log_vfd) {
+ char verbose_name[1024];
+
+ HDsnprintf(verbose_name, sizeof(verbose_name), "swmr_writer.log.%u", random_seed);
+
+ H5Pset_fapl_log(fapl, verbose_name, H5FD_LOG_ALL, (size_t)(512 * 1024 * 1024));
+ } /* end if */
+
+ /*
+ * Set up to open the file with VFD SWMR configured.
+ */
+
+ /* Enable page buffering */
+ if(H5Pset_page_buffer_size(fapl, 4096, 0, 0) < 0)
+ return -1;
+
+ /* Allocate memory for the configuration structure */
+ if((config = (H5F_vfd_swmr_config_t *)HDmalloc(sizeof(H5F_vfd_swmr_config_t))) == NULL)
+ return -1;
+
+ config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION;
+ config->tick_len = 4;
+ config->max_lag = 6;
+ config->vfd_swmr_writer = TRUE;
+ config->md_pages_reserved = 2;
+ HDstrcpy(config->md_file_path, "./my_md_file");
+
+ /* Enable VFD SWMR configuration */
+ if(H5Pset_vfd_swmr_config(fapl, config) < 0)
+ return -1;
+
+ /* Open the file with VFD SWMR configured */
+ if((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0)
+ return -1;
+
+ /* Close file access property list */
+ if(H5Pclose(fapl) < 0)
+ return -1;
+
+ /* Emit informational message */
+ if(verbose)
+ HDfprintf(verbose_file, "Opening datasets\n");
+
+ /* Open the datasets */
+ for(u = 0; u < NLEVELS; u++)
+ for(v = 0; v < symbol_count[u]; v++) {
+ if((symbol_info[u][v].dsid = H5Dopen2(fid, symbol_info[u][v].name, H5P_DEFAULT)) < 0)
+ return -1;
+ symbol_info[u][v].nrecords = 0;
+ } /* end for */
+
+ if(config)
+ HDfree(config);
+ return fid;
+} /* open_skeleton() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: add_records
+ *
+ * Purpose: Writes a specified number of records to random datasets in
+ * the SWMR test file.
+ *
+ * Parameters: hid_t fid
+ * The file ID of the SWMR HDF5 file
+ *
+ * hbool_t verbose
+ * Whether or not to emit verbose console messages
+ *
+ * FILE *verbose_file
+ * File handle for verbose output
+ *
+ * unsigned long nrecords
+ * # of records to write to the datasets
+ *
+ * unsigned long flush_count
+ * # of records to write before flushing the file to disk
+ *
+ * Return: Success: 0
+ * Failure: -1
+ *
+ *-------------------------------------------------------------------------
+ */
+static int
+add_records(hid_t fid, hbool_t verbose, FILE *verbose_file,
+ unsigned long nrecords, unsigned long flush_count)
+{
+ hid_t tid; /* Datatype ID for records */
+ hid_t mem_sid; /* Memory dataspace ID */
+ hsize_t start[2] = {0, 0}, count[2] = {1, 1}; /* Hyperslab selection values */
+ hsize_t dim[2] = {1, 0}; /* Dataspace dimensions */
+ symbol_t record; /* The record to add to the dataset */
+ unsigned long rec_to_flush; /* # of records left to write before flush */
+ unsigned long u, v; /* Local index variables */
+
+ HDassert(fid >= 0);
+
+ /* Reset the record */
+ /* (record's 'info' field might need to change for each record written, also) */
+ HDmemset(&record, 0, sizeof(record));
+
+ /* Create a dataspace for the record to add */
+ if((mem_sid = H5Screate(H5S_SCALAR)) < 0)
+ return -1;
+
+ /* Create datatype for appending records */
+ if((tid = create_symbol_datatype()) < 0)
+ return -1;
+
+ /* Add records to random datasets, according to frequency distribution */
+ rec_to_flush = flush_count;
+ for(u = 0; u < nrecords; u++) {
+ symbol_info_t *symbol; /* Symbol to write record to */
+ hid_t file_sid; /* Dataset's space ID */
+
+ /* Get a random dataset, according to the symbol distribution */
+ symbol = choose_dataset();
+
+ /* Set the record's ID (equal to its position) */
+ record.rec_id = symbol->nrecords;
+
+ /* Get the coordinate to write */
+ start[1] = symbol->nrecords;
+
+ /* Cork the metadata cache, to prevent the object header from being
+ * flushed before the data has been written */
+ if(H5Odisable_mdc_flushes(symbol->dsid) < 0)
+ return -1;
+
+ /* Extend the dataset's dataspace to hold the new record */
+ symbol->nrecords++;
+ dim[1] = symbol->nrecords;
+ if(H5Dset_extent(symbol->dsid, dim) < 0)
+ return -1;
+
+ /* Get the dataset's dataspace */
+ if((file_sid = H5Dget_space(symbol->dsid)) < 0)
+ return -1;
+
+ /* Choose the last record in the dataset */
+ if(H5Sselect_hyperslab(file_sid, H5S_SELECT_SET, start, NULL, count, NULL) < 0)
+ return -1;
+
+ /* Write record to the dataset */
+ if(H5Dwrite(symbol->dsid, tid, mem_sid, file_sid, H5P_DEFAULT, &record) < 0)
+ return -1;
+
+ /* Uncork the metadata cache */
+ if(H5Oenable_mdc_flushes(symbol->dsid) < 0)
+ return -1;
+
+ /* Close the dataset's dataspace */
+ if(H5Sclose(file_sid) < 0)
+ return -1;
+
+ /* Check for flushing file */
+ if(flush_count > 0) {
+ /* Decrement count of records to write before flushing */
+ rec_to_flush--;
+
+ /* Check for counter being reached */
+ if(0 == rec_to_flush) {
+ /* Flush contents of file */
+ if(H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0)
+ return -1;
+
+ /* Reset flush counter */
+ rec_to_flush = flush_count;
+ } /* end if */
+ } /* end if */
+ } /* end for */
+
+ /* Close the memory dataspace */
+ if(H5Sclose(mem_sid) < 0)
+ return -1;
+
+ /* Close the datatype */
+ if(H5Tclose(tid) < 0)
+ return -1;
+
+ /* Emit informational message */
+ if(verbose)
+ HDfprintf(verbose_file, "Closing datasets\n");
+
+ /* Close the datasets */
+ for(u = 0; u < NLEVELS; u++)
+ for(v = 0; v < symbol_count[u]; v++)
+ if(H5Dclose(symbol_info[u][v].dsid) < 0)
+ return -1;
+
+ return 0;
+}
+
+static void
+usage(void)
+{
+ printf("\n");
+ printf("Usage error!\n");
+ printf("\n");
+ printf("Usage: swmr_writer [-q] [-o] [-f <# of records to write between flushing\n");
+ printf(" file contents>] [-r <random seed>] <# of records>\n");
+ printf("\n");
+ printf("<# of records to write between flushing file contents> should be 0\n");
+ printf("(for no flushing) or between 1 and (<# of records> - 1).\n");
+ printf("\n");
+ printf("<# of records> must be specified.\n");
+ printf("\n");
+ printf("Defaults to verbose (no '-q' given), latest format when opening file (no '-o' given),\n");
+ printf("flushing every 10000 records ('-f 10000'), and will generate a random seed (no -r given).\n");
+ printf("\n");
+ HDexit(1);
+}
+
+int main(int argc, const char *argv[])
+{
+ hid_t fid; /* File ID for file opened */
+ long nrecords = 0; /* # of records to append */
+ long flush_count = 10000; /* # of records to write between flushing file */
+ hbool_t verbose = TRUE; /* Whether to emit some informational messages */
+ FILE *verbose_file = NULL; /* File handle for verbose output */
+ hbool_t old = FALSE; /* Whether to use non-latest-format when opening file */
+ hbool_t use_seed = FALSE; /* Set to TRUE if a seed was set on the command line */
+ unsigned random_seed = 0; /* Random # seed */
+ unsigned u; /* Local index variable */
+ int temp;
+
+ /* Parse command line options */
+ if(argc < 2)
+ usage();
+ if(argc > 1) {
+ u = 1;
+ while(u < (unsigned)argc) {
+ if(argv[u][0] == '-') {
+ switch(argv[u][1]) {
+ /* # of records to write between flushing file */
+ case 'f':
+ flush_count = HDatol(argv[u + 1]);
+ if(flush_count < 0)
+ usage();
+ u += 2;
+ break;
+
+ /* Be quiet */
+ case 'q':
+ verbose = FALSE;
+ u++;
+ break;
+
+ /* Random # seed */
+ case 'r':
+ use_seed = TRUE;
+ temp = HDatoi(argv[u + 1]);
+ random_seed = (unsigned)temp;
+ u += 2;
+ break;
+
+ /* Use non-latest-format when opening file */
+ case 'o':
+ old = TRUE;
+ u++;
+ break;
+
+ default:
+ usage();
+ break;
+ } /* end switch */
+ } /* end if */
+ else {
+ /* Get the number of records to append */
+ nrecords = HDatol(argv[u]);
+ if(nrecords <= 0)
+ usage();
+
+ u++;
+ } /* end else */
+ } /* end while */
+ } /* end if */
+ if(nrecords <= 0)
+ usage();
+ if(flush_count >= nrecords)
+ usage();
+
+ /* Set the random seed */
+ if(!use_seed) {
+ struct timeval t;
+
+ HDgettimeofday(&t, NULL);
+ random_seed = (unsigned)(t.tv_usec);
+ } /* end if */
+ HDsrandom(random_seed);
+
+ /* Open output file */
+ if(verbose) {
+ char verbose_name[1024];
+
+ HDsnprintf(verbose_name, sizeof(verbose_name), "swmr_writer.out.%u", random_seed);
+ if(NULL == (verbose_file = HDfopen(verbose_name, "w"))) {
+ HDfprintf(stderr, "Can't open verbose output file!\n");
+ HDexit(1);
+ }
+ } /* end if */
+
+ /* Emit informational message */
+ if(verbose) {
+ HDfprintf(verbose_file, "Parameters:\n");
+ HDfprintf(verbose_file, "\t# of records between flushes = %ld\n", flush_count);
+ HDfprintf(verbose_file, "\t# of records to write = %ld\n", nrecords);
+ } /* end if */
+
+ /* ALWAYS emit the random seed for possible debugging */
+ HDfprintf(stdout, "Using writer random seed: %u\n", random_seed);
+
+ /* Emit informational message */
+ if(verbose)
+ HDfprintf(verbose_file, "Generating symbol names\n");
+
+ /* Generate dataset names */
+ if(generate_symbols() < 0)
+ return -1;
+
+ /* Emit informational message */
+ if(verbose)
+ HDfprintf(verbose_file, "Opening skeleton file: %s\n", FILENAME);
+
+ /* Open file skeleton */
+ if((fid = open_skeleton(FILENAME, verbose, verbose_file, random_seed, old)) < 0) {
+ HDfprintf(stderr, "Error opening skeleton file!\n");
+ HDexit(1);
+ } /* end if */
+
+ /* Send a message to indicate "H5Fopen" is complete--releasing the file lock */
+ h5_send_message(WRITER_MESSAGE, NULL, NULL);
+
+ /* Emit informational message */
+ if(verbose)
+ HDfprintf(verbose_file, "Adding records\n");
+
+ /* Append records to datasets */
+ if(add_records(fid, verbose, verbose_file, (unsigned long)nrecords, (unsigned long)flush_count) < 0) {
+ HDfprintf(stderr, "Error appending records to datasets!\n");
+ HDexit(1);
+ } /* end if */
+
+ /* Emit informational message */
+ if(verbose)
+ HDfprintf(verbose_file, "Releasing symbols\n");
+
+ /* Clean up the symbols */
+ if(shutdown_symbols() < 0) {
+ HDfprintf(stderr, "Error releasing symbols!\n");
+ HDexit(1);
+ } /* end if */
+
+ /* Emit informational message */
+ if(verbose)
+ HDfprintf(verbose_file, "Closing objects\n");
+
+ /* Close objects opened */
+ if(H5Fclose(fid) < 0) {
+ HDfprintf(stderr, "Error closing file!\n");
+ HDexit(1);
+ } /* end if */
+
+ return 0;
+}
+
generated by cgit v0.12 at 2024-07-17 15:00:13 (GMT)