summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorvchoi <vchoi@jelly.ad.hdfgroup.org>2021-11-17 17:25:20 (GMT)
committervchoi <vchoi@jelly.ad.hdfgroup.org>2021-11-17 17:25:20 (GMT)
commit97a37aa84913808792418e6814126bdee856288e (patch)
treee613e8f34a3988777909e0fa813dd8b7ee60df44
parentf3293556b7af7191496909f4577e983da19a9e68 (diff)
downloadhdf5-97a37aa84913808792418e6814126bdee856288e.zip
hdf5-97a37aa84913808792418e6814126bdee856288e.tar.gz
hdf5-97a37aa84913808792418e6814126bdee856288e.tar.bz2
1) Core changes for adding NFS/updater support as described in the RFC.
src/H5Pfapl.c src/H5Fvfd_swmr.c src/H5Fpublic.h src/H5Fpkg.h src/H5Fprivate.h 2) For VFD SWMR testing, add private property for checksum generation of metadata files: src/H5Fint.c src/H5Fvfd_swmr.c src/H5Pfapl.c src/H5Fpkg.h src/H5Fprivate.h 3) Fix the following in H5F_vfd_swmr_init() and H5F_vfd_swmr_close_or_flush(): (a) Allocate metadata file index right after metadata file header. (b) Set tick number to 0 when creating header and index for file open case. (c) Remove tick number increment at file close. src/H5Fvfd_swmr.c src/H5Ftest.c 4) To be consistent with the RFC, change the name for field "chksum" to "checksum" in struct H5FD_vfd_swmr_idx_entry_t: src/H5FDprivate.h src/H5FDtest.c src/H5FDvfd_swmr.c src/H5Ftest.c src/H5PB.c 4) Add tests for NFS/updater test/vfd_swmr.c 5) Modify common routine init_vfd_swmr_config() to accept updater_file_path test/vfd_swmr_common.c test/vfd_swmr_common.h 6) Changes to the tests due to the common routine init_vfd_swmr_config(): test/vfd_swmr_addrem_writer.c test/vfd_swmr_attrdset_writer.c test/vfd_swmr_bigset_writer.c test/vfd_swmr_dsetchks_writer.c test/vfd_swmr_dsetops_writer.c test/vfd_swmr_generator.c test/vfd_swmr_gfail_writer.c test/vfd_swmr_gperf_writer.c test/vfd_swmr_group_writer.c test/vfd_swmr_reader.c test/vfd_swmr_remove_reader.c test/vfd_swmr_remove_writer.c test/vfd_swmr_sparse_reader.c test/vfd_swmr_sparse_writer.c test/vfd_swmr_vlstr_reader.c test/vfd_swmr_vlstr_writer.c test/vfd_swmr_writer.c test/page_buffer.c
-rw-r--r--src/H5FDprivate.h2
-rw-r--r--src/H5FDtest.c2
-rw-r--r--src/H5FDvfd_swmr.c4
-rw-r--r--src/H5Fint.c8
-rw-r--r--src/H5Fpkg.h27
-rw-r--r--src/H5Fprivate.h249
-rw-r--r--src/H5Fpublic.h43
-rw-r--r--src/H5Ftest.c26
-rw-r--r--src/H5Fvfd_swmr.c604
-rw-r--r--src/H5PB.c2
-rw-r--r--src/H5Pfapl.c62
-rw-r--r--test/page_buffer.c2
-rw-r--r--test/vfd_swmr.c1378
-rw-r--r--test/vfd_swmr_addrem_writer.c5
-rw-r--r--test/vfd_swmr_attrdset_writer.c5
-rw-r--r--test/vfd_swmr_bigset_writer.c6
-rw-r--r--test/vfd_swmr_common.c13
-rw-r--r--test/vfd_swmr_common.h8
-rw-r--r--test/vfd_swmr_dsetchks_writer.c5
-rw-r--r--test/vfd_swmr_dsetops_writer.c5
-rw-r--r--test/vfd_swmr_generator.c5
-rw-r--r--test/vfd_swmr_gfail_writer.c5
-rw-r--r--test/vfd_swmr_gperf_writer.c5
-rw-r--r--test/vfd_swmr_group_writer.c5
-rw-r--r--test/vfd_swmr_reader.c5
-rw-r--r--test/vfd_swmr_remove_reader.c5
-rw-r--r--test/vfd_swmr_remove_writer.c5
-rw-r--r--test/vfd_swmr_sparse_reader.c5
-rw-r--r--test/vfd_swmr_sparse_writer.c5
-rw-r--r--test/vfd_swmr_vlstr_reader.c5
-rw-r--r--test/vfd_swmr_vlstr_writer.c5
-rw-r--r--test/vfd_swmr_writer.c5
-rw-r--r--test/vfd_swmr_zoo_writer.c5
33 files changed, 2147 insertions, 374 deletions
diff --git a/src/H5FDprivate.h b/src/H5FDprivate.h
index 0a8850f..7c0dd77 100644
--- a/src/H5FDprivate.h
+++ b/src/H5FDprivate.h
@@ -168,7 +168,7 @@ typedef struct H5FD_vfd_swmr_idx_entry_t {
uint64_t hdf5_page_offset;
uint64_t md_file_page_offset;
uint32_t length;
- uint32_t chksum;
+ uint32_t checksum;
void * entry_ptr;
uint64_t tick_of_last_change;
hbool_t clean;
diff --git a/src/H5FDtest.c b/src/H5FDtest.c
index a55d34c..521886f 100644
--- a/src/H5FDtest.c
+++ b/src/H5FDtest.c
@@ -162,7 +162,7 @@ H5FD__vfd_swmr_reader_md_test(H5FD_t *file, unsigned num_entries, H5FD_vfd_swmr_
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL,
"incorrect md_file_page_offset read from metadata file")
- if (vfd_index[i].chksum != index[i].chksum)
+ if (vfd_index[i].checksum != index[i].checksum)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "incorrect chksum read from metadata file")
}
}
diff --git a/src/H5FDvfd_swmr.c b/src/H5FDvfd_swmr.c
index 568f5d9..73d8df4 100644
--- a/src/H5FDvfd_swmr.c
+++ b/src/H5FDvfd_swmr.c
@@ -857,7 +857,7 @@ H5FD__vfd_swmr_read(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id
* signature.
*/
if (file->pb_configured && entry->length == init_size &&
- H5_checksum_metadata(buf, entry->length, 0) != entry->chksum) {
+ H5_checksum_metadata(buf, entry->length, 0) != entry->checksum) {
H5FD_vfd_swmr_md_header tmp_header;
if (H5FD__vfd_swmr_header_deserialize(file, &tmp_header) != TRUE)
@@ -1331,7 +1331,7 @@ H5FD__vfd_swmr_index_deserialize(const H5FD_vfd_swmr_t *file, H5FD_vfd_swmr_md_i
UINT32DECODE(p, md_index->entries[i].hdf5_page_offset);
UINT32DECODE(p, md_index->entries[i].md_file_page_offset);
UINT32DECODE(p, md_index->entries[i].length);
- UINT32DECODE(p, md_index->entries[i].chksum);
+ UINT32DECODE(p, md_index->entries[i].checksum);
}
}
else
diff --git a/src/H5Fint.c b/src/H5Fint.c
index c3fc346..8518785 100644
--- a/src/H5Fint.c
+++ b/src/H5Fint.c
@@ -1853,6 +1853,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id)
hbool_t ci_write = FALSE; /* Whether MDC CI write requested */
hbool_t file_create = FALSE; /* Creating a new file or not */
H5F_vfd_swmr_config_t *vfd_swmr_config_ptr = NULL; /* Points to VFD SMWR config info */
+ H5F_generate_md_ck_cb_t cb_info = {NULL};
H5F_t * ret_value = NULL; /* Actual return value */
FUNC_ENTER_NOAPI(NULL)
@@ -1876,6 +1877,10 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "file access is writer but VFD SWMR config is reader")
if ((flags & H5F_ACC_RDWR) == 0 && vfd_swmr_config_ptr->writer)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "file access is reader but VFD SWMR config is writer")
+
+ /* Retrieve the private property for VFD SWMR testing */
+ if (H5P_get(a_plist, H5F_ACS_GENERATE_MD_CK_CB_NAME, &cb_info) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get generate_md_ck_cb info")
}
/*
@@ -2111,6 +2116,9 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id)
/* Initialization for VFD SWMR writer and reader */
if (1 == shared->nrefs) {
+ /* Private property for VFD SWMR testing: generate checksum for metadata file */
+ if(cb_info.func)
+ shared->generate_md_ck_cb = cb_info.func;
if (H5F_vfd_swmr_init(file, file_create) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTSET, NULL, "file open fail with initialization for VFD SWMR")
}
diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h
index 706e41d..31630ef 100644
--- a/src/H5Fpkg.h
+++ b/src/H5Fpkg.h
@@ -455,13 +455,36 @@ struct H5F_shared_t {
uint32_t old_mdf_idx_len;
uint32_t old_mdf_idx_entries_used;
- /* Metadata file for VFD SWMR writer */
+ /* Metadata file and updater file for VFD SWMR writer */
int vfd_swmr_md_fd; /* POSIX: file descriptor for the
- * metadata file
+ * metadata file or -1 if the metadata file
+ * is not currently open.
+ * The vfd_swmr_config.generate_updater_files
+ * is FALSE.
*/
+ /* NFS:
+ * The vfd_swmr_config.generate_updater_files
+ * is TRUE and:
+ * --if vfd_swmr_config.writer is FALSE,
+ * this field is the file descriptor of the local
+ * copy of the metadata file, or -1 if the local
+ * copy is not currently open.
+ * --if vfd_swmr_config.writer is TRUE, this field
+ * is not used and is set to -1.
+ */
+ H5F_generate_md_ck_t generate_md_ck_cb;
+ /* For testing only:
+ * Invoke the user-defined callback if exists to
+ * generate checksum for the metadata file
+ */
+
haddr_t vfd_swmr_md_eoa; /* POSIX: eoa for the metadata
* file
*/
+ uint64_t updater_seq_num;/* Sequence number of the next updater file to be
+ * genereated. This field must be initialized to zero,
+ * and incremented after each updater file is generated.
+ */
/* Free space manager for the metadata file */
H5FS_t * fs_man_md; /* Free-space manager */
diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h
index cef77dc..05ace44 100644
--- a/src/H5Fprivate.h
+++ b/src/H5Fprivate.h
@@ -647,11 +647,18 @@ uint64_decode(uint8_t **pp)
{ \
/* int32_t version = */ 0, /* int32_t tick_len = */ 0, \
/* int32_t max_lag = */ 0, /* hbool_t vfd_swmr_writer = */ FALSE, \
+ /* hbool_t maintain_metadata_file = */ FALSE, \
+ /* hbool_t generate_updater_files = */ FALSE, \
/* hbool_t flush_raw_data = */ FALSE, /* int32_t md_pages_reserved = */ 0, \
- /* int32_t pb_expansion_threshold = */ 0, /* char md_file_path[] = */ "", \
+ /* int32_t pb_expansion_threshold = */ 0, \
+ /* char md_file_path[] = */ "", \
+ /* char updater_file_path[] = */ "", \
/* char log_file_path[] = */ "" \
}
+/* For VFD SWMR testing only: private property to generate checksum for metadata file via callback */
+#define H5F_ACS_GENERATE_MD_CK_CB_NAME "generate md ck callback"
+
/* ======================== File Mount properties ====================*/
#define H5F_MNT_SYM_LOCAL_NAME "local" /* Whether absolute symlinks local to file. */
@@ -784,6 +791,72 @@ uint64_decode(uint8_t **pp)
#define H5SM_TABLE_MAGIC "SMTB" /* Shared Message Table */
#define H5SM_LIST_MAGIC "SMLI" /* Shared Message List */
+/*
+ * VFD SWMR
+ */
+
+/* Updater file header */
+#define H5F_UD_VERSION 0 /* Version of the updater file format */
+#define H5F_UD_HEADER_OFF 0 /* Updater file header offset */
+#define H5F_UD_HEADER_MAGIC "VUDH" /* Updater file header magic */
+#define H5F_SIZEOF_CHKSUM 4 /* Size of checksum */
+
+/* Flags in the updater file header */
+#define CREATE_METADATA_FILE_ONLY_FLAG 0x0001
+#define FINAL_UPDATE_FLAG 0x0002
+
+/* Size of updater file header */
+#define H5F_UD_HEADER_SIZE \
+ (H5_SIZEOF_MAGIC /* Signature */ \
+ + 2 /* Version number */ \
+ + 2 /* Flags */ \
+ + 4 /* Page size */ \
+ + 8 /* Sequence number */ \
+ + 8 /* Tick number */ \
+ + 8 /* Chnage list offset */ \
+ + 8 /* Change list length */ \
+ + H5F_SIZEOF_CHKSUM /* Updater file header checksum */ \
+ )
+
+#define H5F_UD_CL_MAGIC "VUCL" /* Updater file change list magic */
+
+/* Size of an updater file change list entry */
+#define H5F_UD_CL_ENTRY_SIZE \
+ (4 /* Updater file page offset */ \
+ + 4 /* Metadata file page offset */ \
+ + 4 /* HDF5 file page offset */ \
+ + 4 /* Length */ \
+ + H5F_SIZEOF_CHKSUM /* Updater file change list entry checksum */ \
+ )
+
+/* Size of updater file change list */
+#define H5F_UD_CL_SIZE(N) /* N is number of change list entries */ \
+ (H5_SIZEOF_MAGIC /* Signature */ \
+ + 8 /* Tick num */ \
+ + 4 /* Metadata file header updater file page offset */ \
+ + 4 /* Metadata file header length */ \
+ + 4 /* Metadata file header checksum */ \
+ + 4 /* Metadata file index updater file page offset */ \
+ + 8 /* Metadata file index metadata file offset */ \
+ + 4 /* Metadata file index length */ \
+ + 4 /* Metadata file index checksum */ \
+ + 4 /* Number of change list entries */ \
+ + (N * H5F_UD_CL_ENTRY_SIZE) /* Change list entries */ \
+ + H5F_SIZEOF_CHKSUM /* Updater file change list checksum */ \
+ )
+
+/*
+ * For VFD SWMR testing only:
+ */
+
+/* Callback routine to generate checksum for metadata file specified by md_path */
+typedef herr_t (*H5F_generate_md_ck_t)(char *md_path, uint64_t updater_seq_num);
+
+/* Structure for "generate checksum callback" private property */
+typedef struct H5F_generate_md_ck_cb_t {
+ H5F_generate_md_ck_t func;
+} H5F_generate_md_ck_cb_t;
+
/****************************/
/* Library Private Typedefs */
/****************************/
@@ -854,6 +927,180 @@ typedef enum H5F_prefix_open_t {
H5F_PREFIX_EFILE = 2 /* External file prefix */
} H5F_prefix_open_t;
+/*
+ * VFD SWMR
+ */
+
+/*----------------------------------------------------------------------------
+ *
+ * struct H5F_vfd_swmr_updater_cl_entry_t
+ *
+ * An array of instances of H5F_vfd_swmr_updater_cl_entry_t of length equal to
+ * the number of metadata pages and multi-page metadata entries modified in
+ * the past tick is used ot aseemble the assoicated data in preparation for
+ * writing an updater file.
+ *
+ * Each entry in this array pertains to a given modified metdata page or
+ * multi-page metadata entry, and contains the following fields:
+ *
+ * entry_image_ptr: void pointer to a buffer containing the image of the
+ * target metadata page or multi-page metadata entry as modified in
+ * this tick, or NULL if undefined.
+ *
+ * entry_image_ud_file_page_offset: Page offset of the entry in the
+ * updater file, or 0 if undefined.
+ *
+ * entry_image_md_file_page_offset: Page offset of the entry in the
+ * metadata file, or 0 if undefined.
+ *
+ * entry_image_h5_file_page_offset: Page offset of the entry in the
+ * HDF5 file. In this case, a page offset of zero is valid,
+ * so we havd no easy marker for an invalid value. Instead,
+ * presume that this field is invalid if the entry_image_md_file_page_offset
+ * is invalid.
+ *
+ * entry_image_len: The size of the metadata page or multi-page metadata
+ * entry in bytes.
+ * entry_image_checksum: Checksum of the entry image.
+ *
+ *----------------------------------------------------------------------------
+ */
+typedef struct H5F_vfd_swmr_updater_cl_entry_t {
+ void *entry_image_ptr;
+ uint32_t entry_image_ud_file_page_offset;
+ uint32_t entry_image_md_file_page_offset;
+ uint32_t entry_image_h5_file_page_offset;
+ size_t entry_image_len;
+ uint32_t entry_image_checksum;
+} H5F_vfd_swmr_updater_cl_entry_t;
+
+/*----------------------------------------------------------------------------
+ *
+ * struct H5F_vfd_swmr_updater_t
+ *
+ * Instances of this structure are used to assemble the data required to
+ * write a metadata file updater file.
+ *
+ * Updater file header related fields:
+ *
+ * version: Version of the updater file format to be used. At present this
+ * must be zero.
+ *
+ * flags: This field contains any flags to be set in the updater file header.
+ * Currently defined flags are:
+ *
+ * 0x0001 CREATE_METADATA_FILE_ONLY_FLAG
+ * If set, the auxiliary process should create the metadata file,
+ * but leave it empty. This flag may only be set if sequence_num
+ * is zero.
+ *
+ * 0x0002 FINAL_UPDATE_FLAG
+ * If set, the VFD SWMR writer is closing the target file, and this
+ * updater contains the final set of updates to the metadata file.
+ * On receipt, the auxiliary process should apply the enclosed
+ * changes to the metadata file, unlink it, and exit.
+ *
+ * sequence_num: This field contains the sequence number of this updater file.
+ * The sequence number of the first updater file must be zero, and
+ * this sequence number must be increased by one for each new updater
+ * file. Note that under some circumstances, the sequence number
+ * will not match the tick_num.
+ *
+ * tick_num: Number of the tick for which this updater file is to be generated.
+ * This value should match that of the index used to fill our this
+ * structure.
+ *
+ * header_image_ptr: void pointer to the buffer in which the
+ * updater file header is constructed.
+ * This field is NULL if the buffer is undefined.
+ *
+ * header_image_len: This field contains the length of the updater file
+ * header in bytes.
+ *
+ * change_list_image_ptr: void pointer to a buffer containing the on disk image
+ * of the updater file change list, or NULL if that buffer does not exist.
+ *
+ * change_list_offset: This field contains the offset in bytes of the change
+ * list in the updater file. This will typically be the offset of
+ * the first byte in the updater file after the header.
+ *
+ * change_list_len: This field contains the size in bytes of the on disk image
+ * of the change list in the updater file.
+ *
+ * Updater File Change List Related Fields:
+ *
+ * The updater file change list is a section of the updater file that details the
+ * locations and lengths of all metadata file entries that must be modified for
+ * this tick.
+ *
+ * md_file_header_image_ptr: void pointer to a buffer containing the on disk image
+ * of the metadata file header as updated for tick_num.
+ *
+ * md_file_header_ud_file_page_offset: This field contains the updater file
+ * page offset of the metadata file header image. Note that we do
+ * not store the metadata file page offset of the metadata file header,
+ * as it is always written to offset 0 in the metadata file.
+ *
+ * md_file_header_len: This field contains the size of the metadata file header
+ * image in bytes.
+ *
+ * md_file_index_image: void pointer to a buffer containing the on disk image
+ * of the metadata file index as updated for tick_num.
+ *
+ * md_file_index_md_file_offset: This field contains the offset of the
+ * metadata file index in the metadata file in bytes.
+ *
+ * This value will either be the size of the metadata file header
+ * (if the metadata file header and index are adjacent), or a page
+ * aligned value.
+ *
+ * md_file_index_ud_file_page_offset: This field contains the page offset of the
+ * metadata file index in the updater file.
+ *
+ * md_file_index_len: This field contains the size of the metadata file index in
+ * bytes.
+ *
+ * num_change_list_entries: This field contains the number of entries in the
+ * array of H5F_vfd_swmr_updater_cl_ entry_t whose base address
+ * is stored in the change_list field below. This value is also the
+ * number of metadata pages and multi-page metadata entries that have
+ * been modified in the past tick
+ *
+ * If this field is zero, there is no change list, and the change_list
+ * field below is NULL.
+ *
+ * change_list: This field contains the base address of a dynamically allocated
+ * array of H5F_vfd_swmr_updater_cl_entry_t of length num_change_list_entries,
+ * or NULL if undefined.
+ *
+ *----------------------------------------------------------------------------
+ */
+typedef struct H5F_vfd_swmr_updater_t {
+ /* Updater file header related fields */
+ uint16_t version;
+ uint16_t flags;
+ uint32_t page_size;
+ uint64_t sequence_number;
+ uint64_t tick_num;
+ void *header_image_ptr;
+ size_t header_image_len;
+ void *change_list_image_ptr;
+ uint64_t change_list_offset;
+ size_t change_list_len;
+ /* Updater file change list related fields */
+ void *md_file_header_image_ptr;
+ uint32_t md_file_header_image_chksum;
+ uint32_t md_file_header_ud_file_page_offset;
+ size_t md_file_header_len;
+ void *md_file_index_image_ptr;
+ uint32_t md_file_index_image_chksum;
+ uint64_t md_file_index_md_file_offset;
+ uint32_t md_file_index_ud_file_page_offset;
+ size_t md_file_index_len;
+ uint32_t num_change_list_entries;
+ H5F_vfd_swmr_updater_cl_entry_t *change_list;
+} H5F_vfd_swmr_updater_t;
+
/*****************************/
/* Library-private Variables */
/*****************************/
diff --git a/src/H5Fpublic.h b/src/H5Fpublic.h
index d241816..5e0473a 100644
--- a/src/H5Fpublic.h
+++ b/src/H5Fpublic.h
@@ -275,6 +275,31 @@ typedef herr_t (*H5F_flush_cb_t)(hid_t object_id, void *udata);
* A boolean flag indicating whether the file opened with this FAPL entry
* will be opened R/W. (i.e. as a VFD SWMR writer)
*
+ * maintain_metadata_file
+ * A boolean flag indicating whether the writer should create and
+ * maintain the metadata file. Note that this field is only revelant
+ * if the above writer flag is TRUE.
+ * If this flag is TRUE, the writer must create and maintain the
+ * metadata file in the location specified in the md_file_path.
+ * Observe that at least one of maintain_metadata_file and
+ * generate_updater_files fields must be TRUE if writer is TRUE.
+ *
+ * generate_updater_files
+ * A boolean flag indicating whether the writer should generate a
+ * sequence of updater files describing how the metadata file
+ * should be updated at the end of each tick.
+ * If the flag is TRUE, all modifications to the metadata file
+ * (including its creation) are described in an ordered sequence of
+ * updater files. These files are read in order by auxiliary processes,
+ * and used to generate local copies of the metadata file as required.
+ * This mechanism exists to allow VFD SWMR to operate on storage
+ * systems that do not support POSIX semantics.
+ * This field is only used by the VFD SWMR writer. VFD SWMR readers
+ * ignore this field, as they always look to the specified metadata
+ * file, regardless of whether it is generated and maintained
+ * directly by the VFD SWMR writer, or by an auxiliary process that
+ * polls for new updater files, and applies them as they appear.
+ *
* flush_raw_data:
* A boolean flag indicating whether raw data should be flushed
* as part of the end of tick processing. If set to TRUE, raw
@@ -307,11 +332,16 @@ typedef herr_t (*H5F_flush_cb_t)(hid_t object_id, void *udata);
* of tick is triggered.
*
* md_file_path:
- * POSIX: this field contains the path of the metadata file.
- * NFS: it contains the path and base name of the metadata file
- * updater files.
- * Object store: it contains the base URL for the objects used
- * to store metadata file updater objects.
+ * If both the writer and maintain_metadata_file fields are TRUE, this
+ * field contains the path of the metadata file.
+ * If writer is FALSE, this field contains the path of the (possibly
+ * local copy of the) metadata file.
+ *
+ * updater_file_path:
+ * If generate_updater_files is TRUE, the contents of this field depends
+ * on whether the writer field is TRUE. If it is, the field contains
+ * the path and base name of the metadatea file updater files.
+ * If writer is FALSE, the field is ignored.
*
* log_file_path:
* This field contains the path to the log file. If defined, this path should
@@ -324,10 +354,13 @@ typedef struct H5F_vfd_swmr_config_t {
uint32_t tick_len;
uint32_t max_lag;
hbool_t writer;
+ hbool_t maintain_metadata_file;
+ hbool_t generate_updater_files;
hbool_t flush_raw_data;
uint32_t md_pages_reserved;
uint32_t pb_expansion_threshold;
char md_file_path[H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1];
+ char updater_file_path[H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1];
char log_file_path[H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1];
} H5F_vfd_swmr_config_t;
diff --git a/src/H5Ftest.c b/src/H5Ftest.c
index b76c579..e9cecc1 100644
--- a/src/H5Ftest.c
+++ b/src/H5Ftest.c
@@ -344,24 +344,10 @@ H5F__vfd_swmr_writer_create_open_flush_test(hid_t file_id, hbool_t file_create)
/* Verify the minimum size for the metadata file */
if (HDstat(f->shared->vfd_swmr_config.md_file_path, &stat_buf) < 0)
HGOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to stat the metadata file")
- if (stat_buf.st_size <
- (HDoff_t)((hsize_t)f->shared->vfd_swmr_config.md_pages_reserved * f->shared->fs_page_size))
- HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "incorrect metadata file size")
if (file_create) { /* Creating file */
- uint32_t hdr_magic;
-
- /* Seek to the beginning of the file */
- if (HDlseek(md_fd, (HDoff_t)H5FD_MD_HEADER_OFF, SEEK_SET) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_SEEKERROR, FAIL, "error seeking metadata file")
-
- /* Try to read the magic for header */
- if (HDread(md_fd, &hdr_magic, H5_SIZEOF_MAGIC) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_READERROR, FAIL, "error reading metadata file")
-
- /* Verify that there is no header magic in the metadata file */
- if (HDmemcmp(&hdr_magic, H5FD_MD_HEADER_MAGIC, (size_t)H5_SIZEOF_MAGIC) == 0)
- HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "error finding header magic in the metadata file")
+ if(stat_buf.st_size != 0)
+ HGOTO_ERROR(H5E_FILE, H5E_READERROR, FAIL, "metadata file should be empty for file create")
}
else { /* Opening or flushing the file */
@@ -494,7 +480,7 @@ H5F__vfd_swmr_decode_md_idx(int md_fd, H5FD_vfd_swmr_md_header *md_hdr, H5FD_vfd
UINT32DECODE(p, md_idx->entries[i].hdf5_page_offset);
UINT32DECODE(p, md_idx->entries[i].md_file_page_offset);
UINT32DECODE(p, md_idx->entries[i].length);
- UINT32DECODE(p, md_idx->entries[i].chksum);
+ UINT32DECODE(p, md_idx->entries[i].checksum);
} /* end for */
} /* end if */
@@ -546,7 +532,7 @@ H5F__vfd_swmr_verify_md_hdr_and_idx(H5F_t *f, H5FD_vfd_swmr_md_header *md_hdr, H
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "incorrect index_length read from metadata file")
/* Verify index_offset read from header in the metadata file is the size of md header */
- if (md_hdr->index_offset != f->shared->fs_page_size)
+ if (md_hdr->index_offset != H5FD_MD_HEADER_SIZE)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "incorrect index_offset read from metadata file")
/* Verify num_entries read from index in the metadata file is num_entries */
@@ -573,8 +559,8 @@ H5F__vfd_swmr_verify_md_hdr_and_idx(H5F_t *f, H5FD_vfd_swmr_md_header *md_hdr, H
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL,
"incorrect md_file_page_offset read from metadata file")
- if (md_idx->entries[i].chksum != index[i].chksum)
- HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "incorrect chksum read from metadata file")
+ if (md_idx->entries[i].checksum != index[i].checksum)
+ HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "incorrect checksum read from metadata file")
}
}
diff --git a/src/H5Fvfd_swmr.c b/src/H5Fvfd_swmr.c
index 8da238c..fe07e74 100644
--- a/src/H5Fvfd_swmr.c
+++ b/src/H5Fvfd_swmr.c
@@ -25,6 +25,7 @@
/****************/
#include "H5Fmodule.h" /* This source code file is part of the H5F module */
+#define H5FD_FRIEND /*suppress error about including H5FDpkg */
/***********/
/* Headers */
@@ -36,7 +37,7 @@
#include "H5Dprivate.h" /* Datasets */
#include "H5Eprivate.h" /* Error handling */
#include "H5Fpkg.h" /* File access */
-#include "H5FDprivate.h" /* File drivers */
+#include "H5FDpkg.h" /* File drivers */
#include "H5Gprivate.h" /* Groups */
#include "H5Iprivate.h" /* IDs */
#include "H5Lprivate.h" /* Links */
@@ -97,13 +98,19 @@ const unsigned int log_err_mesg_length = 14;
/********************/
static herr_t H5F__vfd_swmr_update_end_of_tick_and_tick_num(H5F_shared_t *, hbool_t);
-static herr_t H5F__vfd_swmr_construct_write_md_hdr(H5F_shared_t *, uint32_t);
+static herr_t H5F__vfd_swmr_construct_write_md_hdr(H5F_shared_t *, uint32_t, uint8_t *);
static herr_t H5F__vfd_swmr_construct_write_md_idx(H5F_shared_t *, uint32_t,
- struct H5FD_vfd_swmr_idx_entry_t[]);
+ struct H5FD_vfd_swmr_idx_entry_t[], uint8_t *);
static herr_t H5F__idx_entry_cmp(const void *_entry1, const void *_entry2);
static herr_t H5F__vfd_swmr_create_index(H5F_shared_t *);
static herr_t H5F__vfd_swmr_writer__wait_a_tick(H5F_t *);
+static herr_t H5F__vfd_swmr_construct_ud_hdr(H5F_vfd_swmr_updater_t *updater);
+static herr_t H5F__vfd_swmr_construct_ud_cl(H5F_vfd_swmr_updater_t *updater);
+static herr_t H5F__generate_updater_file(H5F_t *f, uint32_t num_entries, uint16_t flags,
+ uint8_t *md_file_hdr_image_ptr, size_t md_file_hdr_image_len,
+ uint8_t *md_file_index_image_ptr, uint64_t md_file_index_offset, size_t md_file_index_image_len);
+
/*********************/
/* Package Variables */
/*********************/
@@ -171,9 +178,11 @@ herr_t
H5F_vfd_swmr_init(H5F_t *f, hbool_t file_create)
{
hsize_t md_size; /* Size of the metadata file */
- haddr_t hdr_addr, idx_addr; /* Addresses returned from H5MV_alloc() */
- herr_t ret_value = SUCCEED; /* Return value */
+ haddr_t hdr_addr; /* Address returned from H5MV_alloc() */
H5F_shared_t *shared = f->shared;
+ uint8_t md_idx_image[H5FD_MD_INDEX_SIZE(0)]; /* Buffer for metadata file index */
+ uint8_t md_hdr_image[H5FD_MD_HEADER_SIZE]; /* Buffer for metadata file header */
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI(FAIL)
@@ -184,43 +193,26 @@ H5F_vfd_swmr_init(H5F_t *f, hbool_t file_create)
if (H5F_SHARED_INTENT(shared) & H5F_ACC_RDWR) {
HDassert(shared->vfd_swmr_config.writer);
+ HDassert(shared->vfd_swmr_config.maintain_metadata_file ||
+ shared->vfd_swmr_config.generate_updater_files);
SIMPLEQ_INIT(&shared->lower_defrees);
shared->vfd_swmr_writer = TRUE;
- shared->tick_num = 1;
-
- if (H5PB_vfd_swmr__set_tick(shared) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_SYSTEM, FAIL, "Can't update page buffer current tick")
+ shared->tick_num = 0;
/* Create the metadata file */
- if (((shared->vfd_swmr_md_fd = HDopen(shared->vfd_swmr_config.md_file_path, O_CREAT | O_RDWR,
+ if (((shared->vfd_swmr_md_fd = HDopen(shared->vfd_swmr_config.md_file_path, O_CREAT | O_RDWR | O_TRUNC,
H5_POSIX_CREATE_MODE_RW))) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "unable to create the metadata file")
md_size = (hsize_t)shared->vfd_swmr_config.md_pages_reserved * shared->fs_page_size;
- HDassert(shared->fs_page_size >= H5FD_MD_HEADER_SIZE);
-
- /* Allocate an entire page from the shadow file for the header. */
- if ((hdr_addr = H5MV_alloc(f, shared->fs_page_size)) == HADDR_UNDEF) {
+ if ((hdr_addr = H5MV_alloc(f, md_size)) == HADDR_UNDEF) {
HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error allocating shadow-file header");
}
HDassert(H5F_addr_eq(hdr_addr, H5FD_MD_HEADER_OFF));
- idx_addr = H5MV_alloc(f, md_size - shared->fs_page_size);
- if (idx_addr == HADDR_UNDEF) {
- HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error allocating shadow-file index");
- }
-
- HDassert(H5F_addr_eq(idx_addr, shared->fs_page_size));
-
- shared->writer_index_offset = idx_addr;
-
- /* Set the metadata file size to md_pages_reserved */
- if (-1 == HDftruncate(shared->vfd_swmr_md_fd, (HDoff_t)md_size))
- HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "truncate fail for the metadata file");
-
- /* Set eof for metadata file to md_pages_reserved */
+ shared->writer_index_offset = H5FD_MD_HEADER_SIZE;
shared->vfd_swmr_md_eoa = (haddr_t)md_size;
/* When opening an existing HDF5 file, create header and empty
@@ -228,12 +220,34 @@ H5F_vfd_swmr_init(H5F_t *f, hbool_t file_create)
*/
if (!file_create) {
- if (H5F__vfd_swmr_construct_write_md_idx(shared, 0, NULL) < 0)
+ if (H5F__vfd_swmr_construct_write_md_idx(shared, 0, NULL, md_idx_image) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "fail to create index in md");
- if (H5F__vfd_swmr_construct_write_md_hdr(shared, 0) < 0)
+ if (H5F__vfd_swmr_construct_write_md_hdr(shared, 0, md_hdr_image) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "fail to create header in md");
+
}
+
+ /* For VFD SWMR testing: invoke callback if set to generate metadata file checksum */
+ if(shared->generate_md_ck_cb) {
+ if(shared->generate_md_ck_cb(shared->vfd_swmr_config.md_file_path, shared->updater_seq_num) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_SYSTEM, FAIL, "error from generate_md_ck_cb()")
+ }
+
+ /* Generate updater files if configuration indicates so */
+ if(shared->vfd_swmr_config.generate_updater_files) {
+ shared->updater_seq_num = 0;
+ if(H5F__generate_updater_file(f, 0, file_create ? CREATE_METADATA_FILE_ONLY_FLAG : 0,
+ md_hdr_image, H5FD_MD_HEADER_SIZE,
+ md_idx_image, shared->writer_index_offset, H5FD_MD_INDEX_SIZE(0)) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_SYSTEM, FAIL, "can't generate updater file")
+ }
+
+ shared->tick_num = 1;
+
+ if (H5PB_vfd_swmr__set_tick(shared) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_SYSTEM, FAIL, "Can't update page buffer current tick")
+
}
else { /* VFD SWMR reader */
@@ -254,7 +268,6 @@ H5F_vfd_swmr_init(H5F_t *f, hbool_t file_create)
&(shared->mdf_idx_entries_used), shared->mdf_idx) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTLOAD, FAIL, "unable to load/decode metadata file");
- HDassert(shared->tick_num != 0);
vfd_swmr_reader_did_increase_tick_to(shared->tick_num);
#if 0 /* JRM */
@@ -310,6 +323,8 @@ H5F_vfd_swmr_close_or_flush(H5F_t *f, hbool_t closing)
{
H5F_shared_t * shared = f->shared;
shadow_defree_t *curr;
+ uint8_t md_idx_image[H5FD_MD_INDEX_SIZE(0)]; /* Buffer for metadata file index */
+ uint8_t md_hdr_image[H5FD_MD_HEADER_SIZE]; /* Buffer for metadata file header */
herr_t ret_value = SUCCEED;
FUNC_ENTER_NOAPI(FAIL)
@@ -318,22 +333,26 @@ H5F_vfd_swmr_close_or_flush(H5F_t *f, hbool_t closing)
HDassert(shared->vfd_swmr_md_fd >= 0);
/* Write empty index to the md file */
- if (H5F__vfd_swmr_construct_write_md_idx(shared, 0, NULL) < 0)
+ if (H5F__vfd_swmr_construct_write_md_idx(shared, 0, NULL, md_idx_image) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "fail to create index in md");
/* Write header to the md file */
- if (H5F__vfd_swmr_construct_write_md_hdr(shared, 0) < 0)
+ if (H5F__vfd_swmr_construct_write_md_hdr(shared, 0, md_hdr_image) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "fail to create header in md");
if (closing) { /* For file close */
- ++shared->tick_num;
-
/* Close the md file */
if (HDclose(shared->vfd_swmr_md_fd) < 0)
HSYS_GOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "unable to close the metadata file");
shared->vfd_swmr_md_fd = -1;
+ /* For VFD SWMR testing: invoke callback if set to generate metadata file checksum */
+ if(shared->generate_md_ck_cb) {
+ if(shared->generate_md_ck_cb(shared->vfd_swmr_config.md_file_path, shared->updater_seq_num) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_SYSTEM, FAIL, "error from generate_md_ck_cb()")
+ }
+
/* Unlink the md file */
if (HDunlink(shared->vfd_swmr_config.md_file_path) < 0)
HSYS_GOTO_ERROR(H5E_FILE, H5E_CANTREMOVE, FAIL, "unable to unlink the metadata file");
@@ -350,6 +369,13 @@ H5F_vfd_swmr_close_or_flush(H5F_t *f, hbool_t closing)
}
HDassert(TAILQ_EMPTY(&shared->shadow_defrees));
+
+ if(shared->vfd_swmr_config.generate_updater_files) {
+ if(H5F__generate_updater_file(f, 0, FINAL_UPDATE_FLAG,
+ md_hdr_image, H5FD_MD_HEADER_SIZE,
+ md_idx_image, shared->writer_index_offset, H5FD_MD_INDEX_SIZE(0)) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_SYSTEM, FAIL, "can't generate updater file")
+ }
}
else { /* For file flush */
/* Update end_of_tick */
@@ -438,6 +464,8 @@ H5F_update_vfd_swmr_metadata_file(H5F_t *f, uint32_t num_entries, H5FD_vfd_swmr_
shadow_defree_t *shadow_defree;
haddr_t md_addr; /* Address in the metadata file */
uint32_t i; /* Local index variable */
+ uint8_t *md_idx_image = NULL;
+ uint8_t md_hdr_image[H5FD_MD_HEADER_SIZE]; /* Buffer for metadata file header */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI(FAIL)
@@ -460,13 +488,15 @@ H5F_update_vfd_swmr_metadata_file(H5F_t *f, uint32_t num_entries, H5FD_vfd_swmr_
* --Compute checksum, update the index entry, write entry to
* the metadata file
*
- * --Set entry_ptr to NULL
+ * --Set entry_ptr to NULL when not generating updater files
*/
for (i = 0; i < num_entries; i++) {
if (index[i].entry_ptr == NULL)
continue;
+ HDassert(index[i].tick_of_last_change == f->shared->tick_num);
+
/* Prepend previous image of the entry to the delayed list */
if (index[i].md_file_page_offset) {
if (shadow_image_defer_free(shared, &index[i]) == -1) {
@@ -482,16 +512,16 @@ H5F_update_vfd_swmr_metadata_file(H5F_t *f, uint32_t num_entries, H5FD_vfd_swmr_
/* Compute checksum and update the index entry */
index[i].md_file_page_offset = md_addr / shared->fs_page_size;
- index[i].chksum = H5_checksum_metadata(index[i].entry_ptr, index[i].length, 0);
+ index[i].checksum = H5_checksum_metadata(index[i].entry_ptr, index[i].length, 0);
#if 0 /* JRM */
HDfprintf(stderr,
- "writing index[%d] fo/mdfo/l/chksum/fc/lc = %lld/%lld/%ld/%lx/%lx/%lx\n",
+ "writing index[%d] fo/mdfo/l/checksum/fc/lc = %lld/%lld/%ld/%lx/%lx/%lx\n",
i,
index[i].hdf5_page_offset,
index[i].md_file_page_offset,
index[i].length,
- index[i].chksum,
+ index[i].checksum,
(((char*)(index[i].entry_ptr))[0]),
(((char*)(index[i].entry_ptr))[4095]));
@@ -500,26 +530,34 @@ H5F_update_vfd_swmr_metadata_file(H5F_t *f, uint32_t num_entries, H5FD_vfd_swmr_
HDassert(shared->fs_page_size == 4096);
#endif /* JRM */
- /* Seek and write the entry to the metadata file */
- if (HDlseek(shared->vfd_swmr_md_fd, (HDoff_t)md_addr, SEEK_SET) < 0)
+ if(shared->vfd_swmr_config.maintain_metadata_file) {
- HGOTO_ERROR(H5E_FILE, H5E_SEEKERROR, FAIL, "unable to seek in the metadata file")
+ /* Seek and write the entry to the metadata file */
+ if (HDlseek(shared->vfd_swmr_md_fd, (HDoff_t)md_addr, SEEK_SET) < 0)
- if (HDwrite(shared->vfd_swmr_md_fd, index[i].entry_ptr, index[i].length) != (ssize_t)index[i].length)
+ HGOTO_ERROR(H5E_FILE, H5E_SEEKERROR, FAIL, "unable to seek in the metadata file")
- HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL,
- "error in writing the page/multi-page entry to metadata file")
+ if (HDwrite(shared->vfd_swmr_md_fd, index[i].entry_ptr, index[i].length) != (ssize_t)index[i].length)
+
+ HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL,
+ "error in writing the page/multi-page entry to metadata file")
+ }
+
+ if(!shared->vfd_swmr_config.generate_updater_files)
+ index[i].entry_ptr = NULL;
- index[i].entry_ptr = NULL;
}
+ if ((md_idx_image = HDmalloc(H5FD_MD_INDEX_SIZE(num_entries))) == NULL)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for md index")
+
/* Construct and write index to the metadata file */
- if (H5F__vfd_swmr_construct_write_md_idx(shared, num_entries, index) < 0)
+ if (H5F__vfd_swmr_construct_write_md_idx(shared, num_entries, index, md_idx_image) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "fail to construct & write index to md")
/* Construct and write header to the md file */
- if (H5F__vfd_swmr_construct_write_md_hdr(shared, num_entries) < 0)
+ if (H5F__vfd_swmr_construct_write_md_hdr(shared, num_entries, md_hdr_image) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "fail to construct & write header to md")
@@ -534,26 +572,47 @@ H5F_update_vfd_swmr_metadata_file(H5F_t *f, uint32_t num_entries, H5FD_vfd_swmr_
* --remove the associated entries from the list
*/
- if (shared->tick_num <= shared->vfd_swmr_config.max_lag)
- goto done; // It is too early for any reclamations to be due.
+ /* if (shared->tick_num <= shared->vfd_swmr_config.max_lag),
+ it is too early for any reclamations to be due.
+ */
+ if (shared->tick_num > shared->vfd_swmr_config.max_lag) {
- TAILQ_FOREACH_REVERSE_SAFE(shadow_defree, &shared->shadow_defrees, shadow_defree_queue, link, prev)
- {
+ TAILQ_FOREACH_REVERSE_SAFE(shadow_defree, &shared->shadow_defrees, shadow_defree_queue, link, prev)
+ {
- if (shadow_defree->tick_num + shared->vfd_swmr_config.max_lag > shared->tick_num) {
- break; // No more entries are due for reclamation.
- }
+ if (shadow_defree->tick_num + shared->vfd_swmr_config.max_lag > shared->tick_num) {
+ break; // No more entries are due for reclamation.
+ }
+
+ if (H5MV_free(f, shadow_defree->offset, shadow_defree->length) < 0) {
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush clean entry");
+ }
+
+ TAILQ_REMOVE(&shared->shadow_defrees, shadow_defree, link);
- if (H5MV_free(f, shadow_defree->offset, shadow_defree->length) < 0) {
- HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush clean entry");
+ H5FL_FREE(shadow_defree_t, shadow_defree);
}
+ }
- TAILQ_REMOVE(&shared->shadow_defrees, shadow_defree, link);
- H5FL_FREE(shadow_defree_t, shadow_defree);
+ /* For VFD SWMR testing: invoke callback if set to generate metadata file checksum */
+ if(shared->generate_md_ck_cb) {
+ if(shared->generate_md_ck_cb(shared->vfd_swmr_config.md_file_path, shared->updater_seq_num) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_SYSTEM, FAIL, "error from generate_md_ck_cb()")
}
+ /* Generate updater files with num_entries */
+ if(shared->vfd_swmr_config.generate_updater_files)
+ if(H5F__generate_updater_file(f, num_entries, 0,
+ md_hdr_image, H5FD_MD_HEADER_SIZE,
+ md_idx_image, shared->writer_index_offset, H5FD_MD_INDEX_SIZE(num_entries)) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_SYSTEM, FAIL, "can't generate updater file")
+
done:
+
+ if (md_idx_image)
+ HDfree(md_idx_image);
+
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5F_update_vfd_swmr_metadata_file() */
@@ -1571,9 +1630,8 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
-H5F__vfd_swmr_construct_write_md_hdr(H5F_shared_t *shared, uint32_t num_entries)
+H5F__vfd_swmr_construct_write_md_hdr(H5F_shared_t *shared, uint32_t num_entries, uint8_t *image)
{
- uint8_t image[H5FD_MD_HEADER_SIZE]; /* Buffer for header */
uint8_t *p = NULL; /* Pointer to buffer */
uint32_t metadata_chksum; /* Computed metadata checksum value */
/* Size of header and index */
@@ -1607,15 +1665,17 @@ H5F__vfd_swmr_construct_write_md_hdr(H5F_shared_t *shared, uint32_t num_entries)
/* Sanity checks on header */
HDassert(p - image == (ptrdiff_t)hdr_size);
- /* Set to beginning of the file */
- if (HDlseek(shared->vfd_swmr_md_fd, H5FD_MD_HEADER_OFF, SEEK_SET) < 0)
+ if(shared->vfd_swmr_config.maintain_metadata_file) {
+ /* Set to beginning of the file */
+ if (HDlseek(shared->vfd_swmr_md_fd, H5FD_MD_HEADER_OFF, SEEK_SET) < 0)
- HGOTO_ERROR(H5E_VFL, H5E_SEEKERROR, FAIL, "unable to seek in metadata file")
+ HGOTO_ERROR(H5E_VFL, H5E_SEEKERROR, FAIL, "unable to seek in metadata file")
- nwritten = HDwrite(shared->vfd_swmr_md_fd, image, hdr_size);
- /* Write header to the metadata file */
- if (nwritten != (ssize_t)hdr_size) {
- HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in writing header to metadata file")
+ nwritten = HDwrite(shared->vfd_swmr_md_fd, image, hdr_size);
+ /* Write header to the metadata file */
+ if (nwritten != (ssize_t)hdr_size) {
+ HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in writing header to metadata file")
+ }
}
done:
@@ -1648,9 +1708,8 @@ done:
*/
static herr_t
H5F__vfd_swmr_construct_write_md_idx(H5F_shared_t *shared, uint32_t num_entries,
- struct H5FD_vfd_swmr_idx_entry_t index[])
+ struct H5FD_vfd_swmr_idx_entry_t index[], uint8_t *image)
{
- uint8_t *image = NULL; /* Pointer to buffer */
uint8_t *p = NULL; /* Pointer to buffer */
uint32_t metadata_chksum; /* Computed metadata checksum value */
/* Size of index */
@@ -1663,11 +1722,6 @@ H5F__vfd_swmr_construct_write_md_idx(H5F_shared_t *shared, uint32_t num_entries,
HDassert(num_entries == 0 || index != NULL);
- /* Allocate space for the buffer to hold the index */
- if ((image = HDmalloc(idx_size)) == NULL)
-
- HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for md index")
-
/*
* Encode metadata file index
*/
@@ -1688,7 +1742,7 @@ H5F__vfd_swmr_construct_write_md_idx(H5F_shared_t *shared, uint32_t num_entries,
UINT32ENCODE(p, index[i].hdf5_page_offset);
UINT32ENCODE(p, index[i].md_file_page_offset);
UINT32ENCODE(p, index[i].length);
- UINT32ENCODE(p, index[i].chksum);
+ UINT32ENCODE(p, index[i].checksum);
}
/* Calculate checksum for index */
@@ -1703,22 +1757,20 @@ H5F__vfd_swmr_construct_write_md_idx(H5F_shared_t *shared, uint32_t num_entries,
/* Verify the md file descriptor exists */
HDassert(shared->vfd_swmr_md_fd >= 0);
- if (HDlseek(shared->vfd_swmr_md_fd, (HDoff_t)shared->writer_index_offset, SEEK_SET) < 0)
- HGOTO_ERROR(H5E_VFL, H5E_SEEKERROR, FAIL, "unable to seek in metadata file")
+ if(shared->vfd_swmr_config.maintain_metadata_file) {
- nwritten = HDwrite(shared->vfd_swmr_md_fd, image, idx_size);
- /* Write index to the metadata file */
- if (nwritten != (ssize_t)idx_size) {
- HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in writing index to metadata file")
+ if (HDlseek(shared->vfd_swmr_md_fd, (HDoff_t)shared->writer_index_offset, SEEK_SET) < 0)
+ HGOTO_ERROR(H5E_VFL, H5E_SEEKERROR, FAIL, "unable to seek in metadata file")
+
+ nwritten = HDwrite(shared->vfd_swmr_md_fd, image, idx_size);
+ /* Write index to the metadata file */
+ if (nwritten != (ssize_t)idx_size) {
+ HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in writing index to metadata file")
+ }
}
done:
- if (image) {
-
- HDfree(image);
- }
-
FUNC_LEAVE_NOAPI(ret_value)
} /* H5F__vfd_swmr_construct_write_idx() */
@@ -2010,3 +2062,381 @@ H5F_post_vfd_swmr_log_entry(H5F_t *f, int entry_type_code, char *log_info)
}
return;
}
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: H5F__vfd_swmr_construct_ud_hdr
+ *
+ * Purpose: Encode updater header in the buffer updater->header_image_ptr
+ *
+ * Return: Success: SUCCEED
+ * Failure: FAIL
+ *
+ * Programmer: Vailin Choi -- 10/2021
+ *
+ * Changes: None.
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5F__vfd_swmr_construct_ud_hdr(H5F_vfd_swmr_updater_t *updater)
+{
+ uint8_t *p = NULL; /* Pointer to buffer */
+ uint8_t *image = (uint8_t *)updater->header_image_ptr;
+ uint32_t metadata_chksum; /* Computed metadata checksum value */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_STATIC_NOERR
+
+ /*
+ * Encode metadata file header
+ */
+ p = image;
+
+ /* Encode magic for header */
+ HDmemcpy(p, H5F_UD_HEADER_MAGIC, (size_t)H5_SIZEOF_MAGIC);
+ p += H5_SIZEOF_MAGIC;
+
+ /* Encode version number, flags, page size, sequence number, tick number, change list offset, change list length */
+ UINT16ENCODE(p, H5F_UD_VERSION);
+ UINT16ENCODE(p, updater->flags);
+ UINT32ENCODE(p, updater->page_size);
+ UINT64ENCODE(p, updater->sequence_number);
+ UINT64ENCODE(p, updater->tick_num);
+
+ UINT64ENCODE(p, updater->change_list_offset);
+ UINT64ENCODE(p, updater->change_list_len);
+
+ /* Calculate checksum for header */
+ metadata_chksum = H5_checksum_metadata(image, (size_t)(p - image), 0);
+
+ /* Encode checksum for header */
+ UINT32ENCODE(p, metadata_chksum);
+
+ /* Sanity checks on header */
+ HDassert(p - image == (ptrdiff_t)updater->header_image_len);
+
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* H5F__vfd_swmr_construct_ud_hdr() */
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: H5F__vfd_swmr_construct_ud_cl
+ *
+ * Purpose: Encode updater change list in the buffer
+ * updater->change_list_image_ptr
+ *
+ * Return: Success: SUCCEED
+ * Failure: FAIL
+ *
+ * Programmer: Vailin Choi -- 10/2021
+ *
+ * Changes: None.
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5F__vfd_swmr_construct_ud_cl(H5F_vfd_swmr_updater_t *updater)
+{
+ uint8_t *p = NULL; /* Pointer to buffer */
+ uint8_t *image = (uint8_t *)updater->change_list_image_ptr;
+ uint32_t metadata_chksum; /* Computed metadata checksum value */
+ unsigned i; /* Local index variable */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_STATIC_NOERR
+
+ /*
+ * Encode ud cl
+ */
+ p = image;
+
+ /* Encode magic for ud cl */
+ HDmemcpy(p, H5F_UD_CL_MAGIC, H5_SIZEOF_MAGIC);
+ p += H5_SIZEOF_MAGIC;
+
+ /* Encode tick number */
+ UINT64ENCODE(p, updater->tick_num);
+
+ /* Encode Metadata File Header Updater File Page Offset*/
+ UINT32ENCODE(p, updater->md_file_header_ud_file_page_offset);
+
+ /* Encode Metadata File Header Length */
+ UINT32ENCODE(p, updater->md_file_header_len);
+
+ /* Calculate checksum on the image of the metadata file header */
+ updater->md_file_header_image_chksum = H5_checksum_metadata(
+ updater->md_file_header_image_ptr, (size_t)updater->md_file_header_len, 0);
+
+ /* Encode Metadata File Header Checksum */
+ UINT32ENCODE(p, updater->md_file_header_image_chksum);
+
+ /* Encode Metadata File Index Updater File Page Offset*/
+ UINT32ENCODE(p, updater->md_file_index_ud_file_page_offset);
+
+ /* Encode Metadata File Index Metadata File Offset */
+ UINT64ENCODE(p, updater->md_file_index_md_file_offset);
+
+ /* Encode Metadata File Index Length */
+ UINT32ENCODE(p, updater->md_file_index_len);
+
+ /* Calculate checksum on the image of the metadata file index */
+ updater->md_file_index_image_chksum = H5_checksum_metadata(
+ updater->md_file_index_image_ptr, (size_t)updater->md_file_index_len, 0);
+
+ /* Encode Metadata File Index Checksum */
+ UINT32ENCODE(p, updater->md_file_index_image_chksum);
+
+ UINT32ENCODE(p, updater->num_change_list_entries);
+
+ /* Encode the ud cl entries */
+ for (i = 0; i < updater->num_change_list_entries; i++) {
+ UINT32ENCODE(p, updater->change_list[i].entry_image_ud_file_page_offset);
+ UINT32ENCODE(p, updater->change_list[i].entry_image_md_file_page_offset);
+ UINT32ENCODE(p, updater->change_list[i].entry_image_h5_file_page_offset);
+ UINT32ENCODE(p, updater->change_list[i].entry_image_len);
+ UINT32ENCODE(p, updater->change_list[i].entry_image_checksum);
+ }
+
+ /* Calculate checksum for ud cl */
+ metadata_chksum = H5_checksum_metadata(image, (size_t)(p - image), 0);
+
+ /* Encode checksum for index */
+ UINT32ENCODE(p, metadata_chksum);
+
+ /* Sanity checks on index */
+ HDassert(p - image == (ptrdiff_t)updater->change_list_len);
+
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* H5F__vfd_swmr_construct_ud_cl() */
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: H5F_generate_updater_file()
+ *
+ * Purpose: Generate updater file:
+ * --assemble and initialize data in the updater struct
+ * --determine num_change_list entries
+ * --allocate buffers
+ * --construct on disk image (serialize) of the updater header and change list
+ * --create updater file using a temporay file name:
+ * --<shared->vfd_swmr_config.updater_file_path>.ud_tmp
+ * --allocate space and write the following to the updater file
+ * --updater file header
+ * --updater file change list
+ * --metadata entries
+ * --metadata file index
+ * --metadata file header
+ * --close the updater file
+ * --rename the updater file with the correct name:
+ * <shared->vfd_swmr_config.updater_file_path>.<shared->updater_seq_num>
+ *
+ * --increment shared->updater_seq_num
+ * --free buffers
+ *
+ * Return: SUCCEED/FAIL
+ *
+ * Programmer: Vailin Choi 8/24/2021
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5F__generate_updater_file(H5F_t *f, uint32_t num_entries, uint16_t flags,
+ uint8_t *md_file_hdr_image_ptr, size_t md_file_hdr_image_len,
+ uint8_t *md_file_index_image_ptr, uint64_t md_file_index_offset,
+ size_t md_file_index_image_len)
+{
+ H5F_shared_t *shared = f->shared; /* shared file pointer */
+ H5F_vfd_swmr_updater_t updater; /* Updater struct */
+ uint32_t next_page_offset;
+ H5FD_t *ud_file = NULL; /* Low-level file struct */
+ char namebuf[H5F__MAX_VFD_SWMR_FILE_NAME_LEN];
+ char newname[H5F__MAX_VFD_SWMR_FILE_NAME_LEN];
+ unsigned i;
+ hsize_t alloc_size;
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+
+ /* Updater file header fields */
+ updater.version = H5F_UD_VERSION;
+ updater.flags = flags;
+ updater.page_size = (uint32_t)shared->fs_page_size;
+ updater.sequence_number = shared->updater_seq_num;
+ updater.tick_num = shared->tick_num;
+ updater.header_image_ptr = NULL;
+ updater.header_image_len = H5F_UD_HEADER_SIZE;
+ updater.change_list_image_ptr = NULL;
+ updater.change_list_offset = 0;
+ updater.change_list_len = 0;
+
+ /* Updater file change list fields */
+
+ /* md_file_header related fields */
+ updater.md_file_header_ud_file_page_offset = 0;
+ updater.md_file_header_image_ptr = md_file_hdr_image_ptr; /* parameter */
+ updater.md_file_header_len = md_file_hdr_image_len; /* parameter */
+
+ /* md_file_index related fields */
+ updater.md_file_index_ud_file_page_offset = 0;
+ updater.md_file_index_image_ptr = md_file_index_image_ptr; /* parameter */
+ updater.md_file_index_md_file_offset = md_file_index_offset; /* parameter */
+ updater.md_file_index_len = md_file_index_image_len; /* parameter */
+
+ updater.num_change_list_entries = 0;
+ updater.change_list = NULL;
+
+ /* Scan index to determine updater.num_change_list_entries */
+ for (i = 0; i < num_entries; i++) {
+ if (shared->mdf_idx[i].entry_ptr != NULL &&
+ shared->mdf_idx[i].tick_of_last_change == shared->tick_num)
+ updater.num_change_list_entries += 1;
+ }
+
+ if (flags == CREATE_METADATA_FILE_ONLY_FLAG)
+ HDassert(updater.sequence_number == 0);
+ /* For file creation, just generate a header with this flag set */
+ else {
+ /* Update 2 updater file header fields: change_list_len, change_list_offset */
+ updater.change_list_len = H5F_UD_CL_SIZE(updater.num_change_list_entries);
+ updater.change_list_offset = updater.header_image_len;
+ }
+
+
+ /* Create the updater file with a temporary file name */
+ HDsprintf(namebuf, "%s.ud_tmp", shared->vfd_swmr_config.updater_file_path);
+
+ if((ud_file = H5FD_open(namebuf, H5F_ACC_TRUNC|H5F_ACC_RDWR|H5F_ACC_CREAT, H5P_FILE_ACCESS_DEFAULT, HADDR_UNDEF)) == NULL)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "fail to open updater file");
+
+ if ((updater.header_image_ptr = HDmalloc(updater.header_image_len)) == NULL)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for ud header")
+
+ /* Serialize updater file hdr in updater.header_image_ptr */
+ if (H5F__vfd_swmr_construct_ud_hdr(&updater) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "fail to create updater file header ");
+
+ /* Allocate space in updater file for updater file header */
+ if(H5FD__alloc_real(ud_file, H5FD_MEM_DEFAULT, updater.header_image_len, NULL, NULL) == HADDR_UNDEF)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to allocate file memory")
+
+ /* Write updater file header */
+ if(H5FD_write(ud_file, H5FD_MEM_DEFAULT, H5F_UD_HEADER_OFF, updater.header_image_len, updater.header_image_ptr) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "ud file write failed")
+
+
+ if (flags != CREATE_METADATA_FILE_ONLY_FLAG) {
+
+ next_page_offset = ((uint32_t)(updater.header_image_len + updater.change_list_len) / updater.page_size) + 1;
+
+ if(updater.num_change_list_entries) {
+
+ /* Allocate space for change list entries */
+ if((updater.change_list = HDmalloc(sizeof(H5F_vfd_swmr_updater_cl_entry_t) *
+ updater.num_change_list_entries)) == NULL)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for ud cl")
+
+
+ /* Initialize change list entries */
+ for (i = 0; i < num_entries; i++) {
+
+ if (shared->mdf_idx[i].entry_ptr != NULL &&
+ shared->mdf_idx[i].tick_of_last_change == shared->tick_num) {
+
+ updater.change_list[i].entry_image_ptr = shared->mdf_idx[i].entry_ptr;
+ updater.change_list[i].entry_image_ud_file_page_offset = 0;
+ updater.change_list[i].entry_image_md_file_page_offset = (uint32_t)shared->mdf_idx[i].md_file_page_offset;
+ updater.change_list[i].entry_image_h5_file_page_offset = (uint32_t)shared->mdf_idx[i].hdf5_page_offset;
+ updater.change_list[i].entry_image_len = shared->mdf_idx[i].length;
+ updater.change_list[i].entry_image_checksum = shared->mdf_idx[i].checksum;
+
+ shared->mdf_idx[i].entry_ptr = NULL;
+ }
+ }
+
+ /* Set up page aligned space for all metadata pages */
+ for(i = 0; i < updater.num_change_list_entries; i++) {
+ updater.change_list[i].entry_image_ud_file_page_offset = next_page_offset;
+ next_page_offset += (((uint32_t)updater.change_list[i].entry_image_len / updater.page_size) + 1);
+ }
+ }
+
+ /* Set up page aligned space for the metadata file index */
+ updater.md_file_index_ud_file_page_offset = next_page_offset;
+
+ /* Set up page aligned space for the metadata file header */
+ next_page_offset += (((uint32_t)updater.md_file_index_len / updater.page_size) + 1);
+ updater.md_file_header_ud_file_page_offset = next_page_offset;
+
+ if ((updater.change_list_image_ptr = HDmalloc(updater.change_list_len)) == NULL)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for ud cl ")
+
+ /* Serialize updater file change list in updater.change_list_image_ptr */
+ if (H5F__vfd_swmr_construct_ud_cl(&updater) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "fail to create updater file cl");
+
+ /* Allocate space in updater file for updater file change list */
+ if(H5FD__alloc_real(ud_file, H5FD_MEM_DEFAULT, updater.header_image_len+updater.change_list_len, NULL, NULL) == HADDR_UNDEF)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to allocate file memory")
+
+ /* Write updater file change list */
+ if(H5FD_write(ud_file, H5FD_MEM_DEFAULT, updater.header_image_len,
+ updater.change_list_len, updater.change_list_image_ptr) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "ud file write failed")
+
+
+ /* Allocate and write metadata pages */
+ for (i = 0; i < updater.num_change_list_entries; i++) {
+ alloc_size = updater.change_list[i].entry_image_ud_file_page_offset * updater.page_size +
+ updater.change_list[i].entry_image_len;
+
+ if(H5FD__alloc_real(ud_file, H5FD_MEM_DEFAULT, alloc_size, NULL, NULL) == HADDR_UNDEF)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to allocate file memory")
+
+ if(H5FD_write(ud_file, H5FD_MEM_DEFAULT,
+ updater.change_list[i].entry_image_ud_file_page_offset * updater.page_size,
+ updater.change_list[i].entry_image_len,
+ updater.change_list[i].entry_image_ptr) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "ud file write failed")
+ }
+
+ /* Allocate and write metadata file index */
+ alloc_size = updater.md_file_index_ud_file_page_offset * updater.page_size + updater.md_file_index_len;
+ if(H5FD__alloc_real(ud_file, H5FD_MEM_DEFAULT, alloc_size, NULL, NULL) == HADDR_UNDEF)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to allocate file memory")
+
+ if(H5FD_write(ud_file, H5FD_MEM_DEFAULT, updater.md_file_index_ud_file_page_offset * updater.page_size,
+ updater.md_file_index_len, updater.md_file_index_image_ptr) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "ud file write failed")
+
+ /* Allocate and write metadata file header */
+ alloc_size = updater.md_file_header_ud_file_page_offset * updater.page_size + updater.md_file_header_len;
+ if(H5FD__alloc_real(ud_file, H5FD_MEM_DEFAULT, alloc_size, NULL, NULL) == HADDR_UNDEF)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to allocate file memory")
+
+ if(H5FD_write(ud_file, H5FD_MEM_DEFAULT, updater.md_file_header_ud_file_page_offset * updater.page_size,
+ updater.md_file_header_len, updater.md_file_header_image_ptr) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "ud file write failed")
+ }
+
+ /* Close the updater file and rename the file */
+ if(H5FD_close(ud_file) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "unable to close updater file")
+ HDsprintf(newname, "%s.%lu", shared->vfd_swmr_config.updater_file_path, shared->updater_seq_num);
+ HDrename(namebuf, newname);
+
+ ++shared->updater_seq_num;
+
+done:
+ if(updater.header_image_ptr)
+ HDfree(updater.header_image_ptr);
+ if(updater.change_list_image_ptr)
+ HDfree(updater.change_list_image_ptr);
+ if(updater.change_list)
+ HDfree(updater.change_list);
+
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* H5F__generate_updater_file() */
diff --git a/src/H5PB.c b/src/H5PB.c
index 57ef7fb..ef93bd7 100644
--- a/src/H5PB.c
+++ b/src/H5PB.c
@@ -2121,7 +2121,7 @@ H5PB_vfd_swmr__update_index(H5F_t *f, uint32_t *idx_ent_added_ptr, uint32_t *idx
/* partial initialization of new entry -- rest done later */
ie_ptr->hdf5_page_offset = target_page;
ie_ptr->md_file_page_offset = 0; /* undefined at this point */
- ie_ptr->chksum = 0; /* undefined at this point */
+ ie_ptr->checksum = 0; /* undefined at this point */
/* ie_ptr->entry_ptr initialized below */
/* ie_ptr->tick_of_last_change initialized below */
/* ie_ptr->clean initialized below */
diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c
index 3b39f12..f833d65 100644
--- a/src/H5Pfapl.c
+++ b/src/H5Pfapl.c
@@ -318,6 +318,16 @@
#define H5F_ACS_VFD_SWMR_CONFIG_ENC H5P__facc_vfd_swmr_config_enc
#define H5F_ACS_VFD_SWMR_CONFIG_DEC H5P__facc_vfd_swmr_config_dec
+/* Private property for VFD SWMR testing:
+ * Callback function to generate checksum for metadata file
+ */
+#define H5F_ACS_GENERATE_MD_CK_CB_SIZE sizeof(H5F_generate_md_ck_cb_t)
+
+#define H5F_ACS_GENERATE_MD_CK_CB_DEF \
+ { \
+ NULL \
+ }
+
/******************/
/* Local Typedefs */
/******************/
@@ -515,6 +525,9 @@ static const hbool_t H5F_def_ignore_disabled_file_locks_g =
H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEF; /* Default ignore disabled file locks flag */
static const H5F_vfd_swmr_config_t H5F_def_vfd_swmr_config_g =
H5F_ACS_VFD_SWMR_CONFIG_DEF; /* Default vfd swmr configuration */
+/* For VFD SWMR testing only: Default to generate checksum for metadata file */
+static const H5F_generate_md_ck_t H5F_def_generate_md_ck_cb_g =
+ H5F_ACS_GENERATE_MD_CK_CB_DEF;
/*-------------------------------------------------------------------------
* Function: H5P__facc_reg_prop
@@ -633,6 +646,14 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass)
NULL) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class")
+ /* Register the private property of whether to generate checksum for metadata file.
+ * It's used for VFD SWMR testing only. */
+ /* (Note: this property should not have an encode/decode callback -QAK) */
+ if (H5P__register_real(pclass, H5F_ACS_GENERATE_MD_CK_CB_NAME, H5F_ACS_GENERATE_MD_CK_CB_SIZE,
+ &H5F_def_generate_md_ck_cb_g, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+ NULL) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class")
+
/* Register the data type of multi driver info */
if (H5P__register_real(pclass, H5F_ACS_MULTI_TYPE_NAME, H5F_ACS_MULTI_TYPE_SIZE, &H5F_def_mem_type_g,
NULL, NULL, NULL, H5F_ACS_MULTI_TYPE_ENC, H5F_ACS_MULTI_TYPE_DEC, NULL, NULL, NULL,
@@ -4097,11 +4118,15 @@ H5P__facc_vfd_swmr_config_enc(const void *value, void **_pp, size_t *size)
INT32ENCODE(*pp, (int32_t)config->tick_len);
INT32ENCODE(*pp, (int32_t)config->max_lag);
H5_ENCODE_UNSIGNED(*pp, config->writer);
+ H5_ENCODE_UNSIGNED(*pp, config->maintain_metadata_file);
+ H5_ENCODE_UNSIGNED(*pp, config->generate_updater_files);
H5_ENCODE_UNSIGNED(*pp, config->flush_raw_data);
INT32ENCODE(*pp, (int32_t)config->md_pages_reserved);
INT32ENCODE(*pp, (int32_t)config->pb_expansion_threshold);
HDmemcpy(*pp, (const uint8_t *)(config->md_file_path), (size_t)(H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1));
*pp += H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1;
+ HDmemcpy(*pp, (const uint8_t *)(config->updater_file_path), (size_t)(H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1));
+ *pp += H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1;
HDmemcpy(*pp, (const uint8_t *)(config->log_file_path),
(size_t)(H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1));
*pp += H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1;
@@ -4109,7 +4134,7 @@ H5P__facc_vfd_swmr_config_enc(const void *value, void **_pp, size_t *size)
} /* end if */
/* Compute encoded size */
- *size += ((5 * sizeof(int32_t)) + (2 * sizeof(unsigned)) + (2 * (H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1)));
+ *size += ((5 * sizeof(int32_t)) + (4 * sizeof(unsigned)) + (3 * (H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1)));
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5P__facc_vfd_swmr_config_enc() */
@@ -4150,6 +4175,8 @@ H5P__facc_vfd_swmr_config_dec(const void **_pp, void *_value)
UINT32DECODE(*pp, config->max_lag);
H5_DECODE_UNSIGNED(*pp, config->writer);
+ H5_DECODE_UNSIGNED(*pp, config->maintain_metadata_file);
+ H5_DECODE_UNSIGNED(*pp, config->generate_updater_files);
H5_DECODE_UNSIGNED(*pp, config->flush_raw_data);
/* int */
@@ -4159,6 +4186,9 @@ H5P__facc_vfd_swmr_config_dec(const void **_pp, void *_value)
HDstrcpy(config->md_file_path, (const char *)(*pp));
*pp += H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1;
+ HDstrcpy(config->updater_file_path, (const char *)(*pp));
+ *pp += H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1;
+
HDstrcpy(config->log_file_path, (const char *)(*pp));
*pp += H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1;
@@ -5703,12 +5733,30 @@ H5Pset_vfd_swmr_config(hid_t plist_id, H5F_vfd_swmr_config_t *config_ptr)
if (config_ptr->pb_expansion_threshold > H5F__MAX_PB_EXPANSION_THRESHOLD)
HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "pb_expansion_threshold out of range")
- /* Must provide the path for the metadata file */
- name_len = HDstrlen(config_ptr->md_file_path);
- if (name_len == 0)
- HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "md_file_path is empty")
- else if (name_len > H5F__MAX_VFD_SWMR_FILE_NAME_LEN)
- HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "md_file_path is too long")
+ /* If writer is TRUE, at least one of maintain_metadata_file and generate_updater_files must be TRUE */
+ if(config_ptr->writer) {
+ if(!config_ptr->maintain_metadata_file && !config_ptr->generate_updater_files)
+ HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "either maintain_metadata_file or generate_updater_files must be TRUE")
+ }
+
+ if((config_ptr->writer && config_ptr->maintain_metadata_file) || !config_ptr->writer) {
+ /* Must provide the path and base name of the metadata file */
+ name_len = HDstrlen(config_ptr->md_file_path);
+ if (name_len == 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "md_file_path is empty")
+ else if (name_len > H5F__MAX_VFD_SWMR_FILE_NAME_LEN)
+ HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "md_file_path is too long")
+ }
+
+ if(config_ptr->writer && config_ptr->generate_updater_files) {
+ /* Must provide the path and base name of the metadata updater files */
+ name_len = HDstrlen(config_ptr->updater_file_path);
+ if (name_len == 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "updater_file_path is empty")
+ else if (name_len > H5F__MAX_VFD_SWMR_FILE_NAME_LEN)
+ HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "updater_file_path is too long")
+ }
+
name_len = HDstrlen(config_ptr->log_file_path);
if (name_len > H5F__MAX_VFD_SWMR_FILE_NAME_LEN)
diff --git a/test/page_buffer.c b/test/page_buffer.c
index 61e9696..53c9b74 100644
--- a/test/page_buffer.c
+++ b/test/page_buffer.c
@@ -101,6 +101,8 @@ swmr_fapl_augment(hid_t fapl, const char *filename, uint32_t max_lag)
.tick_len = 4,
.max_lag = max_lag,
.writer = true,
+ .maintain_metadata_file = true,
+ .generate_updater_files = false,
.md_pages_reserved = 128};
char * bname = NULL;
char * dname = NULL;
diff --git a/test/vfd_swmr.c b/test/vfd_swmr.c
index 3ed73c1..e93a399 100644
--- a/test/vfd_swmr.c
+++ b/test/vfd_swmr.c
@@ -41,6 +41,7 @@
#define FS_PAGE_SIZE 512
#define FILENAME "vfd_swmr_file.h5"
#define MD_FILENAME "vfd_swmr_metadata_file"
+#define UD_FILENAME "vfd_swmr_updater_file"
#define FILENAME2 "vfd_swmr_file2.h5"
#define MD_FILENAME2 "vfd_swmr_metadata_file2"
@@ -48,61 +49,50 @@
#define FILENAME3 "vfd_swmr_file3.h5"
#define MD_FILENAME3 "vfd_swmr_metadata_file3"
+#define FILENAME4 "vfd_swmr_file4.h5"
+#define MD_FILE "vfd_swmr_md_file"
+#define UD_FILE "vfd_swmr_ud_file"
+
#define FNAME "non_vfd_swmr_file.h5"
-/* test routines for VFD SWMR */
-static unsigned test_fapl(void);
-static unsigned test_file_end_tick(void);
-static unsigned test_file_fapl(void);
-static unsigned test_writer_md(void);
+/* Defines used by verify_updater_flags() and verify_ud_chk() helper routine */
-/* helper routines */
-static hid_t init_vfd_swmr_config_fapl(H5F_vfd_swmr_config_t *config, uint32_t tick_len, uint32_t max_lag,
- hbool_t is_writer, uint32_t md_pages_reserved,
- const char *md_file_path, size_t pbuf_size);
+/* Offset of "flags" in updater file header */
+#define UD_HD_FLAGS_OFFSET 6
-/*-------------------------------------------------------------------------
- * Function: init_vfd_swmr_config_fapl
- *
- * Purpose: Helper routine to initialize the fields for VFD SWMR configuration
- *
- * Return: void
- *
- *-------------------------------------------------------------------------
- */
-static hid_t
-init_vfd_swmr_config_fapl(H5F_vfd_swmr_config_t *config, uint32_t tick_len, uint32_t max_lag,
- hbool_t is_writer, uint32_t md_pages_reserved, const char *md_file_path,
- size_t pbuf_size)
-{
- hid_t fapl;
+/* Offset of "sequence number" in updater file header */
+#define UD_HD_SEQ_NUM_OFFSET 12
- HDmemset(config, 0, sizeof(*config));
+/* Offset of "change list length" in updater file header */
+#define UD_HD_CHANGE_LIST_LEN_OFFSET 36
- config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION;
- config->tick_len = tick_len;
- config->max_lag = max_lag;
- config->writer = is_writer;
- config->md_pages_reserved = md_pages_reserved;
- HDstrcpy(config->md_file_path, md_file_path);
+/* Offset of "number of change list entries" in updater file change list header */
+#define UD_CL_NUM_CHANGE_LIST_ENTRIES_OFFSET H5F_UD_HEADER_SIZE + 44
- /* Create a copy of the file access property list */
- if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
- return H5I_INVALID_HID;
+/* Size of "sequence number", "tick number" and "change list length" fields in the updater file header */
+#define UD_SIZE_8 8
- if (H5Pset_vfd_swmr_config(fapl, config) < 0) {
- (void)H5Pclose(fapl);
- return H5I_INVALID_HID;
- }
+/* Size of checksum and "number of change list entries" field in the updater change list header */
+#define UD_SIZE_4 4
- /* Enable page buffering */
- if (pbuf_size != 0 && H5Pset_page_buffer_size(fapl, pbuf_size, 0, 0) < 0) {
- (void)H5Pclose(fapl);
- return H5I_INVALID_HID;
- }
+/* Size of "flags" field in the updater file header */
+#define UD_SIZE_2 2
+
+/* test routines for VFD SWMR */
+static unsigned test_fapl(void);
+static unsigned test_file_end_tick(void);
+static unsigned test_file_fapl(void);
+static unsigned test_writer_md(void);
+
+static unsigned test_updater_flags(void);
+static unsigned test_updater_flags_same_file_opens(void);
+static herr_t verify_updater_flags(char *ud_name, uint16_t expected_flags);
+
+static void clean_chk_ud_files(char *md_file_path, char *updater_file_path);
+static herr_t verify_ud_chk(char *md_file_path, char *ud_file_path);
+static herr_t md_ck_cb(char *md_file_path, uint64_t tick_num);
+static unsigned test_updater_generate_md_checksums(hbool_t file_create);
- return fapl;
-} /* init_vfd_swmr_config_fapl() */
/*-------------------------------------------------------------------------
* Function: test_fapl()
@@ -114,6 +104,12 @@ init_vfd_swmr_config_fapl(H5F_vfd_swmr_config_t *config, uint32_t tick_len, uint
* --max_lag: should be >= 3
* --md_pages_reserved: should be >= 2
* --md_file_path: should contain the metadata file path (POSIX)
+ * --at least one of maintain_metadata_file and generate_updater_files
+ * must be true
+ * --if both the writer and maintain_metadata_file fields are true,
+ * then md_file_path field shouldn't be empty
+ * --if both the writer and generate_updater_files fields are true,
+ * then updater_file_path field shouldn't be empty
* B) Verify that info set in the fapl is retrieved correctly.
*
* Return: 0 if test is sucessful
@@ -195,9 +191,40 @@ test_fapl(void)
if (ret >= 0)
TEST_ERROR;
+ my_config->writer = TRUE;
+ /* Should fail: at least one of maintain_metadata_file and generate_updater_files must be true */
+ H5E_BEGIN_TRY
+ {
+ ret = H5Pset_vfd_swmr_config(fapl, my_config);
+ }
+ H5E_END_TRY;
+ if (ret >= 0)
+ TEST_ERROR;
+
+ my_config->writer = TRUE;
+ my_config->maintain_metadata_file = TRUE;
+ /* Should fail: empty md_file_path */
+ H5E_BEGIN_TRY
+ {
+ ret = H5Pset_vfd_swmr_config(fapl, my_config);
+ }
+ H5E_END_TRY;
+ if (ret >= 0)
+ TEST_ERROR;
+
+ my_config->generate_updater_files = TRUE;
+ /* Should fail: empty updater_file_path */
+ H5E_BEGIN_TRY
+ {
+ ret = H5Pset_vfd_swmr_config(fapl, my_config);
+ }
+ H5E_END_TRY;
+ if (ret >= 0)
+ TEST_ERROR;
+
/* Set md_file_path */
HDstrcpy(my_config->md_file_path, MD_FILENAME);
- my_config->writer = TRUE;
+ my_config->generate_updater_files = FALSE;
/* Should succeed in setting the configuration info */
if (H5Pset_vfd_swmr_config(fapl, my_config) < 0)
@@ -215,6 +242,33 @@ test_fapl(void)
TEST_ERROR;
if (my_config->md_pages_reserved != 2)
TEST_ERROR;
+ if (my_config->generate_updater_files)
+ TEST_ERROR;
+ if (HDstrcmp(my_config->md_file_path, MD_FILENAME) != 0)
+ TEST_ERROR;
+
+ my_config->generate_updater_files = TRUE;
+ /* Set updater_file_path */
+ HDstrcpy(my_config->updater_file_path, UD_FILENAME);
+
+ /* Should succeed in setting the configuration info */
+ if (H5Pset_vfd_swmr_config(fapl, my_config) < 0)
+ TEST_ERROR;
+
+ /* Clear the configuration structure */
+ HDmemset(my_config, 0, sizeof(H5F_vfd_swmr_config_t));
+
+ /* Retrieve the configuration info just set */
+ if (H5Pget_vfd_swmr_config(fapl, my_config) < 0)
+ TEST_ERROR;
+
+ /* Verify the configuration info */
+ if (!my_config->generate_updater_files)
+ TEST_ERROR;
+ if (HDstrcmp(my_config->updater_file_path, UD_FILENAME) != 0)
+ TEST_ERROR;
+ if (!my_config->maintain_metadata_file)
+ TEST_ERROR;
if (HDstrcmp(my_config->md_file_path, MD_FILENAME) != 0)
TEST_ERROR;
@@ -290,8 +344,17 @@ test_file_fapl(void)
if ((file_config = (H5F_vfd_swmr_config_t *)HDmalloc(sizeof(H5F_vfd_swmr_config_t))) == NULL)
FAIL_STACK_ERROR;
- /* Configured as VFD SWMR reader + no page buffering */
- fapl1 = init_vfd_swmr_config_fapl(config1, 4, 6, FALSE, 2, MD_FILENAME, 0);
+
+ /*
+ * Configured as VFD SWMR reader + no page buffering
+ */
+
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config1, 4, 7, FALSE, TRUE, FALSE, TRUE, 2, MD_FILENAME, NULL);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl1 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 0, config1);
if (fapl1 == H5I_INVALID_HID)
TEST_ERROR
@@ -307,8 +370,17 @@ test_file_fapl(void)
if (H5Pclose(fapl1) < 0)
FAIL_STACK_ERROR
- /* Configured as VFD SWMR writer + no page buffering */
- fapl1 = init_vfd_swmr_config_fapl(config1, 4, 6, TRUE, 2, MD_FILENAME, 0);
+ /*
+ * Configured as VFD SWMR writer + no page buffering
+ */
+
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config1, 4, 7, TRUE, TRUE, TRUE, TRUE, 2, MD_FILENAME, UD_FILENAME);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl1 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 0, config1);
+
if (fapl1 == H5I_INVALID_HID)
TEST_ERROR
@@ -321,13 +393,10 @@ test_file_fapl(void)
if (fid >= 0)
TEST_ERROR;
- /* Create a copy of the file creation property list */
- if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0)
+ if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, 4096)) < 0) {
+ HDprintf("vfd_swmr_create_fcpl() failed");
FAIL_STACK_ERROR
-
- /* Set file space strategy */
- if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0)
- FAIL_STACK_ERROR;
+ }
/* Should fail to create: no page buffering */
H5E_BEGIN_TRY
@@ -341,8 +410,17 @@ test_file_fapl(void)
if (H5Pclose(fapl1) < 0)
FAIL_STACK_ERROR
- /* Configured as VFD SWMR writer + page buffering */
- fapl1 = init_vfd_swmr_config_fapl(config1, 4, 6, TRUE, 2, MD_FILENAME, 4096);
+ /*
+ * Configured as VFD SWMR writer + page buffering
+ */
+
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config1, 4, 7, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME, NULL);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl1 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config1);
+
if (fapl1 == H5I_INVALID_HID)
TEST_ERROR
@@ -404,8 +482,17 @@ test_file_fapl(void)
if (H5Pclose(fapl1) < 0)
FAIL_STACK_ERROR;
- /* Set up different VFD SWMR configuration + page_buffering */
- fapl2 = init_vfd_swmr_config_fapl(config2, 4, 10, TRUE, 2, MD_FILENAME, 4096);
+ /*
+ * Set up different VFD SWMR configuration + page_buffering
+ */
+
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config2, 4, 10, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME, NULL);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl2 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config2);
+
if (fapl2 == H5I_INVALID_HID)
TEST_ERROR
@@ -435,10 +522,20 @@ test_file_fapl(void)
/* The file previously opened as VDF SWMR writer is still open */
/* with VFD SWMR configuration in config2 */
- /* Set up as VFD SWMR writer in config1 but different from config2 */
- fapl1 = init_vfd_swmr_config_fapl(config1, 3, 8, TRUE, 3, MD_FILENAME, 4096);
+
+ /*
+ * Set up as VFD SWMR writer in config1 but different from config2
+ */
+
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config1, 3, 8, TRUE, TRUE, FALSE, TRUE, 3, MD_FILENAME, NULL);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl1 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config1);
+
if (fapl1 == H5I_INVALID_HID)
- TEST_ERROR;
+ TEST_ERROR
/* Re-open the same file with config1 */
/* Should fail to open since config1 is different from config2 setting */
@@ -454,10 +551,17 @@ test_file_fapl(void)
if (H5Pclose(fapl1) < 0)
FAIL_STACK_ERROR;
- /* Set up as VFD SWMR reader in config1 which is same as config2 */
- fapl1 = init_vfd_swmr_config_fapl(config1, 4, 10, TRUE, 2, MD_FILENAME, 4096);
+ /*
+ * Set up as VFD SWMR reader in config1 which is same as config2
+ */
+
+ init_vfd_swmr_config(config1, 4, 10, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME, NULL);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl1 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config1);
+
if (fapl1 == H5I_INVALID_HID)
- TEST_ERROR;
+ TEST_ERROR
/* Re-open the same file as VFD SWMR writer */
/* Should succeed since config1 is same as the setting in config2 */
@@ -530,6 +634,7 @@ error:
return 1;
} /* test_file_fapl() */
+
/*-------------------------------------------------------------------------
* Function: test_file_end_tick()
*
@@ -589,28 +694,52 @@ test_file_end_tick(void)
if ((config3 = HDmalloc(sizeof(*config3))) == NULL)
FAIL_STACK_ERROR;
- /* Configured file 1 as VFD SWMR writer + page buffering */
- fapl1 = init_vfd_swmr_config_fapl(config1, 10, 15, TRUE, 2, MD_FILENAME, 4096);
+ /*
+ * Configured file 1 as VFD SWMR writer + page buffering
+ */
+
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config1, 10, 15, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME, NULL);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl1 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config1);
+
if (fapl1 == H5I_INVALID_HID)
- FAIL_STACK_ERROR;
+ TEST_ERROR
+
+ /*
+ * Configured file 2 as VFD SWMR writer + page buffering
+ */
+
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config2, 5, 6, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME2, NULL);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl2 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config2);
- /* Configured file 2 as VFD SWMR writer + page buffering */
- fapl2 = init_vfd_swmr_config_fapl(config2, 5, 6, TRUE, 2, MD_FILENAME2, 4096);
if (fapl2 == H5I_INVALID_HID)
- FAIL_STACK_ERROR;
+ TEST_ERROR
- /* Configured file 3 as VFD SWMR writer + page buffering */
- fapl3 = init_vfd_swmr_config_fapl(config3, 3, 6, TRUE, 2, MD_FILENAME3, 4096);
- if (fapl3 == H5I_INVALID_HID)
- FAIL_STACK_ERROR;
+ /*
+ * Configured file 3 as VFD SWMR writer + page buffering
+ */
- /* Create a copy of the file creation property list */
- if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0)
- FAIL_STACK_ERROR
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config3, 3, 6, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME3, NULL);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl3 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config3);
+
+ if (fapl3 == H5I_INVALID_HID)
+ TEST_ERROR
- /* Set file space strategy */
- if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0)
+ if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, 4096)) < 0) {
+ HDprintf("vfd_swmr_create_fcpl() failed");
FAIL_STACK_ERROR;
+ }
/* Create file 1 with VFD SWMR writer */
if ((fid1 = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl1)) < 0)
@@ -776,18 +905,23 @@ test_writer_create_open_flush(void)
if ((my_config = HDmalloc(sizeof(H5F_vfd_swmr_config_t))) == NULL)
FAIL_STACK_ERROR;
- /* Set up the VFD SWMR configuration + page buffering */
- fapl = init_vfd_swmr_config_fapl(my_config, 1, 3, TRUE, 2, MD_FILENAME, 4096);
- if (fapl == H5I_INVALID_HID)
- FAIL_STACK_ERROR;
+ /*
+ * Set up the VFD SWMR configuration + page buffering
+ */
- /* Create a copy of the file creation property list */
- if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0)
- FAIL_STACK_ERROR
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(my_config, 1, 3, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME, NULL);
- /* Set file space strategy */
- if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0)
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, my_config);
+ if (fapl == H5I_INVALID_HID)
+ TEST_ERROR
+
+ if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, 4096)) < 0) {
+ HDprintf("vfd_swmr_create_fcpl() failed");
FAIL_STACK_ERROR;
+ }
/* Create an HDF5 file with VFD SWMR configured */
if ((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl)) < 0)
@@ -797,7 +931,6 @@ test_writer_create_open_flush(void)
if (H5F__vfd_swmr_writer_create_open_flush_test(fid, TRUE) < 0)
FAIL_STACK_ERROR;
-#ifdef LATER /* Will activate the test when flush is implemented */
/* Flush the HDF5 file */
if (H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0)
FAIL_STACK_ERROR
@@ -805,7 +938,6 @@ test_writer_create_open_flush(void)
/* Verify info in metadata file when flushing the HDF5 file */
if (H5F__vfd_swmr_writer_create_open_flush_test(fid, FALSE) < 0)
FAIL_STACK_ERROR
-#endif
/* Close the file */
if (H5Fclose(fid) < 0)
@@ -891,6 +1023,7 @@ test_writer_md(void)
hsize_t chunk_dims[2] = {2, 5}; /* Dataset chunked dimension sizes */
H5FD_vfd_swmr_idx_entry_t *index = NULL; /* Pointer to the index entries */
H5F_vfd_swmr_config_t * my_config = NULL; /* Configuration for VFD SWMR */
+ H5F_t * f = NULL; /* Internal file object pointer */
TESTING("Verify the metadata file for VFD SWMR writer");
@@ -898,25 +1031,28 @@ test_writer_md(void)
if ((my_config = HDmalloc(sizeof(H5F_vfd_swmr_config_t))) == NULL)
FAIL_STACK_ERROR;
- /* Set up the VFD SWMR configuration + page buffering */
- fapl = init_vfd_swmr_config_fapl(my_config, 1, 3, TRUE, 256, MD_FILENAME, FS_PAGE_SIZE);
- if (fapl == H5I_INVALID_HID)
- FAIL_STACK_ERROR
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(my_config, 1, 3, TRUE, TRUE, FALSE, TRUE, 256, MD_FILENAME, NULL);
- /* Create a copy of the file creation property list */
- if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0)
- FAIL_STACK_ERROR
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, FS_PAGE_SIZE, my_config);
+ if (fapl == H5I_INVALID_HID)
+ TEST_ERROR
- /* Set file space strategy */
- if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0)
- FAIL_STACK_ERROR;
- if (H5Pset_file_space_page_size(fcpl, FS_PAGE_SIZE) < 0)
+ if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, FS_PAGE_SIZE)) < 0) {
+ HDprintf("vfd_swmr_create_fcpl() failed");
FAIL_STACK_ERROR;
+ }
/* Create an HDF5 file with VFD SWMR configured */
if ((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl)) < 0)
FAIL_STACK_ERROR;
+ /* Get a pointer to the internal file object */
+ if (NULL == (f = (H5F_t *)H5VL_object(fid)))
+ FAIL_STACK_ERROR;
+
/* Allocate num_entries for the data buffer */
if ((buf = HDcalloc(num_entries, FS_PAGE_SIZE)) == NULL)
FAIL_STACK_ERROR;
@@ -932,6 +1068,7 @@ test_writer_md(void)
index[i].md_file_page_offset = 1 + (num_entries - i) * 5;
index[i].length = (uint32_t)FS_PAGE_SIZE;
index[i].entry_ptr = &buf[i * FS_PAGE_SIZE];
+ index[i].tick_of_last_change = f->shared->tick_num;
}
/* Update with index and verify info in the metadata file */
@@ -970,8 +1107,10 @@ test_writer_md(void)
}
/* (B) Update every other entry in the index */
- for (i = 0; i < num_entries; i += 2)
+ for (i = 0; i < num_entries; i += 2) {
index[i].entry_ptr = &buf[i * FS_PAGE_SIZE];
+ index[i].tick_of_last_change = f->shared->tick_num;
+ }
/* Update with index and verify info in the metadata file */
/* Also verify that 5 entries will be on the delayed list */
@@ -1007,8 +1146,10 @@ test_writer_md(void)
}
/* (C) Update every 3 entry in the index */
- for (i = 0; i < num_entries; i += 3)
+ for (i = 0; i < num_entries; i += 3) {
index[i].entry_ptr = &buf[i * FS_PAGE_SIZE];
+ index[i].tick_of_last_change = f->shared->tick_num;
+ }
/* Update with index and verify info in the metadata file */
/* Also verify that 4 entries will be on the delayed list */
@@ -1042,7 +1183,9 @@ test_writer_md(void)
/* (D) Update two entries in the index */
index[1].entry_ptr = &buf[1 * FS_PAGE_SIZE];
+ index[1].tick_of_last_change = f->shared->tick_num;
index[5].entry_ptr = &buf[5 * FS_PAGE_SIZE];
+ index[5].tick_of_last_change = f->shared->tick_num;
/* Update with index and verify info in the metadata file */
/* Also verify that 2 entries will be on the delayed list */
@@ -1202,21 +1345,23 @@ test_reader_md_concur(void)
if ((config_writer = HDmalloc(sizeof(*config_writer))) == NULL)
FAIL_STACK_ERROR;
- /* Set up the VFD SWMR configuration + page buffering */
- fapl_writer = init_vfd_swmr_config_fapl(config_writer, 1, 3, TRUE, 256, MD_FILENAME, FS_PAGE_SIZE);
- if (fapl_writer == H5I_INVALID_HID)
- FAIL_STACK_ERROR;
+ /*
+ * Set up the VFD SWMR configuration + page buffering
+ */
- /* Create a copy of the file creation property list */
- if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0)
- FAIL_STACK_ERROR
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config_writer, 1, 3, TRUE, TRUE, FALSE, TRUE, 256, MD_FILENAME, NULL);
- /* Set file space strategy */
- if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0)
- FAIL_STACK_ERROR;
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl_writer = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, FS_PAGE_SIZE, config_writer);
+ if (fapl_writer == H5I_INVALID_HID)
+ TEST_ERROR
- if (H5Pset_file_space_page_size(fcpl, FS_PAGE_SIZE) < 0)
+ if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, FS_PAGE_SIZE)) < 0) {
+ HDprintf("vfd_swmr_create_fcpl() failed");
FAIL_STACK_ERROR;
+ }
/* Create an HDF5 file with VFD SWMR configured */
if ((fid_writer = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl_writer)) < 0)
@@ -1276,10 +1421,19 @@ test_reader_md_concur(void)
if ((config_reader = HDmalloc(sizeof(*config_reader))) == NULL)
HDexit(EXIT_FAILURE);
- /* Set up the VFD SWMR configuration as reader + page buffering */
- fapl_reader = init_vfd_swmr_config_fapl(config_reader, 1, 3, FALSE, 256, MD_FILENAME, FS_PAGE_SIZE);
+ /*
+ * Set up the VFD SWMR configuration as reader + page buffering
+ */
+
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config_reader, 1, 3, FALSE, TRUE, FALSE, TRUE, 256, MD_FILENAME, NULL);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl_reader = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, FS_PAGE_SIZE, config_reader);
+
if (fapl_reader == H5I_INVALID_HID)
- HDexit(EXIT_FAILURE);
+ TEST_ERROR
/* Open the test file as reader */
if ((fid_reader = H5Fopen(FILENAME, H5F_ACC_RDONLY, fapl_reader)) < 0)
@@ -1479,6 +1633,9 @@ test_reader_md_concur(void)
if ((fid_writer = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl_writer)) < 0)
FAIL_STACK_ERROR;
+ /* Get the file pointer */
+ file_writer = H5VL_object(fid_writer);
+
/* Send notification 1 to reader to start verfication */
notify = 1;
if (HDwrite(parent_pfd[1], &notify, sizeof(int)) < 0)
@@ -1544,11 +1701,9 @@ test_reader_md_concur(void)
index[i].md_file_page_offset = 1 + (num_entries - i) * 5;
index[i].length = (uint32_t)FS_PAGE_SIZE;
index[i].entry_ptr = &buf[i * FS_PAGE_SIZE];
+ index[i].tick_of_last_change = file_writer->shared->tick_num;
}
- /* Get the file pointer */
- file_writer = H5VL_object(fid_writer);
-
/* Update the metadata file with the index */
if (H5F_update_vfd_swmr_metadata_file(file_writer, num_entries, index) < 0)
TEST_ERROR;
@@ -1605,8 +1760,10 @@ test_reader_md_concur(void)
/* Update 3 entries in the index */
num_entries = 3;
- for (i = 0; i < num_entries; i++)
+ for (i = 0; i < num_entries; i++) {
index[i].entry_ptr = &buf[i * FS_PAGE_SIZE];
+ index[i].tick_of_last_change = file_writer->shared->tick_num;
+ }
/* Update the metadata file with the index */
if (H5F_update_vfd_swmr_metadata_file(file_writer, num_entries, index) < 0)
@@ -1658,8 +1815,10 @@ test_reader_md_concur(void)
/* Update 5 entries in the index */
num_entries = 5;
- for (i = 0; i < num_entries; i++)
+ for (i = 0; i < num_entries; i++) {
index[i].entry_ptr = &buf[i * FS_PAGE_SIZE];
+ index[i].tick_of_last_change = file_writer->shared->tick_num;
+ }
/* Update the metadata file with the index */
if (H5F_update_vfd_swmr_metadata_file(file_writer, num_entries, index) < 0)
@@ -1816,16 +1975,10 @@ test_multiple_file_opens_concur(void)
TESTING("EOT queue entries when opening files concurrently with VFD SWMR");
- /* Create a copy of the file creation property list */
- if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0)
- FAIL_STACK_ERROR
-
- /* Set file space strategy */
- if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0)
- FAIL_STACK_ERROR;
-
- if (H5Pset_file_space_page_size(fcpl, FS_PAGE_SIZE) < 0)
+ if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, FS_PAGE_SIZE)) < 0) {
+ HDprintf("vfd_swmr_create_fcpl() failed");
FAIL_STACK_ERROR;
+ }
/* Create file A */
if ((fid1 = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, H5P_DEFAULT)) < 0)
@@ -1886,7 +2039,14 @@ test_multiple_file_opens_concur(void)
HDexit(EXIT_FAILURE);
/* Set the VFD SWMR configuration in fapl_writer + page buffering */
- fapl_writer = init_vfd_swmr_config_fapl(config_writer, 1, 3, TRUE, 256, MD_FILENAME2, FS_PAGE_SIZE);
+
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config_writer, 1, 3, TRUE, TRUE, FALSE, TRUE, 256, MD_FILENAME2, NULL);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl_writer = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, FS_PAGE_SIZE, config_writer);
+
if (fapl_writer == H5I_INVALID_HID)
HDexit(EXIT_FAILURE);
@@ -1948,8 +2108,13 @@ test_multiple_file_opens_concur(void)
if ((config1 = HDmalloc(sizeof(*config1))) == NULL)
FAIL_STACK_ERROR
- /* Set the VFD SWMR configuration in fapl1 + page buffering */
- fapl1 = init_vfd_swmr_config_fapl(config1, 7, 10, TRUE, 256, MD_FILENAME, FS_PAGE_SIZE);
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config1, 7, 10, TRUE, TRUE, FALSE, TRUE, 256, MD_FILENAME, NULL);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl1 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, FS_PAGE_SIZE, config1);
+
if (fapl1 == H5I_INVALID_HID)
FAIL_STACK_ERROR
@@ -1986,8 +2151,13 @@ test_multiple_file_opens_concur(void)
if ((config2 = HDmalloc(sizeof(*config2))) == NULL)
FAIL_STACK_ERROR
- /* Set the VFD SWMR configuration in fapl2 + page buffering */
- fapl2 = init_vfd_swmr_config_fapl(config2, 1, 3, FALSE, 256, MD_FILENAME2, FS_PAGE_SIZE);
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config2, 1, 3, FALSE, TRUE, FALSE, TRUE, 256, MD_FILENAME2, NULL);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl2 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, FS_PAGE_SIZE, config2);
+
if (fapl2 == H5I_INVALID_HID)
FAIL_STACK_ERROR
@@ -2120,21 +2290,24 @@ test_disable_enable_eot_concur(void)
if ((config_writer = HDmalloc(sizeof(*config_writer))) == NULL)
FAIL_STACK_ERROR;
- /* Set up the VFD SWMR configuration + page buffering */
- fapl_writer = init_vfd_swmr_config_fapl(config_writer, 1, 3, TRUE, 256, MD_FILENAME, FS_PAGE_SIZE);
- if (fapl_writer == H5I_INVALID_HID)
- FAIL_STACK_ERROR;
+ /*
+ * Set up the VFD SWMR configuration + page buffering
+ */
- /* Create a copy of the file creation property list */
- if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0)
- FAIL_STACK_ERROR
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config_writer, 1, 3, TRUE, TRUE, FALSE, TRUE, 256, MD_FILENAME, NULL);
- /* Set file space strategy */
- if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0)
- FAIL_STACK_ERROR;
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl_writer = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, FS_PAGE_SIZE, config_writer);
- if (H5Pset_file_space_page_size(fcpl, FS_PAGE_SIZE) < 0)
- FAIL_STACK_ERROR;
+ if (fapl_writer == H5I_INVALID_HID)
+ FAIL_STACK_ERROR
+
+ if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, FS_PAGE_SIZE)) < 0) {
+ HDprintf("vfd_swmr_create_fcpl() failed");
+ FAIL_STACK_ERROR
+ }
/* Create an HDF5 file with VFD SWMR configured */
if ((fid_writer = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl_writer)) < 0)
@@ -2197,10 +2370,19 @@ test_disable_enable_eot_concur(void)
if ((config_reader = HDmalloc(sizeof(*config_reader))) == NULL)
HDexit(EXIT_FAILURE);
- /* Set up the VFD SWMR configuration as reader + page buffering */
- fapl_reader = init_vfd_swmr_config_fapl(config_reader, 1, 3, FALSE, 256, MD_FILENAME, FS_PAGE_SIZE);
+ /*
+ * Set up the VFD SWMR configuration as reader + page buffering
+ */
+
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config_reader, 1, 3, FALSE, TRUE, FALSE, TRUE, 256, MD_FILENAME, NULL);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl_reader = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, FS_PAGE_SIZE, config_reader);
+
if (fapl_reader == H5I_INVALID_HID)
- FAIL_STACK_ERROR;
+ FAIL_STACK_ERROR
/* Open the test file as reader */
if ((fid_reader = H5Fopen(FILENAME, H5F_ACC_RDONLY, fapl_reader)) < 0)
@@ -2400,21 +2582,24 @@ test_file_end_tick_concur(void)
if ((config_writer = HDmalloc(sizeof(*config_writer))) == NULL)
FAIL_STACK_ERROR;
- /* Set up the VFD SWMR configuration + page buffering */
- fapl_writer = init_vfd_swmr_config_fapl(config_writer, 1, 3, TRUE, 256, MD_FILENAME, FS_PAGE_SIZE);
- if (fapl_writer == H5I_INVALID_HID)
- FAIL_STACK_ERROR;
+ /*
+ * Set up the VFD SWMR configuration + page buffering
+ */
- /* Create a copy of the file creation property list */
- if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0)
- FAIL_STACK_ERROR
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config_writer, 1, 3, TRUE, TRUE, FALSE, TRUE, 256, MD_FILENAME, NULL);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl_writer = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, FS_PAGE_SIZE, config_writer);
- /* Set file space strategy */
- if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0)
+ if (fapl_writer == H5I_INVALID_HID)
FAIL_STACK_ERROR;
- if (H5Pset_file_space_page_size(fcpl, FS_PAGE_SIZE) < 0)
+ if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, FS_PAGE_SIZE)) < 0) {
+ HDprintf("vfd_swmr_create_fcpl() failed");
FAIL_STACK_ERROR;
+ }
/* Create an HDF5 file with VFD SWMR configured */
if ((fid_writer = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl_writer)) < 0)
@@ -2477,10 +2662,15 @@ test_file_end_tick_concur(void)
if ((config_reader = HDmalloc(sizeof(*config_reader))) == NULL)
HDexit(EXIT_FAILURE);
- /* Set up the VFD SWMR configuration as reader + page buffering */
- fapl_reader = init_vfd_swmr_config_fapl(config_reader, 1, 3, FALSE, 256, MD_FILENAME, FS_PAGE_SIZE);
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config_reader, 1, 3, FALSE, TRUE, FALSE, TRUE, 256, MD_FILENAME, NULL);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl_reader = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, FS_PAGE_SIZE, config_reader);
+
if (fapl_reader == H5I_INVALID_HID)
- FAIL_STACK_ERROR;
+ FAIL_STACK_ERROR
/* Open the test file as reader */
if ((fid_reader1 = H5Fopen(FILENAME, H5F_ACC_RDONLY, fapl_reader)) < 0)
@@ -2667,23 +2857,33 @@ test_multiple_file_opens(void)
if ((config2 = HDmalloc(sizeof(*config2))) == NULL)
FAIL_STACK_ERROR;
- /* Configured as VFD SWMR writer + page buffering */
- fapl1 = init_vfd_swmr_config_fapl(config1, 4, 6, TRUE, 2, MD_FILENAME, 4096);
+ /*
+ * Configured as VFD SWMR writer + page buffering
+ */
+
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config1, 4, 6, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME, NULL);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl1 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config1);
if (fapl1 == H5I_INVALID_HID)
FAIL_STACK_ERROR;
- /* Configured as VFD SWMR writer + page buffering */
- fapl2 = init_vfd_swmr_config_fapl(config2, 4, 6, TRUE, 2, MD_FILENAME2, 4096);
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config2, 4, 6, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME2, NULL);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl2 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config2);
+
if (fapl2 == H5I_INVALID_HID)
FAIL_STACK_ERROR;
- /* Create a copy of the file creation property list */
- if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0)
- FAIL_STACK_ERROR
-
- /* Set file space strategy */
- if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0)
+ if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, 4096)) < 0) {
+ HDprintf("vfd_swmr_create_fcpl() failed");
FAIL_STACK_ERROR;
+ }
/* Create a file without VFD SWMR */
if ((fid = H5Fcreate(FNAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
@@ -2854,13 +3054,10 @@ test_same_file_opens(void)
if ((config2 = HDmalloc(sizeof(*config2))) == NULL)
FAIL_STACK_ERROR;
- /* Create a copy of the file creation property list */
- if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0)
+ if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, 4096)) < 0) {
+ HDprintf("vfd_swmr_create_fcpl() failed");
FAIL_STACK_ERROR
-
- /* Set file space strategy */
- if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0)
- FAIL_STACK_ERROR;
+ }
/*
* Tests for first column
@@ -2874,8 +3071,17 @@ test_same_file_opens(void)
if (H5Fclose(fid) < 0)
FAIL_STACK_ERROR;
- /* Set the VFD SWMR configuration in fapl1 + page buffering */
- fapl1 = init_vfd_swmr_config_fapl(config1, 4, 10, TRUE, 2, MD_FILENAME, 4096);
+ /*
+ * Set the VFD SWMR configuration in fapl1 + page buffering
+ */
+
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config1, 4, 10, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME, NULL);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl1 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config1);
+
if (fapl1 == H5I_INVALID_HID)
FAIL_STACK_ERROR;
@@ -2893,8 +3099,17 @@ test_same_file_opens(void)
if (H5Fclose(fid2) < 0)
FAIL_STACK_ERROR;
- /* Set the VFD SWMR configuration in fapl2 + page buffering */
- fapl2 = init_vfd_swmr_config_fapl(config2, 3, 8, FALSE, 3, MD_FILENAME, 4096);
+ /*
+ * Set the VFD SWMR configuration in fapl2 + page buffering
+ */
+
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config2, 3, 8, FALSE, TRUE, FALSE, TRUE, 3, MD_FILENAME, NULL);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl2 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config2);
+
if (fapl2 == H5I_INVALID_HID)
FAIL_STACK_ERROR;
@@ -2939,8 +3154,18 @@ test_same_file_opens(void)
* Tests for second column
*/
- /* Set up as VFD SWMR reader + page buffering */
- fapl1 = init_vfd_swmr_config_fapl(config1, 4, 10, FALSE, 2, MD_FILENAME, 4096);
+
+ /*
+ * Set up as VFD SWMR reader + page buffering
+ */
+
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config1, 4, 10, FALSE, TRUE, FALSE, TRUE, 2, MD_FILENAME, NULL);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl1 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config1);
+
if (fapl1 == H5I_INVALID_HID)
FAIL_STACK_ERROR;
@@ -2967,8 +3192,17 @@ test_same_file_opens(void)
if ((fid = H5Fopen(FILENAME, H5F_ACC_RDWR, H5P_DEFAULT)) < 0)
TEST_ERROR;
- /* Set up as VFD SWMR writer + page buffering */
- fapl1 = init_vfd_swmr_config_fapl(config1, 4, 10, TRUE, 2, MD_FILENAME, 4096);
+ /*
+ * Set up as VFD SWMR writer + page buffering
+ */
+
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config1, 4, 10, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME, NULL);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl1 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config1);
+
if (fapl1 == H5I_INVALID_HID)
FAIL_STACK_ERROR;
@@ -3014,8 +3248,17 @@ test_same_file_opens(void)
if ((fid = H5Fopen(FILENAME, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0)
TEST_ERROR;
- /* Set up as VFD SWMR writer + page buffering */
- fapl1 = init_vfd_swmr_config_fapl(config1, 4, 10, TRUE, 2, MD_FILENAME, 4096);
+ /*
+ * Set up as VFD SWMR writer + page buffering
+ */
+
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config1, 4, 10, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME, NULL);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl1 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config1);
+
if (fapl1 == H5I_INVALID_HID)
FAIL_STACK_ERROR;
@@ -3216,28 +3459,52 @@ test_enable_disable_eot(void)
if ((config3 = HDmalloc(sizeof(*config3))) == NULL)
FAIL_STACK_ERROR;
- /* Configured first file as VFD SWMR writer + page buffering */
- fapl1 = init_vfd_swmr_config_fapl(config1, 4, 6, TRUE, 2, MD_FILENAME, 4096);
+ /*
+ * Configured first file as VFD SWMR writer + page buffering
+ */
+
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config1, 4, 6, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME, NULL);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl1 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config1);
+
if (fapl1 == H5I_INVALID_HID)
FAIL_STACK_ERROR;
- /* Configured second file as VFD SWMR writer + page buffering */
- fapl2 = init_vfd_swmr_config_fapl(config2, 4, 6, TRUE, 2, MD_FILENAME2, 4096);
+ /*
+ * Configured second file as VFD SWMR writer + page buffering
+ */
+
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config2, 4, 6, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME2, NULL);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl2 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config2);
+
if (fapl2 == H5I_INVALID_HID)
FAIL_STACK_ERROR;
- /* Configured third file as VFD SWMR writer + page buffering */
- fapl3 = init_vfd_swmr_config_fapl(config3, 4, 6, TRUE, 2, MD_FILENAME3, 4096);
+ /*
+ * Configured third file as VFD SWMR writer + page buffering
+ */
+
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config3, 4, 6, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME3, NULL);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl3 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config3);
+
if (fapl3 == H5I_INVALID_HID)
FAIL_STACK_ERROR;
- /* Create a copy of the file creation property list */
- if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0)
+ if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, 4096)) < 0) {
+ HDprintf("vfd_swmr_create_fcpl() failed");
FAIL_STACK_ERROR
-
- /* Set file space strategy */
- if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0)
- FAIL_STACK_ERROR;
+ }
/* Create a file without VFD SWMR */
if ((fid = H5Fcreate(FNAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
@@ -3425,6 +3692,703 @@ error:
} /* test_enable_disable_eot() */
/*-------------------------------------------------------------------------
+ * Function: verify_updater_flags()
+ *
+ * Purpose: This is the helper routine used to verify whether
+ * "flags" in the updater file is as expected.
+ *
+ * Return: 0 if test is sucessful
+ * 1 if test fails
+ *
+ * Programmer: Vailin Choi; October 2021
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+verify_updater_flags(char *ud_name, uint16_t expected_flags)
+{
+ FILE* ud_fp; /* Updater file pointer */
+ uint16_t flags;
+ size_t ret; /* Return value */
+
+
+ /* Open the updater file */
+ if((ud_fp = HDfopen(ud_name, "r")) == NULL)
+ FAIL_STACK_ERROR;
+
+ /* Seek to the position of "flags" in the updater file's header */
+ if(HDfseek(ud_fp, (HDoff_t)UD_HD_FLAGS_OFFSET, SEEK_SET) < 0)
+ FAIL_STACK_ERROR;
+
+ /* Read "flags" from the updater file */
+ if((ret = HDfread(&flags, UD_SIZE_2, 1, ud_fp)) != (size_t)1)
+ FAIL_STACK_ERROR;
+
+ if (flags != expected_flags)
+ TEST_ERROR;
+
+ if (HDfclose(ud_fp) < 0)
+ FAIL_STACK_ERROR;
+
+ return 0;
+
+error:
+ return 1;
+
+} /* verify_updater_flags() */
+
+/*-------------------------------------------------------------------------
+ * Function: test_updater_flags
+ *
+ * Purpose: Verify "flags" in the updater file is as expected for
+ * file creation.
+ *
+ * Return: 0 if test is sucessful
+ * 1 if test fails
+ *
+ * Programmer: Vailin Choi; October 2021
+ *
+ *-------------------------------------------------------------------------
+ */
+static unsigned
+test_updater_flags(void)
+{
+ hid_t fid = -1; /* File ID */
+ hid_t fcpl = -1; /* File creation property list ID */
+ hid_t fapl = -1; /* File access property list ID */
+ hid_t file_fapl = -1; /* File access property list ID associated with the file */
+ H5F_vfd_swmr_config_t *config = NULL; /* Configuration for VFD SWMR */
+ H5F_vfd_swmr_config_t *file_config = NULL; /* Configuration for VFD SWMR */
+ uint64_t seq_num = 0; /* Sequence number for updater file */
+ uint64_t i = 0; /* Local index variable */
+ char namebuf[H5F__MAX_VFD_SWMR_FILE_NAME_LEN]; /* Updater file path */
+ h5_stat_t sb; /* Info returned by stat system call */
+
+ TESTING("VFD SWMR updater file flags");
+
+ /* Should succeed without VFD SWMR configured */
+ if ((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+ TEST_ERROR;
+
+ /* Close the file */
+ if (H5Fclose(fid) < 0)
+ FAIL_STACK_ERROR;
+
+ /* Allocate memory for the configuration structure */
+ if ((config = (H5F_vfd_swmr_config_t *)HDmalloc(sizeof(H5F_vfd_swmr_config_t))) == NULL)
+ FAIL_STACK_ERROR;
+ if ((file_config = (H5F_vfd_swmr_config_t *)HDmalloc(sizeof(H5F_vfd_swmr_config_t))) == NULL)
+ FAIL_STACK_ERROR;
+
+ /*
+ * Configured as VFD SWMR writer + page buffering + generate updater files
+ */
+
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config, 4, 7, TRUE, TRUE, TRUE, TRUE, 2, MD_FILENAME, UD_FILENAME);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config);
+
+ if (fapl == H5I_INVALID_HID)
+ TEST_ERROR
+
+ if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, 4096)) < 0) {
+ HDprintf("vfd_swmr_create_fcpl() failed");
+ FAIL_STACK_ERROR
+ }
+
+ if ((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl)) < 0)
+ TEST_ERROR;
+
+ /* Get the file's file access property list */
+ if ((file_fapl = H5Fget_access_plist(fid)) < 0)
+ FAIL_STACK_ERROR;
+
+ /* Retrieve the VFD SWMR configuration from file_fapl */
+ if (H5Pget_vfd_swmr_config(file_fapl, file_config) < 0)
+ TEST_ERROR;
+
+ /* Verify the retrieved info is the same as config1 */
+ if (HDmemcmp(config, file_config, sizeof(H5F_vfd_swmr_config_t)) != 0)
+ TEST_ERROR;
+
+ /* Verify the first updater file: "flags" field and file size */
+ HDsprintf(namebuf, "%s.%lu", UD_FILENAME, seq_num);
+
+ /* Verify "flags" of the first updater file */
+ if(verify_updater_flags(namebuf, CREATE_METADATA_FILE_ONLY_FLAG) < 0)
+ TEST_ERROR;
+
+ /* Check updater file size */
+ if (HDstat(namebuf, &sb) == 0 && sb.st_size != H5F_UD_HEADER_SIZE)
+ TEST_ERROR;
+
+ /* Closing */
+ if (H5Fclose(fid) < 0)
+ FAIL_STACK_ERROR;
+
+ /* Look for the last updater file */
+ for (seq_num = 0; ; seq_num++) {
+ HDsprintf(namebuf, "%s.%lu", UD_FILENAME, seq_num);
+ if (HDaccess(namebuf, F_OK) != 0)
+ break;
+ }
+ HDsprintf(namebuf, "%s.%lu", UD_FILENAME, seq_num-1);
+
+ /* Verify "flags" of the last updater file */
+ if(verify_updater_flags(namebuf, FINAL_UPDATE_FLAG) < 0)
+ TEST_ERROR;
+
+ if (H5Pclose(file_fapl) < 0)
+ FAIL_STACK_ERROR;
+ if (H5Pclose(fapl) < 0)
+ FAIL_STACK_ERROR;
+ if (H5Pclose(fcpl) < 0)
+ FAIL_STACK_ERROR;
+
+ /* Remove updater files */
+ for (i = 0; i < seq_num; i++) {
+ HDsprintf(namebuf, "%s.%lu", UD_FILENAME, i);
+ HDremove(namebuf);
+ }
+
+ /* Free buffers */
+ if (config)
+ HDfree(config);
+ if (file_config)
+ HDfree(file_config);
+
+ PASSED();
+ return 0;
+
+error:
+ H5E_BEGIN_TRY
+ {
+ H5Pclose(fapl);
+ H5Pclose(file_fapl);
+ H5Pclose(fcpl);
+ H5Fclose(fid);
+ }
+ H5E_END_TRY;
+ if (config)
+ HDfree(config);
+ if (file_config)
+ HDfree(file_config);
+
+ return 1;
+} /* test_updater_flags() */
+
+/*-------------------------------------------------------------------------
+ * Function: test_updater_flags_same_file_opens()
+ *
+ * Purpose: Verify "flags" in the updater file is as expected for
+ * multiple opens of the same file.
+ *
+ * Return: 0 if test is sucessful
+ * 1 if test fails
+ *
+ * Programmer: Vailin Choi; October 2021
+ *
+ *-------------------------------------------------------------------------
+ */
+static unsigned
+test_updater_flags_same_file_opens(void)
+{
+ hid_t fid = -1; /* File ID */
+ hid_t fid2 = -1; /* File ID */
+ hid_t fcpl = -1; /* File creation property list ID */
+ hid_t fapl1 = -1; /* File access property list ID */
+ H5F_vfd_swmr_config_t *config1 = NULL; /* Configuration for VFD SWMR */
+ uint64_t seq_num = 0; /* Sequence number for updater file */
+ uint64_t i = 0; /* Local index variable */
+ char namebuf[H5F__MAX_VFD_SWMR_FILE_NAME_LEN]; /* Updater file path */
+
+ TESTING("VFD SWMR updater file flags for multiple opens of the same file");
+
+ /* Should succeed without VFD SWMR configured */
+ if ((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+ TEST_ERROR;
+
+ /* Close the file */
+ if (H5Fclose(fid) < 0)
+ FAIL_STACK_ERROR;
+
+ /* Allocate memory for the configuration structure */
+ if ((config1 = HDmalloc(sizeof(*config1))) == NULL)
+ FAIL_STACK_ERROR;
+
+ if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, 4096)) < 0) {
+ HDprintf("vfd_swmr_create_fcpl() failed");
+ FAIL_STACK_ERROR
+ }
+
+ /* Create the test file */
+ if ((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, H5P_DEFAULT)) < 0)
+ TEST_ERROR;
+
+ /* Close the file */
+ if (H5Fclose(fid) < 0)
+ FAIL_STACK_ERROR;
+
+ /*
+ * Set the VFD SWMR configuration in fapl1 + page buffering
+ */
+
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config1, 4, 10, TRUE, TRUE, TRUE, TRUE, 2, MD_FILENAME, UD_FILENAME);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl1 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config1);
+
+ if (fapl1 == H5I_INVALID_HID)
+ FAIL_STACK_ERROR;
+
+ /* Open the file as VFD SWMR writer */
+ /* Keep the file open */
+ if ((fid = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl1)) < 0)
+ TEST_ERROR;
+
+ /* Open the same file again as VFD SWMR writer */
+ /* Should succeed: 1st open--VFD SWMR writer, 2nd open--VFD SWMR writer */
+ if ((fid2 = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl1)) < 0)
+ TEST_ERROR;
+
+ /* Close the second file open */
+ if (H5Fclose(fid2) < 0)
+ FAIL_STACK_ERROR;
+
+
+ /* Verify the first updater file for first file open */
+ HDsprintf(namebuf, "%s.%lu", UD_FILENAME, seq_num);
+
+ /* Verify "flags" of the first updater file is 0*/
+ if(verify_updater_flags(namebuf, 0) < 0)
+ TEST_ERROR;
+
+ /* Look for the last updater file */
+ for (seq_num = 0; ; seq_num++) {
+ HDsprintf(namebuf, "%s.%lu", UD_FILENAME, seq_num);
+ if (HDaccess(namebuf, F_OK) != 0)
+ break;
+ }
+ HDsprintf(namebuf, "%s.%lu", UD_FILENAME, seq_num-1);
+
+ /* Verify "flags" of the last updater file is 0 */
+ if(verify_updater_flags(namebuf, 0) < 0)
+ TEST_ERROR;
+
+ /* Close the 1st open file */
+ if (H5Fclose(fid) < 0)
+ FAIL_STACK_ERROR;
+
+ /* Look for the last updater file */
+ for (seq_num = 0; ; seq_num++) {
+ HDsprintf(namebuf, "%s.%lu", UD_FILENAME, seq_num);
+ if (HDaccess(namebuf, F_OK) != 0)
+ break;
+ }
+ HDsprintf(namebuf, "%s.%lu", UD_FILENAME, seq_num-1);
+
+ /* Verify "flags" of the last updater file after closing file */
+ if(verify_updater_flags(namebuf, FINAL_UPDATE_FLAG) < 0)
+ TEST_ERROR;
+
+ /* Clean up updater files */
+ for (i = 0; i < seq_num; i++) {
+ HDsprintf(namebuf, "%s.%lu", UD_FILENAME, i);
+ HDremove(namebuf);
+ }
+
+ if (H5Pclose(fcpl) < 0)
+ FAIL_STACK_ERROR;
+
+ /* Free buffers */
+ if (config1)
+ HDfree(config1);
+
+ PASSED();
+ return 0;
+
+error:
+ H5E_BEGIN_TRY
+ {
+ H5Pclose(fapl1);
+ H5Pclose(fcpl);
+ H5Fclose(fid);
+ H5Fclose(fid2);
+ }
+ H5E_END_TRY;
+ if (config1)
+ HDfree(config1);
+ return 1;
+} /* test_updater_flags_same_file_opens() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: clean_chk_ud_files()
+ *
+ * Purpose: This is the helper routine used to clean up
+ * the checksum file and the updater files.
+ *
+ * Return: void
+ *
+ * Programmer: Vailin Choi; October 2021
+ *
+ *-------------------------------------------------------------------------
+ */
+static void
+clean_chk_ud_files(char *md_file_path, char *updater_file_path)
+{
+ char chk_name[1024 + 4];
+ char ud_name[1024 + 4];
+ uint64_t i;
+
+ /* Name of the checksum file: <md_file_path>.chk */
+ HDsprintf(chk_name, "%s.chk", md_file_path);
+
+ /* Remove the checksum file if exists.
+ If not, the callback will just continue appending
+ checksums to the existing file */
+ if(HDaccess(chk_name, F_OK) == 0) {
+ HDremove(chk_name);
+ }
+
+ /* Remove all the updater files if exist: <updater_file_path>.<i> */
+ for(i = 0; ; i++) {
+ sprintf(ud_name, "%s.%lu", updater_file_path, i);
+ if(HDaccess(ud_name, F_OK) != 0)
+ break;
+ HDremove(ud_name);
+ }
+
+} /* clean_chk_ud_files() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: verify_ud_chk()
+ *
+ * Purpose: This is the helper routine used by
+ * test_updater_generate_md_checksums() to verify
+ * contents of the checksum file and the updater files.
+ * --verify the sequence number in each updater's file header
+ * corresponds to the ith sequence number of the updater
+ * file name.
+ * --verify the tick number in each updater's file header
+ * corresponds to the tick number stored in the checksum file
+ * --verify the change_list_len in each updater's file header
+ * is consistent with num_change_list_entries in each updater's
+ * change list header
+ *
+ * Return: 0 if test is sucessful
+ * 1 if test fails
+ *
+ * Programmer: Vailin Choi; October 2021
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+verify_ud_chk(char *md_file_path, char *ud_file_path)
+{
+ char chk_name[1024 + 4]; /* Checksum file name */
+ char ud_name[1024 + 4]; /* Updater file name */
+ FILE* chk_fp; /* Checksum file pointer */
+ FILE* ud_fp; /* Updater file pointer */
+ uint64_t ud_seq_num; /* Sequence number in the updater file */
+ uint64_t chk_ud_seq_num; /* Updater sequence number in the checksum file */
+ uint64_t i; /* Local index variable */
+ long size = 0; /* Size of the file */
+ size_t change_list_len; /* change_list_len in the updater file header */
+ uint32_t num_change_list_entries; /* num_change_list_entries in the updater change list header */
+
+ /* Open the checksum file */
+ HDsprintf(chk_name, "%s.chk", md_file_path);
+ if((chk_fp = HDfopen(chk_name, "r")) == NULL)
+ FAIL_STACK_ERROR;
+
+ for(i = 0; ; i++) {
+ /* Generate updater file name: <ud_file_path>.<i> */
+ HDsprintf(ud_name, "%s.%lu", ud_file_path, i);
+
+ /* Open the updater file */
+ if ((ud_fp = HDfopen(ud_name, "r")) == NULL)
+ break;
+ else {
+ /* Seek to the position of the sequence number in the updater file's header */
+ if(HDfseek(ud_fp, (off_t)UD_HD_SEQ_NUM_OFFSET, SEEK_SET) < 0)
+ FAIL_STACK_ERROR;
+
+ /* Read the sequence number from the updater file */
+ if(HDfread(&ud_seq_num, UD_SIZE_8, 1, ud_fp) != 1)
+ FAIL_STACK_ERROR;
+
+ /* Compare the sequence number with i */
+ if(ud_seq_num != i)
+ TEST_ERROR;
+
+ /* Read change_list_len from updater file's header */
+ if(HDfseek(ud_fp, (off_t)UD_HD_CHANGE_LIST_LEN_OFFSET, SEEK_SET) < 0)
+ FAIL_STACK_ERROR;
+
+ if(HDfread(&change_list_len, UD_SIZE_8, 1, ud_fp) != 1)
+ FAIL_STACK_ERROR;
+
+ if(i != 0) {
+
+ /* Read num_change_list_entries from updater file's change list */
+ if(HDfseek(ud_fp, (off_t)UD_CL_NUM_CHANGE_LIST_ENTRIES_OFFSET, SEEK_SET) < 0)
+ FAIL_STACK_ERROR;
+
+ if(HDfread(&num_change_list_entries, UD_SIZE_4, 1, ud_fp) != 1)
+ FAIL_STACK_ERROR;
+
+ if(num_change_list_entries == 0) {
+ if(change_list_len != H5F_UD_CL_SIZE(0))
+ TEST_ERROR;
+ } else {
+ if (change_list_len != H5F_UD_CL_SIZE(num_change_list_entries))
+ TEST_ERROR
+ }
+ }
+
+ /* Close the updater file */
+ if(HDfclose(ud_fp) < 0)
+ FAIL_STACK_ERROR;
+
+ /* Read the updater sequence number from checksum file */
+ if(HDfread(&chk_ud_seq_num, UD_SIZE_8, 1, chk_fp) != 1)
+ FAIL_STACK_ERROR;
+
+ /* Compare sequence number in updater file with sequence number in checksum file */
+ if (ud_seq_num != chk_ud_seq_num)
+ TEST_ERROR;
+
+ /* Advance checksum file to the next sequence number */
+ if(HDfseek(chk_fp, (off_t)UD_SIZE_4, SEEK_CUR) < 0)
+ FAIL_STACK_ERROR;
+ }
+ }
+
+ /* Get the size of the chksum file */
+ if ((size = HDftell(chk_fp)) < 0)
+ FAIL_STACK_ERROR;
+
+ /* Size of sequence number and checksum in the checksum file */
+ if((unsigned)size != (i * (UD_SIZE_8 + UD_SIZE_4)))
+ TEST_ERROR;
+
+ return 0;
+
+error:
+ return (-1);
+
+} /* verify_ud_chk() */
+
+/*-------------------------------------------------------------------------
+ * Function: md_ck_cb()
+ *
+ * Purpose: This is the callback function used by
+ * test_updater_generate_md_checksums() when the
+ * H5F_ACS_GENERATE_MD_CK_CB_NAME property is set in fapl.
+ * --Opens and read the metadata file into a buffer.
+ * --Generate checksum for the metadata file
+ * --Write the tick number and the checksum to the checksum file
+ *
+ * Return: 0 if test is sucessful
+ * 1 if test fails
+ *
+ * Programmer: Vailin Choi; October 2021
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+md_ck_cb(char *md_file_path, uint64_t updater_seq_num)
+{
+ FILE* md_fp = NULL; /* Metadata file pointer */
+ FILE* chk_fp = NULL; /* Checksum file pointer */
+ long size = 0; /* File size returned from HDftell() */
+ void *buf = NULL; /* Buffer for holding the metadata file content */
+ uint32_t chksum = 0; /* The checksum generated for the metadata file */
+ char chk_name[1024 + 4]; /* Buffer for the checksum file name */
+ size_t ret; /* Return value */
+
+ /* Open the metadata file */
+ if((md_fp = HDfopen(md_file_path, "r")) == NULL)
+ FAIL_STACK_ERROR;
+
+ /* Set file pointer at end of file.*/
+ if(HDfseek(md_fp, 0, SEEK_END) < 0)
+ FAIL_STACK_ERROR;
+
+ /* Get the current position of the file pointer.*/
+ if ((size = HDftell(md_fp)) < 0)
+ FAIL_STACK_ERROR;
+
+ if(size != 0) {
+
+ HDrewind(md_fp);
+
+ if((buf = HDmalloc((size_t)size)) == NULL)
+ FAIL_STACK_ERROR;
+
+ /* Read the metadata file to buf */
+ if ((ret = HDfread(buf, 1, (size_t)size, md_fp)) != (size_t)size)
+ FAIL_STACK_ERROR;
+
+ /* Calculate checksum of the metadata file */
+ chksum = H5_checksum_metadata(buf, (size_t)size, 0);
+ }
+
+ /* Close the metadata file */
+ if(md_fp && HDfclose(md_fp) < 0)
+ FAIL_STACK_ERROR;
+
+ /*
+ * Checksum file
+ */
+
+ /* Generate checksum file name: <md_file_path>.chk */
+ HDsprintf(chk_name, "%s.chk", md_file_path);
+
+ /* Open checksum file for append */
+ if((chk_fp = HDfopen(chk_name, "a")) == NULL)
+ FAIL_STACK_ERROR;
+
+ /* Write the updater sequence number to the checksum file */
+ if((ret = HDfwrite(&updater_seq_num, sizeof(uint64_t), 1, chk_fp)) != 1)
+ FAIL_STACK_ERROR;
+
+ /* Write the checksum to the checksum file */
+ if((ret = HDfwrite(&chksum, sizeof(uint32_t), 1, chk_fp)) != 1)
+ FAIL_STACK_ERROR;
+
+ /* Close the checksum file */
+ if(chk_fp && HDfclose(chk_fp) != 0)
+ FAIL_STACK_ERROR;
+
+ if(buf)
+ HDfree(buf);
+
+ return 0;
+
+error:
+ if(buf)
+ HDfree(buf);
+ if(md_fp)
+ HDfclose(md_fp);
+ if(chk_fp)
+ HDfclose(chk_fp);
+
+ return -1;
+} /* md_ck_cb() */
+
+/*-------------------------------------------------------------------------
+ * Function: test_updater_generate_md_checksums()
+ *
+ * Purpose: It enables the generation of checksums for the metadata file
+ * created by the writer end of tick function.
+ * It also verifies the contents of the checksum file and the
+ * updater files.
+ *
+ * The test is invoked when the file is created via H5Fcreate()
+ * and via H5Fopen().
+ *
+ * Return: 0 if test is sucessful
+ * 1 if test fails
+ *
+ * Programmer: Vailin Choi; October 2021
+ *
+ * Note: It is important to clean up the checksum file and the updater files.
+ *
+ *-------------------------------------------------------------------------
+ */
+static unsigned
+test_updater_generate_md_checksums(hbool_t file_create)
+{
+ hid_t fid = -1; /* File ID */
+ hid_t fcpl = -1; /* File creation property list ID */
+ hid_t fapl = -1; /* File access property list ID */
+ H5F_vfd_swmr_config_t config; /* Configuration for VFD SWMR */
+ H5F_generate_md_ck_cb_t cb_info; /* Callback */
+
+ if(file_create) {
+ TESTING("VFD SWMR updater generate checksums for metadata file with H5Fcreate");
+ } else {
+ TESTING("VFD SWMR updater generate checksums for metadata file with H5Fopen");
+ }
+
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(&config, 4, 7, TRUE, TRUE, TRUE, TRUE, 2, MD_FILE, UD_FILE);
+
+ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
+ fapl = vfd_swmr_create_fapl(TRUE, TRUE, FALSE, 4096, &config);
+
+ if (fapl == H5I_INVALID_HID)
+ TEST_ERROR
+
+ if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, 4096)) < 0) {
+ HDprintf("vfd_swmr_create_fcpl() failed");
+ FAIL_STACK_ERROR
+ }
+
+ /* Set up callback to generate checksums for updater's metadata files */
+ cb_info.func = md_ck_cb;
+
+ /* Activate private property to generate checksums for updater's metadata file */
+ H5Pset(fapl, H5F_ACS_GENERATE_MD_CK_CB_NAME, &cb_info);
+
+ /* Use file creation or file open for testing */
+ if(file_create) {
+ if((fid = H5Fcreate(FILENAME4, H5F_ACC_TRUNC, fcpl, fapl)) < 0)
+ TEST_ERROR;
+ } else {
+ fid = H5Fcreate(FILENAME4, H5F_ACC_TRUNC, fcpl, H5P_DEFAULT);
+ H5Fclose(fid);
+
+ fid = H5Fopen(FILENAME4, H5F_ACC_RDWR, fapl);
+ }
+
+ /* Close the file */
+ if (H5Fclose(fid) < 0)
+ FAIL_STACK_ERROR;
+
+ if (H5Pclose(fapl) < 0)
+ FAIL_STACK_ERROR;
+ if (H5Pclose(fcpl) < 0)
+ FAIL_STACK_ERROR;
+
+ /* Verify contents of checksum file and updater files */
+ if(verify_ud_chk(config.md_file_path, config.updater_file_path) < 0)
+ TEST_ERROR;
+
+ /* It's important to clean up the chechsum and updater files. */
+ clean_chk_ud_files(config.md_file_path, config.updater_file_path);
+
+ PASSED();
+
+ return 0;
+
+error:
+ H5E_BEGIN_TRY
+ {
+ H5Pclose(fapl);
+ H5Pclose(fcpl);
+ H5Fclose(fid);
+ }
+ H5E_END_TRY;
+
+ /* It's important to clean up the chechsum and updater files. */
+ clean_chk_ud_files(config.md_file_path, config.updater_file_path);
+
+ return 1;
+
+} /* test_updater_generate_md_checksums() */
+
+
+/*-------------------------------------------------------------------------
* Function: main()
*
* Purpose: Main function for VFD SWMR tests.
@@ -3481,9 +4445,15 @@ main(void)
PUTS_ERROR("Can't get VFD-dependent fapl")
}
+ /* Add nfs/updater testing in this routine */
nerrors += test_fapl();
if (use_file_locking) {
+ nerrors += test_updater_flags();
+ nerrors += test_updater_flags_same_file_opens();
+ nerrors += test_updater_generate_md_checksums(TRUE);
+ nerrors += test_updater_generate_md_checksums(FALSE);
+
nerrors += test_shadow_index_lookup();
nerrors += test_file_fapl();
diff --git a/test/vfd_swmr_addrem_writer.c b/test/vfd_swmr_addrem_writer.c
index f115d2e..a2982fd 100644
--- a/test/vfd_swmr_addrem_writer.c
+++ b/test/vfd_swmr_addrem_writer.c
@@ -94,8 +94,9 @@ open_skeleton(const char *filename, unsigned verbose)
if ((config = HDcalloc(1, sizeof(*config))) == NULL)
goto error;
- /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */
- init_vfd_swmr_config(config, 4, 5, TRUE, TRUE, 128, "./rw-shadow");
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config, 4, 5, TRUE, TRUE, FALSE, TRUE, 128, "rw-shadow", NULL);
/* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
if ((fapl = vfd_swmr_create_fapl(TRUE, TRUE, FALSE, 4096, config)) < 0)
diff --git a/test/vfd_swmr_attrdset_writer.c b/test/vfd_swmr_attrdset_writer.c
index f445321..a722f44 100644
--- a/test/vfd_swmr_attrdset_writer.c
+++ b/test/vfd_swmr_attrdset_writer.c
@@ -1979,8 +1979,9 @@ main(int argc, char **argv)
TEST_ERROR;
}
- /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */
- init_vfd_swmr_config(&config, 4, 7, writer, true, 128, "./attrdset-shadow");
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(&config, 4, 7, writer, TRUE, FALSE, TRUE, 128, "./attrdset-shadow", NULL);
/* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
if ((fapl = vfd_swmr_create_fapl(true, s.use_vfd_swmr, true, 4096, &config)) < 0) {
diff --git a/test/vfd_swmr_bigset_writer.c b/test/vfd_swmr_bigset_writer.c
index 77c7b26..928f7c7 100644
--- a/test/vfd_swmr_bigset_writer.c
+++ b/test/vfd_swmr_bigset_writer.c
@@ -2470,9 +2470,9 @@ main(int argc, char **argv)
continue;
}
- /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */
- init_vfd_swmr_config(&config, s.tick_len, s.max_lag, s.writer, s.flush_raw_data, 128,
- "./bigset-shadow-%zu", i);
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(&config, s.tick_len, s.max_lag, s.writer, TRUE, FALSE, s.flush_raw_data, 128, "./bigset-shadow-%zu", NULL, i);
/* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
if ((fapl = vfd_swmr_create_fapl(true, s.use_vfd_swmr, true, s.page_buf_size, &config)) < 0) {
diff --git a/test/vfd_swmr_common.c b/test/vfd_swmr_common.c
index cfe93aa..90abeed 100644
--- a/test/vfd_swmr_common.c
+++ b/test/vfd_swmr_common.c
@@ -345,10 +345,10 @@ await_signal(hid_t fid)
/* Revised support routines that can be used for all VFD SWMR integration tests
*/
-/* Initialize fields in config with the input parameters */
void
init_vfd_swmr_config(H5F_vfd_swmr_config_t *config, uint32_t tick_len, uint32_t max_lag, hbool_t writer,
- hbool_t flush_raw_data, uint32_t md_pages_reserved, const char *md_file_fmtstr, ...)
+ hbool_t maintain_metadata_file, hbool_t generate_updater_files, hbool_t flush_raw_data,
+ uint32_t md_pages_reserved, const char *md_file_fmtstr, const char *updater_file_path, ...)
{
va_list ap;
@@ -360,13 +360,20 @@ init_vfd_swmr_config(H5F_vfd_swmr_config_t *config, uint32_t tick_len, uint32_t
config->tick_len = tick_len;
config->max_lag = max_lag;
config->writer = writer;
+ config->maintain_metadata_file = maintain_metadata_file;
+ config->generate_updater_files = generate_updater_files;
config->flush_raw_data = flush_raw_data;
config->md_pages_reserved = md_pages_reserved;
- HDva_start(ap, md_file_fmtstr);
+ HDva_start(ap, updater_file_path);
+
evsnprintf(config->md_file_path, sizeof(config->md_file_path), md_file_fmtstr, ap);
+
HDva_end(ap);
+ if(config->generate_updater_files && updater_file_path != NULL)
+ HDstrcpy(config->updater_file_path, updater_file_path);
+
} /* init_vfd_swmr_config() */
/* Initialize the log file path in config, this function should be called after init_vfd_swmr_config. */
diff --git a/test/vfd_swmr_common.h b/test/vfd_swmr_common.h
index f5981a5..42569be 100644
--- a/test/vfd_swmr_common.h
+++ b/test/vfd_swmr_common.h
@@ -73,9 +73,11 @@ H5TEST_DLL void await_signal(hid_t);
H5TEST_DLL hid_t vfd_swmr_create_fapl(bool use_latest_format, bool use_vfd_swmr, bool only_meta_pages,
size_t page_buf_size, H5F_vfd_swmr_config_t *config);
-H5TEST_DLL void init_vfd_swmr_config(H5F_vfd_swmr_config_t *config, uint32_t tick_len, uint32_t max_lag,
- hbool_t writer, hbool_t flush_raw_data, uint32_t md_pages_reserved,
- const char *md_file_fmtstr, ...) H5_ATTR_FORMAT(printf, 7, 8);
+H5TEST_DLL void
+init_vfd_swmr_config(H5F_vfd_swmr_config_t *config, uint32_t tick_len, uint32_t max_lag, hbool_t writer,
+ hbool_t maintain_metadata_file, hbool_t generate_updater_files, hbool_t flush_raw_data,
+ uint32_t md_pages_reserved, const char *md_file_fmtstr,
+ const char *updater_file_path, ...) H5_ATTR_FORMAT(printf, 9, 11);
H5TEST_DLL void init_vfd_swmr_log(H5F_vfd_swmr_config_t *config, const char *log_file_fmtstr, ...)
H5_ATTR_FORMAT(printf, 2, 3);
diff --git a/test/vfd_swmr_dsetchks_writer.c b/test/vfd_swmr_dsetchks_writer.c
index d264ca9..3cf43f9 100644
--- a/test/vfd_swmr_dsetchks_writer.c
+++ b/test/vfd_swmr_dsetchks_writer.c
@@ -2302,8 +2302,9 @@ main(int argc, char **argv)
TEST_ERROR;
}
- /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */
- init_vfd_swmr_config(&config, 4, 7, writer, s.flush_raw_data, 128, "./dsetchks-shadow");
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(&config, 4, 7, writer, TRUE, FALSE, s.flush_raw_data, 128, "./dsetchks-shadow", NULL);
/* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
if ((fapl = vfd_swmr_create_fapl(true, s.use_vfd_swmr, true, 4096, &config)) < 0) {
diff --git a/test/vfd_swmr_dsetops_writer.c b/test/vfd_swmr_dsetops_writer.c
index 3704c44..d999d64 100644
--- a/test/vfd_swmr_dsetops_writer.c
+++ b/test/vfd_swmr_dsetops_writer.c
@@ -1888,8 +1888,9 @@ main(int argc, char **argv)
TEST_ERROR;
}
- /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */
- init_vfd_swmr_config(&config, 4, 7, writer, s.flush_raw_data, 128, "./dsetops-shadow");
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(&config, 4, 7, writer, TRUE, FALSE, s.flush_raw_data, 128, "./dsetops-shadow", NULL);
/* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
if ((fapl = vfd_swmr_create_fapl(true, s.use_vfd_swmr, true, 4096, &config)) < 0) {
diff --git a/test/vfd_swmr_generator.c b/test/vfd_swmr_generator.c
index 16b1d4b..1c549ca 100644
--- a/test/vfd_swmr_generator.c
+++ b/test/vfd_swmr_generator.c
@@ -125,8 +125,9 @@ gen_skeleton(const char *filename, hbool_t verbose, hbool_t vfd_swmr_write, int
if ((config = HDcalloc(1, sizeof(*config))) == NULL)
return -1;
- /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */
- init_vfd_swmr_config(config, 4, 10, vfd_swmr_write, TRUE, 128, "generator-shadow");
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config, 4, 10, vfd_swmr_write, TRUE, FALSE, TRUE, 128, "generator-shadow", NULL);
}
/* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
diff --git a/test/vfd_swmr_gfail_writer.c b/test/vfd_swmr_gfail_writer.c
index 430ed89..2be58e2 100644
--- a/test/vfd_swmr_gfail_writer.c
+++ b/test/vfd_swmr_gfail_writer.c
@@ -544,8 +544,9 @@ main(int argc, char **argv)
TEST_ERROR;
}
- /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */
- init_vfd_swmr_config(&config, s.tick_len, s.max_lag, writer, TRUE, 128, "./group-shadow");
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(&config, s.tick_len, s.max_lag, writer, TRUE, FALSE, TRUE, 128, "./group-shadow", NULL);
/* If old-style option is chosen, use the earliest file format(H5F_LIBVER_EARLIEST)
* as the second parameter of H5Pset_libver_bound() that is called by
diff --git a/test/vfd_swmr_gperf_writer.c b/test/vfd_swmr_gperf_writer.c
index 3342b40..18f3199 100644
--- a/test/vfd_swmr_gperf_writer.c
+++ b/test/vfd_swmr_gperf_writer.c
@@ -2786,8 +2786,9 @@ main(int argc, char **argv)
return EXIT_SUCCESS;
}
- /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */
- init_vfd_swmr_config(&config, s.tick_len, s.max_lag, writer, FALSE, 128, "./group-shadow");
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(&config, s.tick_len, s.max_lag, writer, TRUE, FALSE, FALSE, 128, "./group-shadow", NULL);
/* If the log flag is on, create the log file log-test under the current directory. */
if (s.glog == true)
diff --git a/test/vfd_swmr_group_writer.c b/test/vfd_swmr_group_writer.c
index 5881300..af46f3d 100644
--- a/test/vfd_swmr_group_writer.c
+++ b/test/vfd_swmr_group_writer.c
@@ -5011,8 +5011,9 @@ main(int argc, char **argv)
TEST_ERROR;
}
- /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */
- init_vfd_swmr_config(&config, s.tick_len, s.max_lag, writer, TRUE, 128, "./group-shadow");
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(&config, s.tick_len, s.max_lag, writer, TRUE, FALSE, TRUE, 128, "./group-shadow", NULL);
/* If old-style option is chosen, use the earliest file format(H5F_LIBVER_EARLIEST)
* as the second parameter of H5Pset_libver_bound() that is called by
diff --git a/test/vfd_swmr_reader.c b/test/vfd_swmr_reader.c
index 6265cae..2b960c13 100644
--- a/test/vfd_swmr_reader.c
+++ b/test/vfd_swmr_reader.c
@@ -319,8 +319,9 @@ read_records(const char *filename, hbool_t verbose, FILE *verbose_file, unsigned
goto error;
}
- /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */
- init_vfd_swmr_config(config, 4, 5, FALSE, TRUE, 128, "./rw-shadow");
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config, 4, 5, FALSE, TRUE, FALSE, TRUE, 128, "rw-shadow", NULL);
/* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
if ((fapl = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config)) < 0) {
diff --git a/test/vfd_swmr_remove_reader.c b/test/vfd_swmr_remove_reader.c
index 4389b05..18dbb62 100644
--- a/test/vfd_swmr_remove_reader.c
+++ b/test/vfd_swmr_remove_reader.c
@@ -303,8 +303,9 @@ read_records(const char *filename, unsigned verbose, unsigned long nseconds, uns
if ((config = HDcalloc(1, sizeof(*config))) == NULL)
goto error;
- /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */
- init_vfd_swmr_config(config, 4, 5, FALSE, TRUE, 128, "./rw-shadow");
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config, 4, 5, FALSE, TRUE, FALSE, TRUE, 128, "rw-shadow", NULL);
/* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
if ((fapl = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config)) < 0) {
diff --git a/test/vfd_swmr_remove_writer.c b/test/vfd_swmr_remove_writer.c
index 8df8fa8..41c2300 100644
--- a/test/vfd_swmr_remove_writer.c
+++ b/test/vfd_swmr_remove_writer.c
@@ -86,8 +86,9 @@ open_skeleton(const char *filename, unsigned verbose, unsigned old H5_ATTR_UNUSE
if ((config = (H5F_vfd_swmr_config_t *)HDcalloc(1, sizeof(H5F_vfd_swmr_config_t))) == NULL)
goto error;
- /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */
- init_vfd_swmr_config(config, 4, 5, TRUE, TRUE, 128, "./rw-shadow");
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config, 4, 5, TRUE, TRUE, FALSE, TRUE, 128, "rw-shadow", NULL);
/* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
if ((fapl = vfd_swmr_create_fapl(TRUE, TRUE, FALSE, 4096, config)) < 0)
diff --git a/test/vfd_swmr_sparse_reader.c b/test/vfd_swmr_sparse_reader.c
index 943c375..a737f70 100644
--- a/test/vfd_swmr_sparse_reader.c
+++ b/test/vfd_swmr_sparse_reader.c
@@ -207,8 +207,9 @@ read_records(const char *filename, unsigned verbose, unsigned long nrecords, uns
if ((config = HDcalloc(1, sizeof(H5F_vfd_swmr_config_t))) == NULL)
goto error;
- /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */
- init_vfd_swmr_config(config, 4, 5, FALSE, TRUE, 128, "./rw-shadow");
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config, 4, 5, FALSE, TRUE, FALSE, TRUE, 128, "rw-shadow", NULL);
/* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
if ((fapl = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config)) < 0) {
diff --git a/test/vfd_swmr_sparse_writer.c b/test/vfd_swmr_sparse_writer.c
index 56d19f3..f473249 100644
--- a/test/vfd_swmr_sparse_writer.c
+++ b/test/vfd_swmr_sparse_writer.c
@@ -86,8 +86,9 @@ open_skeleton(const char *filename, unsigned verbose)
if ((config = (H5F_vfd_swmr_config_t *)HDcalloc(1, sizeof(H5F_vfd_swmr_config_t))) == NULL)
goto error;
- /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */
- init_vfd_swmr_config(config, 4, 5, TRUE, TRUE, 128, "./rw-shadow");
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config, 4, 5, TRUE, TRUE, FALSE, TRUE, 128, "rw-shadow", NULL);
/* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
if ((fapl = vfd_swmr_create_fapl(TRUE, TRUE, FALSE, 4096, config)) < 0)
diff --git a/test/vfd_swmr_vlstr_reader.c b/test/vfd_swmr_vlstr_reader.c
index fa9d7b4..2554207 100644
--- a/test/vfd_swmr_vlstr_reader.c
+++ b/test/vfd_swmr_vlstr_reader.c
@@ -115,8 +115,9 @@ main(int argc, char **argv)
if (argc > 0)
errx(EXIT_FAILURE, "unexpected command-line arguments");
- /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */
- init_vfd_swmr_config(&config, 4, 7, false, TRUE, 128, "./vlstr-shadow");
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(&config, 4, 7, false, TRUE, FALSE, TRUE, 128, "./vlstr-shadow", NULL);
/* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
fapl = vfd_swmr_create_fapl(true, use_vfd_swmr, sel == TEST_OOB, 4096, &config);
diff --git a/test/vfd_swmr_vlstr_writer.c b/test/vfd_swmr_vlstr_writer.c
index 4f7376e..e3905e8 100644
--- a/test/vfd_swmr_vlstr_writer.c
+++ b/test/vfd_swmr_vlstr_writer.c
@@ -184,8 +184,9 @@ main(int argc, char **argv)
if (argc > 0)
errx(EXIT_FAILURE, "unexpected command-line arguments");
- /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */
- init_vfd_swmr_config(&config, 4, 7, true, TRUE, 128, "./vlstr-shadow");
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(&config, 4, 7, true, TRUE, FALSE, TRUE, 128, "./vlstr-shadow", NULL);
/* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
fapl = vfd_swmr_create_fapl(true, use_vfd_swmr, sel == TEST_OOB, 4096, &config);
diff --git a/test/vfd_swmr_writer.c b/test/vfd_swmr_writer.c
index 4693554..0fb5fe8 100644
--- a/test/vfd_swmr_writer.c
+++ b/test/vfd_swmr_writer.c
@@ -88,8 +88,9 @@ open_skeleton(const char *filename, hbool_t verbose, FILE *verbose_file, unsigne
if ((config = (H5F_vfd_swmr_config_t *)HDcalloc(1, sizeof(H5F_vfd_swmr_config_t))) == NULL)
return -1;
- /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */
- init_vfd_swmr_config(config, 4, 5, TRUE, TRUE, 128, "./rw-shadow");
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(config, 4, 5, TRUE, TRUE, FALSE, TRUE, 128, "rw-shadow", NULL);
/* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
if ((fapl = vfd_swmr_create_fapl(TRUE, TRUE, FALSE, 4096, config)) < 0)
diff --git a/test/vfd_swmr_zoo_writer.c b/test/vfd_swmr_zoo_writer.c
index 13794d9..d40f2d9 100644
--- a/test/vfd_swmr_zoo_writer.c
+++ b/test/vfd_swmr_zoo_writer.c
@@ -473,8 +473,9 @@ main(int argc, char **argv)
parse_command_line_options(argc, argv);
- /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */
- init_vfd_swmr_config(&vfd_swmr_config, TICK_LEN, 7, writer, TRUE, 128, "./zoo-shadow");
+ /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
+ * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+ init_vfd_swmr_config(&vfd_swmr_config, TICK_LEN, 7, writer, TRUE, FALSE, TRUE, 128, "./zoo-shadow", NULL);
/* ? turn off use latest format argument via 1st argument? since later on it reset to early format */
/* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */