summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorvchoi-hdfgroup <55293060+vchoi-hdfgroup@users.noreply.github.com>2021-11-16 22:59:28 (GMT)
committerGitHub <noreply@github.com>2021-11-16 22:59:28 (GMT)
commitd4bd58d1ecd97436806288bf57b6351dc025af38 (patch)
tree8f9a1bfab01ede517230e87ef65b91139fa603e7
parent33630ee54581f7f54e4ecd8e53ec3e94ba13c493 (diff)
parenta58fbf52f0202546b2a75227e62b48817280ff2f (diff)
downloadhdf5-d4bd58d1ecd97436806288bf57b6351dc025af38.zip
hdf5-d4bd58d1ecd97436806288bf57b6351dc025af38.tar.gz
hdf5-d4bd58d1ecd97436806288bf57b6351dc025af38.tar.bz2
Merge pull request #44 from HDFGroup/feature/vfd_swmr
Feature/vfd swmr
-rw-r--r--src/H5Fint.c22
-rw-r--r--src/H5Fpkg.h69
-rw-r--r--src/H5Fvfd_swmr.c126
-rw-r--r--src/H5Pfapl.c4
-rw-r--r--test/testvfdswmr.sh.in34
-rw-r--r--test/vfd_swmr_bigset_writer.c32
-rw-r--r--test/vfd_swmr_common.c13
-rw-r--r--test/vfd_swmr_common.h3
-rw-r--r--test/vfd_swmr_gperf_writer.c197
9 files changed, 392 insertions, 108 deletions
diff --git a/src/H5Fint.c b/src/H5Fint.c
index e650878..c3fc346 100644
--- a/src/H5Fint.c
+++ b/src/H5Fint.c
@@ -2013,7 +2013,8 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id)
/* Short cuts */
shared = file->shared;
- lf = shared->lf;
+
+ lf = shared->lf;
/* Set the file locking flag. If the file is already open, the file
* requested file locking flag must match that of the open file.
@@ -2092,6 +2093,22 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id)
/* Checked if configured for VFD SWMR */
if (H5F_VFD_SWMR_CONFIG(file)) {
+
+ /* Set up the VFD SWMR LOG file */
+ /* Kent*/
+ if (HDstrlen(vfd_swmr_config_ptr->log_file_path) > 0)
+ shared->vfd_swmr_log_on = TRUE;
+ if (TRUE == shared->vfd_swmr_log_on) {
+ /* Create the log file */
+ if ((shared->vfd_swmr_log_file_ptr = HDfopen(vfd_swmr_config_ptr->log_file_path, "w")) == NULL)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to create the log file")
+ if (H5_timer_init(&(shared->vfd_swmr_log_start_time)) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't initialize HDF5 timer.")
+ if (H5_timer_start(&(shared->vfd_swmr_log_start_time)) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't obtain the time from the HDF5 timer.")
+ }
+ /* End of Kent */
+
/* Initialization for VFD SWMR writer and reader */
if (1 == shared->nrefs) {
if (H5F_vfd_swmr_init(file, file_create) < 0)
@@ -2210,6 +2227,9 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id)
/* Success */
ret_value = file;
+#if 1 /*Kent: write to the log file when H5F_open ends. Tested, can be commented out if necessary.*/
+ H5F_POST_VFD_SWMR_LOG_ENTRY(file, FILE_OPEN, "File open ends");
+#endif
done:
if ((NULL == ret_value) && file) {
if (file->shared->root_grp && file->shared->nrefs == 1) {
diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h
index c94c5af..706e41d 100644
--- a/src/H5Fpkg.h
+++ b/src/H5Fpkg.h
@@ -468,7 +468,17 @@ struct H5F_shared_t {
H5F_fs_state_t fs_state_md; /* State of the free space
* manager
*/
-
+ /* Log file for VFD SWMR */
+ FILE *vfd_swmr_log_file_ptr; /* File pointer for the
+ * log file.
+ */
+ hbool_t vfd_swmr_log_on; /* flag to indicate if
+ * the log file is
+ * created. */
+ H5_timer_t vfd_swmr_log_start_time; /* The starting time for
+ * calculating the time
+ * stamp of a log message.
+ */
/* Delayed free space release doubly linked list */
shadow_defree_queue_t shadow_defrees;
@@ -608,4 +618,61 @@ H5_DLL htri_t H5F__same_file_test(hid_t file_id1, hid_t file_id2);
H5_DLL herr_t H5F__reparse_file_lock_variable_test(void);
#endif /* H5F_TESTING */
+/* VFD SMWR LOG REPORTING MACROS */
+
+/* H5F_POST_VFD_SWMR_LOG_ENTRY is the macro that can help the developers debug VFD SWMR features.
+ * It calls an internal reporting function H5F_post_vfd_swmr_log_entry() that receives
+ * a log entry_type_code, which generates a log tag, and the message log_info, which
+ * the developers want to save into a log file.
+ *
+ * The macro H5F_POST_VFD_SWMR_LOG_ENTRY_PRODUCTION(f, c, number_entry_production, m) is
+ * called by H5F_POST_VFD_SWMR_LOG_ENTRY when the HDF5 library is built with the
+ * production mode. Number_entry_production will control the number of entry tags that
+ * applications can receive. Currently this number is set to 1 and is subject to change
+ * when more tags are useful to be present to applications.
+ *
+ * The first argument of the macro is the HDF5 file pointer(H5F_t *).
+ * Its value needs to be checked to avoid a failure caused by "Low-Level File I/O "
+ * in the testhdf5 program, which involves the test of a non-existing HDF5 file.
+ */
+
+H5_DLL void H5F_post_vfd_swmr_log_entry(H5F_t *f, int entry_type_code, char *log_info);
+
+#define H5F_POST_VFD_SWMR_LOG_ENTRY_INTERNAL(fp, entry_type_code, log_info) \
+ do { \
+ if (fp != NULL) { \
+ if (fp->shared != NULL) { \
+ if (fp->shared->vfd_swmr_log_on == TRUE) { \
+ H5F_post_vfd_swmr_log_entry(fp, entry_type_code, log_info); \
+ } \
+ } \
+ } \
+ } while (0)
+
+#define H5F_POST_VFD_SWMR_LOG_ENTRY_PRODUCTION(fp, entry_type_code, max_code, log_info) \
+ do { \
+ if (entry_type_code < max_code) { \
+ H5F_POST_VFD_SWMR_LOG_ENTRY_INTERNAL(fp, entry_type_code, log_info); \
+ } \
+ } while (0)
+
+/* Note: change H5F_POST_VFD_SWMR_LOG_ENTRY_PRODUCTION(f, c, 1, m) on the following lines to
+ * H5F_POST_VFD_SWMR_LOG_ENTRY_PRODUCTION(f, c, your_number_entry_production, m)
+ * as necessary.
+ */
+#ifndef NDEBUG
+#define H5F_POST_VFD_SWMR_LOG_ENTRY(f, c, m) H5F_POST_VFD_SWMR_LOG_ENTRY_INTERNAL(f, c, m)
+#else
+#define H5F_POST_VFD_SWMR_LOG_ENTRY(f, c, m) H5F_POST_VFD_SWMR_LOG_ENTRY_PRODUCTION(f, c, 1, m)
+#endif
+
+/* Macros for VFD SWMR log entry code
+ * Note: this should be consistent with const char *H5Fvfd_swmr_log_tags[] declared at
+ * H5Fvfd_swmr.c .
+ */
+#define EOT_PROCESSING_TIME 0
+#define FILE_OPEN 1
+#define FILE_CLOSE 2
+#define EOT_TRIGGER_TIME 3
+#define EOT_META_FILE_INDEX 4
#endif /* H5Fpkg_H */
diff --git a/src/H5Fvfd_swmr.c b/src/H5Fvfd_swmr.c
index 69e03a0..8da238c 100644
--- a/src/H5Fvfd_swmr.c
+++ b/src/H5Fvfd_swmr.c
@@ -54,6 +54,44 @@
#define nanosecs_per_second 1000000000 /* nanoseconds per second */
#define nanosecs_per_tenth_sec 100000000 /* nanoseconds per 0.1 second */
+/* Declare an array of string to identify the VFD SMWR Log tags.
+ * Note this array is used to generate the entry tag by the log reporting macro
+ * H5F_POST_VFD_SWMR_LOG_ENTRY.
+ *
+ * The following is the first version. Developers can add/modify the tags as necessary.
+ *
+ * If the entry code is 0, H5Fvfd_swmr_log_tags[0] is used to report the entry tag.
+ * H5F_POST_VFD_SWMR_LOG_ENTRY(f, EOT_PROCESSING_TIME, log_msg) will put the log_msg attached to
+ * the entry tag "EOT_PROCESSING_TIME".
+ * The entry code number is listed in the comment for convenience.
+ * Currently for the production mode, only the "EOT_PROCESSING_TIME" is present.
+ */
+
+/* clang-format off */
+static const char *H5Fvfd_swmr_log_tags[] = {
+ "EOT_PROCESSING_TIME", /* 0 */
+ "FILE_OPEN", /* 1 */
+ "FILE_CLOSE", /* 2 */
+ "EOT_TRIGGER_TIME", /* 3 */
+ "EOT_META_FILE_INDEX" /* 4 */
+ };
+/* clang-format on */
+
+/* This string defines the format of the VFD SWMR log file.
+ * The current maximum length of entry tag string is set to 26.
+ * One can enlarge or reduce this number as necessary.
+ * For example, to enlarge the maximum length of entry tag string to 30,
+ * Just change 26 to 30 in the following line, like
+ * const char *log_fmt_str="%-30s: %.3lf s: %s\n";
+ */
+const char *log_fmt_str = "%-26s: %.3lf s: %s\n";
+
+/* The length of the EOT processing time log message, subject to change */
+const unsigned int eot_pt_log_mesg_length = 48;
+
+/* The length of error message in the log */
+const unsigned int log_err_mesg_length = 14;
+
/********************/
/* Local Prototypes */
/********************/
@@ -157,7 +195,6 @@ H5F_vfd_swmr_init(H5F_t *f, hbool_t file_create)
/* Create the metadata file */
if (((shared->vfd_swmr_md_fd = HDopen(shared->vfd_swmr_config.md_file_path, O_CREAT | O_RDWR,
H5_POSIX_CREATE_MODE_RW))) < 0)
-
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "unable to create the metadata file")
md_size = (hsize_t)shared->vfd_swmr_config.md_pages_reserved * shared->fs_page_size;
@@ -319,7 +356,21 @@ H5F_vfd_swmr_close_or_flush(H5F_t *f, hbool_t closing)
if (H5F__vfd_swmr_update_end_of_tick_and_tick_num(shared, TRUE) < 0)
HDONE_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to update end of tick");
}
+#if 1 /*Kent Save the end of close info. to the log file, subject to comment out. */
+ if (closing)
+ H5F_POST_VFD_SWMR_LOG_ENTRY(f, FILE_CLOSE, "VFD SWMR File close ends");
+#endif
done:
+
+ /* Kent: Stop the timer and close the VFD SWMR log file if it is turned on.
+ * Please REVIEW to ensure this is the right place to
+ * close the log file.
+ */
+ if (shared->vfd_swmr_log_on && closing) {
+ H5_timer_stop(&(shared->vfd_swmr_log_start_time));
+ HDfclose(shared->vfd_swmr_log_file_ptr);
+ }
+ /* Kent */
FUNC_LEAVE_NOAPI(ret_value)
}
@@ -755,12 +806,28 @@ H5F_vfd_swmr_writer_end_of_tick(H5F_t *f, hbool_t wait_for_reader)
herr_t ret_value = SUCCEED; /* Return value */
hbool_t incr_tick = FALSE;
+ /* Kent: define the local variables to calculate the EOT time
+ and write them to the log file. */
+ H5_timevals_t current_time;
+ double start_elapsed_time, end_elapsed_time;
+ unsigned int temp_time;
+ char * log_msg;
+
FUNC_ENTER_NOAPI(FAIL)
HDassert(shared);
HDassert(shared->page_buf);
HDassert(shared->vfd_swmr_writer);
+ /* Kent */
+ /* Obtain the starting time for the logging info: the processing time of this function. */
+ if (shared->vfd_swmr_log_on == true) {
+ if (H5_timer_get_times(shared->vfd_swmr_log_start_time, &current_time) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get time from H5_timer_get_times")
+ start_elapsed_time = current_time.elapsed;
+ }
+ /* Kent */
+
if (!vfd_swmr_writer_may_increase_tick_to(shared->tick_num + 1, wait_for_reader))
goto update_eot;
@@ -883,6 +950,19 @@ update_eot:
HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to insert entry into the EOT queue")
done:
+ /* Kent: Calcuate the processing time and write the time info to the log file */
+ if (shared->vfd_swmr_log_on == true) {
+ if (H5_timer_get_times(shared->vfd_swmr_log_start_time, &current_time) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get time from H5_timer_get_times")
+ end_elapsed_time = current_time.elapsed;
+ if (NULL != (log_msg = HDmalloc(eot_pt_log_mesg_length * sizeof(char)))) {
+ temp_time = (unsigned int)((end_elapsed_time - start_elapsed_time) * 1000);
+ HDsprintf(log_msg, "Writer time is %u milliseconds", temp_time);
+ H5F_POST_VFD_SWMR_LOG_ENTRY(f, EOT_PROCESSING_TIME, log_msg);
+ HDfree(log_msg);
+ }
+ }
+ /* Kent */
FUNC_LEAVE_NOAPI(ret_value)
}
@@ -1886,3 +1966,47 @@ H5F_vfd_swmr_process_eot_queue(hbool_t entering_api)
done:
FUNC_LEAVE_NOAPI(ret_value)
}
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: H5F_post_vfd_swmr_log_entry
+ *
+ * Purpose: Write the log information to the log file.
+ *
+ * Parameters:
+ * H5F_t *f IN: HDF5 file pointer
+ * int entry_type_code IN: The entry type code to identify the
+ * log entry tag.
+ * char *log_info IN: The information to be stored in the
+ * log file.
+ * Return: None
+ *
+ *-------------------------------------------------------------------------
+ */
+
+void
+H5F_post_vfd_swmr_log_entry(H5F_t *f, int entry_type_code, char *log_info)
+{
+ double temp_time;
+ H5_timevals_t current_time;
+ char * gettime_error;
+
+ /* Obtain the current time.
+ If failed, write an error message to the log file.
+ else obtain the elapsed time in seconds since the log file
+ was created and write the time to the log file. */
+ if (H5_timer_get_times(f->shared->vfd_swmr_log_start_time, &current_time) < 0) {
+ if (NULL != (gettime_error = HDmalloc(log_err_mesg_length * sizeof(char)))) {
+ HDsprintf(gettime_error, "gettime_error");
+ HDfprintf(f->shared->vfd_swmr_log_file_ptr, "%-26s: %s\n", H5Fvfd_swmr_log_tags[entry_type_code],
+ gettime_error);
+ HDfree(gettime_error);
+ }
+ }
+ else {
+ temp_time = current_time.elapsed;
+ HDfprintf(f->shared->vfd_swmr_log_file_ptr, log_fmt_str, H5Fvfd_swmr_log_tags[entry_type_code],
+ temp_time, log_info);
+ }
+ return;
+}
diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c
index d20503f..3b39f12 100644
--- a/src/H5Pfapl.c
+++ b/src/H5Pfapl.c
@@ -5710,6 +5710,10 @@ H5Pset_vfd_swmr_config(hid_t plist_id, H5F_vfd_swmr_config_t *config_ptr)
else if (name_len > H5F__MAX_VFD_SWMR_FILE_NAME_LEN)
HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "md_file_path is too long")
+ name_len = HDstrlen(config_ptr->log_file_path);
+ if (name_len > H5F__MAX_VFD_SWMR_FILE_NAME_LEN)
+ HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "log_file_path is too long")
+
/* Set the modified config */
if (H5P_set(plist, H5F_ACS_VFD_SWMR_CONFIG_NAME, config_ptr) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set metadata cache initial config")
diff --git a/test/testvfdswmr.sh.in b/test/testvfdswmr.sh.in
index 4dc1e82..535aafa 100644
--- a/test/testvfdswmr.sh.in
+++ b/test/testvfdswmr.sh.in
@@ -1047,31 +1047,31 @@ fi
#
BIGSET_n=25 # -n option: # of iterations
BIGSET_few_s=10 # -s option: # of datasets (for few_big test)
-BIGSET_many_s=100 # -s option: # of datasets (for many_small test)
+BIGSET_many_s=50 # -s option: # of datasets (for many_small test)
#
#
# Setting for exhaustive and quick runs
#
if [[ "$HDF5TestExpress" -eq 0 ]] ; then # exhaustive run
- BIGSET_n=200
- BIGSET_few_s=100
- BIGSET_many_s=1000
+ BIGSET_n=100
+ BIGSET_few_s=25
+ BIGSET_many_s=100
elif [[ "$HDF5TestExpress" -gt 1 ]]; then # quick run
BIGSET_n=10
BIGSET_few_s=3
- BIGSET_many_s=50
+ BIGSET_many_s=25
fi
#
#
-for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -t" "-d 1 -t -F" "-d 1 -t -R" "-d 1 -V" "-d 1 -M" "-d 1 -V -F" "-d 1 -M -F"; do
+for options in "-d 1" "-d 1 -F" "-d 2 -l 16" "-d 2 -F -l 16" "-d 1 -t" "-d 1 -t -F" "-d 1 -t -R" "-d 1 -V" "-d 1 -M" "-d 1 -V -F" "-d 1 -M -F"; do
if [ ${do_many_small:-no} = no ]; then
continue
fi
#
- # Test many small datasets of one and two dimensions.
+ # Test many small datasets of two or three dimensions.
#
# Perform 25 iterations on 100 extensible datasets configured with
- # 2D 16x16 chunks or 3D 8x16x16 chunks of 32-bit unsigned integer elements,
+ # 2D 16x16 chunks or 3D 1x16x16 chunks of 32-bit unsigned integer elements,
# expanding each dataset by a chunk in one dimension (up to 25x1
# 16x16 chunks) on each iteration.
#
@@ -1079,16 +1079,16 @@ for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -t" "-d 1 -t -F" "-d 1 -t
# in *two* dimensions (up to 25x25 16x16 chunks).
#
# If testing 3D datasets (-t option), extending each dataset along the
- # first dimension (up to 25 8x16x16)
+ # first dimension (up to 25 1x16x16)
#
echo launch vfd_swmr_bigset_writer many small, options $options
catch_out_err_and_rc vfd_swmr_bigset_writer \
- ../vfd_swmr_bigset_writer -n $BIGSET_n $options -s $BIGSET_many_s -e 8 -r 16 -c 16 -q -l 3 &
+ ../vfd_swmr_bigset_writer -n $BIGSET_n $options -s $BIGSET_many_s -e 1 -r 16 -c 16 -q &
pid_writer=$!
catch_out_err_and_rc vfd_swmr_bigset_reader \
- ../vfd_swmr_bigset_reader -n $BIGSET_n $options -s $BIGSET_many_s -e 8 -r 16 -c 16 -q -l 3 &
+ ../vfd_swmr_bigset_reader -n $BIGSET_n $options -s $BIGSET_many_s -e 1 -r 16 -c 16 -q &
pid_reader=$!
# Wait for the reader to finish before signalling the
@@ -1116,12 +1116,12 @@ for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -t" "-d 1 -t -F" "-d 1 -t
done
# bigset test for bigger chunks
-for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -t" "-d 1 -t -F" "-d 1 -t -R" "-d 1 -V" "-d 1 -M" "-d 1 -V -F" "-d 1 -M -F"; do
+for options in "-d 1" "-d 1 -F" "-d 2 -l 10" "-d 2 -F -l 10" "-d 1 -t -l 10" "-d 1 -t -F -l 10" "-d 1 -t -R" "-d 1 -V" "-d 1 -M" "-d 1 -V -F" "-d 1 -M -F"; do
#
- # Test a few big datasets of one and two dimensions.
+ # Test a few big datasets of two or three dimensions.
#
# Perform 25 iterations on 10 extensible datasets configured with
- # 2D 256x256 chunks or 3D 64x256x256 of 32-bit unsigned integer elements,
+ # 2D 256x256 chunks or 3D 8x256x256 of 32-bit unsigned integer elements,
# expanding each dataset by a chunk in one dimension (up to 25x1
# 256x256 chunks) on each iteration.
#
@@ -1129,7 +1129,7 @@ for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -t" "-d 1 -t -F" "-d 1 -t
# in *two* dimensions (up to 25x25 256x256 chunks).
#
# If testing 3D datasets (-t option), extending each dataset along the
- # first dimension (up to 25 64x256x256)
+ # first dimension (up to 25 8x256x256)
#
if [ ${do_few_big:-no} = no ]; then
@@ -1137,11 +1137,11 @@ for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -t" "-d 1 -t -F" "-d 1 -t
fi
echo launch vfd_swmr_bigset_writer few big, options $options ......may take some time......
catch_out_err_and_rc vfd_swmr_bigset_writer \
- ../vfd_swmr_bigset_writer -n $BIGSET_n $options -s $BIGSET_few_s -e 64 -r 256 -c 256 -q -l 3 &
+ ../vfd_swmr_bigset_writer -n $BIGSET_n $options -s $BIGSET_few_s -e 8 -r 256 -c 256 -q &
pid_writer=$!
catch_out_err_and_rc vfd_swmr_bigset_reader \
- ../vfd_swmr_bigset_reader -n $BIGSET_n $options -s $BIGSET_few_s -e 64 -r 256 -c 256 -q -l 3 &
+ ../vfd_swmr_bigset_reader -n $BIGSET_n $options -s $BIGSET_few_s -e 8 -r 256 -c 256 -q &
pid_reader=$!
# Wait for the reader to finish before signalling the
diff --git a/test/vfd_swmr_bigset_writer.c b/test/vfd_swmr_bigset_writer.c
index 956973a..77c7b26 100644
--- a/test/vfd_swmr_bigset_writer.c
+++ b/test/vfd_swmr_bigset_writer.c
@@ -149,6 +149,7 @@ typedef struct {
bool test_3d;
enum { vds_off, vds_single, vds_multi } vds;
bool use_vfd_swmr;
+ bool use_legacy_swmr;
bool use_named_pipe;
bool do_perf;
bool cross_chunk_read;
@@ -214,6 +215,7 @@ state_initializer(void)
.test_3d = false,
.vds = vds_off,
.use_vfd_swmr = true,
+ .use_legacy_swmr = false,
.use_named_pipe = true,
.do_perf = false,
.cross_chunk_read = false,
@@ -256,6 +258,7 @@ usage(const char *progname)
"-P: do the performance measurement\n"
"-R: flush raw data\n"
"-S: do not use VFD SWMR\n"
+ "-T: use legacy SWMR (-S and -N must also be specified)\n"
"-V: use virtual datasets and a single\n"
" source file\n"
"-a steps: `steps` between adding attributes\n"
@@ -339,7 +342,7 @@ state_init(state_t *s, int argc, char **argv)
if (tfile)
HDfree(tfile);
- while ((ch = getopt(argc, argv, "CFMNPRSVa:bc:d:e:f:g:j:k:l:m:n:o:p:qr:s:tu:v:w:")) != -1) {
+ while ((ch = getopt(argc, argv, "CFMNPRSTVa:bc:d:e:f:g:j:k:l:m:n:o:p:qr:s:tu:v:w:")) != -1) {
switch (ch) {
case 'C':
/* This flag indicates cross-over chunk read during data validation */
@@ -362,6 +365,9 @@ state_init(state_t *s, int argc, char **argv)
case 'S':
s->use_vfd_swmr = false;
break;
+ case 'T':
+ s->use_legacy_swmr = true;
+ break;
case 'V':
s->vds = vds_single;
break;
@@ -745,6 +751,18 @@ state_init(state_t *s, int argc, char **argv)
TEST_ERROR;
}
+ if (s->use_legacy_swmr) {
+ if (s->use_vfd_swmr) {
+ HDfprintf(stderr, "Can't use both VFD SWMR and Legacy SWMR\n");
+ TEST_ERROR;
+ }
+
+ if (s->use_named_pipe) {
+ HDfprintf(stderr, "Can't use named pipe for the Legacy SWMR\n");
+ TEST_ERROR;
+ }
+ }
+
return true;
error:
@@ -826,14 +844,14 @@ state_destroy(state_t *s)
if (s->vds != vds_multi) {
if (H5Fvfd_swmr_end_tick(s->file[0]) < 0) {
- HDfprintf(stderr, "H5Fclose failed\n");
+ HDfprintf(stderr, "H5Fvfd_swmr_end_tick failed\n");
TEST_ERROR;
}
}
else {
for (j = 0; j < NELMTS(s->file); j++)
if (H5Fvfd_swmr_end_tick(s->file[j]) < 0) {
- HDfprintf(stderr, "H5Fclose failed\n");
+ HDfprintf(stderr, "H5Fvfd_swmr_end_tick failed\n");
TEST_ERROR;
}
}
@@ -2081,7 +2099,7 @@ verify_dsets(state_t s, np_state_t *np, mat_t *mat)
* the validation of the chunks */
if (s.use_named_pipe && below_speed_limit(&(last.time), &(s.ival))) {
AT();
- HDfprintf(stderr, "verify_extensible_dset took too long to finish\n");
+ HDfprintf(stderr, "Warning: verify_extensible_dset took too long to finish\n");
}
/* For checking the time lapse between the writer's finishing writing a batch of chunks
@@ -2544,6 +2562,12 @@ main(int argc, char **argv)
TEST_ERROR;
}
+ /* Enable the Legacy SWMR writing mode if specified */
+ if (s.use_legacy_swmr && H5Fstart_swmr_write(s.file[0]) < 0) {
+ HDfprintf(stderr, "failed to start the Legacy SWMR writing mode\n");
+ TEST_ERROR;
+ }
+
/* Start to write chunks. The writer writes as many chunks as possible within a tick, then
* notify the reader. But it doesn't receive back the reader's notice. */
if (!write_dsets(s, &np, mat)) {
diff --git a/test/vfd_swmr_common.c b/test/vfd_swmr_common.c
index f57f39a..cfe93aa 100644
--- a/test/vfd_swmr_common.c
+++ b/test/vfd_swmr_common.c
@@ -369,6 +369,19 @@ init_vfd_swmr_config(H5F_vfd_swmr_config_t *config, uint32_t tick_len, uint32_t
} /* init_vfd_swmr_config() */
+/* Initialize the log file path in config, this function should be called after init_vfd_swmr_config. */
+void
+init_vfd_swmr_log(H5F_vfd_swmr_config_t *config, const char *log_file_fmtstr, ...)
+
+{
+ va_list ap;
+
+ HDva_start(ap, log_file_fmtstr);
+ evsnprintf(config->log_file_path, sizeof(config->log_file_path), log_file_fmtstr, ap);
+ HDva_end(ap);
+
+} /* init_vfd_swmr_log() */
+
/* Perform common VFD SWMR configuration on the file-access property list:
* configure page buffering, set reasonable VFD SWMR defaults.
*/
diff --git a/test/vfd_swmr_common.h b/test/vfd_swmr_common.h
index b2fbbbd..f5981a5 100644
--- a/test/vfd_swmr_common.h
+++ b/test/vfd_swmr_common.h
@@ -77,6 +77,9 @@ H5TEST_DLL void init_vfd_swmr_config(H5F_vfd_swmr_config_t *config, uint32_t tic
hbool_t writer, hbool_t flush_raw_data, uint32_t md_pages_reserved,
const char *md_file_fmtstr, ...) H5_ATTR_FORMAT(printf, 7, 8);
+H5TEST_DLL void init_vfd_swmr_log(H5F_vfd_swmr_config_t *config, const char *log_file_fmtstr, ...)
+ H5_ATTR_FORMAT(printf, 2, 3);
+
H5TEST_DLL hid_t vfd_swmr_create_fcpl(H5F_fspace_strategy_t fs_strategy, hsize_t fs_page_size);
H5TEST_DLL void dbgf(int, const char *, ...) H5_ATTR_FORMAT(printf, 2, 3);
diff --git a/test/vfd_swmr_gperf_writer.c b/test/vfd_swmr_gperf_writer.c
index e21f931..3342b40 100644
--- a/test/vfd_swmr_gperf_writer.c
+++ b/test/vfd_swmr_gperf_writer.c
@@ -12,6 +12,12 @@
/* Description of this program:
* This program checks the performance of group creations for VFD SWMR.
+ * It also has an option to turn on the VFD SWMR logging feature and generates
+ * a log message for group creations. The performance test illustration is in
+ * section I. The log feature illustration is in section II.
+ *
+ * I. Performance tests with options
+ *
* Currently the group creation time, H5Fopen and H5Fclose time are measured.
* After compiling the program,
* ./vfd_swmr_gperf_writer -n 1000 -P -N 5 -q
@@ -29,6 +35,18 @@
* The program is run with max_lag = 8, tick_len = 4;
* The page buffer size is 16384 bytes. The page size is 8192 bytes.
*
+ * II. How to generate log message
+ *
+ * ./vfd_swmr_gperf_writer -n 100000 -P -L -q
+ * will generate 100000 groups, each group has one attribute. It also writes the log message
+ * to file "log-test" under the same directory.
+ * To turn on the log feature, one just needs to provide the log file path as
+ * indicated by the line init_vfd_swmr_log(&config, "./log-test") in the main
+ * function. The init_vfd_swmr_log is defined inside the vfd_swmr_common.c.
+ * The "log-test" should include something like:
+ * 'EOT_PROCESSING_TIME : 0.040 s: Writer time is 1 milliseconds'
+ * The standard output of performance measurement can help check the correctness in the "log-test".
+ *
*/
#define H5F_FRIEND /*suppress error about including H5Fpkg */
@@ -63,6 +81,7 @@ typedef struct {
uint32_t max_lag;
uint32_t tick_len;
bool gperf;
+ bool glog;
double min_gc_time;
double max_gc_time;
double mean_gc_time;
@@ -84,7 +103,7 @@ typedef struct {
.file = H5I_INVALID_HID, .one_by_one_sid = H5I_INVALID_HID, .filename = "", \
.filetype = H5T_NATIVE_UINT32, .asteps = 1, .nsteps = 100, .use_vfd_swmr = true, \
.old_style_grp = false, .grp_op_pattern = ' ', .grp_op_test = false, .at_pattern = ' ', \
- .attr_test = false, .tick_len = 4, .max_lag = 7, .gperf = false, .min_gc_time = 100., \
+ .attr_test = false, .tick_len = 4, .max_lag = 7, .gperf = false, .glog = false, .min_gc_time = 100., \
.max_gc_time = 0., .mean_gc_time = 0., .total_gc_time = 0., .total_time = 0., .mean_time = 0., \
.fo_total_time = 0., .fc_total_time = 0., .num_attrs = 1, .vlstr_test = false, .ps = 4096, \
.pbs = 4096, .nglevels = 0 \
@@ -93,88 +112,91 @@ typedef struct {
static void
usage(const char *progname)
{
- fprintf(stderr,
- "usage: ./%s -P -n 1000 -N 5 -q (create 1000 groups, each group has 5 attributes)\n"
- "Options: \n"
- " [-P] [-S] [-G] [-t tick_len] [-m max_lag][-B pbs] [-s ps]\n"
- " [-n ngroups] [-l ng_levels] [-O grp_op_pattern]\n"
- " [-N num_attrs] [-V] [-b] [-A at_pattern] [-a steps] [-q]\n"
- "\n"
- "-P: carry out the performance test\n"
- "-S: do not use VFD SWMR\n"
- "-G: old-style type of group\n"
- "-t tick_len: length of a tick in tenths of a second.\n"
- "-m max_lag: maximum expected lag(in ticks) between writer and readers\n"
- "-B pbs: page Buffer Size in bytes:\n"
- " The default value is 4K(4096).\n"
- "-s ps: page size used by page aggregation, page buffer and \n"
- " the metadata file. \n"
- "-n ngroups: the number of groups\n"
- "-l ng_levels: the number of level of nested groups. \n"
- " If all the groups are under the root group, \n"
- " this number should be 0.\n"
- "-N num_attrs: the number of attributes \n"
- "-V use variable length string attributes for performance test\n"
- "-b: write data in big-endian byte order\n"
- " (For the performance test, -V overwrites -b)\n"
- "-A at_pattern: `at_pattern' for different attribute tests\n"
- " The value of `at_pattern` is one of the following:\n"
- " `compact` - Attributes added in compact storage\n"
- " `dense` - An attribute added in dense storage\n"
- " `compact-del` - Attributes added and then one\n"
- " attribute deleted, in compact \n"
- " `dense-del` - Attributes added until the storage\n"
- " is dense then an attribute deleted\n"
- " the storge still in dense\n"
- " `compact-add-to-dense` - Attributes added first in compact\n"
- " then in dense storage\n"
- " `dense-del-to-compact` - Attributes added until the storage\n"
- " is dense, then several attributes \n"
- " deleted, the storage changed to\n"
- " compact\n"
- " `modify` - An attribute added then modified\n"
- " `add-vstr` - A VL string attribute added\n"
- " `remove-vstr` - A VL string attribute added then\n"
- " deleted\n"
- " `modify-vstr` - A VL string attribute added then \n"
- " modified \n"
- " `add-ohr-block` - An attribute is added and this forces\n"
- " the creation of object header\n"
- " continuation block \n"
- " `del-ohr-block` - An attribute is added and this forces\n"
- " the creation of object header\n"
- " continuation block and then this \n"
- " attribute is deleted so the \n"
- " object header continuation block is \n"
- " removed. \n"
- "-O grp_op_pattern: `grp_op_pattern' for different group operation tests\n"
- " The value of `grp_op_pattern` is one of the following:\n"
- " `grp-creation` - A group is created.\n"
- " `grp-deletion` - An existing group is deleted.\n"
- " `grp-move` - A group is moved to become \n"
- " another group. \n"
- " `grp-ins-links` - Links are inserted, including\n"
- " both hard and soft links. \n"
- " `grp-del-links` - Links are deleted, including\n"
- " both hard ans soft links. \n"
- " `grp-compact-t-dense` - Links are inserted to the group.\n"
- " The link storage of this group \n"
- " changed from compact to dense. \n"
- " The links include both hard and\n"
- " soft links. \n"
- " `grp-dense-t-compact` - Links are inserted to the group\n"
- " The link storage of this group \n"
- " changed from compact to dense. \n"
- " Then several links are deleted.\n"
- " The link storage changed from \n"
- " dense to compact again. \n"
- " The links include both hard and\n"
- " soft links. \n"
- "-a steps: `steps` between adding attributes\n"
- " (Don't recommend to use this option for performance test.)\n"
- "-q: silence printouts, few messages\n"
- "\n",
- progname);
+ fprintf(
+ stderr,
+ "usage: ./%s -P -n 1000 -N 5 -q (create 1000 groups, each group has 5 attributes)\n"
+ "usage: ./%s -P -L -n 100000 -q (create 100000 groups and generate log message to file 'log-test')\n"
+ "Options: \n"
+ " [-P] [-S] [-G] [-L] [-t tick_len] [-m max_lag][-B pbs] [-s ps]\n"
+ " [-n ngroups] [-l ng_levels] [-O grp_op_pattern]\n"
+ " [-N num_attrs] [-V] [-b] [-A at_pattern] [-a steps] [-q]\n"
+ "\n"
+ "-P: carry out the performance test\n"
+ "-S: do not use VFD SWMR\n"
+ "-G: old-style type of group\n"
+ "-L: Turn on the logging feature.\n"
+ "-t tick_len: length of a tick in tenths of a second.\n"
+ "-m max_lag: maximum expected lag(in ticks) between writer and readers\n"
+ "-B pbs: page Buffer Size in bytes:\n"
+ " The default value is 4K(4096).\n"
+ "-s ps: page size used by page aggregation, page buffer and \n"
+ " the metadata file. \n"
+ "-n ngroups: the number of groups\n"
+ "-l ng_levels: the number of level of nested groups. \n"
+ " If all the groups are under the root group, \n"
+ " this number should be 0.\n"
+ "-N num_attrs: the number of attributes \n"
+ "-V use variable length string attributes for performance test\n"
+ "-b: write data in big-endian byte order\n"
+ " (For the performance test, -V overwrites -b)\n"
+ "-A at_pattern: `at_pattern' for different attribute tests\n"
+ " The value of `at_pattern` is one of the following:\n"
+ " `compact` - Attributes added in compact storage\n"
+ " `dense` - An attribute added in dense storage\n"
+ " `compact-del` - Attributes added and then one\n"
+ " attribute deleted, in compact \n"
+ " `dense-del` - Attributes added until the storage\n"
+ " is dense then an attribute deleted\n"
+ " the storge still in dense\n"
+ " `compact-add-to-dense` - Attributes added first in compact\n"
+ " then in dense storage\n"
+ " `dense-del-to-compact` - Attributes added until the storage\n"
+ " is dense, then several attributes \n"
+ " deleted, the storage changed to\n"
+ " compact\n"
+ " `modify` - An attribute added then modified\n"
+ " `add-vstr` - A VL string attribute added\n"
+ " `remove-vstr` - A VL string attribute added then\n"
+ " deleted\n"
+ " `modify-vstr` - A VL string attribute added then \n"
+ " modified \n"
+ " `add-ohr-block` - An attribute is added and this forces\n"
+ " the creation of object header\n"
+ " continuation block \n"
+ " `del-ohr-block` - An attribute is added and this forces\n"
+ " the creation of object header\n"
+ " continuation block and then this \n"
+ " attribute is deleted so the \n"
+ " object header continuation block is \n"
+ " removed. \n"
+ "-O grp_op_pattern: `grp_op_pattern' for different group operation tests\n"
+ " The value of `grp_op_pattern` is one of the following:\n"
+ " `grp-creation` - A group is created.\n"
+ " `grp-deletion` - An existing group is deleted.\n"
+ " `grp-move` - A group is moved to become \n"
+ " another group. \n"
+ " `grp-ins-links` - Links are inserted, including\n"
+ " both hard and soft links. \n"
+ " `grp-del-links` - Links are deleted, including\n"
+ " both hard ans soft links. \n"
+ " `grp-compact-t-dense` - Links are inserted to the group.\n"
+ " The link storage of this group \n"
+ " changed from compact to dense. \n"
+ " The links include both hard and\n"
+ " soft links. \n"
+ " `grp-dense-t-compact` - Links are inserted to the group\n"
+ " The link storage of this group \n"
+ " changed from compact to dense. \n"
+ " Then several links are deleted.\n"
+ " The link storage changed from \n"
+ " dense to compact again. \n"
+ " The links include both hard and\n"
+ " soft links. \n"
+ "-a steps: `steps` between adding attributes\n"
+ " (Don't recommend to use this option for performance test.)\n"
+ "-q: silence printouts, few messages\n"
+ "\n",
+ progname, progname);
exit(EXIT_FAILURE);
}
@@ -203,7 +225,7 @@ state_init(state_t *s, int argc, char **argv)
if (argc == 1)
usage(s->progname);
- while ((ch = getopt(argc, argv, "PSGa:bVt:m:B:s:n:qA:N:l:O:")) != -1) {
+ while ((ch = getopt(argc, argv, "PSGLa:bVt:m:B:s:n:qA:N:l:O:")) != -1) {
switch (ch) {
case 'P':
s->gperf = true;
@@ -214,6 +236,9 @@ state_init(state_t *s, int argc, char **argv)
case 'G':
s->old_style_grp = true;
break;
+ case 'L':
+ s->glog = true;
+ break;
case 'a':
case 'n':
case 'N':
@@ -2764,6 +2789,10 @@ main(int argc, char **argv)
/* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */
init_vfd_swmr_config(&config, s.tick_len, s.max_lag, writer, FALSE, 128, "./group-shadow");
+ /* If the log flag is on, create the log file log-test under the current directory. */
+ if (s.glog == true)
+ init_vfd_swmr_log(&config, "./log-test");
+
/* If old-style option is chosen, use the earliest file format(H5F_LIBVER_EARLIEST)
* as the second parameter of H5Pset_libver_bound() that is called by
* vfd_swmr_create_fapl. Otherwise, the latest file format(H5F_LIBVER_LATEST)