summaryrefslogtreecommitdiffstats
path: root/testpar
diff options
context:
space:
mode:
authorjhendersonHDF <jhenderson@hdfgroup.org>2023-03-17 20:45:07 (GMT)
committerGitHub <noreply@github.com>2023-03-17 20:45:07 (GMT)
commit1392b9fc17bfa74f0661000de212a5f54ff41956 (patch)
tree348872ba16208b701b570582a87f3222b28db825 /testpar
parent549fbcba86437860f8ce090a511039e9d8c06e6a (diff)
downloadhdf5-1392b9fc17bfa74f0661000de212a5f54ff41956.zip
hdf5-1392b9fc17bfa74f0661000de212a5f54ff41956.tar.gz
hdf5-1392b9fc17bfa74f0661000de212a5f54ff41956.tar.bz2
Subfiling VFD - fix issues with I/O concentrator selection strategies (#2571)
Fix multiple bugs with the SELECT_IOC_EVERY_NTH_RANK and SELECT_IOC_TOTAL I/O concentrator selection strategies and add a regression test for them
Diffstat (limited to 'testpar')
-rw-r--r--testpar/t_subfiling_vfd.c195
1 files changed, 195 insertions, 0 deletions
diff --git a/testpar/t_subfiling_vfd.c b/testpar/t_subfiling_vfd.c
index e1f9e05..380c068 100644
--- a/testpar/t_subfiling_vfd.c
+++ b/testpar/t_subfiling_vfd.c
@@ -95,6 +95,7 @@ static hid_t create_subfiling_ioc_fapl(MPI_Comm comm, MPI_Info info, hbool_t cus
static void test_create_and_close(void);
static void test_config_file(void);
static void test_stripe_sizes(void);
+static void test_selection_strategies(void);
static void test_read_different_stripe_size(void);
static void test_subfiling_precreate_rank_0(void);
static void test_subfiling_write_many_read_one(void);
@@ -105,6 +106,7 @@ static test_func tests[] = {
test_create_and_close,
test_config_file,
test_stripe_sizes,
+ test_selection_strategies,
test_read_different_stripe_size,
test_subfiling_precreate_rank_0,
test_subfiling_write_many_read_one,
@@ -795,6 +797,199 @@ test_stripe_sizes(void)
#undef SUBF_NITER
/*
+ * Test the different I/O Concentator selection strategies
+ * for the Subfiling VFD
+ */
+#define SUBF_FILENAME "test_subfiling_selection_strategies.h5"
+#define NUM_RANKS_CHOICES 2
+#define NUM_CRITERIA_FORMATS 2
+static void
+test_selection_strategies(void)
+{
+ H5FD_subfiling_params_t cfg;
+ hid_t file_id = H5I_INVALID_HID;
+ hid_t fapl_id = H5I_INVALID_HID;
+ char *tmp_filename = NULL;
+
+ curr_nerrors = nerrors;
+
+ if (MAINPROCESS)
+ TESTING_2("I/O concentrator selection strategies");
+
+ tmp_filename = HDmalloc(PATH_MAX);
+ VRFY(tmp_filename, "HDmalloc succeeded");
+
+ for (H5FD_subfiling_ioc_select_t strategy = 0; strategy < ioc_selection_options; strategy++) {
+ /* Skip 1 IOC per node strategy since we assume it's
+ * the default strategy tested in this file. Skip
+ * "with config" strategy since it isn't supported.
+ */
+ if (strategy == SELECT_IOC_ONE_PER_NODE || strategy == SELECT_IOC_WITH_CONFIG)
+ continue;
+
+ /* Test with 1 MPI rank and then all MPI ranks */
+ for (size_t num_ranks_choice = 0; num_ranks_choice < NUM_RANKS_CHOICES; num_ranks_choice++) {
+ int num_active_ranks = mpi_size;
+
+ if (num_ranks_choice == 0)
+ num_active_ranks = 1;
+
+ /* Test with a selection strategy criteria string
+ * in the 'integer:[integer|string]' form and in
+ * the form of just a single value.
+ */
+ for (size_t criteria_format_choice = 0; criteria_format_choice < NUM_CRITERIA_FORMATS;
+ criteria_format_choice++) {
+ MPI_Comm file_comm = comm_g;
+ char criteria_buf[256];
+ char sel_criteria[128]; /* Use char buffer for criteria as we may support
+ the "with config" strategy in the future */
+ int expected_num_subfiles;
+
+ cfg.ioc_selection = strategy;
+ cfg.stripe_size = H5FD_SUBFILING_DEFAULT_STRIPE_SIZE;
+ cfg.stripe_count = H5FD_SUBFILING_DEFAULT_STRIPE_COUNT;
+
+ switch (strategy) {
+ case SELECT_IOC_EVERY_NTH_RANK: {
+ int stride;
+
+ /* Try to select a reasonable stride value */
+ if (num_active_ranks <= 2)
+ stride = 1;
+ else if (num_active_ranks <= 8)
+ stride = 2;
+ else if (num_active_ranks <= 32)
+ stride = 4;
+ else if (num_active_ranks <= 128)
+ stride = 8;
+ else
+ stride = 16;
+
+ HDsnprintf(sel_criteria, 128, "%d", stride);
+
+ expected_num_subfiles = ((num_active_ranks - 1) / stride) + 1;
+
+ break;
+ }
+
+ case SELECT_IOC_TOTAL: {
+ int n_iocs;
+
+ /* Try to select a reasonable number of IOCs */
+ if (num_active_ranks <= 2)
+ n_iocs = 1;
+ else if (num_active_ranks <= 8)
+ n_iocs = 2;
+ else if (num_active_ranks <= 32)
+ n_iocs = 4;
+ else if (num_active_ranks <= 128)
+ n_iocs = 8;
+ else
+ n_iocs = 16;
+
+ HDsnprintf(sel_criteria, 128, "%d", n_iocs);
+
+ expected_num_subfiles = n_iocs;
+
+ break;
+ }
+
+ case SELECT_IOC_ONE_PER_NODE:
+ case SELECT_IOC_WITH_CONFIG:
+ default:
+ HDprintf("invalid IOC selection strategy\n");
+ MPI_Abort(comm_g, -1);
+ }
+
+ if (criteria_format_choice == 0) {
+ HDsnprintf(criteria_buf, 256, "%d:%s", strategy, sel_criteria);
+ }
+ else if (criteria_format_choice == 1) {
+ HDsnprintf(criteria_buf, 256, "%s", sel_criteria);
+ }
+
+ VRFY(HDsetenv(H5FD_SUBFILING_IOC_SELECTION_CRITERIA, criteria_buf, 1) >= 0,
+ "HDsetenv succeeded");
+
+ HDassert(num_active_ranks == mpi_size || num_active_ranks == 1);
+
+ if ((num_active_ranks == mpi_size) || (mpi_rank == 0)) {
+ h5_stat_t file_info;
+ FILE *subfile_ptr;
+ int num_digits;
+
+ if (num_active_ranks < mpi_size)
+ file_comm = MPI_COMM_SELF;
+
+ fapl_id = create_subfiling_ioc_fapl(file_comm, info_g, TRUE, &cfg,
+ H5FD_IOC_DEFAULT_THREAD_POOL_SIZE);
+ VRFY((fapl_id >= 0), "FAPL creation succeeded");
+
+ file_id = H5Fcreate(SUBF_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);
+ VRFY((file_id >= 0), "H5Fcreate succeeded");
+
+ /*
+ * Get the file inode value so we can construct the subfile names
+ */
+ VRFY((HDstat(SUBF_FILENAME, &file_info) >= 0), "HDstat succeeded");
+
+ num_digits = (int)(HDlog10(expected_num_subfiles) + 1);
+
+ /* Ensure all the subfiles are present */
+ for (int i = 0; i < expected_num_subfiles; i++) {
+ HDsnprintf(tmp_filename, PATH_MAX, H5FD_SUBFILING_FILENAME_TEMPLATE, SUBF_FILENAME,
+ (uint64_t)file_info.st_ino, num_digits, i + 1, expected_num_subfiles);
+
+ /* Ensure file exists */
+ subfile_ptr = HDfopen(tmp_filename, "r");
+ VRFY(subfile_ptr, "HDfopen on subfile succeeded");
+ VRFY((HDfclose(subfile_ptr) >= 0), "HDfclose on subfile succeeded");
+ }
+
+ /* Ensure no extra subfiles are present */
+ HDsnprintf(tmp_filename, PATH_MAX, H5FD_SUBFILING_FILENAME_TEMPLATE, SUBF_FILENAME,
+ (uint64_t)file_info.st_ino, num_digits, expected_num_subfiles + 1,
+ expected_num_subfiles);
+
+ /* Ensure file doesn't exist */
+ subfile_ptr = HDfopen(tmp_filename, "r");
+ VRFY(subfile_ptr == NULL, "HDfopen on subfile correctly failed");
+
+ VRFY((H5Fclose(file_id) >= 0), "File close succeeded");
+
+ mpi_code_g = MPI_Barrier(file_comm);
+ VRFY((mpi_code_g == MPI_SUCCESS), "MPI_Barrier succeeded");
+
+ H5E_BEGIN_TRY
+ {
+ H5Fdelete(SUBF_FILENAME, fapl_id);
+ }
+ H5E_END_TRY;
+
+ VRFY((H5Pclose(fapl_id) >= 0), "FAPL close succeeded");
+
+ VRFY(HDunsetenv(H5FD_SUBFILING_IOC_SELECTION_CRITERIA) >= 0, "HDunsetenv succeeded");
+ }
+
+ mpi_code_g = MPI_Barrier(comm_g);
+ VRFY((mpi_code_g == MPI_SUCCESS), "MPI_Barrier succeeded");
+ }
+ }
+ }
+
+ mpi_code_g = MPI_Barrier(comm_g);
+ VRFY((mpi_code_g == MPI_SUCCESS), "MPI_Barrier succeeded");
+
+ HDfree(tmp_filename);
+
+ CHECK_PASSED();
+}
+#undef SUBF_FILENAME
+#undef NUM_RANKS_CHOICES
+#undef NUM_CRITERIA_FORMATS
+
+/*
* Test that opening a file with a different stripe
* size/count than was used when creating the file
* results in the original stripe size/count being