summaryrefslogtreecommitdiffstats
path: root/examples/ph5_subfiling.c
diff options
context:
space:
mode:
Diffstat (limited to 'examples/ph5_subfiling.c')
-rw-r--r--examples/ph5_subfiling.c251
1 files changed, 220 insertions, 31 deletions
diff --git a/examples/ph5_subfiling.c b/examples/ph5_subfiling.c
index 9142749..7d72448 100644
--- a/examples/ph5_subfiling.c
+++ b/examples/ph5_subfiling.c
@@ -12,7 +12,7 @@
/*
* Example of using HDF5's Subfiling VFD to write to an
- * HDF5 file that is striped across multiple sub-files
+ * HDF5 file that is striped across multiple subfiles
*
* If the HDF5_NOCLEANUP environment variable is set, the
* files that this example creates will not be removed as
@@ -35,12 +35,13 @@
#define EXAMPLE_FILE "h5_subfiling_default_example.h5"
#define EXAMPLE_FILE2 "h5_subfiling_custom_example.h5"
+#define EXAMPLE_FILE3 "h5_subfiling_precreate_example.h5"
#define EXAMPLE_DSET_NAME "DSET"
#define EXAMPLE_DSET_DIMS 2
-/* Have each MPI rank write 64MiB of data */
-#define EXAMPLE_DSET_NY 16777216
+/* Have each MPI rank write 16MiB of data */
+#define EXAMPLE_DSET_NY 4194304
/* Dataset datatype */
#define EXAMPLE_DSET_DATATYPE H5T_NATIVE_INT
@@ -56,6 +57,11 @@ cleanup(char *filename, hid_t fapl_id)
H5Fdelete(filename, fapl_id);
}
+/*
+ * An example of using the HDF5 Subfiling VFD with
+ * its default settings of 1 subfile per node, with
+ * a stripe size of 32MiB
+ */
static void
subfiling_write_default(hid_t fapl_id, int mpi_size, int mpi_rank)
{
@@ -64,12 +70,19 @@ subfiling_write_default(hid_t fapl_id, int mpi_size, int mpi_rank)
hsize_t start[EXAMPLE_DSET_DIMS];
hsize_t count[EXAMPLE_DSET_DIMS];
hid_t file_id;
+ hid_t subfiling_fapl;
hid_t dset_id;
hid_t filespace;
char filename[512];
char *par_prefix;
/*
+ * Make a copy of the FAPL so we don't disturb
+ * it for the other examples
+ */
+ subfiling_fapl = H5Pcopy(fapl_id);
+
+ /*
* Set Subfiling VFD on FAPL using default settings
* (use IOC VFD, 1 IOC per node, 32MiB stripe size)
*
@@ -77,7 +90,7 @@ subfiling_write_default(hid_t fapl_id, int mpi_size, int mpi_rank)
* can be adjusted with environment variables as well
* in this case.
*/
- H5Pset_fapl_subfiling(fapl_id, NULL);
+ H5Pset_fapl_subfiling(subfiling_fapl, NULL);
/*
* OPTIONAL: Set alignment of objects in HDF5 file to
@@ -94,7 +107,7 @@ subfiling_write_default(hid_t fapl_id, int mpi_size, int mpi_rank)
* files, so it is a good idea to keep an eye
* on this.
*/
- H5Pset_alignment(fapl_id, 0, 33554432); /* Align to default 32MiB stripe size */
+ H5Pset_alignment(subfiling_fapl, 0, 33554432); /* Align to default 32MiB stripe size */
/* Parse any parallel prefix and create filename */
par_prefix = getenv("HDF5_PARAPREFIX");
@@ -105,7 +118,7 @@ subfiling_write_default(hid_t fapl_id, int mpi_size, int mpi_rank)
/*
* Create a new file collectively
*/
- file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);
+ file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, subfiling_fapl);
/*
* Create the dataspace for the dataset. The first
@@ -155,9 +168,15 @@ subfiling_write_default(hid_t fapl_id, int mpi_size, int mpi_rank)
H5Sclose(filespace);
H5Fclose(file_id);
- cleanup(EXAMPLE_FILE, fapl_id);
+ cleanup(EXAMPLE_FILE, subfiling_fapl);
+
+ H5Pclose(subfiling_fapl);
}
+/*
+ * An example of using the HDF5 Subfiling VFD with
+ * custom settings
+ */
static void
subfiling_write_custom(hid_t fapl_id, int mpi_size, int mpi_rank)
{
@@ -168,17 +187,23 @@ subfiling_write_custom(hid_t fapl_id, int mpi_size, int mpi_rank)
hsize_t start[EXAMPLE_DSET_DIMS];
hsize_t count[EXAMPLE_DSET_DIMS];
hid_t file_id;
- hid_t ioc_fapl;
+ hid_t subfiling_fapl;
hid_t dset_id;
hid_t filespace;
char filename[512];
char *par_prefix;
/*
+ * Make a copy of the FAPL so we don't disturb
+ * it for the other examples
+ */
+ subfiling_fapl = H5Pcopy(fapl_id);
+
+ /*
* Get a default Subfiling and IOC configuration
*/
- H5Pget_fapl_subfiling(fapl_id, &subf_config);
- H5Pget_fapl_ioc(fapl_id, &ioc_config);
+ H5Pget_fapl_subfiling(subfiling_fapl, &subf_config);
+ H5Pget_fapl_ioc(subfiling_fapl, &ioc_config);
/*
* Set Subfiling configuration to use a 1MiB
@@ -198,32 +223,18 @@ subfiling_write_custom(hid_t fapl_id, int mpi_size, int mpi_rank)
* configuration.
*/
ioc_config.thread_pool_size = 2;
- ioc_config.subf_config = subf_config.shared_cfg;
-
- /*
- * Create a File Access Property List for
- * the IOC VFD and set our new configuration
- * on it. We make a copy of the original
- * FAPL here so we get the MPI parameters
- * set on it
- */
- ioc_fapl = H5Pcopy(fapl_id);
- H5Pset_fapl_ioc(ioc_fapl, &ioc_config);
/*
- * Close FAPLs in the default configurations
- * we retrieved and update the subfiling
- * configuration with our new IOC FAPL
+ * Set our new configuration on the IOC
+ * FAPL used for Subfiling
*/
- H5Pclose(ioc_config.under_fapl_id);
- H5Pclose(subf_config.ioc_fapl_id);
- subf_config.ioc_fapl_id = ioc_fapl;
+ H5Pset_fapl_ioc(subf_config.ioc_fapl_id, &ioc_config);
/*
* Finally, set our new Subfiling configuration
* on the original FAPL
*/
- H5Pset_fapl_subfiling(fapl_id, &subf_config);
+ H5Pset_fapl_subfiling(subfiling_fapl, &subf_config);
/*
* OPTIONAL: Set alignment of objects in HDF5 file to
@@ -240,7 +251,7 @@ subfiling_write_custom(hid_t fapl_id, int mpi_size, int mpi_rank)
* files, so it is a good idea to keep an eye
* on this.
*/
- H5Pset_alignment(fapl_id, 0, 1048576); /* Align to custom 1MiB stripe size */
+ H5Pset_alignment(subfiling_fapl, 0, 1048576); /* Align to custom 1MiB stripe size */
/* Parse any parallel prefix and create filename */
par_prefix = getenv("HDF5_PARAPREFIX");
@@ -251,7 +262,7 @@ subfiling_write_custom(hid_t fapl_id, int mpi_size, int mpi_rank)
/*
* Create a new file collectively
*/
- file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);
+ file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, subfiling_fapl);
/*
* Create the dataspace for the dataset. The first
@@ -301,7 +312,179 @@ subfiling_write_custom(hid_t fapl_id, int mpi_size, int mpi_rank)
H5Sclose(filespace);
H5Fclose(file_id);
- cleanup(EXAMPLE_FILE2, fapl_id);
+ cleanup(EXAMPLE_FILE2, subfiling_fapl);
+
+ H5Pclose(subfiling_fapl);
+}
+
+/*
+ * An example of pre-creating an HDF5 file on MPI rank
+ * 0 when using the HDF5 Subfiling VFD. In this case,
+ * the subfiling stripe count must be set so that rank
+ * 0 knows how many subfiles to pre-create.
+ */
+static void
+subfiling_write_precreate(hid_t fapl_id, int mpi_size, int mpi_rank)
+{
+ EXAMPLE_DSET_C_DATATYPE *data;
+ H5FD_subfiling_config_t subf_config;
+ hsize_t dset_dims[EXAMPLE_DSET_DIMS];
+ hsize_t start[EXAMPLE_DSET_DIMS];
+ hsize_t count[EXAMPLE_DSET_DIMS];
+ hid_t file_id;
+ hid_t subfiling_fapl;
+ hid_t dset_id;
+ hid_t filespace;
+ char filename[512];
+ char *par_prefix;
+
+ /*
+ * Make a copy of the FAPL so we don't disturb
+ * it for the other examples
+ */
+ subfiling_fapl = H5Pcopy(fapl_id);
+
+ /*
+ * Get a default Subfiling and IOC configuration
+ */
+ H5Pget_fapl_subfiling(subfiling_fapl, &subf_config);
+
+ /*
+ * Set the Subfiling stripe count so that rank
+ * 0 knows how many subfiles the logical HDF5
+ * file should consist of. In this case, use
+ * 5 subfiles with a default stripe size of
+ * 32MiB.
+ */
+ subf_config.shared_cfg.stripe_count = 5;
+
+ /*
+ * OPTIONAL: Set alignment of objects in HDF5 file to
+ * be equal to the Subfiling stripe size.
+ * Choosing a Subfiling stripe size and HDF5
+ * object alignment value that are some
+ * multiple of the disk block size can
+ * generally help performance by ensuring
+ * that I/O is well-aligned and doesn't
+ * excessively cross stripe boundaries.
+ *
+ * Note that this option can substantially
+ * increase the size of the resulting HDF5
+ * files, so it is a good idea to keep an eye
+ * on this.
+ */
+ H5Pset_alignment(subfiling_fapl, 0, 1048576); /* Align to custom 1MiB stripe size */
+
+ /* Parse any parallel prefix and create filename */
+ par_prefix = getenv("HDF5_PARAPREFIX");
+
+ snprintf(filename, sizeof(filename), "%s%s%s", par_prefix ? par_prefix : "", par_prefix ? "/" : "",
+ EXAMPLE_FILE3);
+
+ /* Set dataset dimensionality */
+ dset_dims[0] = mpi_size;
+ dset_dims[1] = EXAMPLE_DSET_NY;
+
+ if (mpi_rank == 0) {
+ /*
+ * Make sure only this rank opens the file
+ */
+ H5Pset_mpi_params(subfiling_fapl, MPI_COMM_SELF, MPI_INFO_NULL);
+
+ /*
+ * Set the Subfiling VFD on our FAPL using
+ * our custom configuration
+ */
+ H5Pset_fapl_subfiling(subfiling_fapl, &subf_config);
+
+ /*
+ * Create a new file on rank 0
+ */
+ file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, subfiling_fapl);
+
+ /*
+ * Create the dataspace for the dataset. The first
+ * dimension varies with the number of MPI ranks
+ * while the second dimension is fixed.
+ */
+ filespace = H5Screate_simple(EXAMPLE_DSET_DIMS, dset_dims, NULL);
+
+ /*
+ * Create the dataset with default properties
+ */
+ dset_id = H5Dcreate2(file_id, EXAMPLE_DSET_NAME, EXAMPLE_DSET_DATATYPE, filespace, H5P_DEFAULT,
+ H5P_DEFAULT, H5P_DEFAULT);
+
+ /*
+ * Initialize data buffer
+ */
+ data = malloc(dset_dims[0] * dset_dims[1] * sizeof(EXAMPLE_DSET_C_DATATYPE));
+ for (size_t i = 0; i < dset_dims[0] * dset_dims[1]; i++) {
+ data[i] = i;
+ }
+
+ /*
+ * Rank 0 writes to the whole dataset
+ */
+ H5Dwrite(dset_id, EXAMPLE_DSET_DATATYPE, H5S_BLOCK, filespace, H5P_DEFAULT, data);
+
+ /*
+ * Close/release resources.
+ */
+
+ free(data);
+
+ H5Dclose(dset_id);
+ H5Sclose(filespace);
+ H5Fclose(file_id);
+ }
+
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ /*
+ * Use all MPI ranks to re-open the file and
+ * read back the dataset that was created
+ */
+ H5Pset_mpi_params(subfiling_fapl, MPI_COMM_WORLD, MPI_INFO_NULL);
+
+ /*
+ * Use the same subfiling configuration as rank 0
+ * used to create the file
+ */
+ H5Pset_fapl_subfiling(subfiling_fapl, &subf_config);
+
+ /*
+ * Re-open the file on all ranks
+ */
+ file_id = H5Fopen(filename, H5F_ACC_RDONLY, subfiling_fapl);
+
+ /*
+ * Open the dataset that was created
+ */
+ dset_id = H5Dopen2(file_id, EXAMPLE_DSET_NAME, H5P_DEFAULT);
+
+ /*
+ * Initialize data buffer
+ */
+ data = malloc(dset_dims[0] * dset_dims[1] * sizeof(EXAMPLE_DSET_C_DATATYPE));
+
+ /*
+ * Read the dataset on all ranks
+ */
+ H5Dread(dset_id, EXAMPLE_DSET_DATATYPE, H5S_BLOCK, H5S_ALL, H5P_DEFAULT, data);
+
+ /*
+ * Close/release resources.
+ */
+
+ free(data);
+
+ H5Dclose(dset_id);
+ H5Fclose(file_id);
+
+ cleanup(EXAMPLE_FILE3, subfiling_fapl);
+
+ H5Pclose(subfiling_fapl);
}
int
@@ -338,6 +521,12 @@ main(int argc, char **argv)
/* Use Subfiling VFD with custom settings */
subfiling_write_custom(fapl_id, mpi_size, mpi_rank);
+ /*
+ * Use Subfiling VFD to precreate the HDF5
+ * file on MPI rank 0
+ */
+ subfiling_write_precreate(fapl_id, mpi_size, mpi_rank);
+
H5Pclose(fapl_id);
if (mpi_rank == 0)