diff options
Diffstat (limited to 'examples/ph5_subfiling.c')
| -rw-r--r-- | examples/ph5_subfiling.c | 251 |
1 files changed, 220 insertions, 31 deletions
diff --git a/examples/ph5_subfiling.c b/examples/ph5_subfiling.c index 9142749..7d72448 100644 --- a/examples/ph5_subfiling.c +++ b/examples/ph5_subfiling.c @@ -12,7 +12,7 @@ /* * Example of using HDF5's Subfiling VFD to write to an - * HDF5 file that is striped across multiple sub-files + * HDF5 file that is striped across multiple subfiles * * If the HDF5_NOCLEANUP environment variable is set, the * files that this example creates will not be removed as @@ -35,12 +35,13 @@ #define EXAMPLE_FILE "h5_subfiling_default_example.h5" #define EXAMPLE_FILE2 "h5_subfiling_custom_example.h5" +#define EXAMPLE_FILE3 "h5_subfiling_precreate_example.h5" #define EXAMPLE_DSET_NAME "DSET" #define EXAMPLE_DSET_DIMS 2 -/* Have each MPI rank write 64MiB of data */ -#define EXAMPLE_DSET_NY 16777216 +/* Have each MPI rank write 16MiB of data */ +#define EXAMPLE_DSET_NY 4194304 /* Dataset datatype */ #define EXAMPLE_DSET_DATATYPE H5T_NATIVE_INT @@ -56,6 +57,11 @@ cleanup(char *filename, hid_t fapl_id) H5Fdelete(filename, fapl_id); } +/* + * An example of using the HDF5 Subfiling VFD with + * its default settings of 1 subfile per node, with + * a stripe size of 32MiB + */ static void subfiling_write_default(hid_t fapl_id, int mpi_size, int mpi_rank) { @@ -64,12 +70,19 @@ subfiling_write_default(hid_t fapl_id, int mpi_size, int mpi_rank) hsize_t start[EXAMPLE_DSET_DIMS]; hsize_t count[EXAMPLE_DSET_DIMS]; hid_t file_id; + hid_t subfiling_fapl; hid_t dset_id; hid_t filespace; char filename[512]; char *par_prefix; /* + * Make a copy of the FAPL so we don't disturb + * it for the other examples + */ + subfiling_fapl = H5Pcopy(fapl_id); + + /* * Set Subfiling VFD on FAPL using default settings * (use IOC VFD, 1 IOC per node, 32MiB stripe size) * @@ -77,7 +90,7 @@ subfiling_write_default(hid_t fapl_id, int mpi_size, int mpi_rank) * can be adjusted with environment variables as well * in this case. */ - H5Pset_fapl_subfiling(fapl_id, NULL); + H5Pset_fapl_subfiling(subfiling_fapl, NULL); /* * OPTIONAL: Set alignment of objects in HDF5 file to @@ -94,7 +107,7 @@ subfiling_write_default(hid_t fapl_id, int mpi_size, int mpi_rank) * files, so it is a good idea to keep an eye * on this. */ - H5Pset_alignment(fapl_id, 0, 33554432); /* Align to default 32MiB stripe size */ + H5Pset_alignment(subfiling_fapl, 0, 33554432); /* Align to default 32MiB stripe size */ /* Parse any parallel prefix and create filename */ par_prefix = getenv("HDF5_PARAPREFIX"); @@ -105,7 +118,7 @@ subfiling_write_default(hid_t fapl_id, int mpi_size, int mpi_rank) /* * Create a new file collectively */ - file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, subfiling_fapl); /* * Create the dataspace for the dataset. The first @@ -155,9 +168,15 @@ subfiling_write_default(hid_t fapl_id, int mpi_size, int mpi_rank) H5Sclose(filespace); H5Fclose(file_id); - cleanup(EXAMPLE_FILE, fapl_id); + cleanup(EXAMPLE_FILE, subfiling_fapl); + + H5Pclose(subfiling_fapl); } +/* + * An example of using the HDF5 Subfiling VFD with + * custom settings + */ static void subfiling_write_custom(hid_t fapl_id, int mpi_size, int mpi_rank) { @@ -168,17 +187,23 @@ subfiling_write_custom(hid_t fapl_id, int mpi_size, int mpi_rank) hsize_t start[EXAMPLE_DSET_DIMS]; hsize_t count[EXAMPLE_DSET_DIMS]; hid_t file_id; - hid_t ioc_fapl; + hid_t subfiling_fapl; hid_t dset_id; hid_t filespace; char filename[512]; char *par_prefix; /* + * Make a copy of the FAPL so we don't disturb + * it for the other examples + */ + subfiling_fapl = H5Pcopy(fapl_id); + + /* * Get a default Subfiling and IOC configuration */ - H5Pget_fapl_subfiling(fapl_id, &subf_config); - H5Pget_fapl_ioc(fapl_id, &ioc_config); + H5Pget_fapl_subfiling(subfiling_fapl, &subf_config); + H5Pget_fapl_ioc(subfiling_fapl, &ioc_config); /* * Set Subfiling configuration to use a 1MiB @@ -198,32 +223,18 @@ subfiling_write_custom(hid_t fapl_id, int mpi_size, int mpi_rank) * configuration. */ ioc_config.thread_pool_size = 2; - ioc_config.subf_config = subf_config.shared_cfg; - - /* - * Create a File Access Property List for - * the IOC VFD and set our new configuration - * on it. We make a copy of the original - * FAPL here so we get the MPI parameters - * set on it - */ - ioc_fapl = H5Pcopy(fapl_id); - H5Pset_fapl_ioc(ioc_fapl, &ioc_config); /* - * Close FAPLs in the default configurations - * we retrieved and update the subfiling - * configuration with our new IOC FAPL + * Set our new configuration on the IOC + * FAPL used for Subfiling */ - H5Pclose(ioc_config.under_fapl_id); - H5Pclose(subf_config.ioc_fapl_id); - subf_config.ioc_fapl_id = ioc_fapl; + H5Pset_fapl_ioc(subf_config.ioc_fapl_id, &ioc_config); /* * Finally, set our new Subfiling configuration * on the original FAPL */ - H5Pset_fapl_subfiling(fapl_id, &subf_config); + H5Pset_fapl_subfiling(subfiling_fapl, &subf_config); /* * OPTIONAL: Set alignment of objects in HDF5 file to @@ -240,7 +251,7 @@ subfiling_write_custom(hid_t fapl_id, int mpi_size, int mpi_rank) * files, so it is a good idea to keep an eye * on this. */ - H5Pset_alignment(fapl_id, 0, 1048576); /* Align to custom 1MiB stripe size */ + H5Pset_alignment(subfiling_fapl, 0, 1048576); /* Align to custom 1MiB stripe size */ /* Parse any parallel prefix and create filename */ par_prefix = getenv("HDF5_PARAPREFIX"); @@ -251,7 +262,7 @@ subfiling_write_custom(hid_t fapl_id, int mpi_size, int mpi_rank) /* * Create a new file collectively */ - file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, subfiling_fapl); /* * Create the dataspace for the dataset. The first @@ -301,7 +312,179 @@ subfiling_write_custom(hid_t fapl_id, int mpi_size, int mpi_rank) H5Sclose(filespace); H5Fclose(file_id); - cleanup(EXAMPLE_FILE2, fapl_id); + cleanup(EXAMPLE_FILE2, subfiling_fapl); + + H5Pclose(subfiling_fapl); +} + +/* + * An example of pre-creating an HDF5 file on MPI rank + * 0 when using the HDF5 Subfiling VFD. In this case, + * the subfiling stripe count must be set so that rank + * 0 knows how many subfiles to pre-create. + */ +static void +subfiling_write_precreate(hid_t fapl_id, int mpi_size, int mpi_rank) +{ + EXAMPLE_DSET_C_DATATYPE *data; + H5FD_subfiling_config_t subf_config; + hsize_t dset_dims[EXAMPLE_DSET_DIMS]; + hsize_t start[EXAMPLE_DSET_DIMS]; + hsize_t count[EXAMPLE_DSET_DIMS]; + hid_t file_id; + hid_t subfiling_fapl; + hid_t dset_id; + hid_t filespace; + char filename[512]; + char *par_prefix; + + /* + * Make a copy of the FAPL so we don't disturb + * it for the other examples + */ + subfiling_fapl = H5Pcopy(fapl_id); + + /* + * Get a default Subfiling and IOC configuration + */ + H5Pget_fapl_subfiling(subfiling_fapl, &subf_config); + + /* + * Set the Subfiling stripe count so that rank + * 0 knows how many subfiles the logical HDF5 + * file should consist of. In this case, use + * 5 subfiles with a default stripe size of + * 32MiB. + */ + subf_config.shared_cfg.stripe_count = 5; + + /* + * OPTIONAL: Set alignment of objects in HDF5 file to + * be equal to the Subfiling stripe size. + * Choosing a Subfiling stripe size and HDF5 + * object alignment value that are some + * multiple of the disk block size can + * generally help performance by ensuring + * that I/O is well-aligned and doesn't + * excessively cross stripe boundaries. + * + * Note that this option can substantially + * increase the size of the resulting HDF5 + * files, so it is a good idea to keep an eye + * on this. + */ + H5Pset_alignment(subfiling_fapl, 0, 1048576); /* Align to custom 1MiB stripe size */ + + /* Parse any parallel prefix and create filename */ + par_prefix = getenv("HDF5_PARAPREFIX"); + + snprintf(filename, sizeof(filename), "%s%s%s", par_prefix ? par_prefix : "", par_prefix ? "/" : "", + EXAMPLE_FILE3); + + /* Set dataset dimensionality */ + dset_dims[0] = mpi_size; + dset_dims[1] = EXAMPLE_DSET_NY; + + if (mpi_rank == 0) { + /* + * Make sure only this rank opens the file + */ + H5Pset_mpi_params(subfiling_fapl, MPI_COMM_SELF, MPI_INFO_NULL); + + /* + * Set the Subfiling VFD on our FAPL using + * our custom configuration + */ + H5Pset_fapl_subfiling(subfiling_fapl, &subf_config); + + /* + * Create a new file on rank 0 + */ + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, subfiling_fapl); + + /* + * Create the dataspace for the dataset. The first + * dimension varies with the number of MPI ranks + * while the second dimension is fixed. + */ + filespace = H5Screate_simple(EXAMPLE_DSET_DIMS, dset_dims, NULL); + + /* + * Create the dataset with default properties + */ + dset_id = H5Dcreate2(file_id, EXAMPLE_DSET_NAME, EXAMPLE_DSET_DATATYPE, filespace, H5P_DEFAULT, + H5P_DEFAULT, H5P_DEFAULT); + + /* + * Initialize data buffer + */ + data = malloc(dset_dims[0] * dset_dims[1] * sizeof(EXAMPLE_DSET_C_DATATYPE)); + for (size_t i = 0; i < dset_dims[0] * dset_dims[1]; i++) { + data[i] = i; + } + + /* + * Rank 0 writes to the whole dataset + */ + H5Dwrite(dset_id, EXAMPLE_DSET_DATATYPE, H5S_BLOCK, filespace, H5P_DEFAULT, data); + + /* + * Close/release resources. + */ + + free(data); + + H5Dclose(dset_id); + H5Sclose(filespace); + H5Fclose(file_id); + } + + MPI_Barrier(MPI_COMM_WORLD); + + /* + * Use all MPI ranks to re-open the file and + * read back the dataset that was created + */ + H5Pset_mpi_params(subfiling_fapl, MPI_COMM_WORLD, MPI_INFO_NULL); + + /* + * Use the same subfiling configuration as rank 0 + * used to create the file + */ + H5Pset_fapl_subfiling(subfiling_fapl, &subf_config); + + /* + * Re-open the file on all ranks + */ + file_id = H5Fopen(filename, H5F_ACC_RDONLY, subfiling_fapl); + + /* + * Open the dataset that was created + */ + dset_id = H5Dopen2(file_id, EXAMPLE_DSET_NAME, H5P_DEFAULT); + + /* + * Initialize data buffer + */ + data = malloc(dset_dims[0] * dset_dims[1] * sizeof(EXAMPLE_DSET_C_DATATYPE)); + + /* + * Read the dataset on all ranks + */ + H5Dread(dset_id, EXAMPLE_DSET_DATATYPE, H5S_BLOCK, H5S_ALL, H5P_DEFAULT, data); + + /* + * Close/release resources. + */ + + free(data); + + H5Dclose(dset_id); + H5Fclose(file_id); + + cleanup(EXAMPLE_FILE3, subfiling_fapl); + + H5Pclose(subfiling_fapl); } int @@ -338,6 +521,12 @@ main(int argc, char **argv) /* Use Subfiling VFD with custom settings */ subfiling_write_custom(fapl_id, mpi_size, mpi_rank); + /* + * Use Subfiling VFD to precreate the HDF5 + * file on MPI rank 0 + */ + subfiling_write_precreate(fapl_id, mpi_size, mpi_rank); + H5Pclose(fapl_id); if (mpi_rank == 0) |