diff options
Diffstat (limited to 'HDF5Examples/C/H5PAR/ph5_subfiling.c')
| -rw-r--r-- | HDF5Examples/C/H5PAR/ph5_subfiling.c | 551 |
1 files changed, 551 insertions, 0 deletions
diff --git a/HDF5Examples/C/H5PAR/ph5_subfiling.c b/HDF5Examples/C/H5PAR/ph5_subfiling.c new file mode 100644 index 0000000..7d72448 --- /dev/null +++ b/HDF5Examples/C/H5PAR/ph5_subfiling.c @@ -0,0 +1,551 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Example of using HDF5's Subfiling VFD to write to an + * HDF5 file that is striped across multiple subfiles + * + * If the HDF5_NOCLEANUP environment variable is set, the + * files that this example creates will not be removed as + * the example finishes. + * + * In general, the current working directory in which compiling + * is done, is not suitable for parallel I/O and there is no + * standard pathname for parallel file systems. In some cases, + * the parallel file name may even need some parallel file type + * prefix such as: "pfs:/GF/...". Therefore, this example parses + * the HDF5_PARAPREFIX environment variable for a prefix, if one + * is needed. + */ + +#include <stdlib.h> + +#include "hdf5.h" + +#if defined(H5_HAVE_PARALLEL) && defined(H5_HAVE_SUBFILING_VFD) + +#define EXAMPLE_FILE "h5_subfiling_default_example.h5" +#define EXAMPLE_FILE2 "h5_subfiling_custom_example.h5" +#define EXAMPLE_FILE3 "h5_subfiling_precreate_example.h5" + +#define EXAMPLE_DSET_NAME "DSET" +#define EXAMPLE_DSET_DIMS 2 + +/* Have each MPI rank write 16MiB of data */ +#define EXAMPLE_DSET_NY 4194304 + +/* Dataset datatype */ +#define EXAMPLE_DSET_DATATYPE H5T_NATIVE_INT +typedef int EXAMPLE_DSET_C_DATATYPE; + +/* Cleanup created files */ +static void +cleanup(char *filename, hid_t fapl_id) +{ + hbool_t do_cleanup = getenv(HDF5_NOCLEANUP) ? 0 : 1; + + if (do_cleanup) + H5Fdelete(filename, fapl_id); +} + +/* + * An example of using the HDF5 Subfiling VFD with + * its default settings of 1 subfile per node, with + * a stripe size of 32MiB + */ +static void +subfiling_write_default(hid_t fapl_id, int mpi_size, int mpi_rank) +{ + EXAMPLE_DSET_C_DATATYPE *data; + hsize_t dset_dims[EXAMPLE_DSET_DIMS]; + hsize_t start[EXAMPLE_DSET_DIMS]; + hsize_t count[EXAMPLE_DSET_DIMS]; + hid_t file_id; + hid_t subfiling_fapl; + hid_t dset_id; + hid_t filespace; + char filename[512]; + char *par_prefix; + + /* + * Make a copy of the FAPL so we don't disturb + * it for the other examples + */ + subfiling_fapl = H5Pcopy(fapl_id); + + /* + * Set Subfiling VFD on FAPL using default settings + * (use IOC VFD, 1 IOC per node, 32MiB stripe size) + * + * Note that all of Subfiling's configuration settings + * can be adjusted with environment variables as well + * in this case. + */ + H5Pset_fapl_subfiling(subfiling_fapl, NULL); + + /* + * OPTIONAL: Set alignment of objects in HDF5 file to + * be equal to the Subfiling stripe size. + * Choosing a Subfiling stripe size and HDF5 + * object alignment value that are some + * multiple of the disk block size can + * generally help performance by ensuring + * that I/O is well-aligned and doesn't + * excessively cross stripe boundaries. + * + * Note that this option can substantially + * increase the size of the resulting HDF5 + * files, so it is a good idea to keep an eye + * on this. + */ + H5Pset_alignment(subfiling_fapl, 0, 33554432); /* Align to default 32MiB stripe size */ + + /* Parse any parallel prefix and create filename */ + par_prefix = getenv("HDF5_PARAPREFIX"); + + snprintf(filename, sizeof(filename), "%s%s%s", par_prefix ? par_prefix : "", par_prefix ? "/" : "", + EXAMPLE_FILE); + + /* + * Create a new file collectively + */ + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, subfiling_fapl); + + /* + * Create the dataspace for the dataset. The first + * dimension varies with the number of MPI ranks + * while the second dimension is fixed. + */ + dset_dims[0] = mpi_size; + dset_dims[1] = EXAMPLE_DSET_NY; + filespace = H5Screate_simple(EXAMPLE_DSET_DIMS, dset_dims, NULL); + + /* + * Create the dataset with default properties + */ + dset_id = H5Dcreate2(file_id, EXAMPLE_DSET_NAME, EXAMPLE_DSET_DATATYPE, filespace, H5P_DEFAULT, + H5P_DEFAULT, H5P_DEFAULT); + + /* + * Each MPI rank writes from a contiguous memory + * region to the hyperslab in the file + */ + start[0] = mpi_rank; + start[1] = 0; + count[0] = 1; + count[1] = dset_dims[1]; + H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, NULL, count, NULL); + + /* + * Initialize data buffer + */ + data = malloc(count[0] * count[1] * sizeof(EXAMPLE_DSET_C_DATATYPE)); + for (size_t i = 0; i < count[0] * count[1]; i++) { + data[i] = mpi_rank + i; + } + + /* + * Write to dataset + */ + H5Dwrite(dset_id, EXAMPLE_DSET_DATATYPE, H5S_BLOCK, filespace, H5P_DEFAULT, data); + + /* + * Close/release resources. + */ + + free(data); + + H5Dclose(dset_id); + H5Sclose(filespace); + H5Fclose(file_id); + + cleanup(EXAMPLE_FILE, subfiling_fapl); + + H5Pclose(subfiling_fapl); +} + +/* + * An example of using the HDF5 Subfiling VFD with + * custom settings + */ +static void +subfiling_write_custom(hid_t fapl_id, int mpi_size, int mpi_rank) +{ + EXAMPLE_DSET_C_DATATYPE *data; + H5FD_subfiling_config_t subf_config; + H5FD_ioc_config_t ioc_config; + hsize_t dset_dims[EXAMPLE_DSET_DIMS]; + hsize_t start[EXAMPLE_DSET_DIMS]; + hsize_t count[EXAMPLE_DSET_DIMS]; + hid_t file_id; + hid_t subfiling_fapl; + hid_t dset_id; + hid_t filespace; + char filename[512]; + char *par_prefix; + + /* + * Make a copy of the FAPL so we don't disturb + * it for the other examples + */ + subfiling_fapl = H5Pcopy(fapl_id); + + /* + * Get a default Subfiling and IOC configuration + */ + H5Pget_fapl_subfiling(subfiling_fapl, &subf_config); + H5Pget_fapl_ioc(subfiling_fapl, &ioc_config); + + /* + * Set Subfiling configuration to use a 1MiB + * stripe size and the SELECT_IOC_EVERY_NTH_RANK + * selection method. By default, without a setting + * in the H5FD_SUBFILING_IOC_SELECTION_CRITERIA + * environment variable, this will use every MPI + * rank as an I/O concentrator. + */ + subf_config.shared_cfg.stripe_size = 1048576; + subf_config.shared_cfg.ioc_selection = SELECT_IOC_EVERY_NTH_RANK; + + /* + * Set IOC configuration to use 2 worker threads + * per IOC instead of the default setting and + * update IOC configuration with new subfiling + * configuration. + */ + ioc_config.thread_pool_size = 2; + + /* + * Set our new configuration on the IOC + * FAPL used for Subfiling + */ + H5Pset_fapl_ioc(subf_config.ioc_fapl_id, &ioc_config); + + /* + * Finally, set our new Subfiling configuration + * on the original FAPL + */ + H5Pset_fapl_subfiling(subfiling_fapl, &subf_config); + + /* + * OPTIONAL: Set alignment of objects in HDF5 file to + * be equal to the Subfiling stripe size. + * Choosing a Subfiling stripe size and HDF5 + * object alignment value that are some + * multiple of the disk block size can + * generally help performance by ensuring + * that I/O is well-aligned and doesn't + * excessively cross stripe boundaries. + * + * Note that this option can substantially + * increase the size of the resulting HDF5 + * files, so it is a good idea to keep an eye + * on this. + */ + H5Pset_alignment(subfiling_fapl, 0, 1048576); /* Align to custom 1MiB stripe size */ + + /* Parse any parallel prefix and create filename */ + par_prefix = getenv("HDF5_PARAPREFIX"); + + snprintf(filename, sizeof(filename), "%s%s%s", par_prefix ? par_prefix : "", par_prefix ? "/" : "", + EXAMPLE_FILE2); + + /* + * Create a new file collectively + */ + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, subfiling_fapl); + + /* + * Create the dataspace for the dataset. The first + * dimension varies with the number of MPI ranks + * while the second dimension is fixed. + */ + dset_dims[0] = mpi_size; + dset_dims[1] = EXAMPLE_DSET_NY; + filespace = H5Screate_simple(EXAMPLE_DSET_DIMS, dset_dims, NULL); + + /* + * Create the dataset with default properties + */ + dset_id = H5Dcreate2(file_id, EXAMPLE_DSET_NAME, EXAMPLE_DSET_DATATYPE, filespace, H5P_DEFAULT, + H5P_DEFAULT, H5P_DEFAULT); + + /* + * Each MPI rank writes from a contiguous memory + * region to the hyperslab in the file + */ + start[0] = mpi_rank; + start[1] = 0; + count[0] = 1; + count[1] = dset_dims[1]; + H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, NULL, count, NULL); + + /* + * Initialize data buffer + */ + data = malloc(count[0] * count[1] * sizeof(EXAMPLE_DSET_C_DATATYPE)); + for (size_t i = 0; i < count[0] * count[1]; i++) { + data[i] = mpi_rank + i; + } + + /* + * Write to dataset + */ + H5Dwrite(dset_id, EXAMPLE_DSET_DATATYPE, H5S_BLOCK, filespace, H5P_DEFAULT, data); + + /* + * Close/release resources. + */ + + free(data); + + H5Dclose(dset_id); + H5Sclose(filespace); + H5Fclose(file_id); + + cleanup(EXAMPLE_FILE2, subfiling_fapl); + + H5Pclose(subfiling_fapl); +} + +/* + * An example of pre-creating an HDF5 file on MPI rank + * 0 when using the HDF5 Subfiling VFD. In this case, + * the subfiling stripe count must be set so that rank + * 0 knows how many subfiles to pre-create. + */ +static void +subfiling_write_precreate(hid_t fapl_id, int mpi_size, int mpi_rank) +{ + EXAMPLE_DSET_C_DATATYPE *data; + H5FD_subfiling_config_t subf_config; + hsize_t dset_dims[EXAMPLE_DSET_DIMS]; + hsize_t start[EXAMPLE_DSET_DIMS]; + hsize_t count[EXAMPLE_DSET_DIMS]; + hid_t file_id; + hid_t subfiling_fapl; + hid_t dset_id; + hid_t filespace; + char filename[512]; + char *par_prefix; + + /* + * Make a copy of the FAPL so we don't disturb + * it for the other examples + */ + subfiling_fapl = H5Pcopy(fapl_id); + + /* + * Get a default Subfiling and IOC configuration + */ + H5Pget_fapl_subfiling(subfiling_fapl, &subf_config); + + /* + * Set the Subfiling stripe count so that rank + * 0 knows how many subfiles the logical HDF5 + * file should consist of. In this case, use + * 5 subfiles with a default stripe size of + * 32MiB. + */ + subf_config.shared_cfg.stripe_count = 5; + + /* + * OPTIONAL: Set alignment of objects in HDF5 file to + * be equal to the Subfiling stripe size. + * Choosing a Subfiling stripe size and HDF5 + * object alignment value that are some + * multiple of the disk block size can + * generally help performance by ensuring + * that I/O is well-aligned and doesn't + * excessively cross stripe boundaries. + * + * Note that this option can substantially + * increase the size of the resulting HDF5 + * files, so it is a good idea to keep an eye + * on this. + */ + H5Pset_alignment(subfiling_fapl, 0, 1048576); /* Align to custom 1MiB stripe size */ + + /* Parse any parallel prefix and create filename */ + par_prefix = getenv("HDF5_PARAPREFIX"); + + snprintf(filename, sizeof(filename), "%s%s%s", par_prefix ? par_prefix : "", par_prefix ? "/" : "", + EXAMPLE_FILE3); + + /* Set dataset dimensionality */ + dset_dims[0] = mpi_size; + dset_dims[1] = EXAMPLE_DSET_NY; + + if (mpi_rank == 0) { + /* + * Make sure only this rank opens the file + */ + H5Pset_mpi_params(subfiling_fapl, MPI_COMM_SELF, MPI_INFO_NULL); + + /* + * Set the Subfiling VFD on our FAPL using + * our custom configuration + */ + H5Pset_fapl_subfiling(subfiling_fapl, &subf_config); + + /* + * Create a new file on rank 0 + */ + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, subfiling_fapl); + + /* + * Create the dataspace for the dataset. The first + * dimension varies with the number of MPI ranks + * while the second dimension is fixed. + */ + filespace = H5Screate_simple(EXAMPLE_DSET_DIMS, dset_dims, NULL); + + /* + * Create the dataset with default properties + */ + dset_id = H5Dcreate2(file_id, EXAMPLE_DSET_NAME, EXAMPLE_DSET_DATATYPE, filespace, H5P_DEFAULT, + H5P_DEFAULT, H5P_DEFAULT); + + /* + * Initialize data buffer + */ + data = malloc(dset_dims[0] * dset_dims[1] * sizeof(EXAMPLE_DSET_C_DATATYPE)); + for (size_t i = 0; i < dset_dims[0] * dset_dims[1]; i++) { + data[i] = i; + } + + /* + * Rank 0 writes to the whole dataset + */ + H5Dwrite(dset_id, EXAMPLE_DSET_DATATYPE, H5S_BLOCK, filespace, H5P_DEFAULT, data); + + /* + * Close/release resources. + */ + + free(data); + + H5Dclose(dset_id); + H5Sclose(filespace); + H5Fclose(file_id); + } + + MPI_Barrier(MPI_COMM_WORLD); + + /* + * Use all MPI ranks to re-open the file and + * read back the dataset that was created + */ + H5Pset_mpi_params(subfiling_fapl, MPI_COMM_WORLD, MPI_INFO_NULL); + + /* + * Use the same subfiling configuration as rank 0 + * used to create the file + */ + H5Pset_fapl_subfiling(subfiling_fapl, &subf_config); + + /* + * Re-open the file on all ranks + */ + file_id = H5Fopen(filename, H5F_ACC_RDONLY, subfiling_fapl); + + /* + * Open the dataset that was created + */ + dset_id = H5Dopen2(file_id, EXAMPLE_DSET_NAME, H5P_DEFAULT); + + /* + * Initialize data buffer + */ + data = malloc(dset_dims[0] * dset_dims[1] * sizeof(EXAMPLE_DSET_C_DATATYPE)); + + /* + * Read the dataset on all ranks + */ + H5Dread(dset_id, EXAMPLE_DSET_DATATYPE, H5S_BLOCK, H5S_ALL, H5P_DEFAULT, data); + + /* + * Close/release resources. + */ + + free(data); + + H5Dclose(dset_id); + H5Fclose(file_id); + + cleanup(EXAMPLE_FILE3, subfiling_fapl); + + H5Pclose(subfiling_fapl); +} + +int +main(int argc, char **argv) +{ + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Info info = MPI_INFO_NULL; + hid_t fapl_id; + int mpi_size; + int mpi_rank; + int mpi_thread_required = MPI_THREAD_MULTIPLE; + int mpi_thread_provided = 0; + + /* HDF5 Subfiling VFD requires MPI_Init_thread with MPI_THREAD_MULTIPLE */ + MPI_Init_thread(&argc, &argv, mpi_thread_required, &mpi_thread_provided); + if (mpi_thread_provided < mpi_thread_required) { + printf("MPI_THREAD_MULTIPLE not supported\n"); + MPI_Abort(comm, -1); + } + + MPI_Comm_size(comm, &mpi_size); + MPI_Comm_rank(comm, &mpi_rank); + + /* + * Set up File Access Property List with MPI + * parameters for the Subfiling VFD to use + */ + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + H5Pset_mpi_params(fapl_id, comm, info); + + /* Use Subfiling VFD with default settings */ + subfiling_write_default(fapl_id, mpi_size, mpi_rank); + + /* Use Subfiling VFD with custom settings */ + subfiling_write_custom(fapl_id, mpi_size, mpi_rank); + + /* + * Use Subfiling VFD to precreate the HDF5 + * file on MPI rank 0 + */ + subfiling_write_precreate(fapl_id, mpi_size, mpi_rank); + + H5Pclose(fapl_id); + + if (mpi_rank == 0) + printf("PHDF5 example finished with no errors\n"); + + MPI_Finalize(); + + return 0; +} + +#else + +/* dummy program since HDF5 is not parallel-enabled */ +int +main(void) +{ + printf( + "Example program cannot run - HDF5 must be built with parallel support and Subfiling VFD support\n"); + return 0; +} + +#endif /* H5_HAVE_PARALLEL && H5_HAVE_SUBFILING_VFD */ |