/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * * Copyright by the Board of Trustees of the University of Illinois. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * * terms governing use, modification, and redistribution, is contained in * * the COPYING file, which can be found at the root of the source code * * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * * If you do not have access to either file, you may request a copy from * * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* * Programmer: Robb Matzke * Monday, August 2, 1999 * * Purpose: The public header file for the subfiling driver. */ #ifndef H5FDsubfiling_H #define H5FDsubfiling_H #define H5FD_SUBFILING (H5FD_subfiling_init()) #define H5FD_SUBFILING_VALUE H5_VFD_SUBFILING #if 1 /* JRM */ /* For now, H5FDsubfiling_priv.h needs mercury. Since the code that needs it will \ * move to its own header, just hack it for now. \ */ #include "mercury_thread.h" #include "mercury_thread_mutex.h" #include "mercury_thread_pool.h" #endif /* JRM */ #include "H5FDsubfiling_priv.h" #ifndef H5FD_SUBFILING_FAPL_T_MAGIC #define H5FD_CURR_SUBFILING_FAPL_T_VERSION 1 #define H5FD_SUBFILING_FAPL_T_MAGIC 0xFED01331 #endif /**************************************************************************** * * Structure: H5FD_subfiling_fapl_t * * Purpose: * * H5FD_subfiling_fapl_t is a public structure that is used to pass * subfiling configuration data to the appropriate subfiling VFD via * the FAPL. A pointer to an instance of this structure is a parameter * to H5Pset_fapl_subfiling() and H5Pget_fapl_subfiling(). * * `magic` (uint32_t) * * Magic is a somewhat unique number which identifies this VFD from * other VFDs. Used in combination with a version number, we can * validate a user generated file access property list (fapl). * This field should be set to H5FD_SUBFILING_FAPL_T_MAGIC. * * `version` (uint32_t) * * Version number of the H5FD_subfiling_fapl_t structure. Any instance * passed to the above calls must have a recognized version number, or * an error will be flagged. * * This field should be set to H5FD_CURR_SUBFILING_FAPL_T_VERSION. * *** IO Concentrator Info *** *** These fields will be replicated in the stacked IOC VFD which *** provides the extended support for aggregating reads and writes *** and allows global file access to node-local storage containers. * * `stripe_count` (int32_t) * * The integer value which identifies the total number of * subfiles that have been algorithmically been selected to * to contain the segments of raw data which make up an HDF5 * file. This value is used to implement the RAID-0 functionality * when reading or writing datasets. * * `stripe_depth` (int64_t) * * The stripe depth defines a limit on the maximum number of contiguous * bytes that can be read or written in a single operation on any * selected subfile. Larger IO operations can exceed this limit * by utilizing MPI derived types to construct an IO request which * gathers additional data segments from memory for the IO request. * * `ioc_selection` (enum io_selection datatype) * * The io_selection_t defines a specific algorithm by which IO * concentrators (IOCs) and sub-files are identified. The available * algorithms are: SELECT_IOC_ONE_PER_NODE, SELECT_IOC_EVERY_NTH_RANK, * SELECT_IOC_WITH_CONFIG, and SELECT_IOC_TOTAL. * *** STACKING and other VFD support *** i.e. FAPL caching *** * * `ioc_fapl_id` (hid_t) * * A valid file access property list (fapl) is cached on each * process and thus enables selection of an alternative provider * for subsequent file operations. * By default, Sub-filing employs an additional support VFD that * provides file IO proxy capabilities to all MPI ranks in a * distributed parallel application. This IO indirection * thus allows application access all sub-files even while * these may actually be node-local and thus not directly * accessible to remote ranks. * *** Subfiling file Info * * `subfile_dir` char[] * * A file directory name where subfiling files should be * placed. Under normal circumstances, this directory name * should match the directory path of the user defined HDF5 * file. * * `subfile_path` char[] * * The full pathname of the user HDF5 file. * WARNING -- this code is commented out #define H5FD_SUBFILING_PATH_MAX 4096 typedef struct config_common_t { uint32_t magic; uint32_t version; int32_t stripe_count; int64_t stripe_depth; ioc_selection_t ioc_selection; hid_t ioc_fapl_id; char subfile_dir[H5FD_SUBFILING_PATH_MAX +1]; char subfile_path[H5FD_SUBFILING_PATH_MAX +1]; char h5_filename[H5FD_SUBFILING_PATH_MAX +1]; } config_common_t; ****************************************************************************/ /* * In addition to the common configuration fields, we can have * VFD specific fields. Here's one for the subfiling VFD. * * `require_ioc` (hbool_t) * * Require_IOC is a boolean flag with a default value of TRUE. * This flag indicates that the stacked H5FDioc VFD should be * employed for sub-filing operations. The default flag can be * overridden with an environment variable: H5_REQUIRE_IOC=0 * */ //! /** * Configure struct for H5Pget_fapl_subfiling() / H5Pset_fapl_subfiling() */ typedef struct H5FD_subfiling_config_t { config_common_t common; hbool_t require_ioc; } H5FD_subfiling_config_t; //! #ifdef __cplusplus extern "C" { #endif extern FILE *sf_logfile; extern FILE *client_log; H5_DLL hid_t H5FD_subfiling_init(void); /** * \ingroup FAPL * * \brief Queries subfiling file driver properties * * \fapl_id * \param[out] config_out The subfiling fapl data. * * \returns \herr_t * * \details H5Pget_fapl_subfiling() queries the #H5FD_SUBFILING driver properties as set * by H5Pset_fapl_subfiling(). * * \since 1.14.0 * */ H5_DLL herr_t H5Pget_fapl_subfiling(hid_t fapl_id, H5FD_subfiling_config_t *config_out); /** * \ingroup FAPL * * \brief Modifies the file access property list to use the #H5FD_SUBFILING driver * * \fapl_id * \param[in] vfd_config #H5FD_SUBFILING driver specific properties. If NULL, then * the IO concentrator VFD will be used. * \returns \herr_t * * \details H5Pset_fapl_core() modifies the file access property list to use the * #H5FD_SUBFILING driver. * * \todo Expand details! * * \since 1.14.0 * */ H5_DLL herr_t H5Pset_fapl_subfiling(hid_t fapl_id, H5FD_subfiling_config_t *vfd_config); H5_DLL herr_t H5FD__get_file_ino(const char *name, uint64_t *st_ino); H5_DLL char * H5FD__get_file_directory(void *h5file); H5_DLL herr_t H5FD__dataset_write_contiguous(hid_t h5_file_id, haddr_t dataset_baseAddr, size_t dtype_extent, int mpi_rank, int mpi_size, void *_dset, hid_t mem_type_id, hid_t mem_space_id, hid_t file_space_id, hid_t plist_id, const void *buf); H5_DLL herr_t H5FD__dataset_read_contiguous(hid_t h5_file_id, haddr_t dataset_baseAddr, size_t dtype_extent, int mpi_rank, int mpi_size, void *_dset, hid_t mem_type_id, hid_t mem_space_id, hid_t file_space_id, hid_t plist_id, void *buf); H5_DLL char *get_ioc_selection_criteria(ioc_selection_t *); H5_DLL void *get__subfiling_object(int64_t object_id); H5_DLL hid_t fid_map_to_context(uint64_t h5_fid); /* return arguments are vector of vectors - function return is the length * (depth) of the sub vectors. Note that we don't need to include the * MPI_Datatype return argument! */ H5_DLL int subfiling_open_file(sf_work_request_t *msg, int subfile_rank, int flags); H5_DLL int init__indep_io(void *_sf_context, size_t depth, int ioc_total, int64_t *sf_source_data_offset, int64_t *sf_datasize, int64_t *f_offset, int *first_index, int *n_containers, int64_t offset, int64_t elements, int dtype_extent); H5_DLL int H5FD__open_subfiles(void *_config_info, uint64_t inode_id, int flags); H5_DLL int H5FD__close_subfiles(hid_t context_id); H5_DLL int H5FD__read_independent(hid_t H5FD__fid, int64_t offset, int64_t elements, int dtype_extent, void *data); H5_DLL int H5FD__write_independent(hid_t H5FD__fid, int64_t offset, int64_t elements, int dtype_extent, const void *data); H5_DLL herr_t H5FD__read_vector(hid_t h5_fid, hssize_t count, haddr_t *addrs, hsize_t sizes[], void *bufs[] /* in */); H5_DLL herr_t H5FD__write_vector(hid_t h5_fid, hssize_t count, haddr_t *addrs, hsize_t sizes[], void *bufs[] /* in */); H5_DLL int H5FD__truncate(hid_t h5_fid, haddr_t addr); H5_DLL int H5FD__shutdown_local_ioc(hid_t fid); H5_DLL void manage_client_logfile(int client_rank, int flag_value); #if 0 /* JRM */ H5_DLL int initialize_ioc_threads(void *sf_context); #endif /* JRM */ H5_DLL herr_t H5FD__write_vector_internal(hid_t h5_fid, hssize_t count, haddr_t addrs[], size_t sizes[], const void *bufs[] /* data_in */); H5_DLL herr_t H5FD__read_vector_internal(hid_t h5_fid, hssize_t count, haddr_t addrs[], size_t sizes[], void *bufs[] /* data_out */); #if 0 /* JRM */ H5_DLL int queue_write_indep(sf_work_request_t *msg, int subfile_rank, int source, MPI_Comm comm); #else /* JRM */ H5_DLL int queue_write_indep(sf_work_request_t *msg, int subfile_rank, int source, MPI_Comm comm, int counter); #endif /* JRM */ H5_DLL int queue_read_indep(sf_work_request_t *msg, int subfile_rank, int source, MPI_Comm comm); H5_DLL int sf_read_data(int fd, int64_t file_offset, void *data_buffer, int64_t data_size, int subfile_rank); H5_DLL int sf_write_data(int fd, int64_t file_offset, void *data_buffer, int64_t data_size, int subfile_rank); #ifdef __cplusplus } #endif #endif