Merged my subfiling code fixes into the new selection_io_branch

1 files changed, 40 insertions, 105 deletions

diff --git a/src/H5FDioc.h b/src/H5FDioc.h
index 7e9779e..aeee9e1 100644
--- a/src/H5FDioc.h
+++ b/src/H5FDioc.h
@@ -20,118 +20,33 @@
 #ifndef H5FDioc_H
 #define H5FDioc_H
 
-#include "H5FDsubfiling_priv.h"
-
 #define H5FD_IOC (H5FD_ioc_init())
 
 #ifndef H5FD_IOC_FAPL_T_MAGIC
-#define H5FD_CURR_IOC_FAPL_T_VERSION 1
-#define H5FD_IOC_FAPL_T_MAGIC        0xFED21331
+#define H5FD_CURR_IOC_FAPL_T_VERSION     1
+#define H5FD_IOC_FAPL_T_MAGIC 0xFED21331
 #endif
 
 /* Maximum length of a filename/path string in the Write-Only channel,
  * including the NULL-terminator.
  */
-#define H5FD_IOC_PATH_MAX         4096
+#define H5FD_IOC_PATH_MAX 4096
 #define H5FD_IOC_THREAD_POOL_SIZE 4
 
-/****************************************************************************
- *
- * Structure: H5FD_subfiling_fapl_t
- *
- * Purpose:
- *
- *     H5FD_subfiling_fapl_t is a public structure that is used to pass
- *     subfiling configuration data to the appropriate subfiling VFD via
- *     the FAPL.  A pointer to an instance of this structure is a parameter
- *     to H5Pset_fapl_subfiling() and H5Pget_fapl_subfiling().
- *
- * `magic`   (uint32_t)
- *
- *     Magic is a somewhat unique number which identifies this VFD from
- *     other VFDs.  Used in combination with a version number, we can
- *     validate a user generated file access property list (fapl).
- *     This field should be set to H5FD_SUBFILING_FAPL_T_MAGIC.
- *
- * `version` (uint32_t)
- *
- *     Version number of the H5FD_subfiling_fapl_t structure.  Any instance
- *     passed to the above calls must have a recognized version number, or
- *     an error will be flagged.
- *
- *     This field should be set to H5FD_CURR_SUBFILING_FAPL_T_VERSION.
- *
- ***   IO Concentrator Info ***
- ***   These fields will be replicated in the stacked IOC VFD which
- ***   provides the extended support for aggregating reads and writes
- ***   and allows global file access to node-local storage containers.
- *
- * `stripe_count` (int32_t)
- *
- *     The integer value which identifies the total number of
- *     subfiles that have been algorthmically been selected to
- *     to contain the segments of raw data which make up an HDF5
- *     file.  This value is used to implement the RAID-0 functionality
- *     when reading or writing datasets.
- *
- * `stripe_depth` (int64_t)
- *
- *     The stripe depth defines a limit on the maximum number of contiguous
- *     bytes that can be read or written in a single operation on any
- *     selected subfile.  Larger IO operations can exceed this limit
- *     by utilizing MPI derived types to construct an IO request which
- *     gathers additional data segments from memory for the IO request.
- *
- * `ioc_selection` (enum io_selection datatype)
- *
- *     The io_selection_t defines a specific algorithm by which IO
- *     concentrators (IOCs) and sub-files are identified.  The available
- *     algorthms are: SELECT_IOC_ONE_PER_NODE, SELECT_IOC_EVERY_NTH_RANK,
- *     SELECT_IOC_WITH_CONFIG, and SELECT_IOC_TOTAL.
- *
- ***   STACKING and other VFD support
- ***   i.e. FAPL caching
- ***
- *
- * `ioc_fapl_id` (hid_t)
- *
- *     A valid file access property list (fapl) is cached on each
- *     process and thus enables selection of an alternative provider
- *     for subsequent file operations.
- *     By defalt, Sub-filing employs an additional support VFD that
- *     provides file IO proxy capabilities to all MPI ranks in a
- *     distributed parallel application.  This IO indirection
- *     thus allows application access all sub-files even while
- *     these may actually be node-local and thus not directly
- *     accessable to remote ranks.
- *
- ***   Subfiling file Info
- *
- * `subfile_dir`  char[]
- *
- *     A file directory name where subfiling files should be
- *     placed. Under normal circumstances, this directory name
- *     should match the directory path of the user defined HDF5
- *     file.
- *
- * `subfile_path` char[]
- *
- *     The full pathname of the user HDF5 file.
- *
-#define H5FD_SUBFILING_PATH_MAX 4096
-
-typedef struct config_common_t {
-    uint32_t        magic;
-    uint32_t        version;
-    int32_t         stripe_count;
-    int64_t         stripe_depth;
-    ioc_selection_t ioc_selection;
-        hid_t           ioc_fapl_id;
-    char            subfile_dir[H5FD_SUBFILING_PATH_MAX +1];
-        char            subfile_path[H5FD_SUBFILING_PATH_MAX +1];
-} config_common_t;
+/*
+ *    Define the various constants to allow different allocations
+ *    of subfile ranks.  The choices are self explanatory, starting
+ *    with the default of one IO Concentrator (IOC) per node and
+ *    lastly, defining a fixed number.
+ */
+typedef enum {
+  SELECT_IOC_ONE_PER_NODE = 0, /* Default */
+  SELECT_IOC_EVERY_NTH_RANK,   /* Starting at rank 0, select-next += N */
+  SELECT_IOC_WITH_CONFIG,      /* NOT IMPLEMENTED: Read-from-file       */
+  SELECT_IOC_TOTAL,            /* Starting at rank 0, mpi_size / total   */
+  ioc_selection_options        /* (Uses same selection as every Nth rank) */
+} ioc_selection_t;
 
- ****************************************************************************/
 /*
  * In addition to the common configuration fields, we can have
  * VFD specific fields.  Here's one for the IO Concentrator VFD.
@@ -142,6 +57,21 @@ typedef struct config_common_t {
  *
  * ----------------------------------------------------------------------------
  */
+#define H5FD_SUBFILING_PATH_MAX 4096
+
+typedef struct config_common_t {
+  uint32_t magic;                /* set to H5FD_SUBFILING_FAPL_T_MAGIC */
+  uint32_t version;              /* set to H5FD_CURR_SUBFILING_FAPL_T_VERSION */
+  int32_t stripe_count;          /* How many io concentrators */
+  int64_t stripe_depth;          /* Max # of bytes in contigious IO to an IOC */
+  ioc_selection_t ioc_selection; /* Method to select IO Concentrators */
+  hid_t ioc_fapl_id;             /* The hid_t value of the stacked VFD  */
+  int64_t context_id;            /* The value used to lookup an IOC context */
+  char file_dir[H5FD_SUBFILING_PATH_MAX + 1]; /* Directory where we find files */
+  char file_path[H5FD_SUBFILING_PATH_MAX + 1]; /* The user define filename */
+} config_common_t;
+
+
 typedef struct H5FD_ioc_config_t {
     config_common_t common;
     int32_t         thread_pool_count;
@@ -150,15 +80,20 @@ typedef struct H5FD_ioc_config_t {
 #ifdef __cplusplus
 extern "C" {
 #endif
-H5_DLL hid_t  H5FD_ioc_init(void);
+H5_DLL hid_t H5FD_ioc_init(void);
 H5_DLL herr_t H5Pset_fapl_ioc(hid_t fapl_id, H5FD_ioc_config_t *config_ptr);
 H5_DLL herr_t H5Pget_fapl_ioc(hid_t fapl_id, H5FD_ioc_config_t *config_ptr);
-H5_DLL void   H5FD_ioc_set_shutdown_flag(int flag);
-H5_DLL void   H5FD_ioc_wait_thread_main(void);
-H5_DLL void   H5FD_ioc_finalize_threads(void);
+H5_DLL void H5FD_ioc_set_shutdown_flag(int flag);
+H5_DLL void H5FD_ioc_wait_thread_main(void);
+H5_DLL void H5FD_ioc_finalize_threads(void);
+H5_DLL int initialize_ioc_threads(void *_sf_context);
+H5_DLL int tpool_add_work(void *work);
+H5_DLL void begin_thread_exclusive(void);
+H5_DLL void end_thread_exclusive(void);
 
 #ifdef __cplusplus
 }
 #endif
 
 #endif
+