[svn-r11950] Purpose:

Enhanced collective chunk IO supports

Description:
1. When using collective IO with chunking storage without any tuning, performance may become worse under some circumstances.
2. Current HDF5 handles raw-data IO per chunk. So for many small chunks, many small IOs will be passed into MPI-IO. That may cause bad performance.
3. For one IO per chunk case, sometimes performance with collective is worse than performance with independent. An obvious case is when only one process is doing IO and all other process are not doing IO, the collective IO will only add overheads for communication. We want to avoid this case. Some management inside our library needs to be done.


Solution:
- Added managements of collective IO supports for chunking storage
         inside parallel HDF5
         1) Implemented One IO with collective mode for all chunks in the
            application by building one MPI derived datatype accross all
            chunks.
         2) Implemented the decision-making support to do collective IO inside
            MPI-IO per chunk.
         3) Added the decision-making support to do one IO accross all chunks
            or to do multiple IOs with each IO per chunk.
         4) Added the support to handle the case some processes won't do any IOs in
            collectively.
         5) Some MPI-IO package(mpich 1.2.6 or lower, e.g.) cannot handle
            collective IO correctly for the case when some processes have no
            contributions to IOs, a special macro is added to change
            collective IO mode to independent IO mode inside HDF5 library.



Platforms tested:

Parallel:

IBM AIX 5.2(copper)
Linux (heping) mpich-1.2.6
SDSC Teragrid mpich-1.2.5
Linux(Tungsten) mpich-1.2.6
Altix(NCSA cobalt)

Seq:
Linux(heping)

Misc. update:

1 files changed, 55 insertions, 30 deletions

diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h
index 97e7763..df30800 100644
--- a/src/H5Dpkg.h
+++ b/src/H5Dpkg.h
@@ -34,7 +34,8 @@
 #include "H5Gprivate.h"		/* Groups 			  	*/
 #include "H5Oprivate.h"		/* Object headers		  	*/
 #include "H5Sprivate.h"		/* Dataspaces 				*/
-#include "H5Tprivate.h"		/* Datatypes 				*/
+#include "H5Tprivate.h"		/* Datatype functions			*/
+#include "H5SLprivate.h"	/* Skip lists				*/
 
 /**************************/
 /* Package Private Macros */
@@ -67,14 +68,14 @@
 struct H5D_io_info_t;
 typedef herr_t (*H5D_io_read_func_t)(struct H5D_io_info_t *io_info,
     size_t nelmts, size_t elmt_size,
-    const H5S_t *file_space, const H5S_t *mem_space,
+    const H5S_t *file_space, const H5S_t *mem_space,haddr_t addr,
     void *buf/*out*/);
 
 
 /* Write directly from app buffer to file */
 typedef herr_t (*H5D_io_write_func_t)(struct H5D_io_info_t *io_info,
     size_t nelmts, size_t elmt_size,
-    const H5S_t *file_space, const H5S_t *mem_space,
+    const H5S_t *file_space, const H5S_t *mem_space,haddr_t addr,
     const void *buf);
 
 /* Function pointers for I/O on particular types of dataset layouts */
@@ -187,6 +188,39 @@ typedef enum {
     H5D_ALLOC_WRITE             /* Dataset is being extended */
 } H5D_time_alloc_t;
 
+
+/* Structure holding information about a chunk's selection for mapping */
+typedef struct H5D_chunk_info_t {
+    hsize_t index;              /* "Index" of chunk in dataset */
+    size_t chunk_points;        /* Number of elements selected in chunk */
+    H5S_t *fspace;              /* Dataspace describing chunk & selection in it */
+    hsize_t coords[H5O_LAYOUT_NDIMS];   /* Coordinates of chunk in file dataset's dataspace */
+    H5S_t *mspace;              /* Dataspace describing selection in memory corresponding to this chunk */
+    unsigned mspace_shared;     /* Indicate that the memory space for a chunk is shared and shouldn't be freed */
+} H5D_chunk_info_t;
+
+/* Main structure holding the mapping between file chunks and memory */
+typedef struct fm_map {
+    H5SL_t *fsel;               /* Skip list containing file dataspaces for all chunks */
+    hsize_t last_index;         /* Index of last chunk operated on */
+    H5D_chunk_info_t *last_chunk_info;  /* Pointer to last chunk's info */
+    const H5S_t *file_space;    /* Pointer to the file dataspace */
+    const H5S_t *mem_space;     /* Pointer to the memory dataspace */
+    unsigned mem_space_copy;    /* Flag to indicate that the memory dataspace must be copied */
+    hsize_t f_dims[H5O_LAYOUT_NDIMS];   /* File dataspace dimensions */
+    H5S_t *mchunk_tmpl;         /* Dataspace template for new memory chunks */
+    unsigned f_ndims;           /* Number of dimensions for file dataspace */
+    H5S_sel_iter_t mem_iter;    /* Iterator for elements in memory selection */
+    unsigned m_ndims;           /* Number of dimensions for memory dataspace */
+    hsize_t chunks[H5O_LAYOUT_NDIMS];   /* Number of chunks in each dimension */
+    hsize_t chunk_dim[H5O_LAYOUT_NDIMS];    /* Size of chunk in each dimension */
+    hsize_t down_chunks[H5O_LAYOUT_NDIMS];   /* "down" size of number of chunks in each dimension */
+    H5O_layout_t *layout;       /* Dataset layout information*/
+    H5S_sel_type msel_type;     /* Selection type in memory */
+    hsize_t total_chunks;       /* Number of total chunks */
+    hbool_t *select_chunk;         /* store the information about whether this chunk is selected or not */
+} fm_map;
+
 /*****************************/
 /* Package Private Variables */
 /*****************************/
@@ -215,10 +249,12 @@ H5_DLL size_t H5D_select_mgath (const void *_buf,
 H5_DLL herr_t H5D_select_read(H5D_io_info_t *io_info,
     size_t nelmts, size_t elmt_size,
     const H5S_t *file_space, const H5S_t *mem_space,
+			      haddr_t addr,
     void *buf/*out*/);
 H5_DLL herr_t H5D_select_write(H5D_io_info_t *io_info,
     size_t nelmts, size_t elmt_size,
     const H5S_t *file_space, const H5S_t *mem_space,
+			       haddr_t addr,
     const void *buf/*out*/);
 
 /* Functions that operate on contiguous storage */
@@ -296,42 +332,31 @@ H5_DLL ssize_t H5D_efl_writevv(const H5D_io_info_t *io_info,
 H5_DLL herr_t H5D_mpio_select_read(H5D_io_info_t *io_info,
     size_t nelmts, size_t elmt_size,
     const struct H5S_t *file_space, const struct H5S_t *mem_space,
-    void *buf/*out*/);
+    haddr_t addr,void *buf/*out*/);
 
 /* MPI-IO function to read , it will select either regular or irregular read */
 H5_DLL herr_t H5D_mpio_select_write(H5D_io_info_t *io_info,
     size_t nelmts, size_t elmt_size,
     const struct H5S_t *file_space, const struct H5S_t *mem_space,
-    const void *buf);
-
-/* MPI-IO function to read directly from app buffer to file rky980813 */
-H5_DLL herr_t H5D_mpio_spaces_read(H5D_io_info_t *io_info,
-    size_t nelmts, size_t elmt_size,
-    const struct H5S_t *file_space, const struct H5S_t *mem_space,
-    void *buf/*out*/);
-
-/* MPI-IO function to write directly from app buffer to file rky980813 */
-H5_DLL herr_t H5D_mpio_spaces_write(H5D_io_info_t *io_info,
-    size_t nelmts, size_t elmt_size,
-    const struct H5S_t *file_space, const struct H5S_t *mem_space,
-    const void *buf);
-
-/* MPI-IO function to read directly from app buffer to file rky980813 */
-H5_DLL herr_t H5D_mpio_spaces_span_read(H5D_io_info_t *io_info,
-    size_t nelmts, size_t elmt_size,
-    const struct H5S_t *file_space, const struct H5S_t *mem_space,
-    void *buf/*out*/);
-
-/* MPI-IO function to write directly from app buffer to file rky980813 */
-H5_DLL herr_t H5D_mpio_spaces_span_write(H5D_io_info_t *io_info,
-    size_t nelmts, size_t elmt_size,
-    const struct H5S_t *file_space, const struct H5S_t *mem_space,
-    const void *buf);
-
+    haddr_t addr,const void *buf);
+
+/* MPI-IO function to handle contiguous collective IO */
+H5_DLL herr_t
+H5D_contig_collective_io(H5D_io_info_t *io_info,
+			 const H5S_t *file_space,const H5S_t *mem_space,
+			 void *_buf,hbool_t do_write);
+
+/* MPI-IO function to handle chunked collective IO */
+H5_DLL herr_t
+H5D_chunk_collective_io(H5D_io_info_t * io_info,fm_map *fm, void*buf,
+			hbool_t do_write);
 /* MPI-IO function to check if a direct I/O transfer is possible between
  * memory and the file */
 H5_DLL htri_t H5D_mpio_opt_possible(const H5D_io_info_t *io_info, const H5S_t *mem_space,
     const H5S_t *file_space, const H5T_path_t *tpath);
+
+H5_DLL herr_t  H5D_mpio_chunk_adjust_iomode(H5D_io_info_t *io_info,const fm_map *fm);
+
 #endif /* H5_HAVE_PARALLEL */
 
 /* Testing functions */