[svn-r10545] Purpose:

Activating collective IO supports for irregular selction inside HDF5 dataset.
This support doesn't include to build the final complicated MPI derived datatype support for chunking storage.

Description:
Support collective chunk IO for both contiguous and chunking storage for irregular selection( using H5S_SELECT_OR for multiple hyperslab selection)
Solution:
Using MPI derived datatype to realize this feature.
Problems still need to be investigated:
 Big size irregular hyperslab selection test may cause MPI hang or abnormalexit with MPICH family on various platforms. This is really hard to debug since sometimes it can work and sometimes it cannot work. We will continue investigating those cases. This may not be parallel HDF5 bugs since with the recent version of poe at IBM platforms, all tests passed.


Platforms tested:
1. Linux heping 2.4.21-27.0.1 with mpich
2. AIX 5.1 copper with mpcc_r
3. Altix cobalt SGI linux 2.4.21-sgi304rp05031014_10149 with icc -lmpi
4. Linux Cluster SDSC TG, intel 8-r2 with mpich 1.2.5
5. NCSA Linux Cluster Tungsten, MPICH-TCP-1.2.5.2, Intel 8.0 under lustre
6. NCSA Linux Cluster Tungsten, MPICH-LAM-INTEL711, sometimes not working
7. NCSA Linux CLuster Tungsten, champion-pro-1.2.0-3, not working for other collective IO tests, but work for irregular selection collective IO test.

Misc. update:

1 files changed, 72 insertions, 73 deletions

diff --git a/src/H5Dmpio.c b/src/H5Dmpio.c
index 976f282..ec2da95 100644
--- a/src/H5Dmpio.c
+++ b/src/H5Dmpio.c
@@ -36,13 +36,14 @@
 
 #ifdef H5_HAVE_PARALLEL
 
+/* For regular hyperslab selection. */
 static herr_t
 H5D_mpio_spaces_xfer(H5D_io_info_t *io_info, size_t elmt_size,
                      const H5S_t *file_space, const H5S_t *mem_space,
                      void *buf/*out*/, 
 		     hbool_t do_write);
 
-
+/* For irregular hyperslab selection. */
 static herr_t
 H5D_mpio_spaces_span_xfer(H5D_io_info_t *io_info, size_t elmt_size,
                      const H5S_t *file_space, const H5S_t *mem_space,
@@ -88,7 +89,8 @@ H5D_mpio_opt_possible( const H5D_t *dset, const H5S_t *mem_space, const H5S_t *f
         HGOTO_DONE(FALSE);
 
     /* Check whether both selections are "regular" */
-#ifndef KYANG
+    /*#ifndef KYANG*/
+#ifndef H5_MPI_COMPLEX_DERIVED_DATATYPE_WORKS
     c1=H5S_SELECT_IS_REGULAR(file_space);
     c2=H5S_SELECT_IS_REGULAR(mem_space);
     if(c1==FAIL || c2==FAIL)
@@ -369,24 +371,25 @@ done:
 } /* end H5D_mpio_spaces_xfer() */
 
 
-/** The following function has been tested, don't call this
-    function until you don't see this line. Nov. 11,2004, KY**/
-
 static herr_t
-H5D_mpio_spaces_span_xfer(H5D_io_info_t *io_info, size_t elmt_size,
-    const H5S_t *file_space, const H5S_t *mem_space,
-    void *_buf /*out*/, hbool_t do_write )
+H5D_mpio_spaces_span_xfer(H5D_io_info_t *io_info, 
+			  size_t elmt_size,
+			  const H5S_t *file_space, 
+			  const H5S_t *mem_space,
+			  void *_buf /*out*/, 
+			  hbool_t do_write )
 {
-    haddr_t	 addr;                  /* Address of dataset (or selection) within file */
-    size_t	 mpi_buf_count, mpi_file_count;       /* Number of "objects" to transfer */
-    hsize_t	 mpi_buf_offset, mpi_file_offset;       /* Offset within dataset where selection (ie. MPI type) begins */
-    MPI_Datatype mpi_buf_type, mpi_file_type;   /* MPI types for buffer (memory) and file */
-    hbool_t	 mbt_is_derived=0,      /* Whether the buffer (memory) type is derived and needs to be free'd */
-		 mft_is_derived=0;      /* Whether the file type is derived and needs to be free'd */
-    hbool_t	 plist_is_setup=0;      /* Whether the dxpl has been customized */
-    uint8_t	*buf=(uint8_t *)_buf;   /* Alias for pointer arithmetic */
-    int          mpi_code;              /* MPI return code */
-    herr_t	 ret_value = SUCCEED;   /* Return value */
+    haddr_t	 addr;                              /* Address of dataset (or selection) within file */
+    size_t	 mpi_buf_count, mpi_file_count;     /* Number of "objects" to transfer */
+    hsize_t	 mpi_buf_offset, mpi_file_offset;   /* Offset within dataset where selection (ie. MPI type) begins */
+    MPI_Datatype mpi_buf_type, mpi_file_type;      /* MPI types for buffer (memory) and file */
+    hbool_t	 mbt_is_derived=0,                 /* Whether the buffer (memory) type is derived and needs to be free'd */
+		 mft_is_derived=0;                /* Whether the file type is derived and needs to be free'd */
+    hbool_t	 plist_is_setup=0;                /* Whether the dxpl has been customized */
+    uint8_t	*buf=(uint8_t *)_buf;             /* Alias for pointer arithmetic */
+    int          mpi_code;                        /* MPI return code */
+    herr_t	 ret_value = SUCCEED;             /* Return value */
+
 
     FUNC_ENTER_NOAPI_NOINIT(H5D_mpio_spaces_span_xfer);
 
@@ -397,10 +400,11 @@ H5D_mpio_spaces_span_xfer(H5D_io_info_t *io_info, size_t elmt_size,
     assert (mem_space);
     assert (buf);
     assert (IS_H5FD_MPIO(io_info->dset->ent.file));
+
     /* Make certain we have the correct type of property list */
     assert(TRUE==H5P_isa_class(io_info->dxpl_id,H5P_DATASET_XFER));
 
-    printf("coming to span tree xfer \n");
+
     /* create the MPI buffer type */
     if(H5S_SELECT_IS_REGULAR(mem_space)==TRUE){
       if (H5S_mpio_space_type( mem_space, elmt_size,
@@ -409,67 +413,69 @@ H5D_mpio_spaces_span_xfer(H5D_io_info_t *io_info, size_t elmt_size,
 			       &mpi_buf_count,
 			       &mpi_buf_offset,
 			       &mbt_is_derived )<0)
-    	HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL,"couldn't create MPI buf type");}
-      else {
-    if (H5S_mpio_space_span_type( mem_space, elmt_size,
+    	HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL,"couldn't create MPI buf type");
+    }
+    else {
+      if (H5S_mpio_space_span_type( mem_space, elmt_size,
 			       /* out: */
 			       &mpi_buf_type,
 			       &mpi_buf_count,
 			       &mpi_buf_offset,
 			       &mbt_is_derived )<0)
     	HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL,"couldn't create MPI buf type");
-      }
-    printf("mpi_buf_count %d\n",mpi_buf_count);
+    }
+
     /* create the MPI file type */
 
-       if(H5S_SELECT_IS_REGULAR(file_space)== TRUE){ 
- if ( H5S_mpio_space_type( file_space, elmt_size,
+   if(H5S_SELECT_IS_REGULAR(file_space)== TRUE){ 
+     if ( H5S_mpio_space_type( file_space, elmt_size,
 			       /* out: */
 			       &mpi_file_type,
 			       &mpi_file_count,
 			       &mpi_file_offset,
 			       &mft_is_derived )<0)
     	HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL,"couldn't create MPI file type");
-       }
-       else {
-	 if ( H5S_mpio_space_span_type( file_space, elmt_size,
+   }
+   else {
+     if ( H5S_mpio_space_span_type( file_space, elmt_size,
 			       /* out: */
 			       &mpi_file_type,
 			       &mpi_file_count,
 			       &mpi_file_offset,
 			       &mft_is_derived )<0)
     	HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL,"couldn't create MPI file type");
-       }
-    /* Get the base address of the contiguous dataset or the chunk */
-    if(io_info->dset->shared->layout.type == H5D_CONTIGUOUS)
-       addr = H5D_contig_get_addr(io_info->dset) + mpi_file_offset;
-    else {
-        haddr_t   chunk_addr; /* for collective chunk IO */
+   }
 
-        assert(io_info->dset->shared->layout.type == H5D_CHUNKED); 
-        chunk_addr=H5D_istore_get_addr(io_info,NULL);
-        addr = H5F_BASE_ADDR(io_info->dset->ent.file) + chunk_addr + mpi_file_offset;
-    }
-
-    /*
-     * Pass buf type, file type to the file driver. Request an MPI type
-     * transfer (instead of an elementary byteblock transfer).
-     */
-    if(H5FD_mpi_setup_collective(io_info->dxpl_id, mpi_buf_type, mpi_file_type)<0)
+   /* Get the base address of the contiguous dataset or the chunk */
+   if(io_info->dset->shared->layout.type == H5D_CONTIGUOUS)
+       addr = H5D_contig_get_addr(io_info->dset) + mpi_file_offset;
+   else {
+       haddr_t   chunk_addr; /* for collective chunk IO */
+       assert(io_info->dset->shared->layout.type == H5D_CHUNKED); 
+       chunk_addr=H5D_istore_get_addr(io_info,NULL);
+       addr = H5F_BASE_ADDR(io_info->dset->ent.file) + chunk_addr + mpi_file_offset;
+   }
+
+   /*
+    * Pass buf type, file type to the file driver. Request an MPI type
+    * transfer (instead of an elementary byteblock transfer).
+    */
+   if(H5FD_mpi_setup_collective(io_info->dxpl_id, mpi_buf_type, mpi_file_type)<0)
         HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set MPI-I/O properties");
-    plist_is_setup=1;
+   plist_is_setup=1;
 
-    /* Adjust the buffer pointer to the beginning of the selection */
-    buf+=mpi_buf_offset;
+   /* Adjust the buffer pointer to the beginning of the selection */
+   buf+=mpi_buf_offset;
 
-    /* transfer the data */
-    if (do_write) {
-    	if (H5F_block_write(io_info->dset->ent.file, H5FD_MEM_DRAW, addr, mpi_buf_count, io_info->dxpl_id, buf) <0)
-	    HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,"MPI write failed");
-    } else {
+   /* transfer the data */
+   if (do_write) {
+       if (H5F_block_write(io_info->dset->ent.file, H5FD_MEM_DRAW, addr, mpi_buf_count, io_info->dxpl_id, buf) <0)
+	   HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,"MPI write failed");
+    } 
+   else {
     	if (H5F_block_read (io_info->dset->ent.file, H5FD_MEM_DRAW, addr, mpi_buf_count, io_info->dxpl_id, buf) <0)
 	    HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL,"MPI read failed");
-    }
+   }
 
 done:
     /* Reset the dxpl settings */
@@ -572,30 +578,24 @@ H5D_mpio_spaces_write(H5D_io_info_t *io_info,
  * Function:	H5D_mpio_spaces_span_read
  *
  * Purpose:	MPI-IO function to read directly from app buffer to file for
-                span-tree
+                span-tree 
  *
  * Return:	non-negative on success, negative on failure.
  *
  * Programmer:	KY
- * Note      :  Don't call this routine
- *              until you don't see this line. 11/11/2004, KY
  *
  * Modifications:
  *
- * rky 980918
- * Added must_convert parameter to let caller know we can't optimize the xfer.
- *
- *      QAK - 2002/04/02
- *      Removed the must_convert parameter and move preconditions to
- *      H5S_mpio_opt_possible() routine
  *
  *-------------------------------------------------------------------------
  */
 herr_t
 H5D_mpio_spaces_span_read(H5D_io_info_t *io_info,
-    size_t UNUSED nelmts, size_t elmt_size,
-    const H5S_t *file_space, const H5S_t *mem_space,
-    void *buf/*out*/)
+			  size_t UNUSED nelmts, 
+			  size_t elmt_size,
+			  const H5S_t *file_space, 
+			  const H5S_t *mem_space,
+			  void *buf/*out*/)
 {
     herr_t ret_value;
 
@@ -616,8 +616,6 @@ H5D_mpio_spaces_span_read(H5D_io_info_t *io_info,
  * Return:	non-negative on success, negative on failure.
  *
  * Programmer:	KY
- * Note:        Don't call this funtion until you don't see this line.
- *              KY,  11/11/04
                  
  *
  * Modifications:
@@ -633,18 +631,19 @@ H5D_mpio_spaces_span_read(H5D_io_info_t *io_info,
  */
 herr_t
 H5D_mpio_spaces_span_write(H5D_io_info_t *io_info,
-    size_t UNUSED nelmts, size_t elmt_size,
-    const H5S_t *file_space, const H5S_t *mem_space,
-    const void *buf)
+			   size_t UNUSED nelmts, 
+			   size_t elmt_size,
+			   const H5S_t *file_space, 
+			   const H5S_t *mem_space,
+			   const void *buf)
 {
     herr_t ret_value;
 
     FUNC_ENTER_NOAPI_NOFUNC(H5D_mpio_spaces_span_write);
 
-    printf(" coming to spaces_span_write function\n");
     fflush(stdout);
     /*OKAY: CAST DISCARDS CONST QUALIFIER*/
-    printf("element size %d\n",elmt_size);
+
     ret_value = H5D_mpio_spaces_span_xfer(io_info, elmt_size, file_space,
         mem_space, (void*)buf, 1/*write*/);