[svn-r5681] Purpose:

    New feature.

Description:
    There is some discussion among the SAF team as to whether it is better
    to use MPI derived types for raw data transfers (thus needing a
    MPI_File_set_view() call), or whether it is better to use a sequence of
    low-level MPI types (i.e. MPI_BYTE) for the raw data transfer.

Solution:
    Added an internal flag to determine whether derived types are preferred
    (the default), or whether they should be avoided.  An environment variable
    ("HDF5_MPI_PREFER_DERIVED_TYPES") can be set by users to control whether MPI
    types should be used or not.  Set the environment variable to "0" (i.e.:
    'setenv HDF5_MPI_PREFER_DERIVED_TYPES 0') to avoid using MPI derived types.

Platforms tested:
    IRIX64 6.5 (modi4) w/parallel

5 files changed, 103 insertions, 43 deletions

diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt
index 953f889..d25dbb0 100644
--- a/release_docs/RELEASE.txt
+++ b/release_docs/RELEASE.txt
@@ -71,6 +71,13 @@ New Features
 
   o Parallel Library
   ==================
+    * Added environment variable flag to control whether creating MPI derived
+      typed is preferred or not.  This can affect performance, depending on
+      which way the MPI-I/O library is optimized for.  The default is set to
+      prefer MPI derived types for collective raw data transfers, setting the
+      HDF5_MPI_PREFER_DERIVED_TYPES environment variable to "0" (i.e.:
+      "setenv HDF5_MPI_PREFER_DERIVED_TYPES 0") changes the preference to avoid
+      using then whenever possible.  QAK - 2002/06/19
     * Changed MPI I/O routines to avoid creating MPI derived types (and thus
       needing to set the file view) for contiguous selections within datasets,
       which should result in some performance improvement for those types of
diff --git a/src/H5D.c b/src/H5D.c
index fea42af..25911d2 100644
--- a/src/H5D.c
+++ b/src/H5D.c
@@ -1703,7 +1703,7 @@ printf("%s: check 1.0, nelmts=%d\n",FUNC,(int)nelmts);
      * mem-and-file-dataspace-xfer functions
      * (the latter in case the arguments to sconv_funcs
      * turn out to be inappropriate for MPI-IO).  */
-    if (H5_mpi_opt_types_g &&
+    if (H5S_mpi_opt_types_g &&
         IS_H5FD_MPIO(dataset->ent.file)) {
 	/* Only collective write should call this since it eventually
 	 * calls MPI_File_set_view which is a collective call.
@@ -2190,7 +2190,7 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
      * mem-and-file-dataspace-xfer functions
      * (the latter in case the arguments to sconv_funcs
      * turn out to be inappropriate for MPI-IO).  */
-    if (H5_mpi_opt_types_g &&
+    if (H5S_mpi_opt_types_g &&
         IS_H5FD_MPIO(dataset->ent.file)) {
 	/* Only collective write should call this since it eventually
 	 * calls MPI_File_set_view which is a collective call.
diff --git a/src/H5S.c b/src/H5S.c
index f0edaf9..62b4755 100644
--- a/src/H5S.c
+++ b/src/H5S.c
@@ -41,8 +41,9 @@ static size_t			H5S_aconv_g = 0;	/*entries allocated*/
 static size_t			H5S_nconv_g = 0;	/*entries used*/
 
 #ifdef H5_HAVE_PARALLEL
-/* Global var whose value can be set from environment variable also*/
-hbool_t         H5_mpi_opt_types_g = TRUE;
+/* Global vars whose value can be set from environment variable also*/
+hbool_t H5S_mpi_opt_types_g = TRUE;
+hbool_t H5S_mpi_prefer_derived_types_g = TRUE;
 #endif
 
 /* Declare a free list to manage the H5S_simple_t struct */
@@ -94,16 +95,14 @@ H5S_init_interface(void)
     {
         /* Allow MPI buf-and-file-type optimizations? */
         const char *s = HDgetenv ("HDF5_MPI_OPT_TYPES");
-#ifdef H5FDmpio_DEBUG
-	hbool_t         oldtmp = H5_mpi_opt_types_g ;
-#endif
-        if (s && HDisdigit(*s)) {
-            H5_mpi_opt_types_g = (int)HDstrtol (s, NULL, 0);
-        }
-#ifdef H5FDmpio_DEBUG
-    	fprintf(stdout, "H5_mpi_opt_types_g was %ld became %ld\n",
-		    oldtmp, H5_mpi_opt_types_g);
-#endif
+        if (s && HDisdigit(*s))
+            H5S_mpi_opt_types_g = (int)HDstrtol (s, NULL, 0);
+    }
+    {
+        /* Prefer MPI derived types for collective data transfers? */
+        const char *s = HDgetenv ("HDF5_MPI_PREFER_DERIVED_TYPES");
+        if (s && HDisdigit(*s))
+            H5S_mpi_prefer_derived_types_g = (int)HDstrtol (s, NULL, 0);
     }
 #endif
 
diff --git a/src/H5Smpio.c b/src/H5Smpio.c
index f44992b..99c55b6 100644
--- a/src/H5Smpio.c
+++ b/src/H5Smpio.c
@@ -49,7 +49,7 @@
 static int             interface_initialize_g = 0;
 
 static herr_t
-H5S_mpio_all_type( const H5S_t *space, const size_t elmt_size,
+H5S_mpio_all_type( const H5S_t *space, size_t elmt_size, hbool_t prefer_derived_types,
 		     /* out: */
 		     MPI_Datatype *new_type,
 		     hsize_t *count,
@@ -57,7 +57,7 @@ H5S_mpio_all_type( const H5S_t *space, const size_t elmt_size,
 		     hbool_t *use_view,
 		     hbool_t *is_derived_type );
 static herr_t
-H5S_mpio_hyper_type( const H5S_t *space, const size_t elmt_size,
+H5S_mpio_hyper_type( const H5S_t *space, size_t elmt_size, hbool_t prefer_derived_types,
 		     /* out: */
 		     MPI_Datatype *new_type,
 		     hsize_t *count,
@@ -65,7 +65,7 @@ H5S_mpio_hyper_type( const H5S_t *space, const size_t elmt_size,
 		     hbool_t *use_view,
 		     hbool_t *is_derived_type );
 static herr_t
-H5S_mpio_hyper_contig_type( const H5S_t *space, const size_t elmt_size,
+H5S_mpio_hyper_contig_type( const H5S_t *space, size_t elmt_size, hbool_t prefer_derived_types,
 		     /* out: */
 		     MPI_Datatype *new_type,
 		     hsize_t *count,
@@ -73,7 +73,7 @@ H5S_mpio_hyper_contig_type( const H5S_t *space, const size_t elmt_size,
 		     hbool_t *use_view,
 		     hbool_t *is_derived_type );
 static herr_t
-H5S_mpio_space_type( const H5S_t *space, const size_t elmt_size,
+H5S_mpio_space_type( const H5S_t *space, size_t elmt_size, hbool_t prefer_derived_types,
 		     /* out: */
 		     MPI_Datatype *new_type,
 		     hsize_t *count,
@@ -85,7 +85,7 @@ H5S_mpio_spaces_xfer(H5F_t *f, const struct H5O_layout_t *layout,
                      size_t elmt_size,
                      const H5S_t *file_space, const H5S_t *mem_space,
                      hid_t dxpl_id, void *buf/*out*/,
-		     hbool_t *must_convert/*out*/, const hbool_t do_write);
+		     hbool_t *must_convert/*out*/, hbool_t do_write);
 
 /*-------------------------------------------------------------------------
  * Function:	H5S_mpio_all_type
@@ -108,10 +108,14 @@ H5S_mpio_spaces_xfer(H5F_t *f, const struct H5O_layout_t *layout,
  *      Quincey Koziol, June 18, 2002
  *      Added 'extra_offset' and 'use_view' parameters
  *
+ *      Quincey Koziol, June 19, 2002
+ *      Added 'prefer_derived_types' flag to choose whether MPI derived types
+ *      should be created or not.
+ *
  *-------------------------------------------------------------------------
  */
 herr_t
-H5S_mpio_all_type( const H5S_t *space, const size_t elmt_size,
+H5S_mpio_all_type( const H5S_t *space, size_t elmt_size, hbool_t prefer_derived_types,
 		     /* out: */
 		     MPI_Datatype *new_type,
 		     hsize_t *count,
@@ -132,12 +136,25 @@ H5S_mpio_all_type( const H5S_t *space, const size_t elmt_size,
     for (u=0; u<space->extent.u.simple.rank; ++u)
 	total_bytes *= space->extent.u.simple.size[u];
 
-    /* fill in the return values */
-    *new_type = MPI_BYTE;
-    *count=total_bytes;
-    *extra_offset = 0;
-    *use_view = 0;
-    *is_derived_type = 0;
+    /* Check if we should prefer creating a derived type */
+    if(prefer_derived_types) {
+        /* fill in the return values */
+        H5_CHECK_OVERFLOW(total_bytes, hsize_t, int);
+        if (MPI_Type_contiguous( (int)total_bytes, MPI_BYTE, new_type ))
+            HRETURN_ERROR(H5E_DATASPACE, H5E_MPI, FAIL,"couldn't create MPI contiguous type");
+        *count = 1;
+        *extra_offset = 0;
+        *use_view = 1;
+        *is_derived_type = 1;
+    } /* end if */
+    else {
+        /* fill in the return values */
+        *new_type = MPI_BYTE;
+        *count=total_bytes;
+        *extra_offset = 0;
+        *use_view = 0;
+        *is_derived_type = 0;
+    } /* end else */
 
 #ifdef H5Smpi_DEBUG
     fprintf(stdout, "Leave %s total_bytes=%Hu\n", FUNC, total_bytes );
@@ -171,10 +188,14 @@ H5S_mpio_all_type( const H5S_t *space, const size_t elmt_size,
  *      Added 'extra_offset' and 'use_view' parameters.  Also accomodate
  *      selection offset in MPI type built.
  *
+ *      Quincey Koziol, June 19, 2002
+ *      Added 'prefer_derived_types' flag to choose whether MPI derived types
+ *      should be created or not. (Ignored for this routine)
+ *
  *-------------------------------------------------------------------------
  */
 herr_t
-H5S_mpio_hyper_type( const H5S_t *space, const size_t elmt_size,
+H5S_mpio_hyper_type( const H5S_t *space, size_t elmt_size, hbool_t UNUSED prefer_derived_types,
 		     /* out: */
 		     MPI_Datatype *new_type,
 		     hsize_t *count,
@@ -499,10 +520,14 @@ done:
  *
  * Modifications:
  *
+ *      Quincey Koziol, June 19, 2002
+ *      Added 'prefer_derived_types' flag to choose whether MPI derived types
+ *      should be created or not.
+ *
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5S_mpio_hyper_contig_type( const H5S_t *space, const size_t elmt_size,
+H5S_mpio_hyper_contig_type( const H5S_t *space, size_t elmt_size, hbool_t prefer_derived_types,
 		     /* out: */
 		     MPI_Datatype *new_type,
 		     hsize_t *count,
@@ -548,12 +573,25 @@ H5S_mpio_hyper_contig_type( const H5S_t *space, const size_t elmt_size,
     for(i=0,byte_offset=0; i<ndims; i++)
         byte_offset+=offset[i]*slab[i];
 
-    /* fill in the return values */
-    *new_type = MPI_BYTE;
-    *count=total_bytes;
-    *extra_offset = byte_offset;
-    *use_view = 0;
-    *is_derived_type = 0;
+    /* Check if we should prefer creating a derived type */
+    if(prefer_derived_types) {
+        /* fill in the return values */
+        H5_CHECK_OVERFLOW(total_bytes, hsize_t, int);
+        if (MPI_Type_contiguous( (int)total_bytes, MPI_BYTE, new_type ))
+            HRETURN_ERROR(H5E_DATASPACE, H5E_MPI, FAIL,"couldn't create MPI contiguous type");
+        *count = 1;
+        *extra_offset = byte_offset;
+        *use_view = 1;
+        *is_derived_type = 1;
+    } /* end if */
+    else {
+        /* fill in the return values */
+        *new_type = MPI_BYTE;
+        *count=total_bytes;
+        *extra_offset = byte_offset;
+        *use_view = 0;
+        *is_derived_type = 0;
+    } /* end else */
 
 #ifdef H5Smpi_DEBUG
     HDfprintf(stdout, "Leave %s total_bytes=%Hu\n", FUNC, total_bytes );
@@ -584,10 +622,14 @@ H5S_mpio_hyper_contig_type( const H5S_t *space, const size_t elmt_size,
  *      Quincey Koziol, June 18, 2002
  *      Added 'extra_offset' and 'use_view' parameters
  *
+ *      Quincey Koziol, June 19, 2002
+ *      Added 'prefer_derived_types' flag to choose whether MPI derived types
+ *      should be created or not.
+ *
  *-------------------------------------------------------------------------
  */
 herr_t
-H5S_mpio_space_type( const H5S_t *space, const size_t elmt_size,
+H5S_mpio_space_type( const H5S_t *space, size_t elmt_size, hbool_t prefer_derived_types,
 		     /* out: */
 		     MPI_Datatype *new_type,
 		     hsize_t *count,
@@ -614,7 +656,7 @@ H5S_mpio_space_type( const H5S_t *space, const size_t elmt_size,
             switch(space->select.type) {
                 case H5S_SEL_NONE:
                 case H5S_SEL_ALL:
-		    err = H5S_mpio_all_type( space, elmt_size,
+                    err = H5S_mpio_all_type( space, elmt_size, prefer_derived_types,
 				/* out: */ new_type, count, extra_offset, use_view, is_derived_type );
 		    if (err<0)
 			HRETURN_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL,"couldn't convert \"all\" selection to MPI type");
@@ -627,11 +669,11 @@ H5S_mpio_space_type( const H5S_t *space, const size_t elmt_size,
 
                 case H5S_SEL_HYPERSLABS:
                     if(H5S_select_contiguous(space)) {
-                        err = H5S_mpio_hyper_contig_type( space, elmt_size,
+                        err = H5S_mpio_hyper_contig_type( space, elmt_size, prefer_derived_types,
                             /* out: */ new_type, count, extra_offset, use_view, is_derived_type );
                     } /* end if */
                     else {
-                        err = H5S_mpio_hyper_type( space, elmt_size,
+                        err = H5S_mpio_hyper_type( space, elmt_size, prefer_derived_types,
                             /* out: */ new_type, count, extra_offset, use_view, is_derived_type );
                     } /* end else */
 		    if (err<0)
@@ -688,6 +730,11 @@ H5S_mpio_space_type( const H5S_t *space, const size_t elmt_size,
  *      used.  Also, switch to getting the 'use_view' and 'extra_offset'
  *      settings for each selection.
  *
+ *      Quincey Koziol, June 19, 2002
+ *      Use 'prefer_derived_types' flag (from HDF5_MPI_PREFER_DERIVED_TYPES
+ *      environment variable) to choose whether MPI derived types should be
+ *      preferred or not.
+ *
  *-------------------------------------------------------------------------
  */
 herr_t
@@ -696,7 +743,7 @@ H5S_mpio_spaces_xfer(H5F_t *f, const struct H5O_layout_t *layout,
                      const H5S_t *file_space, const H5S_t *mem_space,
 		     hid_t dxpl_id, void *_buf /*out*/,
 		     hbool_t *must_convert /*out*/,
-		     const hbool_t do_write )
+		     hbool_t do_write )
 {
     haddr_t	 addr;                  /* Address of dataset (or selection) within file */
     hsize_t	 mpi_buf_count, mpi_file_count;       /* Number of "objects" to transfer */
@@ -707,6 +754,7 @@ H5S_mpio_spaces_xfer(H5F_t *f, const struct H5O_layout_t *layout,
     hbool_t	 mbt_is_derived=0,      /* Whether the buffer (memory) type is derived and needs to be free'd */
 		 mft_is_derived=0;      /* Whether the file type is derived and needs to be free'd */
     hbool_t	 plist_is_setup=0;      /* Whether the dxpl has been customized */
+    hbool_t	 prefer_derived_types=0;/* Whether to prefer MPI derived types or not */
     uint8_t	*buf=(uint8_t *)_buf;   /* Alias for pointer arithmetic */
     int		 err;                   /* Error detection value */
     herr_t	 ret_value = SUCCEED;   /* Return value */
@@ -763,8 +811,12 @@ H5S_mpio_spaces_xfer(H5F_t *f, const struct H5O_layout_t *layout,
     }
 #endif
 
+    /* Get the preference for MPI derived types */
+    /* (Set via the "HDF5_MPI_PREFER_DERIVED_TYPES" environment variable for now) */
+    prefer_derived_types= H5S_mpi_prefer_derived_types_g;
+
     /* create the MPI buffer type */
-    err = H5S_mpio_space_type( mem_space, elmt_size,
+    err = H5S_mpio_space_type( mem_space, elmt_size, prefer_derived_types,
 			       /* out: */
 			       &mpi_buf_type,
 			       &mpi_buf_count,
@@ -775,7 +827,7 @@ H5S_mpio_spaces_xfer(H5F_t *f, const struct H5O_layout_t *layout,
     	HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL,"couldn't create MPI buf type");
 
     /* create the MPI file type */
-    err = H5S_mpio_space_type( file_space, elmt_size,
+    err = H5S_mpio_space_type( file_space, elmt_size, prefer_derived_types,
 			       /* out: */
 			       &mpi_file_type,
 			       &mpi_file_count,
diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h
index 6c11dbb..18fbe00 100644
--- a/src/H5Sprivate.h
+++ b/src/H5Sprivate.h
@@ -262,8 +262,10 @@ __DLL__ herr_t H5S_mpio_spaces_write(H5F_t *f,
 				     const void *buf,
 				     hbool_t *must_convert /*out*/ );
 #ifndef _H5S_IN_H5S_C
-/* Global var whose value comes from environment variable */
-__DLLVAR__ hbool_t		H5_mpi_opt_types_g;
+/* Global vars whose value comes from environment variable */
+/* (Defined in H5S.c) */
+__DLLVAR__ hbool_t		H5S_mpi_opt_types_g;
+__DLLVAR__ hbool_t		H5S_mpi_prefer_derived_types_g;
 #endif /* _H5S_IN_H5S_C */
 
 #endif /* H5_HAVE_PARALLEL */