merged:

HDFFV-10652
HDFFV-10443

1 files changed, 96 insertions, 25 deletions

diff --git a/src/H5Dmpio.c b/src/H5Dmpio.c
index baec4c4..20b2066 100644
--- a/src/H5Dmpio.c
+++ b/src/H5Dmpio.c
@@ -37,7 +37,6 @@
 #include "H5Eprivate.h"       /* Error handling    */
 #include "H5Fprivate.h"       /* File access       */
 #include "H5FDprivate.h"      /* File drivers      */
-#include "H5FDmpi.h"          /* MPI-based file drivers	*/
 #include "H5Iprivate.h"       /* IDs               */
 #include "H5MMprivate.h"      /* Memory management */
 #include "H5Oprivate.h"       /* Object headers    */
@@ -83,16 +82,30 @@
 /* Macros to represent the regularity of the selection for multiple chunk IO case. */
 #define H5D_CHUNK_SELECT_REG          1
 
+/* Macros for reasons to not enable read-proc-and-bcast. */
+#define H5D_MPIO_PROC0_BCAST 0x00
+#define H5D_MPIO_NOT_H5S_ALL 0x01
+#define H5D_MPIO_GREATER_THAN_2GB 0x02
 
 /******************/
 /* Local Typedefs */
 /******************/
 /* Combine chunk address and chunk info into a struct for better performance. */
 typedef struct H5D_chunk_addr_info_t {
-  haddr_t chunk_addr;
-  H5D_chunk_info_t chunk_info;
+    haddr_t chunk_addr;
+    H5D_chunk_info_t chunk_info;
 } H5D_chunk_addr_info_t;
 
+/* Rank 0 Bcast values */
+typedef enum H5D_mpio_no_rank0_bcast_cause_t {
+    H5D_MPIO_RANK0_BCAST = 0x00,
+    H5D_MPIO_RANK0_NOT_H5S_ALL = 0x01,
+    H5D_MPIO_RANK0_NOT_CONTIGUOUS = 0x02,
+    H5D_MPIO_RANK0_NOT_FIXED_SIZE = 0x04,
+    H5D_MPIO_RANK0_GREATER_THAN_2GB = 0x08
+} H5D_mpio_no_rank0_bcast_cause_t;
+
+
 /*
  * Information about a single chunk when performing collective filtered I/O. All
  * of the fields of one of these structs are initialized at the start of collective
@@ -281,9 +294,12 @@ H5D__mpio_opt_possible(const H5D_io_info_t *io_info, const H5S_t *file_space,
     const H5S_t *mem_space, const H5D_type_info_t *type_info)
 {
     H5FD_mpio_xfer_t io_xfer_mode;      /* MPI I/O transfer mode */
-    unsigned local_cause = 0;                /* Local reason(s) for breaking collective mode */
-    unsigned global_cause = 0;               /* Global reason(s) for breaking collective mode */
+    unsigned local_cause[2] = {0,0};    /* [0] Local reason(s) for breaking collective mode */
+                                        /* [1] Flag if dataset is both: H5S_ALL and small */
+    unsigned global_cause[2] = {0,0};   /* Global reason(s) for breaking collective mode */
+    htri_t is_vl_storage;               /* Whether the dataset's datatype is stored in a variable-length form */
     htri_t ret_value = SUCCEED;         /* Return value */
+    hbool_t H5FD_MPIO_Proc0_BCast;      /* Flag if dataset is both: H5S_ALL and < 2GB */
 
     FUNC_ENTER_PACKAGE
 
@@ -296,36 +312,37 @@ H5D__mpio_opt_possible(const H5D_io_info_t *io_info, const H5S_t *file_space,
 
     /* For independent I/O, get out quickly and don't try to form consensus */
     if(H5CX_get_io_xfer_mode(&io_xfer_mode) < 0)
-        HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get MPI-I/O transfer mode")
+        /* Set error flag, but keep going */
+        local_cause[0] |= H5D_MPIO_ERROR_WHILE_CHECKING_COLLECTIVE_POSSIBLE;
     if(io_xfer_mode == H5FD_MPIO_INDEPENDENT)
-        local_cause |= H5D_MPIO_SET_INDEPENDENT;
+        local_cause[0] |= H5D_MPIO_SET_INDEPENDENT;
 
     /* Optimized MPI types flag must be set */
     /* (based on 'HDF5_MPI_OPT_TYPES' environment variable) */
     if(!H5FD_mpi_opt_types_g)
-        local_cause |= H5D_MPIO_MPI_OPT_TYPES_ENV_VAR_DISABLED;
+        local_cause[0] |= H5D_MPIO_MPI_OPT_TYPES_ENV_VAR_DISABLED;
 
     /* Don't allow collective operations if datatype conversions need to happen */
     if(!type_info->is_conv_noop)
-        local_cause |= H5D_MPIO_DATATYPE_CONVERSION;
+        local_cause[0] |= H5D_MPIO_DATATYPE_CONVERSION;
 
     /* Don't allow collective operations if data transform operations should occur */
     if(!type_info->is_xform_noop)
-        local_cause |= H5D_MPIO_DATA_TRANSFORMS;
+        local_cause[0] |= H5D_MPIO_DATA_TRANSFORMS;
 
     /* Check whether these are both simple or scalar dataspaces */
     if(!((H5S_SIMPLE == H5S_GET_EXTENT_TYPE(mem_space) || H5S_SCALAR == H5S_GET_EXTENT_TYPE(mem_space))
             && (H5S_SIMPLE == H5S_GET_EXTENT_TYPE(file_space) || H5S_SCALAR == H5S_GET_EXTENT_TYPE(file_space))))
-        local_cause |= H5D_MPIO_NOT_SIMPLE_OR_SCALAR_DATASPACES;
+        local_cause[0] |= H5D_MPIO_NOT_SIMPLE_OR_SCALAR_DATASPACES;
 
     /* Dataset storage must be contiguous or chunked */
     if(!(io_info->dset->shared->layout.type == H5D_CONTIGUOUS ||
             io_info->dset->shared->layout.type == H5D_CHUNKED))
-        local_cause |= H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET;
+        local_cause[0] |= H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET;
 
     /* check if external-file storage is used */
     if(io_info->dset->shared->dcpl_cache.efl.nused > 0)
-        local_cause |= H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET;
+        local_cause[0] |= H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET;
 
     /* The handling of memory space is different for chunking and contiguous
      *  storage.  For contiguous storage, mem_space and file_space won't change
@@ -340,31 +357,85 @@ H5D__mpio_opt_possible(const H5D_io_info_t *io_info, const H5S_t *file_space,
      * is less than 3. The functions needed (MPI_Mprobe and MPI_Imrecv) will
      * not be available.
      */
-    if (io_info->op_type == H5D_IO_OP_WRITE &&
-        io_info->dset->shared->layout.type == H5D_CHUNKED &&
-        io_info->dset->shared->dcpl_cache.pline.nused > 0)
-        local_cause |= H5D_MPIO_PARALLEL_FILTERED_WRITES_DISABLED;
+    if(io_info->op_type == H5D_IO_OP_WRITE &&
+            io_info->dset->shared->layout.type == H5D_CHUNKED &&
+            io_info->dset->shared->dcpl_cache.pline.nused > 0)
+        local_cause[0] |= H5D_MPIO_PARALLEL_FILTERED_WRITES_DISABLED;
 #endif
 
+    /* Check if we are able to do a MPI_Bcast of the data from one rank
+     * instead of having all the processes involved in the collective I/O call.
+     */
+
+    /* Check to see if the process is reading the entire dataset */
+    if(H5S_GET_SELECT_TYPE(file_space) != H5S_SEL_ALL)
+        local_cause[1] |= H5D_MPIO_RANK0_NOT_H5S_ALL; 
+    /* Only perform this optimization for contigous datasets, currently */
+    else if(H5D_CONTIGUOUS != io_info->dset->shared->layout.type)
+        /* Flag to do a MPI_Bcast of the data from one proc instead of 
+         * having all the processes involved in the collective I/O.
+         */
+        local_cause[1] |= H5D_MPIO_RANK0_NOT_CONTIGUOUS; 
+    else if((is_vl_storage = H5T_is_vl_storage(type_info->dset_type)) < 0)
+        local_cause[0] |= H5D_MPIO_ERROR_WHILE_CHECKING_COLLECTIVE_POSSIBLE;
+    else if(is_vl_storage)
+        local_cause[1] |= H5D_MPIO_RANK0_NOT_FIXED_SIZE;
+    else {
+        size_t type_size;       /* Size of dataset's datatype */
+
+        /* Retrieve the size of the dataset's datatype */
+        if(0 == (type_size = H5T_GET_SIZE(type_info->dset_type)))
+            local_cause[0] |= H5D_MPIO_ERROR_WHILE_CHECKING_COLLECTIVE_POSSIBLE;
+        else {
+            hssize_t snelmts;   /* [Signed] # of elements in dataset's dataspace */
+
+            /* Retrieve the size of the dataset's datatype */
+            if((snelmts = H5S_GET_EXTENT_NPOINTS(file_space)) < 0)
+                local_cause[0] |= H5D_MPIO_ERROR_WHILE_CHECKING_COLLECTIVE_POSSIBLE;
+            else {
+                hsize_t dset_size;
+
+                /* Determine dataset size */
+                dset_size = ((hsize_t)snelmts) * type_size;
+
+               /* If the size of the dataset is less than 2GB then do an MPI_Bcast
+                * of the data from one process instead of having all the processes 
+                * involved in the collective I/O.
+                */
+                if(dset_size > ((hsize_t)(2.0F * H5_GB) - 1))
+                    local_cause[1] |= H5D_MPIO_RANK0_GREATER_THAN_2GB;
+            } /* end else */
+        } /* end else */
+    } /* end else */
+    
     /* Check for independent I/O */
-    if(local_cause & H5D_MPIO_SET_INDEPENDENT)
-        global_cause = local_cause;
+    if(local_cause[0] & H5D_MPIO_SET_INDEPENDENT)
+        global_cause[0] = local_cause[0];
     else {
         int mpi_code;               /* MPI error code */
 
         /* Form consensus opinion among all processes about whether to perform
          * collective I/O
          */
-        if(MPI_SUCCESS != (mpi_code = MPI_Allreduce(&local_cause, &global_cause, 1, MPI_UNSIGNED, MPI_BOR, io_info->comm)))
+        if(MPI_SUCCESS != (mpi_code = MPI_Allreduce(&local_cause, &global_cause, 2, MPI_UNSIGNED, MPI_BOR, io_info->comm)))
             HMPI_GOTO_ERROR(FAIL, "MPI_Allreduce failed", mpi_code)
+
     } /* end else */
 
     /* Set the local & global values of no-collective-cause in the API context */
-    H5CX_set_mpio_local_no_coll_cause(local_cause);
-    H5CX_set_mpio_global_no_coll_cause(global_cause);
+    H5CX_set_mpio_local_no_coll_cause(local_cause[0]);
+    H5CX_set_mpio_global_no_coll_cause(global_cause[0]);
+
+    /* Set read-with-rank0-and-bcast flag if possible */
+    if(global_cause[0] == 0 && global_cause[1] == 0) {
+        H5CX_set_mpio_rank0_bcast(TRUE);
+#ifdef H5_HAVE_INSTRUMENTED_LIBRARY
+        H5CX_test_set_mpio_coll_rank0_bcast(TRUE);
+#endif /* H5_HAVE_INSTRUMENTED_LIBRARY */
+    } /* end if */
 
     /* Set the return value, based on the global cause */
-    ret_value = global_cause > 0 ? FALSE : TRUE;
+    ret_value = global_cause[0] > 0 ? FALSE : TRUE;
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
@@ -3076,8 +3147,8 @@ H5D__filtered_collective_chunk_entry_io(H5D_filtered_collective_io_info_t *chunk
         chunk_entry->chunk_states.new_chunk.length = chunk_entry->chunk_states.chunk_current.length;
 
         /* Currently, these chunk reads are done independently and will likely
-	 * cause issues with collective metadata reads enabled. In the future,
-	 * this should be refactored to use collective chunk reads - JTH */
+         * cause issues with collective metadata reads enabled. In the future,
+         * this should be refactored to use collective chunk reads - JTH */
 
         /* Get the original state of parallel I/O transfer mode */
         if(H5CX_get_io_xfer_mode(&xfer_mode) < 0)