diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/H5Dio.c | 2 | ||||
| -rw-r--r-- | src/H5Dmpio.c | 66 | ||||
| -rw-r--r-- | src/H5Dpkg.h | 3 | ||||
| -rw-r--r-- | src/H5Dprivate.h | 2 | ||||
| -rw-r--r-- | src/H5Pdxpl.c | 51 | ||||
| -rw-r--r-- | src/H5Ppublic.h | 14 | ||||
| -rw-r--r-- | src/H5trace.c | 54 |
7 files changed, 163 insertions, 29 deletions
diff --git a/src/H5Dio.c b/src/H5Dio.c index 917fd89..0493a9e 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -996,7 +996,7 @@ H5D__ioinfo_adjust(H5D_io_info_t *io_info, const H5D_t *dset, hid_t dxpl_id, HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't retrieve MPI communicator") /* Check if we can set direct MPI-IO read/write functions */ - if((opt = H5D__mpio_opt_possible(io_info, file_space, mem_space, type_info, fm)) < 0) + if((opt = H5D__mpio_opt_possible(io_info, file_space, mem_space, type_info, fm, dx_plist)) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "invalid check for direct IO dataspace ") /* Check if we can use the optimized parallel I/O routines */ diff --git a/src/H5Dmpio.c b/src/H5Dmpio.c index 9b8fa27..c2d964e 100644 --- a/src/H5Dmpio.c +++ b/src/H5Dmpio.c @@ -156,10 +156,12 @@ static herr_t H5D__mpio_get_sum_chunk(const H5D_io_info_t *io_info, htri_t H5D__mpio_opt_possible(const H5D_io_info_t *io_info, const H5S_t *file_space, const H5S_t *mem_space, const H5D_type_info_t *type_info, - const H5D_chunk_map_t *fm) + const H5D_chunk_map_t *fm, H5P_genplist_t *dx_plist) { - int local_opinion = TRUE; /* This process's idea of whether to perform collective I/O or not */ - int consensus; /* Consensus opinion of all processes */ + /* variables to set cause of broken collective I/O */ + int local_cause = 0; + int global_cause = 0; + int mpi_code; /* MPI error code */ htri_t ret_value = TRUE; @@ -171,51 +173,54 @@ H5D__mpio_opt_possible(const H5D_io_info_t *io_info, const H5S_t *file_space, HDassert(file_space); HDassert(type_info); + /* For independent I/O, get out quickly and don't try to form consensus */ - if(io_info->dxpl_cache->xfer_mode == H5FD_MPIO_INDEPENDENT) + if(io_info->dxpl_cache->xfer_mode == H5FD_MPIO_INDEPENDENT) { + local_cause = H5D_MPIO_SET_INDEPENDENT; + global_cause = H5D_MPIO_SET_INDEPENDENT; HGOTO_DONE(FALSE); + } + + /* Optimized MPI types flag must be set and it must be collective IO */ + /* (Don't allow parallel I/O for the MPI-posix driver, since it doesn't do real collective I/O) */ + if(!(H5S_mpi_opt_types_g && io_info->dxpl_cache->xfer_mode == H5FD_MPIO_COLLECTIVE + && !IS_H5FD_MPIPOSIX(io_info->dset->oloc.file))) { + local_cause |= H5D_MPIO_SET_MPIPOSIX; + } /* end if */ /* Don't allow collective operations if datatype conversions need to happen */ if(!type_info->is_conv_noop) { - local_opinion = FALSE; - goto broadcast; + local_cause |= H5D_MPIO_DATATYPE_CONVERSION; } /* end if */ /* Don't allow collective operations if data transform operations should occur */ if(!type_info->is_xform_noop) { - local_opinion = FALSE; - goto broadcast; - } /* end if */ - - /* Optimized MPI types flag must be set and it must be collective IO */ - /* (Don't allow parallel I/O for the MPI-posix driver, since it doesn't do real collective I/O) */ - if(!(H5S_mpi_opt_types_g && io_info->dxpl_cache->xfer_mode == H5FD_MPIO_COLLECTIVE - && !IS_H5FD_MPIPOSIX(io_info->dset->oloc.file))) { - local_opinion = FALSE; - goto broadcast; + local_cause |= H5D_MPIO_DATA_TRANSFORMS; } /* end if */ /* Check whether these are both simple or scalar dataspaces */ if(!((H5S_SIMPLE == H5S_GET_EXTENT_TYPE(mem_space) || H5S_SCALAR == H5S_GET_EXTENT_TYPE(mem_space)) && (H5S_SIMPLE == H5S_GET_EXTENT_TYPE(file_space) || H5S_SCALAR == H5S_GET_EXTENT_TYPE(file_space)))) { - local_opinion = FALSE; - goto broadcast; + local_cause |= H5D_MPIO_NOT_SIMPLE_OR_SCALAR_DATASPACES; } /* end if */ /* Can't currently handle point selections */ if(H5S_SEL_POINTS == H5S_GET_SELECT_TYPE(mem_space) || H5S_SEL_POINTS == H5S_GET_SELECT_TYPE(file_space)) { - local_opinion = FALSE; - goto broadcast; + local_cause |= H5D_MPIO_POINT_SELECTIONS; } /* end if */ /* Dataset storage must be contiguous or chunked */ if(!(io_info->dset->shared->layout.type == H5D_CONTIGUOUS || io_info->dset->shared->layout.type == H5D_CHUNKED)) { - local_opinion = FALSE; - goto broadcast; + local_cause |= H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; } /* end if */ + /* check if external-file storage is used */ + if (io_info->dset->shared->dcpl_cache.efl.nused > 0) { + local_cause |= H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; + } + /* The handling of memory space is different for chunking and contiguous * storage. For contiguous storage, mem_space and file_space won't change * when it it is doing disk IO. For chunking storage, mem_space will @@ -226,21 +231,28 @@ H5D__mpio_opt_possible(const H5D_io_info_t *io_info, const H5S_t *file_space, /* Don't allow collective operations if filters need to be applied */ if(io_info->dset->shared->layout.type == H5D_CHUNKED) { if(io_info->dset->shared->dcpl_cache.pline.nused > 0) { - local_opinion = FALSE; - goto broadcast; + local_cause |= H5D_MPIO_FILTERS; } /* end if */ } /* end if */ -broadcast: /* Form consensus opinion among all processes about whether to perform * collective I/O */ - if(MPI_SUCCESS != (mpi_code = MPI_Allreduce(&local_opinion, &consensus, 1, MPI_INT, MPI_LAND, io_info->comm))) + if(MPI_SUCCESS != (mpi_code = MPI_Allreduce(&local_cause, &global_cause, 1, MPI_INT, MPI_BOR, io_info->comm))) HMPI_GOTO_ERROR(FAIL, "MPI_Allreduce failed", mpi_code) - ret_value = consensus > 0 ? TRUE : FALSE; + ret_value = global_cause > 0 ? FALSE : TRUE; + done: + /* Write the local value of no-collective-cause to the DXPL. */ + if(H5P_set(dx_plist, H5D_MPIO_LOCAL_NO_COLLECTIVE_CAUSE_NAME, &local_cause) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "couldn't set local no collective cause property") + + /* Write the global value of no-collective-cause to the DXPL. */ + if(H5P_set(dx_plist, H5D_MPIO_GLOBAL_NO_COLLECTIVE_CAUSE_NAME, &global_cause) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "couldn't set global no collective cause property") + FUNC_LEAVE_NOAPI(ret_value) } /* H5D__mpio_opt_possible() */ diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index dfc19b8..ed6da8f 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -693,7 +693,8 @@ H5_DLL herr_t H5D__chunk_collective_write(H5D_io_info_t *io_info, * memory and the file */ H5_DLL htri_t H5D__mpio_opt_possible(const H5D_io_info_t *io_info, const H5S_t *file_space, const H5S_t *mem_space, - const H5D_type_info_t *type_info, const H5D_chunk_map_t *fm); + const H5D_type_info_t *type_info, const H5D_chunk_map_t *fm, + H5P_genplist_t *dx_plist); #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h index 2211f79..85051c3 100644 --- a/src/H5Dprivate.h +++ b/src/H5Dprivate.h @@ -74,6 +74,8 @@ #define H5D_XFER_MPIO_CHUNK_OPT_RATIO_NAME "mpio_chunk_opt_ratio" #define H5D_MPIO_ACTUAL_CHUNK_OPT_MODE_NAME "actual_chunk_opt_mode" #define H5D_MPIO_ACTUAL_IO_MODE_NAME "actual_io_mode" +#define H5D_MPIO_LOCAL_NO_COLLECTIVE_CAUSE_NAME "local_no_collective_cause" /* cause of broken collective I/O in each process */ +#define H5D_MPIO_GLOBAL_NO_COLLECTIVE_CAUSE_NAME "global_no_collective_cause" /* cause of broken collective I/O in all processes */ #endif /* H5_HAVE_PARALLEL */ #define H5D_XFER_EDC_NAME "err_detect" /* EDC */ #define H5D_XFER_FILTER_CB_NAME "filter_cb" /* Filter callback function */ diff --git a/src/H5Pdxpl.c b/src/H5Pdxpl.c index 87e6c89..4c98dd6 100644 --- a/src/H5Pdxpl.c +++ b/src/H5Pdxpl.c @@ -101,6 +101,9 @@ /* Definitions for chunk io mode property. */ #define H5D_MPIO_ACTUAL_IO_MODE_SIZE sizeof(H5D_mpio_actual_io_mode_t) #define H5D_MPIO_ACTUAL_IO_MODE_DEF H5D_MPIO_NO_COLLECTIVE +/* Definitions for cause of broken collective io property */ +#define H5D_MPIO_NO_COLLECTIVE_CAUSE_SIZE sizeof(uint32_t) +#define H5D_MPIO_NO_COLLECTIVE_CAUSE_DEF H5D_MPIO_COLLECTIVE /* Definitions for memory MPI type property */ #define H5FD_MPI_XFER_MEM_MPI_TYPE_SIZE sizeof(MPI_Datatype) #define H5FD_MPI_XFER_MEM_MPI_TYPE_DEF MPI_DATATYPE_NULL @@ -209,6 +212,7 @@ H5P__dxfr_reg_prop(H5P_genclass_t *pclass) unsigned def_mpio_chunk_opt_ratio = H5D_XFER_MPIO_CHUNK_OPT_RATIO_DEF; H5D_mpio_actual_chunk_opt_mode_t def_mpio_actual_chunk_opt_mode = H5D_MPIO_ACTUAL_CHUNK_OPT_MODE_DEF; H5D_mpio_actual_io_mode_t def_mpio_actual_io_mode = H5D_MPIO_ACTUAL_IO_MODE_DEF; + H5D_mpio_no_collective_cause_t def_mpio_no_collective_cause = H5D_MPIO_NO_COLLECTIVE_CAUSE_DEF; MPI_Datatype btype = H5FD_MPI_XFER_MEM_MPI_TYPE_DEF; /* Default value for MPI buffer type */ MPI_Datatype ftype = H5FD_MPI_XFER_FILE_MPI_TYPE_DEF; /* Default value for MPI file type */ #endif /* H5_HAVE_PARALLEL */ @@ -281,6 +285,14 @@ H5P__dxfr_reg_prop(H5P_genclass_t *pclass) if(H5P_register_real(pclass, H5D_MPIO_ACTUAL_IO_MODE_NAME, H5D_MPIO_ACTUAL_IO_MODE_SIZE, &def_mpio_actual_io_mode, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the local cause of broken collective I/O */ + if(H5P_register_real(pclass, H5D_MPIO_LOCAL_NO_COLLECTIVE_CAUSE_NAME, H5D_MPIO_NO_COLLECTIVE_CAUSE_SIZE, &def_mpio_no_collective_cause, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + + /* Register the global cause of broken collective I/O */ + if(H5P_register_real(pclass, H5D_MPIO_GLOBAL_NO_COLLECTIVE_CAUSE_NAME, H5D_MPIO_NO_COLLECTIVE_CAUSE_SIZE, &def_mpio_no_collective_cause, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the MPI memory type property */ if(H5P_register_real(pclass, H5FD_MPI_XFER_MEM_MPI_TYPE_NAME, H5FD_MPI_XFER_MEM_MPI_TYPE_SIZE, &btype, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) @@ -1354,5 +1366,44 @@ H5Pget_mpio_actual_io_mode(hid_t plist_id, H5D_mpio_actual_io_mode_t *actual_io_ done: FUNC_LEAVE_API(ret_value) } /* end H5Pget_mpio_actual_io_mode() */ + +/*------------------------------------------------------------------------- + * Function: H5Pget_mpio_no_collective_cause + * + * Purpose: Retrieves cause for the broke collective I/O + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Jonathan Kim + * Aug 3, 2012 + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_mpio_no_collective_cause(hid_t plist_id, uint32_t *local_no_collective_cause, uint32_t *global_no_collective_cause) +{ + H5P_genplist_t *plist; + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "i*Dn*Dn", plist_id, local_no_collective_cause, + global_no_collective_cause); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Return values */ + if(local_no_collective_cause) + if(H5P_get(plist, H5D_MPIO_LOCAL_NO_COLLECTIVE_CAUSE_NAME, local_no_collective_cause) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get local value") + if(global_no_collective_cause) + if(H5P_get(plist, H5D_MPIO_GLOBAL_NO_COLLECTIVE_CAUSE_NAME, global_no_collective_cause) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get global value") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_mpio_no_collective_cause() */ + + #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index c06ab39..8681f65 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -153,6 +153,19 @@ typedef enum H5D_mpio_actual_io_mode_t { H5D_MPIO_CONTIGUOUS_COLLECTIVE = 0x4 } H5D_mpio_actual_io_mode_t; +/* Broken collective IO property */ +typedef enum H5D_mpio_no_collective_cause_t { + H5D_MPIO_COLLECTIVE = 0x00, + H5D_MPIO_SET_INDEPENDENT = 0x01, + H5D_MPIO_DATATYPE_CONVERSION = 0x02, + H5D_MPIO_DATA_TRANSFORMS = 0x04, + H5D_MPIO_SET_MPIPOSIX = 0x08, + H5D_MPIO_NOT_SIMPLE_OR_SCALAR_DATASPACES = 0x10, + H5D_MPIO_POINT_SELECTIONS = 0x20, + H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET = 0x40, + H5D_MPIO_FILTERS = 0x80 +} H5D_mpio_no_collective_cause_t; + /********************/ /* Public Variables */ /********************/ @@ -400,6 +413,7 @@ H5_DLL herr_t H5Pget_type_conv_cb(hid_t dxpl_id, H5T_conv_except_func_t *op, voi #ifdef H5_HAVE_PARALLEL H5_DLL herr_t H5Pget_mpio_actual_chunk_opt_mode(hid_t plist_id, H5D_mpio_actual_chunk_opt_mode_t *actual_chunk_opt_mode); H5_DLL herr_t H5Pget_mpio_actual_io_mode(hid_t plist_id, H5D_mpio_actual_io_mode_t *actual_io_mode); +H5_DLL herr_t H5Pget_mpio_no_collective_cause(hid_t plist_id, uint32_t *local_no_collective_cause, uint32_t *global_no_collective_cause); #endif /* H5_HAVE_PARALLEL */ /* Link creation property list (LCPL) routines */ diff --git a/src/H5trace.c b/src/H5trace.c index e0b5794..9d0fd5b 100644 --- a/src/H5trace.c +++ b/src/H5trace.c @@ -535,6 +535,60 @@ H5_trace(const double *returning, const char *func, const char *type, ...) } /* end else */ break; + case 'n': + if(ptr) { + if(vp) + fprintf(out, "0x%lx", (unsigned long)vp); + else + fprintf(out, "NULL"); + } /* end if */ + else { + H5D_mpio_no_collective_cause_t nocol_cause_mode = (H5D_mpio_no_collective_cause_t)va_arg(ap, int); + + switch(nocol_cause_mode) { + case H5D_MPIO_COLLECTIVE: + fprintf(out, "H5D_MPIO_COLLECTIVE"); + break; + + case H5D_MPIO_SET_INDEPENDENT: + fprintf(out, "H5D_MPIO_SET_INDEPENDENT"); + break; + + case H5D_MPIO_DATATYPE_CONVERSION: + fprintf(out, "H5D_MPIO_DATATYPE_CONVERSION"); + break; + + case H5D_MPIO_DATA_TRANSFORMS: + fprintf(out, "H5D_MPIO_DATA_TRANSFORMS"); + break; + + case H5D_MPIO_SET_MPIPOSIX: + fprintf(out, "H5D_MPIO_SET_MPIPOSIX"); + break; + + case H5D_MPIO_NOT_SIMPLE_OR_SCALAR_DATASPACES: + fprintf(out, "H5D_MPIO_NOT_SIMPLE_OR_SCALAR_DATASPACES"); + break; + + case H5D_MPIO_POINT_SELECTIONS: + fprintf(out, "H5D_MPIO_POINT_SELECTIONS"); + break; + + case H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET: + fprintf(out, "H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET"); + break; + + case H5D_MPIO_FILTERS: + fprintf(out, "H5D_MPIO_FILTERS"); + break; + + default: + fprintf(out, "%ld", (long)nocol_cause_mode); + break; + } /* end switch */ + } /* end else */ + break; + case 'o': if(ptr) { if(vp) |