diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/H5Dio.c | 6 | ||||
| -rw-r--r-- | src/H5Dmpio.c | 189 | ||||
| -rw-r--r-- | src/H5Dpkg.h | 12 | ||||
| -rw-r--r-- | src/H5Smpio.c | 299 | ||||
| -rw-r--r-- | src/H5Sprivate.h | 9 |
5 files changed, 515 insertions, 0 deletions
diff --git a/src/H5Dio.c b/src/H5Dio.c index 941801c..0eb0dd7 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -3315,7 +3315,13 @@ H5D_ioinfo_init(H5D_t *dset, const H5D_dxpl_cache_t *dxpl_cache, hid_t dxpl_id, } /* end if */ else { /* Indicate that the I/O will _NOT_ be parallel */ + +#ifdef KYANG + io_info->ops.read = H5D_mpio_spaces_span_read; + io_info->ops.write = H5D_mpio_spaces_span_write; +#else *use_par_opt_io=FALSE; +#endif #endif /* H5_HAVE_PARALLEL */ io_info->ops.read = H5D_select_read; diff --git a/src/H5Dmpio.c b/src/H5Dmpio.c index 08c6a48..4dfb17b 100644 --- a/src/H5Dmpio.c +++ b/src/H5Dmpio.c @@ -45,6 +45,12 @@ H5D_mpio_spaces_xfer(H5D_io_info_t *io_info, size_t elmt_size, void *buf/*out*/, hbool_t do_write); + +static herr_t +H5D_mpio_spaces_span_xfer(H5D_io_info_t *io_info, size_t elmt_size, + const H5S_t *file_space, const H5S_t *mem_space, + void *buf/*out*/, + hbool_t do_write); /*------------------------------------------------------------------------- * Function: H5D_mpio_opt_possible @@ -364,6 +370,107 @@ done: FUNC_LEAVE_NOAPI(ret_value); } /* end H5D_mpio_spaces_xfer() */ + +/** The following function has been tested, don't call this + function until you don't see this line. Nov. 11,2004, KY**/ + +static herr_t +H5D_mpio_spaces_span_xfer(H5D_io_info_t *io_info, size_t elmt_size, + const H5S_t *file_space, const H5S_t *mem_space, + void *_buf /*out*/, hbool_t do_write ) +{ + haddr_t addr; /* Address of dataset (or selection) within file */ + size_t mpi_buf_count, mpi_file_count; /* Number of "objects" to transfer */ + hsize_t mpi_buf_offset, mpi_file_offset; /* Offset within dataset where selection (ie. MPI type) begins */ + MPI_Datatype mpi_buf_type, mpi_file_type; /* MPI types for buffer (memory) and file */ + hbool_t mbt_is_derived=0, /* Whether the buffer (memory) type is derived and needs to be free'd */ + mft_is_derived=0; /* Whether the file type is derived and needs to be free'd */ + hbool_t plist_is_setup=0; /* Whether the dxpl has been customized */ + uint8_t *buf=(uint8_t *)_buf; /* Alias for pointer arithmetic */ + int mpi_code; /* MPI return code */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_mpio_spaces_xfer); + + /* Check args */ + assert (io_info); + assert (io_info->dset); + assert (file_space); + assert (mem_space); + assert (buf); + assert (IS_H5FD_MPIO(io_info->dset->ent.file)); + /* Make certain we have the correct type of property list */ + assert(TRUE==H5P_isa_class(io_info->dxpl_id,H5P_DATASET_XFER)); + + /* create the MPI buffer type */ + if (H5S_mpio_space_span_type( mem_space, elmt_size, + /* out: */ + &mpi_buf_type, + &mpi_buf_count, + &mpi_buf_offset, + &mbt_is_derived )<0) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL,"couldn't create MPI buf type"); + + /* create the MPI file type */ + if ( H5S_mpio_space_span_type( file_space, elmt_size, + /* out: */ + &mpi_file_type, + &mpi_file_count, + &mpi_file_offset, + &mft_is_derived )<0) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL,"couldn't create MPI file type"); + + /* Get the base address of the contiguous dataset or the chunk */ + if(io_info->dset->shared->layout.type == H5D_CONTIGUOUS) + addr = H5D_contig_get_addr(io_info->dset) + mpi_file_offset; + else { + haddr_t chunk_addr; /* for collective chunk IO */ + + assert(io_info->dset->shared->layout.type == H5D_CHUNKED); + chunk_addr=H5D_istore_get_addr(io_info,NULL); + addr = H5F_BASE_ADDR(io_info->dset->ent.file) + chunk_addr + mpi_file_offset; + } + + /* + * Pass buf type, file type to the file driver. Request an MPI type + * transfer (instead of an elementary byteblock transfer). + */ + if(H5FD_mpi_setup_collective(io_info->dxpl_id, mpi_buf_type, mpi_file_type)<0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set MPI-I/O properties"); + plist_is_setup=1; + + /* Adjust the buffer pointer to the beginning of the selection */ + buf+=mpi_buf_offset; + + /* transfer the data */ + if (do_write) { + if (H5F_block_write(io_info->dset->ent.file, H5FD_MEM_DRAW, addr, mpi_buf_count, io_info->dxpl_id, buf) <0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,"MPI write failed"); + } else { + if (H5F_block_read (io_info->dset->ent.file, H5FD_MEM_DRAW, addr, mpi_buf_count, io_info->dxpl_id, buf) <0) + HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL,"MPI read failed"); + } + +done: + /* Reset the dxpl settings */ + if(plist_is_setup) { + if(H5FD_mpi_teardown_collective(io_info->dxpl_id)<0) + HDONE_ERROR(H5E_DATASPACE, H5E_CANTFREE, FAIL, "unable to reset dxpl values"); + } /* end if */ + + /* free the MPI buf and file types */ + if (mbt_is_derived) { + if (MPI_SUCCESS != (mpi_code= MPI_Type_free( &mpi_buf_type ))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code); + } + if (mft_is_derived) { + if (MPI_SUCCESS != (mpi_code= MPI_Type_free( &mpi_file_type ))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code); + } + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5D_mpio_spaces_span_xfer() */ + /*------------------------------------------------------------------------- * Function: H5D_mpio_spaces_read @@ -438,4 +545,86 @@ H5D_mpio_spaces_write(H5D_io_info_t *io_info, FUNC_LEAVE_NOAPI(ret_value); } /* end H5D_mpio_spaces_write() */ + + + +/*------------------------------------------------------------------------- + * Function: H5D_mpio_spaces_span_read + * + * Purpose: MPI-IO function to read directly from app buffer to file for + span-tree + * + * Return: non-negative on success, negative on failure. + * + * Programmer: KY + * Note : Don't call this routine + * until you don't see this line. 11/11/2004, KY + * + * Modifications: + * + * rky 980918 + * Added must_convert parameter to let caller know we can't optimize the xfer. + * + * QAK - 2002/04/02 + * Removed the must_convert parameter and move preconditions to + * H5S_mpio_opt_possible() routine + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_mpio_spaces_span_read(H5D_io_info_t *io_info, + size_t UNUSED nelmts, size_t elmt_size, + const H5S_t *file_space, const H5S_t *mem_space, + void *buf/*out*/) +{ + herr_t ret_value; + + FUNC_ENTER_NOAPI_NOFUNC(H5D_mpio_spaces_read); + + ret_value = H5D_mpio_spaces_span_xfer(io_info, elmt_size, file_space, + mem_space, buf, 0/*read*/); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5D_mpio_spaces_read() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_mpio_spaces_span_write + * + * Purpose: MPI-IO function to write directly from app buffer to file. + * + * Return: non-negative on success, negative on failure. + * + * Programmer: KY + * Note: Don't call this funtion until you don't see this line. + * KY, 11/11/04 + + * + * Modifications: + * + * rky 980918 + * Added must_convert parameter to let caller know we can't optimize the xfer. + * + * QAK - 2002/04/02 + * Removed the must_convert parameter and move preconditions to + * H5S_mpio_opt_possible() routine + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_mpio_spaces_span_write(H5D_io_info_t *io_info, + size_t UNUSED nelmts, size_t elmt_size, + const H5S_t *file_space, const H5S_t *mem_space, + const void *buf) +{ + herr_t ret_value; + + FUNC_ENTER_NOAPI_NOFUNC(H5D_mpio_spaces_write); + + /*OKAY: CAST DISCARDS CONST QUALIFIER*/ + ret_value = H5D_mpio_spaces_span_xfer(io_info, elmt_size, file_space, + mem_space, (void*)buf, 1/*write*/); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5D_mpio_spaces_write() */ #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 499ceb0..68c8742 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -288,6 +288,18 @@ H5_DLL herr_t H5D_mpio_spaces_write(H5D_io_info_t *io_info, const struct H5S_t *file_space, const struct H5S_t *mem_space, const void *buf); +/* MPI-IO function to read directly from app buffer to file rky980813 */ +H5_DLL herr_t H5D_mpio_spaces_span_read(H5D_io_info_t *io_info, + size_t nelmts, size_t elmt_size, + const struct H5S_t *file_space, const struct H5S_t *mem_space, + void *buf/*out*/); + +/* MPI-IO function to write directly from app buffer to file rky980813 */ +H5_DLL herr_t H5D_mpio_spaces_span_write(H5D_io_info_t *io_info, + size_t nelmts, size_t elmt_size, + const struct H5S_t *file_space, const struct H5S_t *mem_space, + const void *buf); + /* MPI-IO function to check if a direct I/O transfer is possible between * memory and the file */ H5_DLL htri_t H5D_mpio_opt_possible(const H5D_t *file, const H5S_t *mem_space, diff --git a/src/H5Smpio.c b/src/H5Smpio.c index dab77ee..0f5b075 100644 --- a/src/H5Smpio.c +++ b/src/H5Smpio.c @@ -61,6 +61,18 @@ H5S_mpio_hyper_type( const H5S_t *space, size_t elmt_size, hsize_t *extra_offset, hbool_t *is_derived_type ); +static herr_t +H5S_mpio_span_hyper_type( const H5S_t *space, size_t elmt_size, + /* out: */ + MPI_Datatype *new_type, + size_t *count, + hsize_t *extra_offset, + hbool_t *is_derived_type ); + +static herr_t obtain_datatype(const hsize_t size[], + H5S_hyper_span_t* span,MPI_Datatype *span_type, + size_t elmt_size,int dimindex); + /*------------------------------------------------------------------------- * Function: H5S_mpio_all_type @@ -539,4 +551,291 @@ H5S_mpio_space_type( const H5S_t *space, size_t elmt_size, done: FUNC_LEAVE_NOAPI(ret_value); } + + + +/*------------------------------------------------------------------------- + * Function: H5S_mpio_space_span_ type + * + * Purpose: Translate an HDF5 dataspace selection into a general + MPI derivewd datatype built with span-tree. + * + * Currently handle only hyperslab and "all" selections. + * + * Return: non-negative on success, negative on failure. + * + * Outputs: *new_type the MPI type corresponding to the selection + * *count how many objects of the new_type in selection + * (useful if this is the buffer type for xfer) + * *extra_offset Number of bytes of offset within dataset + * *is_derived_type 0 if MPI primitive type, 1 if derived + * + * Programmer: KY + * + * Modifications: + * + * Quincey Koziol, June 18, 2002 + * Added 'extra_offset' parameter + * + *------------------------------------------------------------------------- + */ +herr_t +H5S_mpio_space_span_type( const H5S_t *space, size_t elmt_size, + /* out: */ + MPI_Datatype *new_type, + size_t *count, + hsize_t *extra_offset, + hbool_t *is_derived_type ) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT(H5S_mpio_space_span_type); + + /* Check args */ + assert (space); + + /* Creat MPI type based on the kind of selection */ + switch (H5S_GET_EXTENT_TYPE(space)) { + case H5S_NULL: + case H5S_SCALAR: + case H5S_SIMPLE: + switch(H5S_GET_SELECT_TYPE(space)) { + case H5S_SEL_NONE: + if ( H5S_mpio_none_type( space, elmt_size, + /* out: */ new_type, count, extra_offset, is_derived_type ) <0) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL,"couldn't convert \"all\" selection to MPI type"); + break; + + case H5S_SEL_ALL: + if ( H5S_mpio_all_type( space, elmt_size, + /* out: */ new_type, count, extra_offset, is_derived_type ) <0) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL,"couldn't convert \"all\" selection to MPI type"); + break; + + case H5S_SEL_POINTS: + /* not yet implemented */ + ret_value = FAIL; + break; + + case H5S_SEL_HYPERSLABS: + if(H5S_mpio_span_hyper_type( space, elmt_size, + /* out: */ new_type, count, extra_offset, is_derived_type )<0) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL,"couldn't convert \"all\" selection to MPI type"); + break; + + default: + assert("unknown selection type" && 0); + break; + } /* end switch */ + break; + + case H5S_COMPLEX: + /* not yet implemented */ + HGOTO_ERROR(H5E_DATASPACE, H5E_UNSUPPORTED, FAIL, "complex data spaces are not supported yet"); + + default: + assert("unknown data space type" && 0); + break; + } + +done: + FUNC_LEAVE_NOAPI(ret_value); +} + + +/* The following codes have been used by Kent to test + general collective derived datatype functionality. + It should NOT be called by other routines except with + macro #ifdef KENT #endif + Nov. 11th, 2004 */ + + +static herr_t +H5S_mpio_span_hyper_type( const H5S_t *space, size_t elmt_size, + /* out: */ + MPI_Datatype *new_type, + size_t *count, + hsize_t *extra_offset, + hbool_t *is_derived_type ){ + + MPI_Datatype span_type; + H5S_hyper_span_t *ospan; + H5S_hyper_span_info_t *odown; + hsize_t *size; + int rank; + herr_t ret_value = SUCCEED; + + + FUNC_ENTER_NOAPI_NOINIT(H5S_mpio_span_hyper_type); + + /* Check args */ + assert (space); + /* assert(sizeof(MPI_Aint) >= sizeof(elmt_size));?? */ + + /* Only for simple extent + rank = space->extent.u.simple.rank; + */ + rank = space->extent.rank; + + size = HDcalloc((size_t)rank,sizeof(hsize_t)); + if (0==elmt_size) + goto empty; + size = space->extent.size; + + + odown = space->select.sel_info.hslab->span_lst; + if(odown == NULL) + goto empty; + ospan = odown->head; + if(ospan == NULL) + goto empty; + + obtain_datatype(size,ospan,&span_type,elmt_size,rank); + + *new_type = span_type; + /* fill in the remaining return values */ + *count = 1; + *extra_offset = 0; + *is_derived_type = 1; + HDfree(size); + HGOTO_DONE(SUCCEED); + +empty: + /* special case: empty hyperslab */ + *new_type = MPI_BYTE; + *count = 0; + *extra_offset = 0; + *is_derived_type = 0; + +done: + FUNC_LEAVE_NOAPI(ret_value); + } + + +static herr_t obtain_datatype(const hsize_t size[], H5S_hyper_span_t* span,MPI_Datatype *span_type, + size_t elmt_size,int dimindex) { + + int innercount,outercount; + MPI_Datatype bas_type,temp_type,tempinner_type; + MPI_Datatype *inner_type; + int inner_blocklen; + MPI_Aint inner_disp; + int *blocklen; + MPI_Aint *disp; + MPI_Aint stride; + MPI_Aint extent,lb; + H5S_hyper_span_info_t *down; + H5S_hyper_span_t *tspan; + MPI_Aint sizeaint,sizedtype; + hsize_t total_lowd,total_lowd1; + int i; + + assert(span); + down = span->down; + tspan = span; + + outercount = 0; + while(tspan) { + tspan = tspan->next; + outercount ++; + } + + +#ifdef H5_HAVE_MPI2 + MPI_Type_extent(MPI_Aint,&sizeaint); + MPI_Type_extent(MPI_Datatype,&sizedtype); + blocklen = (int *)HDcalloc((size_t)outercount,sizeof(int)); + disp = (MPI_Aint *)HDcalloc((size_t)outercount,sizeaint); + inner_type = (MPI_Datatype *)HDcalloc((size_t)outercount,sizedtype); +#else + blocklen = (int *)HDcalloc((size_t)outercount,sizeof(int)); + disp = (MPI_Aint *)HDcalloc((size_t)outercount,sizeof(int)); + inner_type = (MPI_Datatype *)HDcalloc((size_t)outercount,sizeof(int)); +#endif + + + tspan = span; + outercount = 0; + + if(down == NULL){ + + if(dimindex > 1) printf("wrong area \n"); + MPI_Type_contiguous((int)elmt_size,MPI_BYTE,&bas_type); + MPI_Type_commit(&bas_type); + + while(tspan){ + disp[outercount] = (MPI_Aint)elmt_size * tspan->low; + blocklen[outercount] = tspan->nelem; + tspan = tspan->next; + outercount ++; + } + + MPI_Type_hindexed(outercount,blocklen,disp,bas_type,&span_type); + + } + else {/* dimindex is the rank of the dimension */ + + if(dimindex <2) printf("something is wrong \n"); + /* Calculate the total bytes of the lower dimension */ + total_lowd = 1; /* one dimension down */ + total_lowd1 = 1; /* two dimensions down */ + + for ( i = 0; i < dimindex-1; i++) + total_lowd = total_lowd * size[i]; + + for ( i = 0; i < dimindex-2; i++) + total_lowd1 = total_lowd1 * size[i]; + HDfprintf(stdout, " one dimension down size %Hu",total_lowd); + HDfprintf(stdout, " two dimension down size %Hu",total_lowd1); + while(tspan){ +/* Displacement should be in byte and should have dimension information */ +/* First using MPI Type vector to build derived data type for this span only */ +/* Need to calculate the disp in byte for this dimension. */ + /* Calculate the total bytes of the lower dimension */ + + disp[outercount] = tspan->low*total_lowd*elmt_size; + blocklen[outercount] = 1; + + /* generating inner derived datatype by using MPI_Type_hvector */ + obtain_datatype(size,tspan->down->head,&temp_type,elmt_size,dimindex-1); +/* inner_type[count] = temp_type; */ + +#ifdef H5_HAVE_MPI2 + MPI_Type_get_extent(temp_type,&lb,&extent); +#else + MPI_Type_lb(temp_type,&lb); + MPI_Type_extent(temp_type,&extent); +#endif + /* building the inner vector datatype */ + /* The following calculation of stride is wrong since stride is calculated + from the first element of the block to the first element of the next + block. */ + /*stride = total_lowd1 * (size[dimindex-1]*elmt_size-extent-lb);*/ + stride = total_lowd*elmt_size; + innercount = tspan->nelem; + MPI_Type_hvector(innercount,1,stride,temp_type,&tempinner_type); + MPI_Type_commit(&tempinner_type); + MPI_Type_free(&temp_type); + inner_type[outercount] = tempinner_type; + outercount ++; + tspan = tspan->next; + } + + /* building the whole vector datatype */ + MPI_Type_struct(outercount,blocklen,disp,inner_type,&span_type); + } + + MPI_Type_commit(span_type); + if(down == NULL) + MPI_Type_free(&bas_type); + else { + MPI_Type_free(inner_type); + } + + HDfree(inner_type); + HDfree(blocklen); + HDfree(disp); + +} + #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h index a0fdf09..61eb9a9 100644 --- a/src/H5Sprivate.h +++ b/src/H5Sprivate.h @@ -279,6 +279,15 @@ H5S_mpio_space_type( const H5S_t *space, size_t elmt_size, size_t *count, hsize_t *extra_offset, hbool_t *is_derived_type ); + +H5_DLL herr_t +H5S_mpio_space_span_type( const H5S_t *space, size_t elmt_size, + /* out: */ + MPI_Datatype *new_type, + size_t *count, + hsize_t *extra_offset, + hbool_t *is_derived_type ); + #endif /* H5_HAVE_PARALLEL */ #endif /* _H5Sprivate_H */ |