From 21a18a1cd2547b83c53395b4a6bb11499199f923 Mon Sep 17 00:00:00 2001 From: Richard Warren Date: Sat, 5 Oct 2019 10:18:17 -0400 Subject: Make the initial bug fixes to allow >2GB writes with Independent IO --- src/H5FDmpio.c | 38 ++++++++++++++++++++++++++++---------- src/H5Smpio.c | 27 +++++++++++++-------------- src/H5Sprivate.h | 5 +++++ testpar/t_file.c | 3 +-- 4 files changed, 47 insertions(+), 26 deletions(-) diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c index 71e9fe1..458d190 100644 --- a/src/H5FDmpio.c +++ b/src/H5FDmpio.c @@ -22,16 +22,17 @@ #include "H5FDdrvr_module.h" /* This source code file is part of the H5FD driver module */ -#include "H5private.h" /* Generic Functions */ +#include "H5private.h" /* Generic Functions */ #include "H5CXprivate.h" /* API Contexts */ -#include "H5Dprivate.h" /* Dataset functions */ -#include "H5Eprivate.h" /* Error handling */ -#include "H5Fprivate.h" /* File access */ -#include "H5FDprivate.h" /* File drivers */ -#include "H5FDmpi.h" /* MPI-based file drivers */ -#include "H5Iprivate.h" /* IDs */ -#include "H5MMprivate.h" /* Memory management */ +#include "H5Dprivate.h" /* Dataset functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* File access */ +#include "H5FDprivate.h" /* File drivers */ +#include "H5FDmpi.h" /* MPI-based file drivers */ +#include "H5Iprivate.h" /* IDs */ +#include "H5MMprivate.h" /* Memory management */ #include "H5Pprivate.h" /* Property lists */ +#include "H5Sprivate.h" /* Large MPIO data type creation */ #ifdef H5_HAVE_PARALLEL @@ -1324,6 +1325,7 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD__mpio_read() */ + /*------------------------------------------------------------------------- * Function: H5FD__mpio_write @@ -1366,6 +1368,7 @@ H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, #endif int size_i; hbool_t use_view_this_time = FALSE; + hbool_t derived_type = FALSE; H5FD_mpio_xfer_t xfer_mode; /* I/O transfer mode */ herr_t ret_value = SUCCEED; @@ -1391,8 +1394,6 @@ H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, if(H5FD_mpi_haddr_to_MPIOff(addr, &mpi_off) < 0) HGOTO_ERROR(H5E_INTERNAL, H5E_BADRANGE, FAIL, "can't convert from haddr to MPI off") size_i = (int)size; - if((hsize_t)size_i != size) - HGOTO_ERROR(H5E_INTERNAL, H5E_BADRANGE, FAIL, "can't convert from size to size_i") #ifdef H5FDmpio_DEBUG if(H5FD_mpio_Debug[(int)'w']) @@ -1430,6 +1431,20 @@ H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, */ mpi_off = 0; } /* end if */ + else if(size != (hsize_t)size_i) { + /* If HERE, then we need to work around the integer size limit + * of 2GB. The input size_t size variable cannot fit into an integer, + * but we can get around that limitation by creating a different datatype + * and then setting the integer size (or element count) to 1 when using + * the derived_type. + */ + + if (H5S_mpio_create_large_type(size, 0, MPI_BYTE, &buf_type) < 0) + HGOTO_ERROR(H5E_INTERNAL, H5E_CANTGET, FAIL, "can't create MPI-I/O datatype") + + derived_type = TRUE; + size_i = 1; + } /* Write the data. */ if(use_view_this_time) { @@ -1506,6 +1521,9 @@ H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, file->local_eof = addr + (haddr_t)bytes_written; done: + if(derived_type) { + MPI_Type_free(&buf_type); + } #ifdef H5FDmpio_DEBUG if(H5FD_mpio_Debug[(int)'t']) HDfprintf(stdout, "%s: Leaving, proc %d: ret_value = %d\n", FUNC, file->mpi_rank, ret_value ); diff --git a/src/H5Smpio.c b/src/H5Smpio.c index aeec566..287bde1 100644 --- a/src/H5Smpio.c +++ b/src/H5Smpio.c @@ -34,6 +34,7 @@ #include "H5FLprivate.h" /* Free Lists */ #include "H5MMprivate.h" /* Memory management */ #include "H5Spkg.h" /* Dataspaces */ +#include "H5Sprivate.h" /* H5S prototypes, etc. */ #include "H5VMprivate.h" /* Vector and array functions */ #ifdef H5_HAVE_PARALLEL @@ -44,7 +45,7 @@ #define H5S_MPIO_INITIAL_ALLOC_COUNT 256 #define TWO_GIG_LIMIT 2147483648 #ifndef H5S_MAX_MPI_COUNT -#define H5S_MAX_MPI_COUNT 536870911 /* (2^29)-1 */ +#define H5S_MAX_MPI_COUNT 1073741824 #endif @@ -88,8 +89,6 @@ static herr_t H5S__release_datatype(H5S_mpio_mpitype_list_t *type_list); static herr_t H5S__obtain_datatype(H5S_hyper_span_info_t *spans, const hsize_t *down, size_t elmt_size, const MPI_Datatype *elmt_type, MPI_Datatype *span_type, H5S_mpio_mpitype_list_t *type_list, uint64_t op_gen); -static herr_t H5S__mpio_create_large_type(hsize_t num_elements, MPI_Aint stride_bytes, - MPI_Datatype old_type, MPI_Datatype *new_type); /*****************************/ @@ -183,7 +182,7 @@ H5S__mpio_all_type(const H5S_t *space, size_t elmt_size, } /* end if */ else { /* Create a LARGE derived datatype for this transfer */ - if(H5S__mpio_create_large_type(total_bytes, 0, MPI_BYTE, new_type) < 0) + if(H5S_mpio_create_large_type(total_bytes, 0, MPI_BYTE, new_type) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "couldn't create a large datatype from the all selection") *count = 1; *is_derived_type = TRUE; @@ -824,7 +823,7 @@ if(H5DEBUG(S)) { } /* end if */ else /* Create the compound datatype for this operation (> 2GB) */ - if(H5S__mpio_create_large_type(elmt_size, 0, MPI_BYTE, &inner_type) < 0) + if(H5S_mpio_create_large_type(elmt_size, 0, MPI_BYTE, &inner_type) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "couldn't create a large inner datatype in hyper selection") /******************************************************* @@ -878,7 +877,7 @@ if(H5DEBUG(S)) * Again we need to check that the number of BLOCKS can fit into * a 32 bit integer */ if(bigio_count < d[i].block) { - if(H5S__mpio_create_large_type(d[i].block, 0, inner_type, &block_type) < 0) + if(H5S_mpio_create_large_type(d[i].block, 0, inner_type, &block_type) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "couldn't create a large block datatype in hyper selection") } /* end if */ else @@ -899,7 +898,7 @@ if(H5DEBUG(S)) * we call the large type creation function to handle that */ if(bigio_count < d[i].count) { - if(H5S__mpio_create_large_type(d[i].count, stride_in_bytes, block_type, &outer_type) < 0) + if(H5S_mpio_create_large_type(d[i].count, stride_in_bytes, block_type, &outer_type) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "couldn't create a large outer datatype in hyper selection") } /* end if */ /* otherwise a regular create_hvector will do */ @@ -1020,7 +1019,7 @@ H5S__mpio_span_hyper_type(const H5S_t *space, size_t elmt_size, HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) } /* end if */ else - if(H5S__mpio_create_large_type(elmt_size, 0, MPI_BYTE, &elmt_type) < 0) + if(H5S_mpio_create_large_type(elmt_size, 0, MPI_BYTE, &elmt_type) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "couldn't create a large element datatype in span_hyper selection") elmt_type_is_derived = TRUE; @@ -1203,7 +1202,7 @@ H5S__obtain_datatype(H5S_hyper_span_info_t *spans, const hsize_t *down, /* create the block type from elmt_type while checking the 32 bit int limit */ if(blocklen[u] > bigio_count) { - if(H5S__mpio_create_large_type(blocklen[u], 0, *elmt_type, &temp_type) < 0) + if(H5S_mpio_create_large_type(blocklen[u], 0, *elmt_type, &temp_type) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "couldn't create a large element datatype in span_hyper selection") } /* end if */ else @@ -1455,7 +1454,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5S__mpio_create_large_type + * Function: H5S_mpio_create_large_type * * Purpose: Create a large datatype of size larger than what a 32 bit integer * can hold. @@ -1468,8 +1467,8 @@ done: * *------------------------------------------------------------------------- */ -static herr_t -H5S__mpio_create_large_type(hsize_t num_elements, MPI_Aint stride_bytes, +herr_t +H5S_mpio_create_large_type(hsize_t num_elements, MPI_Aint stride_bytes, MPI_Datatype old_type, MPI_Datatype *new_type) { int num_big_types; /* num times the 2G datatype will be repeated */ @@ -1481,7 +1480,7 @@ H5S__mpio_create_large_type(hsize_t num_elements, MPI_Aint stride_bytes, MPI_Aint disp[2], old_extent; herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_STATIC + FUNC_ENTER_NOAPI_NOINIT /* Calculate how many Big MPI datatypes are needed to represent the buffer */ num_big_types = (int)(num_elements/bigio_count); @@ -1559,7 +1558,7 @@ H5S__mpio_create_large_type(hsize_t num_elements, MPI_Aint stride_bytes, done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5S__mpio_create_large_type() */ +} /* end H5S_mpio_create_large_type() */ #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h index 0a9d2e7..831c454 100644 --- a/src/H5Sprivate.h +++ b/src/H5Sprivate.h @@ -308,6 +308,11 @@ H5_DLL herr_t H5S_sel_iter_close(H5S_sel_iter_t *sel_iter); #ifdef H5_HAVE_PARALLEL H5_DLL hsize_t H5S_mpio_set_bigio_count(hsize_t new_count); +H5_DLL herr_t H5S_mpio_create_large_type(hsize_t num_elements, + MPI_Aint stride_bytes, + MPI_Datatype old_type, + MPI_Datatype *new_type); + H5_DLL herr_t H5S_mpio_space_type(const H5S_t *space, size_t elmt_size, /* out: */ MPI_Datatype *new_type, int *count, diff --git a/testpar/t_file.c b/testpar/t_file.c index 204095b..e5a4f8a 100644 --- a/testpar/t_file.c +++ b/testpar/t_file.c @@ -145,7 +145,7 @@ test_page_buffer_access(void) ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 1, (hsize_t)0); VRFY((ret == 0), ""); - ret = H5Pset_file_space_page_size(fcpl, sizeof(int)*100); + ret = H5Pset_file_space_page_size(fcpl, sizeof(int)*128); VRFY((ret == 0), ""); ret = H5Pset_page_buffer_size(fapl, sizeof(int)*100000, 0, 0); VRFY((ret == 0), ""); @@ -180,7 +180,6 @@ test_page_buffer_access(void) data[i] = -1; if(MAINPROCESS) { hid_t fapl_self = H5I_INVALID_HID; - fapl_self = create_faccess_plist(MPI_COMM_SELF, MPI_INFO_NULL, facc_type); ret = H5Pset_page_buffer_size(fapl_self, sizeof(int)*1000, 0, 0); -- cgit v0.12