diff options
Diffstat (limited to 'src')
106 files changed, 13528 insertions, 1478 deletions
diff --git a/src/.indent.pro b/src/.indent.pro index e0fda85..fa375f0 100644 --- a/src/.indent.pro +++ b/src/.indent.pro @@ -68,7 +68,7 @@ -T H5E_minor_t -T H5E_t -T H5F_create_t --T H5F_file_t +-T H5F_shared_t -T H5F_isop_t -T H5F_istore_key_t -T H5F_istore_ud1_t diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a0a10a5..26a3c26 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -8,6 +8,7 @@ set (H5_SOURCES ${HDF5_SRC_DIR}/H5.c ${HDF5_SRC_DIR}/H5checksum.c ${HDF5_SRC_DIR}/H5dbg.c + ${HDF5_SRC_DIR}/H5mpi.c ${HDF5_SRC_DIR}/H5system.c ${HDF5_SRC_DIR}/H5timer.c ${HDF5_SRC_DIR}/H5trace.c @@ -227,11 +228,14 @@ set (H5FD_SOURCES ${HDF5_SRC_DIR}/H5FDcore.c ${HDF5_SRC_DIR}/H5FDdirect.c ${HDF5_SRC_DIR}/H5FDfamily.c + ${HDF5_SRC_DIR}/H5FDhdfs.c ${HDF5_SRC_DIR}/H5FDint.c ${HDF5_SRC_DIR}/H5FDlog.c ${HDF5_SRC_DIR}/H5FDmpi.c ${HDF5_SRC_DIR}/H5FDmpio.c ${HDF5_SRC_DIR}/H5FDmulti.c + ${HDF5_SRC_DIR}/H5FDros3.c + ${HDF5_SRC_DIR}/H5FDs3comms.c ${HDF5_SRC_DIR}/H5FDsec2.c ${HDF5_SRC_DIR}/H5FDspace.c ${HDF5_SRC_DIR}/H5FDstdio.c @@ -243,11 +247,14 @@ set (H5FD_HDRS ${HDF5_SRC_DIR}/H5FDcore.h ${HDF5_SRC_DIR}/H5FDdirect.h ${HDF5_SRC_DIR}/H5FDfamily.h + ${HDF5_SRC_DIR}/H5FDhdfs.h ${HDF5_SRC_DIR}/H5FDlog.h ${HDF5_SRC_DIR}/H5FDmpi.h ${HDF5_SRC_DIR}/H5FDmpio.h ${HDF5_SRC_DIR}/H5FDmulti.h ${HDF5_SRC_DIR}/H5FDpublic.h + ${HDF5_SRC_DIR}/H5FDros3.h + ${HDF5_SRC_DIR}/H5FDs3comms.c ${HDF5_SRC_DIR}/H5FDsec2.h ${HDF5_SRC_DIR}/H5FDstdio.h ${HDF5_SRC_DIR}/H5FDwindows.h @@ -387,6 +394,15 @@ set (H5L_HDRS IDE_GENERATED_PROPERTIES ("H5L" "${H5L_HDRS}" "${H5L_SOURCES}" ) +set (H5M_SOURCES + ${HDF5_SRC_DIR}/H5M.c +) +set (H5M_HDRS + ${HDF5_SRC_DIR}/H5Mpublic.h +) +IDE_GENERATED_PROPERTIES ("H5M" "${H5M_HDRS}" "${H5M_SOURCES}") + + set (H5MF_SOURCES ${HDF5_SRC_DIR}/H5MF.c ${HDF5_SRC_DIR}/H5MFaggr.c @@ -477,6 +493,8 @@ set (H5P_SOURCES ${HDF5_SRC_DIR}/H5Pint.c ${HDF5_SRC_DIR}/H5Plapl.c ${HDF5_SRC_DIR}/H5Plcpl.c + ${HDF5_SRC_DIR}/H5Pmapl.c + ${HDF5_SRC_DIR}/H5Pmcpl.c ${HDF5_SRC_DIR}/H5Pocpl.c ${HDF5_SRC_DIR}/H5Pocpypl.c ${HDF5_SRC_DIR}/H5Pstrcpl.c @@ -710,6 +728,7 @@ set (common_SRCS ${H5HP_SOURCES} ${H5I_SOURCES} ${H5L_SOURCES} + ${H5M_SOURCES} ${H5MF_SOURCES} ${H5MM_SOURCES} ${H5MP_SOURCES} @@ -753,6 +772,7 @@ set (H5_PUBLIC_HEADERS ${H5HL_HDRS} ${H5I_HDRS} ${H5L_HDRS} + ${H5M_HDRS} ${H5MF_HDRS} ${H5MM_HDRS} ${H5MP_HDRS} @@ -839,6 +859,9 @@ set (H5_PRIVATE_HEADERS ${HDF5_SRC_DIR}/H5Lpkg.h ${HDF5_SRC_DIR}/H5Lprivate.h + ${HDF5_SRC_DIR}/H5Mpkg.h + ${HDF5_SRC_DIR}/H5Mprivate.h + ${HDF5_SRC_DIR}/H5MFprivate.h ${HDF5_SRC_DIR}/H5MMprivate.h @@ -967,13 +990,13 @@ if (LOCAL_BATCH_TEST) endif () endif () -if (NOT EXISTS ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c) +if (NOT EXISTS "${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c") add_executable (H5detect ${HDF5_SRC_DIR}/H5detect.c) target_include_directories (H5detect PRIVATE "${HDF5_SRC_DIR};${HDF5_BINARY_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>") target_compile_definitions(H5detect PUBLIC ${HDF_EXTRA_C_FLAGS} ${HDF_EXTRA_FLAGS}) TARGET_C_PROPERTIES (H5detect STATIC) target_link_libraries (H5detect - PRIVATE "$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_LIBRARIES}>" $<$<PLATFORM_ID:Windows>:ws2_32.lib> + PRIVATE "$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_LIBRARIES}>" $<$<OR:$<PLATFORM_ID:Windows>,$<PLATFORM_ID:MinGW>>:ws2_32.lib> PRIVATE $<$<PLATFORM_ID:Emscripten>:"-O0"> ) @@ -1014,7 +1037,7 @@ if (NOT EXISTS ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c) add_custom_command ( OUTPUT ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c ${HDF5_GENERATED_SOURCE_DIR}/gen_SRCS.stamp1 - COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR}$<TARGET_FILE:H5detect> + COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:H5detect> ARGS ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c COMMAND ${CMAKE_COMMAND} ARGS -E touch ${HDF5_GENERATED_SOURCE_DIR}/gen_SRCS.stamp1 @@ -1063,14 +1086,14 @@ target_include_directories (H5make_libsettings PRIVATE "${HDF5_SRC_DIR};${HDF5_B target_compile_definitions(H5make_libsettings PUBLIC ${HDF_EXTRA_C_FLAGS} ${HDF_EXTRA_FLAGS}) TARGET_C_PROPERTIES (H5make_libsettings STATIC) target_link_libraries (H5make_libsettings - PRIVATE "$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_LIBRARIES}>" $<$<PLATFORM_ID:Windows>:ws2_32.lib> + PRIVATE "$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_LIBRARIES}>" $<$<OR:$<PLATFORM_ID:Windows>,$<PLATFORM_ID:MinGW>>:ws2_32.lib> PRIVATE $<$<PLATFORM_ID:Emscripten>:"-O0"> ) add_custom_command ( OUTPUT ${HDF5_BINARY_DIR}/H5lib_settings.c ${HDF5_BINARY_DIR}/gen_SRCS.stamp2 - COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR}$<TARGET_FILE:H5make_libsettings> + COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:H5make_libsettings> ARGS ${HDF5_BINARY_DIR}/H5lib_settings.c COMMAND ${CMAKE_COMMAND} ARGS -E touch ${HDF5_GENERATED_SOURCE_DIR}/gen_SRCS.stamp2 @@ -1092,8 +1115,8 @@ if (BUILD_SHARED_LIBS) set_source_files_properties (${HDF5_BINARY_DIR}/shared/H5lib_settings.c PROPERTIES GENERATED TRUE) endif () -## all_packages="AC,B,B2,D,F,FA,FL,FS,HL,I,O,S,ST,T,Z" -#all_packages="AC,B2,CX,D,F,HL,I,O,S,ST,T,Z" +## all_packages="AC,B,B2,D,EA,F,FA,FL,FS,HL,I,O,S,ST,T,Z" +#all_packages="AC,B2,CX,D,EA,F,HL,I,O,S,ST,T,Z" option (HDF5_ENABLE_DEBUG_APIS "Turn on extra debug output in all packages" OFF) #----------------------------------------------------------------------------- @@ -1112,10 +1135,9 @@ if (NOT ONLY_SHARED_LIBS) PUBLIC ${HDF_EXTRA_C_FLAGS} ${HDF_EXTRA_FLAGS} - $<IF:$<CONFIG:Debug>,DEBUG,NDEBUG> PRIVATE $<$<BOOL:${HDF5_ENABLE_TRACE}>:H5_DEBUG_API> # Enable tracing of the API - $<$<BOOL:${HDF5_ENABLE_DEBUG_APIS}>:H5Z_DEBUG;H5T_DEBUG;H5ST_DEBUG;H5S_DEBUG;H5O_DEBUG;H5I_DEBUG;H5HL_DEBUG;H5F_DEBUG;H5D_DEBUG;H5B2_DEBUG;H5AC_DEBUG> + $<$<BOOL:${HDF5_ENABLE_DEBUG_APIS}>:H5Z_DEBUG;H5T_DEBUG;H5ST_DEBUG;H5S_DEBUG;H5O_DEBUG;H5I_DEBUG;H5HL_DEBUG;H5F_DEBUG;H5EA_DEBUG;H5D_DEBUG;H5B2_DEBUG;H5AC_DEBUG> ) TARGET_C_PROPERTIES (${HDF5_LIB_TARGET} STATIC) target_link_libraries (${HDF5_LIB_TARGET} @@ -1142,6 +1164,7 @@ if (BUILD_SHARED_LIBS) add_library (${HDF5_LIBSH_TARGET} SHARED ${common_SRCS} ${shared_gen_SRCS} ${H5_PUBLIC_HEADERS} ${H5_PRIVATE_HEADERS} ${H5_GENERATED_HEADERS}) target_include_directories (${HDF5_LIBSH_TARGET} PRIVATE "${HDF5_SRC_DIR};${HDF5_BINARY_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>" + PUBLIC "$<$<BOOL:${HDF5_ENABLE_HDFS}>:${HDFS_INCLUDE_DIR}>" INTERFACE "$<INSTALL_INTERFACE:$<INSTALL_PREFIX>/include>" ) target_compile_definitions(${HDF5_LIBSH_TARGET} @@ -1149,11 +1172,10 @@ if (BUILD_SHARED_LIBS) "H5_BUILT_AS_DYNAMIC_LIB" ${HDF_EXTRA_C_FLAGS} ${HDF_EXTRA_FLAGS} - $<IF:$<CONFIG:Debug>,DEBUG,NDEBUG> PRIVATE $<$<BOOL:${HDF5_ENABLE_THREADSAFE}>:H5_HAVE_THREADSAFE> $<$<BOOL:${HDF5_ENABLE_TRACE}>:H5_DEBUG_API> # Enable tracing of the API - $<$<BOOL:${HDF5_ENABLE_DEBUG_APIS}>:H5Z_DEBUG;H5T_DEBUG;H5ST_DEBUG;H5S_DEBUG;H5O_DEBUG;H5I_DEBUG;H5HL_DEBUG;H5F_DEBUG;H5D_DEBUG;H5B2_DEBUG;H5AC_DEBUG> + $<$<BOOL:${HDF5_ENABLE_DEBUG_APIS}>:H5Z_DEBUG;H5T_DEBUG;H5ST_DEBUG;H5S_DEBUG;H5O_DEBUG;H5I_DEBUG;H5HL_DEBUG;H5F_DEBUG;H5EA_DEBUG;H5D_DEBUG;H5B2_DEBUG;H5AC_DEBUG> ) TARGET_C_PROPERTIES (${HDF5_LIBSH_TARGET} SHARED) target_link_libraries (${HDF5_LIBSH_TARGET} @@ -166,6 +166,7 @@ H5_init_library(void) H5_debug_g.pkg[H5_PKG_HG].name = "hg"; H5_debug_g.pkg[H5_PKG_HL].name = "hl"; H5_debug_g.pkg[H5_PKG_I].name = "i"; + H5_debug_g.pkg[H5_PKG_M].name = "m"; H5_debug_g.pkg[H5_PKG_MF].name = "mf"; H5_debug_g.pkg[H5_PKG_MM].name = "mm"; H5_debug_g.pkg[H5_PKG_O].name = "o"; @@ -312,6 +313,7 @@ H5_term_library(void) pending += DOWN(A_top); pending += DOWN(D_top); pending += DOWN(G_top); + pending += DOWN(M_top); pending += DOWN(R_top); pending += DOWN(S_top); pending += DOWN(T_top); @@ -337,6 +339,7 @@ H5_term_library(void) pending += DOWN(A); pending += DOWN(D); pending += DOWN(G); + pending += DOWN(M); pending += DOWN(R); pending += DOWN(S); pending += DOWN(T); diff --git a/src/H5ACmpio.c b/src/H5ACmpio.c index b60b933..e4b81fa 100644 --- a/src/H5ACmpio.c +++ b/src/H5ACmpio.c @@ -821,9 +821,8 @@ H5AC__log_cleaned_entry(const H5AC_info_t *entry_ptr) { H5AC_t * cache_ptr; H5AC_aux_t * aux_ptr; - herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_PACKAGE + FUNC_ENTER_PACKAGE_NOERR /* Sanity check */ HDassert(entry_ptr); @@ -853,8 +852,7 @@ H5AC__log_cleaned_entry(const H5AC_info_t *entry_ptr) /* Decrement the dirty byte count */ aux_ptr->dirty_bytes -= entry_ptr->size; -done: - FUNC_LEAVE_NOAPI(ret_value) + FUNC_LEAVE_NOAPI(SUCCEED) } /* H5AC__log_cleaned_entry() */ diff --git a/src/H5Cmpio.c b/src/H5Cmpio.c index 84ec16c..e784487 100644 --- a/src/H5Cmpio.c +++ b/src/H5Cmpio.c @@ -222,7 +222,7 @@ H5C_apply_candidate_list(H5F_t * f, HDfprintf(stdout, "%s", tbl_buf); #endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - if(f->coll_md_write) { + if(f->shared->coll_md_write) { /* Sanity check */ HDassert(NULL == cache_ptr->coll_write_list); @@ -386,7 +386,7 @@ H5C_apply_candidate_list(H5F_t * f, HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "flush candidates failed") /* If we've deferred writing to do it collectively, take care of that now */ - if(f->coll_md_write) { + if(f->shared->coll_md_write) { /* Sanity check */ HDassert(cache_ptr->coll_write_list); @@ -457,8 +457,8 @@ done: * * Failure: H5I_INVALID_HID * - * Programmer: Robb Matzke - * Tuesday, February 3, 1998 + * Programmer: Robb Matzke + * Tuesday, February 3, 1998 * *------------------------------------------------------------------------- */ @@ -512,8 +512,8 @@ done: * * Failure: H5I_INVALID_HID * - * Programmer: Neil Fortner - * Wednesday, October 29, 2008 + * Programmer: Neil Fortner + * Wednesday, October 29, 2008 * *------------------------------------------------------------------------- */ @@ -610,9 +610,9 @@ done: /*------------------------------------------------------------------------- - * Function: H5Diterate + * Function: H5Diterate * - * Purpose: This routine iterates over all the elements selected in a memory + * Purpose: This routine iterates over all the elements selected in a memory * buffer. The callback function is called once for each element selected * in the dataspace. The selection in the dataspace is modified so * that any elements already iterated over are removed from the selection @@ -659,11 +659,11 @@ done: * indicating failure. The iterator can be restarted at the next * element. * - * Return: Returns the return value of the last operator if it was non-zero, + * Return: Returns the return value of the last operator if it was non-zero, * or zero if all elements were processed. Otherwise returns a * negative value. * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Friday, June 11, 1999 * *------------------------------------------------------------------------- @@ -706,16 +706,16 @@ done: /*------------------------------------------------------------------------- - * Function: H5Dvlen_reclaim + * Function: H5Dvlen_reclaim * - * Purpose: Frees the buffers allocated for storing variable-length data + * Purpose: Frees the buffers allocated for storing variable-length data * in memory. Only frees the VL data in the selection defined in the * dataspace. The dataset transfer property list is required to find the * correct allocation/free methods for the VL data in the buffer. * - * Return: Non-negative on success, negative on failure + * Return: Non-negative on success, negative on failure * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Thursday, June 10, 1999 * *------------------------------------------------------------------------- @@ -756,9 +756,9 @@ done: /*------------------------------------------------------------------------- - * Function: H5Dvlen_get_buf_size + * Function: H5Dvlen_get_buf_size * - * Purpose: This routine checks the number of bytes required to store the VL + * Purpose: This routine checks the number of bytes required to store the VL * data from the dataset, using the space_id for the selection in the * dataset on disk and the type_id for the memory representation of the * VL data, in memory. The *size value is modified according to how many @@ -772,9 +772,9 @@ done: * Kinda kludgy, but easier than the other method of trying to figure out * the sizes without actually reading the data in... - QAK * - * Return: Non-negative on success, negative on failure + * Return: Non-negative on success, negative on failure * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Wednesday, August 11, 1999 * *------------------------------------------------------------------------- @@ -1095,3 +1095,150 @@ done: FUNC_LEAVE_API(ret_value); } /* H5Dget_chunk_storage_size() */ + +/*------------------------------------------------------------------------- + * Function: H5Dget_num_chunks + * + * Purpose: Retrieves the number of chunks that have non-empty intersection + * with a specified selection. + * + * Note: Currently, this function only gets the number of all written + * chunks, regardless the dataspace. + * + * Parameters: + * hid_t dset_id; IN: Chunked dataset ID + * hid_t fspace_id; IN: File dataspace ID + * hsize_t *nchunks; OUT: Number of non-empty chunks + * + * Return: Non-negative on success, negative on failure + * + * Programmer: Binh-Minh Ribler + * May 2019 (HDFFV-10677) + * + *------------------------------------------------------------------------- + */ +herr_t +H5Dget_num_chunks(hid_t dset_id, hid_t fspace_id, hsize_t *nchunks) +{ + H5VL_object_t *vol_obj = NULL; /* Dataset for this operation */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "ii*h", dset_id, fspace_id, nchunks); + + /* Check arguments */ + if(NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(dset_id, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid dataset identifier") + if(NULL == nchunks) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid argument (null)") + + /* Get the number of written chunks */ + if(H5VL_dataset_optional(vol_obj, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, H5VL_NATIVE_DATASET_GET_NUM_CHUNKS, fspace_id, nchunks) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "Can't get number of chunks") + +done: + FUNC_LEAVE_API(ret_value); +} /* H5Dget_num_chunks() */ + + +/*------------------------------------------------------------------------- + * Function: H5Dget_chunk_info + * + * Purpose: Retrieves information about a chunk specified by its index. + * + * Parameters: + * hid_t dset_id; IN: Chunked dataset ID + * hid_t fspace_id; IN: File dataspace ID + * hsize_t index; IN: Index of written chunk + * hsize_t *offset OUT: Logical position of the chunk’s + * first element in the dataspace + * unsigned *filter_mask OUT: Mask for identifying the filters in use + * haddr_t *addr OUT: Address of the chunk + * hsize_t *size OUT: Size of the chunk + * + * Return: Non-negative on success, negative on failure + * + * Programmer: Binh-Minh Ribler + * May 2019 (HDFFV-10677) + * + *------------------------------------------------------------------------- + */ +herr_t +H5Dget_chunk_info(hid_t dset_id, hid_t fspace_id, hsize_t chk_index, hsize_t *offset, unsigned *filter_mask, haddr_t *addr, hsize_t *size) +{ + H5VL_object_t *vol_obj = NULL; /* Dataset for this operation */ + hsize_t nchunks = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE7("e", "iih*h*Iu*a*h", dset_id, fspace_id, chk_index, offset, + filter_mask, addr, size); + + /* Check arguments */ + if(NULL == offset && NULL == filter_mask && NULL == addr && NULL == size) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid arguments, must have at least one non-null output argument") + if(NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(dset_id, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid dataset identifier") + + /* Get the number of written chunks to check range */ + if(H5VL_dataset_optional(vol_obj, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, H5VL_NATIVE_DATASET_GET_NUM_CHUNKS, fspace_id, &nchunks) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "Can't get number of chunks") + + /* Check range for chunk index */ + if(chk_index >= nchunks) + HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "chunk index is out of range") + + /* Call private function to get the chunk info given the chunk's index */ + if(H5VL_dataset_optional(vol_obj, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_IDX, fspace_id, chk_index, offset, filter_mask, addr, size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "Can't get chunk info by index") + +done: + FUNC_LEAVE_API(ret_value); +} /* H5Dget_chunk_info() */ + + +/*------------------------------------------------------------------------- + * Function: H5Dget_chunk_info_by_coord + * + * Purpose: Retrieves information about a chunk specified by its logical + * coordinates. + * + * Parameters: + * hid_t dset_id; IN: Chunked dataset ID + * hsize_t *offset IN: Logical position of the chunk’s + * first element in the dataspace + * unsigned *filter_mask OUT: Mask for identifying the filters in use + * haddr_t *addr OUT: Address of the chunk + * hsize_t *size OUT: Size of the chunk + * + * Return: Non-negative on success, negative on failure + * + * Programmer: Binh-Minh Ribler + * May 2019 (HDFFV-10677) + * + *------------------------------------------------------------------------- + */ +herr_t +H5Dget_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, unsigned *filter_mask, haddr_t *addr, hsize_t *size) +{ + H5VL_object_t *vol_obj = NULL; /* Dataset for this operation */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE5("e", "i*h*Iu*a*h", dset_id, offset, filter_mask, addr, size); + + /* Check arguments */ + if(NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(dset_id, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid dataset identifier") + if(NULL == filter_mask && NULL == addr && NULL == size) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid arguments, must have at least one non-null output argument") + if(NULL == offset) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid argument (null)") + + /* Call private function to get the chunk info given the chunk's index */ + if(H5VL_dataset_optional(vol_obj, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_COORD, offset, filter_mask, addr, size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "Can't get chunk info by its logical coordinates") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Dget_chunk_info_by_coord() */ diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index 7a07225..e3bbd59 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -216,6 +216,18 @@ typedef struct H5D_chunk_readvv_ud_t { const H5D_t *dset; /* Dataset to operate on */ } H5D_chunk_readvv_ud_t; +typedef struct H5D_chunk_info_iter_ud_t { + hsize_t scaled[H5O_LAYOUT_NDIMS]; /* Logical offset of the chunk */ + hsize_t ndims; /* Number of dimensions in the dataset */ + uint32_t nbytes; /* Size of stored data in the chunk */ + unsigned filter_mask; /* Excluded filters */ + haddr_t chunk_addr; /* Address of the chunk in file */ + hsize_t chunk_idx; /* Chunk index, where the iteration needs to stop */ + hsize_t curr_idx; /* Current index, where the iteration is */ + unsigned idx_hint; /* Index of chunk in cache, if present */ + hbool_t found; /* Whether the chunk was found */ +} H5D_chunk_info_iter_ud_t; + /* Callback info for file selection iteration */ typedef struct H5D_chunk_file_iter_ud_t { H5D_chunk_map_t *fm; /* File->memory chunk mapping info */ @@ -252,6 +264,11 @@ static herr_t H5D__chunk_flush(H5D_t *dset); static herr_t H5D__chunk_io_term(const H5D_chunk_map_t *fm); static herr_t H5D__chunk_dest(H5D_t *dset); +/* Chunk query operation callbacks */ +static int H5D__get_num_chunks_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata); +static int H5D__get_chunk_info_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata); +static int H5D__get_chunk_info_by_coord_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata); + /* "Nonexistent" layout operation callback */ static ssize_t H5D__nonexistent_readvv(const H5D_io_info_t *io_info, @@ -393,7 +410,6 @@ H5D__chunk_direct_write(const H5D_t *dset, uint32_t filters, hsize_t *offset, H5D_chk_idx_info_t idx_info; /* Chunked index info */ hsize_t scaled[H5S_MAX_RANK]; /* Scaled coordinates for this chunk */ hbool_t need_insert = FALSE; /* Whether the chunk needs to be inserted into the index */ - H5D_io_info_t io_info; /* to hold the dset info */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE_TAG(dset->oloc.addr) @@ -401,13 +417,17 @@ H5D__chunk_direct_write(const H5D_t *dset, uint32_t filters, hsize_t *offset, /* Sanity checks */ HDassert(layout->type == H5D_CHUNKED); - io_info.dset = dset; - /* Allocate dataspace and initialize it if it hasn't been. */ - if(!H5D__chunk_is_space_alloc(&layout->storage)) + if(!H5D__chunk_is_space_alloc(&layout->storage)) { + H5D_io_info_t io_info; /* to hold the dset info */ + + io_info.dset = dset; + io_info.f_sh = H5F_SHARED(dset->oloc.file); + /* Allocate storage */ if(H5D__alloc_storage(&io_info, H5D_ALLOC_WRITE, FALSE, NULL) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize storage") + } /* end if */ /* Calculate the index of this chunk */ H5VM_chunk_scaled(dset->shared->ndims, offset, layout->u.chunk.dim, scaled); @@ -467,7 +487,7 @@ H5D__chunk_direct_write(const H5D_t *dset, uint32_t filters, hsize_t *offset, } /* end if */ /* Write the data to the file */ - if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.chunk_block.offset, data_size, buf) < 0) + if(H5F_shared_block_write(H5F_SHARED(dset->oloc.file), H5FD_MEM_DRAW, udata.chunk_block.offset, data_size, buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to write raw data to file") /* Insert the chunk record into the index */ @@ -570,9 +590,9 @@ H5D__chunk_direct_read(const H5D_t *dset, hsize_t *offset, uint32_t* filters, HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "chunk address isn't defined") /* Read the chunk data into the supplied buffer */ - if(H5F_block_read(dset->oloc.file, H5FD_MEM_DRAW, udata.chunk_block.offset, udata.chunk_block.length, buf) < 0) + if(H5F_shared_block_read(H5F_SHARED(dset->oloc.file), H5FD_MEM_DRAW, udata.chunk_block.offset, udata.chunk_block.length, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "unable to read raw data chunk") - + /* Return the filter mask */ *filters = udata.filter_mask; @@ -1245,7 +1265,7 @@ H5D__chunk_io_init(const H5D_io_info_t *io_info, const H5D_type_info_t *type_inf } /* end else */ /* Build the memory selection for each chunk */ - if(sel_hyper_flag && H5S_select_shape_same(file_space, mem_space) == TRUE) { + if(sel_hyper_flag && H5S_SELECT_SHAPE_SAME(file_space, mem_space) == TRUE) { /* Reset chunk template information */ fm->mchunk_tmpl = NULL; @@ -1811,7 +1831,7 @@ H5D__create_chunk_file_map_hyper(H5D_chunk_map_t *fm, const H5D_io_info_t while(sel_points) { /* Check for intersection of current chunk and file selection */ /* (Casting away const OK - QAK) */ - if(TRUE == H5S_hyper_intersect_block((H5S_t *)fm->file_space, coords, end)) { + if(TRUE == H5S_SELECT_INTERSECT_BLOCK(fm->file_space, coords, end)) { H5D_chunk_info_t *new_chunk_info; /* chunk information to insert into skip list */ hssize_t schunk_points; /* Number of elements in chunk selection */ @@ -2030,12 +2050,8 @@ H5D__create_chunk_mem_map_hyper(const H5D_chunk_map_t *fm) /* Sanity check */ HDassert(H5S_SEL_HYPERSLABS == chunk_sel_type); - /* Release the current selection */ - if(H5S_SELECT_RELEASE(chunk_info->mspace) < 0) - HGOTO_ERROR(H5E_DATASPACE, H5E_CANTRELEASE, FAIL, "unable to release selection") - /* Copy the file chunk's selection */ - if(H5S_select_copy(chunk_info->mspace, chunk_info->fspace, FALSE) < 0) + if(H5S_SELECT_COPY(chunk_info->mspace, chunk_info->fspace, FALSE) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCOPY, FAIL, "unable to copy selection") /* Compute the adjustment for this chunk */ @@ -3336,7 +3352,7 @@ H5D__chunk_flush_entry(const H5D_t *dset, H5D_rdcc_ent_t *ent, hbool_t reset) /* Write the data to the file */ HDassert(H5F_addr_defined(udata.chunk_block.offset)); H5_CHECK_OVERFLOW(udata.chunk_block.length, hsize_t, size_t); - if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.chunk_block.offset, (size_t)udata.chunk_block.length, buf) < 0) + if(H5F_shared_block_write(H5F_SHARED(dset->oloc.file), H5FD_MEM_DRAW, udata.chunk_block.offset, (size_t)udata.chunk_block.length, buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to write raw data to file") /* Insert the chunk record into the index */ @@ -3807,7 +3823,7 @@ H5D__chunk_lock(const H5D_io_info_t *io_info, H5D_chunk_ud_t *udata, * size in memory, so allocate memory big enough. */ if(NULL == (chunk = H5D__chunk_mem_alloc(my_chunk_alloc, (udata->new_unfilt_chunk ? old_pline : pline)))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for raw data chunk") - if(H5F_block_read(dset->oloc.file, H5FD_MEM_DRAW, chunk_addr, my_chunk_alloc, chunk) < 0) + if(H5F_shared_block_read(H5F_SHARED(dset->oloc.file), H5FD_MEM_DRAW, chunk_addr, my_chunk_alloc, chunk) < 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, NULL, "unable to read raw data chunk") if(old_pline && old_pline->nused) { @@ -4537,7 +4553,7 @@ H5D__chunk_allocate(const H5D_io_info_t *io_info, hbool_t full_overwrite, hsize_ } /* end if */ else { #endif /* H5_HAVE_PARALLEL */ - if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.chunk_block.offset, chunk_size, *fill_buf) < 0) + if(H5F_shared_block_write(H5F_SHARED(dset->oloc.file), H5FD_MEM_DRAW, udata.chunk_block.offset, chunk_size, *fill_buf) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to write raw data to file") #ifdef H5_HAVE_PARALLEL } /* end else */ @@ -4922,7 +4938,7 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_info_t *chunk_info, HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set transfer mode") /* Low-level write (collective) */ - if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, (haddr_t)0, (blocks) ? (size_t)1 : (size_t)0, fill_buf) < 0) + if(H5F_shared_block_write(H5F_SHARED(dset->oloc.file), H5FD_MEM_DRAW, (haddr_t)0, (blocks) ? (size_t)1 : (size_t)0, fill_buf) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to write raw data to file") /* Barrier so processes don't race ahead */ @@ -6957,3 +6973,373 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__chunk_format_convert() */ + +/*------------------------------------------------------------------------- + * Function: H5D__get_num_chunks_cb + * + * Purpose: Callback function that increments the number of written + * chunks in the dataset. + * + * Note: Currently, this function only gets the number of all written + * chunks, regardless the dataspace. + * + * Return: H5_ITER_CONT + * + * Programmer: Binh-Minh Ribler + * June 2019 (HDFFV-10677) + * + *------------------------------------------------------------------------- + */ +static int +H5D__get_num_chunks_cb(const H5D_chunk_rec_t H5_ATTR_UNUSED *chunk_rec, void *_udata) +{ + hsize_t *num_chunks = (hsize_t *)_udata; + int ret_value = H5_ITER_CONT; /* Callback return value */ + + FUNC_ENTER_STATIC_NOERR + + HDassert(num_chunks); + + (*num_chunks)++; + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5D__get_num_chunks_cb() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__get_num_chunks + * + * Purpose: Gets the number of written chunks in a dataset. + * + * Note: Currently, this function only gets the number of all written + * chunks, regardless the dataspace. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Binh-Minh Ribler + * June 2019 (HDFFV-10677) + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__get_num_chunks(const H5D_t *dset, const H5S_t H5_ATTR_UNUSED *space, hsize_t *nchunks) +{ + H5D_chk_idx_info_t idx_info; /* Chunked index info */ + hsize_t num_chunks = 0; /* Number of written chunks */ + H5D_rdcc_ent_t *ent; /* Cache entry */ + const H5D_rdcc_t *rdcc = NULL; /* Raw data chunk cache */ + const H5O_layout_t *layout; /* Dataset layout */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_TAG(dset->oloc.addr) + + HDassert(dset); + HDassert(dset->shared); + HDassert(space); + HDassert(nchunks); + + layout = &(dset->shared->layout); /* Dataset layout */ + rdcc = &(dset->shared->cache.chunk); /* raw data chunk cache */ + HDassert(rdcc); + + /* Search for cached chunks that haven't been written out */ + for(ent = rdcc->head; ent; ent = ent->next) + /* Flush the chunk out to disk, to make certain the size is correct later */ + if(H5D__chunk_flush_entry(dset, ent, FALSE) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot flush indexed storage buffer") + + /* Compose chunked index info struct */ + idx_info.f = dset->oloc.file; + idx_info.pline = &dset->shared->dcpl_cache.pline; + idx_info.layout = &dset->shared->layout.u.chunk; + idx_info.storage = &dset->shared->layout.storage.u.chunk; + + /* If the dataset is not written, number of chunks will be 0 */ + if(!H5F_addr_defined(idx_info.storage->idx_addr)) { + *nchunks = 0; + } + else { + /* Iterate over the allocated chunks */ + if((dset->shared->layout.storage.u.chunk.ops->iterate)(&idx_info, H5D__get_num_chunks_cb, &num_chunks) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to retrieve allocated chunk information from index") + *nchunks = num_chunks; + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__get_num_chunks() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__get_chunk_info_cb + * + * Purpose: Get the chunk info of the queried chunk, given by its index. + * + * Return: Success: H5_ITER_CONT or H5_ITER_STOP + * H5_ITER_STOP indicates the queried chunk is found + * Failure: Negative (H5_ITER_ERROR) + * + * Programmer: Binh-Minh Ribler + * June 2019 (HDFFV-10677) + * + *------------------------------------------------------------------------- + */ +static int +H5D__get_chunk_info_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) +{ + H5D_chunk_info_iter_ud_t *chunk_info = (H5D_chunk_info_iter_ud_t *)_udata; + hsize_t ii = 0; /* Dimension index */ + int ret_value = H5_ITER_CONT; /* Callback return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check args */ + HDassert(chunk_rec); + HDassert(chunk_info); + + /* If this is the queried chunk, retrieve its info and stop iterating */ + if (chunk_info->curr_idx == chunk_info->chunk_idx) { + chunk_info->filter_mask = chunk_rec->filter_mask; + chunk_info->chunk_addr = chunk_rec->chunk_addr; + chunk_info->nbytes = chunk_rec->nbytes; + for (ii = 0; ii < chunk_info->ndims; ii++) + chunk_info->scaled[ii] = chunk_rec->scaled[ii]; + chunk_info->found = TRUE; + + /* Stop iterating */ + ret_value = H5_ITER_STOP; + } + /* Go to the next chunk */ + else + chunk_info->curr_idx++; + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5D__get_chunk_info_cb() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__get_chunk_info + * + * Purpose: Iterate over the chunks in the dataset to get the info + * of the desired chunk. + * + * Note: Currently, the domain of the index in this function is of all + * the written chunks, regardless the dataspace. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Binh-Minh Ribler + * June 2019 (HDFFV-10677) + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__get_chunk_info(const H5D_t *dset, const H5S_t H5_ATTR_UNUSED *space, hsize_t chk_index, hsize_t *offset, unsigned *filter_mask, haddr_t *addr, hsize_t *size) +{ + H5D_chk_idx_info_t idx_info; /* Chunked index info */ + H5D_chunk_info_iter_ud_t udata; /* User data for callback */ + const H5D_rdcc_t *rdcc = NULL; /* Raw data chunk cache */ + H5D_rdcc_ent_t *ent; /* Cache entry index */ + hsize_t ii = 0; /* Dimension index */ + herr_t ret_value = SUCCEED;/* Return value */ + + FUNC_ENTER_PACKAGE_TAG(dset->oloc.addr) + + HDassert(dset); + HDassert(dset->shared); + HDassert(space); + + /* Get the raw data chunk cache */ + rdcc = &(dset->shared->cache.chunk); + HDassert(rdcc); + + /* Search for cached chunks that haven't been written out */ + for(ent = rdcc->head; ent; ent = ent->next) + /* Flush the chunk out to disk, to make certain the size is correct later */ + if(H5D__chunk_flush_entry(dset, ent, FALSE) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot flush indexed storage buffer") + + /* Compose chunked index info struct */ + idx_info.f = dset->oloc.file; + idx_info.pline = &dset->shared->dcpl_cache.pline; + idx_info.layout = &dset->shared->layout.u.chunk; + idx_info.storage = &dset->shared->layout.storage.u.chunk; + + /* Set addr & size for when dset is not written or queried chunk is not found */ + if (addr) + *addr = HADDR_UNDEF; + if (size) + *size = 0; + + /* If the chunk is written, get its info, otherwise, return without error */ + if(H5F_addr_defined(idx_info.storage->idx_addr)) { + /* Initialize before iteration */ + udata.chunk_idx = chk_index; + udata.curr_idx = 0; + udata.ndims = dset->shared->ndims; + udata.nbytes = 0; + udata.filter_mask = 0; + udata.chunk_addr = HADDR_UNDEF; + udata.found = FALSE; + + /* Iterate over the allocated chunks */ + if((dset->shared->layout.storage.u.chunk.ops->iterate)(&idx_info, H5D__get_chunk_info_cb, &udata) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to retrieve allocated chunk information from index") + + /* Obtain requested info if the chunk is found */ + if(udata.found) { + if(filter_mask) + *filter_mask = udata.filter_mask; + if(addr) + *addr = udata.chunk_addr; + if(size) + *size = udata.nbytes; + if(offset) + for(ii = 0; ii < udata.ndims; ii++) + offset[ii] = udata.scaled[ii] * dset->shared->layout.u.chunk.dim[ii]; + } + } /* end if H5F_addr_defined */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__get_chunk_info() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__get_chunk_info_by_coord_cb + * + * Purpose: Get the chunk info of the desired chunk, given its offset + * coordinates. + * + * Return: Success: H5_ITER_CONT or H5_ITER_STOP + * Failure: Negative (H5_ITER_ERROR) + * + * Programmer: Binh-Minh Ribler + * June 2019 (HDFFV-10677) + * + *------------------------------------------------------------------------- + */ +static int +H5D__get_chunk_info_by_coord_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) +{ + hsize_t ii; + H5D_chunk_info_iter_ud_t *chunk_info = (H5D_chunk_info_iter_ud_t *)_udata; + hbool_t different = FALSE; /* TRUE when a scaled value pair mismatch */ + int ret_value = H5_ITER_CONT; /* Callback return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check args */ + HDassert(chunk_rec); + HDassert(chunk_info); + + /* Going through the scaled, stop when a mismatch is found */ + for (ii = 0; ii < chunk_info->ndims && !different; ii++) + if (chunk_info->scaled[ii] != chunk_rec->scaled[ii]) + different = TRUE; + + /* Same scaled coords means the chunk is found, copy the chunk info */ + if (!different) { + chunk_info->nbytes = chunk_rec->nbytes; + chunk_info->filter_mask = chunk_rec->filter_mask; + chunk_info->chunk_addr = chunk_rec->chunk_addr; + chunk_info->found = TRUE; + + /* Stop iterating */ + ret_value = H5_ITER_STOP; + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5D__get_chunk_info_by_coord_cb() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__get_chunk_info_by_coord + * + * Purpose: Iterate over the chunks in the dataset to get the info + * of the desired chunk, given by its offset coordinates. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Binh-Minh Ribler + * June 2019 (HDFFV-10677) + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__get_chunk_info_by_coord(const H5D_t *dset, const hsize_t *offset, unsigned* filter_mask, haddr_t *addr, hsize_t *size) +{ + const H5O_layout_t *layout = NULL; /* Dataset layout */ + const H5D_rdcc_t *rdcc = NULL; /* Raw data chunk cache */ + H5D_rdcc_ent_t *ent; /* Cache entry index */ + H5D_chk_idx_info_t idx_info; /* Chunked index info */ + H5D_chunk_info_iter_ud_t udata; /* User data for callback */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_TAG(dset->oloc.addr) + + /* Check args */ + HDassert(dset); + HDassert(dset->shared); + HDassert(offset); + + /* Get dataset layout and raw data chunk cache */ + layout = &(dset->shared->layout); + rdcc = &(dset->shared->cache.chunk); + HDassert(layout); + HDassert(rdcc); + HDassert(H5D_CHUNKED == layout->type); + + /* Search for cached chunks that haven't been written out */ + for(ent = rdcc->head; ent; ent = ent->next) + /* Flush the chunk out to disk, to make certain the size is correct later */ + if(H5D__chunk_flush_entry(dset, ent, FALSE) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot flush indexed storage buffer") + + /* Set addr & size for when dset is not written or queried chunk is not found */ + if (addr) + *addr = HADDR_UNDEF; + if (size) + *size = 0; + + /* Compose chunked index info struct */ + idx_info.f = dset->oloc.file; + idx_info.pline = &dset->shared->dcpl_cache.pline; + idx_info.layout = &dset->shared->layout.u.chunk; + idx_info.storage = &dset->shared->layout.storage.u.chunk; + + /* If the dataset is not written, return without errors */ + if(H5F_addr_defined(idx_info.storage->idx_addr)) { + /* Calculate the scaled of this chunk */ + H5VM_chunk_scaled(dset->shared->ndims, offset, layout->u.chunk.dim, udata.scaled); + udata.scaled[dset->shared->ndims] = 0; + + /* Initialize before iteration */ + udata.ndims = dset->shared->ndims; + udata.nbytes = 0; + udata.filter_mask = 0; + udata.chunk_addr = HADDR_UNDEF; + udata.found = FALSE; + + /* Iterate over the allocated chunks to find the requested chunk */ + if((dset->shared->layout.storage.u.chunk.ops->iterate)(&idx_info, H5D__get_chunk_info_by_coord_cb, &udata) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to retrieve information of the chunk by its scaled coordinates") + + /* Obtain requested info if the chunk is found */ + if (udata.found) { + if(filter_mask) + *filter_mask = udata.filter_mask; + if(addr) + *addr = udata.chunk_addr; + if(size) + *size = udata.nbytes; + } + } /* end if H5F_addr_defined */ + +done: + FUNC_LEAVE_NOAPI_TAG(ret_value) +} /* end H5D__get_chunk_info_by_coord() */ + diff --git a/src/H5Dcontig.c b/src/H5Dcontig.c index 8733633..c9f9fc2 100644 --- a/src/H5Dcontig.c +++ b/src/H5Dcontig.c @@ -58,7 +58,7 @@ /* Callback info for sieve buffer readvv operation */ typedef struct H5D_contig_readvv_sieve_ud_t { - H5F_t *file; /* File for dataset */ + H5F_shared_t *f_sh; /* Shared file for dataset */ H5D_rdcdc_t *dset_contig; /* Cached information about contiguous data */ const H5D_contig_storage_t *store_contig; /* Contiguous storage info for this I/O operation */ unsigned char *rbuf; /* Pointer to buffer to fill */ @@ -66,14 +66,14 @@ typedef struct H5D_contig_readvv_sieve_ud_t { /* Callback info for [plain] readvv operation */ typedef struct H5D_contig_readvv_ud_t { - H5F_t *file; /* File for dataset */ + H5F_shared_t *f_sh; /* Shared file for dataset */ haddr_t dset_addr; /* Address of dataset */ unsigned char *rbuf; /* Pointer to buffer to fill */ } H5D_contig_readvv_ud_t; /* Callback info for sieve buffer writevv operation */ typedef struct H5D_contig_writevv_sieve_ud_t { - H5F_t *file; /* File for dataset */ + H5F_shared_t *f_sh; /* Shared file for dataset */ H5D_rdcdc_t *dset_contig; /* Cached information about contiguous data */ const H5D_contig_storage_t *store_contig; /* Contiguous storage info for this I/O operation */ const unsigned char *wbuf; /* Pointer to buffer to write */ @@ -81,7 +81,7 @@ typedef struct H5D_contig_writevv_sieve_ud_t { /* Callback info for [plain] writevv operation */ typedef struct H5D_contig_writevv_ud_t { - H5F_t *file; /* File for dataset */ + H5F_shared_t *f_sh; /* Shared file for dataset */ haddr_t dset_addr; /* Address of dataset */ const unsigned char *wbuf; /* Pointer to buffer to write */ } H5D_contig_writevv_ud_t; @@ -718,7 +718,7 @@ H5D__contig_readvv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, void *_udata) { H5D_contig_readvv_sieve_ud_t *udata = (H5D_contig_readvv_sieve_ud_t *)_udata; /* User data for H5VM_opvv() operator */ - H5F_t *file = udata->file; /* File for dataset */ + H5F_shared_t *f_sh = udata->f_sh; /* Shared file for dataset */ H5D_rdcdc_t *dset_contig = udata->dset_contig; /* Cached information about contiguous data */ const H5D_contig_storage_t *store_contig = udata->store_contig; /* Contiguous storage info for this I/O operation */ unsigned char *buf; /* Pointer to buffer to fill */ @@ -750,7 +750,7 @@ H5D__contig_readvv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, if(NULL == dset_contig->sieve_buf) { /* Check if we can actually hold the I/O request in the sieve buffer */ if(len > dset_contig->sieve_buf_size) { - if(H5F_block_read(file, H5FD_MEM_DRAW, addr, len, buf) < 0) + if(H5F_shared_block_read(f_sh, H5FD_MEM_DRAW, addr, len, buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "block read failed") } /* end if */ else { @@ -762,7 +762,7 @@ H5D__contig_readvv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, dset_contig->sieve_loc = addr; /* Make certain we don't read off the end of the file */ - if(HADDR_UNDEF == (rel_eoa = H5F_get_eoa(file, H5FD_MEM_DRAW))) + if(HADDR_UNDEF == (rel_eoa = H5F_shared_get_eoa(f_sh, H5FD_MEM_DRAW))) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to determine file size") /* Set up the buffer parameters */ @@ -773,7 +773,7 @@ H5D__contig_readvv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, H5_CHECKED_ASSIGN(dset_contig->sieve_size, size_t, min, hsize_t); /* Read the new sieve buffer */ - if(H5F_block_read(file, H5FD_MEM_DRAW, dset_contig->sieve_loc, dset_contig->sieve_size, dset_contig->sieve_buf) < 0) + if(H5F_shared_block_read(f_sh, H5FD_MEM_DRAW, dset_contig->sieve_loc, dset_contig->sieve_size, dset_contig->sieve_buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "block read failed") /* Grab the data out of the buffer (must be first piece of data in buffer ) */ @@ -804,7 +804,7 @@ H5D__contig_readvv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, /* Flush the sieve buffer, if it's dirty */ if(dset_contig->sieve_dirty) { /* Write to file */ - if(H5F_block_write(file, H5FD_MEM_DRAW, sieve_start, sieve_size, dset_contig->sieve_buf) < 0) + if(H5F_shared_block_write(f_sh, H5FD_MEM_DRAW, sieve_start, sieve_size, dset_contig->sieve_buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "block write failed") /* Reset sieve buffer dirty flag */ @@ -813,7 +813,7 @@ H5D__contig_readvv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, } /* end if */ /* Read directly into the user's buffer */ - if(H5F_block_read(file, H5FD_MEM_DRAW, addr, len, buf) < 0) + if(H5F_shared_block_read(f_sh, H5FD_MEM_DRAW, addr, len, buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "block read failed") } /* end if */ /* Element size fits within the buffer size */ @@ -821,7 +821,7 @@ H5D__contig_readvv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, /* Flush the sieve buffer if it's dirty */ if(dset_contig->sieve_dirty) { /* Write to file */ - if(H5F_block_write(file, H5FD_MEM_DRAW, sieve_start, sieve_size, dset_contig->sieve_buf) < 0) + if(H5F_shared_block_write(f_sh, H5FD_MEM_DRAW, sieve_start, sieve_size, dset_contig->sieve_buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "block write failed") /* Reset sieve buffer dirty flag */ @@ -832,7 +832,7 @@ H5D__contig_readvv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, dset_contig->sieve_loc = addr; /* Make certain we don't read off the end of the file */ - if(HADDR_UNDEF == (rel_eoa = H5F_get_eoa(file, H5FD_MEM_DRAW))) + if(HADDR_UNDEF == (rel_eoa = H5F_shared_get_eoa(f_sh, H5FD_MEM_DRAW))) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to determine file size") /* Only need this when resizing sieve buffer */ @@ -847,7 +847,7 @@ H5D__contig_readvv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, H5_CHECKED_ASSIGN(dset_contig->sieve_size, size_t, min, hsize_t); /* Read the new sieve buffer */ - if(H5F_block_read(file, H5FD_MEM_DRAW, dset_contig->sieve_loc, dset_contig->sieve_size, dset_contig->sieve_buf) < 0) + if(H5F_shared_block_read(f_sh, H5FD_MEM_DRAW, dset_contig->sieve_loc, dset_contig->sieve_size, dset_contig->sieve_buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "block read failed") /* Grab the data out of the buffer (must be first piece of data in buffer ) */ @@ -885,7 +885,7 @@ H5D__contig_readvv_cb(hsize_t dst_off, hsize_t src_off, size_t len, void *_udata FUNC_ENTER_STATIC /* Write data */ - if(H5F_block_read(udata->file, H5FD_MEM_DRAW, (udata->dset_addr + dst_off), + if(H5F_shared_block_read(udata->f_sh, H5FD_MEM_DRAW, (udata->dset_addr + dst_off), len, (udata->rbuf + src_off)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "block write failed") @@ -931,11 +931,11 @@ H5D__contig_readvv(const H5D_io_info_t *io_info, HDassert(mem_off_arr); /* Check if data sieving is enabled */ - if(H5F_HAS_FEATURE(io_info->dset->oloc.file, H5FD_FEAT_DATA_SIEVE)) { + if(H5F_SHARED_HAS_FEATURE(io_info->f_sh, H5FD_FEAT_DATA_SIEVE)) { H5D_contig_readvv_sieve_ud_t udata; /* User data for H5VM_opvv() operator */ /* Set up user data for H5VM_opvv() */ - udata.file = io_info->dset->oloc.file; + udata.f_sh = io_info->f_sh; udata.dset_contig = &(io_info->dset->shared->cache.contig); udata.store_contig = &(io_info->store->contig); udata.rbuf = (unsigned char *)io_info->u.rbuf; @@ -950,7 +950,7 @@ H5D__contig_readvv(const H5D_io_info_t *io_info, H5D_contig_readvv_ud_t udata; /* User data for H5VM_opvv() operator */ /* Set up user data for H5VM_opvv() */ - udata.file = io_info->dset->oloc.file; + udata.f_sh = io_info->f_sh; udata.dset_addr = io_info->store->contig.dset_addr; udata.rbuf = (unsigned char *)io_info->u.rbuf; @@ -983,7 +983,7 @@ H5D__contig_writevv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, void *_udata) { H5D_contig_writevv_sieve_ud_t *udata = (H5D_contig_writevv_sieve_ud_t *)_udata; /* User data for H5VM_opvv() operator */ - H5F_t *file = udata->file; /* File for dataset */ + H5F_shared_t *f_sh = udata->f_sh; /* Shared file for dataset */ H5D_rdcdc_t *dset_contig = udata->dset_contig; /* Cached information about contiguous data */ const H5D_contig_storage_t *store_contig = udata->store_contig; /* Contiguous storage info for this I/O operation */ const unsigned char *buf; /* Pointer to buffer to fill */ @@ -1015,7 +1015,7 @@ H5D__contig_writevv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, if(NULL == dset_contig->sieve_buf) { /* Check if we can actually hold the I/O request in the sieve buffer */ if(len > dset_contig->sieve_buf_size) { - if(H5F_block_write(file, H5FD_MEM_DRAW, addr, len, buf) < 0) + if(H5F_shared_block_write(f_sh, H5FD_MEM_DRAW, addr, len, buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "block write failed") } /* end if */ else { @@ -1031,7 +1031,7 @@ H5D__contig_writevv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, dset_contig->sieve_loc = addr; /* Make certain we don't read off the end of the file */ - if(HADDR_UNDEF == (rel_eoa = H5F_get_eoa(file, H5FD_MEM_DRAW))) + if(HADDR_UNDEF == (rel_eoa = H5F_shared_get_eoa(f_sh, H5FD_MEM_DRAW))) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to determine file size") /* Set up the buffer parameters */ @@ -1044,7 +1044,7 @@ H5D__contig_writevv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, /* Check if there is any point in reading the data from the file */ if(dset_contig->sieve_size > len) { /* Read the new sieve buffer */ - if(H5F_block_read(file, H5FD_MEM_DRAW, dset_contig->sieve_loc, dset_contig->sieve_size, dset_contig->sieve_buf) < 0) + if(H5F_shared_block_read(f_sh, H5FD_MEM_DRAW, dset_contig->sieve_loc, dset_contig->sieve_size, dset_contig->sieve_buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "block read failed") } /* end if */ @@ -1084,7 +1084,7 @@ H5D__contig_writevv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, /* Flush the sieve buffer, if it's dirty */ if(dset_contig->sieve_dirty) { /* Write to file */ - if(H5F_block_write(file, H5FD_MEM_DRAW, sieve_start, sieve_size, dset_contig->sieve_buf) < 0) + if(H5F_shared_block_write(f_sh, H5FD_MEM_DRAW, sieve_start, sieve_size, dset_contig->sieve_buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "block write failed") /* Reset sieve buffer dirty flag */ @@ -1097,7 +1097,7 @@ H5D__contig_writevv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, } /* end if */ /* Write directly from the user's buffer */ - if(H5F_block_write(file, H5FD_MEM_DRAW, addr, len, buf) < 0) + if(H5F_shared_block_write(f_sh, H5FD_MEM_DRAW, addr, len, buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "block write failed") } /* end if */ /* Element size fits within the buffer size */ @@ -1132,7 +1132,7 @@ H5D__contig_writevv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, /* Flush the sieve buffer if it's dirty */ if(dset_contig->sieve_dirty) { /* Write to file */ - if(H5F_block_write(file, H5FD_MEM_DRAW, sieve_start, sieve_size, dset_contig->sieve_buf) < 0) + if(H5F_shared_block_write(f_sh, H5FD_MEM_DRAW, sieve_start, sieve_size, dset_contig->sieve_buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "block write failed") /* Reset sieve buffer dirty flag */ @@ -1143,7 +1143,7 @@ H5D__contig_writevv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, dset_contig->sieve_loc = addr; /* Make certain we don't read off the end of the file */ - if(HADDR_UNDEF == (rel_eoa = H5F_get_eoa(file, H5FD_MEM_DRAW))) + if(HADDR_UNDEF == (rel_eoa = H5F_shared_get_eoa(f_sh, H5FD_MEM_DRAW))) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to determine file size") /* Only need this when resizing sieve buffer */ @@ -1160,7 +1160,7 @@ H5D__contig_writevv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, /* Check if there is any point in reading the data from the file */ if(dset_contig->sieve_size > len) { /* Read the new sieve buffer */ - if(H5F_block_read(file, H5FD_MEM_DRAW, dset_contig->sieve_loc, dset_contig->sieve_size, dset_contig->sieve_buf) < 0) + if(H5F_shared_block_read(f_sh, H5FD_MEM_DRAW, dset_contig->sieve_loc, dset_contig->sieve_size, dset_contig->sieve_buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "block read failed") } /* end if */ @@ -1200,7 +1200,7 @@ H5D__contig_writevv_cb(hsize_t dst_off, hsize_t src_off, size_t len, void *_udat FUNC_ENTER_STATIC /* Write data */ - if(H5F_block_write(udata->file, H5FD_MEM_DRAW, (udata->dset_addr + dst_off), len, (udata->wbuf + src_off)) < 0) + if(H5F_shared_block_write(udata->f_sh, H5FD_MEM_DRAW, (udata->dset_addr + dst_off), len, (udata->wbuf + src_off)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "block write failed") done: @@ -1245,11 +1245,11 @@ H5D__contig_writevv(const H5D_io_info_t *io_info, HDassert(mem_off_arr); /* Check if data sieving is enabled */ - if(H5F_HAS_FEATURE(io_info->dset->oloc.file, H5FD_FEAT_DATA_SIEVE)) { + if(H5F_SHARED_HAS_FEATURE(io_info->f_sh, H5FD_FEAT_DATA_SIEVE)) { H5D_contig_writevv_sieve_ud_t udata; /* User data for H5VM_opvv() operator */ /* Set up user data for H5VM_opvv() */ - udata.file = io_info->dset->oloc.file; + udata.f_sh = io_info->f_sh; udata.dset_contig = &(io_info->dset->shared->cache.contig); udata.store_contig = &(io_info->store->contig); udata.wbuf = (const unsigned char *)io_info->u.wbuf; @@ -1264,7 +1264,7 @@ H5D__contig_writevv(const H5D_io_info_t *io_info, H5D_contig_writevv_ud_t udata; /* User data for H5VM_opvv() operator */ /* Set up user data for H5VM_opvv() */ - udata.file = io_info->dset->oloc.file; + udata.f_sh = io_info->f_sh; udata.dset_addr = io_info->store->contig.dset_addr; udata.wbuf = (const unsigned char *)io_info->u.wbuf; diff --git a/src/H5Dint.c b/src/H5Dint.c index 1e9c6ff..ada542e 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -2932,7 +2932,7 @@ H5D__flush_sieve_buf(H5D_t *dataset) HDassert(dataset->shared->layout.type != H5D_COMPACT); /* We should never have a sieve buffer for compact storage */ /* Write dirty data sieve buffer to file */ - if(H5F_block_write(dataset->oloc.file, H5FD_MEM_DRAW, dataset->shared->cache.contig.sieve_loc, + if(H5F_shared_block_write(H5F_SHARED(dataset->oloc.file), H5FD_MEM_DRAW, dataset->shared->cache.contig.sieve_loc, dataset->shared->cache.contig.sieve_size, dataset->shared->cache.contig.sieve_buf) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "block write failed") diff --git a/src/H5Dio.c b/src/H5Dio.c index 9343b80..1e6e70d 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -490,7 +490,7 @@ H5D__read(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, * Note that in general, this requires us to touch up the memory buffer as * well. */ - if(TRUE == H5S_select_shape_same(mem_space, file_space) && + if(TRUE == H5S_SELECT_SHAPE_SAME(mem_space, file_space) && H5S_GET_EXTENT_NDIMS(mem_space) != H5S_GET_EXTENT_NDIMS(file_space)) { void *adj_buf = NULL; /* Pointer to the location in buf corresponding */ /* to the beginning of the projected mem space. */ @@ -725,7 +725,7 @@ H5D__write(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, * Note that in general, this requires us to touch up the memory buffer * as well. */ - if(TRUE == H5S_select_shape_same(mem_space, file_space) && + if(TRUE == H5S_SELECT_SHAPE_SAME(mem_space, file_space) && H5S_GET_EXTENT_NDIMS(mem_space) != H5S_GET_EXTENT_NDIMS(file_space)) { void *adj_buf = NULL; /* Pointer to the location in buf corresponding */ /* to the beginning of the projected mem space. */ @@ -857,6 +857,7 @@ H5D__ioinfo_init(H5D_t *dset, const H5D_type_info_t *type_info, /* Set up "normal" I/O fields */ io_info->dset = dset; + io_info->f_sh = H5F_SHARED(dset->oloc.file); io_info->store = store; /* Set I/O operations to initial values */ diff --git a/src/H5Dmpio.c b/src/H5Dmpio.c index 467b1b7..0dbbe9f 100644 --- a/src/H5Dmpio.c +++ b/src/H5Dmpio.c @@ -457,7 +457,7 @@ H5D__mpio_select_read(const H5D_io_info_t *io_info, const H5D_type_info_t H5_ATT FUNC_ENTER_PACKAGE H5_CHECK_OVERFLOW(mpi_buf_count, hsize_t, size_t); - if(H5F_block_read(io_info->dset->oloc.file, H5FD_MEM_DRAW, store_contig->dset_addr, (size_t)mpi_buf_count, io_info->u.rbuf) < 0) + if(H5F_shared_block_read(io_info->f_sh, H5FD_MEM_DRAW, store_contig->dset_addr, (size_t)mpi_buf_count, io_info->u.rbuf) < 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "can't finish collective parallel read") done: @@ -487,7 +487,7 @@ H5D__mpio_select_write(const H5D_io_info_t *io_info, const H5D_type_info_t H5_AT /*OKAY: CAST DISCARDS CONST QUALIFIER*/ H5_CHECK_OVERFLOW(mpi_buf_count, hsize_t, size_t); - if(H5F_block_write(io_info->dset->oloc.file, H5FD_MEM_DRAW, store_contig->dset_addr, (size_t)mpi_buf_count, io_info->u.wbuf) < 0) + if(H5F_shared_block_write(io_info->f_sh, H5FD_MEM_DRAW, store_contig->dset_addr, (size_t)mpi_buf_count, io_info->u.wbuf) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "can't finish collective parallel write") done: @@ -3153,7 +3153,7 @@ H5D__filtered_collective_chunk_entry_io(H5D_filtered_collective_io_info_t *chunk if(H5CX_set_io_xfer_mode(H5FD_MPIO_INDEPENDENT) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set MPI-I/O transfer mode") - if(H5F_block_read(io_info->dset->oloc.file, H5FD_MEM_DRAW, chunk_entry->chunk_states.chunk_current.offset, + if(H5F_shared_block_read(io_info->f_sh, H5FD_MEM_DRAW, chunk_entry->chunk_states.chunk_current.offset, chunk_entry->chunk_states.new_chunk.length, chunk_entry->buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read raw data chunk") diff --git a/src/H5Dnone.c b/src/H5Dnone.c index be421b6..9346220 100644 --- a/src/H5Dnone.c +++ b/src/H5Dnone.c @@ -242,9 +242,9 @@ H5D__none_idx_iterate(const H5D_chk_idx_info_t *idx_info, unsigned u; /* Local index variable */ int curr_dim; /* Current rank */ hsize_t idx; /* Array index of chunk */ - int ret_value = -1; /* Return value */ + int ret_value = H5_ITER_CONT; /* Return value */ - FUNC_ENTER_STATIC_NOERR + FUNC_ENTER_STATIC /* Sanity checks */ HDassert(idx_info); @@ -266,34 +266,35 @@ H5D__none_idx_iterate(const H5D_chk_idx_info_t *idx_info, HDassert(ndims > 0); /* Iterate over all the chunks in the dataset's dataspace */ - for(u = 0; u < idx_info->layout->nchunks; u++) { - /* Calculate the index of this chunk */ - idx = H5VM_array_offset_pre(ndims, idx_info->layout->max_down_chunks, chunk_rec.scaled); - - /* Calculate the address of the chunk */ - chunk_rec.chunk_addr = idx_info->storage->idx_addr + idx * idx_info->layout->size; - - /* Make "generic chunk" callback */ - if((ret_value = (*chunk_cb)(&chunk_rec, chunk_udata)) < 0) - HERROR(H5E_DATASET, H5E_CALLBACK, "failure in generic chunk iterator callback"); - - /* Update coordinates of chunk in dataset */ - curr_dim = (int)(ndims - 1); - while(curr_dim >= 0) { - /* Increment coordinate in current dimension */ - chunk_rec.scaled[curr_dim]++; - - /* Check if we went off the end of the current dimension */ - if(chunk_rec.scaled[curr_dim] >= idx_info->layout->chunks[curr_dim]) { - /* Reset coordinate & move to next faster dimension */ - chunk_rec.scaled[curr_dim] = 0; - curr_dim--; - } /* end if */ - else - break; - } /* end while */ + for(u = 0; u < idx_info->layout->nchunks && ret_value == H5_ITER_CONT; u++) { + /* Calculate the index of this chunk */ + idx = H5VM_array_offset_pre(ndims, idx_info->layout->max_down_chunks, chunk_rec.scaled); + + /* Calculate the address of the chunk */ + chunk_rec.chunk_addr = idx_info->storage->idx_addr + idx * idx_info->layout->size; + + /* Make "generic chunk" callback */ + if((ret_value = (*chunk_cb)(&chunk_rec, chunk_udata)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CALLBACK, H5_ITER_ERROR, "failure in generic chunk iterator callback") + + /* Update coordinates of chunk in dataset */ + curr_dim = (int)(ndims - 1); + while(curr_dim >= 0) { + /* Increment coordinate in current dimension */ + chunk_rec.scaled[curr_dim]++; + + /* Check if we went off the end of the current dimension */ + if(chunk_rec.scaled[curr_dim] >= idx_info->layout->chunks[curr_dim]) { + /* Reset coordinate & move to next faster dimension */ + chunk_rec.scaled[curr_dim] = 0; + curr_dim--; + } /* end if */ + else + break; + } /* end while */ } /* end for */ +done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__none_idx_iterate() */ diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 638c8a5..723acf9 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -47,11 +47,13 @@ /* [Simple] Macro to construct a H5D_io_info_t from it's components */ #define H5D_BUILD_IO_INFO_WRT(io_info, ds, str, buf) \ (io_info)->dset = ds; \ + (io_info)->f_sh = H5F_SHARED((ds)->oloc.file); \ (io_info)->store = str; \ (io_info)->op_type = H5D_IO_OP_WRITE; \ (io_info)->u.wbuf = buf #define H5D_BUILD_IO_INFO_RD(io_info, ds, str, buf) \ (io_info)->dset = ds; \ + (io_info)->f_sh = H5F_SHARED((ds)->oloc.file); \ (io_info)->store = str; \ (io_info)->op_type = H5D_IO_OP_READ; \ (io_info)->u.rbuf = buf @@ -207,6 +209,8 @@ typedef enum H5D_io_op_type_t { typedef struct H5D_io_info_t { const H5D_t *dset; /* Pointer to dataset being operated on */ +/* QAK: Delete the f_sh field when oloc has a shared file pointer? */ + H5F_shared_t *f_sh; /* Pointer to shared file struct that dataset is within */ #ifdef H5_HAVE_PARALLEL MPI_Comm comm; /* MPI communicator for file */ hbool_t using_mpi_vfd; /* Whether the file is using an MPI-based VFD */ @@ -565,6 +569,9 @@ H5_DLL herr_t H5D__get_space_status(const H5D_t *dset, H5D_space_status_t *alloc H5_DLL herr_t H5D__alloc_storage(const H5D_io_info_t *io_info, H5D_time_alloc_t time_alloc, hbool_t full_overwrite, hsize_t old_dim[]); H5_DLL herr_t H5D__get_storage_size(const H5D_t *dset, hsize_t *storage_size); H5_DLL herr_t H5D__get_chunk_storage_size(H5D_t *dset, const hsize_t *offset, hsize_t *storage_size); +H5_DLL herr_t H5D__get_num_chunks(const H5D_t *dset, const H5S_t *space, hsize_t *nchunks); +H5_DLL herr_t H5D__get_chunk_info(const H5D_t *dset, const H5S_t *space, hsize_t chk_idx, hsize_t *coord, unsigned *filter_mask, haddr_t *offset, hsize_t *size); +H5_DLL herr_t H5D__get_chunk_info_by_coord(const H5D_t *dset, const hsize_t *coord, unsigned *filter_mask, haddr_t *addr, hsize_t *size); H5_DLL haddr_t H5D__get_offset(const H5D_t *dset); H5_DLL void *H5D__vlen_get_buf_size_alloc(size_t size, void *info); H5_DLL herr_t H5D__vlen_get_buf_size(void *elem, hid_t type_id, unsigned ndim, const hsize_t *point, void *op_data); diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index 63f1a24..7234d16 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -142,6 +142,9 @@ H5_DLL hid_t H5Dget_create_plist(hid_t dset_id); H5_DLL hid_t H5Dget_access_plist(hid_t dset_id); H5_DLL hsize_t H5Dget_storage_size(hid_t dset_id); H5_DLL herr_t H5Dget_chunk_storage_size(hid_t dset_id, const hsize_t *offset, hsize_t *chunk_bytes); +H5_DLL herr_t H5Dget_num_chunks(hid_t dset_id, hid_t fspace_id, hsize_t *nchunks); +H5_DLL herr_t H5Dget_chunk_info_by_coord(hid_t dset_id, const hsize_t *coord, unsigned *filter_mask, haddr_t *addr, hsize_t *size); +H5_DLL herr_t H5Dget_chunk_info(hid_t dset_id, hid_t fspace_id, hsize_t chk_idx, hsize_t *coord, unsigned *filter_mask, haddr_t *addr, hsize_t *size); H5_DLL haddr_t H5Dget_offset(hid_t dset_id); H5_DLL herr_t H5Dread(hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id, hid_t file_space_id, hid_t plist_id, void *buf/*out*/); @@ -877,7 +877,7 @@ H5EA_close(H5EA_t *ea)) if(pending_delete) { H5EA_hdr_t *hdr; /* Another pointer to extensible array header */ -#ifndef NDEBUG +#ifdef H5EA_DEBUG { unsigned hdr_status = 0; /* Header's status in the metadata cache */ @@ -890,7 +890,7 @@ H5EA_close(H5EA_t *ea)) HDassert(hdr_status & H5AC_ES__IS_PINNED); HDassert(!(hdr_status & H5AC_ES__IS_PROTECTED)); } -#endif /* NDEBUG */ +#endif /* H5EA_DEBUG */ /* Lock the array header into memory */ /* (OK to pass in NULL for callback context, since we know the header must be in the cache) */ @@ -987,19 +987,25 @@ END_FUNC(PRIV) /* end H5EA_delete() */ * Purpose: Iterate over the elements of an extensible array * (copied and modified from FA_iterate() in H5FA.c) * - * Return: SUCCEED/FAIL + * Return: H5_ITER_CONT/H5_ITER_ERROR * * Programmer: Vailin Choi; Feb 2015 * + * Modification: + * Prototype changed (HDFFV-10661) + * - herr_t to int + * - SUCCEED/FAIL to H5_ITER_CONT/H5_ITER_ERROR + * June 6, 2019 -BMR *------------------------------------------------------------------------- */ BEGIN_FUNC(PRIV, ERR, -herr_t, SUCCEED, FAIL, +int, H5_ITER_CONT, H5_ITER_ERROR, H5EA_iterate(H5EA_t *ea, H5EA_operator_t op, void *udata)) /* Local variables */ uint8_t *elmt = NULL; hsize_t u; + int cb_ret = H5_ITER_CONT; /* Return value from callback */ /* * Check arguments. @@ -1013,9 +1019,7 @@ H5EA_iterate(H5EA_t *ea, H5EA_operator_t op, void *udata)) H5E_THROW(H5E_CANTALLOC, "memory allocation failed for extensible array element") /* Iterate over all elements in array */ - for(u = 0; u < ea->hdr->stats.stored.max_idx_set; u++) { - int cb_ret; /* Return value from callback */ - + for(u = 0; u < ea->hdr->stats.stored.max_idx_set && cb_ret == H5_ITER_CONT; u++) { /* Get array element */ if(H5EA_get(ea, u, elmt) < 0) H5E_THROW(H5E_CANTGET, "unable to delete fixed array") diff --git a/src/H5EAcache.c b/src/H5EAcache.c index da67e6b..d55a2b4 100644 --- a/src/H5EAcache.c +++ b/src/H5EAcache.c @@ -581,11 +581,7 @@ H5EA__cache_hdr_notify(H5AC_notify_action_t action, void *_thing)) break; default: -#ifdef NDEBUG H5E_THROW(H5E_BADVALUE, "unknown action from metadata cache") -#else /* NDEBUG */ - HDassert(0 && "Unknown action?!?"); -#endif /* NDEBUG */ } /* end switch */ } /* end if */ else @@ -978,11 +974,7 @@ H5EA__cache_iblock_notify(H5AC_notify_action_t action, void *_thing)) break; default: -#ifdef NDEBUG H5E_THROW(H5E_BADVALUE, "unknown action from metadata cache") -#else /* NDEBUG */ - HDassert(0 && "Unknown action?!?"); -#endif /* NDEBUG */ } /* end switch */ CATCH @@ -1390,11 +1382,7 @@ H5EA__cache_sblock_notify(H5AC_notify_action_t action, void *_thing)) break; default: -#ifdef NDEBUG H5E_THROW(H5E_BADVALUE, "unknown action from metadata cache") -#else /* NDEBUG */ - HDassert(0 && "Unknown action?!?"); -#endif /* NDEBUG */ } /* end switch */ CATCH @@ -1803,11 +1791,7 @@ H5EA__cache_dblock_notify(H5AC_notify_action_t action, void *_thing)) break; default: -#ifdef NDEBUG H5E_THROW(H5E_BADVALUE, "unknown action from metadata cache") -#else /* NDEBUG */ - HDassert(0 && "Unknown action?!?"); -#endif /* NDEBUG */ } /* end switch */ CATCH @@ -2184,11 +2168,7 @@ H5EA__cache_dblk_page_notify(H5AC_notify_action_t action, void *_thing)) break; default: -#ifdef NDEBUG H5E_THROW(H5E_BADVALUE, "unknown action from metadata cache") -#else /* NDEBUG */ - HDassert(0 && "Unknown action?!?"); -#endif /* NDEBUG */ } /* end switch */ CATCH diff --git a/src/H5EAhdr.c b/src/H5EAhdr.c index ec40298..8a2e1d1 100644 --- a/src/H5EAhdr.c +++ b/src/H5EAhdr.c @@ -49,10 +49,10 @@ /* Local Macros */ /****************/ -#ifndef NDEBUG +#ifdef H5EA_DEBUG /* Max. # of bits for max. nelmts index */ #define H5EA_MAX_NELMTS_IDX_MAX 64 -#endif /* NDEBUG */ +#endif /* H5EA_DEBUG */ /* # of elements in a data block for a particular super block */ #define H5EA_SBLK_DBLK_NELMTS(s, m) \ @@ -366,7 +366,7 @@ H5EA__hdr_create(H5F_t *f, const H5EA_create_t *cparam, void *ctx_udata)) HDassert(f); HDassert(cparam); -#ifndef NDEBUG +#ifdef H5EA_DEBUG { unsigned sblk_idx; /* Super block index for first "actual" super block */ size_t dblk_nelmts; /* Number of data block elements */ @@ -398,7 +398,7 @@ H5EA__hdr_create(H5F_t *f, const H5EA_create_t *cparam, void *ctx_udata)) if(cparam->max_dblk_page_nelmts_bits > cparam->max_nelmts_bits) H5E_THROW(H5E_BADVALUE, "max. # of elements per data block page bits must be <= max. # of elements bits") } -#endif /* NDEBUG */ +#endif /* H5EA_DEBUG */ /* Allocate space for the shared information */ if(NULL == (hdr = H5EA__hdr_alloc(f))) @@ -724,7 +724,7 @@ H5EA__hdr_delete(H5EA_hdr_t *hdr)) HDassert(hdr); HDassert(!hdr->file_rc); -#ifndef NDEBUG +#ifdef H5EA_DEBUG { unsigned hdr_status = 0; /* Array header's status in the metadata cache */ @@ -736,7 +736,7 @@ H5EA__hdr_delete(H5EA_hdr_t *hdr)) HDassert(hdr_status & H5AC_ES__IN_CACHE); HDassert(hdr_status & H5AC_ES__IS_PROTECTED); } /* end block */ -#endif /* NDEBUG */ +#endif /* H5EA_DEBUG */ /* Check for index block */ if(H5F_addr_defined(hdr->idx_blk_addr)) { diff --git a/src/H5EAprivate.h b/src/H5EAprivate.h index 1195256..6067ed0 100644 --- a/src/H5EAprivate.h +++ b/src/H5EAprivate.h @@ -150,8 +150,8 @@ H5_DLL herr_t H5EA_patch_file(H5EA_t *fa, H5F_t *f); H5_DLL herr_t H5EA_get_stats(const H5EA_t *ea, H5EA_stat_t *stats); /* Debugging routines */ -#ifdef H5EA_DEBUGGING -#endif /* H5EA_DEBUGGING */ +#ifdef H5EA_DEBUG +#endif /* H5EA_DEBUG */ #endif /* _H5EAprivate_H */ diff --git a/src/H5EAtest.c b/src/H5EAtest.c index 814e64f..f7731eb 100644 --- a/src/H5EAtest.c +++ b/src/H5EAtest.c @@ -282,9 +282,9 @@ herr_t, SUCCEED, -, H5EA__test_decode(const void *_raw, void *_elmt, size_t nelmts, void *_ctx)) /* Local variables */ -#ifndef NDEBUG +#ifdef H5EA_DEBUG H5EA__test_ctx_t *ctx = (H5EA__test_ctx_t *)_ctx; /* Callback context to destroy */ -#endif /* NDEBUG */ +#endif /* H5EA_DEBUG */ uint64_t *elmt = (uint64_t *)_elmt; /* Convenience pointer to native elements */ const uint8_t *raw = (const uint8_t *)_raw; /* Convenience pointer to raw elements */ @@ -292,7 +292,9 @@ H5EA__test_decode(const void *_raw, void *_elmt, size_t nelmts, void *_ctx)) HDassert(raw); HDassert(elmt); HDassert(nelmts); +#ifdef H5EA_DEBUG HDassert(H5EA__TEST_BOGUS_VAL == ctx->bogus); +#endif /* H5EA_DEBUG */ /* Decode raw elements into native elements */ while(nelmts) { @@ -686,20 +686,26 @@ END_FUNC(PRIV) /* end H5FA_delete() */ * Note: This is not very efficient, we should be iterating directly * over the fixed array's direct block [pages]. * - * Return: SUCCEED/FAIL + * Return: H5_ITER_CONT/H5_ITER_ERROR * * Programmer: Vailin Choi * Thursday, April 30, 2009 * + * Modification: + * Prototype changed (HDFFV-10661) + * - herr_t to int + * - SUCCEED/FAIL to H5_ITER_CONT/H5_ITER_ERROR + * June 6, 2019 -BMR *------------------------------------------------------------------------- */ BEGIN_FUNC(PRIV, ERR, -herr_t, SUCCEED, FAIL, +int, H5_ITER_CONT, H5_ITER_ERROR, H5FA_iterate(H5FA_t *fa, H5FA_operator_t op, void *udata)) /* Local variables */ uint8_t *elmt = NULL; hsize_t u; + int cb_ret = H5_ITER_CONT; /* Return value from callback */ /* * Check arguments. @@ -713,9 +719,7 @@ H5FA_iterate(H5FA_t *fa, H5FA_operator_t op, void *udata)) H5E_THROW(H5E_CANTALLOC, "memory allocation failed for fixed array element") /* Iterate over all elements in array */ - for(u = 0; u < fa->hdr->stats.nelmts; u++) { - int cb_ret; /* Return value from callback */ - + for(u = 0; u < fa->hdr->stats.nelmts && cb_ret == H5_ITER_CONT; u++) { /* Get array element */ if(H5FA_get(fa, u, elmt) < 0) H5E_THROW(H5E_CANTGET, "unable to delete fixed array") diff --git a/src/H5FDhdfs.c b/src/H5FDhdfs.c new file mode 100644 index 0000000..83d0202 --- /dev/null +++ b/src/H5FDhdfs.c @@ -0,0 +1,2166 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Read-Only HDFS Virtual File Driver (VFD) * + * Copyright (c) 2018, The HDF Group. * + * * + * All rights reserved. * + * * + * NOTICE: * + * All information contained herein is, and remains, the property of The HDF * + * Group. The intellectual and technical concepts contained herein are * + * proprietary to The HDF Group. Dissemination of this information or * + * reproduction of this material is strictly forbidden unless prior written * + * permission is obtained from The HDF Group. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Programmer: Jacob Smith + * 2018-04-23 + * + * Purpose: Provide read-only access to files on the Hadoop Distributed + * File System (HDFS). + */ + +/* This source code file is part of the H5FD driver module */ +#include "H5FDdrvr_module.h" + +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5FDprivate.h" /* File drivers */ +#include "H5FDhdfs.h" /* hdfs file driver */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5Iprivate.h" /* IDs */ +#include "H5MMprivate.h" /* Memory management */ + +#ifdef H5_HAVE_LIBHDFS +#include "hdfs.h" +#endif + +/* toggle function call prints: 1 turns on */ +#define HDFS_DEBUG 0 + +/* toggle stats collection and reporting */ +#define HDFS_STATS 0 + +/* The driver identification number, initialized at runtime */ +static hid_t H5FD_HDFS_g = 0; + +#if HDFS_STATS + +/* arbitrarily large value, such that any reasonable size read will be "less" + * than this value and set a true minimum + * not 0 because that may be a valid recorded minimum in degenerate cases + */ +#define HDFS_STATS_STARTING_MIN 0xfffffffful + +/* Configuration definitions for stats collection and breakdown + * + * 2^10 = 1024 + * Reads up to 1024 bytes (1 kB) fall in bin 0 + * 2^(10+(1*16)) = 2^26 = 64MB + * Reads of 64MB or greater fall in "overflow" bin[BIN_COUNT] + */ +#define HDFS_STATS_BASE 2 +#define HDFS_STATS_INTERVAL 1 +#define HDFS_STATS_START_POWER 10 +#define HDFS_STATS_BIN_COUNT 16 /* MUST BE GREATER THAN 0 */ + + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Calculate `BASE ^ (START_POWER + (INTERVAL * bin_i))` + * Stores result at `(unsigned long long *) out_ptr`. + * Used in computing boundaries between stats bins. + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + */ +#define HDFS_STATS_POW(bin_i, out_ptr) { \ + unsigned long long donotshadowresult = 1; \ + unsigned donotshadowindex = 0; \ + for (donotshadowindex = 0; \ + donotshadowindex < (((bin_i) * HDFS_STATS_INTERVAL) + \ + HDFS_STATS_START_POWER); \ + donotshadowindex++) \ + { \ + donotshadowresult *= HDFS_STATS_BASE; \ + } \ + *(out_ptr) = donotshadowresult; \ +} + +/* array to hold pre-computed boundaries for stats bins */ +static unsigned long long hdfs_stats_boundaries[HDFS_STATS_BIN_COUNT]; + + +/*************************************************************************** + * + * Structure: hdfs_statsbin + * + * Purpose: + * + * Structure for storing per-file hdfs VFD usage statistics. + * + * + * + * `count` (unsigned long long) + * + * Number of reads with size in this bin's range. + * + * `bytes` (unsigned long long) + * + * Total number of bytes read through this bin. + * + * `min` (unsigned long long) + * + * Smallest read size in this bin. + * + * `max` (unsigned long long) + * + * Largest read size in this bin. + * + * + * + * Programmer: Jacob Smith + * + * Changes: None + * + ***************************************************************************/ +typedef struct { + unsigned long long count; + unsigned long long bytes; + unsigned long long min; + unsigned long long max; +} hdfs_statsbin; + +#endif /* HDFS_STATS */ + +#ifdef H5_HAVE_LIBHDFS + +/* "unique" identifier for `hdfs_t` structures. + * Randomly generated by unweighted dice rolls. + */ +#define HDFS_HDFST_MAGIC 0x1AD5DE84 + + +/*************************************************************************** + * + * Structure: hdfs_t + * + * Purpose: + * + * Contain/retain information associated with a file hosted on Hadoop + * Distributed File System (HDFS). Instantiated and populated via + * `H5FD_hdfs_handle_open()` and cleaned up via `H5FD_hdfs_handle_close()`. + * + * + * + * `magic` (unisgned long) + * + * Number to indicate that this structure is of the promised + * type and should still be valid; should be HDFS_HDFST_MAGIC throughout + * the lifespan of the structure. Upon deletion of the structure, the + * programmer should set magic to anything but HDFS_HDFST_MAGIC, to + * indicate that the structure is to no longer be trusted. + * + * `filesystem` (hdfsFS) + * + * A libhdfs file system handle. + * + * `fileinfo` (hdfsFileInfo*) + * + * A pointer to a libhdfs file info structure. + * + * `file` (hdfsFile) + * + * A libhdfs file handle. + * + * + * + * Programmer: Jacob Smith + * May 2018 + * + * Changes: None + * + *************************************************************************** + */ +typedef struct { + unsigned long magic; + hdfsFS filesystem; + hdfsFileInfo *fileinfo; + hdfsFile file; +} hdfs_t; + + +/*-------------------------------------------------------------------------- + * Function: H5FD_hdfs_handle_open + * + * Purpose: Create a HDFS file handle, 'opening' the target file. + * + * Return: Success: Pointer to HDFS container/handle of opened file. + * Failure: NULL + * + * Programmer: Gerd Herber + * May 2018 + * + * Changes: None. + *-------------------------------------------------------------------------- + */ +static hdfs_t * +H5FD_hdfs_handle_open( + const char *path, + const char *namenode_name, + const int32_t namenode_port, + const char *user_name, + const char *kerberos_ticket_cache, + const int32_t stream_buffer_size) +{ + struct hdfsBuilder *builder = NULL; + hdfs_t *handle = NULL; + hdfs_t *ret_value = NULL; + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDfprintf(stdout, "called H5FD_hdfs_handle_open.\n"); +#endif + + if (path == NULL || path[0] == '\0') { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "path cannot be null.\n") + } + if (namenode_name == NULL /* || namenode_name[0] == '\0' */ ) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "namenode name cannot be null.\n") + } + if (namenode_port < 0 || namenode_port > 65535) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "namenode port must be non-negative and <= 65535.\n") + } + if (stream_buffer_size < 0) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "buffer size must non-negative.\n") + } + + handle = (hdfs_t *)H5MM_malloc(sizeof(hdfs_t)); + if (handle == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL, + "could not malloc space for handle.\n") + } + + handle->magic = (unsigned long)HDFS_HDFST_MAGIC; + handle->filesystem = NULL; /* TODO: not a pointer; NULL may cause bug */ + handle->fileinfo = NULL; + handle->file = NULL; /* TODO: not a pointer; NULL may cause bug */ + + builder = hdfsNewBuilder(); + if (!builder) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "(hdfs) failed to create builder") + } + hdfsBuilderSetNameNode(builder, namenode_name); + hdfsBuilderSetNameNodePort(builder, (tPort)namenode_port); + if (user_name != NULL && user_name[0] != '\0') { + hdfsBuilderSetUserName(builder, user_name); + } + if (kerberos_ticket_cache != NULL && kerberos_ticket_cache[0] != '\0') { + hdfsBuilderSetKerbTicketCachePath(builder, kerberos_ticket_cache); + } + /* Call to `hdfsBuilderConnect` releases builder, regardless of success. */ + handle->filesystem = hdfsBuilderConnect(builder); + if (!handle->filesystem) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "(hdfs) could not connect to default namenode") + } + handle->fileinfo = hdfsGetPathInfo(handle->filesystem, path); + if (!handle->fileinfo) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "hdfsGetPathInfo failed") + } + handle->file = hdfsOpenFile( + handle->filesystem, + path, + O_RDONLY, + stream_buffer_size, + 0, + 0); + if (!handle->file) { + HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, + "(hdfs) could not open") + } + + ret_value = handle; + +done: + if (ret_value == NULL && handle != NULL) { + /* error; clean up */ + HDassert(handle->magic == HDFS_HDFST_MAGIC); + handle->magic++; + if (handle->file != NULL) { + if (FAIL == (hdfsCloseFile(handle->filesystem, handle->file))) { + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, + "unable to close hdfs file handle") + } + } + if (handle->fileinfo != NULL) { + hdfsFreeFileInfo(handle->fileinfo, 1); + } + if (handle->filesystem != NULL) { + if (FAIL == (hdfsDisconnect(handle->filesystem))) { + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, + "unable to disconnect from hdfs") + } + } + H5MM_xfree(handle); + } + + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5FD_hdfs_handle_open() */ + + +/*-------------------------------------------------------------------------- + * Function: H5FD_hdfs_handle_close + * + * Purpose: 'Close' an HDFS file container/handle, releasing underlying + * resources. + * + * Return: Success: `SUCCEED` (0) + * Failure: `FAIL` (-1) + * + * Programmer: Gerd Herber + * May 2018 + * + * Changes: None. + *-------------------------------------------------------------------------- + */ +static herr_t +H5FD_hdfs_handle_close(hdfs_t *handle) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDfprintf(stdout, "called H5FD_hdfs_close.\n"); +#endif + + if (handle == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle cannot be null.\n") + } + if (handle->magic != HDFS_HDFST_MAGIC) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle has invalid magic.\n") + } + + handle->magic++; + if (handle->file != NULL) { + if (FAIL == (hdfsCloseFile(handle->filesystem, handle->file))) { + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, + "unable to close hdfs file handle") + } + } + if (handle->fileinfo != NULL) { + hdfsFreeFileInfo(handle->fileinfo, 1); + } + if (handle->filesystem != NULL) { + if (FAIL == (hdfsDisconnect(handle->filesystem))) { + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, + "unable to disconnect hdfs file system") + } + } + + H5MM_xfree(handle); + +done: + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5FD_hdfs_close() */ + +#endif /* H5_HAVE_LIBHDFS */ + + +/*************************************************************************** + * + * Structure: H5FD_hdfs_t + * + * Purpose: + * + * H5FD_hdfs_t is a structure used to store all information needed to + * maintain R/O access to a single HDF5 file in an HDFS file system. + * This structure is created when such a file is "opened" and + * discarded when it is "closed". + * + * + * `pub` (H5FD_t) + * + * Instance of H5FD_t which contains all fields common to all VFDs. + * It must be the first item in this structure, since at higher levels, + * this structure will be treated as an instance of H5FD_t. + * + * `fa` (H5FD_hdfs_fapl_t) + * + * Instance of `H5FD_hdfs_fapl_t` containing the HDFS configuration data + * needed to "open" the HDF5 file. + * + * `eoa` (haddr_t) + * + * End of addressed space in file. After open, it should always + * equal the file size. + * + * `hdfs_handle` (hdfs_t *) + * + * Instance of HDFS Request handle associated with the target resource. + * Responsible for communicating with remote host and presenting file + * contents as indistinguishable from a file on the local filesystem. + * + * *** present only if HDFS_SATS is flagged to enable stats collection *** + * + * `meta` (hdfs_statsbin[]) + * `raw` (hdfs_statsbin[]) + * + * Only present if hdfs stats collection is enabled. + * + * Arrays of `hdfs_statsbin` structures to record raw- and metadata reads. + * + * Records count and size of reads performed by the VFD, and is used to + * print formatted usage statistics to stdout upon VFD shutdown. + * + * Reads of each raw- and metadata type are recorded in an individual bin + * determined by the size of the read. The last bin of each type is + * reserved for "big" reads, with no defined upper bound. + * + * *** end HDFS_STATS *** + * + * + * + * Programmer: Jacob Smith + * + * Changes: None. + * + *************************************************************************** + */ +typedef struct H5FD_hdfs_t { + H5FD_t pub; + H5FD_hdfs_fapl_t fa; + haddr_t eoa; +#ifdef H5_HAVE_LIBHDFS + hdfs_t *hdfs_handle; +#endif +#if HDFS_STATS + hdfs_statsbin meta[HDFS_STATS_BIN_COUNT + 1]; + hdfs_statsbin raw[HDFS_STATS_BIN_COUNT + 1]; +#endif +} H5FD_hdfs_t; + +/* + * These macros check for overflow of various quantities. These macros + * assume that HDoff_t is signed and haddr_t and size_t are unsigned. + * + * ADDR_OVERFLOW: Checks whether a file address of type `haddr_t' + * is too large to be represented by the second argument + * of the file seek function. + * Only included if HDFS code should compile. + * + */ +#define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1) +#ifdef H5_HAVE_LIBHDFS +#define ADDR_OVERFLOW(A) (HADDR_UNDEF==(A) || ((A) & ~(haddr_t)MAXADDR)) +#endif /* H5_HAVE_LIBHDFS */ + +/* Prototypes */ +static herr_t H5FD_hdfs_term(void); +static void *H5FD_hdfs_fapl_get(H5FD_t *_file); +static void *H5FD_hdfs_fapl_copy(const void *_old_fa); +static herr_t H5FD_hdfs_fapl_free(void *_fa); +static H5FD_t *H5FD_hdfs_open(const char *name, unsigned flags, hid_t fapl_id, + haddr_t maxaddr); +static herr_t H5FD_hdfs_close(H5FD_t *_file); +static int H5FD_hdfs_cmp(const H5FD_t *_f1, const H5FD_t *_f2); +static herr_t H5FD_hdfs_query(const H5FD_t *_f1, unsigned long *flags); +static haddr_t H5FD_hdfs_get_eoa(const H5FD_t *_file, H5FD_mem_t type); +static herr_t H5FD_hdfs_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr); +static haddr_t H5FD_hdfs_get_eof(const H5FD_t *_file, H5FD_mem_t type); +static herr_t H5FD_hdfs_get_handle(H5FD_t *_file, hid_t fapl, + void** file_handle); +static herr_t H5FD_hdfs_read(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, + haddr_t addr, size_t size, void *buf); +static herr_t H5FD_hdfs_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, + haddr_t addr, size_t size, const void *buf); +static herr_t H5FD_hdfs_truncate(H5FD_t *_file, hid_t dxpl_id, + hbool_t closing); +static herr_t H5FD_hdfs_lock(H5FD_t *_file, hbool_t rw); +static herr_t H5FD_hdfs_unlock(H5FD_t *_file); +static herr_t H5FD_hdfs_validate_config(const H5FD_hdfs_fapl_t * fa); + +static const H5FD_class_t H5FD_hdfs_g = { + "hdfs", /* name */ + MAXADDR, /* maxaddr */ + H5F_CLOSE_WEAK, /* fc_degree */ + H5FD_hdfs_term, /* terminate */ + NULL, /* sb_size */ + NULL, /* sb_encode */ + NULL, /* sb_decode */ + sizeof(H5FD_hdfs_fapl_t), /* fapl_size */ + H5FD_hdfs_fapl_get, /* fapl_get */ + H5FD_hdfs_fapl_copy, /* fapl_copy */ + H5FD_hdfs_fapl_free, /* fapl_free */ + 0, /* dxpl_size */ + NULL, /* dxpl_copy */ + NULL, /* dxpl_free */ + H5FD_hdfs_open, /* open */ + H5FD_hdfs_close, /* close */ + H5FD_hdfs_cmp, /* cmp */ + H5FD_hdfs_query, /* query */ + NULL, /* get_type_map */ + NULL, /* alloc */ + NULL, /* free */ + H5FD_hdfs_get_eoa, /* get_eoa */ + H5FD_hdfs_set_eoa, /* set_eoa */ + H5FD_hdfs_get_eof, /* get_eof */ + H5FD_hdfs_get_handle, /* get_handle */ + H5FD_hdfs_read, /* read */ + H5FD_hdfs_write, /* write */ + NULL, /* flush */ + H5FD_hdfs_truncate, /* truncate */ + H5FD_hdfs_lock, /* lock */ + H5FD_hdfs_unlock, /* unlock */ + H5FD_FLMAP_DICHOTOMY /* fl_map */ +}; + +#ifdef H5_HAVE_LIBHDFS +/* Declare a free list to manage the H5FD_hdfs_t struct */ +H5FL_DEFINE_STATIC(H5FD_hdfs_t); +#endif /* H5_HAVE_LIBHDFS */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__init_package + * + * Purpose: Initializes any interface-specific data or routines. + * + * Return: Non-negative on success/Negative on failure + * + * Changes: Rename as appropriate for hdfs vfd. + * Jacob Smith 2018 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__init_package(void) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + if (H5FD_hdfs_init() < 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, + "unable to initialize hdfs VFD") + } + +done: + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5FD__init_package() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_init + * + * Purpose: Initialize this driver by registering the driver with the + * library. + * + * Return: Success: The driver ID for the hdfs driver. + * Failure: Negative + * + * Programmer: Robb Matzke + * Thursday, July 29, 1999 + * + * Changes: Rename as appropriate for hdfs vfd. + * Jacob Smith 2018 + * + *------------------------------------------------------------------------- + */ +hid_t +H5FD_hdfs_init(void) +{ + hid_t ret_value = H5I_INVALID_HID; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_init() called.\n"); +#endif + + if (H5I_VFL != H5I_get_type(H5FD_HDFS_g)) { + H5FD_HDFS_g = H5FD_register( + &H5FD_hdfs_g, + sizeof(H5FD_class_t), + FALSE); + } + +#if HDFS_STATS + /* pre-compute statsbin boundaries + */ + for (unsigned bin_i = 0; bin_i < HDFS_STATS_BIN_COUNT; bin_i++) { + unsigned long long value = 0; + HDFS_STATS_POW(bin_i, &value) + hdfs_stats_boundaries[bin_i] = value; + } +#endif + + ret_value = H5FD_HDFS_g; + +done: + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5FD_hdfs_init() */ + + +/*--------------------------------------------------------------------------- + * Function: H5FD_hdfs_term + * + * Purpose: Shut down the VFD + * + * Returns: SUCCEED (Can't fail) + * + * Programmer: Quincey Koziol + * Friday, Jan 30, 2004 + * + * Changes: Rename as appropriate for hdfs vfd. + * Jacob Smith 2018 + * + *--------------------------------------------------------------------------- + */ +static herr_t +H5FD_hdfs_term(void) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_term() called.\n"); +#endif + + /* Reset VFL ID */ + H5FD_HDFS_g = 0; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5FD_hdfs_term() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pset_fapl_hdfs + * + * Purpose: Modify the file access property list to use the H5FD_HDFS + * driver defined in this source file. All driver specfic + * properties are passed in as a pointer to a suitably + * initialized instance of H5FD_hdfs_fapl_t + * + * Return: SUCCEED/FAIL + * + * Programmer: John Mainzer + * 9/10/17 + * + * Changes: Rename as appropriate for hdfs vfd. + * Jacob Smith 2018 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_fapl_hdfs(hid_t fapl_id, + H5FD_hdfs_fapl_t *fa) +{ + H5P_genplist_t *plist = NULL; /* Property list pointer */ + herr_t ret_value = FAIL; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*x", fapl_id, fa); + + HDassert(fa != NULL); + +#if HDFS_DEBUG + HDfprintf(stdout, "H5Pset_fapl_hdfs() called.\n"); +#endif + + plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS); + if (plist == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, \ + "not a file access property list") + } + + if (FAIL == H5FD_hdfs_validate_config(fa)) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "invalid hdfs config") + } + + ret_value = H5P_set_driver(plist, H5FD_HDFS, (void *)fa); + +done: + FUNC_LEAVE_API(ret_value) + +} /* H5Pset_fapl_hdfs() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_validate_config() + * + * Purpose: Test to see if the supplied instance of H5FD_hdfs_fapl_t + * contains internally consistant data. Return SUCCEED if so, + * and FAIL otherwise. + * + * Note the difference between internally consistant and + * correct. As we will have to try to access the target + * object to determine whether the supplied data is correct, + * we will settle for internal consistancy at this point + * + * Return: SUCCEED if instance of H5FD_hdfs_fapl_t contains internally + * consistant data, FAIL otherwise. + * + * Programmer: Jacob Smith + * 9/10/17 + * + * Changes: None. + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_hdfs_validate_config(const H5FD_hdfs_fapl_t * fa) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + + HDassert(fa != NULL); + + if ( fa->version != H5FD__CURR_HDFS_FAPL_T_VERSION ) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "Unknown H5FD_hdfs_fapl_t version"); + } + + if ( fa->namenode_port > 65535 ) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "Invalid namenode port number"); + } + if ( fa->namenode_port < 0 ) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "Invalid namenode port number"); + } + +done: + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5FD_hdfs_validate_config() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_fapl_hdfs + * + * Purpose: Returns information about the hdfs file access property + * list though the function arguments. + * + * Return: Success: Non-negative + * + * Failure: Negative + * + * Programmer: John Mainzer + * 9/10/17 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_fapl_hdfs(hid_t fapl_id, + H5FD_hdfs_fapl_t *fa_out) +{ + const H5FD_hdfs_fapl_t *fa = NULL; + H5P_genplist_t *plist = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*x", fapl_id, fa_out); + +#if HDFS_DEBUG + HDfprintf(stdout, "H5Pget_fapl_hdfs() called.\n"); +#endif + + if (fa_out == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "fa_out is NULL") + } + plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS); + if (plist == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, + "not a file access list") + } + if (H5FD_HDFS != H5P_peek_driver(plist)) { + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, + "incorrect VFL driver") + } + + fa = (const H5FD_hdfs_fapl_t *)H5P_peek_driver_info(plist); + if (fa == NULL) { + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, + "bad VFL driver info") + } + + /* Copy the hdfs fapl data out */ + HDmemcpy(fa_out, fa, sizeof(H5FD_hdfs_fapl_t)); + +done: + FUNC_LEAVE_API(ret_value) + +} /* H5Pget_fapl_hdfs() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_fapl_get + * + * Purpose: Gets a file access property list which could be used to + * create an identical file. + * + * Return: Success: Ptr to new file access property list value. + * + * Failure: NULL + * + * Programmer: John Mainzer + * 9/8/17 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static void * +H5FD_hdfs_fapl_get(H5FD_t *_file) +{ + H5FD_hdfs_t *file = (H5FD_hdfs_t*)_file; + H5FD_hdfs_fapl_t *fa = NULL; + void *ret_value = NULL; + + FUNC_ENTER_NOAPI_NOINIT + + fa = (H5FD_hdfs_fapl_t *)H5MM_calloc(sizeof(H5FD_hdfs_fapl_t)); + if (fa == NULL) { + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, + "memory allocation failed") + } + + /* Copy the fields of the structure */ + HDmemcpy(fa, &(file->fa), sizeof(H5FD_hdfs_fapl_t)); + + ret_value = fa; + +done: + if (ret_value == NULL && fa != NULL) { + H5MM_xfree(fa); /* clean up on error */ + } + + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5FD_hdfs_fapl_get() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_fapl_copy + * + * Purpose: Copies the hdfs-specific file access properties. + * + * Return: Success: Ptr to a new property list + * + * Failure: NULL + * + * Programmer: John Mainzer + * 9/8/17 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static void * +H5FD_hdfs_fapl_copy(const void *_old_fa) +{ + const H5FD_hdfs_fapl_t *old_fa = (const H5FD_hdfs_fapl_t*)_old_fa; + H5FD_hdfs_fapl_t *new_fa = NULL; + void *ret_value = NULL; + + FUNC_ENTER_NOAPI_NOINIT + + new_fa = (H5FD_hdfs_fapl_t *)H5MM_malloc(sizeof(H5FD_hdfs_fapl_t)); + if (new_fa == NULL) { + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, + "memory allocation failed") + } + + HDmemcpy(new_fa, old_fa, sizeof(H5FD_hdfs_fapl_t)); + ret_value = new_fa; + +done: + if (ret_value == NULL && new_fa != NULL) { + H5MM_xfree(new_fa); /* clean up on error */ + } + + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5FD_hdfs_fapl_copy() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_fapl_free + * + * Purpose: Frees the hdfs-specific file access properties. + * + * Return: SUCCEED (cannot fail) + * + * Programmer: John Mainzer + * 9/8/17 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_hdfs_fapl_free(void *_fa) +{ + H5FD_hdfs_fapl_t *fa = (H5FD_hdfs_fapl_t*)_fa; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(fa != NULL); /* sanity check */ + + H5MM_xfree(fa); + + FUNC_LEAVE_NOAPI(SUCCEED) + +} /* H5FD_hdfs_fapl_free() */ + +#if HDFS_STATS + +/*---------------------------------------------------------------------------- + * + * Function: hdfs_reset_stats() + * + * Purpose: + * + * Reset the stats collection elements in this virtual file structure. + * + * Clears any set data in stats bins; initializes/zeroes values. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * - Occurs if the file is invalid somehow + * + * Programmer: Jacob Smith + * 2017-12-08 + * + * Changes: None. + * + *---------------------------------------------------------------------------- + */ +static herr_t +hdfs_reset_stats(H5FD_hdfs_t *file) +{ + unsigned i = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDprintf("hdfs_reset_stats() called\n"); +#endif + + if (file == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "file was null") + } + + for (i = 0; i <= HDFS_STATS_BIN_COUNT; i++) { + file->raw[i].bytes = 0; + file->raw[i].count = 0; + file->raw[i].min = (unsigned long long)HDFS_STATS_STARTING_MIN; + file->raw[i].max = 0; + + file->meta[i].bytes = 0; + file->meta[i].count = 0; + file->meta[i].min = (unsigned long long)HDFS_STATS_STARTING_MIN; + file->meta[i].max = 0; + } + +done: + FUNC_LEAVE_NOAPI(ret_value); + +} /* hdfs_reset_stats */ +#endif /* HDFS_STATS */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_hdfs_open() + * + * Purpose: + * + * Create and/or opens a file as an HDF5 file. + * + * Any flag except H5F_ACC_RDONLY will cause an error. + * + * Return: + * + * Success: A pointer to a new file data structure. + * The public fields will be initialized by the caller, which is + * always H5FD_open(). + * + * Failure: NULL + * + * Programmer: Jacob Smith + * 2017-11-02 + * + * Changes: None. + * + *------------------------------------------------------------------------- + */ +#ifdef H5_HAVE_LIBHDFS +static H5FD_t * +H5FD_hdfs_open( + const char *path, + unsigned flags, + hid_t fapl_id, + haddr_t maxaddr) +{ + H5FD_t *ret_value = NULL; + H5FD_hdfs_t *file = NULL; + hdfs_t *handle = NULL; + H5FD_hdfs_fapl_t fa; + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_open() called.\n"); +#endif /* HDFS_DEBUG */ + + /* Sanity check on file offsets */ + HDcompile_assert(sizeof(HDoff_t) >= sizeof(size_t)); + + /* Check arguments */ + if (!path || !*path) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "invalid file name") + } + if (0 == maxaddr || HADDR_UNDEF == maxaddr) { + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, + "bogus maxaddr") + } + if (ADDR_OVERFLOW(maxaddr)) { + HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, NULL, + "bogus maxaddr") + } + if (flags != H5F_ACC_RDONLY) { + HGOTO_ERROR(H5E_ARGS, H5E_UNSUPPORTED, NULL, + "only Read-Only access allowed") + } + if (fapl_id == H5P_DEFAULT || fapl_id == H5P_FILE_ACCESS_DEFAULT) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "fapl cannot be H5P_DEFAULT") + } + if (FAIL == H5Pget_fapl_hdfs(fapl_id, &fa)) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "can't get property list") + } + + handle = H5FD_hdfs_handle_open( + path, + fa.namenode_name, + fa.namenode_port, + fa.user_name, + fa.kerberos_ticket_cache, + fa.stream_buffer_size); + + if (handle == NULL) { + HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, + "could not open") + } + + HDassert(handle->magic == HDFS_HDFST_MAGIC); + + /* create new file struct + */ + file = H5FL_CALLOC(H5FD_hdfs_t); + if (file == NULL) { + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, + "unable to allocate file struct") + } + file->hdfs_handle = handle; + HDmemcpy(&(file->fa), &fa, sizeof(H5FD_hdfs_fapl_t)); + +#if HDFS_STATS + if (FAIL == hdfs_reset_stats(file)) { + HGOTO_ERROR(H5E_INTERNAL, H5E_UNINITIALIZED, NULL, + "unable to reset file statistics") + } +#endif /* HDFS_STATS */ + + ret_value = (H5FD_t*)file; + +done: + if (ret_value == NULL) { + if (handle != NULL) { + if (FAIL == H5FD_hdfs_handle_close(handle)) { + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, + "unable to close HDFS file handle") + } + } + if (file != NULL) { + file = H5FL_FREE(H5FD_hdfs_t, file); + } + } /* end if null return value (error) */ + + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5FD_hdfs_open() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static H5FD_t * +H5FD_hdfs_open( + const char H5_ATTR_UNUSED *path, + unsigned H5_ATTR_UNUSED flags, + hid_t H5_ATTR_UNUSED fapl_id, + haddr_t H5_ATTR_UNUSED maxaddr) +{ + H5FD_t *ret_value = NULL; + + FUNC_ENTER_NOAPI_NOINIT + + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, NULL, + "Illegal open of unsupported virtual file (hdfs)"); + +done: + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5FD_hdfs_open() */ + +#endif /* H5_HAVE_LIBHDFS */ + +#if HDFS_STATS + +/*---------------------------------------------------------------------------- + * + * Function: hdfs_fprint_stats() + * + * Purpose: + * + * Tabulate and pretty-print statistics for this virtual file. + * + * Should be called upon file close. + * + * Shows number of reads and bytes read, broken down by + * "raw" (H5FD_MEM_DRAW) + * or "meta" (any other flag) + * + * Prints filename and listing of total number of reads and bytes read, + * both as a grand total and separate meta- and rawdata reads. + * + * If any reads were done, prints out two tables: + * + * 1. overview of raw- and metadata reads + * - min (smallest size read) + * - average of size read + * - k,M,G suffixes by powers of 1024 (2^10) + * - max (largest size read) + * 2. tabulation of "bins", sepraring reads into exponentially-larger + * ranges of size. + * - columns for number of reads, total bytes, and average size, with + * separate sub-colums for raw- and metadata reads. + * - each row represents one bin, identified by the top of its range + * + * Bin ranges can be modified with pound-defines at the top of this file. + * + * Bins without any reads in their bounds are not printed. + * + * An "overflow" bin is also present, to catch "big" reads. + * + * Output for all bins (and range ceiling and average size report) + * is divied by powers of 1024. By corollary, four digits before the decimal + * is valid. + * + * - 41080 bytes is represented by 40.177k, not 41.080k + * - 1004.831M represents approx. 1052642000 bytes + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * - occurs if the file passed in is invalid + * - TODO: if stream is invalid? how can we check this? + * + * Programmer: Jacob Smith + * + * Changes: None. + * + *---------------------------------------------------------------------------- + */ +static herr_t +hdfs_fprint_stats( + FILE *stream, + const H5FD_hdfs_t *file) +{ + herr_t ret_value = SUCCEED; + parsed_url_t *purl = NULL; + unsigned i = 0; + unsigned long count_meta = 0; + unsigned long count_raw = 0; + double average_meta = 0.0; + double average_raw = 0.0; + unsigned long long min_meta = (unsigned long long)HDFS_STATS_STARTING_MIN; + unsigned long long min_raw = (unsigned long long)HDFS_STATS_STARTING_MIN; + unsigned long long max_meta = 0; + unsigned long long max_raw = 0; + unsigned long long bytes_raw = 0; + unsigned long long bytes_meta = 0; + double re_dub = 0.0; /* re-usable double variable */ + unsigned suffix_i = 0; + const char suffixes[] = { ' ', 'K', 'M', 'G', 'T', 'P' }; + + FUNC_ENTER_NOAPI_NOINIT + + if (stream == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "file stream cannot be null" ) + } + if (file == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "file cannot be null") + } + if (file->hdfs_handle == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "hdfs handle cannot be null") + } + if (file->hdfs_handle->magic != HDFS_HDFST_MAGIC) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "hdfs handle has invalid magic") + } + + /******************* + * AGGREGATE STATS * + *******************/ + + for (i = 0; i <= HDFS_STATS_BIN_COUNT; i++) { + const hdfs_statsbin *r = &file->raw[i]; + const hdfs_statsbin *m = &file->meta[i]; + + if (m->min < min_meta) { min_meta = m->min; } + if (r->min < min_raw) { min_raw = r->min; } + if (m->max > max_meta) { max_meta = m->max; } + if (r->max > max_raw) { max_raw = r->max; } + + count_raw += r->count; + count_meta += m->count; + bytes_raw += r->bytes; + bytes_meta += m->bytes; + } + if (count_raw > 0) { + average_raw = (double)bytes_raw / (double)count_raw; + } + if (count_meta > 0) { + average_meta = (double)bytes_meta / (double)count_meta; + } + + /****************** + * PRINT OVERVIEW * + ******************/ + + HDfprintf(stream, "TOTAL READS: %llu (%llu meta, %llu raw)\n", + count_raw + count_meta, count_meta, count_raw); + HDfprintf(stream, "TOTAL BYTES: %llu (%llu meta, %llu raw)\n", + bytes_raw + bytes_meta, bytes_meta, bytes_raw); + + if (count_raw + count_meta == 0) { + goto done; + } + + /************************* + * PRINT AGGREGATE STATS * + *************************/ + + HDfprintf(stream, "SIZES meta raw\n"); + HDfprintf(stream, " min "); + if (count_meta == 0) { + HDfprintf(stream, " 0.000 "); + } else { + re_dub = (double)min_meta; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) { + re_dub /= 1024.0; + } + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]); + } + + if (count_raw == 0) { + HDfprintf(stream, " 0.000 \n"); + } else { + re_dub = (double)min_raw; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) { + re_dub /= 1024.0; + } + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]); + } + + HDfprintf(stream, " avg "); + re_dub = (double)average_meta; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) { + re_dub /= 1024.0; + } + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]); + + re_dub = (double)average_raw; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) { + re_dub /= 1024.0; + } + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]); + + HDfprintf(stream, " max "); + re_dub = (double)max_meta; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) { + re_dub /= 1024.0; + } + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]); + + re_dub = (double)max_raw; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) { + re_dub /= 1024.0; + } + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]); + + /****************************** + * PRINT INDIVIDUAL BIN STATS * + ******************************/ + + HDfprintf(stream, + "BINS # of reads total bytes average size\n"); + HDfprintf(stream, + " up-to meta raw meta raw meta raw\n"); + + for (i = 0; i <= HDFS_STATS_BIN_COUNT; i++) { + const hdfs_statsbin *m; + const hdfs_statsbin *r; + unsigned long long range_end = 0; + char bm_suffix = ' '; /* bytes-meta */ + double bm_val = 0.0; + char br_suffix = ' '; /* bytes-raw */ + double br_val = 0.0; + char am_suffix = ' '; /* average-meta */ + double am_val = 0.0; + char ar_suffix = ' '; /* average-raw */ + double ar_val = 0.0; + + m = &file->meta[i]; + r = &file->raw[i]; + if (r->count == 0 && m->count == 0) { + continue; + } + + range_end = hdfs_stats_boundaries[i]; + + if (i == HDFS_STATS_BIN_COUNT) { + range_end = hdfs_stats_boundaries[i-1]; + HDfprintf(stream, ">"); + } else { + HDfprintf(stream, " "); + } + + bm_val = (double)m->bytes; + for (suffix_i = 0; bm_val >= 1024.0; suffix_i++) { + bm_val /= 1024.0; + } + HDassert(suffix_i < sizeof(suffixes)); + bm_suffix = suffixes[suffix_i]; + + br_val = (double)r->bytes; + for (suffix_i = 0; br_val >= 1024.0; suffix_i++) { + br_val /= 1024.0; + } + HDassert(suffix_i < sizeof(suffixes)); + br_suffix = suffixes[suffix_i]; + + if (m->count > 0) { + am_val = (double)(m->bytes) / (double)(m->count); + } + for (suffix_i = 0; am_val >= 1024.0; suffix_i++) { + am_val /= 1024.0; + } + HDassert(suffix_i < sizeof(suffixes)); + am_suffix = suffixes[suffix_i]; + + if (r->count > 0) { + ar_val = (double)(r->bytes) / (double)(r->count); + } + for (suffix_i = 0; ar_val >= 1024.0; suffix_i++) { + ar_val /= 1024.0; + } + HDassert(suffix_i < sizeof(suffixes)); + ar_suffix = suffixes[suffix_i]; + + re_dub = (double)range_end; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) { + re_dub /= 1024.0; + } + HDassert(suffix_i < sizeof(suffixes)); + + HDfprintf( + stream, + " %8.3f%c %7d %7d %8.3f%c %8.3f%c %8.3f%c %8.3f%c\n", + re_dub, suffixes[suffix_i], /* bin ceiling */ + m->count, /* metadata reads */ + r->count, /* rawdata reads */ + bm_val, bm_suffix, /* metadata bytes */ + br_val, br_suffix, /* rawdata bytes */ + am_val, am_suffix, /* metadata average */ + ar_val, ar_suffix); /* rawdata average */ + fflush(stream); + } + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* hdfs_fprint_stats */ +#endif /* HDFS_STATS */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_hdfs_close() + * + * Purpose: + * + * Close an HDF5 file. + * + * Return: + * + * SUCCEED/FAIL + * + * Programmer: Jacob Smith + * 2017-11-02 + * + * Changes: None. + * + *------------------------------------------------------------------------- + */ +#ifdef H5_HAVE_LIBHDFS + +static herr_t +H5FD_hdfs_close(H5FD_t *_file) +{ + herr_t ret_value = SUCCEED; + H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file; + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_close() called.\n"); +#endif + + /* Sanity checks + */ + HDassert(file != NULL); + HDassert(file->hdfs_handle != NULL); + HDassert(file->hdfs_handle->magic == HDFS_HDFST_MAGIC); + + /* Close the underlying request handle + */ + if (file->hdfs_handle != NULL) { + if (FAIL == H5FD_hdfs_handle_close(file->hdfs_handle)) { + HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, + "unable to close HDFS file handle") + } + } + +#if HDFS_STATS + /* TODO: mechanism to re-target stats printout */ + if (FAIL == hdfs_fprint_stats(stdout, file)) { + HGOTO_ERROR(H5E_INTERNAL, H5E_ERROR, FAIL, + "problem while writing file statistics") + } +#endif /* HDFS_STATS */ + + /* Release the file info + */ + file = H5FL_FREE(H5FD_hdfs_t, file); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_hdfs_close() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static herr_t +H5FD_hdfs_close(H5FD_t H5_ATTR_UNUSED *_file) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + + HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, + "Illegal close of unsupported Virtual File (hdfs)") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_hdfs_close() */ + +#endif /* H5_HAVE_LIBHDFS */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_hdfs_cmp() + * + * Purpose: + * + * Compares two files using this driver by their HDFS-provided file info, + * field-by-field. + * + * Return: + * + Equivalent: 0 + * + Not Equivalent: -1 + * + * Programmer: Gerd Herber + * May 2018 + * + * Changes: + * + * + Replace `if (ret_value == 0)` chain with `HGOTO_DONE` jumps. + * Jacob Smith 17 May 2018 + * + *------------------------------------------------------------------------- + */ +#ifdef H5_HAVE_LIBHDFS + +static int +H5FD_hdfs_cmp( + const H5FD_t *_f1, + const H5FD_t *_f2) +{ + int ret_value = 0; + const H5FD_hdfs_t *f1 = (const H5FD_hdfs_t *)_f1; + const H5FD_hdfs_t *f2 = (const H5FD_hdfs_t *)_f2; + hdfsFileInfo *finfo1 = NULL; + hdfsFileInfo *finfo2 = NULL; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_cmp() called.\n"); +#endif /* HDFS_DEBUG */ + + HDassert(f1->hdfs_handle != NULL); + HDassert(f2->hdfs_handle != NULL); + HDassert(f1->hdfs_handle->magic == HDFS_HDFST_MAGIC); + HDassert(f2->hdfs_handle->magic == HDFS_HDFST_MAGIC); + + finfo1 = f1->hdfs_handle->fileinfo; + finfo2 = f2->hdfs_handle->fileinfo; + HDassert(finfo1 != NULL); + HDassert(finfo2 != NULL); + + if (finfo1->mKind != finfo2->mKind) { HGOTO_DONE(-1); } + if (finfo1->mName != finfo2->mName) { HGOTO_DONE(-1); } + if (finfo1->mLastMod != finfo2->mLastMod) { HGOTO_DONE(-1); } + if (finfo1->mSize != finfo2->mSize) { HGOTO_DONE(-1); } + if (finfo1->mReplication != finfo2->mReplication) { HGOTO_DONE(-1); } + if (finfo1->mBlockSize != finfo2->mBlockSize) { HGOTO_DONE(-1); } + if (strcmp(finfo1->mOwner, finfo2->mOwner)) { HGOTO_DONE(-1); } + if (strcmp(finfo1->mGroup, finfo2->mGroup)) { HGOTO_DONE(-1); } + if (finfo1->mPermissions != finfo2->mPermissions) { HGOTO_DONE(-1); } + if (finfo1->mLastAccess != finfo2->mLastAccess) { HGOTO_DONE(-1); } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD_hdfs_cmp() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static int +H5FD_hdfs_cmp( + const H5FD_t H5_ATTR_UNUSED *_f1, + const H5FD_t H5_ATTR_UNUSED *_f2) +{ + int ret_value = 0; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD_hdfs_cmp() */ + +#endif /* H5_HAVE_LIBHDFS */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_query + * + * Purpose: Set the flags that this VFL driver is capable of supporting. + * (listed in H5FDpublic.h) + * + * Note that since the HDFS VFD is read only, most flags + * are irrelevant. + * + * The term "set" is highly misleading... + * stores/copies the supported flags in the out-pointer `flags`. + * + * Return: SUCCEED (Can't fail) + * + * Programmer: John Mainzer + * 9/11/17 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_hdfs_query( + const H5FD_t H5_ATTR_UNUSED *_file, + unsigned long *flags) /* out variable */ +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_query() called.\n"); +#endif + + if (flags) { + *flags = 0; + *flags |= H5FD_FEAT_DATA_SIEVE; + } + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5FD_hdfs_query() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_hdfs_get_eoa() + * + * Purpose: + * + * Gets the end-of-address marker for the file. The EOA marker + * is the first address past the last byte allocated in the + * format address space. + * + * Return: + * + * The end-of-address marker. + * + * Programmer: Jacob Smith + * 2017-11-02 + * + * Changes: None. + * + *------------------------------------------------------------------------- + */ +#ifdef H5_HAVE_LIBHDFS + +static haddr_t +H5FD_hdfs_get_eoa( + const H5FD_t *_file, + H5FD_mem_t H5_ATTR_UNUSED type) +{ + const H5FD_hdfs_t *file = (const H5FD_hdfs_t *)_file; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_get_eoa() called.\n"); +#endif + + FUNC_LEAVE_NOAPI(file->eoa) +} /* end H5FD_hdfs_get_eoa() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static haddr_t +H5FD_hdfs_get_eoa( + const H5FD_t H5_ATTR_UNUSED *_file, + H5FD_mem_t H5_ATTR_UNUSED type) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_get_eoa() called.\n"); +#endif + + FUNC_LEAVE_NOAPI(0) +} /* end H5FD_hdfs_get_eoa() */ + +#endif /* H5_HAVE_LIBHDFS */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_hdfs_set_eoa() + * + * Purpose: + * + * Set the end-of-address marker for the file. + * + * Return: + * + * SUCCEED (can't fail) + * + * Programmer: Jacob Smith + * 2017-11-03 + * + * Changes: None. + * + *------------------------------------------------------------------------- + */ +#ifdef H5_HAVE_LIBHDFS + +static herr_t +H5FD_hdfs_set_eoa( + H5FD_t *_file, + H5FD_mem_t H5_ATTR_UNUSED type, + haddr_t addr) +{ + H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_set_eoa() called.\n"); +#endif + + file->eoa = addr; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5FD_hdfs_set_eoa() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static herr_t +H5FD_hdfs_set_eoa( + H5FD_t H5_ATTR_UNUSED *_file, + H5FD_mem_t H5_ATTR_UNUSED type, + haddr_t H5_ATTR_UNUSED addr) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_set_eoa() called.\n"); +#endif + + FUNC_LEAVE_NOAPI(FAIL) +} /* H5FD_hdfs_set_eoa() */ + +#endif /* H5_HAVE_LIBHDFS */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_hdfs_get_eof() + * + * Purpose: + * + * Returns the end-of-file marker. + * + * Return: + * + * EOF: the first address past the end of the "file", either the + * filesystem file or the HDF5 file. + * + * Programmer: Jacob Smith + * 2017-11-02 + * + *------------------------------------------------------------------------- + */ +#ifdef H5_HAVE_LIBHDFS + +static haddr_t +H5FD_hdfs_get_eof( + const H5FD_t *_file, + H5FD_mem_t H5_ATTR_UNUSED type) +{ + const H5FD_hdfs_t *file = (const H5FD_hdfs_t *)_file; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_get_eof() called.\n"); +#endif + + HDassert(file->hdfs_handle != NULL); + HDassert(file->hdfs_handle->magic == HDFS_HDFST_MAGIC); + + FUNC_LEAVE_NOAPI((size_t) file->hdfs_handle->fileinfo->mSize) +} /* end H5FD_hdfs_get_eof() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static haddr_t +H5FD_hdfs_get_eof( + const H5FD_t H5_ATTR_UNUSED *_file, + H5FD_mem_t H5_ATTR_UNUSED type) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_get_eof() called.\n"); +#endif + + FUNC_LEAVE_NOAPI((size_t)0) +} /* end H5FD_hdfs_get_eof() */ + +#endif /* H5_HAVE_LIBHDFS */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_hdfs_get_handle() + * + * Purpose: + * + * Returns the HDFS handle (hdfs_t) of hdfs file driver. + * + * Returns: + * + * SUCCEED/FAIL + * + * Programmer: Jacob Smith + * 2017-11-02 + * + * Changes: None. + * + *------------------------------------------------------------------------- + */ +#ifdef H5_HAVE_LIBHDFS + +static herr_t +H5FD_hdfs_get_handle( + H5FD_t *_file, + hid_t H5_ATTR_UNUSED fapl, + void **file_handle) +{ + herr_t ret_value = SUCCEED; + H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file; + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_get_handle() called.\n"); +#endif /* HDFS_DEBUG */ + + if (!file_handle) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "file handle not valid") + } + + *file_handle = file->hdfs_handle; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_hdfs_get_handle() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static herr_t +H5FD_hdfs_get_handle( + H5FD_t H5_ATTR_UNUSED *_file, + hid_t H5_ATTR_UNUSED fapl, + void H5_ATTR_UNUSED **file_handle) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_get_handle() called.\n"); +#endif /* HDFS_DEBUG */ + + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, + "Illegal get-handle of unsupported virtual file (hdfs)"); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_hdfs_get_handle() */ + +#endif /* H5_HAVE_LIBHDFS */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_hdfs_read() + * + * Purpose: + * + * Reads SIZE bytes of data from FILE beginning at address ADDR + * into buffer BUF according to data transfer properties in DXPL_ID. + * + * Return: + * + * Success: `SUCCEED` + * - Result is stored in caller-supplied buffer BUF. + * Failure: `FAIL` + * - Unable to complete read. + * - Contents of buffer `buf` are undefined. + * + * Programmer: Jacob Smith + * 2017-11-?? + * + * Changes: None. + * + *------------------------------------------------------------------------- + */ +#ifdef H5_HAVE_LIBHDFS + +static herr_t +H5FD_hdfs_read( + H5FD_t *_file, + H5FD_mem_t H5_ATTR_UNUSED type, + hid_t H5_ATTR_UNUSED dxpl_id, + haddr_t addr, /* start offset */ + size_t size, /* length of read */ + void *buf) /* out */ +{ + herr_t ret_value = SUCCEED; + H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file; + size_t filesize = 0; +#if HDFS_STATS + /* working variables for storing stats */ + hdfs_statsbin *bin = NULL; + unsigned bin_i = 0; +#endif /* HDFS_STATS */ + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_read() called.\n"); +#endif /* HDFS_DEBUG */ + + HDassert(file != NULL); + HDassert(file->hdfs_handle != NULL); + HDassert(file->hdfs_handle->magic == HDFS_HDFST_MAGIC); + HDassert(buf != NULL); + + filesize = (size_t) file->hdfs_handle->fileinfo->mSize; + + if ((addr > filesize) || ((addr + size) > filesize)) { + HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, + "range exceeds file address") + } + + if (FAIL == hdfsPread( + file->hdfs_handle->filesystem, + file->hdfs_handle->file, + (tOffset)addr, + buf, + (tSize)size)) + { + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, + "unable to execute read") + } + +#if HDFS_STATS + + /* Find which "bin" this read fits in. Can be "overflow" bin. + */ + for (bin_i = 0; bin_i < HDFS_STATS_BIN_COUNT; bin_i++) { + if ((unsigned long long)size < hdfs_stats_boundaries[bin_i]) { + break; + } + } + bin = (type == H5FD_MEM_DRAW) + ? &file->raw[bin_i] + : &file->meta[bin_i]; + + /* Store collected stats in appropriate bin + */ + if (bin->count == 0) { + bin->min = size; + bin->max = size; + } + else { + if (size < bin->min) { bin->min = size; } + if (size > bin->max) { bin->max = size; } + } + bin->count++; + bin->bytes += (unsigned long long)size; + +#endif /* HDFS_STATS */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_hdfs_read() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static herr_t +H5FD_hdfs_read( + H5FD_t H5_ATTR_UNUSED *_file, + H5FD_mem_t H5_ATTR_UNUSED type, + hid_t H5_ATTR_UNUSED dxpl_id, + haddr_t H5_ATTR_UNUSED addr, + size_t H5_ATTR_UNUSED size, + void H5_ATTR_UNUSED *buf) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_read() called.\n"); +#endif /* HDFS_DEBUG */ + + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, + "Illegal get-handle of unsupported virtual file (hdfs)"); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_hdfs_read() */ + +#endif /* H5_HAVE_LIBHDFS */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_hdfs_write() + * + * Purpose: + * + * Write bytes to file. + * UNSUPPORTED IN READ-ONLY HDFS VFD. + * + * Return: + * + * FAIL (Not possible with Read-Only S3 file.) + * + * Programmer: Jacob Smith + * 2017-10-23 + * + * Changes: None. + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_hdfs_write( + H5FD_t H5_ATTR_UNUSED *_file, + H5FD_mem_t H5_ATTR_UNUSED type, + hid_t H5_ATTR_UNUSED dxpl_id, + haddr_t H5_ATTR_UNUSED addr, + size_t H5_ATTR_UNUSED size, + const void H5_ATTR_UNUSED *buf) +{ + herr_t ret_value = FAIL; + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_write() called.\n"); +#endif + + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, + "cannot write to read-only file.") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD_hdfs_write() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_hdfs_truncate() + * + * Purpose: + * + * Makes sure that the true file size is the same (or larger) + * than the end-of-address. + * + * NOT POSSIBLE ON READ-ONLY S3 FILES. + * + * Return: + * + * FAIL (Not possible on Read-Only S3 files.) + * + * Programmer: Jacob Smith + * 2017-10-23 + * + * Changes: None. + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_hdfs_truncate( + H5FD_t H5_ATTR_UNUSED *_file, + hid_t H5_ATTR_UNUSED dxpl_id, + hbool_t H5_ATTR_UNUSED closing) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_truncate() called.\n"); +#endif + + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, + "cannot truncate read-only file.") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_hdfs_truncate() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_hdfs_lock() + * + * Purpose: + * + * Place an advisory lock on a file. + * No effect on Read-Only S3 file. + * + * Suggestion: remove lock/unlock from class + * > would result in error at H5FD_[un]lock() (H5FD.c) + * + * Return: + * + * SUCCEED (No-op always succeeds) + * + * Programmer: Jacob Smith + * 2017-11-03 + * + * Changes: None. + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_hdfs_lock( + H5FD_t H5_ATTR_UNUSED *_file, + hbool_t H5_ATTR_UNUSED rw) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5FD_hdfs_lock() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_hdfs_unlock() + * + * Purpose: + * + * Remove the existing lock on the file. + * No effect on Read-Only S3 file. + * + * Return: + * + * SUCCEED (No-op always succeeds) + * + * Programmer: Jacob Smith + * 2017-11-03 + * + * Changes: None. + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_hdfs_unlock(H5FD_t H5_ATTR_UNUSED *_file) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5FD_hdfs_unlock() */ + diff --git a/src/H5FDhdfs.h b/src/H5FDhdfs.h new file mode 100644 index 0000000..3d4128d --- /dev/null +++ b/src/H5FDhdfs.h @@ -0,0 +1,122 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Read-Only HDFS Virtual File Driver (VFD) * + * Copyright (c) 2018, The HDF Group. * + * * + * All rights reserved. * + * * + * NOTICE: * + * All information contained herein is, and remains, the property of The HDF * + * Group. The intellectual and technical concepts contained herein are * + * proprietary to The HDF Group. Dissemination of this information or * + * reproduction of this material is strictly forbidden unless prior written * + * permission is obtained from The HDF Group. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Programmer: Jacob Smith + * 2018-04-23 + * + * Purpose: The public header file for the hdfs driver. + */ + +#ifndef H5FDhdfs_H +#define H5FDhdfs_H + +#define H5FD_HDFS (H5FD_hdfs_init()) + +#ifdef __cplusplus +extern "C" { +#endif + +/**************************************************************************** + * + * Structure: H5FD_hdfs_fapl_t + * + * Purpose: + * + * H5FD_hdfs_fapl_t is a public structure that is used to pass + * configuration information to the appropriate HDFS VFD via the FAPL. + * A pointer to an instance of this structure is a parameter to + * H5Pset_fapl_hdfs() and H5Pget_fapl_hdfs(). + * + * + * + * `version` (int32_t) + * + * Version number of the `H5FD_hdfs_fapl_t` structure. Any instance passed + * to the above calls must have a recognized version number, or an error + * will be flagged. + * + * This field should be set to `H5FD__CURR_HDFS_FAPL_T_VERSION`. + * + * `namenode_name` (const char[]) + * + * Name of "Name Node" to access as the HDFS server. + * + * Must not be longer than `H5FD__HDFS_NODE_NAME_SPACE`. + * + * TBD: Can be NULL. + * + * `namenode_port` (int32_t) TBD + * + * Port number to use to connect with Name Node. + * + * TBD: If 0, uses a default port. + * + * `kerberos_ticket_cache` (const char[]) + * + * Path to the location of the Kerberos authentication cache. + * + * Must not be longer than `H5FD__HDFS_KERB_CACHE_PATH_SPACE`. + * + * TBD: Can be NULL. + * + * `user_name` (const char[]) + * + * Username to use when accessing file. + * + * Must not be longer than `H5FD__HDFS_USER_NAME_SPACE`. + * + * TBD: Can be NULL. + * + * `stream_buffer_size` (int32_t) + * + * Size (in bytes) of the file read stream buffer. + * + * TBD: If -1, relies on a default value. + * + * + * + * Programmer: Jacob Smith + * 2018-04-23 + * + * Changes: None + * + ****************************************************************************/ + +#define H5FD__CURR_HDFS_FAPL_T_VERSION 1 + +#define H5FD__HDFS_NODE_NAME_SPACE 128 +#define H5FD__HDFS_USER_NAME_SPACE 128 +#define H5FD__HDFS_KERB_CACHE_PATH_SPACE 128 + +typedef struct H5FD_hdfs_fapl_t { + int32_t version; + char namenode_name[H5FD__HDFS_NODE_NAME_SPACE + 1]; + int32_t namenode_port; + char user_name[H5FD__HDFS_USER_NAME_SPACE + 1]; + char kerberos_ticket_cache[H5FD__HDFS_KERB_CACHE_PATH_SPACE + 1]; + int32_t stream_buffer_size; +} H5FD_hdfs_fapl_t; + +H5_DLL hid_t H5FD_hdfs_init(void); +H5_DLL herr_t H5Pget_fapl_hdfs(hid_t fapl_id, H5FD_hdfs_fapl_t *fa_out); +H5_DLL herr_t H5Pset_fapl_hdfs(hid_t fapl_id, H5FD_hdfs_fapl_t *fa); + +#ifdef __cplusplus +} +#endif + +#endif /* ifndef H5FDhdfs_H */ + + diff --git a/src/H5FDmpi.c b/src/H5FDmpi.c index 7d58b7a..b2959a5 100644 --- a/src/H5FDmpi.c +++ b/src/H5FDmpi.c @@ -267,115 +267,6 @@ H5FD_mpi_haddr_to_MPIOff(haddr_t addr, MPI_Offset *mpi_off/*out*/) FUNC_LEAVE_NOAPI(ret_value) } - -/*------------------------------------------------------------------------- - * Function: H5FD_mpi_comm_info_dup - * - * Purpose: Make duplicates of communicator and Info object. - * If the Info object is in fact MPI_INFO_NULL, no duplicate - * is made but the same value assigned to the new Info object - * handle. - * - * Return: Success: Non-negative. The new communicator and Info - * object handles are returned via comm_new and - * info_new pointers. - * - * Failure: Negative. - * - * Programmer: Albert Cheng - * Jan 8, 2003 - * - * Modifications: - *------------------------------------------------------------------------- - */ -herr_t -H5FD_mpi_comm_info_dup(MPI_Comm comm, MPI_Info info, MPI_Comm *comm_new, MPI_Info *info_new) -{ - herr_t ret_value=SUCCEED; - MPI_Comm comm_dup=MPI_COMM_NULL; - MPI_Info info_dup=MPI_INFO_NULL; - int mpi_code; - - FUNC_ENTER_NOAPI(FAIL) - - /* Check arguments */ - if (MPI_COMM_NULL == comm) - HGOTO_ERROR(H5E_INTERNAL, H5E_BADVALUE, FAIL, "not a valid argument") - if (!comm_new || !info_new) - HGOTO_ERROR(H5E_INTERNAL, H5E_BADVALUE, FAIL, "bad pointers") - - /* Dup them. Using temporary variables for error recovery cleanup. */ - if (MPI_SUCCESS != (mpi_code=MPI_Comm_dup(comm, &comm_dup))) - HMPI_GOTO_ERROR(FAIL, "MPI_Comm_dup failed", mpi_code) - if (MPI_INFO_NULL != info){ - if (MPI_SUCCESS != (mpi_code=MPI_Info_dup(info, &info_dup))) - HMPI_GOTO_ERROR(FAIL, "MPI_Info_dup failed", mpi_code) - }else{ - /* No dup, just copy it. */ - info_dup = info; - } - - /* Set MPI_ERRORS_RETURN on comm_dup so that MPI failures are not fatal, - and return codes can be checked and handled. May 23, 2017 FTW */ - if (MPI_SUCCESS != (mpi_code = MPI_Comm_set_errhandler(comm_dup, MPI_ERRORS_RETURN))) - HMPI_GOTO_ERROR(FAIL, "MPI_Errhandler_set failed", mpi_code) - - /* copy them to the return arguments */ - *comm_new = comm_dup; - *info_new = info_dup; - -done: - if (FAIL == ret_value){ - /* need to free anything created here */ - if (MPI_COMM_NULL != comm_dup) - MPI_Comm_free(&comm_dup); - if (MPI_INFO_NULL != info_dup) - MPI_Info_free(&info_dup); - } - - FUNC_LEAVE_NOAPI(ret_value) -} - - -/*------------------------------------------------------------------------- - * Function: H5FD_mpi_comm_info_free - * - * Purpose: Free the communicator and Info object. - * If comm or info is in fact MPI_COMM_NULL or MPI_INFO_NULL - * respectively, no action occurs to it. - * - * Return: Success: Non-negative. The values the pointers refer - * to will be set to the corresponding NULL - * handles. - * - * Failure: Negative. - * - * Programmer: Albert Cheng - * Jan 8, 2003 - * - * Modifications: - *------------------------------------------------------------------------- - */ -herr_t -H5FD_mpi_comm_info_free(MPI_Comm *comm, MPI_Info *info) -{ - herr_t ret_value=SUCCEED; - - FUNC_ENTER_NOAPI(FAIL) - - /* Check arguments */ - if (!comm || !info) - HGOTO_ERROR(H5E_INTERNAL, H5E_BADVALUE, FAIL, "not a valid argument") - - if (MPI_COMM_NULL != *comm) - MPI_Comm_free(comm); - if (MPI_INFO_NULL != *info) - MPI_Info_free(info); - -done: - FUNC_LEAVE_NOAPI(ret_value) -} - #ifdef NOT_YET /*------------------------------------------------------------------------- diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c index d5aa170..71e9fe1 100644 --- a/src/H5FDmpio.c +++ b/src/H5FDmpio.c @@ -58,27 +58,24 @@ static char H5FD_mpi_native_g[] = "native"; * driver doesn't bother to keep it updated since it's an expensive operation. */ typedef struct H5FD_mpio_t { - H5FD_t pub; /*public stuff, must be first */ - MPI_File f; /*MPIO file handle */ - MPI_Comm comm; /*communicator */ - MPI_Info info; /*file information */ - int mpi_rank; /* This process's rank */ - int mpi_size; /* Total number of processes */ - haddr_t eof; /*end-of-file marker */ - haddr_t eoa; /*end-of-address marker */ - haddr_t last_eoa; /* Last known end-of-address marker */ - haddr_t local_eof; /* Local end-of-file address for each process */ + H5FD_t pub; /* Public stuff, must be first */ + MPI_File f; /* MPIO file handle */ + MPI_Comm comm; /* MPI Communicator */ + MPI_Info info; /* MPI info object */ + int mpi_rank; /* This process's rank */ + int mpi_size; /* Total number of processes */ + haddr_t eof; /* End-of-file marker */ + haddr_t eoa; /* End-of-address marker */ + haddr_t last_eoa; /* Last known end-of-address marker */ + haddr_t local_eof; /* Local end-of-file address for each process */ } H5FD_mpio_t; /* Private Prototypes */ /* Callbacks */ static herr_t H5FD__mpio_term(void); -static void *H5FD__mpio_fapl_get(H5FD_t *_file); -static void *H5FD__mpio_fapl_copy(const void *_old_fa); -static herr_t H5FD__mpio_fapl_free(void *_fa); static H5FD_t *H5FD__mpio_open(const char *name, unsigned flags, hid_t fapl_id, - haddr_t maxaddr); + haddr_t maxaddr); static herr_t H5FD__mpio_close(H5FD_t *_file); static herr_t H5FD__mpio_query(const H5FD_t *_f1, unsigned long *flags); static haddr_t H5FD__mpio_get_eoa(const H5FD_t *_file, H5FD_mem_t type); @@ -106,10 +103,10 @@ static const H5FD_class_mpi_t H5FD_mpio_g = { NULL, /*sb_size */ NULL, /*sb_encode */ NULL, /*sb_decode */ - sizeof(H5FD_mpio_fapl_t), /*fapl_size */ - H5FD__mpio_fapl_get, /*fapl_get */ - H5FD__mpio_fapl_copy, /*fapl_copy */ - H5FD__mpio_fapl_free, /*fapl_free */ + 0, /*fapl_size */ + NULL, /*fapl_get */ + NULL, /*fapl_copy */ + NULL, /*fapl_free */ 0, /*dxpl_size */ NULL, /*dxpl_copy */ NULL, /*dxpl_free */ @@ -167,7 +164,7 @@ USAGE herr_t H5FD__init_package() RETURNS - Non-negative on success/Negative on failure + SUCCEED/FAIL DESCRIPTION Initializes any interface-specific data or routines. (Just calls @@ -249,9 +246,9 @@ done: /*--------------------------------------------------------------------------- - * Function: H5FD__mpio_term + * Function: H5FD__mpio_term * - * Purpose: Shut down the VFD + * Purpose: Shut down the VFD * * Returns: Non-negative on success or negative on failure * @@ -263,7 +260,7 @@ done: static herr_t H5FD__mpio_term(void) { - FUNC_ENTER_STATIC + FUNC_ENTER_STATIC_NOERR /* Reset VFL ID */ H5FD_MPIO_g = 0; @@ -273,40 +270,39 @@ H5FD__mpio_term(void) /*------------------------------------------------------------------------- - * Function: H5Pset_fapl_mpio + * Function: H5Pset_fapl_mpio * - * Purpose: Store the user supplied MPIO communicator comm and info in - * the file access property list FAPL_ID which can then be used - * to create and/or open the file. This function is available - * only in the parallel HDF5 library and is not collective. + * Purpose: Store the user supplied MPIO communicator comm and info in + * the file access property list FAPL_ID which can then be used + * to create and/or open the file. This function is available + * only in the parallel HDF5 library and is not collective. * - * comm is the MPI communicator to be used for file open as - * defined in MPI_FILE_OPEN of MPI-2. This function makes a - * duplicate of comm. Any modification to comm after this function - * call returns has no effect on the access property list. + * comm is the MPI communicator to be used for file open as + * defined in MPI_FILE_OPEN of MPI-2. This function makes a + * duplicate of comm. Any modification to comm after this function + * call returns has no effect on the access property list. * - * info is the MPI Info object to be used for file open as - * defined in MPI_FILE_OPEN of MPI-2. This function makes a - * duplicate of info. Any modification to info after this - * function call returns has no effect on the access property - * list. + * info is the MPI Info object to be used for file open as + * defined in MPI_FILE_OPEN of MPI-2. This function makes a + * duplicate of info. Any modification to info after this + * function call returns has no effect on the access property + * list. * * If fapl_id has previously set comm and info values, they * will be replaced and the old communicator and Info object * are freed. * - * Return: Success: Non-negative - * Failure: Negative + * Return: Success: Non-negative + * Failure: Negative * - * Programmer: Albert Cheng - * Feb 3, 1998 + * Programmer: Albert Cheng + * Feb 3, 1998 * *------------------------------------------------------------------------- */ herr_t H5Pset_fapl_mpio(hid_t fapl_id, MPI_Comm comm, MPI_Info info) { - H5FD_mpio_fapl_t fa; H5P_genplist_t *plist; /* Property list pointer */ herr_t ret_value; @@ -319,15 +315,16 @@ H5Pset_fapl_mpio(hid_t fapl_id, MPI_Comm comm, MPI_Info info) if(NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a file access list") if(MPI_COMM_NULL == comm) - HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a valid communicator") + HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "MPI_COMM_NULL is not a valid communicator") - /* Initialize driver specific properties */ - HDmemset(&fa, 0, sizeof(H5FD_mpio_fapl_t)); - fa.comm = comm; - fa.info = info; + /* Set the MPI communicator and info object */ + if(H5P_set(plist, H5F_ACS_MPI_PARAMS_COMM_NAME, &comm) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set MPI communicator") + if(H5P_set(plist, H5F_ACS_MPI_PARAMS_INFO_NAME, &info) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set MPI info object") /* duplication is done during driver setting. */ - ret_value = H5P_set_driver(plist, H5FD_MPIO, &fa); + ret_value = H5P_set_driver(plist, H5FD_MPIO, NULL); done: FUNC_LEAVE_API(ret_value) @@ -335,25 +332,25 @@ done: /*------------------------------------------------------------------------- - * Function: H5Pget_fapl_mpio - * - * Purpose: If the file access property list is set to the H5FD_MPIO - * driver then this function returns duplicates of the MPI - * communicator and Info object stored through the comm and - * info pointers. It is the responsibility of the application - * to free the returned communicator and Info object. - * - * Return: Success: Non-negative with the communicator and - * Info object returned through the comm and - * info arguments if non-null. Since they are - * duplicates of the stored objects, future - * modifications to the access property list do - * not affect them and it is the responsibility - * of the application to free them. - * Failure: Negative - * - * Programmer: Robb Matzke - * Thursday, February 26, 1998 + * Function: H5Pget_fapl_mpio + * + * Purpose: If the file access property list is set to the H5FD_MPIO + * driver then this function returns duplicates of the MPI + * communicator and Info object stored through the comm and + * info pointers. It is the responsibility of the application + * to free the returned communicator and Info object. + * + * Return: Success: Non-negative with the communicator and + * Info object returned through the comm and + * info arguments if non-null. Since they are + * duplicates of the stored objects, future + * modifications to the access property list do + * not affect them and it is the responsibility + * of the application to free them. + * Failure: Negative + * + * Programmer: Robb Matzke + * Thursday, February 26, 1998 * *------------------------------------------------------------------------- */ @@ -361,76 +358,67 @@ herr_t H5Pget_fapl_mpio(hid_t fapl_id, MPI_Comm *comm/*out*/, MPI_Info *info/*out*/) { H5P_genplist_t *plist; /* Property list pointer */ - const H5FD_mpio_fapl_t *fa; /* MPIO fapl info */ - MPI_Comm comm_tmp = MPI_COMM_NULL; - hbool_t comm_copied = FALSE; /* MPI Comm has been duplicated */ - int mpi_code; /* MPI return code */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) H5TRACE3("e", "ixx", fapl_id, comm, info); + /* Set comm and info in case we have problems */ + if(comm) + *comm = MPI_COMM_NULL; + if(info) + *info = MPI_INFO_NULL; + /* Check arguments */ if(NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a file access list") if(H5FD_MPIO != H5P_peek_driver(plist)) - HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver") - if(NULL == (fa = (const H5FD_mpio_fapl_t *)H5P_peek_driver_info(plist))) - HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "bad VFL driver info") - - /* Store the duplicated communicator in a temporary variable for error */ - /* recovery in case the INFO duplication fails. */ - if(comm) { - if(MPI_SUCCESS != (mpi_code = MPI_Comm_dup(fa->comm, &comm_tmp))) - HMPI_GOTO_ERROR(FAIL, "MPI_Comm_dup failed", mpi_code) - comm_copied = TRUE; - } /* end if */ - - if(info) { - if(MPI_INFO_NULL != fa->info) { - if(MPI_SUCCESS != (mpi_code = MPI_Info_dup(fa->info, info))) - HMPI_GOTO_ERROR(FAIL, "MPI_Info_dup failed", mpi_code) - } /* end if */ - else - /* do not dup it */ - *info = MPI_INFO_NULL; - } /* end if */ + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "VFL driver is not MPI-I/O") - /* Store the copied communicator, now that the Info object has been - * successfully copied. - */ + /* Get the MPI communicator and info object */ if(comm) - *comm = comm_tmp; + if(H5P_get(plist, H5F_ACS_MPI_PARAMS_COMM_NAME, comm) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get MPI communicator") + if(info) + if(H5P_get(plist, H5F_ACS_MPI_PARAMS_INFO_NAME, info) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get MPI info object") done: - if(ret_value < 0) - /* need to free anything created here */ - if(comm_copied) - MPI_Comm_free(&comm_tmp); + /* Clean up anything duplicated on errors. The free calls will set + * the output values to MPI_COMM|INFO_NULL. + */ + if(ret_value != SUCCEED) { + if(comm) + if(H5_mpi_comm_free(comm) < 0) + HDONE_ERROR(H5E_PLIST, H5E_CANTFREE, FAIL, "unable to free MPI communicator") + if(info) + if(H5_mpi_info_free(info) < 0) + HDONE_ERROR(H5E_PLIST, H5E_CANTFREE, FAIL, "unable to free MPI info object") + } FUNC_LEAVE_API(ret_value) } /* end H5Pget_fapl_mpio() */ /*------------------------------------------------------------------------- - * Function: H5Pset_dxpl_mpio + * Function: H5Pset_dxpl_mpio * - * Purpose: Set the data transfer property list DXPL_ID to use transfer - * mode XFER_MODE. The property list can then be used to control - * the I/O transfer mode during data I/O operations. The valid - * transfer modes are: + * Purpose: Set the data transfer property list DXPL_ID to use transfer + * mode XFER_MODE. The property list can then be used to control + * the I/O transfer mode during data I/O operations. The valid + * transfer modes are: * - * H5FD_MPIO_INDEPENDENT: - * Use independent I/O access (the default). + * H5FD_MPIO_INDEPENDENT: + * Use independent I/O access (the default). * - * H5FD_MPIO_COLLECTIVE: - * Use collective I/O access. + * H5FD_MPIO_COLLECTIVE: + * Use collective I/O access. * - * Return: Success: Non-negative - * Failure: Negative + * Return: Success: Non-negative + * Failure: Negative * - * Programmer: Albert Cheng - * April 2, 1998 + * Programmer: Albert Cheng + * April 2, 1998 * *------------------------------------------------------------------------- */ @@ -461,18 +449,18 @@ done: /*------------------------------------------------------------------------- - * Function: H5Pget_dxpl_mpio + * Function: H5Pget_dxpl_mpio * - * Purpose: Queries the transfer mode current set in the data transfer - * property list DXPL_ID. This is not collective. + * Purpose: Queries the transfer mode current set in the data transfer + * property list DXPL_ID. This is not collective. * - * Return: Success: Non-negative, with the transfer mode returned - * through the XFER_MODE argument if it is - * non-null. - * Failure: Negative + * Return: Success: Non-negative, with the transfer mode returned + * through the XFER_MODE argument if it is + * non-null. + * Failure: Negative * - * Programmer: Albert Cheng - * April 2, 1998 + * Programmer: Albert Cheng + * April 2, 1998 * *------------------------------------------------------------------------- */ @@ -500,21 +488,20 @@ done: /*------------------------------------------------------------------------- - * Function: H5Pset_dxpl_mpio_collective_opt + * Function: H5Pset_dxpl_mpio_collective_opt * - * Purpose: To set a flag to choose linked chunk I/O or multi-chunk I/O - * without involving decision-making inside HDF5 + * Purpose: To set a flag to choose linked chunk I/O or multi-chunk I/O + * without involving decision-making inside HDF5 * - * Note: The library will do linked chunk I/O or multi-chunk I/O without - * involving communications for decision-making process. - * The library won't behave as it asks for only when we find - * that the low-level MPI-IO package doesn't support this. + * Note: The library will do linked chunk I/O or multi-chunk I/O without + * involving communications for decision-making process. + * The library won't behave as it asks for only when we find + * that the low-level MPI-IO package doesn't support this. * - * Return: Success: Non-negative - * Failure: Negative + * Return: Success: Non-negative + * Failure: Negative * - * Programmer: Kent Yang - * ? ?, ? + * Programmer: Kent Yang * *------------------------------------------------------------------------- */ @@ -543,21 +530,20 @@ done: /*------------------------------------------------------------------------- - * Function: H5Pset_dxpl_mpio_chunk_opt + * Function: H5Pset_dxpl_mpio_chunk_opt * - * Purpose: To set a flag to choose linked chunk I/O or multi-chunk I/O - * without involving decision-making inside HDF5 + * Purpose: To set a flag to choose linked chunk I/O or multi-chunk I/O + * without involving decision-making inside HDF5 * - * Note: The library will do linked chunk I/O or multi-chunk I/O without - * involving communications for decision-making process. - * The library won't behave as it asks for only when we find - * that the low-level MPI-IO package doesn't support this. + * Note: The library will do linked chunk I/O or multi-chunk I/O without + * involving communications for decision-making process. + * The library won't behave as it asks for only when we find + * that the low-level MPI-IO package doesn't support this. * - * Return: Success: Non-negative - * Failure: Negative + * Return: Success: Non-negative + * Failure: Negative * - * Programmer: Kent Yang - * ? ?, ? + * Programmer: Kent Yang * *------------------------------------------------------------------------- */ @@ -586,19 +572,18 @@ done: /*------------------------------------------------------------------------- - * Function: H5Pset_dxpl_mpio_chunk_opt_num + * Function: H5Pset_dxpl_mpio_chunk_opt_num * - * Purpose: To set a threshold for doing linked chunk IO + * Purpose: To set a threshold for doing linked chunk IO * - * Note: If the number is greater than the threshold set by the user, - * the library will do linked chunk I/O; otherwise, I/O will be - * done for every chunk. + * Note: If the number is greater than the threshold set by the user, + * the library will do linked chunk I/O; otherwise, I/O will be + * done for every chunk. * - * Return: Success: Non-negative - * Failure: Negative + * Return: Success: Non-negative + * Failure: Negative * - * Programmer: Kent Yang - * ? ?, ? + * Programmer: Kent Yang * *------------------------------------------------------------------------- */ @@ -627,22 +612,21 @@ done: /*------------------------------------------------------------------------- - * Function: H5Pset_dxpl_mpio_chunk_opt_ratio + * Function: H5Pset_dxpl_mpio_chunk_opt_ratio * - * Purpose: To set a threshold for doing collective I/O for each chunk + * Purpose: To set a threshold for doing collective I/O for each chunk * - * Note: The library will calculate the percentage of the number of - * process holding selections at each chunk. If that percentage - * of number of process in the individual chunk is greater than - * the threshold set by the user, the library will do collective - * chunk I/O for this chunk; otherwise, independent I/O will be - * done for this chunk. + * Note: The library will calculate the percentage of the number of + * process holding selections at each chunk. If that percentage + * of number of process in the individual chunk is greater than + * the threshold set by the user, the library will do collective + * chunk I/O for this chunk; otherwise, independent I/O will be + * done for this chunk. * - * Return: Success: Non-negative - * Failure: Negative + * Return: Success: Non-negative + * Failure: Negative * - * Programmer: Kent Yang - * ? ?, ? + * Programmer: Kent Yang * *------------------------------------------------------------------------- */ @@ -671,158 +655,14 @@ done: /*------------------------------------------------------------------------- - * Function: H5FD__mpio_fapl_get - * - * Purpose: Returns a file access property list which could be used to - * create another file the same as this one. - * - * Return: Success: Ptr to new file access property list with all - * fields copied from the file pointer. - * Failure: NULL - * - * Programmer: Robb Matzke - * Friday, August 13, 1999 - * - *------------------------------------------------------------------------- - */ -static void * -H5FD__mpio_fapl_get(H5FD_t *_file) -{ - H5FD_mpio_t *file = (H5FD_mpio_t*)_file; - H5FD_mpio_fapl_t *fa = NULL; - void *ret_value; /* Return value */ - - FUNC_ENTER_STATIC - - /* Sanity checks */ - HDassert(file); - HDassert(H5FD_MPIO == file->pub.driver_id); - - /* Check arguments */ - if(NULL == (fa = (H5FD_mpio_fapl_t *)H5MM_calloc(sizeof(H5FD_mpio_fapl_t)))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - - /* Duplicate communicator and Info object. */ - if(FAIL == H5FD_mpi_comm_info_dup(file->comm, file->info, &fa->comm, &fa->info)) - HGOTO_ERROR(H5E_INTERNAL, H5E_CANTCOPY, NULL, "Communicator/Info duplicate failed") - - /* Set return value */ - ret_value = fa; - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD__mpio_fapl_get() */ - - -/*------------------------------------------------------------------------- - * Function: H5FD__mpio_fapl_copy - * - * Purpose: Copies the mpio-specific file access properties. - * - * Return: Success: Ptr to a new property list - * Failure: NULL - * - * Programmer: Albert Cheng - * Jan 8, 2003 - * - *------------------------------------------------------------------------- - */ -static void * -H5FD__mpio_fapl_copy(const void *_old_fa) -{ - const H5FD_mpio_fapl_t *old_fa = (const H5FD_mpio_fapl_t*)_old_fa; - H5FD_mpio_fapl_t *new_fa = NULL; - void *ret_value = NULL; - - FUNC_ENTER_STATIC - -#ifdef H5FDmpio_DEBUG -if(H5FD_mpio_Debug[(int)'t']) - HDfprintf(stderr, "%s: entering\n", FUNC); -#endif - - if(NULL == (new_fa = (H5FD_mpio_fapl_t *)H5MM_malloc(sizeof(H5FD_mpio_fapl_t)))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - - /* Copy the general information */ - H5MM_memcpy(new_fa, old_fa, sizeof(H5FD_mpio_fapl_t)); - - /* Duplicate communicator and Info object. */ - if(H5FD_mpi_comm_info_dup(old_fa->comm, old_fa->info, &new_fa->comm, &new_fa->info) < 0) - HGOTO_ERROR(H5E_INTERNAL, H5E_CANTCOPY, NULL, "Communicator/Info duplicate failed") - - /* Set return value */ - ret_value = new_fa; - -done: - if(NULL == ret_value) - /* cleanup */ - if(new_fa) - H5MM_xfree(new_fa); - -#ifdef H5FDmpio_DEBUG -if(H5FD_mpio_Debug[(int)'t']) - HDfprintf(stderr, "%s: leaving\n", FUNC); -#endif - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD__mpio_fapl_copy() */ - - -/*------------------------------------------------------------------------- - * Function: H5FD__mpio_fapl_free - * - * Purpose: Frees the mpio-specific file access properties. - * - * Return: Success: 0 - * Failure: -1 - * - * Programmer: Albert Cheng - * Jan 8, 2003 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5FD__mpio_fapl_free(void *_fa) -{ - H5FD_mpio_fapl_t *fa = (H5FD_mpio_fapl_t*)_fa; - herr_t ret_value = SUCCEED; - - FUNC_ENTER_STATIC - -#ifdef H5FDmpio_DEBUG -if(H5FD_mpio_Debug[(int)'t']) - HDfprintf(stderr, "%s: Entering\n", FUNC); -#endif - - /* Sanity checks */ - HDassert(fa); - - /* Free the internal communicator and INFO object */ - HDassert(MPI_COMM_NULL != fa->comm); - H5FD_mpi_comm_info_free(&fa->comm, &fa->info); - H5MM_xfree(fa); - -#ifdef H5FDmpio_DEBUG -if(H5FD_mpio_Debug[(int)'t']) - HDfprintf(stderr, "%s: leaving\n", FUNC); -#endif - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD__mpio_fapl_free() */ - - -/*------------------------------------------------------------------------- - * Function: H5FD_set_mpio_atomicity - * - * Purpose: Sets the atomicity mode + * Function: H5FD_set_mpio_atomicity * - * Return: Success: Non-negative + * Purpose: Sets the atomicity mode * - * Failure: Negative + * Return: SUCCEED/FAIL * - * Programmer: Mohamad Chaarawi - * Feb 14, 2012 + * Programmer: Mohamad Chaarawi + * Feb 14, 2012 * *------------------------------------------------------------------------- */ @@ -838,7 +678,7 @@ H5FD_set_mpio_atomicity(H5FD_t *_file, hbool_t flag) #ifdef H5FDmpio_DEBUG if(H5FD_mpio_Debug[(int)'t']) - HDfprintf(stdout, "%s: Entering\n", FUNC); + HDfprintf(stdout, "%s: Entering\n", FUNC); #endif if(FALSE == flag) @@ -853,7 +693,7 @@ H5FD_set_mpio_atomicity(H5FD_t *_file, hbool_t flag) done: #ifdef H5FDmpio_DEBUG if(H5FD_mpio_Debug[(int)'t']) - HDfprintf(stdout, "%s: Leaving\n", FUNC); + HDfprintf(stdout, "%s: Leaving\n", FUNC); #endif FUNC_LEAVE_NOAPI(ret_value) @@ -861,16 +701,14 @@ done: /*------------------------------------------------------------------------- - * Function: H5FD_get_mpio_atomicity + * Function: H5FD_get_mpio_atomicity * - * Purpose: Returns the atomicity mode + * Purpose: Returns the atomicity mode * - * Return: Success: Non-negative + * Return: SUCCEED/FAIL * - * Failure: Negative - * - * Programmer: Mohamad Chaarawi - * Feb 14, 2012 + * Programmer: Mohamad Chaarawi + * Feb 14, 2012 * *------------------------------------------------------------------------- */ @@ -886,7 +724,7 @@ H5FD_get_mpio_atomicity(H5FD_t *_file, hbool_t *flag) #ifdef H5FDmpio_DEBUG if(H5FD_mpio_Debug[(int)'t']) - HDfprintf(stdout, "%s: Entering\n", FUNC); + HDfprintf(stdout, "%s: Entering\n", FUNC); #endif /* Get atomicity value */ @@ -901,7 +739,7 @@ H5FD_get_mpio_atomicity(H5FD_t *_file, hbool_t *flag) done: #ifdef H5FDmpio_DEBUG if(H5FD_mpio_Debug[(int)'t']) - HDfprintf(stdout, "%s: Leaving\n", FUNC); + HDfprintf(stdout, "%s: Leaving\n", FUNC); #endif FUNC_LEAVE_NOAPI(ret_value) @@ -912,14 +750,14 @@ done: * Function: H5FD__mpio_open * * Purpose: Opens a file with name NAME. The FLAGS are a bit field with - * purpose similar to the second argument of open(2) and which - * are defined in H5Fpublic.h. The file access property list - * FAPL_ID contains the properties driver properties and MAXADDR - * is the largest address which this file will be expected to - * access. This is collective. + * purpose similar to the second argument of open(2) and which + * are defined in H5Fpublic.h. The file access property list + * FAPL_ID contains the properties driver properties and MAXADDR + * is the largest address which this file will be expected to + * access. This is collective. * - * Return: Success: A new file pointer. - * Failure: NULL + * Return: Success: A new file pointer + * Failure: NULL * * Programmer: Robert Kim Yates * January 30, 1998 @@ -930,46 +768,38 @@ static H5FD_t * H5FD__mpio_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t H5_ATTR_UNUSED maxaddr) { - H5FD_mpio_t *file = NULL; - MPI_File fh; - unsigned file_opened = 0; /* Flag to indicate that the file was successfully opened */ - int mpi_amode; - int mpi_rank; /* MPI rank of this process */ - int mpi_size; /* Total number of MPI processes */ - int mpi_code; /* mpi return code */ - MPI_Offset size; - const H5FD_mpio_fapl_t *fa = NULL; - H5FD_mpio_fapl_t _fa; + H5FD_mpio_t *file = NULL; + MPI_File fh; + hbool_t file_opened = FALSE; /* Flag to indicate that the file was successfully opened */ + int mpi_amode; + int mpi_rank; /* MPI rank of this process */ + int mpi_size; /* Total number of MPI processes */ + int mpi_code; /* MPI return code */ + MPI_Offset size; H5P_genplist_t *plist; /* Property list pointer */ - MPI_Comm comm_dup = MPI_COMM_NULL; - MPI_Info info_dup = MPI_INFO_NULL; - H5FD_t *ret_value; /* Return value */ + MPI_Comm comm = MPI_COMM_NULL; + MPI_Info info = MPI_INFO_NULL; + H5FD_t *ret_value = NULL; /* Return value */ FUNC_ENTER_STATIC #ifdef H5FDmpio_DEBUG if(H5FD_mpio_Debug[(int)'t']) - HDfprintf(stdout, "%s: Entering - name = \"%s\", flags = 0x%x, fapl_id = %d, maxaddr = %lu\n", FUNC, name, flags, (int)fapl_id, (unsigned long)maxaddr); + HDfprintf(stdout, "%s: Entering - name = \"%s\", flags = 0x%x, fapl_id = %d, maxaddr = %lu\n", FUNC, name, flags, (int)fapl_id, (unsigned long)maxaddr); #endif - /* Obtain a pointer to mpio-specific file access properties */ + /* Get a pointer to the fapl */ if(NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list") - if(H5P_FILE_ACCESS_DEFAULT == fapl_id || H5FD_MPIO != H5P_peek_driver(plist)) { - _fa.comm = MPI_COMM_SELF; /*default*/ - _fa.info = MPI_INFO_NULL; /*default*/ - fa = &_fa; - } /* end if */ - else - if(NULL == (fa = (const H5FD_mpio_fapl_t *)H5P_peek_driver_info(plist))) - HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, NULL, "bad VFL driver info") - /* Duplicate communicator and Info object for use by this file. */ - if(FAIL == H5FD_mpi_comm_info_dup(fa->comm, fa->info, &comm_dup, &info_dup)) - HGOTO_ERROR(H5E_INTERNAL, H5E_CANTCOPY, NULL, "Communicator/Info duplicate failed") + /* Get the MPI communicator and info object from the property list */ + if(H5P_get(plist, H5F_ACS_MPI_PARAMS_COMM_NAME, &comm) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get MPI communicator") + if(H5P_get(plist, H5F_ACS_MPI_PARAMS_INFO_NAME, &info) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get MPI info object") - /* convert HDF5 flags to MPI-IO flags */ - /* some combinations are illegal; let MPI-IO figure it out */ + /* Convert HDF5 flags to MPI-IO flags */ + /* Some combinations are illegal; let MPI-IO figure it out */ mpi_amode = (flags & H5F_ACC_RDWR) ? MPI_MODE_RDWR : MPI_MODE_RDONLY; if(flags & H5F_ACC_CREAT) mpi_amode |= MPI_MODE_CREATE; @@ -978,11 +808,11 @@ H5FD__mpio_open(const char *name, unsigned flags, hid_t fapl_id, #ifdef H5FDmpio_DEBUG /* Check for debug commands in the info parameter */ - if(MPI_INFO_NULL != info_dup) { + if(MPI_INFO_NULL != info) { char debug_str[128]; int flag; - MPI_Info_get(fa->info, H5F_MPIO_DEBUG_KEY, sizeof(debug_str) - 1, debug_str, &flag); + MPI_Info_get(info, H5F_MPIO_DEBUG_KEY, sizeof(debug_str) - 1, debug_str, &flag); if(flag) { int i; @@ -993,22 +823,22 @@ H5FD__mpio_open(const char *name, unsigned flags, hid_t fapl_id, } /* end if */ #endif - if(MPI_SUCCESS != (mpi_code = MPI_File_open(comm_dup, name, mpi_amode, info_dup, &fh))) + if(MPI_SUCCESS != (mpi_code = MPI_File_open(comm, name, mpi_amode, info, &fh))) HMPI_GOTO_ERROR(NULL, "MPI_File_open failed", mpi_code) - file_opened=1; + file_opened = TRUE; /* Get the MPI rank of this process and the total number of processes */ - if (MPI_SUCCESS != (mpi_code=MPI_Comm_rank (comm_dup, &mpi_rank))) + if (MPI_SUCCESS != (mpi_code = MPI_Comm_rank (comm, &mpi_rank))) HMPI_GOTO_ERROR(NULL, "MPI_Comm_rank failed", mpi_code) - if (MPI_SUCCESS != (mpi_code=MPI_Comm_size (comm_dup, &mpi_size))) + if (MPI_SUCCESS != (mpi_code = MPI_Comm_size (comm, &mpi_size))) HMPI_GOTO_ERROR(NULL, "MPI_Comm_size failed", mpi_code) /* Build the return value and initialize it */ if(NULL == (file = (H5FD_mpio_t *)H5MM_calloc(sizeof(H5FD_mpio_t)))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") file->f = fh; - file->comm = comm_dup; - file->info = info_dup; + file->comm = comm; + file->info = info; file->mpi_rank = mpi_rank; file->mpi_size = mpi_size; @@ -1019,7 +849,7 @@ H5FD__mpio_open(const char *name, unsigned flags, hid_t fapl_id, } /* end if */ /* Broadcast file size */ - if(MPI_SUCCESS != (mpi_code = MPI_Bcast(&size, (int)sizeof(MPI_Offset), MPI_BYTE, 0, comm_dup))) + if(MPI_SUCCESS != (mpi_code = MPI_Bcast(&size, (int)sizeof(MPI_Offset), MPI_BYTE, 0, comm))) HMPI_GOTO_ERROR(NULL, "MPI_Bcast failed", mpi_code) /* Determine if the file should be truncated */ @@ -1028,7 +858,7 @@ H5FD__mpio_open(const char *name, unsigned flags, hid_t fapl_id, HMPI_GOTO_ERROR(NULL, "MPI_File_set_size failed", mpi_code) /* Don't let any proc return until all have truncated the file. */ - if (MPI_SUCCESS!= (mpi_code=MPI_Barrier(comm_dup))) + if (MPI_SUCCESS != (mpi_code = MPI_Barrier(comm))) HMPI_GOTO_ERROR(NULL, "MPI_Barrier failed", mpi_code) /* File is zero size now */ @@ -1046,12 +876,12 @@ done: if(ret_value == NULL) { if(file_opened) MPI_File_close(&fh); - if(MPI_COMM_NULL != comm_dup) - MPI_Comm_free(&comm_dup); - if(MPI_INFO_NULL != info_dup) - MPI_Info_free(&info_dup); - if(file) - H5MM_xfree(file); + if(H5_mpi_comm_free(&comm) < 0) + HDONE_ERROR(H5E_VFL, H5E_CANTFREE, NULL, "unable to free MPI communicator") + if(H5_mpi_info_free(&info) < 0) + HDONE_ERROR(H5E_VFL, H5E_CANTFREE, NULL, "unable to free MPI info object") + if(file) + H5MM_xfree(file); } /* end if */ #ifdef H5FDmpio_DEBUG @@ -1068,8 +898,7 @@ done: * * Purpose: Closes a file. This is collective. * - * Return: Success: Non-negative - * Failure: Negative + * Return: SUCCEED/FAIL * * Programmer: Unknown * January 30, 1998 @@ -1079,15 +908,15 @@ done: static herr_t H5FD__mpio_close(H5FD_t *_file) { - H5FD_mpio_t *file = (H5FD_mpio_t*)_file; - int mpi_code; /* MPI return code */ + H5FD_mpio_t *file = (H5FD_mpio_t*)_file; + int mpi_code; /* MPI return code */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC #ifdef H5FDmpio_DEBUG if(H5FD_mpio_Debug[(int)'t']) - HDfprintf(stdout, "%s: Entering\n", FUNC); + HDfprintf(stdout, "%s: Entering\n", FUNC); #endif /* Sanity checks */ @@ -1099,13 +928,14 @@ H5FD__mpio_close(H5FD_t *_file) HMPI_GOTO_ERROR(FAIL, "MPI_File_close failed", mpi_code) /* Clean up other stuff */ - H5FD_mpi_comm_info_free(&file->comm, &file->info); + H5_mpi_comm_free(&file->comm); + H5_mpi_info_free(&file->info); H5MM_xfree(file); done: #ifdef H5FDmpio_DEBUG if(H5FD_mpio_Debug[(int)'t']) - HDfprintf(stdout, "%s: Leaving\n", FUNC); + HDfprintf(stdout, "%s: Leaving\n", FUNC); #endif FUNC_LEAVE_NOAPI(ret_value) @@ -1113,15 +943,14 @@ done: /*------------------------------------------------------------------------- - * Function: H5FD__mpio_query + * Function: H5FD__mpio_query * - * Purpose: Set the flags that this VFL driver is capable of supporting. + * Purpose: Set the flags that this VFL driver is capable of supporting. * (listed in H5FDpublic.h) * - * Return: Success: non-negative - * Failure: negative + * Return: SUCCEED/FAIL * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Friday, August 25, 2000 * *------------------------------------------------------------------------- @@ -1129,7 +958,7 @@ done: static herr_t H5FD__mpio_query(const H5FD_t H5_ATTR_UNUSED *_file, unsigned long *flags /* out */) { - FUNC_ENTER_STATIC + FUNC_ENTER_STATIC_NOERR /* Set the VFL feature flags that this driver supports */ if(flags) { @@ -1146,16 +975,16 @@ H5FD__mpio_query(const H5FD_t H5_ATTR_UNUSED *_file, unsigned long *flags /* out /*------------------------------------------------------------------------- - * Function: H5FD__mpio_get_eoa + * Function: H5FD__mpio_get_eoa * - * Purpose: Gets the end-of-address marker for the file. The EOA marker - * is the first address past the last byte allocated in the - * format address space. + * Purpose: Gets the end-of-address marker for the file. The EOA marker + * is the first address past the last byte allocated in the + * format address space. * - * Return: Success: The end-of-address marker. - * Failure: HADDR_UNDEF + * Return: Success: The end-of-address marker + * Failure: HADDR_UNDEF * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Friday, August 6, 1999 * *------------------------------------------------------------------------- @@ -1163,9 +992,9 @@ H5FD__mpio_query(const H5FD_t H5_ATTR_UNUSED *_file, unsigned long *flags /* out static haddr_t H5FD__mpio_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) { - const H5FD_mpio_t *file = (const H5FD_mpio_t*)_file; + const H5FD_mpio_t *file = (const H5FD_mpio_t*)_file; - FUNC_ENTER_STATIC + FUNC_ENTER_STATIC_NOERR /* Sanity checks */ HDassert(file); @@ -1176,16 +1005,15 @@ H5FD__mpio_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) /*------------------------------------------------------------------------- - * Function: H5FD__mpio_set_eoa + * Function: H5FD__mpio_set_eoa * - * Purpose: Set the end-of-address marker for the file. This function is - * called shortly after an existing HDF5 file is opened in order - * to tell the driver where the end of the HDF5 data is located. + * Purpose: Set the end-of-address marker for the file. This function is + * called shortly after an existing HDF5 file is opened in order + * to tell the driver where the end of the HDF5 data is located. * - * Return: Success: 0 - * Failure: -1 + * Return: SUCCEED/FAIL * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Friday, August 6, 1999 * *------------------------------------------------------------------------- @@ -1193,9 +1021,9 @@ H5FD__mpio_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) static herr_t H5FD__mpio_set_eoa(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr) { - H5FD_mpio_t *file = (H5FD_mpio_t*)_file; + H5FD_mpio_t *file = (H5FD_mpio_t*)_file; - FUNC_ENTER_STATIC + FUNC_ENTER_STATIC_NOERR /* Sanity checks */ HDassert(file); @@ -1208,27 +1036,27 @@ H5FD__mpio_set_eoa(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr) /*------------------------------------------------------------------------- - * Function: H5FD__mpio_get_eof + * Function: H5FD__mpio_get_eof * - * Purpose: Gets the end-of-file marker for the file. The EOF marker - * is the real size of the file. + * Purpose: Gets the end-of-file marker for the file. The EOF marker + * is the real size of the file. * - * The MPIO driver doesn't bother keeping this field updated - * since that's a relatively expensive operation. Fortunately - * the library only needs the EOF just after the file is opened - * in order to determine whether the file is empty, truncated, - * or okay. Therefore, any MPIO I/O function will set its value - * to HADDR_UNDEF which is the error return value of this - * function. + * The MPIO driver doesn't bother keeping this field updated + * since that's a relatively expensive operation. Fortunately + * the library only needs the EOF just after the file is opened + * in order to determine whether the file is empty, truncated, + * or okay. Therefore, any MPIO I/O function will set its value + * to HADDR_UNDEF which is the error return value of this + * function. * * Keeping the EOF updated (during write calls) is expensive * because any process may extend the physical end of the * file. -QAK * - * Return: Success: The end-of-address marker. - * Failure: HADDR_UNDEF + * Return: Success: The end-of-file marker + * Failure: HADDR_UNDEF * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Friday, August 6, 1999 * *------------------------------------------------------------------------- @@ -1236,9 +1064,9 @@ H5FD__mpio_set_eoa(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr) static haddr_t H5FD__mpio_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) { - const H5FD_mpio_t *file = (const H5FD_mpio_t*)_file; + const H5FD_mpio_t *file = (const H5FD_mpio_t*)_file; - FUNC_ENTER_STATIC + FUNC_ENTER_STATIC_NOERR /* Sanity checks */ HDassert(file); @@ -1253,7 +1081,7 @@ H5FD__mpio_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) * * Purpose: Returns the file handle of MPIO file driver. * - * Returns: Non-negative if succeed or negative if fails. + * Returns: SUCCEED/FAIL * * Programmer: Raymond Lu * Sept. 16, 2002 @@ -1279,56 +1107,23 @@ done: /*------------------------------------------------------------------------- - * Function: H5FD_mpio_get_info - * - * Purpose: Returns the file info of MPIO file driver. - * - * Returns: Non-negative if succeed or negative if fails. - * - * Programmer: John Mainzer - * April 4, 2017 - * - * Modifications: - * - *------------------------------------------------------------------------- -*/ -static herr_t -H5FD_mpio_get_info(H5FD_t *_file, void** mpi_info) -{ - H5FD_mpio_t *file = (H5FD_mpio_t *)_file; - herr_t ret_value = SUCCEED; - - FUNC_ENTER_NOAPI_NOINIT - - if(!mpi_info) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "mpi info not valid") - - *mpi_info = &(file->info); - -done: - FUNC_LEAVE_NOAPI(ret_value) - -} /* H5FD_mpio_get_info() */ - - -/*------------------------------------------------------------------------- - * Function: H5FD__mpio_read + * Function: H5FD__mpio_read * - * Purpose: Reads SIZE bytes of data from FILE beginning at address ADDR - * into buffer BUF according to data transfer properties in - * DXPL_ID using potentially complex file and buffer types to - * effect the transfer. + * Purpose: Reads SIZE bytes of data from FILE beginning at address ADDR + * into buffer BUF according to data transfer properties in + * DXPL_ID using potentially complex file and buffer types to + * effect the transfer. * - * Reading past the end of the MPI file returns zeros instead of - * failing. MPI is able to coalesce requests from different - * processes (collective or independent). + * Reading past the end of the MPI file returns zeros instead of + * failing. MPI is able to coalesce requests from different + * processes (collective or independent). * - * Return: Success: Zero. Result is stored in caller-supplied - * buffer BUF. + * Return: Success: SUCCEED. Result is stored in caller-supplied + * buffer BUF. * - * Failure: -1, Contents of buffer BUF are undefined. + * Failure: FAIL. Contents of buffer BUF are undefined. * - * Programmer: rky, 1998-01-30 + * Programmer: rky, 1998-01-30 * *------------------------------------------------------------------------- */ @@ -1531,22 +1326,22 @@ done: /*------------------------------------------------------------------------- - * Function: H5FD__mpio_write + * Function: H5FD__mpio_write * - * Purpose: Writes SIZE bytes of data to FILE beginning at address ADDR - * from buffer BUF according to data transfer properties in - * DXPL_ID using potentially complex file and buffer types to - * effect the transfer. + * Purpose: Writes SIZE bytes of data to FILE beginning at address ADDR + * from buffer BUF according to data transfer properties in + * DXPL_ID using potentially complex file and buffer types to + * effect the transfer. * - * MPI is able to coalesce requests from different processes - * (collective and independent). + * MPI is able to coalesce requests from different processes + * (collective and independent). * - * Return: Success: Zero. USE_TYPES and OLD_USE_TYPES in the - * access params are altered. - * Failure: -1, USE_TYPES and OLD_USE_TYPES in the - * access params may be altered. + * Return: Success: SUCCEED. USE_TYPES and OLD_USE_TYPES in the + * access params are altered. + * Failure: FAIL. USE_TYPES and OLD_USE_TYPES in the + * access params may be altered. * - * Programmer: Robert Kim Yates + * Programmer: Robert Kim Yates * January 30, 1998 * *------------------------------------------------------------------------- @@ -1555,13 +1350,13 @@ static herr_t H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, haddr_t addr, size_t size, const void *buf) { - H5FD_mpio_t *file = (H5FD_mpio_t*)_file; - MPI_Offset mpi_off; - MPI_Status mpi_stat; /* Status from I/O operation */ - MPI_Datatype buf_type = MPI_BYTE; /* MPI description of the selection in memory */ - int mpi_code; /* MPI return code */ + H5FD_mpio_t *file = (H5FD_mpio_t*)_file; + MPI_Offset mpi_off; + MPI_Status mpi_stat; /* Status from I/O operation */ + MPI_Datatype buf_type = MPI_BYTE; /* MPI description of the selection in memory */ + int mpi_code; /* MPI return code */ #if MPI_VERSION >= 3 - MPI_Count bytes_written; + MPI_Count bytes_written; MPI_Count type_size; /* MPI datatype used for I/O's size */ MPI_Count io_size; /* Actual number of bytes requested */ #else @@ -1570,15 +1365,15 @@ H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, int io_size; /* Actual number of bytes requested */ #endif int size_i; - hbool_t use_view_this_time = FALSE; + hbool_t use_view_this_time = FALSE; H5FD_mpio_xfer_t xfer_mode; /* I/O transfer mode */ - herr_t ret_value = SUCCEED; + herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC #ifdef H5FDmpio_DEBUG if(H5FD_mpio_Debug[(int)'t']) - HDfprintf(stdout, "%s: Entering\n", FUNC); + HDfprintf(stdout, "%s: Entering\n", FUNC); #endif /* Sanity checks */ @@ -1615,7 +1410,7 @@ H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, * could mean "use MPI_BYTE" by convention). */ if(xfer_mode == H5FD_MPIO_COLLECTIVE) { - MPI_Datatype file_type; + MPI_Datatype file_type; /* Remember that views are used */ use_view_this_time = TRUE; @@ -1696,7 +1491,7 @@ H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, io_size = type_size * size_i; /* Check for write failure */ - if(bytes_written != io_size) + if(bytes_written != io_size || bytes_written < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") /* Each process will keep track of its perceived EOF value locally, and @@ -1707,13 +1502,13 @@ H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, * potentially be wrong.) */ file->eof = HADDR_UNDEF; - if(bytes_written && ((bytes_written + addr) > file->local_eof)) - file->local_eof = addr + bytes_written; + if(bytes_written && (((haddr_t)bytes_written + addr) > file->local_eof)) + file->local_eof = addr + (haddr_t)bytes_written; done: #ifdef H5FDmpio_DEBUG if(H5FD_mpio_Debug[(int)'t']) - HDfprintf(stdout, "%s: Leaving, proc %d: ret_value = %d\n", FUNC, file->mpi_rank, ret_value ); + HDfprintf(stdout, "%s: Leaving, proc %d: ret_value = %d\n", FUNC, file->mpi_rank, ret_value ); #endif FUNC_LEAVE_NOAPI(ret_value) @@ -1725,9 +1520,7 @@ done: * * Purpose: Makes sure that all data is on disk. This is collective. * - * Return: Success: Non-negative - * - * Failure: Negative + * Return: SUCCEED/FAIL * * Programmer: Robb Matzke * January 30, 1998 @@ -1737,15 +1530,15 @@ done: static herr_t H5FD__mpio_flush(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, hbool_t closing) { - H5FD_mpio_t *file = (H5FD_mpio_t*)_file; - int mpi_code; /* mpi return code */ - herr_t ret_value = SUCCEED; + H5FD_mpio_t *file = (H5FD_mpio_t*)_file; + int mpi_code; /* mpi return code */ + herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC #ifdef H5FDmpio_DEBUG if(H5FD_mpio_Debug[(int)'t']) - HDfprintf(stdout, "%s: Entering\n", FUNC); + HDfprintf(stdout, "%s: Entering\n", FUNC); #endif /* Sanity checks */ @@ -1760,7 +1553,7 @@ H5FD__mpio_flush(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, hbool_t closing) done: #ifdef H5FDmpio_DEBUG if(H5FD_mpio_Debug[(int)'t']) - HDfprintf(stdout, "%s: Leaving\n", FUNC); + HDfprintf(stdout, "%s: Leaving\n", FUNC); #endif FUNC_LEAVE_NOAPI(ret_value) @@ -1782,8 +1575,7 @@ done: * only call MPI_File_set_size() if this value disagrees * with the current eoa. * - * Return: Success: Non-negative - * Failure: Negative + * Return: SUCCEED/FAIL * * Programmer: Quincey Koziol * January 31, 2008 @@ -1793,14 +1585,14 @@ done: static herr_t H5FD__mpio_truncate(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, hbool_t H5_ATTR_UNUSED closing) { - H5FD_mpio_t *file = (H5FD_mpio_t*)_file; + H5FD_mpio_t *file = (H5FD_mpio_t*)_file; herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC #ifdef H5FDmpio_DEBUG if(H5FD_mpio_Debug[(int)'t']) - HDfprintf(stdout, "%s: Entering\n", FUNC); + HDfprintf(stdout, "%s: Entering\n", FUNC); #endif /* Sanity checks */ @@ -1869,7 +1661,7 @@ H5FD__mpio_truncate(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, hbool_t H5_ATTR done: #ifdef H5FDmpio_DEBUG if(H5FD_mpio_Debug[(int)'t']) - HDfprintf(stdout, "%s: Leaving\n", FUNC); + HDfprintf(stdout, "%s: Leaving\n", FUNC); #endif FUNC_LEAVE_NOAPI(ret_value) @@ -1877,14 +1669,14 @@ done: /*------------------------------------------------------------------------- - * Function: H5FD__mpio_mpi_rank + * Function: H5FD__mpio_mpi_rank * - * Purpose: Returns the MPI rank for a process + * Purpose: Returns the MPI rank for a process * - * Return: Success: non-negative - * Failure: negative + * Return: Success: non-negative + * Failure: negative * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Thursday, May 16, 2002 * *------------------------------------------------------------------------- @@ -1892,9 +1684,9 @@ done: static int H5FD__mpio_mpi_rank(const H5FD_t *_file) { - const H5FD_mpio_t *file = (const H5FD_mpio_t*)_file; + const H5FD_mpio_t *file = (const H5FD_mpio_t*)_file; - FUNC_ENTER_STATIC + FUNC_ENTER_STATIC_NOERR /* Sanity checks */ HDassert(file); @@ -1905,14 +1697,14 @@ H5FD__mpio_mpi_rank(const H5FD_t *_file) /*------------------------------------------------------------------------- - * Function: H5FD__mpio_mpi_size + * Function: H5FD__mpio_mpi_size * - * Purpose: Returns the number of MPI processes + * Purpose: Returns the number of MPI processes * - * Return: Success: non-negative - * Failure: negative + * Return: Success: non-negative + * Failure: negative * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Thursday, May 16, 2002 * *------------------------------------------------------------------------- @@ -1920,9 +1712,9 @@ H5FD__mpio_mpi_rank(const H5FD_t *_file) static int H5FD__mpio_mpi_size(const H5FD_t *_file) { - const H5FD_mpio_t *file = (const H5FD_mpio_t*)_file; + const H5FD_mpio_t *file = (const H5FD_mpio_t*)_file; - FUNC_ENTER_STATIC + FUNC_ENTER_STATIC_NOERR /* Sanity checks */ HDassert(file); @@ -1933,15 +1725,14 @@ H5FD__mpio_mpi_size(const H5FD_t *_file) /*------------------------------------------------------------------------- - * Function: H5FD__mpio_communicator + * Function: H5FD__mpio_communicator * - * Purpose: Returns the MPI communicator for the file. + * Purpose: Returns the MPI communicator for the file. * - * Return: Success: The communicator + * Return: Success: The communicator + * Failure: Can't fail * - * Failure: NULL - * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Monday, August 9, 1999 * *------------------------------------------------------------------------- @@ -1949,9 +1740,9 @@ H5FD__mpio_mpi_size(const H5FD_t *_file) static MPI_Comm H5FD__mpio_communicator(const H5FD_t *_file) { - const H5FD_mpio_t *file = (const H5FD_mpio_t*)_file; + const H5FD_mpio_t *file = (const H5FD_mpio_t*)_file; - FUNC_ENTER_STATIC + FUNC_ENTER_STATIC_NOERR /* Sanity checks */ HDassert(file); @@ -1971,8 +1762,6 @@ H5FD__mpio_communicator(const H5FD_t *_file) * Programmer: John Mainzer * April 4, 2017 * - * Modifications: - * *------------------------------------------------------------------------- */ static herr_t diff --git a/src/H5FDprivate.h b/src/H5FDprivate.h index c79d676..ac08f7f 100644 --- a/src/H5FDprivate.h +++ b/src/H5FDprivate.h @@ -95,14 +95,6 @@ typedef struct { const void *driver_info; /* Driver info, for open callbacks */ } H5FD_driver_prop_t; -#ifdef H5_HAVE_PARALLEL -/* MPIO-specific file access properties */ -typedef struct H5FD_mpio_fapl_t { - MPI_Comm comm; /*communicator */ - MPI_Info info; /*file information */ -} H5FD_mpio_fapl_t; -#endif /* H5_HAVE_PARALLEL */ - /*****************************/ /* Library Private Variables */ @@ -160,9 +152,6 @@ H5_DLL herr_t H5FD_set_paged_aggr(H5FD_t *file, hbool_t paged); /* General routines */ H5_DLL haddr_t H5FD_mpi_MPIOff_to_haddr(MPI_Offset mpi_off); H5_DLL herr_t H5FD_mpi_haddr_to_MPIOff(haddr_t addr, MPI_Offset *mpi_off/*out*/); -H5_DLL herr_t H5FD_mpi_comm_info_dup(MPI_Comm comm, MPI_Info info, - MPI_Comm *comm_new, MPI_Info *info_new); -H5_DLL herr_t H5FD_mpi_comm_info_free(MPI_Comm *comm, MPI_Info *info); #ifdef NOT_YET H5_DLL herr_t H5FD_mpio_wait_for_left_neighbor(H5FD_t *file); H5_DLL herr_t H5FD_mpio_signal_right_neighbor(H5FD_t *file); diff --git a/src/H5FDros3.c b/src/H5FDros3.c new file mode 100644 index 0000000..a369ca2 --- /dev/null +++ b/src/H5FDros3.c @@ -0,0 +1,1832 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Read-Only S3 Virtual File Driver (VFD) + * + * Programmer: Jacob Smith <jake.smith@hdfgroup.org> + * 2017-10-13 + * + * Purpose: + * + * Provide read-only access to files hosted on Amazon's S3 service. + * Relies on "s3comms" utility layer to implement the AWS REST API. + */ + +/* This source code file is part of the H5FD driver module */ +#include "H5FDdrvr_module.h" + +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5FDprivate.h" /* File drivers */ +#include "H5FDros3.h" /* ros3 file driver */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5Iprivate.h" /* IDs */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5FDs3comms.h" /* S3 Communications */ + +#ifdef H5_HAVE_ROS3_VFD + +/* toggle function call prints: 1 turns on + */ +#define ROS3_DEBUG 0 + +/* toggle stats collection and reporting + */ +#define ROS3_STATS 0 + +/* The driver identification number, initialized at runtime + */ +static hid_t H5FD_ROS3_g = 0; + +#if ROS3_STATS + +/* arbitrarily large value, such that any reasonable size read will be "less" + * than this value and set a true minimum + * not 0 because that may be a valid recorded minimum in degenerate cases + */ +#define ROS3_STATS_STARTING_MIN 0xfffffffful + +/* Configuration definitions for stats collection and breakdown + * + * 2^10 = 1024 + * Reads up to 1024 bytes (1 kB) fall in bin 0 + * 2^(10+(1*16)) = 2^26 = 64MB + * Reads of 64MB or greater fall in "overflow" bin[BIN_COUNT] + */ +#define ROS3_STATS_BASE 2 +#define ROS3_STATS_INTERVAL 1 +#define ROS3_STATS_START_POWER 10 +#define ROS3_STATS_BIN_COUNT 16 /* MUST BE GREATER THAN 0 */ + + +/* + * Calculate `BASE ^ (START_POWER + (INTERVAL * bin_i))` + * Stores result at `(unsigned long long *) out_ptr`. + * Used in computing boundaries between stats bins. + */ +#define ROS3_STATS_POW(bin_i, out_ptr) { \ + unsigned long long donotshadowresult = 1; \ + unsigned donotshadowindex = 0; \ + for (donotshadowindex = 0; \ + donotshadowindex < (((bin_i) * ROS3_STATS_INTERVAL) + \ + ROS3_STATS_START_POWER); \ + donotshadowindex++) \ + { \ + donotshadowresult *= ROS3_STATS_BASE; \ + } \ + *(out_ptr) = donotshadowresult; \ +} + +/* array to hold pre-computed boundaries for stats bins + */ +static unsigned long long ros3_stats_boundaries[ROS3_STATS_BIN_COUNT]; + +/*************************************************************************** + * + * Structure: ros3_statsbin + * + * Purpose: + * + * Structure for storing per-file ros3 VFD usage statistics. + * + * + * + * `count` (unsigned long long) + * + * Number of reads with size in this bin's range. + * + * `bytes` (unsigned long long) + * + * Total number of bytes read through this bin. + * + * `min` (unsigned long long) + * + * Smallest read size in this bin. + * + * `max` (unsigned long long) + * + * Largest read size in this bin. + * + * + * + * Programmer: Jacob Smith + * + ***************************************************************************/ +typedef struct { + unsigned long long count; + unsigned long long bytes; + unsigned long long min; + unsigned long long max; +} ros3_statsbin; + +#endif /* ROS3_STATS */ + +/*************************************************************************** + * + * Structure: H5FD_ros3_t + * + * Purpose: + * + * H5FD_ros3_t is a structure used to store all information needed to + * maintain R/O access to a single HDF5 file that has been stored as a + * S3 object. This structure is created when such a file is "opened" and + * discarded when it is "closed". + * + * Presents an S3 object as a file to the HDF5 library. + * + * + * + * `pub` (H5FD_t) + * + * Instance of H5FD_t which contains all fields common to all VFDs. + * It must be the first item in this structure, since at higher levels, + * this structure will be treated as an instance of H5FD_t. + * + * `fa` (H5FD_ros3_fapl_t) + * + * Instance of `H5FD_ros3_fapl_t` containing the S3 configuration data + * needed to "open" the HDF5 file. + * + * `eoa` (haddr_t) + * + * End of addressed space in file. After open, it should always + * equal the file size. + * + * `s3r_handle` (s3r_t *) + * + * Instance of S3 Request handle associated with the target resource. + * Responsible for communicating with remote host and presenting file + * contents as indistinguishable from a file on the local filesystem. + * + * *** present only if ROS3_SATS is flagged to enable stats collection *** + * + * `meta` (ros3_statsbin[]) + * `raw` (ros3_statsbin[]) + * + * Only present if ros3 stats collection is enabled. + * + * Arrays of `ros3_statsbin` structures to record raw- and metadata reads. + * + * Records count and size of reads performed by the VFD, and is used to + * print formatted usage statistics to stdout upon VFD shutdown. + * + * Reads of each raw- and metadata type are recorded in an individual bin + * determined by the size of the read. The last bin of each type is + * reserved for "big" reads, with no defined upper bound. + * + * *** end ROS3_STATS *** + * + * + * + * Programmer: Jacob Smith + * + ***************************************************************************/ +typedef struct H5FD_ros3_t { + H5FD_t pub; + H5FD_ros3_fapl_t fa; + haddr_t eoa; + s3r_t *s3r_handle; +#if ROS3_STATS + ros3_statsbin meta[ROS3_STATS_BIN_COUNT + 1]; + ros3_statsbin raw[ROS3_STATS_BIN_COUNT + 1]; +#endif +} H5FD_ros3_t; + +/* + * These macros check for overflow of various quantities. These macros + * assume that HDoff_t is signed and haddr_t and size_t are unsigned. + * + * ADDR_OVERFLOW: Checks whether a file address of type `haddr_t' + * is too large to be represented by the second argument + * of the file seek function. + * Only included if it may be used -- ROS3 VFD is enabled. + * + */ +#define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1) +#define ADDR_OVERFLOW(A) (HADDR_UNDEF==(A) || ((A) & ~(haddr_t)MAXADDR)) + +/* Prototypes */ +static herr_t H5FD_ros3_term(void); +static void *H5FD_ros3_fapl_get(H5FD_t *_file); +static void *H5FD_ros3_fapl_copy(const void *_old_fa); +static herr_t H5FD_ros3_fapl_free(void *_fa); +static H5FD_t *H5FD_ros3_open(const char *name, unsigned flags, hid_t fapl_id, + haddr_t maxaddr); +static herr_t H5FD_ros3_close(H5FD_t *_file); +static int H5FD_ros3_cmp(const H5FD_t *_f1, const H5FD_t *_f2); +static herr_t H5FD_ros3_query(const H5FD_t *_f1, unsigned long *flags); +static haddr_t H5FD_ros3_get_eoa(const H5FD_t *_file, H5FD_mem_t type); +static herr_t H5FD_ros3_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr); +static haddr_t H5FD_ros3_get_eof(const H5FD_t *_file, H5FD_mem_t type); +static herr_t H5FD_ros3_get_handle(H5FD_t *_file, hid_t fapl, + void** file_handle); +static herr_t H5FD_ros3_read(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, + haddr_t addr, size_t size, void *buf); +static herr_t H5FD_ros3_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, + haddr_t addr, size_t size, const void *buf); +static herr_t H5FD_ros3_truncate(H5FD_t *_file, hid_t dxpl_id, + hbool_t closing); +static herr_t H5FD_ros3_lock(H5FD_t *_file, hbool_t rw); +static herr_t H5FD_ros3_unlock(H5FD_t *_file); +static herr_t H5FD_ros3_validate_config(const H5FD_ros3_fapl_t * fa); + +static const H5FD_class_t H5FD_ros3_g = { + "ros3", /* name */ + MAXADDR, /* maxaddr */ + H5F_CLOSE_WEAK, /* fc_degree */ + H5FD_ros3_term, /* terminate */ + NULL, /* sb_size */ + NULL, /* sb_encode */ + NULL, /* sb_decode */ + sizeof(H5FD_ros3_fapl_t), /* fapl_size */ + H5FD_ros3_fapl_get, /* fapl_get */ + H5FD_ros3_fapl_copy, /* fapl_copy */ + H5FD_ros3_fapl_free, /* fapl_free */ + 0, /* dxpl_size */ + NULL, /* dxpl_copy */ + NULL, /* dxpl_free */ + H5FD_ros3_open, /* open */ + H5FD_ros3_close, /* close */ + H5FD_ros3_cmp, /* cmp */ + H5FD_ros3_query, /* query */ + NULL, /* get_type_map */ + NULL, /* alloc */ + NULL, /* free */ + H5FD_ros3_get_eoa, /* get_eoa */ + H5FD_ros3_set_eoa, /* set_eoa */ + H5FD_ros3_get_eof, /* get_eof */ + H5FD_ros3_get_handle, /* get_handle */ + H5FD_ros3_read, /* read */ + H5FD_ros3_write, /* write */ + NULL, /* flush */ + H5FD_ros3_truncate, /* truncate */ + H5FD_ros3_lock, /* lock */ + H5FD_ros3_unlock, /* unlock */ + H5FD_FLMAP_DICHOTOMY /* fl_map */ +}; + +/* Declare a free list to manage the H5FD_ros3_t struct */ +H5FL_DEFINE_STATIC(H5FD_ros3_t); + + +/*------------------------------------------------------------------------- + * Function: H5FD__init_package + * + * Purpose: Initializes any interface-specific data or routines. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Jacob Smith 2017 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__init_package(void) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + if (H5FD_ros3_init() < 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, + "unable to initialize ros3 VFD") + } + +done: + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5FD__init_package() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_ros3_init + * + * Purpose: Initialize this driver by registering the driver with the + * library. + * + * Return: Success: The driver ID for the ros3 driver. + * Failure: Negative + * + * Programmer: Jacob Smith 2017 + * + *------------------------------------------------------------------------- + */ +hid_t +H5FD_ros3_init(void) +{ + hid_t ret_value = H5I_INVALID_HID; + + FUNC_ENTER_NOAPI(FAIL) + +#if ROS3_DEBUG + HDfprintf(stdout, "H5FD_ros3_init() called.\n"); +#endif + + if (H5I_VFL != H5I_get_type(H5FD_ROS3_g)) + H5FD_ROS3_g = H5FD_register(&H5FD_ros3_g, sizeof(H5FD_class_t), FALSE); + +#if ROS3_STATS + /* pre-compute statsbin boundaries + */ + for (unsigned bin_i = 0; bin_i < ROS3_STATS_BIN_COUNT; bin_i++) { + unsigned long long value = 0; + ROS3_STATS_POW(bin_i, &value) + ros3_stats_boundaries[bin_i] = value; + } +#endif + + /* Set return value */ + ret_value = H5FD_ROS3_g; + +done: + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5FD_ros3_init() */ + + +/*--------------------------------------------------------------------------- + * Function: H5FD_ros3_term + * + * Purpose: Shut down the VFD + * + * Returns: SUCCEED (Can't fail) + * + * Programmer: Jacob Smith 2017 + * + *--------------------------------------------------------------------------- + */ +static herr_t +H5FD_ros3_term(void) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if ROS3_DEBUG + HDfprintf(stdout, "H5FD_ros3_term() called.\n"); +#endif + + /* Reset VFL ID */ + H5FD_ROS3_g = 0; + + FUNC_LEAVE_NOAPI(SUCCEED) + +} /* end H5FD_ros3_term() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pset_fapl_ros3 + * + * Purpose: Modify the file access property list to use the H5FD_ROS3 + * driver defined in this source file. All driver specfic + * properties are passed in as a pointer to a suitably + * initialized instance of H5FD_ros3_fapl_t + * + * Return: SUCCEED/FAIL + * + * Programmer: John Mainzer + * 9/10/17 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_fapl_ros3(hid_t fapl_id, + H5FD_ros3_fapl_t *fa) +{ + H5P_genplist_t *plist = NULL; /* Property list pointer */ + herr_t ret_value = FAIL; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*x", fapl_id, fa); + + HDassert(fa != NULL); + +#if ROS3_DEBUG + HDfprintf(stdout, "H5Pset_fapl_ros3() called.\n"); +#endif + + plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS); + if (plist == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, \ + "not a file access property list") + } + + if (FAIL == H5FD_ros3_validate_config(fa)) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid ros3 config") + } + + ret_value = H5P_set_driver(plist, H5FD_ROS3, (void *)fa); + +done: + FUNC_LEAVE_API(ret_value) + +} /* end H5Pset_fapl_ros3() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_ros3_validate_config() + * + * Purpose: Test to see if the supplied instance of H5FD_ros3_fapl_t + * contains internally consistant data. Return SUCCEED if so, + * and FAIL otherwise. + * + * Note the difference between internally consistant and + * correct. As we will have to try to access the target + * object to determine whether the supplied data is correct, + * we will settle for internal consistancy at this point + * + * Return: SUCCEED if instance of H5FD_ros3_fapl_t contains internally + * consistant data, FAIL otherwise. + * + * Programmer: Jacob Smith + * 9/10/17 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_ros3_validate_config(const H5FD_ros3_fapl_t * fa) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + + HDassert(fa != NULL); + + if ( fa->version != H5FD_CURR_ROS3_FAPL_T_VERSION ) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "Unknown H5FD_ros3_fapl_t version"); + } + + /* if set to authenticate, region and id cannot be empty strings + */ + if (fa->authenticate == TRUE) { + if ((fa->aws_region[0] == '\0') || + (fa->secret_id[0] == '\0')) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "Inconsistent authentication information"); + } + } + +done: + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5FD_ros3_validate_config() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_fapl_ros3 + * + * Purpose: Returns information about the ros3 file access property + * list though the function arguments. + * + * Return: Success: Non-negative + * + * Failure: Negative + * + * Programmer: John Mainzer + * 9/10/17 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_fapl_ros3(hid_t fapl_id, + H5FD_ros3_fapl_t *fa_out) +{ + const H5FD_ros3_fapl_t *fa = NULL; + H5P_genplist_t *plist = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*x", fapl_id, fa_out); + +#if ROS3_DEBUG + HDfprintf(stdout, "H5Pget_fapl_ros3() called.\n"); +#endif + + if (fa_out == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "fa_out is NULL") + } + + plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS); + if (plist == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access list") + } + + if (H5FD_ROS3 != H5P_peek_driver(plist)) { + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver") + } + + fa = (const H5FD_ros3_fapl_t *)H5P_peek_driver_info(plist); + if (fa == NULL) { + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "bad VFL driver info") + } + + /* Copy the ros3 fapl data out */ + HDmemcpy(fa_out, fa, sizeof(H5FD_ros3_fapl_t)); + +done: + FUNC_LEAVE_API(ret_value) + +} /* end H5Pget_fapl_ros3() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_ros3_fapl_get + * + * Purpose: Gets a file access property list which could be used to + * create an identical file. + * + * Return: Success: Ptr to new file access property list value. + * + * Failure: NULL + * + * Programmer: John Mainzer + * 9/8/17 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static void * +H5FD_ros3_fapl_get(H5FD_t *_file) +{ + H5FD_ros3_t *file = (H5FD_ros3_t*)_file; + H5FD_ros3_fapl_t *fa = NULL; + void *ret_value = NULL; + + FUNC_ENTER_NOAPI_NOINIT + + fa = (H5FD_ros3_fapl_t *)H5MM_calloc(sizeof(H5FD_ros3_fapl_t)); + if (fa == NULL) { + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, + "memory allocation failed") + } + + /* Copy the fields of the structure */ + HDmemcpy(fa, &(file->fa), sizeof(H5FD_ros3_fapl_t)); + + /* Set return value */ + ret_value = fa; + +done: + if (ret_value == NULL) { + if (fa != NULL) { + H5MM_xfree(fa); + } + } + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5FD_ros3_fapl_get() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_ros3_fapl_copy + * + * Purpose: Copies the ros3-specific file access properties. + * + * Return: Success: Ptr to a new property list + * + * Failure: NULL + * + * Programmer: John Mainzer + * 9/8/17 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static void * +H5FD_ros3_fapl_copy(const void *_old_fa) +{ + const H5FD_ros3_fapl_t *old_fa = (const H5FD_ros3_fapl_t*)_old_fa; + H5FD_ros3_fapl_t *new_fa = NULL; + void *ret_value = NULL; + + FUNC_ENTER_NOAPI_NOINIT + + new_fa = (H5FD_ros3_fapl_t *)H5MM_malloc(sizeof(H5FD_ros3_fapl_t)); + if (new_fa == NULL) { + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, + "memory allocation failed"); + } + + HDmemcpy(new_fa, old_fa, sizeof(H5FD_ros3_fapl_t)); + ret_value = new_fa; + +done: + if (ret_value == NULL) { + if (new_fa != NULL) { + H5MM_xfree(new_fa); + } + } + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5FD_ros3_fapl_copy() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_ros3_fapl_free + * + * Purpose: Frees the ros3-specific file access properties. + * + * Return: SUCCEED (cannot fail) + * + * Programmer: John Mainzer + * 9/8/17 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_ros3_fapl_free(void *_fa) +{ + H5FD_ros3_fapl_t *fa = (H5FD_ros3_fapl_t*)_fa; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(fa != NULL); /* sanity check */ + + H5MM_xfree(fa); + + FUNC_LEAVE_NOAPI(SUCCEED) + +} /* end H5FD_ros3_fapl_free() */ + +#if ROS3_STATS + +/*---------------------------------------------------------------------------- + * + * Function: ros3_reset_stats() + * + * Purpose: + * + * Reset the stats collection elements in this virtual file structure. + * + * Clears any set data in stats bins; initializes/zeroes values. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * - Occurs if the file is invalid somehow + * + * Programmer: Jacob Smith + * 2017-12-08 + * + *---------------------------------------------------------------------------- + */ +static herr_t +ros3_reset_stats(H5FD_ros3_t *file) +{ + unsigned i = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if ROS3_DEBUG + HDprintf("ros3_reset_stats() called\n"); +#endif + + if (file == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "file was null"); + } + + for (i = 0; i <= ROS3_STATS_BIN_COUNT; i++) { + file->raw[i].bytes = 0; + file->raw[i].count = 0; + file->raw[i].min = (unsigned long long)ROS3_STATS_STARTING_MIN; + file->raw[i].max = 0; + + file->meta[i].bytes = 0; + file->meta[i].count = 0; + file->meta[i].min = (unsigned long long)ROS3_STATS_STARTING_MIN; + file->meta[i].max = 0; + } + +done: + FUNC_LEAVE_NOAPI(ret_value); + +} /* end ros3_reset_stats() */ + +#endif /* ROS3_STATS */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_ros3_open() + * + * Purpose: + * + * Create and/or opens a file as an HDF5 file. + * + * Any flag except H5F_ACC_RDONLY will cause an error. + * + * Name (as received from `H5FD_open()`) must conform to web url: + * NAME :: HTTP "://" DOMAIN [PORT] ["/" [URI] [QUERY] ] + * HTTP :: "http" [ "s" ] + * DOMAIN :: e.g., "mybucket.host.org" + * PORT :: ":" <number> (e.g., ":9000" ) + * URI :: <string> (e.g., "path/to/resource.hd5" ) + * QUERY :: "?" <string> (e.g., "arg1=param1&arg2=param2") + * + * Return: + * + * Success: A pointer to a new file data structure. + * The public fields will be initialized by the caller, which is + * always H5FD_open(). + * + * Failure: NULL + * + * Programmer: Jacob Smith + * 2017-11-02 + * + *------------------------------------------------------------------------- + */ +static H5FD_t * +H5FD_ros3_open( + const char *url, + unsigned flags, + hid_t fapl_id, + haddr_t maxaddr) +{ + H5FD_ros3_t *file = NULL; + struct tm *now = NULL; + char iso8601now[ISO8601_SIZE]; + unsigned char signing_key[SHA256_DIGEST_LENGTH]; + s3r_t *handle = NULL; + H5FD_ros3_fapl_t fa; + H5FD_t *ret_value = NULL; + + + + FUNC_ENTER_NOAPI_NOINIT + +#if ROS3_DEBUG + HDfprintf(stdout, "H5FD_ros3_open() called.\n"); +#endif + + /* Sanity check on file offsets */ + HDcompile_assert(sizeof(HDoff_t) >= sizeof(size_t)); + + /* Check arguments */ + if (!url || !*url) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid file name") + if (0 == maxaddr || HADDR_UNDEF == maxaddr) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, "bogus maxaddr") + if (ADDR_OVERFLOW(maxaddr)) + HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, NULL, "bogus maxaddr") + if (flags != H5F_ACC_RDONLY) + HGOTO_ERROR(H5E_ARGS, H5E_UNSUPPORTED, NULL, + "only Read-Only access allowed") + + if (FAIL == H5Pget_fapl_ros3(fapl_id, &fa)) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "can't get property list") + } + + if (CURLE_OK != curl_global_init(CURL_GLOBAL_DEFAULT)) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "unable to initialize curl global (placeholder flags)") + } + + /* open file; procedure depends on whether or not the fapl instructs to + * authenticate requests or not. + */ + if (fa.authenticate == TRUE) { + /* compute signing key (part of AWS/S3 REST API) + * can be re-used by user/key for 7 days after creation. + * find way to re-use/share + */ + now = gmnow(); + HDassert( now != NULL ); + if (ISO8601NOW(iso8601now, now) != (ISO8601_SIZE - 1)) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "problem while writing iso8601 timestamp") + } + if (FAIL == H5FD_s3comms_signing_key(signing_key, + (const char *)fa.secret_key, + (const char *)fa.aws_region, + (const char *)iso8601now) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "problem while computing signing key") + } + + handle = H5FD_s3comms_s3r_open( + url, + (const char *)fa.aws_region, + (const char *)fa.secret_id, + (const unsigned char *)signing_key); + } else { + handle = H5FD_s3comms_s3r_open(url, NULL, NULL, NULL); + } /* if/else should authenticate */ + + if (handle == NULL) { + /* If we want to check CURL's say on the matter in a controlled + * fashion, this is the place to do it, but would need to make a + * few minor changes to s3comms `s3r_t` and `s3r_read()`. + */ + HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "could not open"); + } + + /* create new file struct + */ + file = H5FL_CALLOC(H5FD_ros3_t); + if (file == NULL) { + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, + "unable to allocate file struct") + } + + file->s3r_handle = handle; + HDmemcpy(&(file->fa), &fa, sizeof(H5FD_ros3_fapl_t)); + +#if ROS3_STATS + if (FAIL == ros3_reset_stats(file)) { + HGOTO_ERROR(H5E_INTERNAL, H5E_UNINITIALIZED, NULL, + "unable to reset file statistics") + } +#endif /* ROS3_STATS */ + + ret_value = (H5FD_t*)file; + +done: + if (ret_value == NULL) { + if (handle != NULL) { + if (FAIL == H5FD_s3comms_s3r_close(handle)) { + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, + "unable to close s3 file handle") + } + } + if (file != NULL) { + file = H5FL_FREE(H5FD_ros3_t, file); + } + curl_global_cleanup(); /* early cleanup because open failed */ + } /* end if null return value (error) */ + + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5FD_ros3_open() */ + +#if ROS3_STATS + +/*---------------------------------------------------------------------------- + * + * Function: ros3_fprint_stats() + * + * Purpose: + * + * Tabulate and pretty-print statistics for this virtual file. + * + * Should be called upon file close. + * + * Shows number of reads and bytes read, broken down by + * "raw" (H5FD_MEM_DRAW) + * or "meta" (any other flag) + * + * Prints filename and listing of total number of reads and bytes read, + * both as a grand total and separate meta- and rawdata reads. + * + * If any reads were done, prints out two tables: + * + * 1. overview of raw- and metadata reads + * - min (smallest size read) + * - average of size read + * - k,M,G suffixes by powers of 1024 (2^10) + * - max (largest size read) + * 2. tabulation of "bins", sepraring reads into exponentially-larger + * ranges of size. + * - columns for number of reads, total bytes, and average size, with + * separate sub-colums for raw- and metadata reads. + * - each row represents one bin, identified by the top of its range + * + * Bin ranges can be modified with pound-defines at the top of this file. + * + * Bins without any reads in their bounds are not printed. + * + * An "overflow" bin is also present, to catch "big" reads. + * + * Output for all bins (and range ceiling and average size report) + * is divied by powers of 1024. By corollary, four digits before the decimal + * is valid. + * + * - 41080 bytes is represented by 40.177k, not 41.080k + * - 1004.831M represents approx. 1052642000 bytes + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * - occurs if the file passed in is invalid + * - TODO: if stream is invalid? how can we check this? + * + * Programmer: Jacob Smith + * + *---------------------------------------------------------------------------- + */ +static herr_t +ros3_fprint_stats(FILE *stream, + const H5FD_ros3_t *file) +{ + herr_t ret_value = SUCCEED; + parsed_url_t *purl = NULL; + unsigned i = 0; + unsigned long count_meta = 0; + unsigned long count_raw = 0; + double average_meta = 0.0; + double average_raw = 0.0; + unsigned long long min_meta = (unsigned long long)ROS3_STATS_STARTING_MIN; + unsigned long long min_raw = (unsigned long long)ROS3_STATS_STARTING_MIN; + unsigned long long max_meta = 0; + unsigned long long max_raw = 0; + unsigned long long bytes_raw = 0; + unsigned long long bytes_meta = 0; + double re_dub = 0.0; /* re-usable double variable */ + unsigned suffix_i = 0; + const char suffixes[] = { ' ', 'K', 'M', 'G', 'T', 'P' }; + + + + FUNC_ENTER_NOAPI_NOINIT + + if (stream == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "file stream cannot be null" ); + } + if (file == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "file cannot be null"); + } + if (file->s3r_handle == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "s3 request handle cannot be null"); + } + if (file->s3r_handle->purl == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "parsed url structure cannot be null"); + } + purl = file->s3r_handle->purl; + + /****************** + * PRINT FILENAME * + ******************/ + + HDfprintf(stream, "stats for %s://%s", purl->scheme, purl->host); + if (purl->port != NULL && purl->port[0] != '\0') + HDfprintf(stream, ":%s", purl->port); + if (purl->query != NULL && purl->query[0] != '\0') { + if (purl->path != NULL && purl->path[0] != '\0') + HDfprintf(stream, "/%s", purl->path); + else + HDfprintf(stream, "/"); + HDfprintf(stream, "?%s", purl->query); + } else if (purl->path != NULL && purl->path[0] != '\0') { + HDfprintf(stream, "/%s", purl->path); + } + HDfprintf(stream, "\n"); + + /******************* + * AGGREGATE STATS * + *******************/ + + for (i = 0; i <= ROS3_STATS_BIN_COUNT; i++) { + const ros3_statsbin *r = &file->raw[i]; + const ros3_statsbin *m = &file->meta[i]; + + if (m->min < min_meta) min_meta = m->min; + if (r->min < min_raw) min_raw = r->min; + if (m->max > max_meta) max_meta = m->max; + if (r->max > max_raw) max_raw = r->max; + + count_raw += r->count; + count_meta += m->count; + bytes_raw += r->bytes; + bytes_meta += m->bytes; + } + if (count_raw > 0) + average_raw = (double)bytes_raw / (double)count_raw; + if (count_meta > 0) + average_meta = (double)bytes_meta / (double)count_meta; + + /****************** + * PRINT OVERVIEW * + ******************/ + + HDfprintf(stream, "TOTAL READS: %llu (%llu meta, %llu raw)\n", + count_raw + count_meta, count_meta, count_raw); + HDfprintf(stream, "TOTAL BYTES: %llu (%llu meta, %llu raw)\n", + bytes_raw + bytes_meta, bytes_meta, bytes_raw); + + if (count_raw + count_meta == 0) + goto done; + + /************************* + * PRINT AGGREGATE STATS * + *************************/ + + HDfprintf(stream, "SIZES meta raw\n"); + HDfprintf(stream, " min "); + if (count_meta == 0) { + HDfprintf(stream, " 0.000 "); + } else { + re_dub = (double)min_meta; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) + re_dub /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]); + } + + if (count_raw == 0) { + HDfprintf(stream, " 0.000 \n"); + } else { + re_dub = (double)min_raw; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) + re_dub /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]); + } + + HDfprintf(stream, " avg "); + re_dub = (double)average_meta; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) + re_dub /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]); + + re_dub = (double)average_raw; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) + re_dub /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]); + + HDfprintf(stream, " max "); + re_dub = (double)max_meta; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) + re_dub /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]); + + re_dub = (double)max_raw; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) + re_dub /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]); + + /****************************** + * PRINT INDIVIDUAL BIN STATS * + ******************************/ + + HDfprintf(stream, + "BINS # of reads total bytes average size\n"); + HDfprintf(stream, + " up-to meta raw meta raw meta raw\n"); + + for (i = 0; i <= ROS3_STATS_BIN_COUNT; i++) { + const ros3_statsbin *m; + const ros3_statsbin *r; + unsigned long long range_end = 0; + char bm_suffix = ' '; /* bytes-meta */ + double bm_val = 0.0; + char br_suffix = ' '; /* bytes-raw */ + double br_val = 0.0; + char am_suffix = ' '; /* average-meta */ + double am_val = 0.0; + char ar_suffix = ' '; /* average-raw */ + double ar_val = 0.0; + + m = &file->meta[i]; + r = &file->raw[i]; + if (r->count == 0 && m->count == 0) + continue; + + range_end = ros3_stats_boundaries[i]; + + if (i == ROS3_STATS_BIN_COUNT) { + range_end = ros3_stats_boundaries[i-1]; + HDfprintf(stream, ">"); + } else { + HDfprintf(stream, " "); + } + + bm_val = (double)m->bytes; + for (suffix_i = 0; bm_val >= 1024.0; suffix_i++) + bm_val /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + bm_suffix = suffixes[suffix_i]; + + br_val = (double)r->bytes; + for (suffix_i = 0; br_val >= 1024.0; suffix_i++) + br_val /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + br_suffix = suffixes[suffix_i]; + + if (m->count > 0) + am_val = (double)(m->bytes) / (double)(m->count); + for (suffix_i = 0; am_val >= 1024.0; suffix_i++) + am_val /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + am_suffix = suffixes[suffix_i]; + + if (r->count > 0) + ar_val = (double)(r->bytes) / (double)(r->count); + for (suffix_i = 0; ar_val >= 1024.0; suffix_i++) + ar_val /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + ar_suffix = suffixes[suffix_i]; + + re_dub = (double)range_end; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) + re_dub /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + + HDfprintf(stream, + " %8.3f%c %7d %7d %8.3f%c %8.3f%c %8.3f%c %8.3f%c\n", + re_dub, suffixes[suffix_i], /* bin ceiling */ + m->count, /* metadata reads */ + r->count, /* rawdata reads */ + bm_val, bm_suffix, /* metadata bytes */ + br_val, br_suffix, /* rawdata bytes */ + am_val, am_suffix, /* metadata average */ + ar_val, ar_suffix); /* rawdata average */ + + fflush(stream); + } + +done: + FUNC_LEAVE_NOAPI(ret_value); + +} /* ros3_fprint_stats */ +#endif /* ROS3_STATS */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_ros3_close() + * + * Purpose: + * + * Close an HDF5 file. + * + * Return: + * + * SUCCEED/FAIL + * + * Programmer: Jacob Smith + * 2017-11-02 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_ros3_close(H5FD_t H5_ATTR_UNUSED *_file) +{ + H5FD_ros3_t *file = (H5FD_ros3_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if ROS3_DEBUG + HDfprintf(stdout, "H5FD_ros3_close() called.\n"); +#endif + + /* Sanity checks + */ + HDassert(file != NULL); + HDassert(file->s3r_handle != NULL); + + /* Close the underlying request handle + */ + if (FAIL == H5FD_s3comms_s3r_close(file->s3r_handle)) { + HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, + "unable to close S3 request handle") + } + +#if ROS3_STATS + /* TODO: mechanism to re-target stats printout */ + if (ros3_fprint_stats(stdout, file) == FAIL) { + HGOTO_ERROR(H5E_INTERNAL, H5E_ERROR, FAIL, + "problem while writing file statistics") + } +#endif /* ROS3_STATS */ + + /* Release the file info + */ + file = H5FL_FREE(H5FD_ros3_t, file); + +done: + curl_global_cleanup(); /* cleanup to answer init on open */ + + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5FD_ros3_close() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_ros3_cmp() + * + * Purpose: + * + * Compares two files belonging to this driver using an arbitrary + * (but consistent) ordering: + * + * + url scheme + * + url host + * + url port + * + url path + * + url query + * + fapl aws_region + * + fapl secret_id + * + fapl secret_key + * + * tl;dr -> check URL, check crentials + * + * Return: + * + * - Equivalent: 0 + * - Not Equivalent: -1 + * + * Programmer: Jacob Smith + * 2017-11-06 + * + *------------------------------------------------------------------------- + */ +static int +H5FD_ros3_cmp( + const H5FD_t *_f1, + const H5FD_t *_f2) +{ + const H5FD_ros3_t *f1 = (const H5FD_ros3_t *)_f1; + const H5FD_ros3_t *f2 = (const H5FD_ros3_t *)_f2; + const parsed_url_t *purl1 = NULL; + const parsed_url_t *purl2 = NULL; + int ret_value = 0; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if ROS3_DEBUG + HDfprintf(stdout, "H5FD_ros3_cmp() called.\n"); +#endif + + HDassert(f1->s3r_handle != NULL); + HDassert(f2->s3r_handle != NULL); + + purl1 = (const parsed_url_t *)f1->s3r_handle->purl; + purl2 = (const parsed_url_t *)f2->s3r_handle->purl; + HDassert(purl1 != NULL); + HDassert(purl2 != NULL); + HDassert(purl1->scheme != NULL); + HDassert(purl2->scheme != NULL); + HDassert(purl1->host != NULL); + HDassert(purl2->host != NULL); + + /* URL: SCHEME */ + if (HDstrcmp(purl1->scheme, purl2->scheme)) { + HGOTO_DONE(-1); + } + + /* URL: HOST */ + if (HDstrcmp(purl1->host, purl2->host)) { + HGOTO_DONE(-1); + } + + /* URL: PORT */ + if (purl1->port && purl2->port) { + if (HDstrcmp(purl1->port, purl2->port)) { + HGOTO_DONE(-1); + } + } + else + if (purl1->port) { + HGOTO_DONE(-1); + } + else + if (purl2->port) { + HGOTO_DONE(-1); + } + + /* URL: PATH */ + if (purl1->path && purl2->path) { + if (HDstrcmp(purl1->path, purl2->path)) { + HGOTO_DONE(-1); + } + } + else + if (purl1->path && !purl2->path) { + HGOTO_DONE(-1); + } + else + if (purl2->path && !purl1->path) { + HGOTO_DONE(-1); + } + + /* URL: QUERY */ + if (purl1->query && purl2->query) { + if (HDstrcmp(purl1->query, purl2->query)) { + HGOTO_DONE(-1); + } + } + else + if (purl1->query && !purl2->query) { + HGOTO_DONE(-1); + } + else + if (purl2->query && !purl1->query) { + HGOTO_DONE(-1); + } + + /* FAPL: AWS_REGION */ + if (f1->fa.aws_region[0] != '\0' && f1->fa.aws_region[0] != '\0') { + if (HDstrcmp(f1->fa.aws_region, f2->fa.aws_region)) { + HGOTO_DONE(-1); + } + } + else + if (f1->fa.aws_region[0] != '\0') { + HGOTO_DONE(-1); + } + else + if (f2->fa.aws_region[0] != '\0') { + HGOTO_DONE(-1); + } + + /* FAPL: SECRET_ID */ + if (f1->fa.secret_id[0] != '\0' && f1->fa.secret_id[0] != '\0') { + if (HDstrcmp(f1->fa.secret_id, f2->fa.secret_id)) { + HGOTO_DONE(-1); + } + } + else + if (f1->fa.secret_id[0] != '\0') { + HGOTO_DONE(-1); + } + else + if (f2->fa.secret_id[0] != '\0') { + HGOTO_DONE(-1); + } + + /* FAPL: SECRET_KEY */ + if (f1->fa.secret_key[0] != '\0' && f1->fa.secret_key[0] != '\0') { + if (HDstrcmp(f1->fa.secret_key, f2->fa.secret_key)) { + HGOTO_DONE(-1); + } + } + else + if (f1->fa.secret_key[0] != '\0') { + HGOTO_DONE(-1); + } + else + if (f2->fa.secret_key[0] != '\0') { + HGOTO_DONE(-1); + } + +done: + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5FD_ros3_cmp() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_ros3_query + * + * Purpose: Set the flags that this VFL driver is capable of supporting. + * (listed in H5FDpublic.h) + * + * Note that since the ROS3 VFD is read only, most flags + * are irrelevant. + * + * The term "set" is highly misleading... + * stores/copies the supported flags in the out-pointer `flags`. + * + * Return: SUCCEED (Can't fail) + * + * Programmer: John Mainzer + * 9/11/17 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_ros3_query(const H5FD_t H5_ATTR_UNUSED *_file, + unsigned long *flags /* out */) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if ROS3_DEBUG + HDfprintf(stdout, "H5FD_ros3_query() called.\n"); +#endif + + /* Set the VFL feature flags that this driver supports */ + if (flags) { + *flags = 0; + /* OK to perform data sieving for faster raw data reads & writes */ + *flags |= H5FD_FEAT_DATA_SIEVE; + } /* end if */ + + FUNC_LEAVE_NOAPI(SUCCEED) + +} /* H5FD_ros3_query() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_ros3_get_eoa() + * + * Purpose: + * + * Gets the end-of-address marker for the file. The EOA marker + * is the first address past the last byte allocated in the + * format address space. + * + * Return: + * + * The end-of-address marker. + * + * Programmer: Jacob Smith + * 2017-11-02 + * + *------------------------------------------------------------------------- + */ +static haddr_t +H5FD_ros3_get_eoa(const H5FD_t *_file, + H5FD_mem_t H5_ATTR_UNUSED type) +{ + const H5FD_ros3_t *file = (const H5FD_ros3_t *)_file; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if ROS3_DEBUG + HDfprintf(stdout, "H5FD_ros3_get_eoa() called.\n"); +#endif + + FUNC_LEAVE_NOAPI(file->eoa) + +} /* end H5FD_ros3_get_eoa() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_ros3_set_eoa() + * + * Purpose: + * + * Set the end-of-address marker for the file. + * + * Return: + * + * SUCCEED (can't fail) + * + * Programmer: Jacob Smith + * 2017-11-03 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_ros3_set_eoa(H5FD_t *_file, + H5FD_mem_t H5_ATTR_UNUSED type, + haddr_t addr) +{ + H5FD_ros3_t *file = (H5FD_ros3_t *)_file; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if ROS3_DEBUG + HDfprintf(stdout, "H5FD_ros3_set_eoa() called.\n"); +#endif + + file->eoa = addr; + + FUNC_LEAVE_NOAPI(SUCCEED) + +} /* H5FD_ros3_set_eoa() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_ros3_get_eof() + * + * Purpose: + * + * Returns the end-of-file marker. + * + * Return: + * + * EOF: the first address past the end of the "file", either the + * filesystem file or the HDF5 file. + * + * Programmer: Jacob Smith + * 2017-11-02 + * + *------------------------------------------------------------------------- + */ +static haddr_t +H5FD_ros3_get_eof(const H5FD_t *_file, + H5FD_mem_t H5_ATTR_UNUSED type) +{ + const H5FD_ros3_t *file = (const H5FD_ros3_t *)_file; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if ROS3_DEBUG + HDfprintf(stdout, "H5FD_ros3_get_eof() called.\n"); +#endif + + FUNC_LEAVE_NOAPI(H5FD_s3comms_s3r_get_filesize(file->s3r_handle)) + +} /* end H5FD_ros3_get_eof() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_ros3_get_handle() + * + * Purpose: + * + * Returns the S3 Request handle (s3r_t) of ros3 file driver. + * + * Returns: + * + * SUCCEED/FAIL + * + * Programmer: Jacob Smith + * 2017-11-02 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_ros3_get_handle(H5FD_t *_file, + hid_t H5_ATTR_UNUSED fapl, + void **file_handle) +{ + H5FD_ros3_t *file = (H5FD_ros3_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if ROS3_DEBUG + HDfprintf(stdout, "H5FD_ros3_get_handle() called.\n"); +#endif + + if (!file_handle) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file handle not valid") + } + + *file_handle = file->s3r_handle; + +done: + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5FD_ros3_get_handle() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_ros3_read() + * + * Purpose + * + * Reads SIZE bytes of data from FILE beginning at address ADDR + * into buffer BUF according to data transfer properties in DXPL_ID. + * + * Return: + * + * Success: `SUCCEED` + * - Result is stored in caller-supplied buffer BUF. + * Failure: `FAIL` + * - Unable to complete read. + * - Contents of buffer `buf` are undefined. + * + * Programmer: Jacob Smith + * 2017-11-?? + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_ros3_read(H5FD_t *_file, + H5FD_mem_t H5_ATTR_UNUSED type, + hid_t H5_ATTR_UNUSED dxpl_id, + haddr_t addr, /* start offset */ + size_t size, /* length of read */ + void *buf) /* out */ +{ + H5FD_ros3_t *file = (H5FD_ros3_t *)_file; + size_t filesize = 0; + herr_t ret_value = SUCCEED; +#if ROS3_STATS + /* working variables for storing stats */ + ros3_statsbin *bin = NULL; + unsigned bin_i = 0; +#endif /* ROS3_STATS */ + + + FUNC_ENTER_NOAPI_NOINIT + +#if ROS3_DEBUG + HDfprintf(stdout, "H5FD_ros3_read() called.\n"); +#endif + + HDassert(file != NULL); + HDassert(file->s3r_handle != NULL); + HDassert(buf != NULL); + + filesize = H5FD_s3comms_s3r_get_filesize(file->s3r_handle); + + if ((addr > filesize) || ((addr + size) > filesize)) { + HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "range exceeds file address") + } + + if (H5FD_s3comms_s3r_read(file->s3r_handle, addr, size, buf) == FAIL) { + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unable to execute read") + } + +#if ROS3_STATS + + /* Find which "bin" this read fits in. Can be "overflow" bin. + */ + for (bin_i = 0; bin_i < ROS3_STATS_BIN_COUNT; bin_i++) { + if ((unsigned long long)size < ros3_stats_boundaries[bin_i]) { + break; + } + } + bin = (type == H5FD_MEM_DRAW) + ? &file->raw[bin_i] + : &file->meta[bin_i]; + + /* Store collected stats in appropriate bin + */ + if (bin->count == 0) { + bin->min = size; + bin->max = size; + } + else { + if (size < bin->min) { + bin->min = size; + } + if (size > bin->max) { + bin->max = size; + } + } + bin->count++; + bin->bytes += (unsigned long long)size; + +#endif /* ROS3_STATS */ + +done: + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5FD_ros3_read() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_ros3_write() + * + * Purpose: + * + * Write bytes to file. + * UNSUPPORTED IN READ-ONLY ROS3 VFD. + * + * Return: + * + * FAIL (Not possible with Read-Only S3 file.) + * + * Programmer: Jacob Smith + * 2017-10-23 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_ros3_write(H5FD_t H5_ATTR_UNUSED *_file, + H5FD_mem_t H5_ATTR_UNUSED type, + hid_t H5_ATTR_UNUSED dxpl_id, + haddr_t H5_ATTR_UNUSED addr, + size_t H5_ATTR_UNUSED size, + const void H5_ATTR_UNUSED *buf) +{ + herr_t ret_value = FAIL; + + FUNC_ENTER_NOAPI_NOINIT + +#if ROS3_DEBUG + HDfprintf(stdout, "H5FD_ros3_write() called.\n"); +#endif + + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, + "cannot write to read-only file.") + +done: + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5FD_ros3_write() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_ros3_truncate() + * + * Purpose: + * + * Makes sure that the true file size is the same (or larger) + * than the end-of-address. + * + * NOT POSSIBLE ON READ-ONLY S3 FILES. + * + * Return: + * + * FAIL (Not possible on Read-Only S3 files.) + * + * Programmer: Jacob Smith + * 2017-10-23 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_ros3_truncate(H5FD_t H5_ATTR_UNUSED *_file, + hid_t H5_ATTR_UNUSED dxpl_id, + hbool_t H5_ATTR_UNUSED closing) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if ROS3_DEBUG + HDfprintf(stdout, "H5FD_ros3_truncate() called.\n"); +#endif + + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, + "cannot truncate read-only file.") + +done: + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5FD_ros3_truncate() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_ros3_lock() + * + * Purpose: + * + * Place an advisory lock on a file. + * No effect on Read-Only S3 file. + * + * Suggestion: remove lock/unlock from class + * > would result in error at H5FD_[un]lock() (H5FD.c) + * + * Return: + * + * SUCCEED (No-op always succeeds) + * + * Programmer: Jacob Smith + * 2017-11-03 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_ros3_lock(H5FD_t H5_ATTR_UNUSED *_file, + hbool_t H5_ATTR_UNUSED rw) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(SUCCEED) + +} /* end H5FD_ros3_lock() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_ros3_unlock() + * + * Purpose: + * + * Remove the existing lock on the file. + * No effect on Read-Only S3 file. + * + * Return: + * + * SUCCEED (No-op always succeeds) + * + * Programmer: Jacob Smith + * 2017-11-03 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_ros3_unlock(H5FD_t H5_ATTR_UNUSED *_file) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(SUCCEED) + +} /* end H5FD_ros3_unlock() */ + +#endif /* H5_HAVE_ROS3_VFD */ + diff --git a/src/H5FDros3.h b/src/H5FDros3.h new file mode 100644 index 0000000..457326e --- /dev/null +++ b/src/H5FDros3.h @@ -0,0 +1,105 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Read-Only S3 Virtual File Driver (VFD) + * + * Programmer: John Mainzer + * 2017-10-10 + * + * Purpose: The public header file for the ros3 driver. + */ +#ifndef H5FDros3_H +#define H5FDros3_H + +#ifdef H5_HAVE_ROS3_VFD +#define H5FD_ROS3 (H5FD_ros3_init()) +#else +#define H5FD_ROS3 (H5I_INVALID_HID) +#endif /* H5_HAVE_ROS3_VFD */ + +#ifdef H5_HAVE_ROS3_VFD + +/**************************************************************************** + * + * Structure: H5FD_ros3_fapl_t + * + * Purpose: + * + * H5FD_ros3_fapl_t is a public structure that is used to pass S3 + * authentication data to the appropriate S3 VFD via the FAPL. A pointer + * to an instance of this structure is a parameter to H5Pset_fapl_ros3() + * and H5Pget_fapl_ros3(). + * + * + * + * `version` (int32_t) + * + * Version number of the H5FD_ros3_fapl_t structure. Any instance passed + * to the above calls must have a recognized version number, or an error + * will be flagged. + * + * This field should be set to H5FD_CURR_ROS3_FAPL_T_VERSION. + * + * `authenticate` (hbool_t) + * + * Flag TRUE or FALSE whether or not requests are to be authenticated + * with the AWS4 algorithm. + * If TRUE, `aws_region`, `secret_id`, and `secret_key` must be populated. + * If FALSE, those three components are unused. + * + * `aws_region` (char[]) + * + * String: name of the AWS "region" of the host, e.g. "us-east-1". + * + * `secret_id` (char[]) + * + * String: "Access ID" for the resource. + * + * `secret_key` (char[]) + * + * String: "Secret Access Key" associated with the ID and resource. + * + ****************************************************************************/ + +#define H5FD_CURR_ROS3_FAPL_T_VERSION 1 + +#define H5FD_ROS3_MAX_REGION_LEN 32 +#define H5FD_ROS3_MAX_SECRET_ID_LEN 128 +#define H5FD_ROS3_MAX_SECRET_KEY_LEN 128 + +typedef struct H5FD_ros3_fapl_t { + int32_t version; + hbool_t authenticate; + char aws_region[H5FD_ROS3_MAX_REGION_LEN + 1]; + char secret_id[H5FD_ROS3_MAX_SECRET_ID_LEN + 1]; + char secret_key[H5FD_ROS3_MAX_SECRET_KEY_LEN + 1]; +} H5FD_ros3_fapl_t; + + +#ifdef __cplusplus +extern "C" { +#endif + +H5_DLL hid_t H5FD_ros3_init(void); +H5_DLL herr_t H5Pget_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa_out); +H5_DLL herr_t H5Pset_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa); + +#ifdef __cplusplus +} +#endif + +#endif /* H5_HAVE_ROS3_VFD */ + +#endif /* ifndef H5FDros3_H */ + + diff --git a/src/H5FDs3comms.c b/src/H5FDs3comms.c new file mode 100644 index 0000000..f08e9d5 --- /dev/null +++ b/src/H5FDs3comms.c @@ -0,0 +1,3593 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/***************************************************************************** + * Read-Only S3 Virtual File Driver (VFD) + * + * Source for S3 Communications module + * + * ***NOT A FILE DRIVER*** + * + * Provide functions and structures required for interfacing with Amazon + * Simple Storage Service (S3). + * + * Provide S3 object access as if it were a local file. + * + * Connect to remote host, send and receive HTTP requests and responses + * as part of the AWS REST API, authenticating requests as appropriate. + * + * Programmer: Jacob Smith + * 2017-11-30 + * + *****************************************************************************/ + +/****************/ +/* Module Setup */ +/****************/ + +/***********/ +/* Headers */ +/***********/ + +#include "H5private.h" /* generic functions */ +#include "H5Eprivate.h" /* error handling */ +#include "H5MMprivate.h" /* memory management */ +#include "H5FDs3comms.h" /* S3 Communications */ + +/****************/ +/* Local Macros */ +/****************/ + +#ifdef H5_HAVE_ROS3_VFD + +/* toggle debugging (enable with 1) + */ +#define S3COMMS_DEBUG 0 + +/* manipulate verbosity of CURL output + * operates separately from S3COMMS_DEBUG + * + * 0 -> no explicit curl output + * 1 -> on error, print failure info to stderr + * 2 -> in addition to above, print information for all performs; sets all + * curl handles with CURLOPT_VERBOSE + */ +#define S3COMMS_CURL_VERBOSITY 0 + +/* size to allocate for "bytes=<first_byte>[-<last_byte>]" HTTP Range value + */ +#define S3COMMS_MAX_RANGE_STRING_SIZE 128 + + +/******************/ +/* Local Typedefs */ +/******************/ + +/********************/ +/* Local Structures */ +/********************/ + +/* struct s3r_datastruct + * Structure passed to curl write callback + * pointer to data region and record of bytes written (offset) + */ +struct s3r_datastruct { + unsigned long magic; + char *data; + size_t size; +}; +#define S3COMMS_CALLBACK_DATASTRUCT_MAGIC 0x28c2b2ul + +/********************/ +/* Local Prototypes */ +/********************/ + +size_t curlwritecallback(char *ptr, + size_t size, + size_t nmemb, + void *userdata); + +herr_t H5FD_s3comms_s3r_getsize(s3r_t *handle); + +/*********************/ +/* Package Variables */ +/*********************/ + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + +/*******************/ +/* Local Variables */ +/*******************/ + +/*************/ +/* Functions */ +/*************/ + + +/*---------------------------------------------------------------------------- + * + * Function: curlwritecallback() + * + * Purpose: + * + * Function called by CURL to write received data. + * + * Writes bytes to `userdata`. + * + * Internally manages number of bytes processed. + * + * Return: + * + * - Number of bytes processed. + * - Should equal number of bytes passed to callback. + * - Failure will result in curl error: CURLE_WRITE_ERROR. + * + * Programmer: Jacob Smith + * 2017-08-17 + * + *---------------------------------------------------------------------------- + */ +size_t +curlwritecallback(char *ptr, + size_t size, + size_t nmemb, + void *userdata) +{ + struct s3r_datastruct *sds = (struct s3r_datastruct *)userdata; + size_t product = (size * nmemb); + size_t written = 0; + + if (sds->magic != S3COMMS_CALLBACK_DATASTRUCT_MAGIC) { + return written; + } + + if (size > 0) { + HDmemcpy(&(sds->data[sds->size]), ptr, product); + sds->size += product; + written = product; + } + + return written; + +} /* end curlwritecallback() */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_hrb_node_set() + * + * Purpose: + * + * Create, insert, modify, and remove elements in a field node list. + * + * `name` cannot be null; will return FAIL and list will be unaltered. + * + * Entries are accessed via the lowercase representation of their name: + * "Host", "host", and "hOSt" would all access the same node, + * but name's case is relevant in HTTP request output. + * + * List pointer `L` must always point to either of : + * - header node with lowest alphabetical order (by lowername) + * - NULL, if list is empty + * + * Types of operations: + * + * - CREATE + * - If `L` is NULL and `name` and `value` are not NULL, + * a new node is created at `L`, starting a list. + * - MODIFY + * - If a node is found with a matching lowercase name and `value` + * is not NULL, the existing name, value, and cat values are released + * and replaced with the new data. + * - No modifications are made to the list pointers. + * - REMOVE + * - If `value` is NULL, will attempt to remove node with matching + * lowercase name. + * - If no match found, returns FAIL and list is not modified. + * - When removing a node, all its resources is released. + * - If removing the last node in the list, list pointer is set to NULL. + * - INSERT + * - If no nodes exists with matching lowercase name and `value` + * is not NULL, a new node is created, inserted into list + * alphabetically by lowercase name. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - List was successfully modified + * - FAILURE: `FAIL` + * - Unable to perform operation + * - Forbidden (attempting to remove absent node, e.g.) + * - Internal error + * + * Programmer: Jacob Smith + * 2017-09-22 + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_hrb_node_set( + hrb_node_t **L, + const char *name, + const char *value) +{ + size_t i = 0; + char *valuecpy = NULL; + char *namecpy = NULL; + size_t namelen = 0; + char *lowername = NULL; + char *nvcat = NULL; + hrb_node_t *node_ptr = NULL; + hrb_node_t *new_node = NULL; + hbool_t is_looking = TRUE; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_hrb_node_set.\n"); + HDprintf("NAME: %s\n", name); + HDprintf("VALUE: %s\n", value); + HDprintf("LIST:\n->"); + for (node_ptr = (*L); node_ptr != NULL; node_ptr = node_ptr->next) { + HDfprintf(stdout, "{%s}\n->", node_ptr->cat); + } + HDprintf("(null)\n"); + fflush(stdout); + node_ptr = NULL; +#endif + + if (name == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to operate on null name.\n"); + } + namelen = HDstrlen(name); + + /*********************** + * PREPARE ALL STRINGS * + **********************/ + + /* copy and lowercase name + */ + lowername = (char *)H5MM_malloc(sizeof(char) * (namelen + 1)); + if (lowername == NULL) { + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, + "cannot make space for lowercase name copy.\n"); + } + for (i = 0; i < namelen; i++) { + lowername[i] = (char)tolower((int)name[i]); + } + lowername[namelen] = 0; + + /* If value supplied, copy name, value, and concatenated "name: value". + * If NULL, we will be removing a node or doing nothing, so no need for + * copies + */ + if (value != NULL) { + int ret = 0; + size_t valuelen = HDstrlen(value); + size_t catlen = namelen + valuelen + 2; /* +2 from ": " */ + size_t catwrite = catlen + 3; /* 3 not 1 to quiet compiler warning */ + + + namecpy = (char *)H5MM_malloc(sizeof(char) * (namelen + 1)); + if (namecpy == NULL) { + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, + "cannot make space for name copy.\n"); + } + HDmemcpy(namecpy, name, (namelen + 1)); + + valuecpy = (char *)H5MM_malloc(sizeof(char) * (valuelen + 1)); + if (valuecpy == NULL) { + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, + "cannot make space for value copy.\n"); + } + HDmemcpy(valuecpy, value, (valuelen + 1)); + + nvcat = (char *)H5MM_malloc(sizeof(char) * catwrite); + if (nvcat == NULL) { + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, + "cannot make space for concatenated string.\n"); + } + ret = HDsnprintf(nvcat, catwrite, "%s: %s", name, value); + if (ret < 0 || (size_t)ret > catlen) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "cannot concatenate `%s: %s", name, value); + } + HDassert( catlen == HDstrlen(nvcat) ); + + /* create new_node, should we need it + */ + new_node = (hrb_node_t *)H5MM_malloc(sizeof(hrb_node_t)); + if (new_node == NULL) { + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, + "cannot make space for new set.\n"); + } + + new_node->magic = S3COMMS_HRB_NODE_MAGIC; + new_node->name = NULL; + new_node->value = NULL; + new_node->cat = NULL; + new_node->lowername = NULL; + new_node->next = NULL; + } + + /*************** + * ACT ON LIST * + ***************/ + + if (*L == NULL) { + if (value == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "trying to remove node from empty list"); + } + else { +#if S3COMMS_DEBUG +HDprintf("CREATE NEW\n"); fflush(stdout); +#endif + /******************* + * CREATE NEW LIST * + *******************/ + + new_node->cat = nvcat; + new_node->name = namecpy; + new_node->lowername = lowername; + new_node->value = valuecpy; + + *L = new_node; + goto done; /* bypass further seeking */ + } + } + + /* sanity-check pointer passed in + */ + HDassert( (*L) != NULL ); + HDassert( (*L)->magic == S3COMMS_HRB_NODE_MAGIC ); + node_ptr = (*L); + + /* Check whether to modify/remove first node in list + */ + if (strcmp(lowername, node_ptr->lowername) == 0) { + + is_looking = FALSE; + + if (value == NULL) { +#if S3COMMS_DEBUG +HDprintf("REMOVE HEAD\n"); fflush(stdout); +#endif + /*************** + * REMOVE HEAD * + ***************/ + + *L = node_ptr->next; + +#if S3COMMS_DEBUG +HDprintf("FREEING CAT (node)\n"); fflush(stdout); +#endif + H5MM_xfree(node_ptr->cat); +#if S3COMMS_DEBUG +HDprintf("FREEING LOWERNAME (node)\n"); fflush(stdout); +#endif + H5MM_xfree(node_ptr->lowername); +#if S3COMMS_DEBUG +HDprintf("FREEING NAME (node)\n"); fflush(stdout); +#endif + H5MM_xfree(node_ptr->name); +#if S3COMMS_DEBUG +HDprintf("FREEING VALUE (node)\n"); fflush(stdout); +#endif + H5MM_xfree(node_ptr->value); +#if S3COMMS_DEBUG +HDprintf("MAGIC OK? %s\n", + (node_ptr->magic == S3COMMS_HRB_NODE_MAGIC) ? "YES" : "NO"); +fflush(stdout); +#endif + HDassert( node_ptr->magic == S3COMMS_HRB_NODE_MAGIC ); + node_ptr->magic += 1ul; +#if S3COMMS_DEBUG +HDprintf("FREEING POINTER\n"); fflush(stdout); +#endif + H5MM_xfree(node_ptr); + +#if S3COMMS_DEBUG +HDprintf("FREEING WORKING LOWERNAME\n"); fflush(stdout); +#endif + H5MM_xfree(lowername); lowername = NULL; + } + else { +#if S3COMMS_DEBUG +HDprintf("MODIFY HEAD\n"); fflush(stdout); +#endif + /*************** + * MODIFY HEAD * + ***************/ + + H5MM_xfree(node_ptr->cat); + H5MM_xfree(node_ptr->name); + H5MM_xfree(node_ptr->value); + + node_ptr->name = namecpy; + node_ptr->value = valuecpy; + node_ptr->cat = nvcat; + + H5MM_xfree(lowername); + lowername = NULL; + new_node->magic += 1ul; + H5MM_xfree(new_node); + new_node = NULL; + } + } + else + if (strcmp(lowername, node_ptr->lowername) < 0) { + + is_looking = FALSE; + + if (value == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "trying to remove a node 'before' head"); + } + else { +#if S3COMMS_DEBUG +HDprintf("PREPEND NEW HEAD\n"); fflush(stdout); +#endif + /******************* + * INSERT NEW HEAD * + *******************/ + + new_node->name = namecpy; + new_node->value = valuecpy; + new_node->lowername = lowername; + new_node->cat = nvcat; + new_node->next = node_ptr; + *L = new_node; + } + } + + /*************** + * SEARCH LIST * + ***************/ + + while (is_looking) { + if (node_ptr->next == NULL) { + + is_looking = FALSE; + + if (value == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "trying to remove absent node"); + } + else { +#if S3COMMS_DEBUG +HDprintf("APPEND A NODE\n"); fflush(stdout); +#endif + /******************* + * APPEND NEW NODE * + *******************/ + + HDassert( strcmp(lowername, node_ptr->lowername) > 0 ); + new_node->name = namecpy; + new_node->value = valuecpy; + new_node->lowername = lowername; + new_node->cat = nvcat; + node_ptr->next = new_node; + } + } + else + if (strcmp(lowername, node_ptr->next->lowername) < 0) { + + is_looking = FALSE; + + if (value == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "trying to remove absent node"); + } + else { +#if S3COMMS_DEBUG +HDprintf("INSERT A NODE\n"); fflush(stdout); +#endif + /******************* + * INSERT NEW NODE * + *******************/ + + HDassert( strcmp(lowername, node_ptr->lowername) > 0 ); + new_node->name = namecpy; + new_node->value = valuecpy; + new_node->lowername = lowername; + new_node->cat = nvcat; + new_node->next = node_ptr->next; + node_ptr->next = new_node; + } + } + else + if (strcmp(lowername, node_ptr->next->lowername) == 0) { + + is_looking = FALSE; + + if (value == NULL) { + /***************** + * REMOVE A NODE * + *****************/ + + hrb_node_t *tmp = node_ptr->next; + node_ptr->next = tmp->next; + +#if S3COMMS_DEBUG +HDprintf("REMOVE A NODE\n"); fflush(stdout); +#endif + H5MM_xfree(tmp->cat); + H5MM_xfree(tmp->lowername); + H5MM_xfree(tmp->name); + H5MM_xfree(tmp->value); + + HDassert( tmp->magic == S3COMMS_HRB_NODE_MAGIC ); + tmp->magic += 1ul; + H5MM_xfree(tmp); + + H5MM_xfree(lowername); + lowername = NULL; + } + else { +#if S3COMMS_DEBUG +HDprintf("MODIFY A NODE\n"); fflush(stdout); +#endif + /***************** + * MODIFY A NODE * + *****************/ + + node_ptr = node_ptr->next; + H5MM_xfree(node_ptr->name); + H5MM_xfree(node_ptr->value); + H5MM_xfree(node_ptr->cat); + + HDassert( new_node->magic == S3COMMS_HRB_NODE_MAGIC ); + new_node->magic += 1ul; + H5MM_xfree(new_node); + H5MM_xfree(lowername); + new_node = NULL; + lowername = NULL; + + node_ptr->name = namecpy; + node_ptr->value = valuecpy; + node_ptr->cat = nvcat; + } + } + else { + /**************** + * KEEP LOOKING * + ****************/ + + node_ptr = node_ptr->next; + } + } /* end while is_looking */ + +done: + if (ret_value == FAIL) { + /* clean up + */ + if (nvcat != NULL) { H5MM_xfree(nvcat); } + if (namecpy != NULL) { H5MM_xfree(namecpy); } + if (lowername != NULL) { H5MM_xfree(lowername); } + if (valuecpy != NULL) { H5MM_xfree(valuecpy); } + if (new_node != NULL) { + HDassert( new_node->magic == S3COMMS_HRB_NODE_MAGIC ); + new_node->magic += 1ul; + H5MM_xfree(new_node); + } + } + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_hrb_node_set() */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_hrb_destroy() + * + * Purpose: + * + * Destroy and free resources _directly_ associated with an HTTP Buffer. + * + * Takes a pointer to pointer to the buffer structure. + * This allows for the pointer itself to be NULLed from within the call. + * + * If buffer or buffer pointer is NULL, there is no effect. + * + * Headers list at `first_header` is not touched. + * + * - Programmer should re-use or destroy `first_header` pointer + * (hrb_node_t *) as suits their purposes. + * - Recommend fetching prior to destroy() + * e.g., `reuse_node = hrb_to_die->first_header; destroy(hrb_to_die);` + * or maintaining an external reference. + * - Destroy node/list separately as appropriate + * - Failure to account for this will result in a memory leak. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - successfully released buffer resources + * - if `buf` is NULL or `*buf` is NULL, no effect + * - FAILURE: `FAIL` + * - `buf->magic != S3COMMS_HRB_MAGIC` + * + * Programmer: Jacob Smith + * 2017-07-21 + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_hrb_destroy(hrb_t **_buf) +{ + hrb_t *buf = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_hrb_destroy.\n"); +#endif + + if (_buf != NULL && *_buf != NULL) { + buf = *_buf; + if (buf->magic != S3COMMS_HRB_MAGIC) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "pointer's magic does not match.\n"); + } + + H5MM_xfree(buf->verb); + H5MM_xfree(buf->version); + H5MM_xfree(buf->resource); + buf->magic += 1ul; + H5MM_xfree(buf); + *_buf = NULL; + } /* end if `_buf` has some value */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_s3comms_hrb_destroy() */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_hrb_init_request() + * + * Purpose: + * + * Create a new HTTP Request Buffer + * + * All non-null arguments should be null-terminated strings. + * + * If `verb` is NULL, defaults to "GET". + * If `http_version` is NULL, defaults to "HTTP/1.1". + * + * `resource` cannot be NULL; should be string beginning with slash + * character ('/'). + * + * All strings are copied into the structure, making them safe from + * modification in source strings. + * + * Return: + * + * - SUCCESS: pointer to new `hrb_t` + * - FAILURE: `NULL` + * + * Programmer: Jacob Smith + * 2017-07-21 + * + *---------------------------------------------------------------------------- + */ +hrb_t * +H5FD_s3comms_hrb_init_request(const char *_verb, + const char *_resource, + const char *_http_version) +{ + hrb_t *request = NULL; + char *res = NULL; + size_t reslen = 0; + hrb_t *ret_value = NULL; + char *verb = NULL; + size_t verblen = 0; + char *vrsn = NULL; + size_t vrsnlen = 0; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_hrb_init_request.\n"); +#endif + + if (_resource == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "resource string cannot be null.\n"); + } + + /* populate valid NULLs with defaults + */ + if (_verb == NULL) { + _verb = "GET"; + } + + if (_http_version == NULL) { + _http_version = "HTTP/1.1"; + } + + /* malloc space for and prepare structure + */ + request = (hrb_t *)H5MM_malloc(sizeof(hrb_t)); + if (request == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL, + "no space for request structure"); + } + request->magic = S3COMMS_HRB_MAGIC; + request->body = NULL; + request->body_len = 0; + request->first_header = NULL; + + + + /* malloc and copy strings for the structure + */ + reslen = HDstrlen(_resource); + + if (_resource[0] == '/') { + res = (char *)H5MM_malloc(sizeof(char) * (reslen+1)); + if (res == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL, + "no space for resource string"); + } + HDmemcpy(res, _resource, (reslen+1)); + } + else { + res = (char *)H5MM_malloc(sizeof(char) * (reslen+2)); + if (res == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL, + "no space for resource string"); + } + *res = '/'; + HDmemcpy((&res[1]), _resource, (reslen+1)); + HDassert( (reslen+1) == HDstrlen(res) ); + } /* end if (else resource string not starting with '/') */ + + verblen = HDstrlen(_verb) + 1; + verb = (char *)H5MM_malloc(sizeof(char) * verblen); + if (verb == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "no space for verb string"); + } + HDstrncpy(verb, _verb, verblen); + + vrsnlen = HDstrlen(_http_version) + 1; + vrsn = (char *)H5MM_malloc(sizeof(char) * vrsnlen); + if (vrsn == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "no space for http-version string"); + } + HDstrncpy(vrsn, _http_version, vrsnlen); + + + + /* place new copies into structure + */ + request->resource = res; + request->verb = verb; + request->version = vrsn; + + ret_value = request; + +done: + /* if there is an error, clean up after ourselves + */ + if (ret_value == NULL) { + if (request != NULL) H5MM_xfree(request); + if (vrsn != NULL) H5MM_xfree(vrsn); + if (verb != NULL) H5MM_xfree(verb); + if (res != NULL) H5MM_xfree(res); + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_s3comms_hrb_init_request() */ + + +/**************************************************************************** + * S3R FUNCTIONS + ****************************************************************************/ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_s3r_close() + * + * Purpose: + * + * Close communications through given S3 Request Handle (`s3r_t`) + * and clean up associated resources. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * - fails if handle is null or has invalid magic number + * + * + * Programmer: Jacob Smith + * 2017-08-31 + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_s3r_close(s3r_t *handle) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_s3r_close.\n"); +#endif + + if (handle == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle cannot be null.\n"); + } + if (handle->magic != S3COMMS_S3R_MAGIC) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle has invalid magic.\n"); + } + + curl_easy_cleanup(handle->curlhandle); + + H5MM_xfree(handle->secret_id); + H5MM_xfree(handle->region); + H5MM_xfree(handle->signing_key); + + HDassert( handle->httpverb != NULL ); + H5MM_xfree(handle->httpverb); + + if (FAIL == H5FD_s3comms_free_purl(handle->purl)) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to release parsed url structure") + } + + H5MM_xfree(handle); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD_s3comms_s3r_close */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_s3r_get_filesize() + * + * Purpose: + * + * Retrieve the filesize of an open request handle. + * + * Wrapper "getter" to hide implementation details. + * + * + * Return: + * + * - SUCCESS: size of file, in bytes, if handle is valid. + * - FAILURE: 0, if handle is NULL or undefined. + * + * Programmer: Jacob Smith 2017-01-14 + * + *---------------------------------------------------------------------------- + */ +size_t +H5FD_s3comms_s3r_get_filesize(s3r_t *handle) +{ + size_t ret_value = 0; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + if (handle != NULL) { + ret_value = handle->filesize; + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD_s3comms_s3r_get_filesize */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_s3r_getsize() + * + * Purpose: + * + * Get the number of bytes of handle's target resource. + * + * Sets handle and curlhandle with to enact an HTTP HEAD request on file, + * and parses received headers to extract "Content-Length" from response + * headers, storing file size at `handle->filesize`. + * + * Critical step in opening (initiating) an `s3r_t` handle. + * + * Wraps `s3r_read()`. + * Sets curlhandle to write headers to a temporary buffer (using extant + * write callback) and provides no buffer for body. + * + * Upon exit, unsets HTTP HEAD settings from curl handle, returning to + * initial state. In event of error, curl handle state is undefined and is + * not to be trusted. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * + * Programmer: Jacob Smith + * 2017-08-23 + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_s3r_getsize(s3r_t *handle) +{ + uintmax_t content_length = 0; + CURL *curlh = NULL; + char *end = NULL; + char *headerresponse = NULL; + struct s3r_datastruct sds = { + S3COMMS_CALLBACK_DATASTRUCT_MAGIC, + NULL, + 0 }; + char *start = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_s3r_getsize.\n"); +#endif + + if (handle == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle cannot be null.\n"); + } + if (handle->magic != S3COMMS_S3R_MAGIC) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle has invalid magic.\n"); + } + if (handle->curlhandle == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle has bad (null) curlhandle.\n") + } + + /******************** + * PREPARE FOR HEAD * + ********************/ + + curlh = handle->curlhandle; + + if ( CURLE_OK != + curl_easy_setopt(curlh, + CURLOPT_NOBODY, + 1L) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "error while setting CURL option (CURLOPT_NOBODY). " + "(placeholder flags)"); + } + + if ( CURLE_OK != + curl_easy_setopt(curlh, + CURLOPT_HEADERDATA, + &sds) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "error while setting CURL option (CURLOPT_HEADERDATA). " + "(placeholder flags)"); + } + + HDassert( handle->httpverb == NULL ); + handle->httpverb = (char *)H5MM_malloc(sizeof(char) * 16); + if (handle->httpverb == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, + "unable to allocate space for S3 request HTTP verb"); + } + HDmemcpy(handle->httpverb, "HEAD", 5); + + headerresponse = (char *)H5MM_malloc(sizeof(char) * CURL_MAX_HTTP_HEADER); + if (headerresponse == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, + "unable to allocate space for curl header response"); + } + sds.data = headerresponse; + + /******************* + * PERFORM REQUEST * + *******************/ + + /* these parameters fetch the entire file, + * but, with a NULL destination and NOBODY and HEADERDATA supplied above, + * only http metadata will be sent by server and recorded by s3comms + */ + if (FAIL == + H5FD_s3comms_s3r_read(handle, 0, 0, NULL) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem in reading during getsize.\n"); + } + + if (sds.size > CURL_MAX_HTTP_HEADER) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "HTTP metadata buffer overrun\n"); + } else if (sds.size == 0) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "No HTTP metadata\n"); +#if S3COMMS_DEBUG + } else { + HDfprintf(stderr, "GETSIZE: OK\n"); +#endif + } + + + /****************** + * PARSE RESPONSE * + ******************/ + + start = strstr(headerresponse, + "\r\nContent-Length: "); + if (start == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "could not find \"Content-Length\" in response.\n"); + } + + /* move "start" to beginning of value in line; find end of line + */ + start = start + HDstrlen("\r\nContent-Length: "); + end = strstr(start, "\r\n"); + if (end == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "could not find end of content length line"); + } + + /* place null terminator at end of numbers + */ + *end = '\0'; + + content_length = strtoumax((const char *)start, + NULL, + 0); + + if (UINTMAX_MAX > SIZE_MAX && content_length > SIZE_MAX) { + HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "content_length overflows size_t\n"); + } + + if (content_length == 0 || + errno == ERANGE) /* errno set by strtoumax*/ + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "could not convert found \"Content-Length\" response (\"%s\")", + start); /* range is null-terminated, remember */ + } + + handle->filesize = (size_t)content_length; + + /********************** + * UNDO HEAD SETTINGS * + **********************/ + + if ( CURLE_OK != + curl_easy_setopt(curlh, + CURLOPT_NOBODY, + NULL) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "error while setting CURL option (CURLOPT_NOBODY). " + "(placeholder flags)"); + } + + if ( CURLE_OK != + curl_easy_setopt(curlh, + CURLOPT_HEADERDATA, + NULL) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "error while setting CURL option (CURLOPT_HEADERDATA). " + "(placeholder flags)"); + } + +done: + H5MM_xfree(headerresponse); + sds.magic += 1; /* set to bad magic */ + + FUNC_LEAVE_NOAPI(ret_value); + +} /* H5FD_s3comms_s3r_getsize */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_s3r_open() + * + * Purpose: + * + * Logically 'open' a file hosted on S3. + * + * - create new Request Handle + * - copy supplied url + * - copy authentication info if supplied + * - create CURL handle + * - fetch size of file + * - connect with server and execute HEAD request + * - return request handle ready for reads + * + * To use 'default' port to connect, `port` should be 0. + * + * To prevent AWS4 authentication, pass null pointer to `region`, `id`, + * and `signing_key`. + * + * Uses `H5FD_s3comms_parse_url()` to validate and parse url input. + * + * Return: + * + * - SUCCESS: Pointer to new request handle. + * - FAILURE: NULL + * - occurs if: + * - authentication strings are inconsistent + * - must _all_ be null, or have at least `region` and `id` + * - url is NULL (no filename) + * - unable to parse url (malformed?) + * - error while performing `getsize()` + * + * Programmer: Jacob Smith + * 2017-09-01 + * + *---------------------------------------------------------------------------- + */ +s3r_t * +H5FD_s3comms_s3r_open(const char *url, + const char *region, + const char *id, + const unsigned char *signing_key) +{ + size_t tmplen = 0; + CURL *curlh = NULL; + s3r_t *handle = NULL; + parsed_url_t *purl = NULL; + s3r_t *ret_value = NULL; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_s3r_open.\n"); +#endif + + + + if (url == NULL || url[0] == '\0') { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "url cannot be null.\n"); + } + + if (FAIL == H5FD_s3comms_parse_url(url, &purl)) { + /* probably a malformed url, but could be internal error */ + HGOTO_ERROR(H5E_ARGS, H5E_CANTCREATE, NULL, + "unable to create parsed url structure"); + } + HDassert( purl != NULL ); /* if above passes, this must be true */ + HDassert( purl->magic == S3COMMS_PARSED_URL_MAGIC ); + + handle = (s3r_t *)H5MM_malloc(sizeof(s3r_t)); + if (handle == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL, + "could not malloc space for handle.\n"); + } + + handle->magic = S3COMMS_S3R_MAGIC; + handle->purl = purl; + handle->filesize = 0; + handle->region = NULL; + handle->secret_id = NULL; + handle->signing_key = NULL; + handle->httpverb = NULL; + + /************************************* + * RECORD AUTHENTICATION INFORMATION * + *************************************/ + + if ((region != NULL && *region != '\0') || + (id != NULL && *id != '\0') || + (signing_key != NULL && *signing_key != '\0')) + { + /* if one exists, all three must exist + */ + if (region == NULL || region[0] == '\0') { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "region cannot be null.\n"); + } + if (id == NULL || id[0] == '\0') { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "secret id cannot be null.\n"); + } + if (signing_key == NULL || signing_key[0] == '\0') { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "signing key cannot be null.\n"); + } + + /* copy strings + */ + tmplen = HDstrlen(region) + 1; + handle->region = (char *)H5MM_malloc(sizeof(char) * tmplen); + if (handle->region == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "could not malloc space for handle region copy.\n"); + } + HDmemcpy(handle->region, region, tmplen); + + tmplen = HDstrlen(id) + 1; + handle->secret_id = (char *)H5MM_malloc(sizeof(char) * tmplen); + if (handle->secret_id == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "could not malloc space for handle ID copy.\n"); + } + HDmemcpy(handle->secret_id, id, tmplen); + + tmplen = SHA256_DIGEST_LENGTH; + handle->signing_key = + (unsigned char *)H5MM_malloc(sizeof(unsigned char) * tmplen); + if (handle->signing_key == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "could not malloc space for handle key copy.\n"); + } + HDmemcpy(handle->signing_key, signing_key, tmplen); + } /* if authentication information provided */ + + /************************ + * INITIATE CURL HANDLE * + ************************/ + + curlh = curl_easy_init(); + + if (curlh == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "problem creating curl easy handle!\n"); + } + + if ( CURLE_OK != + curl_easy_setopt(curlh, + CURLOPT_HTTPGET, + 1L) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "error while setting CURL option (CURLOPT_HTTPGET). " + "(placeholder flags)"); + } + + if ( CURLE_OK != + curl_easy_setopt(curlh, + CURLOPT_HTTP_VERSION, + CURL_HTTP_VERSION_1_1) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "error while setting CURL option (CURLOPT_HTTP_VERSION). " + "(placeholder flags)"); + } + + if ( CURLE_OK != + curl_easy_setopt(curlh, + CURLOPT_FAILONERROR, + 1L) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "error while setting CURL option (CURLOPT_FAILONERROR). " + "(placeholder flags)"); + } + + if ( CURLE_OK != + curl_easy_setopt(curlh, + CURLOPT_WRITEFUNCTION, + curlwritecallback) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "error while setting CURL option (CURLOPT_WRITEFUNCTION). " + "(placeholder flags)"); + } + + if ( CURLE_OK != + curl_easy_setopt(curlh, + CURLOPT_URL, + url) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "error while setting CURL option (CURLOPT_URL). " + "(placeholder flags)"); + } + +#if S3COMMS_CURL_VERBOSITY > 1 + /* CURL will print (to stdout) information for each operation + */ + curl_easy_setopt(curlh, CURLOPT_VERBOSE, 1L); +#endif + + handle->curlhandle = curlh; + + /******************* + * OPEN CONNECTION * + * * * * * * * * * * + * GET FILE SIZE * + *******************/ + + if (FAIL == + H5FD_s3comms_s3r_getsize(handle) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "problem in H5FD_s3comms_s3r_getsize.\n"); + } + + /********************* + * FINAL PREPARATION * + *********************/ + + HDassert( handle->httpverb != NULL ); + HDmemcpy(handle->httpverb, "GET", 4); + + ret_value = handle; + +done: + if (ret_value == NULL) { + if (curlh != NULL) { + curl_easy_cleanup(curlh); + } + if (FAIL == H5FD_s3comms_free_purl(purl)) { + HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "unable to free parsed url structure") + } + if (handle != NULL) { + H5MM_xfree(handle->region); + H5MM_xfree(handle->secret_id); + H5MM_xfree(handle->signing_key); + if (handle->httpverb != NULL) { + H5MM_xfree(handle->httpverb); + } + H5MM_xfree(handle); + } + } + + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5FD_s3comms_s3r_open */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_s3r_read() + * + * Purpose: + * + * Read file pointed to by request handle, writing specified + * `offset` .. `offset + len` bytes to buffer `dest`. + * + * If `len` is 0, reads entirety of file starting at `offset`. + * If `offset` and `len` are both 0, reads entire file. + * + * If `offset` or `offset+len` is greater than the file size, read is + * aborted and returns `FAIL`. + * + * Uses configured "curl easy handle" to perform request. + * + * In event of error, buffer should remain unaltered. + * + * If handle is set to authorize a request, creates a new (temporary) + * HTTP Request object (hrb_t) for generating requisite headers, + * which is then translated to a `curl slist` and set in the curl handle + * for the request. + * + * `dest` _may_ be NULL, but no body data will be recorded. + * + * - In general practice, NULL should never be passed in as `dest`. + * - NULL `dest` passed in by internal function `s3r_getsize()`, in + * conjunction with CURLOPT_NOBODY to preempt transmission of file data + * from server. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * + * Programmer: Jacob Smith + * 2017-08-22 + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_s3r_read(s3r_t *handle, + haddr_t offset, + size_t len, + void *dest) +{ + CURL *curlh = NULL; + CURLcode p_status = CURLE_OK; + struct curl_slist *curlheaders = NULL; + hrb_node_t *headers = NULL; + hrb_node_t *node = NULL; + struct tm *now = NULL; + char *rangebytesstr = NULL; + hrb_t *request = NULL; + int ret = 0; /* working variable to check */ + /* return value of HDsnprintf */ + struct s3r_datastruct *sds = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_s3r_read.\n"); +#endif + + /************************************** + * ABSOLUTELY NECESSARY SANITY-CHECKS * + **************************************/ + + if (handle == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle cannot be null.\n"); + } + if (handle->magic != S3COMMS_S3R_MAGIC) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle has invalid magic.\n"); + } + if (handle->curlhandle == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle has bad (null) curlhandle.\n") + } + if (handle->purl == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle has bad (null) url.\n") + } + HDassert( handle->purl->magic == S3COMMS_PARSED_URL_MAGIC ); + if (offset > handle->filesize || (len + offset) > handle->filesize) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to read past EoF") + } + + curlh = handle->curlhandle; + + /********************* + * PREPARE WRITEDATA * + *********************/ + + if (dest != NULL) { + sds = (struct s3r_datastruct *)H5MM_malloc( + sizeof(struct s3r_datastruct)); + if (sds == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, + "could not malloc destination datastructure.\n"); + } + + sds->magic = S3COMMS_CALLBACK_DATASTRUCT_MAGIC; + sds->data = (char *)dest; + sds->size = 0; + if (CURLE_OK != + curl_easy_setopt(curlh, + CURLOPT_WRITEDATA, + sds) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, + "error while setting CURL option (CURLOPT_WRITEDATA). " + "(placeholder flags)"); + } + } + + /********************* + * FORMAT HTTP RANGE * + *********************/ + + if (len > 0) { + rangebytesstr = (char *)H5MM_malloc(sizeof(char) * \ + (S3COMMS_MAX_RANGE_STRING_SIZE+1) ); + if (rangebytesstr == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, + "could not malloc range format string.\n"); + } + ret = HDsnprintf(rangebytesstr, + (S3COMMS_MAX_RANGE_STRING_SIZE), + "bytes="H5_PRINTF_HADDR_FMT"-"H5_PRINTF_HADDR_FMT, + offset, + offset + len - 1); + if (ret <= 0 || ret >= S3COMMS_MAX_RANGE_STRING_SIZE) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to format HTTP Range value"); + } else if (offset > 0) { + rangebytesstr = (char *)H5MM_malloc(sizeof(char) * \ + (S3COMMS_MAX_RANGE_STRING_SIZE+1)); + if (rangebytesstr == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, + "could not malloc range format string.\n"); + } + ret = HDsnprintf(rangebytesstr, + (S3COMMS_MAX_RANGE_STRING_SIZE), + "bytes="H5_PRINTF_HADDR_FMT"-", + offset); + if (ret <= 0 || ret >= S3COMMS_MAX_RANGE_STRING_SIZE) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to format HTTP Range value"); + } + + /******************* + * COMPILE REQUEST * + *******************/ + + if (handle->signing_key == NULL) { + /* Do not authenticate. + */ + if (rangebytesstr != NULL) { + /* Pass in range directly + */ + char *bytesrange_ptr = NULL; /* pointer past "bytes=" portion */ + + bytesrange_ptr = strchr(rangebytesstr, '='); + HDassert( bytesrange_ptr != NULL ); + bytesrange_ptr++; /* move to first char past '=' */ + HDassert( *bytesrange_ptr != '\0' ); + + if (CURLE_OK != + curl_easy_setopt(curlh, + CURLOPT_RANGE, + bytesrange_ptr) ) + { + HGOTO_ERROR(H5E_VFL, H5E_UNINITIALIZED, FAIL, + "error while setting CURL option (CURLOPT_RANGE). "); + } + } + } else { + /* authenticate request + */ + char authorization[512+1]; + /* 512 := approximate max length... + * 67 <len("AWS4-HMAC-SHA256 Credential=///s3/aws4_request," + * "SignedHeaders=,Signature=")> + * + 8 <yyyyMMDD> + * + 64 <hex(sha256())> + * + 128 <max? len(secret_id)> + * + 20 <max? len(region)> + * + 128 <max? len(signed_headers)> + */ + char buffer1[512+1]; /* -> Canonical Request -> Signature */ + char buffer2[256+1]; /* -> String To Sign -> Credential */ + char iso8601now[ISO8601_SIZE]; + char signed_headers[48+1]; + /* should be large enough for nominal listing: + * "host;range;x-amz-content-sha256;x-amz-date" + * + '\0', with "range;" possibly absent + */ + + /* zero start of strings */ + authorization[0] = 0; + buffer1[0] = 0; + buffer2[0] = 0; + iso8601now[0] = 0; + signed_headers[0] = 0; + + /**** VERIFY INFORMATION EXISTS ****/ + + if (handle->region == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle must have non-null region.\n"); + } + if (handle->secret_id == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle must have non-null secret_id.\n"); + } + if (handle->signing_key == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle must have non-null signing_key.\n"); + } + if (handle->httpverb == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle must have non-null httpverb.\n"); + } + if (handle->purl->host == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle must have non-null host.\n"); + } + if (handle->purl->path == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle must have non-null resource.\n"); + } + + /**** CREATE HTTP REQUEST STRUCTURE (hrb_t) ****/ + + request = H5FD_s3comms_hrb_init_request( + (const char *)handle->httpverb, + (const char *)handle->purl->path, + "HTTP/1.1"); + if (request == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "could not allocate hrb_t request.\n"); + } + HDassert( request->magic == S3COMMS_HRB_MAGIC ); + + now = gmnow(); + if (ISO8601NOW(iso8601now, now) != (ISO8601_SIZE - 1)) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "could not format ISO8601 time.\n"); + } + + if (FAIL == + H5FD_s3comms_hrb_node_set( + &headers, + "x-amz-date", + (const char *)iso8601now) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to set x-amz-date header") + } + if (headers == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem building headers list. " + "(placeholder flags)\n"); + } + HDassert( headers->magic == S3COMMS_HRB_NODE_MAGIC ); + + if (FAIL == + H5FD_s3comms_hrb_node_set( + &headers, + "x-amz-content-sha256", + (const char *)EMPTY_SHA256) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to set x-amz-content-sha256 header") + } + if (headers == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem building headers list. " + "(placeholder flags)\n"); + } + HDassert( headers->magic == S3COMMS_HRB_NODE_MAGIC ); + + if (rangebytesstr != NULL) { + if (FAIL == + H5FD_s3comms_hrb_node_set( + &headers, + "Range", + (const char *)rangebytesstr) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to set range header") + } + if (headers == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem building headers list. " + "(placeholder flags)\n"); + } + HDassert( headers->magic == S3COMMS_HRB_NODE_MAGIC ); + } + + if (FAIL == + H5FD_s3comms_hrb_node_set( + &headers, + "Host", + (const char *)handle->purl->host) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to set host header") + } + if (headers == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem building headers list. " + "(placeholder flags)\n"); + } + HDassert( headers->magic == S3COMMS_HRB_NODE_MAGIC ); + + request->first_header = headers; + + /**** COMPUTE AUTHORIZATION ****/ + + if (FAIL == /* buffer1 -> canonical request */ + H5FD_s3comms_aws_canonical_request( + buffer1, + 512, + signed_headers, + 48, + request) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "(placeholder flags)\n"); + } + if ( FAIL == /* buffer2->string-to-sign */ + H5FD_s3comms_tostringtosign(buffer2, + buffer1, + iso8601now, + handle->region) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "(placeholder flags)\n"); + } + if (FAIL == /* buffer1 -> signature */ + H5FD_s3comms_HMAC_SHA256(handle->signing_key, + SHA256_DIGEST_LENGTH, + buffer2, + HDstrlen(buffer2), + buffer1) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "(placeholder flags)\n"); + } + + iso8601now[8] = 0; /* trim to yyyyMMDD */ + ret = S3COMMS_FORMAT_CREDENTIAL(buffer2, + handle->secret_id, + iso8601now, + handle->region, + "s3"); + if (ret == 0 || ret >= S3COMMS_MAX_CREDENTIAL_SIZE) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to format aws4 credential string"); + + ret = HDsnprintf( + authorization, + 512, + "AWS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s", + buffer2, + signed_headers, + buffer1); + if (ret <= 0 || ret >= 512) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to format aws4 authorization string"); + } + + /* append authorization header to http request buffer + */ + if (H5FD_s3comms_hrb_node_set( + &headers, + "Authorization", + (const char *)authorization) + == FAIL) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to set Authorization header") + } + if (headers == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem building headers list. " + "(placeholder flags)\n"); + } + + /* update hrb's "first header" pointer + */ + request->first_header = headers; + + /**** SET CURLHANDLE HTTP HEADERS FROM GENERATED DATA ****/ + + node = request->first_header; + while (node != NULL) { + HDassert( node->magic == S3COMMS_HRB_NODE_MAGIC ); + curlheaders = curl_slist_append(curlheaders, + (const char *)node->cat); + if (curlheaders == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "could not append header to curl slist. " + "(placeholder flags)\n"); + } + node = node->next; + } + + /* sanity-check + */ + if (curlheaders == NULL) { + /* above loop was probably never run */ + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "curlheaders was never populated.\n"); + } + + /* finally, set http headers in curl handle + */ + if (curl_easy_setopt( + curlh, + CURLOPT_HTTPHEADER, + curlheaders) + != CURLE_OK) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "error while setting CURL option " + "(CURLOPT_HTTPHEADER). (placeholder flags)"); + } + + } /* end if should authenticate (info provided) */ + + /******************* + * PERFORM REQUEST * + *******************/ + +#if S3COMMS_CURL_VERBOSITY > 0 + /* In event of error, print detailed information to stderr + * This is not the default behavior. + */ + { + long int httpcode = 0; + char curlerrbuf[CURL_ERROR_SIZE]; + curlerrbuf[0] = '\0'; + + if (CURLE_OK != + curl_easy_setopt(curlh, CURLOPT_ERRORBUFFER, curlerrbuf) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem setting error buffer") + } + + p_status = curl_easy_perform(curlh); + + if (p_status != CURLE_OK) { + if (CURLE_OK != + curl_easy_getinfo(curlh, CURLINFO_RESPONSE_CODE, &httpcode) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem getting response code") + } + HDfprintf(stderr, "CURL ERROR CODE: %d\nHTTP CODE: %d\n", + p_status, httpcode); + HDfprintf(stderr, "%s\n", curl_easy_strerror(p_status)); + HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL, + "problem while performing request.\n"); + } + if (CURLE_OK != + curl_easy_setopt(curlh, CURLOPT_ERRORBUFFER, NULL) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem unsetting error buffer") + } + } /* verbose error reporting */ +#else + p_status = curl_easy_perform(curlh); + + if (p_status != CURLE_OK) { + HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL, + "curl cannot perform request\n") + } +#endif + +#if S3COMMS_DEBUG + if (dest != NULL) { + HDfprintf(stderr, "len: %d\n", (int)len); + HDfprintf(stderr, "CHECKING FOR BUFFER OVERFLOW\n"); + if (sds == NULL) { + HDfprintf(stderr, "sds is NULL!\n"); + } + else { + HDfprintf(stderr, "sds: 0x%lx\n", (long long)sds); + HDfprintf(stderr, "sds->size: %d\n", (int)sds->size); + if (len > sds->size) { + HDfprintf(stderr, "buffer overwrite\n"); + } + } + } + else { + HDfprintf(stderr, "performed on entire file\n"); + } +#endif + +done: + /* clean any malloc'd resources + */ + if (curlheaders != NULL) { + curl_slist_free_all(curlheaders); + curlheaders = NULL; + } + if (rangebytesstr != NULL) { + H5MM_xfree(rangebytesstr); + rangebytesstr = NULL; + } + if (sds != NULL) { + H5MM_xfree(sds); + sds = NULL; + } + if (request != NULL) { + while (headers != NULL) + if (FAIL == + H5FD_s3comms_hrb_node_set(&headers, headers->name, NULL)) + { + HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "cannot release header node") + } + HDassert( NULL == headers ); + if (FAIL == H5FD_s3comms_hrb_destroy(&request)) { + HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "cannot release header request structure") + } + HDassert( NULL == request ); + } + + if (curlh != NULL) { + /* clear any Range */ + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_RANGE, NULL) ) { + HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "cannot unset CURLOPT_RANGE") + } + + /* clear headers */ + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_HTTPHEADER, NULL) ) { + HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "cannot unset CURLOPT_HTTPHEADER") + } + } + + FUNC_LEAVE_NOAPI(ret_value); +} /* H5FD_s3comms_s3r_read */ + + +/**************************************************************************** + * MISCELLANEOUS FUNCTIONS + ****************************************************************************/ + + +/*---------------------------------------------------------------------------- + * + * Function: gmnow() + * + * Purpose: + * + * Get the output of `time.h`'s `gmtime()` call while minimizing setup + * clutter where important. + * + * Return: + * + * Pointer to resulting `struct tm`,as created by gmtime(time_t * T). + * + * Programmer: Jacob Smith + * 2017-07-12 + * + *---------------------------------------------------------------------------- + */ +struct tm * +gmnow(void) +{ + time_t now; + time_t *now_ptr = &now; + struct tm *ret_value = NULL; + + /* Doctor assert, checks against error in time() */ + if ( (time_t)(-1) != time(now_ptr) ) { + ret_value = gmtime(now_ptr); + } + + HDassert( ret_value != NULL ); + + return ret_value; +} /* end gmnow() */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_aws_canonical_request() + * + * Purpose: + * + * Compose AWS "Canonical Request" (and signed headers string) + * as defined in the REST API documentation. + * + * Both destination strings are null-terminated. + * + * Destination string arguments must be provided with adequate space. + * + * Canonical Request format: + * + * <HTTP VERB>"\n" + * <resource path>"\n" + * <query string>"\n" + * <header1>"\n" (`lowercase(name)`":"`trim(value)`) + * <header2>"\n" + * ... (headers sorted by name) + * <header_n>"\n" + * "\n" + * <signed headers>"\n" (`lowercase(header 1 name)`";"`header 2 name`;...) + * <hex-string of sha256sum of body> ("e3b0c4429...", e.g.) + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - writes canonical request to respective `...dest` strings + * - FAILURE: `FAIL` + * - one or more input argument was NULL + * - internal error + * + * Programmer: Jacob Smith + * 2017-10-04 + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_aws_canonical_request( + char *canonical_request_dest, + int _cr_size, + char *signed_headers_dest, + int _sh_size, + hrb_t *http_request) +{ + hrb_node_t *node = NULL; + const char *query_params = ""; /* unused at present */ + herr_t ret_value = SUCCEED; + int ret = 0; + size_t cr_size = (size_t)_cr_size; + size_t sh_size = (size_t)_sh_size; + size_t cr_len = 0; /* working length of canonical request str */ + size_t sh_len = 0; /* working length of signed headers str */ + char tmpstr[256+1]; + tmpstr[256] = 0; /* terminating NULL */ + + /* "query params" refers to the optional element in the URL, e.g. + * http://bucket.aws.com/myfile.txt?max-keys=2&prefix=J + * ^-----------------^ + * + * Not handled/implemented as of 2017-10-xx. + * Element introduced as empty placeholder and reminder. + * Further research to be done if this is ever relevant for the + * VFD use-cases. + */ + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_aws_canonical_request.\n"); +#endif + + if (http_request == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "hrb object cannot be null.\n"); + } + HDassert( http_request->magic == S3COMMS_HRB_MAGIC ); + + if (canonical_request_dest == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "canonical request destination cannot be null.\n"); + } + + if (signed_headers_dest == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "signed headers destination cannot be null.\n"); + } + + /* HTTP verb, resource path, and query string lines + */ + cr_len = (HDstrlen(http_request->verb) + + HDstrlen(http_request->resource) + + HDstrlen(query_params) + + (size_t)3); /* three newline chars */ + if (cr_len >= cr_size) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "not enough space in canonical request"); + } + ret = HDsnprintf( /* TODO: compiler warning */ + canonical_request_dest, + (cr_size-1), + "%s\n%s\n%s\n", + http_request->verb, + http_request->resource, + query_params); + if (ret < 0 || (size_t)ret >= cr_size) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to compose canonical request first line"); + } + + /* write in canonical headers, building signed headers concurrently + */ + node = http_request->first_header; /* assumed sorted */ + while (node != NULL) { + + HDassert(node->magic == S3COMMS_HRB_NODE_MAGIC); + + ret = HDsnprintf( + tmpstr, + 256, + "%s:%s\n", + node->lowername, + node->value); + if (ret < 0 || ret >= 256) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to concatenate HTTP header %s:%s", + node->lowername, + node->value); + } + cr_len += HDstrlen(tmpstr); + if (cr_len + 1 > cr_size) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "not enough space in canonical request"); + } + strcat(canonical_request_dest, tmpstr); + + ret = HDsnprintf( + tmpstr, + 256, + "%s;", + node->lowername); + if (ret < 0 || ret >= 256) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to append semicolon to lowername %s", + node->lowername); + } + sh_len += HDstrlen(tmpstr); + if (sh_len + 1 > sh_size) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "not enough space in signed headers"); + } + strcat(signed_headers_dest, tmpstr); + + node = node->next; + } /* end while node is not NULL */ + + /* remove tailing ';' from signed headers sequence + */ + signed_headers_dest[HDstrlen(signed_headers_dest) - 1] = '\0'; + + /* append signed headers and payload hash + * NOTE: at present, no HTTP body is handled, per the nature of + * requests/range-gets + */ + strcat(canonical_request_dest, "\n"); + strcat(canonical_request_dest, signed_headers_dest); + strcat(canonical_request_dest, "\n"); + strcat(canonical_request_dest, EMPTY_SHA256); + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_aws_canonical_request() */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_bytes_to_hex() + * + * Purpose: + * + * Produce human-readable hex string [0-9A-F] from sequence of bytes. + * + * For each byte (char), writes two-character hexadecimal representation. + * + * No null-terminator appended. + * + * Assumes `dest` is allocated to enough size (msg_len * 2). + * + * Fails if either `dest` or `msg` are null. + * + * `msg_len` message length of 0 has no effect. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - hex string written to `dest` (not null-terminated) + * - FAILURE: `FAIL` + * - `dest == NULL` + * - `msg == NULL` + * + * Programmer: Jacob Smith + * 2017-07-12 + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_bytes_to_hex( + char *dest, + const unsigned char *msg, + size_t msg_len, + hbool_t lowercase) +{ + size_t i = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_bytes_to_hex.\n"); +#endif + + if (dest == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "hex destination cannot be null.\n") + } + if (msg == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "bytes sequence cannot be null.\n") + } + + for (i = 0; i < msg_len; i++) { + int chars_written = HDsnprintf(&(dest[i * 2]), + 3, /* 'X', 'X', '\n' */ + (lowercase == TRUE) ? "%02x" : "%02X", + msg[i]); + if (chars_written != 2) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem while writing hex chars for %c", + msg[i]); + } + } + +done: + FUNC_LEAVE_NOAPI(ret_value); + +} /* end H5FD_s3comms_bytes_to_hex() */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_free_purl() + * + * Purpose: + * + * Release resources from a parsed_url_t pointer. + * + * If pointer is null, nothing happens. + * + * Return: + * + * `SUCCEED` (never fails) + * + * Programmer: Jacob Smith + * 2017-11-01 + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_free_purl(parsed_url_t *purl) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if S3COMMS_DEBUG + HDprintf("called H5FD_s3comms_free_purl.\n"); +#endif + + if (purl != NULL) { + HDassert( purl->magic == S3COMMS_PARSED_URL_MAGIC ); + if (purl->scheme != NULL) { H5MM_xfree(purl->scheme); } + if (purl->host != NULL) { H5MM_xfree(purl->host); } + if (purl->port != NULL) { H5MM_xfree(purl->port); } + if (purl->path != NULL) { H5MM_xfree(purl->path); } + if (purl->query != NULL) { H5MM_xfree(purl->query); } + purl->magic += 1ul; + H5MM_xfree(purl); + } + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5FD_s3comms_free_purl() */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_HMAC_SHA256() + * + * Purpose: + * + * Generate Hash-based Message Authentication Checksum using the SHA-256 + * hashing algorithm. + * + * Given a key, message, and respective lengths (to accommodate null + * characters in either), generate _hex string_ of authentication checksum + * and write to `dest`. + * + * `dest` must be at least `SHA256_DIGEST_LENGTH * 2` characters in size. + * Not enforceable by this function. + * `dest` will _not_ be null-terminated by this function. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - hex string written to `dest` (not null-terminated) + * - FAILURE: `FAIL` + * - `dest == NULL` + * - error while generating hex string output + * + * Programmer: Jacob Smith + * 2017-07-?? + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_HMAC_SHA256( + const unsigned char *key, + size_t key_len, + const char *msg, + size_t msg_len, + char *dest) +{ + unsigned char md[SHA256_DIGEST_LENGTH]; + unsigned int md_len = SHA256_DIGEST_LENGTH; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_HMAC_SHA256.\n"); +#endif + + if (dest == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "destination cannot be null."); + } + + HMAC(EVP_sha256(), + key, + (int)key_len, + (const unsigned char *)msg, + msg_len, + md, + &md_len); + + if (H5FD_s3comms_bytes_to_hex( + dest, + (const unsigned char *)md, + (size_t)md_len, + true) + == FAIL) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "could not convert to hex string."); + } + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* H5FD_s3comms_HMAC_SHA256 */ + + +/*----------------------------------------------------------------------------- + * + * Function: H5FD__s3comms_load_aws_creds_from_file() + * + * Purpose: + * + * Extract AWS configuration information from a target file. + * + * Given a file and a profile name, e.g. "ros3_vfd_test", attempt to locate + * that region in the file. If not found, returns in error and output + * pointers are not modified. + * + * If the profile label is found, attempts to locate and parse configuration + * data, stopping at the first line where: + * + reached end of file + * + line does not start with a recognized setting name + * + * Following AWS documentation, looks for any of: + * + aws_access_key_id + * + aws_secret_access_key + * + region + * + * To be valid, the setting must begin the line with one of the keywords, + * followed immediately by an equals sign '=', and have some data before + * newline at end of line. + * + `spam=eggs` would be INVALID because name is unrecognized + * + `region = us-east-2` would be INVALID because of spaces + * + `region=` would be INVALID because no data. + * + * Upon successful parsing of a setting line, will store the result in the + * corresponding output pointer. If the output pointer is NULL, will skip + * any matching setting line while parsing -- useful to prevent overwrite + * when reading from multiple files. + * + * Return: + * + * + SUCCESS: `SUCCEED` + * + no error. settings may or may not have been loaded. + * + FAILURE: `FAIL` + * + internal error occurred. + * + -1 :: unable to format profile label + * + -2 :: profile name/label not found in file + * + -3 :: some other error + * + * Programmer: Jacob Smith + * 2018-02-27 + * + *----------------------------------------------------------------------------- + */ +static herr_t +H5FD__s3comms_load_aws_creds_from_file( + FILE *file, + const char *profile_name, + char *key_id, + char *access_key, + char *aws_region) +{ + char profile_line[32]; + char buffer[128]; + const char *setting_names[] = { + "region", + "aws_access_key_id", + "aws_secret_access_key", + }; + char * const setting_pointers[] = { + aws_region, + key_id, + access_key, + }; + unsigned setting_count = 3; + herr_t ret_value = SUCCEED; + unsigned buffer_i = 0; + unsigned setting_i = 0; + int found_setting = 0; + char *line_buffer = &(buffer[0]); + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called load_aws_creds_from_file.\n"); +#endif + + /* format target line for start of profile */ + if (32 < HDsnprintf(profile_line, 32, "[%s]", profile_name)) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, + "unable to format profile label") + } + + /* look for start of profile */ + do { + /* clear buffer */ + for (buffer_i=0; buffer_i < 128; buffer_i++) { + buffer[buffer_i] = 0; + } + + line_buffer = fgets(line_buffer, 128, file); + if (line_buffer == NULL) { /* reached end of file */ + goto done; + } + } while (strncmp(line_buffer, profile_line, HDstrlen(profile_line))); + + /* extract credentials from lines */ + do { + /* clear buffer */ + for (buffer_i=0; buffer_i < 128; buffer_i++) { + buffer[buffer_i] = 0; + } + + /* collect a line from file */ + line_buffer = fgets(line_buffer, 128, file); + if (line_buffer == NULL) { + goto done; /* end of file */ + } + + /* loop over names to see if line looks like assignment */ + for (setting_i = 0; setting_i < setting_count; setting_i++) { + size_t setting_name_len = 0; + const char *setting_name = NULL; + char line_prefix[128]; + + setting_name = setting_names[setting_i]; + setting_name_len = HDstrlen(setting_name); + if (HDsnprintf(line_prefix, 128, "%s=", setting_name) < 0) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, + "unable to format line prefix") + } + + /* found a matching name? */ + if (!HDstrncmp(line_buffer, line_prefix, setting_name_len + 1)) { + found_setting = 1; + + /* skip NULL destination buffer */ + if (setting_pointers[setting_i] == NULL) { + break; + } + + /* advance to end of name in string */ + do { + line_buffer++; + } while (*line_buffer != 0 && *line_buffer != '='); + + if (*line_buffer == 0 || *(line_buffer+1) == 0) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "incomplete assignment in file") + } + line_buffer++; /* was pointing at '='; advance */ + + /* copy line buffer into out pointer */ + if (HDstrncpy( + setting_pointers[setting_i], + (const char *)line_buffer, + HDstrlen(line_buffer)) + == NULL) + { + HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, + "unable to copy line into pointer") + } + + /* "trim" tailing whitespace by replacing with null terminator*/ + buffer_i = 0; + while (!isspace(setting_pointers[setting_i][buffer_i])) { + buffer_i++; + } + setting_pointers[setting_i][buffer_i] = '\0'; + + break; /* have read setting; don't compare with others */ + } /* end if possible name match */ + } /* end for each setting name */ + } while (found_setting); + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__s3comms_load_aws_creds_from_file() */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_load_aws_profile() + * + * Purpose : + * + * Read aws profile elements from standard location on system and store + * settings in memory. + * + * Looks for both `~/.aws/config` and `~/.aws/credentials`, the standard + * files for AWS tools. If a file exists (can be opened), looks for the + * given profile name and reads the settings into the relevant buffer. + * + * Any setting duplicated in both files will be set to that from + * `credentials`. + * + * Settings are stored in the supplied buffers as null-terminated strings. + * + * Return: + * + * + SUCCESS: `SUCCEED` (0) + * + no error occurred and all settings were populated + * + FAILURE: `FAIL` (-1) + * + internal error occurred + * + unable to locate profile + * + region, key id, and secret key were not all found and set + * + * Programmer: Jacob Smith + * 2018-02-27 + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_load_aws_profile(const char *profile_name, + char *key_id_out, + char *secret_access_key_out, + char *aws_region_out) +{ + herr_t ret_value = SUCCEED; + FILE *credfile = NULL; + char awspath[117]; + char filepath[128]; + int ret = 0; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_load_aws_profile.\n"); +#endif + +#ifdef H5_HAVE_WIN32_API + ret = HDsnprintf(awspath, 117, "%s/.aws/", getenv("USERPROFILE")) ; +#else + ret = HDsnprintf(awspath, 117, "%s/.aws/", getenv("HOME")) ; +#endif + if (ret < 0 || (size_t)ret >= 117) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, + "unable to format home-aws path") + } + ret = HDsnprintf(filepath, 128, "%s%s", awspath, "credentials"); + if (ret < 0 || (size_t)ret >= 128) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, + "unable to format credentials path") + } + + credfile = fopen(filepath, "r"); + if (credfile != NULL) { + if (H5FD__s3comms_load_aws_creds_from_file( + credfile, + profile_name, + key_id_out, + secret_access_key_out, + aws_region_out) + == FAIL) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to load from aws credentials") + } + if (fclose(credfile) == EOF) { + HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, + "unable to close credentials file") + } + credfile = NULL; + } /* end if credential file opened */ + + ret = HDsnprintf(filepath, 128, "%s%s", awspath, "config"); + if (ret < 0 || (size_t)ret >= 128) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, + "unable to format config path") + } + credfile = fopen(filepath, "r"); + if (credfile != NULL) { + if (H5FD__s3comms_load_aws_creds_from_file( + credfile, + profile_name, + (*key_id_out == 0) ? key_id_out : NULL, + (*secret_access_key_out == 0) ? secret_access_key_out : NULL, + (*aws_region_out == 0) ? aws_region_out : NULL) + == FAIL) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to load from aws config") + } + if (fclose(credfile) == EOF) { + HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, + "unable to close config file") + } + credfile = NULL; + } /* end if credential file opened */ + + /* fail if not all three settings were loaded */ + if (*key_id_out == 0 || + *secret_access_key_out == 0 || + *aws_region_out == 0) + { + ret_value = FAIL; + } + +done: + if (credfile != NULL) { + if (fclose(credfile) == EOF) { + HDONE_ERROR(H5E_ARGS, H5E_ARGS, FAIL, + "problem error-closing aws configuration file") + } + } + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_load_aws_profile() */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_nlowercase() + * + * Purpose: + * + * From string starting at `s`, write `len` characters to `dest`, + * converting all to lowercase. + * + * Behavior is undefined if `s` is NULL or `len` overruns the allocated + * space of either `s` or `dest`. + * + * Provided as convenience. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - upon completion, `dest` is populated + * - FAILURE: `FAIL` + * - `dest == NULL` + * + * Programmer: Jacob Smith + * 2017-09-18 + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_nlowercase( + char *dest, + const char *s, + size_t len) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_nlowercase.\n"); +#endif + + if (dest == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "destination cannot be null.\n"); + } + + if (len > 0) { + HDmemcpy(dest, s, len); + do { + len--; + dest[len] = (char)tolower( (int)dest[len] ); + } while (len > 0); + } + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_nlowercase() */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_parse_url() + * + * Purpose: + * + * Parse URL-like string and stuff URL components into + * `parsed_url` structure, if possible. + * + * Expects null-terminated string of format: + * SCHEME "://" HOST [":" PORT ] ["/" [ PATH ] ] ["?" QUERY] + * where SCHEME :: "[a-zA-Z/.-]+" + * PORT :: "[0-9]" + * + * Stores resulting structure in argument pointer `purl`, if successful, + * creating and populating new `parsed_url_t` structure pointer. + * Empty or absent elements are NULL in new purl structure. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - `purl` pointer is populated + * - FAILURE: `FAIL` + * - unable to parse + * - `purl` is unaltered (probably NULL) + * + * Programmer: Jacob Smith + * 2017-10-30 + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_parse_url( + const char *str, + parsed_url_t **_purl) +{ + parsed_url_t *purl = NULL; /* pointer to new structure */ + const char *tmpstr = NULL; /* working pointer in string */ + const char *curstr = str; /* "start" pointer in string */ + long int len = 0; /* substring length */ + long int urllen = 0; /* length of passed-in url string */ + unsigned int i = 0; + herr_t ret_value = FAIL; + + FUNC_ENTER_NOAPI_NOINIT; + +#if S3COMMS_DEBUG + HDprintf("called H5FD_s3comms_parse_url.\n"); +#endif + + if (str == NULL || *str == '\0') { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "invalid url string"); + } + + urllen = (long int)HDstrlen(str); + + purl = (parsed_url_t *)H5MM_malloc(sizeof(parsed_url_t)); + if (purl == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, + "can't allocate space for parsed_url_t"); + } + purl->magic = S3COMMS_PARSED_URL_MAGIC; + purl->scheme = NULL; + purl->host = NULL; + purl->port = NULL; + purl->path = NULL; + purl->query = NULL; + + /*************** + * READ SCHEME * + ***************/ + + tmpstr = HDstrchr(curstr, ':'); + if (tmpstr == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "invalid SCHEME construction: probably not URL"); + } + len = tmpstr - curstr; + HDassert( (0 <= len) && (len < urllen) ); + + /* check for restrictions + */ + for (i = 0; i < len; i++) { + /* scheme = [a-zA-Z+-.]+ (terminated by ":") */ + if (!isalpha(curstr[i]) && + '+' != curstr[i] && + '-' != curstr[i] && + '.' != curstr[i]) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "invalid SCHEME construction"); + } + } + /* copy lowercased scheme to structure + */ + purl->scheme = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1)); + if (purl->scheme == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, + "can't allocate space for SCHEME"); + } + (void)HDstrncpy(purl->scheme, curstr, (size_t)len); + purl->scheme[len] = '\0'; + for ( i = 0; i < len; i++ ) { + purl->scheme[i] = (char)tolower(purl->scheme[i]); + } + + /* Skip "://" */ + tmpstr += 3; + curstr = tmpstr; + + /************* + * READ HOST * + *************/ + + if (*curstr == '[') { + /* IPv6 */ + while (']' != *tmpstr) { + if (tmpstr == 0) { /* end of string reached! */ + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "reached end of URL: incomplete IPv6 HOST"); + } + tmpstr++; + } + tmpstr++; + } /* end if (IPv6) */ + else { + while (0 != *tmpstr) { + if (':' == *tmpstr || + '/' == *tmpstr || + '?' == *tmpstr) + { + break; + } + tmpstr++; + } + } /* end else (IPv4) */ + len = tmpstr - curstr; + if (len == 0) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "HOST substring cannot be empty"); + } + else + if (len > urllen) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem with length of HOST substring"); + } + + /* copy host + */ + purl->host = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1)); + if (purl->host == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, + "can't allocate space for HOST"); + } + (void)HDstrncpy(purl->host, curstr, (size_t)len); + purl->host[len] = 0; + + /************* + * READ PORT * + *************/ + + if (':' == *tmpstr) { + tmpstr += 1; /* advance past ':' */ + curstr = tmpstr; + while ((0 != *tmpstr) && ('/' != *tmpstr) && ('?' != *tmpstr)) { + tmpstr++; + } + len = tmpstr - curstr; + if (len == 0) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "PORT element cannot be empty"); + } + else + if (len > urllen) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem with length of PORT substring"); + } + for (i = 0; i < len; i ++) { + if (!isdigit(curstr[i])) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "PORT is not a decimal string"); + } + } + + /* copy port + */ + purl->port = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1)); + if (purl->port == NULL) { /* cannot malloc */ + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, + "can't allocate space for PORT"); + } + (void)HDstrncpy(purl->port, curstr, (size_t)len); + purl->port[len] = 0; + } /* end if PORT element */ + + /************* + * READ PATH * + *************/ + + if ('/' == *tmpstr) { + /* advance past '/' */ + tmpstr += 1; + curstr = tmpstr; + + /* seek end of PATH + */ + while ((0 != *tmpstr) && ('?' != *tmpstr)) { + tmpstr++; + } + len = tmpstr - curstr; + if (len > urllen) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem with length of PATH substring"); + } + if (len > 0) { + purl->path = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1)); + if (purl->path == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, + "can't allocate space for PATH"); + } /* cannot malloc path pointer */ + (void)HDstrncpy(purl->path, curstr, (size_t)len); + purl->path[len] = 0; + } + } /* end if PATH element */ + + /************** + * READ QUERY * + **************/ + + if ('?' == *tmpstr) { + tmpstr += 1; + curstr = tmpstr; + while (0 != *tmpstr) { + tmpstr++; + } + len = tmpstr - curstr; + if (len == 0) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "QUERY cannot be empty"); + } else if (len > urllen) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem with length of QUERY substring"); + } + purl->query = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1)); + if (purl->query == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, + "can't allocate space for QUERY"); + } /* cannot malloc path pointer */ + (void)HDstrncpy(purl->query, curstr, (size_t)len); + purl->query[len] = 0; + } /* end if QUERY exists */ + + *_purl = purl; + ret_value = SUCCEED; + +done: + if (ret_value == FAIL) { + H5FD_s3comms_free_purl(purl); + } + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_parse_url() */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_percent_encode_char() + * + * Purpose: + * + * "Percent-encode" utf-8 character `c`, e.g., + * '$' -> "%24" + * '¢' -> "%C2%A2" + * + * `c` cannot be null. + * + * Does not (currently) accept multi-byte characters... + * limit to (?) u+00ff, well below upper bound for two-byte utf-8 encoding + * (u+0080..u+07ff). + * + * Writes output to `repr`. + * `repr` cannot be null. + * Assumes adequate space i `repr`... + * >>> char[4] or [7] for most characters, + * >>> [13] as theoretical maximum. + * + * Representation `repr` is null-terminated. + * + * Stores length of representation (without null terminator) at pointer + * `repr_len`. + * + * Return : SUCCEED/FAIL + * + * - SUCCESS: `SUCCEED` + * - percent-encoded representation written to `repr` + * - 'repr' is null-terminated + * - FAILURE: `FAIL` + * - `c` or `repr` was NULL + * + * Programmer: Jacob Smith + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_percent_encode_char( + char *repr, + const unsigned char c, + size_t *repr_len) +{ + unsigned int i = 0; + int chars_written = 0; + herr_t ret_value = SUCCEED; +#if S3COMMS_DEBUG + unsigned char s[2] = {c, 0}; + unsigned char hex[3] = {0, 0, 0}; +#endif + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_percent_encode_char.\n"); +#endif + + if (repr == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no destination `repr`.\n") + } + +#if S3COMMS_DEBUG + H5FD_s3comms_bytes_to_hex((char *)hex, s, 1, FALSE); + HDfprintf(stdout, " CHAR: \'%s\'\n", s); + HDfprintf(stdout, " CHAR-HEX: \"%s\"\n", hex); +#endif + + if (c <= (unsigned char)0x7f) { + /* character represented in a single "byte" + * and single percent-code + */ +#if S3COMMS_DEBUG + HDfprintf(stdout, " SINGLE-BYTE\n"); +#endif + *repr_len = 3; + chars_written = HDsnprintf(repr, 4, "%%%02X", c); + if (chars_written < 0) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "cannot write char %c", + c); + } + } /* end if single-byte unicode char */ + else { + /* multi-byte, multi-percent representation + */ + unsigned int acc = 0; /* byte accumulator */ + unsigned int k = 0; /* uint character representation */ + unsigned int stack_size = 0; + unsigned char stack[4] = {0, 0, 0, 0}; +#if S3COMMS_DEBUG + HDfprintf(stdout, " MULTI-BYTE\n"); +#endif + stack_size = 0; + k = (unsigned int)c; + *repr_len = 0; + do { + /* push number onto stack in six-bit slices + */ + acc = k; + acc >>= 6; /* cull least */ + acc <<= 6; /* six bits */ + stack[stack_size++] = (unsigned char)(k - acc); + k = acc >> 6; + } while (k > 0); + + /* `stack` now has two to four six-bit 'numbers' to be put into + * UTF-8 byte fields. + */ + +#if S3COMMS_DEBUG + HDfprintf(stdout, " STACK:\n {\n"); + for (i = 0; i < stack_size; i++) { + H5FD_s3comms_bytes_to_hex( + (char *)hex, + (&stack[i]), + 1, + FALSE); + hex[2] = 0; + HDfprintf(stdout, " %s,\n", hex); + } + HDfprintf(stdout, " }\n"); +#endif + + /**************** + * leading byte * + ****************/ + + /* prepend 11[1[1]]0 to first byte */ + /* 110xxxxx, 1110xxxx, or 11110xxx */ + acc = 0xC0; /* 0x11000000 */ + acc += (stack_size > 2) ? 0x20 : 0; /* 0x00100000 */ + acc += (stack_size > 3) ? 0x10 : 0; /* 0x00010000 */ + stack_size--; + chars_written = HDsnprintf( + repr, + 4, + "%%%02X", + (unsigned char)(acc + stack[stack_size])); + if (chars_written < 0) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "cannot write char %c", + c); + } + *repr_len += 3; + + /************************ + * continuation byte(s) * + ************************/ + + /* 10xxxxxx */ + for (i = 0; i < stack_size; i++) { + chars_written = HDsnprintf( + &repr[i*3 + 3], + 4, + "%%%02X", + (unsigned char)(0x80 + stack[stack_size - 1 - i])); + if (chars_written < 0) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "cannot write char %c", + c); + } + *repr_len += 3; + } /* end for each continuation byte */ + } /* end else (multi-byte) */ + + *(repr + *repr_len) = '\0'; + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* H5FD_s3comms_percent_encode_char */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_signing_key() + * + * Purpose: + * + * Create AWS4 "Signing Key" from secret key, AWS region, and timestamp. + * + * Sequentially runs HMAC_SHA256 on strings in specified order, + * generating re-usable checksum (according to documentation, valid for + * 7 days from time given). + * + * `secret` is `access key id` for targeted service/bucket/resource. + * + * `iso8601now` must conform to format, yyyyMMDD'T'hhmmss'Z' + * e.g. "19690720T201740Z". + * + * `region` should be one of AWS service region names, e.g. "us-east-1". + * + * Hard-coded "service" algorithm requirement to "s3". + * + * Inputs must be null-terminated strings. + * + * Writes to `md` the raw byte data, length of `SHA256_DIGEST_LENGTH`. + * Programmer must ensure that `md` is appropriately allocated. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - raw byte data of signing key written to `md` + * - FAILURE: `FAIL` + * - if any input arguments was NULL + * + * Programmer: Jacob Smith + * 2017-07-13 + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_signing_key( + unsigned char *md, + const char *secret, + const char *region, + const char *iso8601now) +{ + char *AWS4_secret = NULL; + size_t AWS4_secret_len = 0; + unsigned char datekey[SHA256_DIGEST_LENGTH]; + unsigned char dateregionkey[SHA256_DIGEST_LENGTH]; + unsigned char dateregionservicekey[SHA256_DIGEST_LENGTH]; + int ret = 0; /* return value of HDsnprintf */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_signing_key.\n"); +#endif + + if (md == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "Destination `md` cannot be NULL.\n") + } + if (secret == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "`secret` cannot be NULL.\n") + } + if (region == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "`region` cannot be NULL.\n") + } + if (iso8601now == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "`iso8601now` cannot be NULL.\n") + } + + AWS4_secret_len = 4 + HDstrlen(secret) + 1; + AWS4_secret = (char*)H5MM_malloc(sizeof(char *) * AWS4_secret_len); + if (AWS4_secret == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "Could not allocate space.\n") + } + + /* prepend "AWS4" to start of the secret key + */ + ret = HDsnprintf(AWS4_secret, AWS4_secret_len,"%s%s", "AWS4", secret); + if ((size_t)ret != (AWS4_secret_len - 1)) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem writing AWS4+secret `%s`", + secret); + } + + /* hash_func, key, len(key), msg, len(msg), digest_dest, digest_len_dest + * we know digest length, so ignore via NULL + */ + HMAC(EVP_sha256(), + (const unsigned char *)AWS4_secret, + (int)HDstrlen(AWS4_secret), + (const unsigned char*)iso8601now, + 8, /* 8 --> length of 8 --> "yyyyMMDD" */ + datekey, + NULL); + HMAC(EVP_sha256(), + (const unsigned char *)datekey, + SHA256_DIGEST_LENGTH, + (const unsigned char *)region, + HDstrlen(region), + dateregionkey, + NULL); + HMAC(EVP_sha256(), + (const unsigned char *)dateregionkey, + SHA256_DIGEST_LENGTH, + (const unsigned char *)"s3", + 2, + dateregionservicekey, + NULL); + HMAC(EVP_sha256(), + (const unsigned char *)dateregionservicekey, + SHA256_DIGEST_LENGTH, + (const unsigned char *)"aws4_request", + 12, + md, + NULL); + +done: + H5MM_xfree(AWS4_secret); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_signing_key() */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_tostringtosign() + * + * Purpose: + * + * Get AWS "String to Sign" from Canonical Request, timestamp, + * and AWS "region". + * + * Common between single request and "chunked upload", + * conforms to: + * "AWS4-HMAC-SHA256\n" + + * <ISO8601 date format> + "\n" + // yyyyMMDD'T'hhmmss'Z' + * <yyyyMMDD> + "/" + <AWS Region> + "/s3/aws4-request\n" + + * hex(SHA256(<CANONICAL-REQUEST>)) + * + * Inputs `creq` (canonical request string), `now` (ISO8601 format), + * and `region` (s3 region designator string) must all be + * null-terminated strings. + * + * Result is written to `dest` with null-terminator. + * It is left to programmer to ensure `dest` has adequate space. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - "string to sign" written to `dest` and null-terminated + * - FAILURE: `FAIL` + * - if any of the inputs are NULL + * - if an error is encountered while computing checksum + * + * Programmer: Jacob Smith + * 2017-07-?? + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_tostringtosign( + char *dest, + const char *req, + const char *now, + const char *region) +{ + unsigned char checksum[SHA256_DIGEST_LENGTH * 2 + 1]; + size_t d = 0; + char day[9]; + char hexsum[SHA256_DIGEST_LENGTH * 2 + 1]; + size_t i = 0; + int ret = 0; /* HDsnprintf return value */ + herr_t ret_value = SUCCEED; + char tmp[128]; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_tostringtosign.\n"); +#endif + + if (dest == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "destination buffer cannot be null.\n") + } + if (req == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "canonical request cannot be null.\n") + } + if (now == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "Timestring cannot be NULL.\n") + } + if (region == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "Region cannot be NULL.\n") + } + + for (i = 0; i < 128; i++) { + tmp[i] = '\0'; + } + for (i = 0; i < SHA256_DIGEST_LENGTH * 2 + 1; i++) { + checksum[i] = '\0'; + hexsum[i] = '\0'; + } + HDstrncpy(day, now, 8); + day[8] = '\0'; + ret = HDsnprintf(tmp, 127, "%s/%s/s3/aws4_request", day, region); + if (ret <= 0 || ret >= 127) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem adding day and region to string") + } + + HDmemcpy((dest + d), "AWS4-HMAC-SHA256\n", 17); + d = 17; + + HDmemcpy((dest+d), now, HDstrlen(now)); + d += HDstrlen(now); + dest[d++] = '\n'; + + HDmemcpy((dest + d), tmp, HDstrlen(tmp)); + d += HDstrlen(tmp); + dest[d++] = '\n'; + + SHA256((const unsigned char *)req, + HDstrlen(req), + checksum); + + if (H5FD_s3comms_bytes_to_hex( + hexsum, + (const unsigned char *)checksum, + SHA256_DIGEST_LENGTH, + true) + == FAIL) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "could not create hex string"); + } + + for (i = 0; i < SHA256_DIGEST_LENGTH * 2; i++) { + dest[d++] = hexsum[i]; + } + + dest[d] = '\0'; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5ros3_tostringtosign() */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_trim() + * + * Purpose: + * + * Remove all whitespace characters from start and end of a string `s` + * of length `s_len`, writing trimmed string copy to `dest`. + * Stores number of characters remaining at `n_written`. + * + * Destination for trimmed copy `dest` cannot be null. + * `dest` must have adequate space allocated for trimmed copy. + * If inadequate space, behavior is undefined, possibly resulting + * in segfault or overwrite of other data. + * + * If `s` is NULL or all whitespace, `dest` is untouched and `n_written` + * is set to 0. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * - `dest == NULL` + * + * Programmer: Jacob Smith + * 2017-09-18 + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_trim(char *dest, + char *s, + size_t s_len, + size_t *n_written) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_trim.\n"); +#endif + + if (dest == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "destination cannot be null.") + } + if (s == NULL) { + s_len = 0; + } + + if (s_len > 0) { + /* Find first non-whitespace character from start; + * reduce total length per character. + */ + while ((s_len > 0) && + isspace((unsigned char)s[0]) && s_len > 0) + { + s++; + s_len--; + } + + /* Find first non-whitespace character from tail; + * reduce length per-character. + * If length is 0 already, there is no non-whitespace character. + */ + if (s_len > 0) { + do { + s_len--; + } while( isspace((unsigned char)s[s_len]) ); + s_len++; + + /* write output into dest + */ + HDmemcpy(dest, s, s_len); + } + } + + *n_written = s_len; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_s3comms_trim() */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_uriencode() + * + * Purpose: + * + * URIencode (percent-encode) every byte except "[a-zA-Z0-9]-._~". + * + * For each character in source string `_s` from `s[0]` to `s[s_len-1]`, + * writes to `dest` either the raw character or its percent-encoded + * equivalent. + * + * See `H5FD_s3comms_bytes_to_hex` for information on percent-encoding. + * + * Space (' ') character encoded as "%20" (not "+") + * + * Forward-slash ('/') encoded as "%2F" only when `encode_slash == true`. + * + * Records number of characters written at `n_written`. + * + * Assumes that `dest` has been allocated with enough space. + * + * Neither `dest` nor `s` can be NULL. + * + * `s_len == 0` will have no effect. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * - source strings `s` or destination `dest` are NULL + * - error while attempting to percent-encode a character + * + * Programmer: Jacob Smith + * 2017-07-?? + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_uriencode( + char *dest, + const char *s, + size_t s_len, + hbool_t encode_slash, + size_t *n_written) +{ + char c = 0; + size_t dest_off = 0; + char hex_buffer[13]; + size_t hex_off = 0; + size_t hex_len = 0; + herr_t ret_value = SUCCEED; + size_t s_off = 0; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "H5FD_s3comms_uriencode called.\n"); +#endif + + if (s == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "source string cannot be NULL"); + } + if (dest == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "destination cannot be NULL"); + } + + /* Write characters to destination, converting to percent-encoded + * "hex-utf-8" strings if necessary. + * e.g., '$' -> "%24" + */ + for (s_off = 0; s_off < s_len; s_off++) { + c = s[s_off]; + if (isalnum(c) || + c == '.' || + c == '-' || + c == '_' || + c == '~' || + (c == '/' && encode_slash == FALSE)) + { + dest[dest_off++] = c; + } + else { + hex_off = 0; + if (H5FD_s3comms_percent_encode_char( + hex_buffer, + (const unsigned char)c, + &hex_len) + == FAIL) + { + hex_buffer[0] = c; + hex_buffer[1] = 0; + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to percent-encode character \'%s\' " + "at %d in \"%s\"", hex_buffer, (int)s_off, s); + } + + for (hex_off = 0; hex_off < hex_len; hex_off++) { + dest[dest_off++] = hex_buffer[hex_off]; + } + } /* end else (not a regular character) */ + } /* end for each character */ + + if (dest_off < s_len) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "buffer overflow"); + + *n_written = dest_off; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD_s3comms_uriencode */ + +#endif /* H5_HAVE_ROS3_VFD */ + diff --git a/src/H5FDs3comms.h b/src/H5FDs3comms.h new file mode 100644 index 0000000..90c6650 --- /dev/null +++ b/src/H5FDs3comms.h @@ -0,0 +1,604 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/***************************************************************************** + * Read-Only S3 Virtual File Driver (VFD) + * + * This is the header for the S3 Communications module + * + * ***NOT A FILE DRIVER*** + * + * Purpose: + * + * - Provide structures and functions related to communicating with + * Amazon S3 (Simple Storage Service). + * - Abstract away the REST API (HTTP, + * networked communications) behind a series of uniform function calls. + * - Handle AWS4 authentication, if appropriate. + * - Fail predictably in event of errors. + * - Eventually, support more S3 operations, such as creating, writing to, + * and removing Objects remotely. + * + * translates: + * `read(some_file, bytes_offset, bytes_length, &dest_buffer);` + * to: + * ``` + * GET myfile HTTP/1.1 + * Host: somewhere.me + * Range: bytes=4096-5115 + * ``` + * and places received bytes from HTTP response... + * ``` + * HTTP/1.1 206 Partial-Content + * Content-Range: 4096-5115/63239 + * + * <bytes> + * ``` + * ...in destination buffer. + * + * TODO: put documentation in a consistent place and point to it from here. + * + * Programmer: Jacob Smith + * 2017-11-30 + * + *****************************************************************************/ + +#include "H5private.h" /* Generic Functions */ + +#ifdef H5_HAVE_ROS3_VFD + +/* Necessary S3 headers */ +#include <curl/curl.h> +#include <openssl/evp.h> +#include <openssl/hmac.h> +#include <openssl/sha.h> + +/***************** + * PUBLIC MACROS * + *****************/ + +/* hexadecimal string of pre-computed sha256 checksum of the empty string + * hex(sha256sum("")) + */ +#define EMPTY_SHA256 \ +"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + +/* string length (plus null terminator) + * example ISO8601-format string: "20170713T145903Z" (YYYYmmdd'T'HHMMSS'_') + */ +#define ISO8601_SIZE 17 + +/* string length (plus null terminator) + * example RFC7231-format string: "Fri, 30 Jun 2017 20:41:55 GMT" + */ +#define RFC7231_SIZE 30 + +/*--------------------------------------------------------------------------- + * + * Macro: ISO8601NOW() + * + * Purpose: + * + * write "YYYYmmdd'T'HHMMSS'Z'" (less single-quotes) to dest + * e.g., "20170630T204155Z" + * + * wrapper for strftime() + * + * It is left to the programmer to check return value of + * ISO8601NOW (should equal ISO8601_SIZE - 1). + * + *--------------------------------------------------------------------------- + */ +#define ISO8601NOW(dest, now_gm) \ +strftime((dest), ISO8601_SIZE, "%Y%m%dT%H%M%SZ", (now_gm)) + +/*--------------------------------------------------------------------------- + * + * Macro: RFC7231NOW() + * + * Purpose: + * + * write "Day, dd Mmm YYYY HH:MM:SS GMT" to dest + * e.g., "Fri, 30 Jun 2017 20:41:55 GMT" + * + * wrapper for strftime() + * + * It is left to the programmer to check return value of + * RFC7231NOW (should equal RFC7231_SIZE - 1). + * + *--------------------------------------------------------------------------- + */ +#define RFC7231NOW(dest, now_gm) \ +strftime((dest), RFC7231_SIZE, "%a, %d %b %Y %H:%M:%S GMT", (now_gm)) + + +/* Reasonable maximum length of a credential string. + * Provided for error-checking S3COMMS_FORMAT_CREDENTIAL (below). + * 17 <- "////aws4_request\0" + * 2 < "s3" (service) + * 8 <- "YYYYmmdd" (date) + * 128 <- (access_id) + * 155 :: sum + */ +#define S3COMMS_MAX_CREDENTIAL_SIZE 155 + + +/*--------------------------------------------------------------------------- + * + * Macro: H5FD_S3COMMS_FORMAT_CREDENTIAL() + * + * Purpose: + * + * Format "S3 Credential" string from inputs, for AWS4. + * + * Wrapper for HDsnprintf(). + * + * _HAS NO ERROR-CHECKING FACILITIES_ + * It is left to programmer to ensure that return value confers success. + * e.g., + * ``` + * assert( S3COMMS_MAX_CREDENTIAL_SIZE >= + * S3COMMS_FORMAT_CREDENTIAL(...) ); + * ``` + * + * "<access-id>/<date>/<aws-region>/<aws-service>/aws4_request" + * assuming that `dest` has adequate space. + * + * ALL inputs must be null-terminated strings. + * + * `access` should be the user's access key ID. + * `date` must be of format "YYYYmmdd". + * `region` should be relevant AWS region, i.e. "us-east-1". + * `service` should be "s3". + * + *--------------------------------------------------------------------------- + */ +#define S3COMMS_FORMAT_CREDENTIAL(dest, access, iso8601_date, region, service) \ +HDsnprintf((dest), S3COMMS_MAX_CREDENTIAL_SIZE, \ + "%s/%s/%s/%s/aws4_request", \ + (access), (iso8601_date), (region), (service)) + +/********************* + * PUBLIC STRUCTURES * + *********************/ + + +/*---------------------------------------------------------------------------- + * + * Structure: hrb_node_t + * + * HTTP Header Field Node + * + * + * + * Maintain a ordered (linked) list of HTTP Header fields. + * + * Provides efficient access and manipulation of a logical sequence of + * HTTP header fields, of particular use when composing an + * "S3 Canonical Request" for authentication. + * + * - The creation of a Canoncial Request involves: + * - convert field names to lower case + * - sort by this lower-case name + * - convert ": " name-value separator in HTTP string to ":" + * - get sorted lowercase names without field or separator + * + * As HTTP headers allow headers in any order (excepting the case of multiple + * headers with the same name), the list ordering can be optimized for Canonical + * Request creation, suggesting alphabtical order. For more expedient insertion + * and removal of elements in the list, linked list seems preferable to a + * dynamically-expanding array. The usually-smaller number of entries (5 or + * fewer) makes performance overhead of traversing the list trivial. + * + * The above requirements of creating at Canonical Request suggests a reasonable + * trade-off of speed for space with the option to compute elements as needed + * or to have the various elements prepared and stored in the structure + * (e.g. name, value, lowername, concatenated name:value) + * The structure currently is implemented to pre-compute. + * + * At all times, the "first" node of the list should be the least, + * alphabetically. For all nodes, the `next` node should be either NULL or + * of greater alphabetical value. + * + * Each node contains its own header field information, plus a pointer to the + * next node. + * + * It is not allowed to have multiple nodes with the same _lowercase_ `name`s + * in the same list + * (i.e., name is case-insensitive for access and modification.) + * + * All data (`name`, `value`, `lowername`, and `cat`) are null-terminated + * strings allocated specifically for their node. + * + * + * + * `magic` (unsigned long) + * + * "unique" idenfier number for the structure type + * + * `name` (char *) + * + * Case-meaningful name of the HTTP field. + * Given case is how it is supplied to networking code. + * e.g., "Range" + * + * `lowername` (char *) + * + * Lowercase copy of name. + * e.g., "range" + * + * `value` (char *) + * + * Case-meaningful value of HTTP field. + * e.g., "bytes=0-9" + * + * `cat` (char *) + * + * Concatenated, null-terminated string of HTTP header line, + * as the field would appear in an HTTP request. + * e.g., "Range: bytes=0-9" + * + * `next` (hrb_node_t *) + * + * Pointers to next node in the list, or NULL sentinel as end of list. + * Next node must have a greater `lowername` as determined by strcmp(). + * + *---------------------------------------------------------------------------- + */ +typedef struct hrb_node_t { + unsigned long magic; + char *name; + char *value; + char *cat; + char *lowername; + struct hrb_node_t *next; +} hrb_node_t; +#define S3COMMS_HRB_NODE_MAGIC 0x7F5757UL + + +/*---------------------------------------------------------------------------- + * + * Structure: hrb_t + * + * HTTP Request Buffer structure + * + * + * + * Logically represent an HTTP request + * + * GET /myplace/myfile.h5 HTTP/1.1 + * Host: over.rainbow.oz + * Date: Fri, 01 Dec 2017 12:35:04 CST + * + * <body> + * + * ...with fast, efficient access to and modification of primary and field + * elements. + * + * Structure for building HTTP requests while hiding much of the string + * processing required "under the hood." + * + * Information about the request target -- the first line -- and the body text, + * if any, are managed directly with this structure. All header fields, e.g., + * "Host" and "Date" above, are created with a linked list of `hrb_node_t` and + * included in the request by a pointer to the head of the list. + * + * + * + * `magic` (unsigned long) + * + * "Magic" number confirming that this is an hrb_t structure and + * what operations are valid for it. + * + * Must be S3COMMS_HRB_MAGIC to be valid. + * + * `body` (char *) : + * + * Pointer to start of HTTP body. + * + * Can be NULL, in which case it is treated as the empty string, "". + * + * `body_len` (size_t) : + * + * Number of bytes (characters) in `body`. 0 if empty or NULL `body`. + * + * `first_header` (hrb_node_t *) : + * + * Pointer to first SORTED header node, if any. + * It is left to the programmer to ensure that this node and associated + * list is destroyed when done. + * + * `resource` (char *) : + * + * Pointer to resource URL string, e.g., "/folder/page.xhtml". + * + * `verb` (char *) : + * + * Pointer to HTTP verb string, e.g., "GET". + * + * `version` (char *) : + * + * Pointer to HTTP version string, e.g., "HTTP/1.1". + * + *---------------------------------------------------------------------------- + */ +typedef struct { + unsigned long magic; + char *body; + size_t body_len; + hrb_node_t *first_header; + char *resource; + char *verb; + char *version; +} hrb_t; +#define S3COMMS_HRB_MAGIC 0x6DCC84UL + + +/*---------------------------------------------------------------------------- + * + * Structure: parsed_url_t + * + * + * Represent a URL with easily-accessed pointers to logical elements within. + * These elements (components) are stored as null-terminated strings (or just + * NULLs). These components should be allocated for the structure, making the + * data as safe as possible from modification. If a component is NULL, it is + * either implicit in or absent from the URL. + * + * "http://mybucket.s3.amazonaws.com:8080/somefile.h5?param=value&arg=value" + * ^--^ ^-----------------------^ ^--^ ^---------^ ^-------------------^ + * Scheme Host Port Resource Query/-ies + * + * + * + * `magic` (unsigned long) + * + * Structure identification and validation identifier. + * Identifies as `parsed_url_t` type. + * + * `scheme` (char *) + * + * String representing which protocol is to be expected. + * _Must_ be present. + * "http", "https", "ftp", e.g. + * + * `host` (char *) + * + * String of host, either domain name, IPv4, or IPv6 format. + * _Must_ be present. + * "over.rainbow.oz", "192.168.0.1", "[0000:0000:0000:0001]" + * + * `port` (char *) + * + * String representation of specified port. Must resolve to a valid unsigned + * integer. + * "9000", "80" + * + * `path` (char *) + * + * Path to resource on host. If not specified, assumes root "/". + * "lollipop_guild.wav", "characters/witches/white.dat" + * + * `query` (char *) + * + * Single string of all query parameters in url (if any). + * "arg1=value1&arg2=value2" + * + *---------------------------------------------------------------------------- + */ +typedef struct { + unsigned long magic; + char *scheme; /* required */ + char *host; /* required */ + char *port; + char *path; + char *query; +} parsed_url_t; +#define S3COMMS_PARSED_URL_MAGIC 0x21D0DFUL + + +/*---------------------------------------------------------------------------- + * + * Structure: s3r_t + * + * + * + * S3 request structure "handle". + * + * Holds persistent information for Amazon S3 requests. + * + * Instantiated through `H5FD_s3comms_s3r_open()`, copies data into self. + * + * Intended to be re-used for operations on a remote object. + * + * Cleaned up through `H5FD_s3comms_s3r_close()`. + * + * _DO NOT_ share handle between threads: curl easy handle `curlhandle` has + * undefined behavior if called to perform in multiple threads. + * + * + * + * `magic` (unsigned long) + * + * "magic" number identifying this structure as unique type. + * MUST equal `S3R_MAGIC` to be valid. + * + * `curlhandle` (CURL) + * + * Pointer to the curl_easy handle generated for the request. + * + * `httpverb` (char *) + * + * Pointer to NULL-terminated string. HTTP verb, + * e.g. "GET", "HEAD", "PUT", etc. + * + * Default is NULL, resulting in a "GET" request. + * + * `purl` (parsed_url_t *) + * + * Pointer to structure holding the elements of URL for file open. + * + * e.g., "http://bucket.aws.com:8080/myfile.dat?q1=v1&q2=v2" + * parsed into... + * { scheme: "http" + * host: "bucket.aws.com" + * port: "8080" + * path: "myfile.dat" + * query: "q1=v1&q2=v2" + * } + * + * Cannot be NULL. + * + * `region` (char *) + * + * Pointer to NULL-terminated string, specifying S3 "region", + * e.g., "us-east-1". + * + * Required to authenticate. + * + * `secret_id` (char *) + * + * Pointer to NULL-terminated string for "secret" access id to S3 resource. + * + * Requred to authenticate. + * + * `signing_key` (unsigned char *) + * + * Pointer to `SHA256_DIGEST_LENGTH`-long string for "re-usable" signing + * key, generated via + * `HMAC-SHA256(HMAC-SHA256(HMAC-SHA256(HMAC-SHA256("AWS4<secret_key>", + * "<yyyyMMDD"), "<aws-region>"), "<aws-service>"), "aws4_request")` + * which may be re-used for several (up to seven (7)) days from creation? + * Computed once upon file open. + * + * Requred to authenticate. + * + *---------------------------------------------------------------------------- + */ +typedef struct { + unsigned long magic; + CURL *curlhandle; + size_t filesize; + char *httpverb; + parsed_url_t *purl; + char *region; + char *secret_id; + unsigned char *signing_key; +} s3r_t; + +#define S3COMMS_S3R_MAGIC 0x44d8d79 + +#ifdef __cplusplus +extern "C" { +#endif + +/******************************************* + * DECLARATION OF HTTP FIELD LIST ROUTINES * + *******************************************/ + +H5_DLL herr_t H5FD_s3comms_hrb_node_set(hrb_node_t **L, + const char *name, + const char *value); + +/*********************************************** + * DECLARATION OF HTTP REQUEST BUFFER ROUTINES * + ***********************************************/ + +H5_DLL herr_t H5FD_s3comms_hrb_destroy(hrb_t **buf); + +H5_DLL hrb_t * H5FD_s3comms_hrb_init_request(const char *verb, + const char *resource, + const char *host); + +/************************************* + * DECLARATION OF S3REQUEST ROUTINES * + *************************************/ + +H5_DLL herr_t H5FD_s3comms_s3r_close(s3r_t *handle); + +H5_DLL size_t H5FD_s3comms_s3r_get_filesize(s3r_t *handle); + +H5_DLL s3r_t * H5FD_s3comms_s3r_open(const char url[], + const char region[], + const char id[], + const unsigned char signing_key[]); + +H5_DLL herr_t H5FD_s3comms_s3r_read(s3r_t *handle, + haddr_t offset, + size_t len, + void *dest); + +/********************************* + * DECLARATION OF OTHER ROUTINES * + *********************************/ + +H5_DLL struct tm * gmnow(void); + +H5_DLL herr_t H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, + int cr_size, + char *signed_headers_dest, + int sh_size, + hrb_t *http_request); + +H5_DLL herr_t H5FD_s3comms_bytes_to_hex(char *dest, + const unsigned char *msg, + size_t msg_len, + hbool_t lowercase); + +H5_DLL herr_t H5FD_s3comms_free_purl(parsed_url_t *purl); + +H5_DLL herr_t H5FD_s3comms_HMAC_SHA256(const unsigned char *key, + size_t key_len, + const char *msg, + size_t msg_len, + char *dest); + +H5_DLL herr_t H5FD_s3comms_load_aws_profile(const char *name, + char *key_id_out, + char *secret_access_key_out, + char *aws_region_out); + +H5_DLL herr_t H5FD_s3comms_nlowercase(char *dest, + const char *s, + size_t len); + +H5_DLL herr_t H5FD_s3comms_parse_url(const char *str, + parsed_url_t **purl); + +H5_DLL herr_t H5FD_s3comms_percent_encode_char(char *repr, + const unsigned char c, + size_t *repr_len); + +H5_DLL herr_t H5FD_s3comms_signing_key(unsigned char *md, + const char *secret, + const char *region, + const char *iso8601now); + +H5_DLL herr_t H5FD_s3comms_tostringtosign(char *dest, + const char *req_str, + const char *now, + const char *region); + +H5_DLL herr_t H5FD_s3comms_trim(char *dest, + char *s, + size_t s_len, + size_t *n_written); + +H5_DLL herr_t H5FD_s3comms_uriencode(char *dest, const char *s, size_t s_len, + hbool_t encode_slash, size_t *n_written); + +#ifdef __cplusplus +} +#endif + +#endif /* H5_HAVE_ROS3_VFD */ + diff --git a/src/H5FSsection.c b/src/H5FSsection.c index 9f86aea..df67bd9 100644 --- a/src/H5FSsection.c +++ b/src/H5FSsection.c @@ -390,7 +390,7 @@ HDfprintf(stderr, "%s: fspace->alloc_sect_size = %Hu, fspace->sect_size = %Hu\n" cache_flags |= H5AC__DIRTIED_FLAG; /* On file close or flushing, does not allow section info to shrink in size */ - if(f->closing || flush_in_progress) { + if(f->shared->closing || flush_in_progress) { if(fspace->sect_size > fspace->alloc_sect_size) cache_flags |= H5AC__DELETED_FLAG | H5AC__TAKE_OWNERSHIP_FLAG; else @@ -441,7 +441,7 @@ HDfprintf(stderr, "%s: Relinquishing section info ownership\n", FUNC); /* Set flag to release section info space in file */ /* On file close or flushing, only need to release section info with size bigger than previous section */ - if(f->closing || flush_in_progress) { + if(f->shared->closing || flush_in_progress) { if(fspace->sect_size > fspace->alloc_sect_size) release_sinfo_space = TRUE; else diff --git a/src/H5Faccum.c b/src/H5Faccum.c index 8d7852b..74a170b 100644 --- a/src/H5Faccum.c +++ b/src/H5Faccum.c @@ -111,7 +111,7 @@ H5FL_BLK_DEFINE_STATIC(meta_accum); *------------------------------------------------------------------------- */ herr_t -H5F__accum_read(H5F_t *f, H5FD_mem_t map_type, haddr_t addr, +H5F__accum_read(H5F_shared_t *f_sh, H5FD_mem_t map_type, haddr_t addr, size_t size, void *buf/*out*/) { H5FD_t *file; /* File driver pointer */ @@ -120,18 +120,18 @@ H5F__accum_read(H5F_t *f, H5FD_mem_t map_type, haddr_t addr, FUNC_ENTER_PACKAGE /* Sanity checks */ - HDassert(f); + HDassert(f_sh); HDassert(buf); /* Translate to file driver I/O info object */ - file = f->shared->lf; + file = f_sh->lf; /* Check if this information is in the metadata accumulator */ - if((f->shared->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) && map_type != H5FD_MEM_DRAW) { + if((f_sh->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) && map_type != H5FD_MEM_DRAW) { H5F_meta_accum_t *accum; /* Alias for file's metadata accumulator */ /* Set up alias for file's metadata accumulator info */ - accum = &f->shared->accum; + accum = &f_sh->accum; if(size < H5F_ACCUM_MAX_SIZE) { /* Sanity check */ @@ -419,7 +419,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F__accum_write(H5F_t *f, H5FD_mem_t map_type, haddr_t addr, +H5F__accum_write(H5F_shared_t *f_sh, H5FD_mem_t map_type, haddr_t addr, size_t size, const void *buf) { H5FD_t *file; /* File driver pointer */ @@ -428,19 +428,19 @@ H5F__accum_write(H5F_t *f, H5FD_mem_t map_type, haddr_t addr, FUNC_ENTER_NOAPI(FAIL) /* Sanity checks */ - HDassert(f); - HDassert(H5F_INTENT(f) & H5F_ACC_RDWR); + HDassert(f_sh); + HDassert(H5F_SHARED_INTENT(f_sh) & H5F_ACC_RDWR); HDassert(buf); /* Translate to file driver pointer */ - file = f->shared->lf; + file = f_sh->lf; /* Check for accumulating metadata */ - if((f->shared->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) && map_type != H5FD_MEM_DRAW) { + if((f_sh->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) && map_type != H5FD_MEM_DRAW) { H5F_meta_accum_t *accum; /* Alias for file's metadata accumulator */ /* Set up alias for file's metadata accumulator info */ - accum = &f->shared->accum; + accum = &f_sh->accum; if(size < H5F_ACCUM_MAX_SIZE) { /* Sanity check */ @@ -732,9 +732,9 @@ H5F__accum_write(H5F_t *f, H5FD_mem_t map_type, haddr_t addr, } /* end if */ else { /* Make certain that data in accumulator is visible before new write */ - if((H5F_INTENT(f) & H5F_ACC_SWMR_WRITE) > 0) + if((H5F_SHARED_INTENT(f_sh) & H5F_ACC_SWMR_WRITE) > 0) /* Flush if dirty and reset accumulator */ - if(H5F__accum_reset(f, TRUE) < 0) + if(H5F__accum_reset(f_sh, TRUE) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTRESET, FAIL, "can't reset accumulator") /* Write the data */ @@ -783,7 +783,7 @@ H5F__accum_write(H5F_t *f, H5FD_mem_t map_type, haddr_t addr, } /* end if */ else { /* Access covers whole accumulator */ /* Reset accumulator, but don't flush */ - if(H5F__accum_reset(f, FALSE) < 0) + if(H5F__accum_reset(f_sh, FALSE) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTRESET, FAIL, "can't reset accumulator") } /* end else */ } /* end if */ @@ -847,7 +847,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F__accum_free(H5F_t *f, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr, +H5F__accum_free(H5F_shared_t *f_sh, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr, hsize_t size) { H5F_meta_accum_t *accum; /* Alias for file's metadata accumulator */ @@ -857,16 +857,16 @@ H5F__accum_free(H5F_t *f, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr, FUNC_ENTER_PACKAGE /* check arguments */ - HDassert(f); + HDassert(f_sh); /* Set up alias for file's metadata accumulator info */ - accum = &f->shared->accum; + accum = &f_sh->accum; /* Translate to file driver pointer */ - file = f->shared->lf; + file = f_sh->lf; /* Adjust the metadata accumulator to remove the freed block, if it overlaps */ - if((f->shared->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) + if((f_sh->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) && H5F_addr_overlap(addr, size, accum->loc, accum->size)) { size_t overlap_size; /* Size of overlap with accumulator */ @@ -1013,28 +1013,28 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F__accum_flush(H5F_t *f) +H5F__accum_flush(H5F_shared_t *f_sh) { herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) /* Sanity checks */ - HDassert(f); + HDassert(f_sh); /* Check if we need to flush out the metadata accumulator */ - if((f->shared->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) && f->shared->accum.dirty) { + if((f_sh->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) && f_sh->accum.dirty) { H5FD_t *file; /* File driver pointer */ /* Translate to file driver pointer */ - file = f->shared->lf; + file = f_sh->lf; /* Flush the metadata contents */ - if(H5FD_write(file, H5FD_MEM_DEFAULT, f->shared->accum.loc + f->shared->accum.dirty_off, f->shared->accum.dirty_len, f->shared->accum.buf + f->shared->accum.dirty_off) < 0) + if(H5FD_write(file, H5FD_MEM_DEFAULT, f_sh->accum.loc + f_sh->accum.dirty_off, f_sh->accum.dirty_len, f_sh->accum.buf + f_sh->accum.dirty_off) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") /* Reset the dirty flag */ - f->shared->accum.dirty = FALSE; + f_sh->accum.dirty = FALSE; } /* end if */ done: @@ -1056,34 +1056,31 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F__accum_reset(H5F_t *f, hbool_t flush) +H5F__accum_reset(H5F_shared_t *f_sh, hbool_t flush) { herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE /* Sanity checks */ - HDassert(f); + HDassert(f_sh); /* Flush any dirty data in accumulator, if requested */ if(flush) - if(H5F__accum_flush(f) < 0) + if(H5F__accum_flush(f_sh) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "can't flush metadata accumulator") /* Check if we need to reset the metadata accumulator information */ - if(f->shared->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) { - /* Sanity check */ - HDassert(!f->closing || FALSE == f->shared->accum.dirty); - + if(f_sh->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) { /* Free the buffer */ - if(f->shared->accum.buf) - f->shared->accum.buf = H5FL_BLK_FREE(meta_accum, f->shared->accum.buf); + if(f_sh->accum.buf) + f_sh->accum.buf = H5FL_BLK_FREE(meta_accum, f_sh->accum.buf); /* Reset the buffer sizes & location */ - f->shared->accum.alloc_size = f->shared->accum.size = 0; - f->shared->accum.loc = HADDR_UNDEF; - f->shared->accum.dirty = FALSE; - f->shared->accum.dirty_len = 0; + f_sh->accum.alloc_size = f_sh->accum.size = 0; + f_sh->accum.loc = HADDR_UNDEF; + f_sh->accum.dirty = FALSE; + f_sh->accum.dirty_len = 0; } /* end if */ done: diff --git a/src/H5Fcwfs.c b/src/H5Fcwfs.c index aa5f8ab..26452b6 100644 --- a/src/H5Fcwfs.c +++ b/src/H5Fcwfs.c @@ -299,7 +299,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F_cwfs_remove_heap(H5F_file_t *shared, H5HG_heap_t *heap) +H5F_cwfs_remove_heap(H5F_shared_t *shared, H5HG_heap_t *heap) { unsigned u; /* Local index variable */ herr_t ret_value = SUCCEED; /* Return value */ diff --git a/src/H5Fefc.c b/src/H5Fefc.c index f3881d4..66d68b2 100644 --- a/src/H5Fefc.c +++ b/src/H5Fefc.c @@ -60,14 +60,14 @@ struct H5F_efc_t { unsigned max_nfiles; /* Maximum size of the external file cache */ unsigned nrefs; /* Number of times this file appears in another file's EFC */ int tag; /* Temporary variable used by H5F__efc_try_close() */ - H5F_file_t *tmp_next; /* Next file in temporary list used by H5F__efc_try_close() */ + H5F_shared_t *tmp_next; /* Next file in temporary list used by H5F__efc_try_close() */ }; /* Private prototypes */ static herr_t H5F__efc_release_real(H5F_efc_t *efc); static herr_t H5F__efc_remove_ent(H5F_efc_t *efc, H5F_efc_ent_t *ent); -static void H5F__efc_try_close_tag1(H5F_file_t *sf, H5F_file_t **tail); -static void H5F__efc_try_close_tag2(H5F_file_t *sf, H5F_file_t **tail); +static void H5F__efc_try_close_tag1(H5F_shared_t *sf, H5F_shared_t **tail); +static void H5F__efc_try_close_tag2(H5F_shared_t *sf, H5F_shared_t **tail); /* Free lists */ H5FL_DEFINE_STATIC(H5F_efc_ent_t); @@ -644,10 +644,10 @@ done: *------------------------------------------------------------------------- */ static void -H5F__efc_try_close_tag1(H5F_file_t *sf, H5F_file_t **tail) +H5F__efc_try_close_tag1(H5F_shared_t *sf, H5F_shared_t **tail) { H5F_efc_ent_t *ent = NULL; /* EFC entry */ - H5F_file_t *esf; /* Convenience pointer to ent->file->shared */ + H5F_shared_t *esf; /* Convenience pointer to ent->file->shared */ FUNC_ENTER_STATIC_NOERR @@ -718,10 +718,10 @@ H5F__efc_try_close_tag1(H5F_file_t *sf, H5F_file_t **tail) *------------------------------------------------------------------------- */ static void -H5F__efc_try_close_tag2(H5F_file_t *sf, H5F_file_t **tail) +H5F__efc_try_close_tag2(H5F_shared_t *sf, H5F_shared_t **tail) { H5F_efc_ent_t *ent = NULL; /* EFC entry */ - H5F_file_t *esf; /* Convenience pointer to ent->file->shared */ + H5F_shared_t *esf; /* Convenience pointer to ent->file->shared */ FUNC_ENTER_STATIC_NOERR @@ -827,11 +827,11 @@ H5F__efc_try_close_tag2(H5F_file_t *sf, H5F_file_t **tail) herr_t H5F__efc_try_close(H5F_t *f) { - H5F_file_t *tail; /* Tail of linked list of found files. Head will be f->shared. */ - H5F_file_t *uncloseable_head = NULL; /* Head of linked list of files found to be uncloseable by the first pass */ - H5F_file_t *uncloseable_tail = NULL; /* Tail of linked list of files found to be uncloseable by the first pass */ - H5F_file_t *sf; /* Temporary file pointer */ - H5F_file_t *next; /* Temporary file pointer */ + H5F_shared_t *tail; /* Tail of linked list of found files. Head will be f->shared. */ + H5F_shared_t *uncloseable_head = NULL; /* Head of linked list of files found to be uncloseable by the first pass */ + H5F_shared_t *uncloseable_tail = NULL; /* Tail of linked list of files found to be uncloseable by the first pass */ + H5F_shared_t *sf; /* Temporary file pointer */ + H5F_shared_t *next; /* Temporary file pointer */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE diff --git a/src/H5Ffake.c b/src/H5Ffake.c index 6072f2e..67bd180 100644 --- a/src/H5Ffake.c +++ b/src/H5Ffake.c @@ -50,7 +50,7 @@ H5F_fake_alloc(uint8_t sizeof_size) /* Allocate faked file struct */ if(NULL == (f = H5FL_CALLOC(H5F_t))) HGOTO_ERROR(H5E_FILE, H5E_NOSPACE, NULL, "can't allocate top file structure") - if(NULL == (f->shared = H5FL_CALLOC(H5F_file_t))) + if(NULL == (f->shared = H5FL_CALLOC(H5F_shared_t))) HGOTO_ERROR(H5E_FILE, H5E_NOSPACE, NULL, "can't allocate shared file structure") /* Only set fields necessary for clients */ @@ -93,7 +93,7 @@ H5F_fake_free(H5F_t *f) if(f) { /* Destroy shared file struct */ if(f->shared) - f->shared = H5FL_FREE(H5F_file_t, f->shared); + f->shared = H5FL_FREE(H5F_shared_t, f->shared); f = H5FL_FREE(H5F_t, f); } /* end if */ diff --git a/src/H5Fint.c b/src/H5Fint.c index 758900b..5e2cf26 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -58,7 +58,7 @@ typedef struct H5F_olist_t { struct { hbool_t local; /* Set flag for "local" file searches */ union { - H5F_file_t *shared; /* Pointer to shared file to look inside */ + H5F_shared_t *shared; /* Pointer to shared file to look inside */ const H5F_t *file; /* Pointer to file to look inside */ } ptr; } file_info; @@ -81,7 +81,7 @@ static herr_t H5F__get_objects(const H5F_t *f, unsigned types, size_t max_index, static int H5F__get_objects_cb(void *obj_ptr, hid_t obj_id, void *key); static herr_t H5F__build_name(const char *prefix, const char *file_name, char **full_name/*out*/); static char *H5F__getenv_prefix_name(char **env_prefix/*in,out*/); -static H5F_t *H5F__new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t *lf); +static H5F_t *H5F__new(H5F_shared_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t *lf); static herr_t H5F__build_actual_name(const H5F_t *f, const H5P_genplist_t *fapl, const char *name, char ** /*out*/ actual_name); static herr_t H5F__flush_phase1(H5F_t *f); static herr_t H5F__flush_phase2(H5F_t *f, hbool_t closing); @@ -104,8 +104,8 @@ static herr_t H5F__flush_phase2(H5F_t *f, hbool_t closing); /* Declare a free list to manage the H5F_t struct */ H5FL_DEFINE(H5F_t); -/* Declare a free list to manage the H5F_file_t struct */ -H5FL_DEFINE(H5F_file_t); +/* Declare a free list to manage the H5F_shared_t struct */ +H5FL_DEFINE(H5F_shared_t); @@ -246,9 +246,9 @@ H5F_get_access_plist(H5F_t *f, hbool_t app_ref) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, H5I_INVALID_HID, "can't set minimum raw data fraction of page buffer") } /* end if */ #ifdef H5_HAVE_PARALLEL - if(H5P_set(new_plist, H5_COLL_MD_READ_FLAG_NAME, &(f->coll_md_read)) < 0) + if(H5P_set(new_plist, H5_COLL_MD_READ_FLAG_NAME, &(f->shared->coll_md_read)) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, H5I_INVALID_HID, "can't set collective metadata read flag") - if(H5P_set(new_plist, H5F_ACS_COLL_MD_WRITE_FLAG_NAME, &(f->coll_md_write)) < 0) + if(H5P_set(new_plist, H5F_ACS_COLL_MD_WRITE_FLAG_NAME, &(f->shared->coll_md_write)) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, H5I_INVALID_HID, "can't set collective metadata read flag") #endif /* H5_HAVE_PARALLEL */ if(H5P_set(new_plist, H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_NAME, &(f->shared->mdc_initCacheImageCfg)) < 0) @@ -498,6 +498,9 @@ H5F__get_objects_cb(void *obj_ptr, hid_t obj_id, void *key) oloc = NULL; break; + case H5I_MAP: + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5_ITER_ERROR, "maps not supported in native VOL connector") + case H5I_UNINIT: case H5I_BADID: case H5I_FILE: @@ -868,7 +871,7 @@ H5F__is_hdf5(const char *name, hid_t fapl_id) /* The file is an hdf5 file if the hdf5 file signature can be found */ if(H5FD_locate_signature(file, &sig_addr) < 0) - HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "unable to locate file signature") + HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "error while trying to locate file signature") ret_value = (HADDR_UNDEF != sig_addr); done: @@ -898,7 +901,7 @@ done: *------------------------------------------------------------------------- */ static H5F_t * -H5F__new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t *lf) +H5F__new(H5F_shared_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t *lf) { H5F_t *f = NULL; H5F_t *ret_value = NULL; @@ -919,7 +922,7 @@ H5F__new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_ size_t u; /* Local index variable */ HDassert(lf != NULL); - if(NULL == (f->shared = H5FL_CALLOC(H5F_file_t))) + if(NULL == (f->shared = H5FL_CALLOC(H5F_shared_t))) HGOTO_ERROR(H5E_FILE, H5E_NOSPACE, NULL, "can't allocate shared file structure") f->shared->flags = flags; @@ -1017,9 +1020,9 @@ H5F__new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_ if(NULL == (f->shared->efc = H5F__efc_create(efc_size))) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "can't create external file cache") #ifdef H5_HAVE_PARALLEL - if(H5P_get(plist, H5_COLL_MD_READ_FLAG_NAME, &(f->coll_md_read)) < 0) + if(H5P_get(plist, H5_COLL_MD_READ_FLAG_NAME, &(f->shared->coll_md_read)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get collective metadata read flag") - if(H5P_get(plist, H5F_ACS_COLL_MD_WRITE_FLAG_NAME, &(f->coll_md_write)) < 0) + if(H5P_get(plist, H5F_ACS_COLL_MD_WRITE_FLAG_NAME, &(f->shared->coll_md_write)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get collective metadata write flag") #endif /* H5_HAVE_PARALLEL */ if(H5P_get(plist, H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_NAME, &(f->shared->mdc_initCacheImageCfg)) < 0) @@ -1038,7 +1041,7 @@ H5F__new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_ if(H5FD_get_fs_type_map(lf, f->shared->fs_type_map) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get free space type mapping from VFD") - if(H5MF_init_merge_flags(f) < 0) + if(H5MF_init_merge_flags(f->shared) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "problem initializing free space merge flags") f->shared->tmp_addr = f->shared->maxaddr; /* Disable temp. space allocation for parallel I/O (for now) */ @@ -1138,7 +1141,7 @@ done: if(H5I_dec_ref(f->shared->fcpl_id) < 0) HDONE_ERROR(H5E_FILE, H5E_CANTDEC, NULL, "can't close property list") - f->shared = H5FL_FREE(H5F_file_t, f->shared); + f->shared = H5FL_FREE(H5F_shared_t, f->shared); } f = H5FL_FREE(H5F_t, f); } @@ -1172,6 +1175,9 @@ H5F__dest(H5F_t *f, hbool_t flush) if(1 == f->shared->nrefs) { int actype; /* metadata cache type (enum value) */ + /* Mark this file as closing */ + f->shared->closing = TRUE; + /* Flush at this point since the file will be closed (phase 1). * Only try to flush the file if it was opened with write access, and if * the caller requested a flush. @@ -1322,7 +1328,7 @@ H5F__dest(H5F_t *f, hbool_t flush) HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "problems closing file") /* Shutdown the page buffer cache */ - if(H5PB_dest(f) < 0) + if(H5PB_dest(f->shared) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "problems closing page buffer cache") @@ -1343,7 +1349,7 @@ H5F__dest(H5F_t *f, hbool_t flush) } /* end if */ /* Destroy other components of the file */ - if(H5F__accum_reset(f, TRUE) < 0) + if(H5F__accum_reset(f->shared, TRUE) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "problems closing file") if(H5FO_dest(f) < 0) @@ -1381,13 +1387,16 @@ H5F__dest(H5F_t *f, hbool_t flush) f->shared->mtab.child = (H5F_mount_t *)H5MM_xfree(f->shared->mtab.child); f->shared->mtab.nalloc = 0; + /* Free the external link file */ + f->shared->extpath = (char *)H5MM_xfree(f->shared->extpath); + /* Clean up the metadata retries array */ for(actype = 0; actype < (int)H5AC_NTYPES; actype++) if(f->shared->retries[actype]) f->shared->retries[actype] = (uint32_t *)H5MM_xfree(f->shared->retries[actype]); /* Destroy shared file struct */ - f->shared = (H5F_file_t *)H5FL_FREE(H5F_file_t, f->shared); + f->shared = (H5F_shared_t *)H5FL_FREE(H5F_shared_t, f->shared); } else if(f->shared->nrefs > 0) { @@ -1401,7 +1410,6 @@ H5F__dest(H5F_t *f, hbool_t flush) /* Free the non-shared part of the file */ f->open_name = (char *)H5MM_xfree(f->open_name); f->actual_name = (char *)H5MM_xfree(f->actual_name); - f->extpath = (char *)H5MM_xfree(f->extpath); if(H5FO_top_dest(f) < 0) HDONE_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "problems closing file") f->shared = NULL; @@ -1488,7 +1496,7 @@ H5F_t * H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) { H5F_t *file = NULL; /*the success return value */ - H5F_file_t *shared = NULL; /*shared part of `file' */ + H5F_shared_t *shared = NULL; /*shared part of `file' */ H5FD_t *lf = NULL; /*file driver part of `shared' */ unsigned tent_flags; /*tentative flags */ H5FD_class_t *drvr; /*file driver class info */ @@ -1647,7 +1655,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) if(page_buf_size) { #ifdef H5_HAVE_PARALLEL /* Collective metadata writes are not supported with page buffering */ - if(file->coll_md_write) + if(file->shared->coll_md_write) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "collective metadata writes are not supported with page buffering") /* Temporary: fail file create when page buffering feature is enabled for parallel */ @@ -1672,7 +1680,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) /* Create the page buffer before initializing the superblock */ if(page_buf_size) - if(H5PB_create(file, page_buf_size, page_buf_min_meta_perc, page_buf_min_raw_perc) < 0) + if(H5PB_create(shared, page_buf_size, page_buf_min_meta_perc, page_buf_min_raw_perc) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to create page buffer") /* Initialize information about the superblock and allocate space for it */ @@ -1694,7 +1702,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) /* Create the page buffer before initializing the superblock */ if(page_buf_size) - if(H5PB_create(file, page_buf_size, page_buf_min_meta_perc, page_buf_min_raw_perc) < 0) + if(H5PB_create(shared, page_buf_size, page_buf_min_meta_perc, page_buf_min_raw_perc) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to create page buffer") /* Open the root group */ @@ -1747,8 +1755,10 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) } /* end if */ /* Formulate the absolute path for later search of target file for external links */ - if(H5_build_extpath(name, &file->extpath) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to build extpath") + if(shared->nrefs == 1) { + if(H5_build_extpath(name, &file->shared->extpath) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to build extpath") + } /* Formulate the actual file name, after following symlinks, etc. */ if(H5F__build_actual_name(file, a_plist, name, &file->actual_name) < 0) @@ -1902,12 +1912,12 @@ H5F__flush_phase2(H5F_t *f, hbool_t closing) #endif /* H5_HAVE_PARALLEL */ /* Flush out the metadata accumulator */ - if(H5F__accum_flush(f) < 0) + if(H5F__accum_flush(f->shared) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_IO, H5E_CANTFLUSH, FAIL, "unable to flush metadata accumulator") /* Flush the page buffer */ - if(H5PB_flush(f) < 0) + if(H5PB_flush(f->shared) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_IO, H5E_CANTFLUSH, FAIL, "page buffer flush failed") @@ -2178,7 +2188,7 @@ H5F_try_close(H5F_t *f, hbool_t *was_closed /*out*/) */ /* Destroy the H5F_t struct and decrement the reference count for the - * shared H5F_file_t struct. If the reference count for the H5F_file_t + * shared H5F_shared_t struct. If the reference count for the H5F_shared_t * struct reaches zero then destroy it also. */ if(H5F__dest(f, TRUE) < 0) @@ -2221,7 +2231,6 @@ H5F__reopen(H5F_t *f) /* Duplicate old file's names */ ret_value->open_name = H5MM_xstrdup(f->open_name); ret_value->actual_name = H5MM_xstrdup(f->actual_name); - ret_value->extpath = H5MM_xstrdup(f->extpath); done: FUNC_LEAVE_NOAPI(ret_value) @@ -2989,8 +2998,10 @@ H5F_set_retries(H5F_t *f) /* Initialize the # of bins for retries */ f->shared->retries_nbins = 0; if(f->shared->read_attempts > 1) { - tmp = HDlog10((double)(f->shared->read_attempts - 1)); - f->shared->retries_nbins = (unsigned)tmp + 1; + /* Use HDceil to ensure that the log10 value is rounded up to the + nearest integer before casting to unsigned */ + tmp = HDceil(HDlog10((double)f->shared->read_attempts)); + f->shared->retries_nbins = (unsigned)tmp; } FUNC_LEAVE_NOAPI(SUCCEED) @@ -3147,31 +3158,6 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5F__get_max_eof_eoa() */ -#ifdef H5_HAVE_PARALLEL - -/*------------------------------------------------------------------------- - * Function: H5F_set_coll_md_read - * - * Purpose: Set the coll_md_read field with a new value. - * - * Return: SUCCEED/FAIL - *------------------------------------------------------------------------- - */ -void -H5F_set_coll_md_read(H5F_t *f, H5P_coll_md_read_flag_t cmr) -{ - /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ - FUNC_ENTER_NOAPI_NOINIT_NOERR - - /* Sanity check */ - HDassert(f); - - f->coll_md_read = cmr; - - FUNC_LEAVE_NOAPI_VOID -} /* H5F_set_coll_md_read() */ -#endif /* H5_HAVE_PARALLEL */ - /*------------------------------------------------------------------------- * Function: H5F_get_metadata_read_retry_info @@ -3411,7 +3397,7 @@ H5F__start_swmr_write(H5F_t *f) } /* end if */ /* Flush and reset the accumulator */ - if(H5F__accum_reset(f, TRUE) < 0) + if(H5F__accum_reset(f->shared, TRUE) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTRESET, FAIL, "can't reset accumulator") /* Turn on SWMR write in shared file open flags */ @@ -3600,6 +3586,9 @@ H5F__get_file(void *obj, H5I_type_t type) oloc = H5A_oloc((H5A_t *)obj); break; + case H5I_MAP: + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "maps not supported in native VOL connector") + case H5I_UNINIT: case H5I_BADID: case H5I_DATASPACE: @@ -3641,7 +3630,7 @@ done: *------------------------------------------------------------------------- */ hid_t -H5F__get_file_id(H5F_t *file) +H5F__get_file_id(H5F_t *file, hbool_t app_ref) { hid_t file_id = H5I_INVALID_HID; /* File ID */ hid_t ret_value = H5I_INVALID_HID; /* Return value */ @@ -3654,13 +3643,13 @@ H5F__get_file_id(H5F_t *file) /* If the ID does not exist, register it with the VOL connector */ if(H5I_INVALID_HID == file_id) { - if((file_id = H5VL_wrap_register(H5I_FILE, file, TRUE)) < 0) + if((file_id = H5VL_wrap_register(H5I_FILE, file, app_ref)) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTREGISTER, H5I_INVALID_HID, "unable to atomize file handle") file->id_exists = TRUE; } /* end if */ else { /* Increment ref count on existing ID */ - if(H5I_inc_ref(file_id, TRUE) < 0) + if(H5I_inc_ref(file_id, app_ref) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTSET, H5I_INVALID_HID, "incrementing file ID failed") } /* end else */ @@ -3684,7 +3673,7 @@ done: *------------------------------------------------------------------------- */ hid_t -H5F_get_file_id(hid_t obj_id, H5I_type_t type) +H5F_get_file_id(hid_t obj_id, H5I_type_t type, hbool_t app_ref) { H5VL_object_t *vol_obj; /* File info */ hid_t file_id = H5I_INVALID_HID; /* File ID for object */ @@ -3697,7 +3686,7 @@ H5F_get_file_id(hid_t obj_id, H5I_type_t type) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5I_INVALID_HID, "invalid identifier") /* Get the file through the VOL */ - if(H5VL_file_optional(vol_obj, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, H5VL_NATIVE_FILE_GET_FILE_ID, (int)type, &file_id) < 0) + if(H5VL_file_optional(vol_obj, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, H5VL_NATIVE_FILE_GET_FILE_ID, (int)type, (int)app_ref, &file_id) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, H5I_INVALID_HID, "unable to get file ID") if(H5I_INVALID_HID == file_id) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, H5I_INVALID_HID, "unable to get the file ID through the VOL") diff --git a/src/H5Fio.c b/src/H5Fio.c index 69e6bb2..34dd0d6 100644 --- a/src/H5Fio.c +++ b/src/H5Fio.c @@ -77,6 +77,50 @@ /*------------------------------------------------------------------------- + * Function: H5F_shared_block_read + * + * Purpose: Reads some data from a file/server/etc into a buffer. + * The data is contiguous. The address is relative to the base + * address for the file. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Robb Matzke + * matzke@llnl.gov + * Jul 10 1997 + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_shared_block_read(H5F_shared_t *f_sh, H5FD_mem_t type, haddr_t addr, size_t size, void *buf/*out*/) +{ + H5FD_mem_t map_type; /* Mapped memory type */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(f_sh); + HDassert(buf); + HDassert(H5F_addr_defined(addr)); + + /* Check for attempting I/O on 'temporary' file address */ + if(H5F_addr_le(f_sh->tmp_addr, (addr + size))) + HGOTO_ERROR(H5E_IO, H5E_BADRANGE, FAIL, "attempting I/O in temporary file space") + + /* Treat global heap as raw data */ + map_type = (type == H5FD_MEM_GHEAP) ? H5FD_MEM_DRAW : type; + + /* Pass through page buffer layer */ + if(H5PB_read(f_sh, map_type, addr, size, buf) < 0) + HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "read through page buffer failed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F_shared_block_read() */ + + +/*------------------------------------------------------------------------- * Function: H5F_block_read * * Purpose: Reads some data from a file/server/etc into a buffer. @@ -113,7 +157,7 @@ H5F_block_read(H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size, void *buf/* map_type = (type == H5FD_MEM_GHEAP) ? H5FD_MEM_DRAW : type; /* Pass through page buffer layer */ - if(H5PB_read(f, map_type, addr, size, buf) < 0) + if(H5PB_read(f->shared, map_type, addr, size, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "read through page buffer failed") done: @@ -122,6 +166,51 @@ done: /*------------------------------------------------------------------------- + * Function: H5F_shared_block_write + * + * Purpose: Writes some data from memory to a file/server/etc. The + * data is contiguous. The address is relative to the base + * address. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Robb Matzke + * matzke@llnl.gov + * Jul 10 1997 + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_shared_block_write(H5F_shared_t *f_sh, H5FD_mem_t type, haddr_t addr, size_t size, const void *buf) +{ + H5FD_mem_t map_type; /* Mapped memory type */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(f_sh); + HDassert(H5F_SHARED_INTENT(f_sh) & H5F_ACC_RDWR); + HDassert(buf); + HDassert(H5F_addr_defined(addr)); + + /* Check for attempting I/O on 'temporary' file address */ + if(H5F_addr_le(f_sh->tmp_addr, (addr + size))) + HGOTO_ERROR(H5E_IO, H5E_BADRANGE, FAIL, "attempting I/O in temporary file space") + + /* Treat global heap as raw data */ + map_type = (type == H5FD_MEM_GHEAP) ? H5FD_MEM_DRAW : type; + + /* Pass through page buffer layer */ + if(H5PB_write(f_sh, map_type, addr, size, buf) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "write through page buffer failed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F_shared_block_write() */ + + +/*------------------------------------------------------------------------- * Function: H5F_block_write * * Purpose: Writes some data from memory to a file/server/etc. The @@ -159,7 +248,7 @@ H5F_block_write(H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size, const void map_type = (type == H5FD_MEM_GHEAP) ? H5FD_MEM_DRAW : type; /* Pass through page buffer layer */ - if(H5PB_write(f, map_type, addr, size, buf) < 0) + if(H5PB_write(f->shared, map_type, addr, size, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "write through page buffer failed") done: @@ -170,7 +259,7 @@ done: /*------------------------------------------------------------------------- * Function: H5F_flush_tagged_metadata * - * Purpose: Flushes metadata with specified tag in the metadata cache + * Purpose: Flushes metadata with specified tag in the metadata cache * to disk. * * Return: Non-negative on success/Negative on failure @@ -192,7 +281,7 @@ H5F_flush_tagged_metadata(H5F_t *f, haddr_t tag) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush tagged metadata") /* Flush and reset the accumulator */ - if(H5F__accum_reset(f, TRUE) < 0) + if(H5F__accum_reset(f->shared, TRUE) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTRESET, FAIL, "can't reset accumulator") /* Flush file buffers to disk. */ diff --git a/src/H5Fmpi.c b/src/H5Fmpi.c index 5bbd717..bb422ac 100644 --- a/src/H5Fmpi.c +++ b/src/H5Fmpi.c @@ -90,14 +90,14 @@ herr_t H5F_get_mpi_handle(const H5F_t *f, MPI_File **f_handle) { herr_t ret_value = SUCCEED; - hid_t fapl = -1; + hid_t fapl_id = H5I_INVALID_HID; FUNC_ENTER_NOAPI(FAIL) HDassert(f && f->shared); /* Dispatch to driver */ - if ((ret_value = H5FD_get_vfd_handle(f->shared->lf, fapl, (void **)f_handle)) < 0) + if ((ret_value = H5FD_get_vfd_handle(f->shared->lf, fapl_id, (void **)f_handle)) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get mpi file handle") done: @@ -106,33 +106,31 @@ done: /*------------------------------------------------------------------------- - * Function: H5F_mpi_get_rank + * Function: H5F_mpi_get_rank * - * Purpose: Retrieves the rank of an MPI process. + * Purpose: Retrieves the rank of an MPI process. * - * Return: Success: The rank (non-negative) + * Return: Success: The rank (non-negative) * - * Failure: Negative + * Failure: Negative * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Friday, January 30, 2004 * - * Modifications: - * *------------------------------------------------------------------------- */ int H5F_mpi_get_rank(const H5F_t *f) { - int ret_value; + int ret_value = -1; - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI((-1)) HDassert(f && f->shared); /* Dispatch to driver */ - if ((ret_value=H5FD_mpi_get_rank(f->shared->lf)) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "driver get_rank request failed") + if ((ret_value = H5FD_mpi_get_rank(f->shared->lf)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, (-1), "driver get_rank request failed") done: FUNC_LEAVE_NOAPI(ret_value) @@ -140,33 +138,31 @@ done: /*------------------------------------------------------------------------- - * Function: H5F_mpi_get_comm + * Function: H5F_mpi_get_comm * - * Purpose: Retrieves the file's communicator + * Purpose: Retrieves the file's communicator * - * Return: Success: The communicator (non-negative) + * Return: Success: The communicator (non-negative) * - * Failure: Negative + * Failure: Negative * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Friday, January 30, 2004 * - * Modifications: - * *------------------------------------------------------------------------- */ MPI_Comm H5F_mpi_get_comm(const H5F_t *f) { - MPI_Comm ret_value; + MPI_Comm ret_value = MPI_COMM_NULL; FUNC_ENTER_NOAPI(MPI_COMM_NULL) HDassert(f && f->shared); /* Dispatch to driver */ - if ((ret_value=H5FD_mpi_get_comm(f->shared->lf))==MPI_COMM_NULL) - HGOTO_ERROR(H5E_VFL, H5E_CANTGET, MPI_COMM_NULL, "driver get_comm request failed") + if ((ret_value = H5FD_mpi_get_comm(f->shared->lf)) == MPI_COMM_NULL) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, MPI_COMM_NULL, "driver get_comm request failed") done: FUNC_LEAVE_NOAPI(ret_value) @@ -174,7 +170,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5F_mpi_get_size + * Function: H5F_shared_mpi_get_size * * Purpose: Retrieves the size of an MPI process. * @@ -185,22 +181,52 @@ done: * Programmer: John Mainzer * Friday, May 6, 2005 * - * Modifications: + *------------------------------------------------------------------------- + */ +int +H5F_shared_mpi_get_size(const H5F_shared_t *f_sh) +{ + int ret_value = -1; + + FUNC_ENTER_NOAPI((-1)) + + HDassert(f_sh); + + /* Dispatch to driver */ + if((ret_value = H5FD_mpi_get_size(f_sh->lf)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, (-1), "driver get_size request failed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F_shared_mpi_get_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_mpi_get_size + * + * Purpose: Retrieves the size of an MPI process. + * + * Return: Success: The size (positive) + * + * Failure: Negative + * + * Programmer: John Mainzer + * Friday, May 6, 2005 * *------------------------------------------------------------------------- */ int H5F_mpi_get_size(const H5F_t *f) { - int ret_value; + int ret_value = -1; - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI((-1)) HDassert(f && f->shared); /* Dispatch to driver */ - if ((ret_value=H5FD_mpi_get_size(f->shared->lf)) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "driver get_size request failed") + if ((ret_value = H5FD_mpi_get_size(f->shared->lf)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, (-1), "driver get_size request failed") done: FUNC_LEAVE_NOAPI(ret_value) @@ -208,16 +234,16 @@ done: /*------------------------------------------------------------------------- - * Function: H5Fset_mpi_atomicity + * Function: H5Fset_mpi_atomicity * - * Purpose: Sets the atomicity mode + * Purpose: Sets the atomicity mode * - * Return: Success: Non-negative + * Return: Success: Non-negative * - * Failure: Negative + * Failure: Negative * - * Programmer: Mohamad Chaarawi - * Feb 14, 2012 + * Programmer: Mohamad Chaarawi + * Feb 14, 2012 * *------------------------------------------------------------------------- */ @@ -248,16 +274,16 @@ done: /*------------------------------------------------------------------------- - * Function: H5Fget_mpi_atomicity + * Function: H5Fget_mpi_atomicity * - * Purpose: Returns the atomicity mode + * Purpose: Returns the atomicity mode * - * Return: Success: Non-negative + * Return: Success: Non-negative * - * Failure: Negative + * Failure: Negative * - * Programmer: Mohamad Chaarawi - * Feb 14, 2012 + * Programmer: Mohamad Chaarawi + * Feb 14, 2012 * *------------------------------------------------------------------------- */ @@ -288,18 +314,18 @@ done: /*------------------------------------------------------------------------- - * Function: H5F_mpi_retrieve_comm + * Function: H5F_mpi_retrieve_comm * - * Purpose: Retrieves an MPI communicator from the file the location ID + * Purpose: Retrieves an MPI communicator from the file the location ID * is in. If the loc_id is invalid, the fapl_id is used to * retrieve the communicator. * - * Return: Success: Non-negative + * Return: Success: Non-negative * - * Failure: Negative + * Failure: Negative * - * Programmer: Mohamad Chaarawi - * Feb 14, 2012 + * Programmer: Mohamad Chaarawi + * Feb 14, 2012 * *------------------------------------------------------------------------- */ @@ -342,14 +368,9 @@ H5F_mpi_retrieve_comm(hid_t loc_id, hid_t acspl_id, MPI_Comm *mpi_comm) if(NULL == (plist = H5P_object_verify(acspl_id, H5P_FILE_ACCESS))) HGOTO_ERROR(H5E_FILE, H5E_BADTYPE, FAIL, "not a file access list") - if(H5FD_MPIO == H5P_peek_driver(plist)) { - const H5FD_mpio_fapl_t *fa; /* MPIO fapl info */ - - if(NULL == (fa = (const H5FD_mpio_fapl_t *)H5P_peek_driver_info(plist))) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad VFL driver info") - - *mpi_comm = fa->comm; - } + if(H5FD_MPIO == H5P_peek_driver(plist)) + if(H5P_peek(plist, H5F_ACS_MPI_PARAMS_COMM_NAME, mpi_comm) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get MPI communicator") } done: diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index 1dd07c1..6cd2d3c 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -217,7 +217,7 @@ typedef struct H5F_mtab_t { } H5F_mtab_t; /* Structure specifically to store superblock. This was originally - * maintained entirely within H5F_file_t, but is now extracted + * maintained entirely within H5F_shared_t, but is now extracted * here because the superblock is now handled by the cache */ typedef struct H5F_super_t { H5AC_info_t cache_info; /* Cache entry information structure */ @@ -238,11 +238,11 @@ typedef struct H5F_super_t { /* * Define the structure to store the file information for HDF5 files. One of * these structures is allocated per file, not per H5Fopen(). That is, set of - * H5F_t structs can all point to the same H5F_file_t struct. The `nrefs' + * H5F_t structs can all point to the same H5F_shared_t struct. The `nrefs' * count in this struct indicates the number of H5F_t structs which are * pointing to this struct. */ -struct H5F_file_t { +struct H5F_shared_t { H5FD_t *lf; /* Lower level file handle for I/O */ H5F_super_t *sblock; /* Pointer to (pinned) superblock for file */ H5O_drvinfo_t *drvinfo; /* Pointer to the (pinned) driver info @@ -309,6 +309,7 @@ struct H5F_file_t { struct H5G_t *root_grp; /* Open root group */ H5FO_t *open_objs; /* Open objects in file */ H5UC_t *grp_btree_shared; /* Ref-counted group B-tree node info */ + hbool_t closing; /* File is in the process of being closed */ /* Cached VOL connector ID & info */ hid_t vol_id; /* ID of VOL connector for the container */ @@ -357,28 +358,31 @@ struct H5F_file_t { /* Object flush info */ H5F_object_flush_t object_flush; /* Information for object flush callback */ hbool_t crt_dset_min_ohdr_flag; /* flag to minimize created dataset object header */ + + char *extpath; /* Path for searching target external link file */ + +#ifdef H5_HAVE_PARALLEL + H5P_coll_md_read_flag_t coll_md_read; /* Do all metadata reads collectively */ + hbool_t coll_md_write; /* Do all metadata writes collectively */ +#endif /* H5_HAVE_PARALLEL */ + }; /* * This is the top-level file descriptor. One of these structures is * allocated every time H5Fopen() is called although they may contain pointers - * to shared H5F_file_t structs. + * to shared H5F_shared_t structs. */ struct H5F_t { - char *open_name; /* Name used to open file */ - char *actual_name; /* Actual name of the file, after resolving symlinks, etc. */ - char *extpath; /* Path for searching target external link file */ - H5F_file_t *shared; /* The shared file info */ - unsigned nopen_objs; /* Number of open object headers */ + char *open_name; /* Name used to open file */ + char *actual_name; /* Actual name of the file, after resolving symlinks, etc. */ + H5F_shared_t *shared; /* The shared file info */ + unsigned nopen_objs; /* Number of open object headers */ H5FO_t *obj_count; /* # of time each object is opened through top file structure */ hbool_t id_exists; /* Whether an ID for this struct exists */ hbool_t closing; /* File is in the process of being closed */ struct H5F_t *parent; /* Parent file that this file is mounted to */ unsigned nmounts; /* Number of children mounted to this file */ -#ifdef H5_HAVE_PARALLEL - H5P_coll_md_read_flag_t coll_md_read; /* Do all metadata reads collectively */ - hbool_t coll_md_write; /* Do all metadata writes collectively */ -#endif /* H5_HAVE_PARALLEL */ }; /*****************************/ @@ -388,8 +392,8 @@ struct H5F_t { /* Declare a free list to manage the H5F_t struct */ H5FL_EXTERN(H5F_t); -/* Declare a free list to manage the H5F_file_t struct */ -H5FL_EXTERN(H5F_file_t); +/* Declare a free list to manage the H5F_shared_t struct */ +H5FL_EXTERN(H5F_shared_t); /******************************/ @@ -408,7 +412,7 @@ H5_DLL herr_t H5F__start_swmr_write(H5F_t *f); H5_DLL herr_t H5F__close(H5F_t *f); H5_DLL herr_t H5F__set_libver_bounds(H5F_t *f, H5F_libver_t low, H5F_libver_t high); H5_DLL H5F_t *H5F__get_file(void *obj, H5I_type_t type); -H5_DLL hid_t H5F__get_file_id(H5F_t *file); +H5_DLL hid_t H5F__get_file_id(H5F_t *file, hbool_t app_ref); /* File mount related routines */ H5_DLL herr_t H5F__mount(H5G_loc_t *loc, const char *name, H5F_t *child, hid_t plist_id); @@ -429,16 +433,16 @@ H5_DLL herr_t H5F__super_ext_remove_msg(H5F_t *f, unsigned id); H5_DLL herr_t H5F__super_ext_close(H5F_t *f, H5O_loc_t *ext_ptr, hbool_t was_created); /* Metadata accumulator routines */ -H5_DLL herr_t H5F__accum_read(H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size, void *buf); -H5_DLL herr_t H5F__accum_write(H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size, const void *buf); -H5_DLL herr_t H5F__accum_free(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size); -H5_DLL herr_t H5F__accum_flush(H5F_t *f); -H5_DLL herr_t H5F__accum_reset(H5F_t *f, hbool_t flush); +H5_DLL herr_t H5F__accum_read(H5F_shared_t *f_sh, H5FD_mem_t type, haddr_t addr, size_t size, void *buf); +H5_DLL herr_t H5F__accum_write(H5F_shared_t *f_sh, H5FD_mem_t type, haddr_t addr, size_t size, const void *buf); +H5_DLL herr_t H5F__accum_free(H5F_shared_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size); +H5_DLL herr_t H5F__accum_flush(H5F_shared_t *f_sh); +H5_DLL herr_t H5F__accum_reset(H5F_shared_t *f_sh, hbool_t flush); /* Shared file list related routines */ -H5_DLL herr_t H5F__sfile_add(H5F_file_t *shared); -H5_DLL H5F_file_t *H5F__sfile_search(H5FD_t *lf); -H5_DLL herr_t H5F__sfile_remove(H5F_file_t *shared); +H5_DLL herr_t H5F__sfile_add(H5F_shared_t *shared); +H5_DLL H5F_shared_t *H5F__sfile_search(H5FD_t *lf); +H5_DLL herr_t H5F__sfile_remove(H5F_shared_t *shared); /* External file cache routines */ H5_DLL H5F_efc_t *H5F__efc_create(unsigned max_nfiles); diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index a8ac158..8c70663 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -278,10 +278,11 @@ typedef struct H5F_t H5F_t; #ifdef H5F_MODULE #define H5F_LOW_BOUND(F) ((F)->shared->low_bound) #define H5F_HIGH_BOUND(F) ((F)->shared->high_bound) +#define H5F_SHARED_INTENT(F_SH) ((F_SH)->flags) #define H5F_INTENT(F) ((F)->shared->flags) #define H5F_OPEN_NAME(F) ((F)->open_name) #define H5F_ACTUAL_NAME(F) ((F)->actual_name) -#define H5F_EXTPATH(F) ((F)->extpath) +#define H5F_EXTPATH(F) ((F)->shared->extpath) #define H5F_SHARED(F) ((F)->shared) #define H5F_SAME_SHARED(F1, F2) ((F1)->shared == (F2)->shared) #define H5F_NOPEN_OBJS(F) ((F)->nopen_objs) @@ -293,6 +294,7 @@ typedef struct H5F_t H5F_t; #define H5F_GET_READ_ATTEMPTS(F) ((F)->shared->read_attempts) #define H5F_DRIVER_ID(F) ((F)->shared->lf->driver_id) #define H5F_GET_FILENO(F,FILENUM) ((FILENUM) = (F)->shared->lf->fileno) +#define H5F_SHARED_HAS_FEATURE(F_SH,FL) ((F_SH)->lf->feature_flags & (FL)) #define H5F_HAS_FEATURE(F,FL) ((F)->shared->lf->feature_flags & (FL)) #define H5F_BASE_ADDR(F) ((F)->shared->sblock->base_addr) #define H5F_SYM_LEAF_K(F) ((F)->shared->sblock->sym_leaf_k) @@ -321,7 +323,7 @@ typedef struct H5F_t H5F_t; #define H5F_USE_TMP_SPACE(F) ((F)->shared->fs.use_tmp_space) #define H5F_IS_TMP_ADDR(F, ADDR) (H5F_addr_le((F)->shared->fs.tmp_addr, (ADDR))) #ifdef H5_HAVE_PARALLEL -#define H5F_COLL_MD_READ(F) ((F)->coll_md_read) +#define H5F_COLL_MD_READ(F) ((F)->shared->coll_md_read) #endif /* H5_HAVE_PARALLEL */ #define H5F_USE_MDC_LOGGING(F) ((F)->shared->use_mdc_logging) #define H5F_START_MDC_LOG_ON_ACCESS(F) ((F)->shared->start_mdc_log_on_access) @@ -336,6 +338,7 @@ typedef struct H5F_t H5F_t; #else /* H5F_MODULE */ #define H5F_LOW_BOUND(F) (H5F_get_low_bound(F)) #define H5F_HIGH_BOUND(F) (H5F_get_high_bound(F)) +#define H5F_SHARED_INTENT(F_SH) (H5F_shared_get_intent(F_SH)) #define H5F_INTENT(F) (H5F_get_intent(F)) #define H5F_OPEN_NAME(F) (H5F_get_open_name(F)) #define H5F_ACTUAL_NAME(F) (H5F_get_actual_name(F)) @@ -351,6 +354,7 @@ typedef struct H5F_t H5F_t; #define H5F_GET_READ_ATTEMPTS(F) (H5F_get_read_attempts(F)) #define H5F_DRIVER_ID(F) (H5F_get_driver_id(F)) #define H5F_GET_FILENO(F,FILENUM) (H5F_get_fileno((F), &(FILENUM))) +#define H5F_SHARED_HAS_FEATURE(F_SH,FL) (H5F_shared_has_feature(F_SH,FL)) #define H5F_HAS_FEATURE(F,FL) (H5F_has_feature(F,FL)) #define H5F_BASE_ADDR(F) (H5F_get_base_addr(F)) #define H5F_SYM_LEAF_K(F) (H5F_sym_leaf_k(F)) @@ -509,6 +513,10 @@ typedef struct H5F_t H5F_t; #define H5F_ACS_PAGE_BUFFER_SIZE_NAME "page_buffer_size" /* the maximum size for the page buffer cache */ #define H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME "page_buffer_min_meta_perc" /* the min metadata percentage for the page buffer cache */ #define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME "page_buffer_min_raw_perc" /* the min raw data percentage for the page buffer cache */ +#ifdef H5_HAVE_PARALLEL +#define H5F_ACS_MPI_PARAMS_COMM_NAME "mpi_params_comm" /* the MPI communicator */ +#define H5F_ACS_MPI_PARAMS_INFO_NAME "mpi_params_info" /* the MPI info struct */ +#endif /* H5_HAVE_PARALLEL */ /* ======================== File Mount properties ====================*/ #define H5F_MNT_SYM_LOCAL_NAME "local" /* Whether absolute symlinks local to file. */ @@ -575,6 +583,7 @@ typedef struct H5F_t H5F_t; #define H5F_SDATA_BLOCK_SIZE_DEF 2048 /* Check for file using paged aggregation */ +#define H5F_SHARED_PAGED_AGGR(F_SH) ((F_SH)->fs_strategy == H5F_FSPACE_STRATEGY_PAGE && (F_SH)->fs_page_size) #define H5F_PAGED_AGGR(F) (F->shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE && F->shared->fs_page_size) /* Metadata read attempt values */ @@ -651,7 +660,7 @@ struct H5P_genplist_t; /* Forward declarations for anonymous H5F objects */ /* Main file structures */ -typedef struct H5F_file_t H5F_file_t; +typedef struct H5F_shared_t H5F_shared_t; /* Block aggregation structure */ typedef struct H5F_blk_aggr_t H5F_blk_aggr_t; @@ -718,16 +727,17 @@ typedef enum H5F_prefix_open_t { /* Private functions */ H5_DLL H5F_t *H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id); H5_DLL herr_t H5F_try_close(H5F_t *f, hbool_t *was_closed/*out*/); -H5_DLL hid_t H5F_get_file_id(hid_t obj_id, H5I_type_t id_type); +H5_DLL hid_t H5F_get_file_id(hid_t obj_id, H5I_type_t id_type, hbool_t app_ref); /* Functions that retrieve values from the file struct */ H5_DLL H5F_libver_t H5F_get_low_bound(const H5F_t *f); H5_DLL H5F_libver_t H5F_get_high_bound(const H5F_t *f); +H5_DLL unsigned H5F_shared_get_intent(const H5F_shared_t *f); H5_DLL unsigned H5F_get_intent(const H5F_t *f); H5_DLL char *H5F_get_open_name(const H5F_t *f); H5_DLL char *H5F_get_actual_name(const H5F_t *f); H5_DLL char *H5F_get_extpath(const H5F_t *f); -H5_DLL H5F_file_t *H5F_get_shared(const H5F_t *f); +H5_DLL H5F_shared_t *H5F_get_shared(const H5F_t *f); H5_DLL hbool_t H5F_same_shared(const H5F_t *f1, const H5F_t *f2); H5_DLL unsigned H5F_get_nopen_objs(const H5F_t *f); H5_DLL unsigned H5F_incr_nopen_objs(H5F_t *f); @@ -786,7 +796,9 @@ H5_DLL char *H5F_mdc_log_location(const H5F_t *f); /* Functions that retrieve values from VFD layer */ H5_DLL hid_t H5F_get_driver_id(const H5F_t *f); H5_DLL herr_t H5F_get_fileno(const H5F_t *f, unsigned long *filenum); +H5_DLL hbool_t H5F_shared_has_feature(const H5F_shared_t *f, unsigned feature); H5_DLL hbool_t H5F_has_feature(const H5F_t *f, unsigned feature); +H5_DLL haddr_t H5F_shared_get_eoa(const H5F_shared_t *f_sh, H5FD_mem_t type); H5_DLL haddr_t H5F_get_eoa(const H5F_t *f, H5FD_mem_t type); H5_DLL herr_t H5F_get_vfd_handle(const H5F_t *file, hid_t fapl, void **file_handle); @@ -797,7 +809,9 @@ H5_DLL herr_t H5F_traverse_mount(struct H5O_loc_t *oloc/*in,out*/); H5_DLL herr_t H5F_flush_mounts(H5F_t *f); /* Functions that operate on blocks of bytes wrt super block */ +H5_DLL herr_t H5F_shared_block_read(H5F_shared_t *f_sh, H5FD_mem_t type, haddr_t addr, size_t size, void *buf/*out*/); H5_DLL herr_t H5F_block_read(H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size, void *buf/*out*/); +H5_DLL herr_t H5F_shared_block_write(H5F_shared_t *f_sh, H5FD_mem_t type, haddr_t addr, size_t size, const void *buf); H5_DLL herr_t H5F_block_write(H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size, const void *buf); /* Functions that flush or evict */ @@ -837,6 +851,7 @@ H5_DLL herr_t H5F_eoa_dirty(H5F_t *f); H5_DLL herr_t H5F_get_mpi_handle(const H5F_t *f, MPI_File **f_handle); H5_DLL int H5F_mpi_get_rank(const H5F_t *f); H5_DLL MPI_Comm H5F_mpi_get_comm(const H5F_t *f); +H5_DLL int H5F_shared_mpi_get_size(const H5F_shared_t *f_sh); H5_DLL int H5F_mpi_get_size(const H5F_t *f); H5_DLL herr_t H5F_mpi_retrieve_comm(hid_t loc_id, hid_t acspl_id, MPI_Comm *mpi_comm); H5_DLL herr_t H5F_get_mpi_info(const H5F_t *f, MPI_Info **f_info); @@ -853,7 +868,7 @@ H5_DLL H5F_t *H5F_prefix_open_file(H5F_t *primary_file, H5F_prefix_open_t prefix H5_DLL herr_t H5F_cwfs_add(H5F_t *f, struct H5HG_heap_t *heap); H5_DLL herr_t H5F_cwfs_find_free_heap(H5F_t *f, size_t need, haddr_t *addr); H5_DLL herr_t H5F_cwfs_advance_heap(H5F_t *f, struct H5HG_heap_t *heap, hbool_t add_heap); -H5_DLL herr_t H5F_cwfs_remove_heap(H5F_file_t *shared, struct H5HG_heap_t *heap); +H5_DLL herr_t H5F_cwfs_remove_heap(H5F_shared_t *shared, struct H5HG_heap_t *heap); /* Debugging functions */ H5_DLL herr_t H5F_debug(H5F_t *f, FILE * stream, int indent, int fwidth); diff --git a/src/H5Fquery.c b/src/H5Fquery.c index c168535..f36f348 100644 --- a/src/H5Fquery.c +++ b/src/H5Fquery.c @@ -75,6 +75,28 @@ /*------------------------------------------------------------------------- + * Function: H5F_shared_get_intent + * + * Purpose: Quick and dirty routine to retrieve the file's 'intent' flags + * (Mainly added to stop non-file routines from poking about in the + * H5F_shared_t data structure) + * + * Return: 'intent' on success/abort on failure (shouldn't fail) + *------------------------------------------------------------------------- + */ +unsigned +H5F_shared_get_intent(const H5F_shared_t *f_sh) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f_sh); + + FUNC_LEAVE_NOAPI(f_sh->flags) +} /* end H5F_shared_get_intent() */ + + +/*------------------------------------------------------------------------- * Function: H5F_get_intent * * Purpose: Quick and dirty routine to retrieve the file's 'intent' flags @@ -207,9 +229,9 @@ H5F_get_extpath(const H5F_t *f) FUNC_ENTER_NOAPI_NOINIT_NOERR HDassert(f); - HDassert(f->extpath); + HDassert(f->shared->extpath); - FUNC_LEAVE_NOAPI(f->extpath) + FUNC_LEAVE_NOAPI(f->shared->extpath) } /* end H5F_get_extpath() */ @@ -221,7 +243,7 @@ H5F_get_extpath(const H5F_t *f) * Return: 'shared' on success/abort on failure (shouldn't fail) *------------------------------------------------------------------------- */ -H5F_file_t * +H5F_shared_t * H5F_get_shared(const H5F_t *f) { /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ @@ -827,6 +849,27 @@ H5F_store_msg_crt_idx(const H5F_t *f) /*------------------------------------------------------------------------- + * Function: H5F_shared_has_feature + * + * Purpose: Check if a file has a particular feature enabled + * + * Return: Success: Non-negative - TRUE or FALSE + * Failure: Negative (should not happen) + *------------------------------------------------------------------------- + */ +hbool_t +H5F_shared_has_feature(const H5F_shared_t *f_sh, unsigned feature) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f_sh); + + FUNC_LEAVE_NOAPI((hbool_t)(f_sh->lf->feature_flags & feature)) +} /* end H5F_shared_has_feature() */ + + +/*------------------------------------------------------------------------- * Function: H5F_has_feature * * Purpose: Check if a file has a particular feature enabled @@ -904,6 +947,32 @@ done: /*------------------------------------------------------------------------- + * Function: H5F_shared_get_eoa + * + * Purpose: Quick and dirty routine to retrieve the file's 'eoa' value + * + * Return: Non-negative on success/Negative on failure + *------------------------------------------------------------------------- + */ +haddr_t +H5F_shared_get_eoa(const H5F_shared_t *f_sh, H5FD_mem_t type) +{ + haddr_t ret_value = HADDR_UNDEF; /* Return value */ + + FUNC_ENTER_NOAPI(HADDR_UNDEF) + + HDassert(f_sh); + + /* Dispatch to driver */ + if(HADDR_UNDEF == (ret_value = H5FD_get_eoa(f_sh->lf, type))) + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, HADDR_UNDEF, "driver get_eoa request failed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F_shared_get_eoa() */ + + +/*------------------------------------------------------------------------- * Function: H5F_get_eoa * * Purpose: Quick and dirty routine to retrieve the file's 'eoa' value @@ -1026,7 +1095,7 @@ H5F_coll_md_read(const H5F_t *f) HDassert(f); - FUNC_LEAVE_NOAPI(f->coll_md_read) + FUNC_LEAVE_NOAPI(f->shared->coll_md_read) } /* end H5F_coll_md_read() */ #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5Fsfile.c b/src/H5Fsfile.c index ebc2ab2..9a9bbab 100644 --- a/src/H5Fsfile.c +++ b/src/H5Fsfile.c @@ -24,7 +24,7 @@ /* Struct for tracking "shared" file structs */ typedef struct H5F_sfile_node_t { - H5F_file_t *shared; /* Pointer to "shared" file struct */ + H5F_shared_t *shared; /* Pointer to "shared" file struct */ struct H5F_sfile_node_t *next; /* Pointer to next node */ } H5F_sfile_node_t; @@ -96,9 +96,9 @@ H5F_sfile_assert_num(unsigned n) *------------------------------------------------------------------------- */ herr_t -H5F__sfile_add(H5F_file_t *shared) +H5F__sfile_add(H5F_shared_t *shared) { - H5F_sfile_node_t *new_shared; /* New shared file node */ + H5F_sfile_node_t *new_shared; /* New shared file node */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -135,11 +135,11 @@ done: * *------------------------------------------------------------------------- */ -H5F_file_t * +H5F_shared_t * H5F__sfile_search(H5FD_t *lf) { H5F_sfile_node_t *curr; /* Current shared file node */ - H5F_file_t *ret_value = NULL; /* Return value */ + H5F_shared_t *ret_value = NULL; /* Return value */ FUNC_ENTER_PACKAGE_NOERR @@ -175,7 +175,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F__sfile_remove(H5F_file_t *shared) +H5F__sfile_remove(H5F_shared_t *shared) { H5F_sfile_node_t *curr; /* Current shared file node */ H5F_sfile_node_t *last; /* Last shared file node */ diff --git a/src/H5Gdeprec.c b/src/H5Gdeprec.c index 95627d2..97a3ccf 100644 --- a/src/H5Gdeprec.c +++ b/src/H5Gdeprec.c @@ -131,6 +131,9 @@ H5G_map_obj_type(H5O_type_t obj_type) ret_value = H5G_TYPE; break; + case H5O_TYPE_MAP: + /* Maps not supported in native VOL connector */ + case H5O_TYPE_UNKNOWN: case H5O_TYPE_NTYPES: default: @@ -354,7 +357,7 @@ H5Glink(hid_t cur_loc_id, H5G_link_t type, const char *cur_name, const char *new tmp_vol_obj.connector = vol_obj->connector; /* Create the link through the VOL */ - if(H5VL_link_create(H5VL_LINK_CREATE_HARD, &tmp_vol_obj, &loc_params2, lcpl_id, H5P_DEFAULT, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, vol_obj->data, loc_params1) < 0) + if(H5VL_link_create(H5VL_LINK_CREATE_HARD, &tmp_vol_obj, &loc_params2, lcpl_id, H5P_DEFAULT, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, vol_obj->data, &loc_params1) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "unable to create link") } /* end if */ else if(type == H5L_TYPE_SOFT) { @@ -434,7 +437,7 @@ H5Glink2(hid_t cur_loc_id, const char *cur_name, H5G_link_t type, HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid location identifier") /* Create the link through the VOL */ - if(H5VL_link_create(H5VL_LINK_CREATE_HARD, vol_obj2, &loc_params2, lcpl_id, H5P_DEFAULT, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, vol_obj1->data, loc_params1) < 0) + if(H5VL_link_create(H5VL_LINK_CREATE_HARD, vol_obj2, &loc_params2, lcpl_id, H5P_DEFAULT, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, vol_obj1->data, &loc_params1) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "unable to create link") } /* end if */ else if(type == H5L_TYPE_SOFT) { diff --git a/src/H5Gloc.c b/src/H5Gloc.c index e47d3be..40d56c6 100644 --- a/src/H5Gloc.c +++ b/src/H5Gloc.c @@ -213,6 +213,9 @@ H5G_loc_real(void *obj, H5I_type_t type, H5G_loc_t *loc) case H5I_DATASPACE: HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to get group location of dataspace") + case H5I_MAP: + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "maps not supported in native VOL connector") + case H5I_GENPROP_CLS: case H5I_GENPROP_LST: HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to get group location of property list") diff --git a/src/H5Gname.c b/src/H5Gname.c index 16c1ca7..86a0c2f 100644 --- a/src/H5Gname.c +++ b/src/H5Gname.c @@ -816,6 +816,9 @@ H5G_name_replace_cb(void *obj_ptr, hid_t obj_id, void *key) obj_path = H5T_nameof((H5T_t *)obj_ptr); break; + case H5I_MAP: + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "maps not supported in native VOL connector") + case H5I_UNINIT: case H5I_BADID: case H5I_FILE: @@ -1112,6 +1115,9 @@ H5G_name_replace(const H5O_link_t *lnk, H5G_names_op_t op, H5F_t *src_file, search_datatype = TRUE; break; + case H5O_TYPE_MAP: + HGOTO_ERROR(H5E_SYM, H5E_BADTYPE, FAIL, "maps not supported in native VOL connector") + case H5O_TYPE_UNKNOWN: case H5O_TYPE_NTYPES: /* Search and replace names through datatype IDs */ diff --git a/src/H5Gtest.c b/src/H5Gtest.c index 0b431b1..f9ab6f2 100644 --- a/src/H5Gtest.c +++ b/src/H5Gtest.c @@ -604,6 +604,9 @@ H5G__user_path_test(hid_t obj_id, char *user_path, size_t *user_path_len, unsign obj_path = H5T_nameof((H5T_t *)obj_ptr); break; + case H5I_MAP: + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "maps not supported in native VOL connector") + case H5I_UNINIT: case H5I_BADID: case H5I_FILE: diff --git a/src/H5HGpkg.h b/src/H5HGpkg.h index 4ed9b10..3119de4 100644 --- a/src/H5HGpkg.h +++ b/src/H5HGpkg.h @@ -117,7 +117,7 @@ typedef struct H5HG_obj_t { } H5HG_obj_t; /* Forward declarations for fields */ -struct H5F_file_t; +struct H5F_shared_t; struct H5HG_heap_t { H5AC_info_t cache_info; /* Information for H5AC cache functions, _must_ be */ @@ -130,7 +130,7 @@ struct H5HG_heap_t { /* If this value is >65535 then all indices */ /* have been used at some time and the */ /* correct new index should be searched for */ - struct H5F_file_t *shared; /* shared file */ + struct H5F_shared_t *shared; /* shared file */ H5HG_obj_t *obj; /*array of object descriptions */ }; @@ -2219,7 +2219,7 @@ H5Iget_file_id(hid_t obj_id) /* Call internal function */ if (H5I_FILE == type || H5I_DATATYPE == type || H5I_GROUP == type || H5I_DATASET == type || H5I_ATTR == type) { - if ((ret_value = H5F_get_file_id(obj_id, type)) < 0) + if ((ret_value = H5F_get_file_id(obj_id, type, TRUE)) < 0) HGOTO_ERROR(H5E_ATOM, H5E_CANTGET, H5I_INVALID_HID, "can't retrieve file ID") } /* end if */ else @@ -2371,6 +2371,11 @@ H5I__id_dump_cb(void *_item, void H5_ATTR_UNUSED *_key, void *_udata) break; } + /* TODO: Maps will have to be added when they are supported in the + * native VOL connector. + */ + case H5I_MAP: + case H5I_UNINIT: case H5I_BADID: case H5I_FILE: diff --git a/src/H5Ipublic.h b/src/H5Ipublic.h index def785f..34a408f 100644 --- a/src/H5Ipublic.h +++ b/src/H5Ipublic.h @@ -38,6 +38,7 @@ typedef enum H5I_type_t { H5I_DATATYPE, /* type ID for Datatype objects */ H5I_DATASPACE, /* type ID for Dataspace objects */ H5I_DATASET, /* type ID for Dataset objects */ + H5I_MAP, /* type ID for Map objects */ H5I_ATTR, /* type ID for Attribute objects */ H5I_VFL, /* type ID for virtual file layer */ H5I_VOL, /* type ID for virtual object layer */ @@ -578,7 +578,7 @@ H5Lcreate_hard(hid_t cur_loc_id, const char *cur_name, tmp_vol_obj.connector = (vol_obj1 != NULL ? vol_obj1->connector : vol_obj2->connector); /* Create the link */ - if(H5VL_link_create(H5VL_LINK_CREATE_HARD, &tmp_vol_obj, &loc_params2, lcpl_id, lapl_id, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, (vol_obj1 ? vol_obj1->data : NULL), loc_params1) < 0) + if(H5VL_link_create(H5VL_LINK_CREATE_HARD, &tmp_vol_obj, &loc_params2, lcpl_id, lapl_id, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, (vol_obj1 ? vol_obj1->data : NULL), &loc_params1) < 0) HGOTO_ERROR(H5E_LINK, H5E_CANTCREATE, FAIL, "unable to create hard link") done: diff --git a/src/H5M.c b/src/H5M.c new file mode 100644 index 0000000..adb4df0 --- /dev/null +++ b/src/H5M.c @@ -0,0 +1,1032 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/****************/ +/* Module Setup */ +/****************/ + +#include "H5Mmodule.h" /* This source code file is part of the H5M module */ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5CXprivate.h" /* API Contexts */ +#include "H5Mpkg.h" /* Maps */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Iprivate.h" /* IDs */ +#include "H5VLprivate.h" /* Virtual Object Layer */ + + + +/****************/ +/* Local Macros */ +/****************/ + + +/******************/ +/* Local Typedefs */ +/******************/ + + +/********************/ +/* Local Prototypes */ +/********************/ + +static herr_t H5M__close_cb(H5VL_object_t *map_vol_obj); + + +/*********************/ +/* Package Variables */ +/*********************/ + +/* Package initialization variable */ +hbool_t H5_PKG_INIT_VAR = FALSE; + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + +/* Map ID class */ +static const H5I_class_t H5I_MAP_CLS[1] = {{ + H5I_MAP, /* ID class value */ + 0, /* Class flags */ + 0, /* # of reserved IDs for class */ + (H5I_free_t)H5M__close_cb /* Callback routine for closing objects of this class */ +}}; + +/* Flag indicating "top" of interface has been initialized */ +static hbool_t H5M_top_package_initialize_s = FALSE; + + + +/*------------------------------------------------------------------------- +NAME + H5M__init_package -- Initialize interface-specific information +USAGE + herr_t H5M__init_package() + +RETURNS + Non-negative on success/Negative on failure +DESCRIPTION + Initializes any interface-specific data or routines. +--------------------------------------------------------------------------- +*/ +herr_t +H5M__init_package(void) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Initialize the atom group for the map IDs */ + if(H5I_register_type(H5I_MAP_CLS) < 0) + HGOTO_ERROR(H5E_MAP, H5E_CANTINIT, FAIL, "unable to initialize interface") + + /* Mark "top" of interface as initialized, too */ + H5M_top_package_initialize_s = TRUE; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5M__init_package() */ + + +/*------------------------------------------------------------------------- + * Function: H5M_top_term_package + * + * Purpose: Close the "top" of the interface, releasing IDs, etc. + * + * Return: Success: Positive if anything was done that might + * affect other interfaces; zero otherwise. + * Failure: Negative. + *------------------------------------------------------------------------- + */ +int +H5M_top_term_package(void) +{ + int n = 0; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + if(H5M_top_package_initialize_s) { + if(H5I_nmembers(H5I_MAP) > 0) { + (void)H5I_clear_type(H5I_MAP, FALSE, FALSE); + n++; /*H5I*/ + } /* end if */ + + /* Mark closed */ + if(0 == n) + H5M_top_package_initialize_s = FALSE; + } /* end if */ + + FUNC_LEAVE_NOAPI(n) +} /* end H5M_top_term_package() */ + + +/*------------------------------------------------------------------------- + * Function: H5M_term_package + * + * Purpose: Terminate this interface. + * + * Note: Finishes shutting down the interface, after + * H5M_top_term_package() is called + * + * Return: Success: Positive if anything was done that might + * affect other interfaces; zero otherwise. + * Failure: Negative. + *------------------------------------------------------------------------- + */ +int +H5M_term_package(void) +{ + int n = 0; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + if(H5_PKG_INIT_VAR) { + /* Sanity checks */ + HDassert(0 == H5I_nmembers(H5I_MAP)); + HDassert(FALSE == H5M_top_package_initialize_s); + + /* Destroy the dataset object id group */ + n += (H5I_dec_type_ref(H5I_MAP) > 0); + + /* Mark closed */ + if(0 == n) + H5_PKG_INIT_VAR = FALSE; + } /* end if */ + + FUNC_LEAVE_NOAPI(n) +} /* end H5M_term_package() */ + + +/*------------------------------------------------------------------------- + * Function: H5M__close_cb + * + * Purpose: Called when the ref count reaches zero on the map's ID + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5M__close_cb(H5VL_object_t *map_vol_obj) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(map_vol_obj); + + /* Close the map */ + if(H5VL_optional(map_vol_obj, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, H5VL_MAP_CLOSE) < 0) + HGOTO_ERROR(H5E_MAP, H5E_CLOSEERROR, FAIL, "unable to close map"); + + /* Free the VOL object */ + if(H5VL_free_object(map_vol_obj) < 0) + HGOTO_ERROR(H5E_MAP, H5E_CANTDEC, FAIL, "unable to free VOL object"); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5M__close_cb() */ + +#ifdef H5_HAVE_MAP_API + + +/*------------------------------------------------------------------------- + * Function: H5Mcreate + * + * Purpose: Creates a new map object for storing key-value pairs. The + * in-file datatype for keys is defined by KEY_TYPE_ID and + * the in-file datatype for values is defined by VAL_TYPE_ID. + * LOC_ID specifies the location to create the map object and + * NAME specifies the name of the link to the object + * (relative to LOC_ID). Other options can be specified + * through the property lists LCPL_ID, MCPL_ID, and MAPL_ID. + * + * Return: Success: The object ID of the new map. + * + * Failure: H5I_INVALID_HID + * + *------------------------------------------------------------------------- + */ +hid_t +H5Mcreate(hid_t loc_id, const char *name, hid_t key_type_id, hid_t val_type_id, + hid_t lcpl_id, hid_t mcpl_id, hid_t mapl_id) +{ + void *map = NULL; /* New map's info */ + H5VL_object_t *vol_obj = NULL; /* object token of loc_id */ + H5VL_loc_params_t loc_params; + hid_t ret_value = H5I_INVALID_HID; /* Return value */ + + FUNC_ENTER_API(H5I_INVALID_HID) + H5TRACE7("i", "i*siiiii", loc_id, name, key_type_id, val_type_id, lcpl_id, + mcpl_id, mapl_id); + + /* Check arguments */ + if(!name) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, H5I_INVALID_HID, "name parameter cannot be NULL") + if(!*name) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, H5I_INVALID_HID, "name parameter cannot be an empty string") + + /* Get link creation property list */ + if(H5P_DEFAULT == lcpl_id) + lcpl_id = H5P_LINK_CREATE_DEFAULT; + else + if(TRUE != H5P_isa_class(lcpl_id, H5P_LINK_CREATE)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5I_INVALID_HID, "lcpl_id is not a link creation property list") + + /* Get map creation property list */ + if(H5P_DEFAULT == mcpl_id) + mcpl_id = H5P_MAP_CREATE_DEFAULT; + else + if(TRUE != H5P_isa_class(mcpl_id, H5P_MAP_CREATE)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5I_INVALID_HID, "mcpl_id is not a map create property list ID") + + /* Verify access property list and set up collective metadata if appropriate */ + if(H5CX_set_apl(&mapl_id, H5P_CLS_MACC, loc_id, TRUE) < 0) + HGOTO_ERROR(H5E_MAP, H5E_CANTSET, H5I_INVALID_HID, "can't set access property list info") + + /* Get the location object */ + if(NULL == (vol_obj = (H5VL_object_t *)H5I_object(loc_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5I_INVALID_HID, "invalid location identifier") + + /* Set location parameters */ + loc_params.type = H5VL_OBJECT_BY_SELF; + loc_params.obj_type = H5I_get_type(loc_id); + + /* Create the map */ + if(H5VL_optional(vol_obj, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, H5VL_MAP_CREATE, &loc_params, name, lcpl_id, + key_type_id, val_type_id, mcpl_id, mapl_id, &map) < 0) + HGOTO_ERROR(H5E_MAP, H5E_CANTINIT, H5I_INVALID_HID, "unable to create map") + + /* Get an atom for the map */ + if((ret_value = H5VL_register(H5I_MAP, map, vol_obj->connector, TRUE)) < 0) + HGOTO_ERROR(H5E_MAP, H5E_CANTREGISTER, H5I_INVALID_HID, "unable to atomize map handle") + +done: + /* Cleanup on failure */ + if(H5I_INVALID_HID == ret_value) + if(map && H5VL_optional(vol_obj, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, H5VL_MAP_CLOSE) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, H5I_INVALID_HID, "unable to release map") + + FUNC_LEAVE_API(ret_value) +} /* end H5Mcreate() */ + + +/*------------------------------------------------------------------------- + * Function: H5Mcreate_anon + * + * Purpose: Creates a new map object for storing key-value pairs. The + * in-file datatype for keys is defined by KEY_TYPE_ID and + * the in-file datatype for values is defined by VAL_TYPE_ID. + * LOC_ID specifies the file to create the map object, but no + * link to the object is created. Other options can be + * specified through the property lists LCPL_ID, MCPL_ID, and + * MAPL_ID. + * + * The resulting ID should be linked into the file with + * H5Olink or it will be deleted when closed. + * + * Return: Success: The object ID of the new map. The map should + * be linked into the group hierarchy before being closed or + * it will be deleted. The dataset should be + * closed when the caller is no longer interested + * in it. + * + * Failure: H5I_INVALID_HID + * + *------------------------------------------------------------------------- + */ +hid_t +H5Mcreate_anon(hid_t loc_id, hid_t key_type_id, hid_t val_type_id, + hid_t mcpl_id, hid_t mapl_id) +{ + void *map = NULL; /* map token from VOL connector */ + H5VL_object_t *vol_obj = NULL; /* object token of loc_id */ + H5VL_loc_params_t loc_params; + hid_t ret_value = H5I_INVALID_HID; /* Return value */ + + FUNC_ENTER_API(H5I_INVALID_HID) + H5TRACE5("i", "iiiii", loc_id, key_type_id, val_type_id, mcpl_id, mapl_id); + + /* Check arguments */ + if(H5P_DEFAULT == mcpl_id) + mcpl_id = H5P_MAP_CREATE_DEFAULT; + else + if(TRUE != H5P_isa_class(mcpl_id, H5P_MAP_CREATE)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5I_INVALID_HID, "not map create property list ID") + + /* Verify access property list and set up collective metadata if appropriate */ + if(H5CX_set_apl(&mapl_id, H5P_CLS_MACC, loc_id, TRUE) < 0) + HGOTO_ERROR(H5E_MAP, H5E_CANTSET, H5I_INVALID_HID, "can't set access property list info") + + /* get the location object */ + if(NULL == (vol_obj = (H5VL_object_t *)H5I_object(loc_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5I_INVALID_HID, "invalid location identifier") + + /* Set location parameters */ + loc_params.type = H5VL_OBJECT_BY_SELF; + loc_params.obj_type = H5I_get_type(loc_id); + + /* Create the map */ + if(H5VL_optional(vol_obj, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, H5VL_MAP_CREATE, &loc_params, NULL, H5P_LINK_CREATE_DEFAULT, + key_type_id, val_type_id, mcpl_id, mapl_id, &map) < 0) + HGOTO_ERROR(H5E_MAP, H5E_CANTINIT, H5I_INVALID_HID, "unable to create map") + + /* Get an atom for the map */ + if((ret_value = H5VL_register(H5I_MAP, map, vol_obj->connector, TRUE)) < 0) + HGOTO_ERROR(H5E_MAP, H5E_CANTREGISTER, H5I_INVALID_HID, "unable to register map") + +done: + /* Cleanup on failure */ + if(H5I_INVALID_HID == ret_value) + if(map && H5VL_optional(vol_obj, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, H5VL_MAP_CLOSE) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, H5I_INVALID_HID, "unable to release map") + + FUNC_LEAVE_API(ret_value) +} /* end H5Mcreate_anon() */ + + +/*------------------------------------------------------------------------ + * Function: H5Mopen + * + * Purpose: Finds a map named NAME at LOC_ID, opens it, and returns + * its ID. The map should be close when the caller is no + * longer interested in it. + * + * Takes a map access property list + * + * Return: Success: Object ID of the map + * + * Failure: H5I_INVALID_HID + * + *------------------------------------------------------------------------- + */ +hid_t +H5Mopen(hid_t loc_id, const char *name, hid_t mapl_id) +{ + void *map = NULL; /* map token from VOL connector */ + H5VL_object_t *vol_obj = NULL; /* object token of loc_id */ + H5VL_loc_params_t loc_params; + hid_t ret_value = H5I_INVALID_HID; /* Return value */ + + FUNC_ENTER_API(H5I_INVALID_HID) + H5TRACE3("i", "i*si", loc_id, name, mapl_id); + + /* Check args */ + if(!name) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, H5I_INVALID_HID, "name parameter cannot be NULL") + if(!*name) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, H5I_INVALID_HID, "name parameter cannot be an empty string") + + /* Verify access property list and set up collective metadata if appropriate */ + if(H5CX_set_apl(&mapl_id, H5P_CLS_MACC, loc_id, FALSE) < 0) + HGOTO_ERROR(H5E_MAP, H5E_CANTSET, H5I_INVALID_HID, "can't set access property list info") + + /* get the location object */ + if(NULL == (vol_obj = (H5VL_object_t *)H5I_object(loc_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5I_INVALID_HID, "invalid location identifier") + + /* Set the location parameters */ + loc_params.type = H5VL_OBJECT_BY_SELF; + loc_params.obj_type = H5I_get_type(loc_id); + + /* Open the map */ + if(H5VL_optional(vol_obj, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, H5VL_MAP_OPEN, &loc_params, name, mapl_id, &map) < 0) + HGOTO_ERROR(H5E_MAP, H5E_CANTOPENOBJ, H5I_INVALID_HID, "unable to open map") + + /* Register an atom for the map */ + if((ret_value = H5VL_register(H5I_MAP, map, vol_obj->connector, TRUE)) < 0) + HGOTO_ERROR(H5E_MAP, H5E_CANTREGISTER, H5I_INVALID_HID, "can't register map atom") + +done: + /* Cleanup on failure */ + if(H5I_INVALID_HID == ret_value) + if(map && H5VL_optional(vol_obj, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, H5VL_MAP_CLOSE) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, H5I_INVALID_HID, "unable to release map") + + FUNC_LEAVE_API(ret_value) +} /* end H5Mopen() */ + + +/*------------------------------------------------------------------------- + * Function: H5Mclose + * + * Purpose: Closes access to a map and releases resources used by it. + * It is illegal to subsequently use that same map ID in + * calls to other map functions. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5Mclose(hid_t map_id) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE1("e", "i", map_id); + + /* Check args */ + if(H5I_MAP != H5I_get_type(map_id)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a map ID") + + /* Decrement the counter on the map. It will be freed if the count + * reaches zero. + */ + if(H5I_dec_app_ref_always_close(map_id) < 0) + HGOTO_ERROR(H5E_MAP, H5E_CANTDEC, FAIL, "can't decrement count on map ID") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Mclose() */ + + +/*------------------------------------------------------------------------- + * Function: H5Mget_key_type + * + * Purpose: Returns a copy of the key datatype for a map. + * + * Return: Success: ID for a copy of the datatype. The data + * type should be released by calling + * H5Tclose(). + * + * Failure: H5I_INVALID_HID + * + *------------------------------------------------------------------------- + */ +hid_t +H5Mget_key_type(hid_t map_id) +{ + H5VL_object_t *vol_obj; /* Map structure */ + hid_t ret_value = H5I_INVALID_HID; /* Return value */ + + FUNC_ENTER_API(H5I_INVALID_HID) + H5TRACE1("i", "i", map_id); + + /* Check args */ + if(NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(map_id, H5I_MAP))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5I_INVALID_HID, "invalid map identifier") + + /* get the datatype */ + if(H5VL_optional(vol_obj, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, H5VL_MAP_GET, H5VL_MAP_GET_KEY_TYPE, &ret_value) < 0) + HGOTO_ERROR(H5E_MAP, H5E_CANTGET, H5I_INVALID_HID, "unable to get datatype") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Mget_key_type() */ + + +/*------------------------------------------------------------------------- + * Function: H5Mget_val_type + * + * Purpose: Returns a copy of the value datatype for a map. + * + * Return: Success: ID for a copy of the datatype. The data + * type should be released by calling + * H5Tclose(). + * + * Failure: H5I_INVALID_HID + * + *------------------------------------------------------------------------- + */ +hid_t +H5Mget_val_type(hid_t map_id) +{ + H5VL_object_t *vol_obj; /* Map structure */ + hid_t ret_value = H5I_INVALID_HID; /* Return value */ + + FUNC_ENTER_API(H5I_INVALID_HID) + H5TRACE1("i", "i", map_id); + + /* Check args */ + if(NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(map_id, H5I_MAP))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5I_INVALID_HID, "invalid map identifier") + + /* get the datatype */ + if(H5VL_optional(vol_obj, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, H5VL_MAP_GET, H5VL_MAP_GET_VAL_TYPE, &ret_value) < 0) + HGOTO_ERROR(H5E_MAP, H5E_CANTGET, H5I_INVALID_HID, "unable to get datatype") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Mget_val_type() */ + + +/*------------------------------------------------------------------------- + * Function: H5Mget_create_plist + * + * Purpose: Returns a copy of the map creation property list. + * + * Return: Success: ID for a copy of the map creation + * property list. The template should be + * released by calling H5P_close(). + * + * Failure: H5I_INVALID_HID + * + *------------------------------------------------------------------------- + */ +hid_t +H5Mget_create_plist(hid_t map_id) +{ + H5VL_object_t *vol_obj; /* Map structure */ + hid_t ret_value = H5I_INVALID_HID; /* Return value */ + + FUNC_ENTER_API(H5I_INVALID_HID) + H5TRACE1("i", "i", map_id); + + /* Check args */ + if(NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(map_id, H5I_MAP))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5I_INVALID_HID, "invalid map identifier") + + /* Get the map creation property list */ + if(H5VL_optional(vol_obj, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, H5VL_MAP_GET, H5VL_MAP_GET_MCPL, &ret_value) < 0) + HGOTO_ERROR(H5E_MAP, H5E_CANTGET, H5I_INVALID_HID, "unable to get map creation properties") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Mget_create_plist() */ + + +/*------------------------------------------------------------------------- + * Function: H5Mget_access_plist + * + * Purpose: Returns a copy of the map access property list. + * + * Description: H5Mget_access_plist returns the map access property + * list identifier of the specified map. + * + * Return: Success: ID for a copy of the map access + * property list. The template should be + * released by calling H5Pclose(). + * + * Failure: H5I_INVALID_HID + * + *------------------------------------------------------------------------- + */ +hid_t +H5Mget_access_plist(hid_t map_id) +{ + H5VL_object_t *vol_obj; /* Map structure */ + hid_t ret_value = H5I_INVALID_HID; /* Return value */ + + FUNC_ENTER_API(H5I_INVALID_HID) + H5TRACE1("i", "i", map_id); + + /* Check args */ + if(NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(map_id, H5I_MAP))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5I_INVALID_HID, "invalid map identifier") + + /* Get the map access property list */ + if(H5VL_optional(vol_obj, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, H5VL_MAP_GET, H5VL_MAP_GET_MAPL, &ret_value) < 0) + HGOTO_ERROR(H5E_MAP, H5E_CANTGET, H5I_INVALID_HID, "unable to get map access properties") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Mget_access_plist() */ + + +/*------------------------------------------------------------------------- + * Function: H5Mget_count + * + * Purpose: Returns the number of key-value pairs stored in the map. + * + * Description: H5Mget_count returns the number of key-value pairs stored + * in the specified map. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5Mget_count(hid_t map_id, hsize_t *count, hid_t dxpl_id) +{ + H5VL_object_t *vol_obj; /* Map structure */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(H5I_INVALID_HID) + H5TRACE3("e", "i*hi", map_id, count, dxpl_id); + + /* Check args */ + if(NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(map_id, H5I_MAP))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5I_INVALID_HID, "invalid map identifier") + + /* Get the default dataset transfer property list if the user didn't provide one */ + if (H5P_DEFAULT == dxpl_id) + dxpl_id = H5P_DATASET_XFER_DEFAULT; + else + if (TRUE != H5P_isa_class(dxpl_id, H5P_DATASET_XFER)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not xfer parms") + + /* Set DXPL for operation */ + H5CX_set_dxpl(dxpl_id); + + /* Get the number of key-value pairs stored in the map */ + if(H5VL_optional(vol_obj, dxpl_id, H5_REQUEST_NULL, H5VL_MAP_GET, H5VL_MAP_GET_COUNT, count) < 0) + HGOTO_ERROR(H5E_MAP, H5E_CANTGET, H5I_INVALID_HID, "unable to get map access properties") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Mget_count() */ + + +/*------------------------------------------------------------------------- + * Function: H5Mput + * + * Purpose: H5Mput adds a key-value pair to the Map specified by + * MAP_ID, or updates the value for the specified key if one + * was set previously. KEY_MEM_TYPE_ID and VAL_MEM_TYPE_ID + * specify the datatypes for the provided KEY and VALUE + * buffers, and if different from those used to create the + * map object, the key and value will be internally converted + * to the datatypes for the map object. Any further options + * can be specified through the property list DXPL_ID. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5Mput(hid_t map_id, hid_t key_mem_type_id, const void *key, + hid_t val_mem_type_id, const void *value, hid_t dxpl_id) +{ + H5VL_object_t *vol_obj = NULL; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE6("e", "ii*xi*xi", map_id, key_mem_type_id, key, val_mem_type_id, value, + dxpl_id); + + /* Check arguments */ + if (key_mem_type_id < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid key memory datatype ID") + if (val_mem_type_id < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid value memory datatype ID") + + /* Get map pointer */ + if (NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(map_id, H5I_MAP))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "map_id is not a map ID") + + /* Get the default dataset transfer property list if the user didn't provide one */ + if (H5P_DEFAULT == dxpl_id) + dxpl_id = H5P_DATASET_XFER_DEFAULT; + else + if (TRUE != H5P_isa_class(dxpl_id, H5P_DATASET_XFER)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not xfer parms") + + /* Set DXPL for operation */ + H5CX_set_dxpl(dxpl_id); + + /* Set the key/value pair */ + if(H5VL_optional(vol_obj, dxpl_id, H5_REQUEST_NULL, H5VL_MAP_PUT, key_mem_type_id, key, val_mem_type_id, value) < 0) + HGOTO_ERROR(H5E_MAP, H5E_CANTSET, FAIL, "unable to put key/value pair") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Mput() */ + + +/*------------------------------------------------------------------------- + * Function: H5Mget + * + * Purpose: H5Mget retrieves, from the Map specified by MAP_ID, the + * value associated with the provided key. KEY_MEM_TYPE_ID + * and VAL_MEM_TYPE_ID specify the datatypes for the provided + * KEY and VALUE buffers. If KEY_MEM_TYPE_ID is different + * from that used to create the map object, the key will be + * internally converted to the datatype for the map object + * for the query, and if VAL_MEM_TYPE_ID is different from + * that used to create the map object, the returned value + * will be converted to VAL_MEM_TYPE_ID before the function + * returns. Any further options can be specified through the + * property list DXPL_ID. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5Mget(hid_t map_id, hid_t key_mem_type_id, const void *key, + hid_t val_mem_type_id, void *value, hid_t dxpl_id) +{ + H5VL_object_t *vol_obj = NULL; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE6("e", "ii*xi*xi", map_id, key_mem_type_id, key, val_mem_type_id, value, + dxpl_id); + + /* Check arguments */ + if (key_mem_type_id < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid key memory datatype ID") + if (val_mem_type_id < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid value memory datatype ID") + + /* Get map pointer */ + if (NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(map_id, H5I_MAP))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "map_id is not a map ID") + + /* Get the default dataset transfer property list if the user didn't provide one */ + if (H5P_DEFAULT == dxpl_id) + dxpl_id = H5P_DATASET_XFER_DEFAULT; + else + if (TRUE != H5P_isa_class(dxpl_id, H5P_DATASET_XFER)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not xfer parms") + + /* Set DXPL for operation */ + H5CX_set_dxpl(dxpl_id); + + /* Get the value for the key */ + if(H5VL_optional(vol_obj, dxpl_id, H5_REQUEST_NULL, H5VL_MAP_GET_VAL, key_mem_type_id, key, val_mem_type_id, value) < 0) + HGOTO_ERROR(H5E_MAP, H5E_CANTGET, FAIL, "unable to get value from map") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Mget() */ + + +/*------------------------------------------------------------------------- + * Function: H5Mexists + * + * Purpose: H5Mexists checks if the provided key is stored in the map + * specified by MAP_ID. If KEY_MEM_TYPE_ID is different from + * that used to create the map object the key will be + * internally converted to the datatype for the map object + * for the query. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5Mexists(hid_t map_id, hid_t key_mem_type_id, const void *key, hbool_t *exists, + hid_t dxpl_id) +{ + H5VL_object_t *vol_obj = NULL; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE5("e", "ii*x*bi", map_id, key_mem_type_id, key, exists, dxpl_id); + + /* Check arguments */ + if (key_mem_type_id < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid key memory datatype ID") + + /* Get map pointer */ + if (NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(map_id, H5I_MAP))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "map_id is not a map ID") + + /* Get the default dataset transfer property list if the user didn't provide one */ + if (H5P_DEFAULT == dxpl_id) + dxpl_id = H5P_DATASET_XFER_DEFAULT; + else + if (TRUE != H5P_isa_class(dxpl_id, H5P_DATASET_XFER)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not xfer parms") + + /* Set DXPL for operation */ + H5CX_set_dxpl(dxpl_id); + + /* Check if key exists */ + if((ret_value = H5VL_optional(vol_obj, dxpl_id, H5_REQUEST_NULL, H5VL_MAP_EXISTS, key_mem_type_id, key, exists)) < 0) + HGOTO_ERROR(H5E_MAP, H5E_CANTGET, ret_value, "unable to check if key exists") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Mexists() */ + + +/*------------------------------------------------------------------------- + * Function: H5Miterate + * + * Purpose: H5Miterate iterates over all key-value pairs stored in the + * map specified by MAP_ID, making the callback specified by + * OP for each. The IDX parameter is an in/out parameter that + * may be used to restart a previously interrupted iteration. + * At the start of iteration IDX should be set to 0, and to + * restart iteration at the same location on a subsequent + * call to H5Miterate, IDX should be the same value as + * returned by the previous call. + * + * H5M_iterate_t is defined as: + * herr_t (*H5M_iterate_t)(hid_t map_id, const void *key, + * void *ctx) + * + * The KEY parameter is the buffer for the key for this + * iteration, converted to the datatype specified by + * KEY_MEM_TYPE_ID. The OP_DATA parameter is a simple pass + * through of the value passed to H5Miterate, which can be + * used to store application-defined data for iteration. A + * negative return value from this function will cause + * H5Miterate to issue an error, while a positive return + * value will cause H5Miterate to stop iterating and return + * this value without issuing an error. A return value of + * zero allows iteration to continue. + * + * Return: Last value returned by op + * + *------------------------------------------------------------------------- + */ +herr_t +H5Miterate(hid_t map_id, hsize_t *idx, hid_t key_mem_type_id, H5M_iterate_t op, + void *op_data, hid_t dxpl_id) +{ + H5VL_object_t *vol_obj = NULL; + H5VL_loc_params_t loc_params; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE6("e", "i*hix*xi", map_id, idx, key_mem_type_id, op, op_data, dxpl_id); + + /* Check arguments */ + if (key_mem_type_id < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid key memory datatype ID") + if (!op) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no operator specified") + + /* Get map pointer */ + if (NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(map_id, H5I_MAP))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "map_id is not a map ID") + + /* Get the default dataset transfer property list if the user didn't provide one */ + if (H5P_DEFAULT == dxpl_id) + dxpl_id = H5P_DATASET_XFER_DEFAULT; + else + if (TRUE != H5P_isa_class(dxpl_id, H5P_DATASET_XFER)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not xfer parms") + + /* Set DXPL for operation */ + H5CX_set_dxpl(dxpl_id); + + /* Set location struct fields */ + loc_params.type = H5VL_OBJECT_BY_SELF; + loc_params.obj_type = H5I_get_type(map_id); + + /* Iterate over keys */ + if((ret_value = H5VL_optional(vol_obj, dxpl_id, H5_REQUEST_NULL, H5VL_MAP_SPECIFIC, &loc_params, H5VL_MAP_ITER, idx, key_mem_type_id, op, op_data)) < 0) + HGOTO_ERROR(H5E_MAP, H5E_BADITER, ret_value, "unable to ierate over keys") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Miterate() */ + + +/*------------------------------------------------------------------------- + * Function: H5Miterate_by_name + * + * Purpose: H5Miterate_by_name iterates over all key-value pairs + * stored in the map specified by MAP_ID, making the callback + * specified by OP for each. The IDX parameter is an in/out + * parameter that may be used to restart a previously + * interrupted iteration. At the start of iteration IDX + * should be set to 0, and to restart iteration at the same + * location on a subsequent call to H5Miterate, IDX should be + * the same value as returned by the previous call. + * + * H5M_iterate_t is defined as: + * herr_t (*H5M_iterate_t)(hid_t map_id, const void *key, + * void *ctx) + * + * The KEY parameter is the buffer for the key for this + * iteration, converted to the datatype specified by + * KEY_MEM_TYPE_ID. The OP_DATA parameter is a simple pass + * through of the value passed to H5Miterate, which can be + * used to store application-defined data for iteration. A + * negative return value from this function will cause + * H5Miterate to issue an error, while a positive return + * value will cause H5Miterate to stop iterating and return + * this value without issuing an error. A return value of + * zero allows iteration to continue. + * + * Return: Last value returned by op + * + *------------------------------------------------------------------------- + */ +herr_t +H5Miterate_by_name(hid_t loc_id, const char *map_name, hsize_t *idx, + hid_t key_mem_type_id, H5M_iterate_t op, void *op_data, hid_t dxpl_id, + hid_t lapl_id) +{ + H5VL_object_t *vol_obj = NULL; + H5VL_loc_params_t loc_params; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE8("e", "i*s*hix*xii", loc_id, map_name, idx, key_mem_type_id, op, + op_data, dxpl_id, lapl_id); + + /* Check arguments */ + if(!map_name) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "map_name parameter cannot be NULL") + if(!*map_name) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "map_name parameter cannot be an empty string") + if (key_mem_type_id < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid key memory datatype ID") + if (!op) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no operator specified") + + /* Get the location object */ + if (NULL == (vol_obj = (H5VL_object_t *)H5I_object(loc_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid location identifier") + + /* Get the default dataset transfer property list if the user didn't provide one */ + if (H5P_DEFAULT == dxpl_id) + dxpl_id = H5P_DATASET_XFER_DEFAULT; + else + if (TRUE != H5P_isa_class(dxpl_id, H5P_DATASET_XFER)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not xfer parms") + + /* Set DXPL for operation */ + H5CX_set_dxpl(dxpl_id); + + /* Set location struct fields */ + loc_params.type = H5VL_OBJECT_BY_NAME; + loc_params.obj_type = H5I_get_type(loc_id); + loc_params.loc_data.loc_by_name.name = map_name; + loc_params.loc_data.loc_by_name.lapl_id = lapl_id; + + /* Iterate over keys */ + if((ret_value = H5VL_optional(vol_obj, dxpl_id, H5_REQUEST_NULL, H5VL_MAP_SPECIFIC, &loc_params, H5VL_MAP_ITER, idx, key_mem_type_id, op, op_data)) < 0) + HGOTO_ERROR(H5E_MAP, H5E_BADITER, ret_value, "unable to ierate over keys") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Miterate_by_name() */ + + +/*------------------------------------------------------------------------- + * Function: H5Mdelete + * + * Purpose: H5Mdelete deletes a key-value pair from the Map + * specified by MAP_ID. KEY_MEM_TYPE_ID specifies the + * datatype for the provided key buffers, and if different + * from that used to create the Map object, the key will be + * internally converted to the datatype for the map object. + * Any further options can be specified through the property + * list DXPL_ID. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5Mdelete(hid_t map_id, hid_t key_mem_type_id, const void *key, + hid_t dxpl_id) +{ + H5VL_object_t *vol_obj = NULL; + H5VL_loc_params_t loc_params; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE4("e", "ii*xi", map_id, key_mem_type_id, key, dxpl_id); + + /* Check arguments */ + if (key_mem_type_id < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid key memory datatype ID") + + /* Get map pointer */ + if (NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(map_id, H5I_MAP))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "map_id is not a map ID") + + /* Get the default dataset transfer property list if the user didn't provide one */ + if (H5P_DEFAULT == dxpl_id) + dxpl_id = H5P_DATASET_XFER_DEFAULT; + else + if (TRUE != H5P_isa_class(dxpl_id, H5P_DATASET_XFER)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not xfer parms") + + /* Set DXPL for operation */ + H5CX_set_dxpl(dxpl_id); + + /* Set location struct fields */ + loc_params.type = H5VL_OBJECT_BY_SELF; + loc_params.obj_type = H5I_get_type(map_id); + + /* Delete the key/value pair */ + if(H5VL_optional(vol_obj, dxpl_id, H5_REQUEST_NULL, H5VL_MAP_SPECIFIC, &loc_params, H5VL_MAP_DELETE, key_mem_type_id, key) < 0) + HGOTO_ERROR(H5E_MAP, H5E_CANTSET, FAIL, "unable to delete key/value pair") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Mdelete() */ + +#endif /* H5_HAVE_MAP_API */ + @@ -55,6 +55,11 @@ if(!H5F_addr_defined(FSM->addr) || !H5F_addr_defined(FSM->sect_addr)) \ *CF = TRUE; +/* For non-paged aggregation: map allocation request type to tracked free-space type */ +/* F_SH -- pointer to H5F_shared_t; T -- H5FD_mem_t */ +#define H5MF_ALLOC_TO_FS_AGGR_TYPE(F_SH, T) \ + ((H5FD_MEM_DEFAULT == (F_SH)->fs_type_map[T]) ? (T) : (F_SH)->fs_type_map[T]) + /******************/ /* Local Typedefs */ /******************/ @@ -93,9 +98,9 @@ static herr_t H5MF__close_shrink_eoa(H5F_t *f); /* General routines */ static herr_t H5MF__get_free_sects(H5F_t *f, H5FS_t *fspace, H5MF_sect_iter_ud_t *sect_udata, size_t *nums); -static hbool_t H5MF__fsm_type_is_self_referential(H5F_t *f, H5F_mem_page_t fsm_type); -static hbool_t H5MF__fsm_is_self_referential(H5F_t *f, H5FS_t *fspace); -static herr_t H5MF__continue_alloc_fsm(H5F_t *f, H5FS_t *sm_hdr_fspace, H5FS_t *sm_sinfo_fspace, +static hbool_t H5MF__fsm_type_is_self_referential(H5F_shared_t *f_sh, H5F_mem_page_t fsm_type); +static hbool_t H5MF__fsm_is_self_referential(H5F_shared_t *f_sh, H5FS_t *fspace); +static herr_t H5MF__continue_alloc_fsm(H5F_shared_t *f_sh, H5FS_t *sm_hdr_fspace, H5FS_t *sm_sinfo_fspace, H5FS_t *lg_hdr_fspace, H5FS_t *lg_sinfo_fspace, hbool_t *continue_alloc_fsm); /* Free-space type manager routines */ @@ -141,7 +146,7 @@ hbool_t H5_PKG_INIT_VAR = FALSE; *------------------------------------------------------------------------- */ herr_t -H5MF_init_merge_flags(H5F_t *f) +H5MF_init_merge_flags(H5F_shared_t *f_sh) { H5MF_aggr_merge_t mapping_type; /* Type of free list mapping */ H5FD_mem_t type; /* Memory type for iteration */ @@ -151,9 +156,8 @@ H5MF_init_merge_flags(H5F_t *f) FUNC_ENTER_NOAPI(FAIL) /* check args */ - HDassert(f); - HDassert(f->shared); - HDassert(f->shared->lf); + HDassert(f_sh); + HDassert(f_sh->lf); /* Iterate over all the free space types to determine if sections of that type * can merge with the metadata or small 'raw' data aggregator @@ -161,21 +165,21 @@ H5MF_init_merge_flags(H5F_t *f) all_same = TRUE; for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) /* Check for any different type mappings */ - if(f->shared->fs_type_map[type] != f->shared->fs_type_map[H5FD_MEM_DEFAULT]) { + if(f_sh->fs_type_map[type] != f_sh->fs_type_map[H5FD_MEM_DEFAULT]) { all_same = FALSE; break; } /* end if */ /* Check for all allocation types mapping to the same free list type */ if(all_same) { - if(f->shared->fs_type_map[H5FD_MEM_DEFAULT] == H5FD_MEM_DEFAULT) + if(f_sh->fs_type_map[H5FD_MEM_DEFAULT] == H5FD_MEM_DEFAULT) mapping_type = H5MF_AGGR_MERGE_SEPARATE; else mapping_type = H5MF_AGGR_MERGE_TOGETHER; } /* end if */ else { /* Check for raw data mapping into same list as metadata */ - if(f->shared->fs_type_map[H5FD_MEM_DRAW] == f->shared->fs_type_map[H5FD_MEM_SUPER]) + if(f_sh->fs_type_map[H5FD_MEM_DRAW] == f_sh->fs_type_map[H5FD_MEM_SUPER]) mapping_type = H5MF_AGGR_MERGE_SEPARATE; else { hbool_t all_metadata_same; /* Whether all metadata go in same free list */ @@ -188,7 +192,7 @@ H5MF_init_merge_flags(H5F_t *f) /* (global heap is treated as raw data) */ if(type != H5FD_MEM_DRAW && type != H5FD_MEM_GHEAP) { /* Check for any different type mappings */ - if(f->shared->fs_type_map[type] != f->shared->fs_type_map[H5FD_MEM_SUPER]) { + if(f_sh->fs_type_map[type] != f_sh->fs_type_map[H5FD_MEM_SUPER]) { all_metadata_same = FALSE; break; } /* end if */ @@ -206,30 +210,30 @@ H5MF_init_merge_flags(H5F_t *f) switch(mapping_type) { case H5MF_AGGR_MERGE_SEPARATE: /* Don't merge any metadata together */ - HDmemset(f->shared->fs_aggr_merge, 0, sizeof(f->shared->fs_aggr_merge)); + HDmemset(f_sh->fs_aggr_merge, 0, sizeof(f_sh->fs_aggr_merge)); /* Check if merging raw data should be allowed */ /* (treat global heaps as raw data) */ - if(H5FD_MEM_DRAW == f->shared->fs_type_map[H5FD_MEM_DRAW] || - H5FD_MEM_DEFAULT == f->shared->fs_type_map[H5FD_MEM_DRAW]) { - f->shared->fs_aggr_merge[H5FD_MEM_DRAW] = H5F_FS_MERGE_RAWDATA; - f->shared->fs_aggr_merge[H5FD_MEM_GHEAP] = H5F_FS_MERGE_RAWDATA; + if(H5FD_MEM_DRAW == f_sh->fs_type_map[H5FD_MEM_DRAW] || + H5FD_MEM_DEFAULT == f_sh->fs_type_map[H5FD_MEM_DRAW]) { + f_sh->fs_aggr_merge[H5FD_MEM_DRAW] = H5F_FS_MERGE_RAWDATA; + f_sh->fs_aggr_merge[H5FD_MEM_GHEAP] = H5F_FS_MERGE_RAWDATA; } /* end if */ break; case H5MF_AGGR_MERGE_DICHOTOMY: /* Merge all metadata together (but not raw data) */ - HDmemset(f->shared->fs_aggr_merge, H5F_FS_MERGE_METADATA, sizeof(f->shared->fs_aggr_merge)); + HDmemset(f_sh->fs_aggr_merge, H5F_FS_MERGE_METADATA, sizeof(f_sh->fs_aggr_merge)); /* Allow merging raw data allocations together */ /* (treat global heaps as raw data) */ - f->shared->fs_aggr_merge[H5FD_MEM_DRAW] = H5F_FS_MERGE_RAWDATA; - f->shared->fs_aggr_merge[H5FD_MEM_GHEAP] = H5F_FS_MERGE_RAWDATA; + f_sh->fs_aggr_merge[H5FD_MEM_DRAW] = H5F_FS_MERGE_RAWDATA; + f_sh->fs_aggr_merge[H5FD_MEM_GHEAP] = H5F_FS_MERGE_RAWDATA; break; case H5MF_AGGR_MERGE_TOGETHER: /* Merge all allocation types together */ - HDmemset(f->shared->fs_aggr_merge, (H5F_FS_MERGE_METADATA | H5F_FS_MERGE_RAWDATA), sizeof(f->shared->fs_aggr_merge)); + HDmemset(f_sh->fs_aggr_merge, (H5F_FS_MERGE_METADATA | H5F_FS_MERGE_RAWDATA), sizeof(f_sh->fs_aggr_merge)); break; default: @@ -254,31 +258,32 @@ done: *------------------------------------------------------------------------- */ void -H5MF__alloc_to_fs_type(H5F_t *f, H5FD_mem_t alloc_type, hsize_t size, H5F_mem_page_t *fs_type) +H5MF__alloc_to_fs_type(H5F_shared_t *f_sh, H5FD_mem_t alloc_type, hsize_t size, H5F_mem_page_t *fs_type) { FUNC_ENTER_PACKAGE_NOERR - HDassert(f); + /* Check arguments */ + HDassert(f_sh); HDassert(fs_type); - if(H5F_PAGED_AGGR(f)) { /* paged aggregation */ - if(size >= f->shared->fs_page_size) { - if(H5F_HAS_FEATURE(f, H5FD_FEAT_PAGED_AGGR)) { /* multi or split driver */ + if(H5F_SHARED_PAGED_AGGR(f_sh)) { /* paged aggregation */ + if(size >= f_sh->fs_page_size) { + if(H5F_SHARED_HAS_FEATURE(f_sh, H5FD_FEAT_PAGED_AGGR)) { /* multi or split driver */ /* For non-contiguous address space, map to large size free-space manager for each alloc_type */ - if(H5FD_MEM_DEFAULT == f->shared->fs_type_map[alloc_type]) - *fs_type = (H5F_mem_page_t) (alloc_type + (H5FD_MEM_NTYPES - 1)); + if(H5FD_MEM_DEFAULT == f_sh->fs_type_map[alloc_type]) + *fs_type = (H5F_mem_page_t)(alloc_type + (H5FD_MEM_NTYPES - 1)); else - *fs_type = (H5F_mem_page_t) (f->shared->fs_type_map[alloc_type] + (H5FD_MEM_NTYPES - 1)); + *fs_type = (H5F_mem_page_t)(f_sh->fs_type_map[alloc_type] + (H5FD_MEM_NTYPES - 1)); } /* end if */ else /* For contiguous address space, map to generic large size free-space manager */ *fs_type = H5F_MEM_PAGE_GENERIC; /* H5F_MEM_PAGE_SUPER */ } /* end if */ else - *fs_type = (H5F_mem_page_t)H5MF_ALLOC_TO_FS_AGGR_TYPE(f, alloc_type); + *fs_type = (H5F_mem_page_t)H5MF_ALLOC_TO_FS_AGGR_TYPE(f_sh, alloc_type); } /* end if */ else /* non-paged aggregation */ - *fs_type = (H5F_mem_page_t)H5MF_ALLOC_TO_FS_AGGR_TYPE(f, alloc_type); + *fs_type = (H5F_mem_page_t)H5MF_ALLOC_TO_FS_AGGR_TYPE(f_sh, alloc_type); FUNC_LEAVE_NOAPI_VOID } /* end H5MF__alloc_to_fs_type() */ @@ -340,7 +345,7 @@ H5MF__open_fstype(H5F_t *f, H5F_mem_page_t type) } /* end else */ /* Set the ring type in the API context */ - if(H5MF__fsm_type_is_self_referential(f, type)) + if(H5MF__fsm_type_is_self_referential(f->shared, type)) fsm_ring = H5AC_RING_MDFSM; else fsm_ring = H5AC_RING_RDFSM; @@ -428,7 +433,7 @@ H5MF__create_fstype(H5F_t *f, H5F_mem_page_t type) } /* end else */ /* Set the ring type in the API context */ - if(H5MF__fsm_type_is_self_referential(f, type)) + if(H5MF__fsm_type_is_self_referential(f->shared, type)) fsm_ring = H5AC_RING_MDFSM; else fsm_ring = H5AC_RING_RDFSM; @@ -544,7 +549,7 @@ H5MF__delete_fstype(H5F_t *f, H5F_mem_page_t type) f->shared->fs_state[type] = H5F_FS_STATE_DELETING; /* Set the ring type in the API context */ - if(H5MF__fsm_type_is_self_referential(f, type)) + if(H5MF__fsm_type_is_self_referential(f->shared, type)) fsm_ring = H5AC_RING_MDFSM; else fsm_ring = H5AC_RING_RDFSM; @@ -620,7 +625,6 @@ done: FUNC_LEAVE_NOAPI_TAG(ret_value) } /* end H5MF__close_fstype() */ - /*------------------------------------------------------------------------- * Function: H5MF__add_sect @@ -649,7 +653,7 @@ H5MF__add_sect(H5F_t *f, H5FD_mem_t alloc_type, H5FS_t *fspace, H5MF_free_sectio HDassert(fspace); HDassert(node); - H5MF__alloc_to_fs_type(f, alloc_type, node->sect_info.size, &fs_type); + H5MF__alloc_to_fs_type(f->shared, alloc_type, node->sect_info.size, &fs_type); /* Construct user data for callbacks */ udata.f = f; @@ -658,7 +662,7 @@ H5MF__add_sect(H5F_t *f, H5FD_mem_t alloc_type, H5FS_t *fspace, H5MF_free_sectio udata.allow_eoa_shrink_only = FALSE; /* Set the ring type in the API context */ - if(H5MF__fsm_is_self_referential(f, fspace)) + if(H5MF__fsm_is_self_referential(f->shared, fspace)) fsm_ring = H5AC_RING_MDFSM; else fsm_ring = H5AC_RING_RDFSM; @@ -708,7 +712,7 @@ H5MF__find_sect(H5F_t *f, H5FD_mem_t alloc_type, hsize_t size, H5FS_t *fspace, HDassert(fspace); /* Set the ring type in the API context */ - if(H5MF__fsm_is_self_referential(f, fspace)) + if(H5MF__fsm_is_self_referential(f->shared, fspace)) fsm_ring = H5AC_RING_MDFSM; else fsm_ring = H5AC_RING_RDFSM; @@ -801,14 +805,14 @@ HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_typ HDassert(f->shared->lf); HDassert(size > 0); - H5MF__alloc_to_fs_type(f, alloc_type, size, &fs_type); + H5MF__alloc_to_fs_type(f->shared, alloc_type, size, &fs_type); #ifdef H5MF_ALLOC_DEBUG_MORE HDfprintf(stderr, "%s: Check 1.0\n", FUNC); #endif /* H5MF_ALLOC_DEBUG_MORE */ /* Set the ring type in the API context */ - if(H5MF__fsm_type_is_self_referential(f, fs_type)) + if(H5MF__fsm_type_is_self_referential(f->shared, fs_type)) fsm_ring = H5AC_RING_MDFSM; else fsm_ring = H5AC_RING_RDFSM; @@ -907,15 +911,15 @@ H5MF__alloc_pagefs(H5F_t *f, H5FD_mem_t alloc_type, hsize_t size) HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_type, size); #endif /* H5MF_ALLOC_DEBUG */ - H5MF__alloc_to_fs_type(f, alloc_type, size, &ptype); + H5MF__alloc_to_fs_type(f->shared, alloc_type, size, &ptype); switch(ptype) { - case H5F_MEM_PAGE_GENERIC: - case H5F_MEM_PAGE_LARGE_BTREE: - case H5F_MEM_PAGE_LARGE_DRAW: - case H5F_MEM_PAGE_LARGE_GHEAP: - case H5F_MEM_PAGE_LARGE_LHEAP: - case H5F_MEM_PAGE_LARGE_OHDR: + case H5F_MEM_PAGE_GENERIC: + case H5F_MEM_PAGE_LARGE_BTREE: + case H5F_MEM_PAGE_LARGE_DRAW: + case H5F_MEM_PAGE_LARGE_GHEAP: + case H5F_MEM_PAGE_LARGE_LHEAP: + case H5F_MEM_PAGE_LARGE_OHDR: { haddr_t eoa; /* EOA for the file */ hsize_t frag_size = 0; /* Fragment size */ @@ -952,7 +956,7 @@ HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_typ } break; - case H5F_MEM_PAGE_META: + case H5F_MEM_PAGE_META: case H5F_MEM_PAGE_DRAW: case H5F_MEM_PAGE_BTREE: case H5F_MEM_PAGE_GHEAP: @@ -980,13 +984,13 @@ HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_typ node = NULL; - /* Insert the new page into the Page Buffer list of new pages so + /* Insert the new page into the Page Buffer list of new pages so we don't read an empty page from disk */ - if(f->shared->page_buf != NULL && H5PB_add_new_page(f, alloc_type, new_page) < 0) + if(f->shared->page_buf != NULL && H5PB_add_new_page(f->shared, alloc_type, new_page) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, HADDR_UNDEF, "can't add new page to Page Buffer new page list") ret_value = new_page; - } + } break; case H5F_MEM_PAGE_NTYPES: @@ -1108,10 +1112,10 @@ HDfprintf(stderr, "%s: Entering - alloc_type = %u, addr = %a, size = %Hu\n", FUN HGOTO_DONE(SUCCEED) HDassert(addr != 0); /* Can't deallocate the superblock :-) */ - H5MF__alloc_to_fs_type(f, alloc_type, size, &fs_type); + H5MF__alloc_to_fs_type(f->shared, alloc_type, size, &fs_type); /* Set the ring type in the API context */ - if(H5MF__fsm_type_is_self_referential(f, fs_type)) + if(H5MF__fsm_type_is_self_referential(f->shared, fs_type)) fsm_ring = H5AC_RING_MDFSM; else fsm_ring = H5AC_RING_RDFSM; @@ -1133,11 +1137,10 @@ HDfprintf(stderr, "%s: Entering - alloc_type = %u, addr = %a, size = %Hu\n", FUN /* If it's metadata, check if the space to free intersects with the file's * metadata accumulator */ - if(H5FD_MEM_DRAW != alloc_type) { + if(H5FD_MEM_DRAW != alloc_type) /* Check if the space to free intersects with the file's metadata accumulator */ - if(H5F__accum_free(f, alloc_type, addr, size) < 0) + if(H5F__accum_free(f->shared, alloc_type, addr, size) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't check free space intersection w/metadata accumulator") - } /* end if */ /* Check if the free space manager for the file has been initialized */ if(!f->shared->fs_man[fs_type]) { @@ -1326,10 +1329,10 @@ HDfprintf(stderr, "%s: Entering: alloc_type = %u, addr = %a, size = %Hu, extra_r } /* end if */ /* Get free space type from allocation type */ - H5MF__alloc_to_fs_type(f, alloc_type, size, &fs_type); + H5MF__alloc_to_fs_type(f->shared, alloc_type, size, &fs_type); /* Set the ring type in the API context */ - if(H5MF__fsm_type_is_self_referential(f, fs_type)) + if(H5MF__fsm_type_is_self_referential(f->shared, fs_type)) fsm_ring = H5AC_RING_MDFSM; else fsm_ring = H5AC_RING_RDFSM; @@ -1477,10 +1480,10 @@ HDfprintf(stderr, "%s: Entering - alloc_type = %u, addr = %a, size = %Hu\n", FUN HDassert(sect_cls); /* Get free space type from allocation type */ - H5MF__alloc_to_fs_type(f, alloc_type, size, &fs_type); + H5MF__alloc_to_fs_type(f->shared, alloc_type, size, &fs_type); /* Set the ring type in the API context */ - if(H5MF__fsm_type_is_self_referential(f, fs_type)) + if(H5MF__fsm_type_is_self_referential(f->shared, fs_type)) fsm_ring = H5AC_RING_MDFSM; else fsm_ring = H5AC_RING_RDFSM; @@ -1553,11 +1556,11 @@ HDfprintf(stderr, "%s: Entering\n", FUNC); if(H5F_PAGED_AGGR(f)) { if((ret_value = H5MF__close_pagefs(f)) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't close free-space managers for 'page' file space") - } + } /* end if */ else { if((ret_value = H5MF__close_aggrfs(f)) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't close free-space managers for 'aggr' file space") - } + } /* end else */ done: #ifdef H5MF_ALLOC_DEBUG @@ -1663,10 +1666,10 @@ HDfprintf(stderr, "%s: Entering\n", FUNC); * * The situation is further complicated if a cache image exists * and had not yet been loaded into the metadata cache. In this - * case, call H5AC_force_cache_image_load() instead of + * case, call H5AC_force_cache_image_load() instead of * H5MF_tidy_self_referential_fsm_hack(). H5AC_force_cache_image_load() - * will load the cache image, and then call - * H5MF_tidy_self_referential_fsm_hack() to discard the cache image + * will load the cache image, and then call + * H5MF_tidy_self_referential_fsm_hack() to discard the cache image * block. */ @@ -1680,12 +1683,12 @@ HDfprintf(stderr, "%s: Entering\n", FUNC); if(H5F_PAGED_AGGR(f)) { H5F_mem_page_t ptype; /* Memory type for iteration */ - /* Iterate over all the free space types that have managers and - * get each free list's space + /* Iterate over all the free space types that have managers and + * get each free list's space */ for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) { /* Test to see if we need to switch rings -- do so if required */ - if(H5MF__fsm_type_is_self_referential(f, ptype)) + if(H5MF__fsm_type_is_self_referential(f->shared, ptype)) needed_ring = H5AC_RING_MDFSM; else needed_ring = H5AC_RING_RDFSM; @@ -1702,12 +1705,12 @@ HDfprintf(stderr, "%s: Entering\n", FUNC); else { H5FD_mem_t type; /* Memory type for iteration */ - /* Iterate over all the free space types that have managers and - * get each free list's space + /* Iterate over all the free space types that have managers and + * get each free list's space */ for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { /* Test to see if we need to switch rings -- do so if required */ - if(H5MF__fsm_type_is_self_referential(f, (H5F_mem_page_t)type)) + if(H5MF__fsm_type_is_self_referential(f->shared, (H5F_mem_page_t)type)) needed_ring = H5AC_RING_MDFSM; else needed_ring = H5AC_RING_RDFSM; @@ -1824,9 +1827,9 @@ HDfprintf(stderr, "%s: Entering\n", FUNC); for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { if(f->shared->fs_man[type]) { /* Test to see if we need to switch rings -- do so if required */ - if(H5MF__fsm_type_is_self_referential(f, (H5F_mem_page_t)type)) + if(H5MF__fsm_type_is_self_referential(f->shared, (H5F_mem_page_t)type)) needed_ring = H5AC_RING_MDFSM; - else + else needed_ring = H5AC_RING_RDFSM; if(needed_ring != curr_ring) { @@ -1974,7 +1977,7 @@ HDfprintf(stderr, "%s: Entering\n", FUNC); /* gather data for the free space manager superblock extension message. * Only need addresses of FSMs and eoa prior to allocation of - * file space for the self referential free space managers. Other + * file space for the self referential free space managers. Other * data was gathered above. */ for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) @@ -1990,7 +1993,7 @@ HDfprintf(stderr, "%s: Entering\n", FUNC); for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) { if(f->shared->fs_man[ptype]) { /* Test to see if we need to switch rings -- do so if required */ - if(H5MF__fsm_type_is_self_referential(f, ptype)) + if(H5MF__fsm_type_is_self_referential(f->shared, ptype)) needed_ring = H5AC_RING_MDFSM; else needed_ring = H5AC_RING_RDFSM; @@ -2125,7 +2128,7 @@ H5MF__close_shrink_eoa(H5F_t *f) for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) { if(f->shared->fs_man[ptype]) { /* Test to see if we need to switch rings -- do so if required */ - if(H5MF__fsm_type_is_self_referential(f, ptype)) + if(H5MF__fsm_type_is_self_referential(f->shared, ptype)) needed_ring = H5AC_RING_MDFSM; else needed_ring = H5AC_RING_RDFSM; @@ -2149,7 +2152,7 @@ H5MF__close_shrink_eoa(H5F_t *f) for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { if(f->shared->fs_man[type]) { /* Test to see if we need to switch rings -- do so if required */ - if(H5MF__fsm_type_is_self_referential(f, (H5F_mem_page_t)type)) + if(H5MF__fsm_type_is_self_referential(f->shared, (H5F_mem_page_t)type)) needed_ring = H5AC_RING_MDFSM; else needed_ring = H5AC_RING_RDFSM; @@ -2269,7 +2272,7 @@ H5MF_get_freespace(H5F_t *f, hsize_t *tot_space, hsize_t *meta_size) } /* end if */ /* Test to see if we need to switch rings -- do so if required */ - if(H5MF__fsm_type_is_self_referential(f, (H5F_mem_page_t)type)) + if(H5MF__fsm_type_is_self_referential(f->shared, (H5F_mem_page_t)type)) needed_ring = H5AC_RING_MDFSM; else needed_ring = H5AC_RING_RDFSM; @@ -2299,7 +2302,7 @@ H5MF_get_freespace(H5F_t *f, hsize_t *tot_space, hsize_t *meta_size) /* Close the free-space managers if they were opened earlier in this routine */ for(type = start_type; type < end_type; H5_INC_ENUM(H5F_mem_page_t, type)) { /* Test to see if we need to switch rings -- do so if required */ - if(H5MF__fsm_type_is_self_referential(f, (H5F_mem_page_t)type)) + if(H5MF__fsm_type_is_self_referential(f->shared, (H5F_mem_page_t)type)) needed_ring = H5AC_RING_MDFSM; else needed_ring = H5AC_RING_RDFSM; @@ -2405,7 +2408,7 @@ H5MF_get_free_sections(H5F_t *f, H5FD_mem_t type, size_t nsects, H5F_sect_info_t size_t nums = 0; /* The number of free-space sections */ /* Test to see if we need to switch rings -- do so if required */ - if(H5MF__fsm_type_is_self_referential(f, ty)) + if(H5MF__fsm_type_is_self_referential(f->shared, ty)) needed_ring = H5AC_RING_MDFSM; else needed_ring = H5AC_RING_RDFSM; @@ -2731,7 +2734,7 @@ H5MF_settle_raw_data_fsm(H5F_t *f, hbool_t *fsm_settled) break; for(mem_type = H5FD_MEM_SUPER; mem_type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5F_mem_t, mem_type)) { - H5MF__alloc_to_fs_type(f, mem_type, alloc_size, &fsm_type); + H5MF__alloc_to_fs_type(f->shared, mem_type, alloc_size, &fsm_type); if(pass_count == 0) { /* this is the first pass */ HDassert(fsm_type > H5F_MEM_PAGE_DEFAULT); @@ -2764,7 +2767,7 @@ H5MF_settle_raw_data_fsm(H5F_t *f, hbool_t *fsm_settled) if(f->shared->fs_man[fsm_type]) { /* Test to see if we need to switch rings -- do so if required */ - if(H5MF__fsm_type_is_self_referential(f, fsm_type)) + if(H5MF__fsm_type_is_self_referential(f->shared, fsm_type)) needed_ring = H5AC_RING_MDFSM; else needed_ring = H5AC_RING_RDFSM; @@ -2876,7 +2879,7 @@ H5MF_settle_raw_data_fsm(H5F_t *f, hbool_t *fsm_settled) break; for(mem_type = H5FD_MEM_SUPER; mem_type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5F_mem_t, mem_type)) { - H5MF__alloc_to_fs_type(f, mem_type, alloc_size, &fsm_type); + H5MF__alloc_to_fs_type(f->shared, mem_type, alloc_size, &fsm_type); if(pass_count == 0) { /* this is the first pass */ HDassert(fsm_type > H5F_MEM_PAGE_DEFAULT); @@ -2890,7 +2893,7 @@ H5MF_settle_raw_data_fsm(H5F_t *f, hbool_t *fsm_settled) HDassert(FALSE); /* Test to see if we need to switch rings -- do so if required */ - if(H5MF__fsm_type_is_self_referential(f, fsm_type)) + if(H5MF__fsm_type_is_self_referential(f->shared, fsm_type)) needed_ring = H5AC_RING_MDFSM; else needed_ring = H5AC_RING_RDFSM; @@ -2913,7 +2916,7 @@ H5MF_settle_raw_data_fsm(H5F_t *f, hbool_t *fsm_settled) * that this is also the deciding factor as to whether a FSM * in in the raw data FSM ring. */ - if(!H5MF__fsm_type_is_self_referential(f, fsm_type)) { + if(!H5MF__fsm_type_is_self_referential(f->shared, fsm_type)) { /* The current ring should be H5AC_RING_RDFSM */ HDassert(curr_ring == H5AC_RING_RDFSM); @@ -3120,8 +3123,8 @@ H5MF_settle_meta_data_fsm(H5F_t *f, hbool_t *fsm_settled) /* should only be called if file is opened R/W */ HDassert(H5F_INTENT(f) & H5F_ACC_RDWR); - H5MF__alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, (size_t)1, &sm_fshdr_fs_type); - H5MF__alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, (size_t)1, &sm_fssinfo_fs_type); + H5MF__alloc_to_fs_type(f->shared, H5FD_MEM_FSPACE_HDR, (size_t)1, &sm_fshdr_fs_type); + H5MF__alloc_to_fs_type(f->shared, H5FD_MEM_FSPACE_SINFO, (size_t)1, &sm_fssinfo_fs_type); HDassert(sm_fshdr_fs_type > H5F_MEM_PAGE_DEFAULT); HDassert(sm_fshdr_fs_type < H5F_MEM_PAGE_LARGE_SUPER); @@ -3137,8 +3140,8 @@ H5MF_settle_meta_data_fsm(H5F_t *f, hbool_t *fsm_settled) sm_sinfo_fspace = f->shared->fs_man[sm_fssinfo_fs_type]; if(H5F_PAGED_AGGR(f)) { - H5MF__alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, f->shared->fs_page_size + 1, &lg_fshdr_fs_type); - H5MF__alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, f->shared->fs_page_size + 1, &lg_fssinfo_fs_type); + H5MF__alloc_to_fs_type(f->shared, H5FD_MEM_FSPACE_HDR, f->shared->fs_page_size + 1, &lg_fshdr_fs_type); + H5MF__alloc_to_fs_type(f->shared, H5FD_MEM_FSPACE_SINFO, f->shared->fs_page_size + 1, &lg_fssinfo_fs_type); HDassert(lg_fshdr_fs_type >= H5F_MEM_PAGE_LARGE_SUPER); HDassert(lg_fshdr_fs_type < H5F_MEM_PAGE_NTYPES); @@ -3278,14 +3281,13 @@ H5MF_settle_meta_data_fsm(H5F_t *f, hbool_t *fsm_settled) lg_sinfo_fspace = f->shared->fs_man[lg_fssinfo_fs_type]; } - if(H5MF__continue_alloc_fsm(f, sm_hdr_fspace, sm_sinfo_fspace, lg_hdr_fspace, lg_sinfo_fspace, &continue_alloc_fsm) < 0) + if(H5MF__continue_alloc_fsm(f->shared, sm_hdr_fspace, sm_sinfo_fspace, lg_hdr_fspace, lg_sinfo_fspace, &continue_alloc_fsm) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate lg sinfo FSM file space") - } while(continue_alloc_fsm); /* All free space managers should have file space allocated for them - * now, and should see no further allocations / deallocations. + * now, and should see no further allocations / deallocations. * For backward compatibility, store the eoa in f->shared->eoa_fsm_fsalloc * which will be set to fsinfo.eoa_pre_fsm_fsalloc when we actually write * the free-space info message to the superblock extension. @@ -3293,7 +3295,7 @@ H5MF_settle_meta_data_fsm(H5F_t *f, hbool_t *fsm_settled) * the new solution. */ /* Get the eoa after allocation of file space for the self referential - * free space managers. Assuming no cache image, this should be the + * free space managers. Assuming no cache image, this should be the * final EOA of the file. */ if(HADDR_UNDEF == (eoa_fsm_fsalloc = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT))) @@ -3328,42 +3330,39 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5MF__continue_alloc_fsm(H5F_t *f, H5FS_t *sm_hdr_fspace, H5FS_t *sm_sinfo_fspace, +H5MF__continue_alloc_fsm(H5F_shared_t *f_sh, H5FS_t *sm_hdr_fspace, H5FS_t *sm_sinfo_fspace, H5FS_t *lg_hdr_fspace, H5FS_t *lg_sinfo_fspace, hbool_t *continue_alloc_fsm) { FUNC_ENTER_NOAPI_NOINIT_NOERR /* Sanity checks */ - HDassert(f); - HDassert(f->shared); + HDassert(f_sh); HDassert(continue_alloc_fsm); /* Check sm_hdr_fspace */ if(sm_hdr_fspace && sm_hdr_fspace->serial_sect_count > 0 && sm_hdr_fspace->sinfo) { H5MF_CHECK_FSM(sm_hdr_fspace, continue_alloc_fsm); - } + } /* end if */ - if(!(*continue_alloc_fsm)) { + if(!(*continue_alloc_fsm)) if(sm_sinfo_fspace && sm_sinfo_fspace != sm_hdr_fspace && sm_sinfo_fspace->serial_sect_count > 0 && sm_sinfo_fspace->sinfo) { H5MF_CHECK_FSM(sm_hdr_fspace, continue_alloc_fsm); - } - } + } /* end if */ - if(H5F_PAGED_AGGR(f) && !(*continue_alloc_fsm)) { + if(H5F_SHARED_PAGED_AGGR(f_sh) && !(*continue_alloc_fsm)) { /* Check lg_hdr_fspace */ if(lg_hdr_fspace && lg_hdr_fspace->serial_sect_count > 0 && lg_hdr_fspace->sinfo) { H5MF_CHECK_FSM(lg_hdr_fspace, continue_alloc_fsm); - } + } /* end if */ /* Check lg_sinfo_fspace */ - if(!(*continue_alloc_fsm)) { + if(!(*continue_alloc_fsm)) if(lg_sinfo_fspace && lg_sinfo_fspace != lg_hdr_fspace && - lg_sinfo_fspace->serial_sect_count > 0 && lg_sinfo_fspace->sinfo) { + lg_sinfo_fspace->serial_sect_count > 0 && lg_sinfo_fspace->sinfo) { H5MF_CHECK_FSM(lg_sinfo_fspace, continue_alloc_fsm); - } - } - } + } /* end if */ + } /* end if */ FUNC_LEAVE_NOAPI(SUCCEED) } /* H5MF__continue_alloc_fsm() */ @@ -3382,8 +3381,8 @@ H5MF__continue_alloc_fsm(H5F_t *f, H5FS_t *sm_hdr_fspace, H5FS_t *sm_sinfo_fspac * *------------------------------------------------------------------------- */ -hbool_t -H5MF__fsm_type_is_self_referential(H5F_t *f, H5F_mem_page_t fsm_type) +static hbool_t +H5MF__fsm_type_is_self_referential(H5F_shared_t *f_sh, H5F_mem_page_t fsm_type) { H5F_mem_page_t sm_fshdr_fsm; H5F_mem_page_t sm_fssinfo_fsm; @@ -3391,26 +3390,25 @@ H5MF__fsm_type_is_self_referential(H5F_t *f, H5F_mem_page_t fsm_type) H5F_mem_page_t lg_fssinfo_fsm; hbool_t result = FALSE; - FUNC_ENTER_PACKAGE_NOERR + FUNC_ENTER_STATIC_NOERR /* Sanity check */ - HDassert(f); - HDassert(f->shared); + HDassert(f_sh); HDassert(fsm_type >= H5F_MEM_PAGE_DEFAULT); HDassert(fsm_type < H5F_MEM_PAGE_NTYPES); - H5MF__alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, (size_t)1, &sm_fshdr_fsm); - H5MF__alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, (size_t)1, &sm_fssinfo_fsm); + H5MF__alloc_to_fs_type(f_sh, H5FD_MEM_FSPACE_HDR, (size_t)1, &sm_fshdr_fsm); + H5MF__alloc_to_fs_type(f_sh, H5FD_MEM_FSPACE_SINFO, (size_t)1, &sm_fssinfo_fsm); - if(H5F_PAGED_AGGR(f)) { - H5MF__alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, f->shared->fs_page_size + 1, &lg_fshdr_fsm); - H5MF__alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, f->shared->fs_page_size + 1, &lg_fssinfo_fsm); + if(H5F_SHARED_PAGED_AGGR(f_sh)) { + H5MF__alloc_to_fs_type(f_sh, H5FD_MEM_FSPACE_HDR, f_sh->fs_page_size + 1, &lg_fshdr_fsm); + H5MF__alloc_to_fs_type(f_sh, H5FD_MEM_FSPACE_SINFO, f_sh->fs_page_size + 1, &lg_fssinfo_fsm); result = (fsm_type == sm_fshdr_fsm) || (fsm_type == sm_fssinfo_fsm) || (fsm_type == lg_fshdr_fsm) || (fsm_type == lg_fssinfo_fsm); } /* end if */ else { - /* In principle, fsm_type should always be less than + /* In principle, fsm_type should always be less than * H5F_MEM_PAGE_LARGE_SUPER whenever paged aggregation * is not enabled. However, since there is code that does * not observe this principle, force the result to FALSE if @@ -3440,7 +3438,7 @@ H5MF__fsm_type_is_self_referential(H5F_t *f, H5F_mem_page_t fsm_type) *------------------------------------------------------------------------- */ static hbool_t -H5MF__fsm_is_self_referential(H5F_t *f, H5FS_t *fspace) +H5MF__fsm_is_self_referential(H5F_shared_t *f_sh, H5FS_t *fspace) { H5F_mem_page_t sm_fshdr_fsm; H5F_mem_page_t sm_fssinfo_fsm; @@ -3449,28 +3447,27 @@ H5MF__fsm_is_self_referential(H5F_t *f, H5FS_t *fspace) FUNC_ENTER_STATIC_NOERR /* Sanity check */ - HDassert(f); - HDassert(f->shared); + HDassert(f_sh); HDassert(fspace); - H5MF__alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, (size_t)1, &sm_fshdr_fsm); - H5MF__alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, (size_t)1, &sm_fssinfo_fsm); + H5MF__alloc_to_fs_type(f_sh, H5FD_MEM_FSPACE_HDR, (size_t)1, &sm_fshdr_fsm); + H5MF__alloc_to_fs_type(f_sh, H5FD_MEM_FSPACE_SINFO, (size_t)1, &sm_fssinfo_fsm); - if(H5F_PAGED_AGGR(f)) { + if(H5F_SHARED_PAGED_AGGR(f_sh)) { H5F_mem_page_t lg_fshdr_fsm; H5F_mem_page_t lg_fssinfo_fsm; - H5MF__alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, f->shared->fs_page_size + 1, &lg_fshdr_fsm); - H5MF__alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, f->shared->fs_page_size + 1, &lg_fssinfo_fsm); + H5MF__alloc_to_fs_type(f_sh, H5FD_MEM_FSPACE_HDR, f_sh->fs_page_size + 1, &lg_fshdr_fsm); + H5MF__alloc_to_fs_type(f_sh, H5FD_MEM_FSPACE_SINFO, f_sh->fs_page_size + 1, &lg_fssinfo_fsm); - result = (fspace == f->shared->fs_man[sm_fshdr_fsm]) || - (fspace == f->shared->fs_man[sm_fssinfo_fsm]) || - (fspace == f->shared->fs_man[lg_fshdr_fsm]) || - (fspace == f->shared->fs_man[lg_fssinfo_fsm]); + result = (fspace == f_sh->fs_man[sm_fshdr_fsm]) || + (fspace == f_sh->fs_man[sm_fssinfo_fsm]) || + (fspace == f_sh->fs_man[lg_fshdr_fsm]) || + (fspace == f_sh->fs_man[lg_fssinfo_fsm]); } /* end if */ else - result = (fspace == f->shared->fs_man[sm_fshdr_fsm]) || - (fspace == f->shared->fs_man[sm_fssinfo_fsm]); + result = (fspace == f_sh->fs_man[sm_fshdr_fsm]) || + (fspace == f_sh->fs_man[sm_fssinfo_fsm]); FUNC_LEAVE_NOAPI(result) } /* H5MF__fsm_is_self_referential() */ diff --git a/src/H5MFaggr.c b/src/H5MFaggr.c index c726341..d716ae2 100644 --- a/src/H5MFaggr.c +++ b/src/H5MFaggr.c @@ -194,9 +194,9 @@ HDfprintf(stderr, "%s: type = %u, size = %Hu\n", FUNC, (unsigned)type, size); * the cache eventually adjusts/evicts ageout entries and ends up flushing out the * same entry that is being serialized (flush_in_progress). */ - if((f->shared->feature_flags & aggr->feature_flag) && f->shared->fs_strategy != H5F_FSPACE_STRATEGY_NONE && (!f->closing || !f->shared->fs_persist)) { + if((f->shared->feature_flags & aggr->feature_flag) && f->shared->fs_strategy != H5F_FSPACE_STRATEGY_NONE && (!f->shared->closing || !f->shared->fs_persist)) { #ifdef REPLACE - if((f->shared->feature_flags & aggr->feature_flag) && f->shared->fs_strategy != H5F_FSPACE_STRATEGY_NONE && !f->closing) { + if((f->shared->feature_flags & aggr->feature_flag) && f->shared->fs_strategy != H5F_FSPACE_STRATEGY_NONE && !f->shared->closing) { #endif haddr_t aggr_frag_addr = HADDR_UNDEF; /* Address of aggregrator fragment */ hsize_t aggr_frag_size = 0; /* Size of aggregator fragment */ diff --git a/src/H5MFpkg.h b/src/H5MFpkg.h index ec4aab4..fc398db 100644 --- a/src/H5MFpkg.h +++ b/src/H5MFpkg.h @@ -55,11 +55,6 @@ #define H5MF_FSPACE_SECT_SMALL 1 /* For paged aggregation: "small" meta/raw data section which is < fsp_size) */ #define H5MF_FSPACE_SECT_LARGE 2 /* For paged aggregation: "large" Section which is >= fsp_size) */ -/* For non-paged aggregation: map allocation request type to tracked free-space type */ -/* F -- pointer to H5F_t; T -- H5FD_mem_t */ -#define H5MF_ALLOC_TO_FS_AGGR_TYPE(F, T) \ - ((H5FD_MEM_DEFAULT == (F)->shared->fs_type_map[T]) ? (T) : (F)->shared->fs_type_map[T]) - /* Get section class type based on size */ #define H5MF_SECT_CLASS_TYPE(F, S) \ ((H5F_PAGED_AGGR(F)) ? \ @@ -182,13 +177,10 @@ H5_DLLVAR H5FS_section_class_t H5MF_FSPACE_SECT_CLS_LARGE[1]; /* Allocator routines */ H5_DLL herr_t H5MF__open_fstype(H5F_t *f, H5F_mem_page_t type); H5_DLL herr_t H5MF__start_fstype(H5F_t *f, H5F_mem_page_t type); - H5_DLL htri_t H5MF__find_sect(H5F_t *f, H5FD_mem_t alloc_type, hsize_t size, H5FS_t *fspace, haddr_t *addr); H5_DLL herr_t H5MF__add_sect(H5F_t *f, H5FD_mem_t alloc_type, H5FS_t *fspace, H5MF_free_section_t *node); - -H5_DLL herr_t H5MF__sects_dump(H5F_t *f, FILE *stream); - -H5_DLL void H5MF__alloc_to_fs_type(H5F_t *f, H5FD_mem_t alloc_type, hsize_t size, H5F_mem_page_t *fs_type); +H5_DLL void H5MF__alloc_to_fs_type(H5F_shared_t *f_sh, H5FD_mem_t alloc_type, + hsize_t size, H5F_mem_page_t *fs_type); /* 'simple/small/large' section routines */ H5_DLL H5MF_free_section_t *H5MF__sect_new(unsigned ctype, haddr_t sect_off, @@ -206,6 +198,11 @@ H5_DLL herr_t H5MF__aggr_absorb(const H5F_t *f, H5F_blk_aggr_t *aggr, H5_DLL herr_t H5MF__aggr_query(const H5F_t *f, const H5F_blk_aggr_t *aggr, haddr_t *addr, hsize_t *size); +/* Debugging routines */ +#ifdef H5MF_ALLOC_DEBUG_DUMP +H5_DLL herr_t H5MF__sects_dump(H5F_t *f, FILE *stream); +#endif /* H5MF_ALLOC_DEBUG_DUMP */ + /* Testing routines */ #ifdef H5MF_TESTING #endif /* H5MF_TESTING */ diff --git a/src/H5MFprivate.h b/src/H5MFprivate.h index bd57f05..de1bdfb 100644 --- a/src/H5MFprivate.h +++ b/src/H5MFprivate.h @@ -48,7 +48,7 @@ /***************************************/ /* File space manager routines */ -H5_DLL herr_t H5MF_init_merge_flags(H5F_t *f); +H5_DLL herr_t H5MF_init_merge_flags(H5F_shared_t *f_sh); H5_DLL herr_t H5MF_get_freespace(H5F_t *f, hsize_t *tot_space, hsize_t *meta_size); H5_DLL herr_t H5MF_close(H5F_t *f); H5_DLL herr_t H5MF_try_close(H5F_t *f); diff --git a/src/H5MFsection.c b/src/H5MFsection.c index 3ebc5c8..f661ef9 100644 --- a/src/H5MFsection.c +++ b/src/H5MFsection.c @@ -776,7 +776,7 @@ H5MF__sect_small_merge(H5FS_section_info_t **_sect1, H5FS_section_info_t *_sect2 /* Note: Large metadata page bypasses the PB cache */ /* Note: Update of raw data page (large or small sized) is handled by the PB cache */ if(udata->f->shared->page_buf != NULL && udata->alloc_type != H5FD_MEM_DRAW) - if(H5PB_remove_entry(udata->f, (*sect1)->sect_info.addr) < 0) + if(H5PB_remove_entry(udata->f->shared, (*sect1)->sect_info.addr) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't free merged section") if(H5MF__sect_free((H5FS_section_info_t *)(*sect1)) < 0) diff --git a/src/H5Mmodule.h b/src/H5Mmodule.h new file mode 100644 index 0000000..00a586f --- /dev/null +++ b/src/H5Mmodule.h @@ -0,0 +1,31 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: This file contains declarations which define macros for the + * H5M package. Including this header means that the source file + * is part of the H5M package. + */ +#ifndef _H5Mmodule_H +#define _H5Mmodule_H + +/* Define the proper control macros for the generic FUNC_ENTER/LEAVE and error + * reporting macros. + */ +#define H5M_MODULE +#define H5_MY_PKG H5M +#define H5_MY_PKG_ERR H5E_MAP +#define H5_MY_PKG_INIT YES + +#endif /* _H5Dmodule_H */ + + diff --git a/src/H5Mpkg.h b/src/H5Mpkg.h new file mode 100644 index 0000000..86ff45c --- /dev/null +++ b/src/H5Mpkg.h @@ -0,0 +1,51 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: This file contains declarations which are visible only within + * the H5M package. Source files outside the H5M package should + * include H5Mprivate.h instead. + */ +#if !(defined H5M_FRIEND || defined H5M_MODULE) +#error "Do not include this file outside the H5M package!" +#endif + +#ifndef _H5Mpkg_H +#define _H5Mpkg_H + +/* Get package's private header */ +#include "H5Mprivate.h" + +/* Other private headers needed by this file */ + +/**************************/ +/* Package Private Macros */ +/**************************/ + + +/****************************/ +/* Package Private Typedefs */ +/****************************/ + + +/*****************************/ +/* Package Private Variables */ +/*****************************/ + + +/******************************/ +/* Package Private Prototypes */ +/******************************/ + +#endif /*_H5Dpkg_H*/ + diff --git a/src/H5Mprivate.h b/src/H5Mprivate.h new file mode 100644 index 0000000..c841e75 --- /dev/null +++ b/src/H5Mprivate.h @@ -0,0 +1,80 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * This file contains private information about the H5M module + */ +#ifndef _H5Mprivate_H +#define _H5Mprivate_H + +/* Include package's public header */ +#include "H5Mpublic.h" + +/* Private headers needed by this file */ +#include "H5FDprivate.h" /* File drivers */ +#include "H5Oprivate.h" /* Object headers */ +#include "H5Sprivate.h" /* Dataspaces */ +#include "H5Zprivate.h" /* Data filters */ + + +/**************************/ +/* Library Private Macros */ +/**************************/ + +/* + * Feature: Define H5M_DEBUG on the compiler command line if you want to + * debug maps. NDEBUG must not be defined in order for this + * to have any effect. + */ +#ifdef NDEBUG +# undef H5M_DEBUG +#endif + +/* ======== Map creation property names ======== */ + +/* ======== Map access property names ======== */ +#define H5M_ACS_KEY_PREFETCH_SIZE_NAME "key_prefetch_size" /* Number of keys to prefetch during map iteration */ +#define H5M_ACS_KEY_ALLOC_SIZE_NAME "key_alloc_size" /* Initial allocation size for keys prefetched during map iteration */ + +/* Default temporary buffer size */ +#define H5D_TEMP_BUF_SIZE (1024 * 1024) + +/* Default I/O vector size */ +#define H5D_IO_VECTOR_SIZE 1024 + +/* Default VL allocation & free info */ +#define H5D_VLEN_ALLOC NULL +#define H5D_VLEN_ALLOC_INFO NULL +#define H5D_VLEN_FREE NULL +#define H5D_VLEN_FREE_INFO NULL + +/* Default virtual dataset list size */ +#define H5D_VIRTUAL_DEF_LIST_SIZE 8 + + +/****************************/ +/* Library Private Typedefs */ +/****************************/ + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/******************************/ +/* Library Private Prototypes */ +/******************************/ + +#endif /* _H5Mprivate_H */ + diff --git a/src/H5Mpublic.h b/src/H5Mpublic.h new file mode 100644 index 0000000..9cbdb32 --- /dev/null +++ b/src/H5Mpublic.h @@ -0,0 +1,126 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * This file contains public declarations for the H5M module. + * + * NOTE: This is an experimental API. Everything in the H5M package + * is subject to revision in a future release. + */ +#ifndef _H5Mpublic_H +#define _H5Mpublic_H + +/* System headers needed by this file */ + +/* Public headers needed by this file */ +#include "H5public.h" +#include "H5Ipublic.h" + + +/*****************/ +/* Public Macros */ +/*****************/ + +/* Macros defining operation IDs for map VOL callbacks (implemented using the + * "optional" VOL callback) */ +#define H5VL_MAP_CREATE 1 +#define H5VL_MAP_OPEN 2 +#define H5VL_MAP_GET_VAL 3 +#define H5VL_MAP_EXISTS 4 +#define H5VL_MAP_PUT 5 +#define H5VL_MAP_GET 6 +#define H5VL_MAP_SPECIFIC 7 +#define H5VL_MAP_CLOSE 8 + + +/*******************/ +/* Public Typedefs */ +/*******************/ + +/* types for map GET callback */ +typedef enum H5VL_map_get_t { + H5VL_MAP_GET_MAPL, /* map access property list */ + H5VL_MAP_GET_MCPL, /* map creation property list */ + H5VL_MAP_GET_KEY_TYPE, /* key type */ + H5VL_MAP_GET_VAL_TYPE, /* value type */ + H5VL_MAP_GET_COUNT /* key count */ +} H5VL_map_get_t; + +/* types for map SPECIFIC callback */ +typedef enum H5VL_map_specific_t { + H5VL_MAP_ITER, /* H5Miterate */ + H5VL_MAP_DELETE /* H5Mdelete */ +} H5VL_map_specific_t; + +/* Callback for H5Miterate() */ +typedef herr_t (*H5M_iterate_t)(hid_t map_id, const void *key, void *op_data); + + +/********************/ +/* Public Variables */ +/********************/ + + +/*********************/ +/* Public Prototypes */ +/*********************/ +#ifdef __cplusplus +extern "C" { +#endif + +/* The map API is only built when requested since there's no support in + * the native file format at this time. It's only supported in a few VOL + * connectors. + */ +#ifdef H5_HAVE_MAP_API + +H5_DLL hid_t H5Mcreate(hid_t loc_id, const char *name, hid_t key_type_id, + hid_t val_type_id, hid_t lcpl_id, hid_t mcpl_id, hid_t mapl_id); +H5_DLL hid_t H5Mcreate_anon(hid_t loc_id, hid_t key_type_id, hid_t val_type_id, + hid_t mcpl_id, hid_t mapl_id); +H5_DLL hid_t H5Mopen(hid_t loc_id, const char *name, hid_t mapl_id); +H5_DLL herr_t H5Mclose(hid_t map_id); +H5_DLL hid_t H5Mget_key_type(hid_t map_id); +H5_DLL hid_t H5Mget_val_type(hid_t map_id); +H5_DLL hid_t H5Mget_create_plist(hid_t map_id); +H5_DLL hid_t H5Mget_access_plist(hid_t map_id); +H5_DLL herr_t H5Mget_count(hid_t map_id, hsize_t *count, hid_t dxpl_id); +H5_DLL herr_t H5Mput(hid_t map_id, hid_t key_mem_type_id, const void *key, + hid_t val_mem_type_id, const void *value, hid_t dxpl_id); +H5_DLL herr_t H5Mget(hid_t map_id, hid_t key_mem_type_id, const void *key, + hid_t val_mem_type_id, void *value, hid_t dxpl_id); +H5_DLL herr_t H5Mexists(hid_t map_id, hid_t key_mem_type_id, const void *key, + hbool_t *exists, hid_t dxpl_id); +H5_DLL herr_t H5Miterate(hid_t map_id, hsize_t *idx, hid_t key_mem_type_id, + H5M_iterate_t op, void *op_data, hid_t dxpl_id); +H5_DLL herr_t H5Miterate_by_name(hid_t loc_id, const char *map_name, + hsize_t *idx, hid_t key_mem_type_id, H5M_iterate_t op, void *op_data, + hid_t dxpl_id, hid_t lapl_id); +H5_DLL herr_t H5Mdelete(hid_t map_id, hid_t key_mem_type_id, + const void *key, hid_t dxpl_id); + +/* Symbols defined for compatibility with previous versions of the HDF5 API. + * + * Use of these symbols is deprecated. + */ +#ifndef H5_NO_DEPRECATED_SYMBOLS +#endif /* H5_NO_DEPRECATED_SYMBOLS */ + +#endif /* H5_HAVE_MAP_API */ + +#ifdef __cplusplus +} +#endif + +#endif /* _H5Mpublic_H */ + @@ -367,7 +367,7 @@ H5Olink(hid_t obj_id, hid_t new_loc_id, const char *new_name, hid_t lcpl_id, tmp_vol_obj.connector = (vol_obj1 != NULL ? vol_obj1->connector : vol_obj2->connector); /* Create a link to the object */ - if(H5VL_link_create(H5VL_LINK_CREATE_HARD, &tmp_vol_obj, &loc_params2, lcpl_id, lapl_id, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, vol_obj1->data, loc_params1) < 0) + if(H5VL_link_create(H5VL_LINK_CREATE_HARD, &tmp_vol_obj, &loc_params2, lcpl_id, lapl_id, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, vol_obj1->data, &loc_params1) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTCREATE, FAIL, "unable to create link") done: @@ -1084,6 +1084,7 @@ H5Oclose(hid_t object_id) case H5I_GROUP: case H5I_DATATYPE: case H5I_DATASET: + case H5I_MAP: if(H5I_object(object_id) == NULL) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a valid object") if(H5I_dec_app_ref(object_id) < 0) diff --git a/src/H5Ocache_image.c b/src/H5Ocache_image.c index 7336211..70d9ef5 100644 --- a/src/H5Ocache_image.c +++ b/src/H5Ocache_image.c @@ -325,7 +325,7 @@ H5O__mdci_delete(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, void *_mesg) - if(f->closing) { + if(f->shared->closing) { /* Get the eoa, and verify that it has the expected value */ if(HADDR_UNDEF == (final_eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT)) ) diff --git a/src/H5Ochunk.c b/src/H5Ochunk.c index 94301b3..9ce5a46 100644 --- a/src/H5Ochunk.c +++ b/src/H5Ochunk.c @@ -331,7 +331,7 @@ H5O__chunk_update_idx(H5F_t *f, H5O_t *oh, unsigned idx) H5O_chk_cache_ud_t chk_udata; /* User data for loading chunk */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_PACKAGE + FUNC_ENTER_PACKAGE_TAG(oh->cache_info.addr) /* check args */ HDassert(f); diff --git a/src/H5Oflush.c b/src/H5Oflush.c index f5ede4a..a03cfa1 100644 --- a/src/H5Oflush.c +++ b/src/H5Oflush.c @@ -479,6 +479,9 @@ H5O_refresh_metadata_reopen(hid_t oid, H5G_loc_t *obj_loc, H5VL_t *vol_connector HGOTO_ERROR(H5E_OHDR, H5E_CANTOPENOBJ, FAIL, "unable to finish refresh for dataset") break; + case H5I_MAP: + HGOTO_ERROR(H5E_OHDR, H5E_BADTYPE, FAIL, "maps not supported in native VOL connector") + case H5I_UNINIT: case H5I_BADID: case H5I_FILE: diff --git a/src/H5Oint.c b/src/H5Oint.c index 45a8046..e37eea5 100644 --- a/src/H5Oint.c +++ b/src/H5Oint.c @@ -1836,6 +1836,9 @@ H5O_get_loc(hid_t object_id) HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, NULL, "unable to get object location from datatype ID") break; + case H5I_MAP: + HGOTO_ERROR(H5E_OHDR, H5E_BADTYPE, NULL, "maps not supported in native VOL connector") + case H5I_UNINIT: case H5I_BADID: case H5I_FILE: diff --git a/src/H5Opublic.h b/src/H5Opublic.h index c868e72..234f4f0 100644 --- a/src/H5Opublic.h +++ b/src/H5Opublic.h @@ -94,6 +94,7 @@ typedef enum H5O_type_t { H5O_TYPE_GROUP, /* Object is a group */ H5O_TYPE_DATASET, /* Object is a dataset */ H5O_TYPE_NAMED_DATATYPE, /* Object is a named data type */ + H5O_TYPE_MAP, /* Object is a map */ H5O_TYPE_NTYPES /* Number of different object types (must be last!) */ } H5O_type_t; @@ -124,8 +124,8 @@ typedef struct { /* Local Prototypes */ /********************/ static herr_t H5PB__insert_entry(H5PB_t *page_buf, H5PB_entry_t *page_entry); -static htri_t H5PB__make_space(H5F_t *f, H5PB_t *page_buf, H5FD_mem_t inserted_type); -static herr_t H5PB__write_entry(H5F_t *f, H5PB_entry_t *page_entry); +static htri_t H5PB__make_space(H5F_shared_t *f_sh, H5PB_t *page_buf, H5FD_mem_t inserted_type); +static herr_t H5PB__write_entry(H5F_shared_t *f_sh, H5PB_entry_t *page_entry); /*********************/ @@ -289,7 +289,7 @@ H5PB_print_stats(const H5PB_t *page_buf) *------------------------------------------------------------------------- */ herr_t -H5PB_create(H5F_t *f, size_t size, unsigned page_buf_min_meta_perc, unsigned page_buf_min_raw_perc) +H5PB_create(H5F_shared_t *f_sh, size_t size, unsigned page_buf_min_meta_perc, unsigned page_buf_min_raw_perc) { H5PB_t *page_buf = NULL; herr_t ret_value = SUCCEED; /* Return value */ @@ -297,20 +297,19 @@ H5PB_create(H5F_t *f, size_t size, unsigned page_buf_min_meta_perc, unsigned pag FUNC_ENTER_NOAPI(FAIL) /* Sanity checks */ - HDassert(f); - HDassert(f->shared); + HDassert(f_sh); /* Check args */ - if(f->shared->fs_strategy != H5F_FSPACE_STRATEGY_PAGE) + if(f_sh->fs_strategy != H5F_FSPACE_STRATEGY_PAGE) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "Enabling Page Buffering requires PAGE file space strategy") /* round down the size if it is larger than the page size */ - else if(size > f->shared->fs_page_size) { + else if(size > f_sh->fs_page_size) { hsize_t temp_size; - temp_size = (size / f->shared->fs_page_size) * f->shared->fs_page_size; + temp_size = (size / f_sh->fs_page_size) * f_sh->fs_page_size; H5_CHECKED_ASSIGN(size, size_t, temp_size, hsize_t); } /* end if */ - else if(0 != size % f->shared->fs_page_size) + else if(0 != size % f_sh->fs_page_size) HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTINIT, FAIL, "Page Buffer size must be >= to the page size") /* Allocate the new page buffering structure */ @@ -318,15 +317,15 @@ H5PB_create(H5F_t *f, size_t size, unsigned page_buf_min_meta_perc, unsigned pag HGOTO_ERROR(H5E_PAGEBUF, H5E_NOSPACE, FAIL, "memory allocation failed") page_buf->max_size = size; - H5_CHECKED_ASSIGN(page_buf->page_size, size_t, f->shared->fs_page_size, hsize_t); + H5_CHECKED_ASSIGN(page_buf->page_size, size_t, f_sh->fs_page_size, hsize_t); page_buf->min_meta_perc = page_buf_min_meta_perc; page_buf->min_raw_perc = page_buf_min_raw_perc; /* Calculate the minimum page count for metadata and raw data - * based on the fractions provided + * based on the fractions provided */ - page_buf->min_meta_count = (unsigned)((size * page_buf_min_meta_perc) / (f->shared->fs_page_size * 100)); - page_buf->min_raw_count = (unsigned)((size * page_buf_min_raw_perc) / (f->shared->fs_page_size * 100)); + page_buf->min_meta_count = (unsigned)((size * page_buf_min_meta_perc) / (f_sh->fs_page_size * 100)); + page_buf->min_raw_count = (unsigned)((size * page_buf_min_raw_perc) / (f_sh->fs_page_size * 100)); if(NULL == (page_buf->slist_ptr = H5SL_create(H5SL_TYPE_HADDR, NULL))) HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTCREATE, FAIL, "can't create skip list") @@ -336,7 +335,7 @@ H5PB_create(H5F_t *f, size_t size, unsigned page_buf_min_meta_perc, unsigned pag if(NULL == (page_buf->page_fac = H5FL_fac_init(page_buf->page_size))) HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTINIT, FAIL, "can't create page factory") - f->shared->page_buf = page_buf; + f_sh->page_buf = page_buf; done: if(ret_value < 0) { @@ -370,18 +369,18 @@ static herr_t H5PB__flush_cb(void *item, void H5_ATTR_UNUSED *key, void *_op_data) { H5PB_entry_t *page_entry = (H5PB_entry_t *)item; /* Pointer to page entry node */ - H5F_t *f = (H5F_t *)_op_data; + H5F_shared_t *f_sh = (H5F_shared_t *)_op_data; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC /* Sanity checks */ HDassert(page_entry); - HDassert(f); + HDassert(f_sh); /* Flush the page if it's dirty */ if(page_entry->is_dirty) - if(H5PB__write_entry(f, page_entry) < 0) + if(H5PB__write_entry(f_sh, page_entry) < 0) HGOTO_ERROR(H5E_PAGEBUF, H5E_WRITEERROR, FAIL, "file write failed") done: @@ -401,21 +400,21 @@ done: *------------------------------------------------------------------------- */ herr_t -H5PB_flush(H5F_t *f) +H5PB_flush(H5F_shared_t *f_sh) { herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) /* Sanity check */ - HDassert(f); + HDassert(f_sh); /* Flush all the entries in the PB skiplist, if we have write access on the file */ - if(f->shared->page_buf && (H5F_ACC_RDWR & H5F_INTENT(f))) { - H5PB_t *page_buf = f->shared->page_buf; + if(f_sh->page_buf && (H5F_ACC_RDWR & H5F_SHARED_INTENT(f_sh))) { + H5PB_t *page_buf = f_sh->page_buf; /* Iterate over all entries in page buffer skip list */ - if(H5SL_iterate(page_buf->slist_ptr, H5PB__flush_cb, (void *)f)) + if(H5SL_iterate(page_buf->slist_ptr, H5PB__flush_cb, f_sh)) HGOTO_ERROR(H5E_PAGEBUF, H5E_BADITER, FAIL, "can't flush page buffer skip list") } /* end if */ @@ -473,21 +472,21 @@ H5PB__dest_cb(void *item, void H5_ATTR_UNUSED *key, void *_op_data) *------------------------------------------------------------------------- */ herr_t -H5PB_dest(H5F_t *f) +H5PB_dest(H5F_shared_t *f_sh) { herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) /* Sanity checks */ - HDassert(f); + HDassert(f_sh); /* flush and destroy the page buffer, if it exists */ - if(f->shared->page_buf) { - H5PB_t *page_buf = f->shared->page_buf; + if(f_sh->page_buf) { + H5PB_t *page_buf = f_sh->page_buf; H5PB_ud1_t op_data; /* Iteration context */ - if(H5PB_flush(f) < 0) + if(H5PB_flush(f_sh) < 0) HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTFLUSH, FAIL, "can't flush page buffer") /* Set up context info */ @@ -507,7 +506,7 @@ H5PB_dest(H5F_t *f) if(H5FL_fac_term(page_buf->page_fac) < 0) HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTRELEASE, FAIL, "can't destroy page buffer page factory") - f->shared->page_buf = H5FL_FREE(H5PB_t, page_buf); + f_sh->page_buf = H5FL_FREE(H5PB_t, page_buf); } /* end if */ done: @@ -529,16 +528,18 @@ done: * *------------------------------------------------------------------------- */ -herr_t -H5PB_add_new_page(H5F_t *f, H5FD_mem_t type, haddr_t page_addr) +herr_t +H5PB_add_new_page(H5F_shared_t *f_sh, H5FD_mem_t type, haddr_t page_addr) { - H5PB_t *page_buf = f->shared->page_buf; - H5PB_entry_t *page_entry = NULL; /* pointer to the corresponding page entry */ - herr_t ret_value = SUCCEED; /* Return value */ + H5PB_t *page_buf; /* Page buffer to operate on */ + H5PB_entry_t *page_entry = NULL; /* Pointer to the corresponding page entry */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) /* Sanity checks */ + HDassert(f_sh); + page_buf = f_sh->page_buf; HDassert(page_buf); /* If there is an existing page, this means that at some point the @@ -624,7 +625,7 @@ H5PB_update_entry(H5PB_t *page_buf, haddr_t addr, size_t size, const void *buf) * Function: H5PB_remove_entry * * Purpose: Remove possible metadata entry with ADDR from the PB cache. - * This is in response to the data corruption bug from fheap.c + * This is in response to the data corruption bug from fheap.c * with page buffering + page strategy. * Note: Large metadata page bypasses the PB cache. * Note: Update of raw data page (large or small sized) is handled by the PB cache. @@ -636,15 +637,17 @@ H5PB_update_entry(H5PB_t *page_buf, haddr_t addr, size_t size, const void *buf) *------------------------------------------------------------------------- */ herr_t -H5PB_remove_entry(const H5F_t *f, haddr_t addr) +H5PB_remove_entry(const H5F_shared_t *f_sh, haddr_t addr) { - H5PB_t *page_buf = f->shared->page_buf; - H5PB_entry_t *page_entry = NULL; /* pointer to the page entry being searched */ - herr_t ret_value = SUCCEED; /* Return value */ + H5PB_t *page_buf; /* Page buffer to operate on */ + H5PB_entry_t *page_entry = NULL; /* Pointer to the page entry being searched */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) /* Sanity checks */ + HDassert(f_sh); + page_buf = f_sh->page_buf; HDassert(page_buf); /* Search for address in the skip list */ @@ -674,7 +677,7 @@ done: /*------------------------------------------------------------------------- * Function: H5PB_read * - * Purpose: Reads in the data from the page containing it if it exists + * Purpose: Reads in the data from the page containing it if it exists * in the PB cache; otherwise reads in the page through the VFD. * * Return: Non-negative on success/Negative on failure @@ -684,7 +687,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5PB_read(H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size, void *buf/*out*/) +H5PB_read(H5F_shared_t *f_sh, H5FD_mem_t type, haddr_t addr, size_t size, void *buf/*out*/) { H5PB_t *page_buf; /* Page buffering info for this file */ H5PB_entry_t *page_entry; /* Pointer to the corresponding page entry */ @@ -701,21 +704,21 @@ H5PB_read(H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size, void *buf/*out*/ FUNC_ENTER_NOAPI(FAIL) /* Sanity checks */ - HDassert(f); + HDassert(f_sh); HDassert(type != H5FD_MEM_GHEAP); /* Get pointer to page buffer info for this file */ - page_buf = f->shared->page_buf; + page_buf = f_sh->page_buf; #ifdef H5_HAVE_PARALLEL - if(H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI)) { + if(H5F_SHARED_HAS_FEATURE(f_sh, H5FD_FEAT_HAS_MPI)) { #if 1 bypass_pb = TRUE; #else /* MSC - why this stopped working ? */ int mpi_size; - if((mpi_size = H5F_mpi_get_size(f)) < 0) + if((mpi_size = H5F_shared_mpi_get_size(f_sh)) < 0) HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "can't retrieve MPI communicator size") if(1 != mpi_size) bypass_pb = TRUE; @@ -729,7 +732,7 @@ H5PB_read(H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size, void *buf/*out*/ */ if(NULL == page_buf || size >= page_buf->page_size || (bypass_pb && H5FD_MEM_DRAW == type)) { - if(H5F__accum_read(f, type, addr, size, buf) < 0) + if(H5F__accum_read(f_sh, type, addr, size, buf) < 0) HGOTO_ERROR(H5E_PAGEBUF, H5E_READERROR, FAIL, "read through metadata accumulator failed") /* Update statistics */ @@ -780,7 +783,7 @@ H5PB_read(H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size, void *buf/*out*/ } /* end else */ /* Translate to file driver I/O info object */ - file = f->shared->lf; + file = f_sh->lf; /* Copy raw data from dirty pages into the read buffer if the read request spans pages in the page buffer*/ @@ -884,7 +887,7 @@ H5PB_read(H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size, void *buf/*out*/ else page_buf->hits[0]++; } /* end if */ - /* if not found */ + /* if not found */ else { void *new_page_buf = NULL; size_t page_size = page_buf->page_size; @@ -895,7 +898,7 @@ H5PB_read(H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size, void *buf/*out*/ htri_t can_make_space; /* check if we can make space in page buffer */ - if((can_make_space = H5PB__make_space(f, page_buf, type)) < 0) + if((can_make_space = H5PB__make_space(f_sh, page_buf, type)) < 0) HGOTO_ERROR(H5E_PAGEBUF, H5E_NOSPACE, FAIL, "make space in Page buffer Failed") /* if make_space returns 0, then we can't use the page @@ -920,7 +923,7 @@ H5PB_read(H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size, void *buf/*out*/ /* Read page through the VFD layer, but make sure we don't read past the EOA. */ /* Retrieve the 'eoa' for the file */ - if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, type))) + if(HADDR_UNDEF == (eoa = H5F_shared_get_eoa(f_sh, type))) HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "driver get_eoa request failed") /* If the entire page falls outside the EOA, then fail */ @@ -981,7 +984,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5PB_write(H5F_t *f, H5FD_mem_t type, haddr_t addr, +H5PB_write(H5F_shared_t *f_sh, H5FD_mem_t type, haddr_t addr, size_t size, const void *buf) { H5PB_t *page_buf; /* Page buffering info for this file */ @@ -999,20 +1002,20 @@ H5PB_write(H5F_t *f, H5FD_mem_t type, haddr_t addr, FUNC_ENTER_NOAPI(FAIL) /* Sanity checks */ - HDassert(f); + HDassert(f_sh); /* Get pointer to page buffer info for this file */ - page_buf = f->shared->page_buf; + page_buf = f_sh->page_buf; #ifdef H5_HAVE_PARALLEL - if(H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI)) { + if(H5F_SHARED_HAS_FEATURE(f_sh, H5FD_FEAT_HAS_MPI)) { #if 1 bypass_pb = TRUE; #else /* MSC - why this stopped working ? */ int mpi_size; - if((mpi_size = H5F_mpi_get_size(f)) < 0) + if((mpi_size = H5F_shared_mpi_get_size(f_sh)) < 0) HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "can't retrieve MPI communicator size") if(1 != mpi_size) bypass_pb = TRUE; @@ -1025,7 +1028,7 @@ H5PB_write(H5F_t *f, H5FD_mem_t type, haddr_t addr, * buffering. */ if(NULL == page_buf || size >= page_buf->page_size || bypass_pb) { - if(H5F__accum_write(f, type, addr, size, buf) < 0) + if(H5F__accum_write(f_sh, type, addr, size, buf) < 0) HGOTO_ERROR(H5E_PAGEBUF, H5E_WRITEERROR, FAIL, "write through metadata accumulator failed") /* Update statistics */ @@ -1084,7 +1087,7 @@ H5PB_write(H5F_t *f, H5FD_mem_t type, haddr_t addr, } /* end else */ /* Translate to file driver I/O info object */ - file = f->shared->lf; + file = f_sh->lf; /* Check if existing pages for raw data need to be updated since raw data access is not atomic */ if(H5FD_MEM_DRAW == type && size >= page_buf->page_size) { @@ -1186,7 +1189,7 @@ H5PB_write(H5F_t *f, H5FD_mem_t type, haddr_t addr, else page_buf->hits[0]++; } /* end if */ - /* If not found */ + /* If not found */ else { void *new_page_buf; size_t page_size = page_buf->page_size; @@ -1196,7 +1199,7 @@ H5PB_write(H5F_t *f, H5FD_mem_t type, haddr_t addr, htri_t can_make_space; /* Check if we can make space in page buffer */ - if((can_make_space = H5PB__make_space(f, page_buf, type)) < 0) + if((can_make_space = H5PB__make_space(f_sh, page_buf, type)) < 0) HGOTO_ERROR(H5E_PAGEBUF, H5E_NOSPACE, FAIL, "make space in Page buffer Failed") /* If make_space returns 0, then we can't use the page @@ -1208,14 +1211,14 @@ H5PB_write(H5F_t *f, H5FD_mem_t type, haddr_t addr, /* Write to VFD and return */ if(H5FD_write(file, type, addr, size, buf) < 0) HGOTO_ERROR(H5E_PAGEBUF, H5E_WRITEERROR, FAIL, "driver write request failed") - + /* Break out of loop */ break; } /* end if */ } /* end if */ /* Don't bother searching if there is no write access */ - if(H5F_ACC_RDWR & H5F_INTENT(f)) + if(H5F_ACC_RDWR & H5F_SHARED_INTENT(f_sh)) /* Lookup & remove the page from the new skip list page if * it exists to see if this is a new page from the MF layer */ @@ -1258,7 +1261,7 @@ H5PB_write(H5F_t *f, H5FD_mem_t type, haddr_t addr, page_entry->type = (H5F_mem_page_t)type; /* Retrieve the 'eoa' for the file */ - if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, type))) + if(HADDR_UNDEF == (eoa = H5F_shared_get_eoa(f_sh, type))) HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "driver get_eoa request failed") /* If the entire page falls outside the EOA, then fail */ @@ -1269,8 +1272,8 @@ H5PB_write(H5F_t *f, H5FD_mem_t type, haddr_t addr, * returned will most likely be HADDR_UNDEF, so skip * that check. */ - if(!H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI)) - if(HADDR_UNDEF == (eof = H5FD_get_eof(f->shared->lf, H5FD_MEM_DEFAULT))) + if(!H5F_SHARED_HAS_FEATURE(f_sh, H5FD_FEAT_HAS_MPI)) + if(HADDR_UNDEF == (eof = H5FD_get_eof(f_sh->lf, H5FD_MEM_DEFAULT))) HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "driver get_eof request failed") /* Adjust the read size to not go beyond the EOA */ @@ -1360,26 +1363,19 @@ done: /*------------------------------------------------------------------------- * Function: H5PB__make_space() * - * Purpose: ??? - * - * This function was created without documentation. - * What follows is my best understanding of Mohamad's intent. - * - * If necessary and if possible, evict a page from the page - * buffer to make space for the supplied page. Depending on - * the page buffer configuration and contents, and the page + * Purpose: If necessary and if possible, evict a page from the page + * buffer to make space for the supplied page. Depending on + * the page buffer configuration and contents, and the page * supplied this may or may not be possible. * - * JRM -- 12/22/16 - * * Return: Non-negative on success/Negative on failure * * Programmer: Mohamad Chaarawi * *------------------------------------------------------------------------- */ -static htri_t -H5PB__make_space(H5F_t *f, H5PB_t *page_buf, H5FD_mem_t inserted_type) +static htri_t +H5PB__make_space(H5F_shared_t *f_sh, H5PB_t *page_buf, H5FD_mem_t inserted_type) { H5PB_entry_t *page_entry; /* Pointer to page eviction candidate */ htri_t ret_value = TRUE; /* Return value */ @@ -1387,7 +1383,7 @@ H5PB__make_space(H5F_t *f, H5PB_t *page_buf, H5FD_mem_t inserted_type) FUNC_ENTER_STATIC /* Sanity check */ - HDassert(f); + HDassert(f_sh); HDassert(page_buf); /* Get oldest entry */ @@ -1444,7 +1440,7 @@ H5PB__make_space(H5F_t *f, H5PB_t *page_buf, H5FD_mem_t inserted_type) /* Flush page if dirty */ if(page_entry->is_dirty) - if(H5PB__write_entry(f, page_entry) < 0) + if(H5PB__write_entry(f_sh, page_entry) < 0) HGOTO_ERROR(H5E_PAGEBUF, H5E_WRITEERROR, FAIL, "file write failed") /* Update statistics */ @@ -1478,7 +1474,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5PB__write_entry(H5F_t *f, H5PB_entry_t *page_entry) +H5PB__write_entry(H5F_shared_t *f_sh, H5PB_entry_t *page_entry) { haddr_t eoa; /* Current EOA for the file */ herr_t ret_value = SUCCEED; /* Return value */ @@ -1486,11 +1482,11 @@ H5PB__write_entry(H5F_t *f, H5PB_entry_t *page_entry) FUNC_ENTER_STATIC /* Sanity check */ - HDassert(f); + HDassert(f_sh); HDassert(page_entry); /* Retrieve the 'eoa' for the file */ - if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, (H5FD_mem_t)page_entry->type))) + if(HADDR_UNDEF == (eoa = H5F_shared_get_eoa(f_sh, (H5FD_mem_t)page_entry->type))) HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "driver get_eoa request failed") /* If the starting address of the page is larger than @@ -1498,14 +1494,14 @@ H5PB__write_entry(H5F_t *f, H5PB_entry_t *page_entry) */ if(page_entry->addr <= eoa) { H5FD_t *file; /* File driver I/O info */ - size_t page_size = f->shared->page_buf->page_size; + size_t page_size = f_sh->page_buf->page_size; /* Adjust the page length if it exceeds the EOA */ if((page_entry->addr + page_size) > eoa) page_size = (size_t)(eoa - page_entry->addr); /* Translate to file driver I/O info object */ - file = f->shared->lf; + file = f_sh->lf; if(H5FD_write(file, (H5FD_mem_t)page_entry->type, page_entry->addr, page_size, page_entry->page_buf_ptr) < 0) HGOTO_ERROR(H5E_PAGEBUF, H5E_WRITEERROR, FAIL, "file write failed") diff --git a/src/H5PBprivate.h b/src/H5PBprivate.h index b94b845..82d010d 100644 --- a/src/H5PBprivate.h +++ b/src/H5PBprivate.h @@ -85,14 +85,14 @@ typedef struct H5PB_t { /***************************************/ /* General routines */ -H5_DLL herr_t H5PB_create(H5F_t *file, size_t page_buffer_size, unsigned page_buf_min_meta_perc, unsigned page_buf_min_raw_perc); -H5_DLL herr_t H5PB_flush(H5F_t *f); -H5_DLL herr_t H5PB_dest(H5F_t *f); -H5_DLL herr_t H5PB_add_new_page(H5F_t *f, H5FD_mem_t type, haddr_t page_addr); +H5_DLL herr_t H5PB_create(H5F_shared_t *f_sh, size_t page_buffer_size, unsigned page_buf_min_meta_perc, unsigned page_buf_min_raw_perc); +H5_DLL herr_t H5PB_flush(H5F_shared_t *f_sh); +H5_DLL herr_t H5PB_dest(H5F_shared_t *f_sh); +H5_DLL herr_t H5PB_add_new_page(H5F_shared_t *f_sh, H5FD_mem_t type, haddr_t page_addr); H5_DLL herr_t H5PB_update_entry(H5PB_t *page_buf, haddr_t addr, size_t size, const void *buf); -H5_DLL herr_t H5PB_remove_entry(const H5F_t *f, haddr_t addr); -H5_DLL herr_t H5PB_read(H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size, void *buf/*out*/); -H5_DLL herr_t H5PB_write(H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size, const void *buf); +H5_DLL herr_t H5PB_remove_entry(const H5F_shared_t *f_sh, haddr_t addr); +H5_DLL herr_t H5PB_read(H5F_shared_t *f_sh, H5FD_mem_t type, haddr_t addr, size_t size, void *buf/*out*/); +H5_DLL herr_t H5PB_write(H5F_shared_t *f_sh, H5FD_mem_t type, haddr_t addr, size_t size, const void *buf); /* Statistics routines */ H5_DLL herr_t H5PB_reset_stats(H5PB_t *page_buf); diff --git a/src/H5Pdcpl.c b/src/H5Pdcpl.c index 6b9edf1..721ba89 100644 --- a/src/H5Pdcpl.c +++ b/src/H5Pdcpl.c @@ -827,7 +827,7 @@ H5P__dcrt_layout_cmp(const void *_layout1, const void *_layout2, if((equal = H5S_extent_equal(layout1->storage.u.virt.list[u].source_dset.virtual_select, layout2->storage.u.virt.list[u].source_dset.virtual_select)) < 0) HGOTO_DONE(-1) if(!equal) HGOTO_DONE(1) - if((equal = H5S_select_shape_same(layout1->storage.u.virt.list[u].source_dset.virtual_select, layout2->storage.u.virt.list[u].source_dset.virtual_select)) < 0) HGOTO_DONE(-1) + if((equal = H5S_SELECT_SHAPE_SAME(layout1->storage.u.virt.list[u].source_dset.virtual_select, layout2->storage.u.virt.list[u].source_dset.virtual_select)) < 0) HGOTO_DONE(-1) if(!equal) HGOTO_DONE(1) @@ -847,7 +847,7 @@ H5P__dcrt_layout_cmp(const void *_layout1, const void *_layout2, if((equal = H5S_extent_equal(layout1->storage.u.virt.list[u].source_select, layout2->storage.u.virt.list[u].source_select)) < 0) HGOTO_DONE(-1) if(!equal) HGOTO_DONE(1) - if((equal = H5S_select_shape_same(layout1->storage.u.virt.list[u].source_select, layout2->storage.u.virt.list[u].source_select)) < 0) HGOTO_DONE(-1) + if((equal = H5S_SELECT_SHAPE_SAME(layout1->storage.u.virt.list[u].source_select, layout2->storage.u.virt.list[u].source_select)) < 0) HGOTO_DONE(-1) if(!equal) HGOTO_DONE(1) } /* end for */ diff --git a/src/H5Pencdec.c b/src/H5Pencdec.c index 8b9962f..2a07a67 100644 --- a/src/H5Pencdec.c +++ b/src/H5Pencdec.c @@ -722,7 +722,7 @@ H5P__decode(const void *buf) /* Get the type of the property list */ type = (H5P_plist_type_t)*p++; - if(type <= H5P_TYPE_USER || type > H5P_TYPE_LINK_ACCESS) + if(type <= H5P_TYPE_USER || type >= H5P_TYPE_MAX_TYPE) HGOTO_ERROR(H5E_PLIST, H5E_BADRANGE, FAIL, "bad type of encoded information: %u", (unsigned)type) /* Create new property list of the specified type */ diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c index 8bb2b12..22252d3 100644 --- a/src/H5Pfapl.c +++ b/src/H5Pfapl.c @@ -30,22 +30,22 @@ /***********/ /* Headers */ /***********/ -#include "H5private.h" /* Generic Functions */ -#include "H5ACprivate.h" /* Metadata cache */ -#include "H5Dprivate.h" /* Datasets */ -#include "H5Eprivate.h" /* Error handling */ -#include "H5Fprivate.h" /* Files */ -#include "H5FDprivate.h" /* File drivers */ -#include "H5Iprivate.h" /* IDs */ -#include "H5MMprivate.h" /* Memory Management */ -#include "H5Ppkg.h" /* Property lists */ -#include "H5VLprivate.h" /* Virtual Object Layer */ +#include "H5private.h" /* Generic Functions */ +#include "H5ACprivate.h" /* Metadata cache */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* Files */ +#include "H5FDprivate.h" /* File drivers */ +#include "H5Iprivate.h" /* IDs */ +#include "H5MMprivate.h" /* Memory Management */ +#include "H5Ppkg.h" /* Property lists */ +#include "H5VLprivate.h" /* Virtual Object Layer */ +#include "H5VMprivate.h" /* Vector Functions */ /* Includes needed to set default file driver */ -#include "H5FDsec2.h" /* POSIX unbuffered I/O */ -#include "H5FDstdio.h" /* Standard C buffered I/O */ +#include "H5FDsec2.h" /* POSIX unbuffered I/O */ +#include "H5FDstdio.h" /* Standard C buffered I/O */ #ifdef H5_HAVE_WINDOWS -#include "H5FDwindows.h" /* Win32 I/O */ +#include "H5FDwindows.h" /* Win32 I/O */ #endif /* Includes needed to set default VOL connector */ @@ -111,7 +111,7 @@ #define H5F_ACS_GARBG_COLCT_REF_DEF 0 #define H5F_ACS_GARBG_COLCT_REF_ENC H5P__encode_unsigned #define H5F_ACS_GARBG_COLCT_REF_DEC H5P__decode_unsigned -/* Definition for file driver ID & info*/ +/* Definition for file driver ID & info */ #define H5F_ACS_FILE_DRV_SIZE sizeof(H5FD_driver_prop_t) #define H5F_ACS_FILE_DRV_DEF {H5_DEFAULT_VFD, NULL} #define H5F_ACS_FILE_DRV_CRT H5P__facc_file_driver_create @@ -231,6 +231,24 @@ #define H5F_ACS_COLL_MD_WRITE_FLAG_DEF FALSE #define H5F_ACS_COLL_MD_WRITE_FLAG_ENC H5P__encode_hbool_t #define H5F_ACS_COLL_MD_WRITE_FLAG_DEC H5P__decode_hbool_t +/* Definition for the file's MPI communicator */ +#define H5F_ACS_MPI_PARAMS_COMM_SIZE sizeof(MPI_Comm) +#define H5F_ACS_MPI_PARAMS_COMM_DEF MPI_COMM_NULL +#define H5F_ACS_MPI_PARAMS_COMM_SET H5P__facc_mpi_comm_set +#define H5F_ACS_MPI_PARAMS_COMM_GET H5P__facc_mpi_comm_get +#define H5F_ACS_MPI_PARAMS_COMM_DEL H5P__facc_mpi_comm_del +#define H5F_ACS_MPI_PARAMS_COMM_COPY H5P__facc_mpi_comm_copy +#define H5F_ACS_MPI_PARAMS_COMM_CMP H5P__facc_mpi_comm_cmp +#define H5F_ACS_MPI_PARAMS_COMM_CLOSE H5P__facc_mpi_comm_close +/* Definition for the file's MPI info */ +#define H5F_ACS_MPI_PARAMS_INFO_SIZE sizeof(MPI_Info) +#define H5F_ACS_MPI_PARAMS_INFO_DEF MPI_INFO_NULL +#define H5F_ACS_MPI_PARAMS_INFO_SET H5P__facc_mpi_info_set +#define H5F_ACS_MPI_PARAMS_INFO_GET H5P__facc_mpi_info_get +#define H5F_ACS_MPI_PARAMS_INFO_DEL H5P__facc_mpi_info_del +#define H5F_ACS_MPI_PARAMS_INFO_COPY H5P__facc_mpi_info_copy +#define H5F_ACS_MPI_PARAMS_INFO_CMP H5P__facc_mpi_info_cmp +#define H5F_ACS_MPI_PARAMS_INFO_CLOSE H5P__facc_mpi_info_close #endif /* H5_HAVE_PARALLEL */ /* Definitions for the initial metadata cache image configuration */ #define H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_SIZE sizeof(H5AC_cache_image_config_t) @@ -334,6 +352,24 @@ static herr_t H5P__facc_vol_copy(const char *name, size_t size, void *value); static int H5P__facc_vol_cmp(const void *value1, const void *value2, size_t size); static herr_t H5P__facc_vol_close(const char *name, size_t size, void *value); +#ifdef H5_HAVE_PARALLEL +/* MPI communicator callbacks */ +static herr_t H5P__facc_mpi_comm_set(hid_t prop_id, const char *name, size_t size, void *value); +static herr_t H5P__facc_mpi_comm_get(hid_t prop_id, const char *name, size_t size, void *value); +static herr_t H5P__facc_mpi_comm_del(hid_t prop_id, const char *name, size_t size, void *value); +static herr_t H5P__facc_mpi_comm_copy(const char *name, size_t size, void *value); +static int H5P__facc_mpi_comm_cmp(const void *value1, const void *value2, size_t size); +static herr_t H5P__facc_mpi_comm_close(const char *name, size_t size, void *value); + +/* MPI info callbacks */ +static herr_t H5P__facc_mpi_info_set(hid_t prop_id, const char *name, size_t size, void *value); +static herr_t H5P__facc_mpi_info_get(hid_t prop_id, const char *name, size_t size, void *value); +static herr_t H5P__facc_mpi_info_del(hid_t prop_id, const char *name, size_t size, void *value); +static herr_t H5P__facc_mpi_info_copy(const char *name, size_t size, void *value); +static int H5P__facc_mpi_info_cmp(const void *value1, const void *value2, size_t size); +static herr_t H5P__facc_mpi_info_close(const char *name, size_t size, void *value); +#endif /* H5_HAVE_PARALLEL */ + /*********************/ /* Package Variables */ @@ -404,6 +440,8 @@ static const hbool_t H5F_def_evict_on_close_flag_g = H5F_ACS_EVICT_ON_CLOSE_FLAG #ifdef H5_HAVE_PARALLEL static const H5P_coll_md_read_flag_t H5F_def_coll_md_read_flag_g = H5F_ACS_COLL_MD_READ_FLAG_DEF; /* Default setting for the collective metedata read flag */ static const hbool_t H5F_def_coll_md_write_flag_g = H5F_ACS_COLL_MD_WRITE_FLAG_DEF; /* Default setting for the collective metedata write flag */ +static const MPI_Comm H5F_def_mpi_params_comm_g = H5F_ACS_MPI_PARAMS_COMM_DEF; /* Default MPI communicator */ +static const MPI_Info H5F_def_mpi_params_info_g = H5F_ACS_MPI_PARAMS_INFO_DEF; /* Default MPI info struct */ #endif /* H5_HAVE_PARALLEL */ static const H5AC_cache_image_config_t H5F_def_mdc_initCacheImageCfg_g = H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_DEF; /* Default metadata cache image settings */ static const size_t H5F_def_page_buf_size_g = H5F_ACS_PAGE_BUFFER_SIZE_DEF; /* Default page buffer size */ @@ -621,6 +659,19 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass) NULL, NULL, NULL, H5F_ACS_COLL_MD_WRITE_FLAG_ENC, H5F_ACS_COLL_MD_WRITE_FLAG_DEC, NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + + /* Register the MPI communicator */ + if(H5P__register_real(pclass, H5F_ACS_MPI_PARAMS_COMM_NAME, H5F_ACS_MPI_PARAMS_COMM_SIZE, &H5F_def_mpi_params_comm_g, + NULL, H5F_ACS_MPI_PARAMS_COMM_SET, H5F_ACS_MPI_PARAMS_COMM_GET, NULL, NULL, + H5F_ACS_MPI_PARAMS_COMM_DEL, H5F_ACS_MPI_PARAMS_COMM_COPY, H5F_ACS_MPI_PARAMS_COMM_CMP, H5F_ACS_MPI_PARAMS_COMM_CLOSE) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + + /* Register the MPI info struct */ + if(H5P__register_real(pclass, H5F_ACS_MPI_PARAMS_INFO_NAME, H5F_ACS_MPI_PARAMS_INFO_SIZE, &H5F_def_mpi_params_info_g, + NULL, H5F_ACS_MPI_PARAMS_INFO_SET, H5F_ACS_MPI_PARAMS_INFO_GET, NULL, NULL, + H5F_ACS_MPI_PARAMS_INFO_DEL, H5F_ACS_MPI_PARAMS_INFO_COPY, H5F_ACS_MPI_PARAMS_INFO_CMP, H5F_ACS_MPI_PARAMS_INFO_CLOSE) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + #endif /* H5_HAVE_PARALLEL */ /* Register the initial metadata cache image configuration */ @@ -634,11 +685,13 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass) NULL, NULL, NULL, H5F_ACS_PAGE_BUFFER_SIZE_ENC, H5F_ACS_PAGE_BUFFER_SIZE_DEC, NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the size of the page buffer minimum metadata size */ if(H5P__register_real(pclass, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_SIZE, &H5F_def_page_buf_min_meta_perc_g, NULL, NULL, NULL, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_ENC, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_DEC, NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the size of the page buffer minimum raw data size */ if(H5P__register_real(pclass, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_SIZE, &H5F_def_page_buf_min_raw_perc_g, NULL, NULL, NULL, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_ENC, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEC, @@ -4738,6 +4791,468 @@ done: /*------------------------------------------------------------------------- + * Function: H5Pget_mpi_params + * + * Purpose: Gets the MPI communicator and info stored in the fapl. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Dana Robinson + * August 2019 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_mpi_params(hid_t plist_id, MPI_Comm *comm, MPI_Info *info) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "i*Mc*Mi", plist_id, comm, info); + + /* Make sure that the property list is a fapl */ + if(TRUE != H5P_isa_class(plist_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "property list is not a file access plist") + + /* Get the plist structure */ + if(NULL == (plist = (H5P_genplist_t *)H5I_object(plist_id))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Get the properties */ + if(H5P_get(plist, H5F_ACS_MPI_PARAMS_COMM_NAME, comm) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get MPI communicator from plist") + if(H5P_get(plist, H5F_ACS_MPI_PARAMS_INFO_NAME, info) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get MPI info from plist") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_mpi_params() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pset_mpi_params + * + * Purpose: Set the MPI communicator and info + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Dana Robinson + * August 2019 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_mpi_params(hid_t plist_id, MPI_Comm comm, MPI_Info info) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "iMcMi", plist_id, comm, info); + + /* Make sure the MPI communicator is valid */ + if(MPI_COMM_NULL == comm) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "not a valid argument") + + /* Make sure that the property list is a fapl */ + if(TRUE != H5P_isa_class(plist_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "property list is not a file access plist") + + /* Get the plist structure */ + if(NULL == (plist = (H5P_genplist_t *)H5I_object(plist_id))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Set values */ + if(H5P_set(plist, H5F_ACS_MPI_PARAMS_COMM_NAME, &comm) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set MPI communicator") + if(H5P_set(plist, H5F_ACS_MPI_PARAMS_INFO_NAME, &info) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set MPI info object") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_mpi_params() */ + + +/*------------------------------------------------------------------------- + * Function: H5P__facc_mpi_comm_set + * + * Purpose: Copies an MPI comminicator property when it's set for a property list + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__facc_mpi_comm_set(hid_t H5_ATTR_UNUSED prop_id, const char H5_ATTR_UNUSED *name, + size_t H5_ATTR_UNUSED size, void *value) +{ + MPI_Comm *comm = (MPI_Comm *)value; + MPI_Comm comm_tmp = MPI_COMM_NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Make a copy of the MPI communicator */ + if(H5_mpi_comm_dup(*comm, &comm_tmp) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "unable to duplicate MPI communicator") + +done: + /* Copy the communicator to the in/out parameter */ + if(ret_value != SUCCEED) + *comm = MPI_COMM_NULL; + else + *comm = comm_tmp; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P__facc_mpi_comm_set() */ + + +/*------------------------------------------------------------------------- + * Function: H5P__facc_mpi_comm_get + * + * Purpose: Copies an MPI comminicator property when it's retrieved from a property list + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__facc_mpi_comm_get(hid_t H5_ATTR_UNUSED prop_id, const char H5_ATTR_UNUSED *name, + size_t H5_ATTR_UNUSED size, void *value) +{ + MPI_Comm *comm = (MPI_Comm *)value; + MPI_Comm comm_tmp = MPI_COMM_NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Make a copy of the MPI communicator */ + if(H5_mpi_comm_dup(*comm, &comm_tmp) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "unable to duplicate MPI communicator") + +done: + /* Copy the communicator to the out parameter */ + if(ret_value != SUCCEED) + *comm = MPI_COMM_NULL; + else + *comm = comm_tmp; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P__facc_mpi_comm_get() */ + + +/*------------------------------------------------------------------------- + * Function: H5P__facc_mpi_comm_del + * + * Purpose: Frees an MPI communicator property + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__facc_mpi_comm_del(hid_t H5_ATTR_UNUSED prop_id, const char H5_ATTR_UNUSED *name, size_t H5_ATTR_UNUSED size, void *value) +{ + MPI_Comm *comm = (MPI_Comm *)value; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Free the MPI communicator */ + if(H5_mpi_comm_free(comm) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTFREE, FAIL, "unable to free MPI communicator") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P__facc_mpi_comm_del() */ + + +/*------------------------------------------------------------------------- + * Function: H5P__facc_mpi_comm_copy + * + * Purpose: Copy callback for the MPI communicator property. + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__facc_mpi_comm_copy(const char H5_ATTR_UNUSED *name, size_t H5_ATTR_UNUSED size, void *value) +{ + MPI_Comm *comm = (MPI_Comm *)value; + MPI_Comm comm_tmp = MPI_COMM_NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Make a copy of the MPI communicator */ + if(H5_mpi_comm_dup(*comm, &comm_tmp) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "unable to duplicate MPI communicator") + +done: + /* Copy the communicator to the in/out parameter */ + if(ret_value != SUCCEED) + *comm = MPI_COMM_NULL; + else + *comm = comm_tmp; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P__facc_mpi_comm_copy() */ + + +/*------------------------------------------------------------------------- + * Function: H5P__facc_mpi_comm_cmp + * + * Purpose: Callback routine which is called whenever the MPI + * communicator property in the file access property list + * is compared. + * + * Return: positive if VALUE1 is greater than VALUE2, negative if + * VALUE2 is greater than VALUE1 and zero if VALUE1 and + * VALUE2 are equal. + * + *------------------------------------------------------------------------- + */ +static int +H5P__facc_mpi_comm_cmp(const void *_comm1, const void *_comm2, size_t H5_ATTR_UNUSED size) +{ + const MPI_Comm *comm1 = (const MPI_Comm *)_comm1; + const MPI_Comm *comm2 = (const MPI_Comm *)_comm2; + int ret_value = 0; + + FUNC_ENTER_STATIC + + /* Compare the MPI communicators */ + if(H5_mpi_comm_cmp(*comm1, *comm2, &ret_value) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, 0, "unable to compare MPI communicator") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P__facc_mpi_comm_cmp() */ + + +/*------------------------------------------------------------------------- + * Function: H5P__facc_mpi_comm_close + * + * Purpose: Close callback for the MPI communicator property. + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__facc_mpi_comm_close(const char H5_ATTR_UNUSED *name, size_t H5_ATTR_UNUSED size, void *value) +{ + MPI_Comm *comm = (MPI_Comm *)value; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Free the MPI communicator */ + if(H5_mpi_comm_free(comm) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTFREE, FAIL, "unable to free MPI communicator") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P__facc_mpi_comm_close() */ + + +/*------------------------------------------------------------------------- + * Function: H5P__facc_mpi_info_set + * + * Purpose: Copies an MPI info object property when it's set for a property list + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__facc_mpi_info_set(hid_t H5_ATTR_UNUSED prop_id, const char H5_ATTR_UNUSED *name, + size_t H5_ATTR_UNUSED size, void *value) +{ + MPI_Info *info = (MPI_Info *)value; + MPI_Info info_tmp = MPI_INFO_NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Make a copy of the MPI info object */ + if(H5_mpi_info_dup(*info, &info_tmp) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "unable to duplicate MPI info object") + +done: + /* Copy the info object to the in/out parameter */ + if(ret_value != SUCCEED) + *info = MPI_INFO_NULL; + else + *info = info_tmp; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P__facc_mpi_info_set() */ + + +/*------------------------------------------------------------------------- + * Function: H5P__facc_mpi_info_get + * + * Purpose: Copies an MPI comminicator property when it's retrieved from a property list + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__facc_mpi_info_get(hid_t H5_ATTR_UNUSED prop_id, const char H5_ATTR_UNUSED *name, + size_t H5_ATTR_UNUSED size, void *value) +{ + MPI_Info *info = (MPI_Info *)value; + MPI_Info info_tmp = MPI_INFO_NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Make a copy of the MPI communicator */ + if(H5_mpi_info_dup(*info, &info_tmp) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "unable to duplicate MPI info object") + +done: + /* Copy the info object to the out parameter */ + if(ret_value != SUCCEED) + *info = MPI_INFO_NULL; + else + *info = info_tmp; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P__facc_mpi_info_get() */ + + +/*------------------------------------------------------------------------- + * Function: H5P__facc_mpi_info_del + * + * Purpose: Frees an MPI info object property + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__facc_mpi_info_del(hid_t H5_ATTR_UNUSED prop_id, const char H5_ATTR_UNUSED *name, size_t H5_ATTR_UNUSED size, void *value) +{ + MPI_Info *info = (MPI_Info *)value; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Free the MPI info object */ + if(H5_mpi_info_free(info) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTFREE, FAIL, "unable to free MPI info object") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P__facc_mpi_info_del() */ + + +/*------------------------------------------------------------------------- + * Function: H5P__facc_mpi_info_copy + * + * Purpose: Copy callback for the MPI info object property. + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__facc_mpi_info_copy(const char H5_ATTR_UNUSED *name, size_t H5_ATTR_UNUSED size, void *value) +{ + MPI_Info *info = (MPI_Info *)value; + MPI_Info info_tmp = MPI_INFO_NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Make a copy of the MPI info object */ + if(H5_mpi_info_dup(*info, &info_tmp) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "unable to duplicate MPI info object") + +done: + /* Copy the info object to the in/out parameter */ + if(ret_value != SUCCEED) + *info = MPI_INFO_NULL; + else + *info = info_tmp; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P__facc_mpi_info_copy() */ + + +/*------------------------------------------------------------------------- + * Function: H5P__facc_mpi_info_cmp + * + * Purpose: Callback routine which is called whenever the MPI + * info object property in the file access property list + * is compared. + * + * Return: positive if VALUE1 is greater than VALUE2, negative if + * VALUE2 is greater than VALUE1 and zero if VALUE1 and + * VALUE2 are equal. + * + *------------------------------------------------------------------------- + */ +static int +H5P__facc_mpi_info_cmp(const void *_info1, const void *_info2, size_t H5_ATTR_UNUSED size) +{ + const MPI_Info *info1 = (const MPI_Info *)_info1; + const MPI_Info *info2 = (const MPI_Info *)_info2; + int ret_value = 0; + + FUNC_ENTER_STATIC + + /* Compare the MPI info objects */ + if(H5_mpi_info_cmp(*info1, *info2, &ret_value) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, 0, "unable to compare MPI info objects") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P__facc_mpi_info_cmp() */ + + +/*------------------------------------------------------------------------- + * Function: H5P__facc_mpi_info_close + * + * Purpose: Close callback for the MPI info object property. + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__facc_mpi_info_close(const char H5_ATTR_UNUSED *name, size_t H5_ATTR_UNUSED size, void *value) +{ + MPI_Info *info = (MPI_Info *)value; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Free the MPI info object */ + if(H5_mpi_info_free(info) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTFREE, FAIL, "unable to free MPI info object") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P__facc_mpi_info_close() */ + + +/*------------------------------------------------------------------------- * Function: H5Pget_coll_metadata_write * * Purpose: Gets information about collective metadata write mode. @@ -5305,4 +5820,3 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P__facc_vol_close() */ - diff --git a/src/H5Pint.c b/src/H5Pint.c index 5683403..36367d7 100644 --- a/src/H5Pint.c +++ b/src/H5Pint.c @@ -151,6 +151,10 @@ hid_t H5P_CLS_DATATYPE_CREATE_ID_g = H5I_INVALID_HID; H5P_genclass_t *H5P_CLS_DATATYPE_CREATE_g = NULL; hid_t H5P_CLS_DATATYPE_ACCESS_ID_g = H5I_INVALID_HID; H5P_genclass_t *H5P_CLS_DATATYPE_ACCESS_g = NULL; +hid_t H5P_CLS_MAP_CREATE_ID_g = H5I_INVALID_HID; +H5P_genclass_t *H5P_CLS_MAP_CREATE_g = NULL; +hid_t H5P_CLS_MAP_ACCESS_ID_g = H5I_INVALID_HID; +H5P_genclass_t *H5P_CLS_MAP_ACCESS_g = NULL; hid_t H5P_CLS_ATTRIBUTE_CREATE_ID_g = H5I_INVALID_HID; H5P_genclass_t *H5P_CLS_ATTRIBUTE_CREATE_g = NULL; hid_t H5P_CLS_ATTRIBUTE_ACCESS_ID_g = H5I_INVALID_HID; @@ -180,6 +184,8 @@ hid_t H5P_LST_GROUP_CREATE_ID_g = H5I_INVALID_HID; hid_t H5P_LST_GROUP_ACCESS_ID_g = H5I_INVALID_HID; hid_t H5P_LST_DATATYPE_CREATE_ID_g = H5I_INVALID_HID; hid_t H5P_LST_DATATYPE_ACCESS_ID_g = H5I_INVALID_HID; +hid_t H5P_LST_MAP_CREATE_ID_g = H5I_INVALID_HID; +hid_t H5P_LST_MAP_ACCESS_ID_g = H5I_INVALID_HID; hid_t H5P_LST_ATTRIBUTE_CREATE_ID_g = H5I_INVALID_HID; hid_t H5P_LST_ATTRIBUTE_ACCESS_ID_g = H5I_INVALID_HID; hid_t H5P_LST_OBJECT_COPY_ID_g = H5I_INVALID_HID; @@ -314,6 +320,7 @@ H5_DLLVAR const H5P_libclass_t H5P_CLS_STRCRT[1]; /* String create */ H5_DLLVAR const H5P_libclass_t H5P_CLS_GCRT[1]; /* Group create */ H5_DLLVAR const H5P_libclass_t H5P_CLS_FCRT[1]; /* File creation */ H5_DLLVAR const H5P_libclass_t H5P_CLS_DCRT[1]; /* Dataset creation */ +H5_DLLVAR const H5P_libclass_t H5P_CLS_MCRT[1]; /* Map creation */ H5_DLLVAR const H5P_libclass_t H5P_CLS_DXFR[1]; /* Data transfer */ H5_DLLVAR const H5P_libclass_t H5P_CLS_FMNT[1]; /* File mount */ H5_DLLVAR const H5P_libclass_t H5P_CLS_ACRT[1]; /* Attribute creation */ @@ -352,6 +359,8 @@ static H5P_libclass_t const * const init_class[] = { H5P_CLS_FMNT, /* File mount */ H5P_CLS_TCRT, /* Datatype creation */ H5P_CLS_TACC, /* Datatype access */ + H5P_CLS_MCRT, /* Map creation */ + H5P_CLS_MACC, /* Map access */ H5P_CLS_ACRT, /* Attribute creation */ H5P_CLS_AACC, /* Attribute access */ H5P_CLS_LCRT, /* Link creation */ @@ -431,7 +440,7 @@ H5P__init_package(void) FUNC_ENTER_PACKAGE /* Sanity check */ - HDcompile_assert(H5P_TYPE_VOL_INITIALIZE == (H5P_TYPE_MAX_TYPE - 1)); + HDcompile_assert(H5P_TYPE_MAP_ACCESS == (H5P_TYPE_MAX_TYPE - 1)); /* * Initialize the Generic Property class & object groups. @@ -442,8 +451,8 @@ H5P__init_package(void) HGOTO_ERROR(H5E_ATOM, H5E_CANTINIT, FAIL, "unable to initialize ID group") /* Repeatedly pass over the list of property list classes for the library, - * initializing each class if it's parent class is initialized, until no - * more progress is made. + * initializing each class if its parent class is initialized, until no + * more progress is made. */ tot_init = 0; do { @@ -547,6 +556,8 @@ H5P_term_package(void) H5P_LST_GROUP_ACCESS_ID_g = H5P_LST_DATATYPE_CREATE_ID_g = H5P_LST_DATATYPE_ACCESS_ID_g = + H5P_LST_MAP_CREATE_ID_g = + H5P_LST_MAP_ACCESS_ID_g = H5P_LST_ATTRIBUTE_CREATE_ID_g = H5P_LST_ATTRIBUTE_ACCESS_ID_g = H5P_LST_OBJECT_COPY_ID_g = @@ -574,6 +585,8 @@ H5P_term_package(void) H5P_CLS_GROUP_ACCESS_g = H5P_CLS_DATATYPE_CREATE_g = H5P_CLS_DATATYPE_ACCESS_g = + H5P_CLS_MAP_CREATE_g = + H5P_CLS_MAP_ACCESS_g = H5P_CLS_STRING_CREATE_g = H5P_CLS_ATTRIBUTE_CREATE_g = H5P_CLS_ATTRIBUTE_ACCESS_g = @@ -594,6 +607,8 @@ H5P_term_package(void) H5P_CLS_GROUP_ACCESS_ID_g = H5P_CLS_DATATYPE_CREATE_ID_g = H5P_CLS_DATATYPE_ACCESS_ID_g = + H5P_CLS_MAP_CREATE_ID_g = + H5P_CLS_MAP_ACCESS_ID_g = H5P_CLS_STRING_CREATE_ID_g = H5P_CLS_ATTRIBUTE_CREATE_ID_g = H5P_CLS_ATTRIBUTE_ACCESS_ID_g = @@ -5437,8 +5452,9 @@ H5P__new_plist_of_type(H5P_plist_type_t type) FUNC_ENTER_PACKAGE - /* Sanity check */ - HDassert(type >= H5P_TYPE_USER && type < H5P_TYPE_MAX_TYPE); + /* Sanity checks */ + HDcompile_assert(H5P_TYPE_MAP_ACCESS == (H5P_TYPE_MAX_TYPE - 1)); + HDassert(type >= H5P_TYPE_USER && type <= H5P_TYPE_MAP_ACCESS); /* Check arguments */ if(type == H5P_TYPE_USER) @@ -5492,6 +5508,14 @@ H5P__new_plist_of_type(H5P_plist_type_t type) class_id = H5P_CLS_DATATYPE_ACCESS_ID_g; break; + case H5P_TYPE_MAP_CREATE: + class_id = H5P_CLS_MAP_CREATE_ID_g; + break; + + case H5P_TYPE_MAP_ACCESS: + class_id = H5P_CLS_MAP_ACCESS_ID_g; + break; + case H5P_TYPE_STRING_CREATE: class_id = H5P_CLS_STRING_CREATE_ID_g; break; diff --git a/src/H5Pmapl.c b/src/H5Pmapl.c new file mode 100644 index 0000000..fe5be0f --- /dev/null +++ b/src/H5Pmapl.c @@ -0,0 +1,217 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Purpose: Map access property list class routines + * + *------------------------------------------------------------------------- + */ + +/****************/ +/* Module Setup */ +/****************/ + +#include "H5Pmodule.h" /* This source code file is part of the H5P module */ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5Mprivate.h" /* Maps */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Iprivate.h" /* IDs */ +#include "H5Ppkg.h" /* Property lists */ + + +/****************/ +/* Local Macros */ +/****************/ + +/* ========= Map Access properties ============ */ +/* Definitions for key prefetch size */ +#define H5M_ACS_KEY_PREFETCH_SIZE_SIZE sizeof(size_t) +#define H5M_ACS_KEY_PREFETCH_SIZE_DEF (size_t)(16 * 1024) +#define H5M_ACS_KEY_PREFETCH_SIZE_ENC H5P__encode_size_t +#define H5M_ACS_KEY_PREFETCH_SIZE_DEC H5P__decode_size_t +/* Definition for key prefetch buffer size */ +#define H5M_ACS_KEY_ALLOC_SIZE_SIZE sizeof(size_t) +#define H5M_ACS_KEY_ALLOC_SIZE_DEF (size_t)(1024 * 1024) +#define H5M_ACS_KEY_ALLOC_SIZE_ENC H5P__encode_size_t +#define H5M_ACS_KEY_ALLOC_SIZE_DEC H5P__decode_size_t + + +/******************/ +/* Local Typedefs */ +/******************/ + + +/********************/ +/* Package Typedefs */ +/********************/ + + +/********************/ +/* Local Prototypes */ +/********************/ + +/* Property class callbacks */ +static herr_t H5P__macc_reg_prop(H5P_genclass_t *pclass); + + +/*********************/ +/* Package Variables */ +/*********************/ + +/* Map access property list class library initialization object */ +const H5P_libclass_t H5P_CLS_MACC[1] = {{ + "map access", /* Class name for debugging */ + H5P_TYPE_MAP_ACCESS, /* Class type */ + + &H5P_CLS_LINK_ACCESS_g, /* Parent class */ + &H5P_CLS_MAP_ACCESS_g, /* Pointer to class */ + &H5P_CLS_MAP_ACCESS_ID_g, /* Pointer to class ID */ + &H5P_LST_MAP_ACCESS_ID_g, /* Pointer to default property list ID */ + H5P__macc_reg_prop, /* Default property registration routine */ + + NULL, /* Class creation callback */ + NULL, /* Class creation callback info */ + NULL, /* Class copy callback */ + NULL, /* Class copy callback info */ + NULL, /* Class close callback */ + NULL /* Class close callback info */ +}}; + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + + + +/*------------------------------------------------------------------------- + * Function: H5P__macc_reg_prop + * + * Purpose: Register the map access property list class's + * properties + * + * Return: Non-negative on success/Negative on failure + *------------------------------------------------------------------------- + */ +static herr_t +H5P__macc_reg_prop(H5P_genclass_t *pclass) +{ + size_t key_prefetch_size = H5M_ACS_KEY_PREFETCH_SIZE_DEF; /* Default key prefetch size for iteration */ + size_t key_alloc_size = H5M_ACS_KEY_ALLOC_SIZE_DEF; /* Default key prefetch allocation size for iteration */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Register the key prefetch size for iteration */ + if(H5P__register_real(pclass, H5M_ACS_KEY_PREFETCH_SIZE_NAME, H5M_ACS_KEY_PREFETCH_SIZE_SIZE, &key_prefetch_size, + NULL, NULL, NULL, H5M_ACS_KEY_PREFETCH_SIZE_ENC, H5M_ACS_KEY_PREFETCH_SIZE_DEC, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + + /* Register the key prefetch allocation size for iteration */ + if(H5P__register_real(pclass, H5M_ACS_KEY_ALLOC_SIZE_NAME, H5M_ACS_KEY_ALLOC_SIZE_SIZE, &key_alloc_size, + NULL, NULL, NULL, H5M_ACS_KEY_ALLOC_SIZE_ENC, H5M_ACS_KEY_ALLOC_SIZE_DEC, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P__macc_reg_prop() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pset_map_iterate_hints + * + * Purpose: H5Pset_map_iterate_hints adjusts the behavior of + * H5Miterate when prefetching keys for iteration. The + * KEY_PREFETCH_SIZE parameter specifies the number of keys + * to prefetch at a time during iteration. The KEY_ALLOC_SIZE + * parameter specifies the initial size of the buffer + * allocated to hold these prefetched keys. If this buffer is + * too small it will be reallocated to a larger size, though + * this may result in an additional I/O. + * + * Move to DAOS VOL code? DSINC + * + * Return: Non-negative on success/Negative on failure + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_map_iterate_hints(hid_t mapl_id, size_t key_prefetch_size, size_t key_alloc_size) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "izz", mapl_id, key_prefetch_size, key_alloc_size); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(mapl_id, H5P_MAP_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID"); + + /* Set sizes */ + if(H5P_set(plist, H5M_ACS_KEY_PREFETCH_SIZE_NAME, &key_prefetch_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET,FAIL, "can't set key prefetch size"); + if(H5P_set(plist, H5M_ACS_KEY_ALLOC_SIZE_NAME, &key_alloc_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET,FAIL, "can't set key allocation size"); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_map_iterate_hints() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_map_iterate_hints + * + * Purpose: Returns the map iterate hints, KEY_PREFETCH_SIZE and + * KEY_ALLOC_SIZE, as set by H5Pset_map_iterate_hints. + * + * Return: Non-negative on success/Negative on failure + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_map_iterate_hints(hid_t mapl_id, size_t *key_prefetch_size, size_t *key_alloc_size) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "i*z*z", mapl_id, key_prefetch_size, key_alloc_size); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(mapl_id, H5P_MAP_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID"); + + /* Get the properties */ + if(key_prefetch_size) { + if(H5P_get(plist, H5M_ACS_KEY_PREFETCH_SIZE_NAME, key_prefetch_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get key prefetch size"); + } /* end if */ + if(key_alloc_size) { + if(H5P_get(plist, H5M_ACS_KEY_ALLOC_SIZE_NAME, key_alloc_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get key allocation size"); + } /* end if */ + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_map_iterate_hints() */ + diff --git a/src/H5Pmcpl.c b/src/H5Pmcpl.c new file mode 100644 index 0000000..41a6659 --- /dev/null +++ b/src/H5Pmcpl.c @@ -0,0 +1,115 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: H5Pmcpl.c + * + * Purpose: Map creation property list class routines + * + *------------------------------------------------------------------------- + */ + +/****************/ +/* Module Setup */ +/****************/ + +#include "H5Pmodule.h" /* This source code file is part of the H5P module */ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5Mprivate.h" /* Maps */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Iprivate.h" /* IDs */ +#include "H5Ppkg.h" /* Property lists */ + + +/****************/ +/* Local Macros */ +/****************/ + + +/******************/ +/* Local Typedefs */ +/******************/ + + +/********************/ +/* Package Typedefs */ +/********************/ + + +/********************/ +/* Local Prototypes */ +/********************/ + +/* Property class callbacks */ +static herr_t H5P__mcrt_reg_prop(H5P_genclass_t *pclass); + + +/*********************/ +/* Package Variables */ +/*********************/ + +/* Map create property list class library initialization object */ +const H5P_libclass_t H5P_CLS_MCRT[1] = {{ + "map create", /* Class name for debugging */ + H5P_TYPE_MAP_CREATE, /* Class type */ + + &H5P_CLS_OBJECT_CREATE_g, /* Parent class */ + &H5P_CLS_MAP_CREATE_g, /* Pointer to class */ + &H5P_CLS_MAP_CREATE_ID_g, /* Pointer to class ID */ + &H5P_LST_MAP_CREATE_ID_g, /* Pointer to default property list ID */ + H5P__mcrt_reg_prop, /* Default property registration routine */ + + NULL, /* Class creation callback */ + NULL, /* Class creation callback info */ + NULL, /* Class copy callback */ + NULL, /* Class copy callback info */ + NULL, /* Class close callback */ + NULL /* Class close callback info */ +}}; + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/***************************/ +/* Local Private Variables */ +/***************************/ + + + +/*------------------------------------------------------------------------- + * Function: H5P__mcrt_reg_prop + * + * Purpose: Register the map creation property list class's properties + * + * Return: Non-negative on success/Negative on failure + *------------------------------------------------------------------------- + */ +static herr_t +H5P__mcrt_reg_prop(H5P_genclass_t H5_ATTR_UNUSED *pclass) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P__mcrt_reg_prop() */ + diff --git a/src/H5Pprivate.h b/src/H5Pprivate.h index 07910c3..9fc1acc 100644 --- a/src/H5Pprivate.h +++ b/src/H5Pprivate.h @@ -79,6 +79,8 @@ typedef enum H5P_plist_type_t { H5P_TYPE_LINK_ACCESS = 17, H5P_TYPE_ATTRIBUTE_ACCESS = 18, H5P_TYPE_VOL_INITIALIZE = 19, + H5P_TYPE_MAP_CREATE = 20, + H5P_TYPE_MAP_ACCESS = 21, H5P_TYPE_MAX_TYPE } H5P_plist_type_t; @@ -125,6 +127,8 @@ H5_DLLVAR H5P_genclass_t *H5P_CLS_GROUP_CREATE_g; H5_DLLVAR H5P_genclass_t *H5P_CLS_GROUP_ACCESS_g; H5_DLLVAR H5P_genclass_t *H5P_CLS_DATATYPE_CREATE_g; H5_DLLVAR H5P_genclass_t *H5P_CLS_DATATYPE_ACCESS_g; +H5_DLLVAR H5P_genclass_t *H5P_CLS_MAP_CREATE_g; +H5_DLLVAR H5P_genclass_t *H5P_CLS_MAP_ACCESS_g; H5_DLLVAR H5P_genclass_t *H5P_CLS_ATTRIBUTE_CREATE_g; H5_DLLVAR H5P_genclass_t *H5P_CLS_ATTRIBUTE_ACCESS_g; H5_DLLVAR H5P_genclass_t *H5P_CLS_OBJECT_COPY_g; @@ -139,6 +143,7 @@ H5_DLLVAR const struct H5P_libclass_t H5P_CLS_AACC[1]; /* Attribute access */ H5_DLLVAR const struct H5P_libclass_t H5P_CLS_DACC[1]; /* Dataset access */ H5_DLLVAR const struct H5P_libclass_t H5P_CLS_GACC[1]; /* Group access */ H5_DLLVAR const struct H5P_libclass_t H5P_CLS_TACC[1]; /* Named datatype access */ +H5_DLLVAR const struct H5P_libclass_t H5P_CLS_MACC[1]; /* Map access */ H5_DLLVAR const struct H5P_libclass_t H5P_CLS_FACC[1]; /* File access */ H5_DLLVAR const struct H5P_libclass_t H5P_CLS_OCPY[1]; /* Object copy */ diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index cad2071..9f26b8b 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -62,6 +62,8 @@ #define H5P_GROUP_ACCESS (H5OPEN H5P_CLS_GROUP_ACCESS_ID_g) #define H5P_DATATYPE_CREATE (H5OPEN H5P_CLS_DATATYPE_CREATE_ID_g) #define H5P_DATATYPE_ACCESS (H5OPEN H5P_CLS_DATATYPE_ACCESS_ID_g) +#define H5P_MAP_CREATE (H5OPEN H5P_CLS_MAP_CREATE_ID_g) +#define H5P_MAP_ACCESS (H5OPEN H5P_CLS_MAP_ACCESS_ID_g) #define H5P_STRING_CREATE (H5OPEN H5P_CLS_STRING_CREATE_ID_g) #define H5P_ATTRIBUTE_CREATE (H5OPEN H5P_CLS_ATTRIBUTE_CREATE_ID_g) #define H5P_ATTRIBUTE_ACCESS (H5OPEN H5P_CLS_ATTRIBUTE_ACCESS_ID_g) @@ -83,6 +85,8 @@ #define H5P_GROUP_ACCESS_DEFAULT (H5OPEN H5P_LST_GROUP_ACCESS_ID_g) #define H5P_DATATYPE_CREATE_DEFAULT (H5OPEN H5P_LST_DATATYPE_CREATE_ID_g) #define H5P_DATATYPE_ACCESS_DEFAULT (H5OPEN H5P_LST_DATATYPE_ACCESS_ID_g) +#define H5P_MAP_CREATE_DEFAULT (H5OPEN H5P_LST_MAP_CREATE_ID_g) +#define H5P_MAP_ACCESS_DEFAULT (H5OPEN H5P_LST_MAP_ACCESS_ID_g) #define H5P_ATTRIBUTE_CREATE_DEFAULT (H5OPEN H5P_LST_ATTRIBUTE_CREATE_ID_g) #define H5P_ATTRIBUTE_ACCESS_DEFAULT (H5OPEN H5P_LST_ATTRIBUTE_ACCESS_ID_g) #define H5P_OBJECT_COPY_DEFAULT (H5OPEN H5P_LST_OBJECT_COPY_ID_g) @@ -191,6 +195,8 @@ H5_DLLVAR hid_t H5P_CLS_GROUP_CREATE_ID_g; H5_DLLVAR hid_t H5P_CLS_GROUP_ACCESS_ID_g; H5_DLLVAR hid_t H5P_CLS_DATATYPE_CREATE_ID_g; H5_DLLVAR hid_t H5P_CLS_DATATYPE_ACCESS_ID_g; +H5_DLLVAR hid_t H5P_CLS_MAP_CREATE_ID_g; +H5_DLLVAR hid_t H5P_CLS_MAP_ACCESS_ID_g; H5_DLLVAR hid_t H5P_CLS_STRING_CREATE_ID_g; H5_DLLVAR hid_t H5P_CLS_ATTRIBUTE_CREATE_ID_g; H5_DLLVAR hid_t H5P_CLS_ATTRIBUTE_ACCESS_ID_g; @@ -211,6 +217,8 @@ H5_DLLVAR hid_t H5P_LST_GROUP_CREATE_ID_g; H5_DLLVAR hid_t H5P_LST_GROUP_ACCESS_ID_g; H5_DLLVAR hid_t H5P_LST_DATATYPE_CREATE_ID_g; H5_DLLVAR hid_t H5P_LST_DATATYPE_ACCESS_ID_g; +H5_DLLVAR hid_t H5P_LST_MAP_CREATE_ID_g; +H5_DLLVAR hid_t H5P_LST_MAP_ACCESS_ID_g; H5_DLLVAR hid_t H5P_LST_ATTRIBUTE_CREATE_ID_g; H5_DLLVAR hid_t H5P_LST_ATTRIBUTE_ACCESS_ID_g; H5_DLLVAR hid_t H5P_LST_OBJECT_COPY_ID_g; @@ -373,6 +381,8 @@ H5_DLL herr_t H5Pset_all_coll_metadata_ops(hid_t plist_id, hbool_t is_collective H5_DLL herr_t H5Pget_all_coll_metadata_ops(hid_t plist_id, hbool_t *is_collective); H5_DLL herr_t H5Pset_coll_metadata_write(hid_t plist_id, hbool_t is_collective); H5_DLL herr_t H5Pget_coll_metadata_write(hid_t plist_id, hbool_t *is_collective); +H5_DLL herr_t H5Pget_mpi_params(hid_t fapl_id, MPI_Comm *comm, MPI_Info *info); +H5_DLL herr_t H5Pset_mpi_params(hid_t fapl_id, MPI_Comm comm, MPI_Info info); #endif /* H5_HAVE_PARALLEL */ H5_DLL herr_t H5Pset_mdc_image_config(hid_t plist_id, H5AC_cache_image_config_t *config_ptr); H5_DLL herr_t H5Pget_mdc_image_config(hid_t plist_id, H5AC_cache_image_config_t *config_ptr /*out*/); @@ -491,6 +501,12 @@ H5_DLL herr_t H5Pget_est_link_info(hid_t plist_id, unsigned *est_num_entries /* H5_DLL herr_t H5Pset_link_creation_order(hid_t plist_id, unsigned crt_order_flags); H5_DLL herr_t H5Pget_link_creation_order(hid_t plist_id, unsigned *crt_order_flags /* out */); +/* Map access property list (MAPL) routines */ +#ifdef H5_HAVE_MAP_API +H5_DLL herr_t H5Pset_map_iterate_hints(hid_t mapl_id, size_t key_prefetch_size, size_t key_alloc_size); +H5_DLL herr_t H5Pget_map_iterate_hints(hid_t mapl_id, size_t *key_prefetch_size /*out*/, size_t *key_alloc_size /*out*/); +#endif /* H5_HAVE_MAP_API */ + /* String creation property list (STRCPL) routines */ H5_DLL herr_t H5Pset_char_encoding(hid_t plist_id, H5T_cset_t encoding); H5_DLL herr_t H5Pget_char_encoding(hid_t plist_id, H5T_cset_t *encoding /*out*/); diff --git a/src/H5Rint.c b/src/H5Rint.c index 2355ec1..07efae2 100644 --- a/src/H5Rint.c +++ b/src/H5Rint.c @@ -472,6 +472,9 @@ H5R__dereference(H5F_t *file, hid_t oapl_id, H5R_type_t ref_type, const void *_r break; } + case H5O_TYPE_MAP: + HGOTO_ERROR(H5E_REFERENCE, H5E_BADTYPE, H5I_INVALID_HID, "maps not supported in native VOL connector") + case H5O_TYPE_UNKNOWN: case H5O_TYPE_NTYPES: default: diff --git a/src/H5Sall.c b/src/H5Sall.c index fbe7041..9d0a65a 100644 --- a/src/H5Sall.c +++ b/src/H5Sall.c @@ -28,11 +28,11 @@ /***********/ /* Headers */ /***********/ -#include "H5private.h" /* Generic Functions */ -#include "H5Eprivate.h" /* Error handling */ -#include "H5Iprivate.h" /* ID Functions */ -#include "H5Spkg.h" /* Dataspace functions */ -#include "H5VMprivate.h" /* Vector functions */ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Iprivate.h" /* ID Functions */ +#include "H5Spkg.h" /* Dataspace functions */ +#include "H5VMprivate.h" /* Vector functions */ /****************/ @@ -63,6 +63,8 @@ static htri_t H5S__all_is_contiguous(const H5S_t *space); static htri_t H5S__all_is_single(const H5S_t *space); static htri_t H5S__all_is_regular(const H5S_t *space); static htri_t H5S__all_shape_same(const H5S_t *space1, const H5S_t *space2); +static htri_t H5S__all_intersect_block(const H5S_t *space, const hsize_t *start, + const hsize_t *end); static herr_t H5S__all_adjust_u(H5S_t *space, const hsize_t *offset); static herr_t H5S__all_project_scalar(const H5S_t *space, hsize_t *offset); static herr_t H5S__all_project_simple(const H5S_t *space, H5S_t *new_space, hsize_t *offset); @@ -108,6 +110,7 @@ const H5S_select_class_t H5S_sel_all[1] = {{ H5S__all_is_single, H5S__all_is_regular, H5S__all_shape_same, + H5S__all_intersect_block, H5S__all_adjust_u, H5S__all_project_scalar, H5S__all_project_simple, @@ -976,6 +979,41 @@ done: /*-------------------------------------------------------------------------- NAME + H5S__all_intersect_block + PURPOSE + Detect intersections of selection with block + USAGE + htri_t H5S__all_intersect_block(space, start, end) + const H5S_t *space; IN: Dataspace with selection to use + const hsize_t *start; IN: Starting coordinate for block + const hsize_t *end; IN: Ending coordinate for block + RETURNS + Non-negative TRUE / FALSE on success, negative on failure + DESCRIPTION + Quickly detect intersections with a block + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +htri_t +H5S__all_intersect_block(const H5S_t H5_ATTR_UNUSED *space, + const hsize_t H5_ATTR_UNUSED *start, const hsize_t H5_ATTR_UNUSED *end) +{ + FUNC_ENTER_STATIC_NOERR + + /* Sanity check */ + HDassert(space); + HDassert(H5S_SEL_ALL == H5S_GET_SELECT_TYPE(space)); + HDassert(start); + HDassert(end); + + FUNC_LEAVE_NOAPI(TRUE) +} /* end H5S__all_intersect_block() */ + + +/*-------------------------------------------------------------------------- + NAME H5S__all_adjust_u PURPOSE Adjust an "all" selection by subtracting an offset diff --git a/src/H5Shyper.c b/src/H5Shyper.c index 2acc135..153f4be 100644 --- a/src/H5Shyper.c +++ b/src/H5Shyper.c @@ -28,30 +28,20 @@ /***********/ /* Headers */ /***********/ -#include "H5private.h" /* Generic Functions */ -#include "H5CXprivate.h" /* API Contexts */ -#include "H5Eprivate.h" /* Error handling */ -#include "H5FLprivate.h" /* Free Lists */ -#include "H5Iprivate.h" /* ID Functions */ -#include "H5MMprivate.h" /* Memory management */ -#include "H5Spkg.h" /* Dataspace functions */ -#include "H5VMprivate.h" /* Vector functions */ +#include "H5private.h" /* Generic Functions */ +#include "H5CXprivate.h" /* API Contexts */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5Iprivate.h" /* ID Functions */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5Spkg.h" /* Dataspace functions */ +#include "H5VMprivate.h" /* Vector functions */ /****************/ /* Local Macros */ /****************/ -/* Macro for checking if two ranges overlap one another */ -/* - * Check for the inverse of whether the ranges are disjoint. If they are - * disjoint, then the low bound of one of the ranges must be greater than the - * high bound of the other. - */ -/* (Assumes that low & high bounds are _inclusive_) */ -#define H5S_RANGE_OVERLAP(L1, H1, L2, H2) \ - (!((L1) > (H2) || (L2) > (H1))) - /* Flags for which hyperslab fragments to compute */ #define H5S_HYPER_COMPUTE_B_NOT_A 0x01 #define H5S_HYPER_COMPUTE_A_AND_B 0x02 @@ -197,6 +187,8 @@ static htri_t H5S__hyper_is_contiguous(const H5S_t *space); static htri_t H5S__hyper_is_single(const H5S_t *space); static htri_t H5S__hyper_is_regular(const H5S_t *space); static htri_t H5S__hyper_shape_same(const H5S_t *space1, const H5S_t *space2); +static htri_t H5S__hyper_intersect_block(const H5S_t *space, const hsize_t *start, + const hsize_t *end); static herr_t H5S__hyper_adjust_u(H5S_t *space, const hsize_t *offset); static herr_t H5S__hyper_project_scalar(const H5S_t *space, hsize_t *offset); static herr_t H5S__hyper_project_simple(const H5S_t *space, H5S_t *new_space, hsize_t *offset); @@ -242,6 +234,7 @@ const H5S_select_class_t H5S_sel_hyper[1] = {{ H5S__hyper_is_single, H5S__hyper_is_regular, H5S__hyper_shape_same, + H5S__hyper_intersect_block, H5S__hyper_adjust_u, H5S__hyper_project_scalar, H5S__hyper_project_simple, @@ -4829,7 +4822,6 @@ static herr_t H5S__hyper_bounds(const H5S_t *space, hsize_t *start, hsize_t *end) { const hsize_t *low_bounds, *high_bounds; /* Pointers to the correct pair of low & high bounds */ - unsigned u; /* Local index variable */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC @@ -4849,22 +4841,32 @@ H5S__hyper_bounds(const H5S_t *space, hsize_t *start, hsize_t *end) high_bounds = space->select.sel_info.hslab->span_lst->high_bounds; } /* end else */ - /* Loop over dimensions */ - for(u = 0; u < space->extent.rank; u++) { - /* Sanity check */ - HDassert(low_bounds[u] <= high_bounds[u]); + /* Check for offset set */ + if(space->select.offset_changed) { + unsigned u; /* Local index variable */ - /* Check for offset moving selection negative */ - if(((hssize_t)low_bounds[u] + space->select.offset[u]) < 0) - HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "offset moves selection out of bounds") + /* Loop over dimensions */ + for(u = 0; u < space->extent.rank; u++) { + /* Sanity check */ + HDassert(low_bounds[u] <= high_bounds[u]); - /* Set the low & high bounds in this dimension */ - start[u] = (hsize_t)((hssize_t)low_bounds[u] + space->select.offset[u]); - if((int)u == space->select.sel_info.hslab->unlim_dim) - end[u] = H5S_UNLIMITED; - else - end[u] = (hsize_t)((hssize_t)high_bounds[u] + space->select.offset[u]); - } /* end for */ + /* Check for offset moving selection negative */ + if(((hssize_t)low_bounds[u] + space->select.offset[u]) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "offset moves selection out of bounds") + + /* Set the low & high bounds in this dimension */ + start[u] = (hsize_t)((hssize_t)low_bounds[u] + space->select.offset[u]); + if((int)u == space->select.sel_info.hslab->unlim_dim) + end[u] = H5S_UNLIMITED; + else + end[u] = (hsize_t)((hssize_t)high_bounds[u] + space->select.offset[u]); + } /* end for */ + } /* end if */ + else { + /* Offset vector is still zeros, just copy low & high bounds */ + H5MM_memcpy(start, low_bounds, sizeof(hsize_t) * space->extent.rank); + H5MM_memcpy(end, high_bounds, sizeof(hsize_t) * space->extent.rank); + } /* end else */ done: FUNC_LEAVE_NOAPI(ret_value) @@ -6240,32 +6242,31 @@ done: /*-------------------------------------------------------------------------- NAME - H5S_hyper_intersect_block + H5S__hyper_intersect_block PURPOSE - Detect intersections in span trees + Detect intersections of selection with block USAGE - htri_t H5S_hyper_intersect_block(space, start, end) - H5S_t *space; IN: First dataspace to operate on span tree - hssize_t *start; IN: Starting coordinate for block - hssize_t *end; IN: Ending coordinate for block + htri_t H5S__hyper_intersect_block(space, start, end) + const H5S_t *space; IN: Dataspace with selection to use + const hsize_t *start; IN: Starting coordinate for block + const hsize_t *end; IN: Ending coordinate for block RETURNS - Non-negative on success, negative on failure + Non-negative TRUE / FALSE on success, negative on failure DESCRIPTION - Quickly detect intersections between span tree and block + Quickly detect intersections between both regular hyperslabs and span trees + with a block GLOBAL VARIABLES COMMENTS, BUGS, ASSUMPTIONS Does not use selection offset. EXAMPLES REVISION LOG --------------------------------------------------------------------------*/ -htri_t -H5S_hyper_intersect_block(H5S_t *space, const hsize_t *start, const hsize_t *end) +static htri_t +H5S__hyper_intersect_block(const H5S_t *space, const hsize_t *start, const hsize_t *end) { - const hsize_t *low_bounds, *high_bounds; /* Pointers to the correct pair of low & high bounds */ - unsigned u; /* Local index variable */ htri_t ret_value = FAIL; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_STATIC /* Sanity check */ HDassert(space); @@ -6277,27 +6278,12 @@ H5S_hyper_intersect_block(H5S_t *space, const hsize_t *start, const hsize_t *end * to be impossible. */ if(space->select.sel_info.hslab->diminfo_valid == H5S_DIMINFO_VALID_NO) - H5S__hyper_rebuild(space); - - /* Check which set of low & high bounds we should be using */ - if(space->select.sel_info.hslab->diminfo_valid == H5S_DIMINFO_VALID_YES) { - low_bounds = space->select.sel_info.hslab->diminfo.low_bounds; - high_bounds = space->select.sel_info.hslab->diminfo.high_bounds; - } /* end if */ - else { - low_bounds = space->select.sel_info.hslab->span_lst->low_bounds; - high_bounds = space->select.sel_info.hslab->span_lst->high_bounds; - } /* end else */ - - /* Loop over selection bounds and block, checking for overlap */ - for(u = 0; u < space->extent.rank; u++) - /* If selection bounds & block don't overlap, can leave now */ - if(!H5S_RANGE_OVERLAP(low_bounds[u], high_bounds[u], start[u], end[u])) - HGOTO_DONE(FALSE) + H5S__hyper_rebuild((H5S_t *)space); /* Casting away const OK -QAK */ /* Check for regular hyperslab intersection */ if(space->select.sel_info.hslab->diminfo_valid == H5S_DIMINFO_VALID_YES) { hbool_t single_block; /* Whether the regular selection is a single block */ + unsigned u; /* Local index variable */ /* Check for a single block */ /* For a regular hyperslab to be single, it must have only one block @@ -6381,7 +6367,7 @@ H5S_hyper_intersect_block(H5S_t *space, const hsize_t *start, const hsize_t *end done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5S_hyper_intersect_block() */ +} /* end H5S__hyper_intersect_block() */ /*-------------------------------------------------------------------------- @@ -7039,6 +7025,47 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5S_hyper_adjust_s() */ +/*-------------------------------------------------------------------------- + NAME + H5Shyper_adjust_s + PURPOSE + Adjust a hyperslab selection by subtracting an offset + USAGE + herr_t H5Shyper_adjust_s(space_id,offset) + hid_t space_id; IN: ID of the dataspace to adjust + const hssize_t *offset; IN: Offset to subtract + RETURNS + Non-negative on success, negative on failure + DESCRIPTION + Moves a hyperslab selection by subtracting an offset from it. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +herr_t +H5Shyper_adjust_s(hid_t space_id, const hssize_t *offset) +{ + H5S_t *space; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*Hs", space_id, offset); + + if(NULL == (space = (H5S_t *)H5I_object_verify(space_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "not a dataspace") + if(H5S_GET_SELECT_TYPE(space) != H5S_SEL_HYPERSLABS) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a hyperslab selection") + if(NULL == offset) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "NULL offset pointer") + + if(H5S_hyper_adjust_s(space, offset) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTSET, FAIL, "can't adjust selection") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Shyper_adjust_s() */ + /*-------------------------------------------------------------------------- NAME diff --git a/src/H5Snone.c b/src/H5Snone.c index 9d64e9d..c262d18 100644 --- a/src/H5Snone.c +++ b/src/H5Snone.c @@ -28,11 +28,11 @@ /***********/ /* Headers */ /***********/ -#include "H5private.h" /* Generic Functions */ -#include "H5Eprivate.h" /* Error handling */ -#include "H5Iprivate.h" /* ID Functions */ -#include "H5Spkg.h" /* Dataspace functions */ -#include "H5VMprivate.h" /* Vector functions */ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Iprivate.h" /* ID Functions */ +#include "H5Spkg.h" /* Dataspace functions */ +#include "H5VMprivate.h" /* Vector functions */ /****************/ @@ -63,9 +63,12 @@ static htri_t H5S__none_is_contiguous(const H5S_t *space); static htri_t H5S__none_is_single(const H5S_t *space); static htri_t H5S__none_is_regular(const H5S_t *space); static htri_t H5S__none_shape_same(const H5S_t *space1, const H5S_t *space2); +static htri_t H5S__none_intersect_block(const H5S_t *space, const hsize_t *start, + const hsize_t *end); static herr_t H5S__none_adjust_u(H5S_t *space, const hsize_t *offset); static herr_t H5S__none_project_scalar(const H5S_t *space, hsize_t *offset); -static herr_t H5S__none_project_simple(const H5S_t *space, H5S_t *new_space, hsize_t *offset); +static herr_t H5S__none_project_simple(const H5S_t *space, H5S_t *new_space, + hsize_t *offset); static herr_t H5S__none_iter_init(const H5S_t *space, H5S_sel_iter_t *iter); /* Selection iteration callbacks */ @@ -108,6 +111,7 @@ const H5S_select_class_t H5S_sel_none[1] = {{ H5S__none_is_single, H5S__none_is_regular, H5S__none_shape_same, + H5S__none_intersect_block, H5S__none_adjust_u, H5S__none_project_scalar, H5S__none_project_simple, @@ -886,6 +890,41 @@ H5S__none_shape_same(const H5S_t H5_ATTR_UNUSED *space1, /*-------------------------------------------------------------------------- NAME + H5S__none_intersect_block + PURPOSE + Detect intersections of selection with block + USAGE + htri_t H5S__none_intersect_block(space, start, end) + const H5S_t *space; IN: Dataspace with selection to use + const hsize_t *start; IN: Starting coordinate for block + const hsize_t *end; IN: Ending coordinate for block + RETURNS + Non-negative TRUE / FALSE on success, negative on failure + DESCRIPTION + Quickly detect intersections with a block + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +htri_t +H5S__none_intersect_block(const H5S_t H5_ATTR_UNUSED *space, + const hsize_t H5_ATTR_UNUSED *start, const hsize_t H5_ATTR_UNUSED *end) +{ + FUNC_ENTER_STATIC_NOERR + + /* Sanity check */ + HDassert(space); + HDassert(H5S_SEL_NONE == H5S_GET_SELECT_TYPE(space)); + HDassert(start); + HDassert(end); + + FUNC_LEAVE_NOAPI(FALSE) +} /* end H5S__none_intersect_block() */ + + +/*-------------------------------------------------------------------------- + NAME H5S__none_adjust_u PURPOSE Adjust an "none" selection by subtracting an offset diff --git a/src/H5Spkg.h b/src/H5Spkg.h index 4752c59..278f08d 100644 --- a/src/H5Spkg.h +++ b/src/H5Spkg.h @@ -91,6 +91,15 @@ * H5S_UNLIMITED) */ #define H5S_MAX_SIZE ((hsize_t)(hssize_t)(-2)) +/* Macro for checking if two ranges overlap one another */ +/* + * Check for the inverse of whether the ranges are disjoint. If they are + * disjoint, then the low bound of one of the ranges must be greater than the + * high bound of the other. + */ +/* (Assumes that low & high bounds are _inclusive_) */ +#define H5S_RANGE_OVERLAP(L1, H1, L2, H2) (!((L1) > (H2) || (L2) > (H1))) + /* * Dataspace extent information @@ -240,6 +249,8 @@ typedef htri_t (*H5S_sel_is_single_func_t)(const H5S_t *space); typedef htri_t (*H5S_sel_is_regular_func_t)(const H5S_t *space); /* Method to determine if two dataspaces' selections are the same shape */ typedef htri_t (*H5S_sel_shape_same_func_t)(const H5S_t *space1, const H5S_t *space2); +/* Method to determine if selection intersects a block */ +typedef htri_t (*H5S_sel_intersect_block_func_t)(const H5S_t *space, const hsize_t *start, const hsize_t *end); /* Method to adjust a selection by an offset */ typedef herr_t (*H5S_sel_adjust_u_func_t)(H5S_t *space, const hsize_t *offset); /* Method to construct single element projection onto scalar dataspace */ @@ -268,6 +279,7 @@ typedef struct { H5S_sel_is_single_func_t is_single; /* Method to determine if current selection is a single block */ H5S_sel_is_regular_func_t is_regular; /* Method to determine if current selection is "regular" */ H5S_sel_shape_same_func_t shape_same; /* Method to determine if two dataspaces' selections are the same shape */ + H5S_sel_intersect_block_func_t intersect_block; /* Method to determine if a dataspaces' selection intersects a block */ H5S_sel_adjust_u_func_t adjust_u; /* Method to adjust a selection by an offset */ H5S_sel_project_scalar project_scalar; /* Method to construct scalar dataspace projection */ H5S_sel_project_simple project_simple; /* Method to construct simple dataspace projection */ @@ -369,7 +381,6 @@ H5_DLL herr_t H5S__hyper_project_intersection(const H5S_t *src_space, /* Testing functions */ #ifdef H5S_TESTING -H5_DLL htri_t H5S__select_shape_same_test(hid_t sid1, hid_t sid2); H5_DLL herr_t H5S__get_rebuild_status_test(hid_t space_id, H5S_diminfo_valid_t *status1, H5S_diminfo_valid_t *status2); H5_DLL herr_t H5S__get_diminfo_status_test(hid_t space_id, diff --git a/src/H5Spoint.c b/src/H5Spoint.c index 875c018..8e1175a 100644 --- a/src/H5Spoint.c +++ b/src/H5Spoint.c @@ -28,14 +28,14 @@ /***********/ /* Headers */ /***********/ -#include "H5private.h" /* Generic Functions */ -#include "H5CXprivate.h" /* API Contexts */ -#include "H5Eprivate.h" /* Error handling */ -#include "H5FLprivate.h" /* Free Lists */ -#include "H5Iprivate.h" /* ID Functions */ -#include "H5MMprivate.h" /* Memory management */ -#include "H5Spkg.h" /* Dataspace functions */ -#include "H5VMprivate.h" /* Vector functions */ +#include "H5private.h" /* Generic Functions */ +#include "H5CXprivate.h" /* API Contexts */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5Iprivate.h" /* ID Functions */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5Spkg.h" /* Dataspace functions */ +#include "H5VMprivate.h" /* Vector functions */ /****************/ @@ -75,6 +75,8 @@ static htri_t H5S__point_is_contiguous(const H5S_t *space); static htri_t H5S__point_is_single(const H5S_t *space); static htri_t H5S__point_is_regular(const H5S_t *space); static htri_t H5S__point_shape_same(const H5S_t *space1, const H5S_t *space2); +static htri_t H5S__point_intersect_block(const H5S_t *space, const hsize_t *start, + const hsize_t *end); static herr_t H5S__point_adjust_u(H5S_t *space, const hsize_t *offset); static herr_t H5S__point_project_scalar(const H5S_t *space, hsize_t *offset); static herr_t H5S__point_project_simple(const H5S_t *space, H5S_t *new_space, @@ -124,6 +126,7 @@ const H5S_select_class_t H5S_sel_point[1] = {{ H5S__point_is_single, H5S__point_is_regular, H5S__point_shape_same, + H5S__point_intersect_block, H5S__point_adjust_u, H5S__point_project_scalar, H5S__point_project_simple, @@ -1998,6 +2001,63 @@ done: /*-------------------------------------------------------------------------- NAME + H5S__point_intersect_block + PURPOSE + Detect intersections of selection with block + USAGE + htri_t H5S__point_intersect_block(space, start, end) + const H5S_t *space; IN: Dataspace with selection to use + const hsize_t *start; IN: Starting coordinate for block + const hsize_t *end; IN: Ending coordinate for block + RETURNS + Non-negative TRUE / FALSE on success, negative on failure + DESCRIPTION + Quickly detect intersections with a block + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +htri_t +H5S__point_intersect_block(const H5S_t *space, const hsize_t *start, + const hsize_t *end) +{ + H5S_pnt_node_t *pnt; /* Point information node */ + htri_t ret_value = FALSE; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Sanity check */ + HDassert(space); + HDassert(H5S_SEL_POINTS == H5S_GET_SELECT_TYPE(space)); + HDassert(start); + HDassert(end); + + /* Loop over points */ + pnt = space->select.sel_info.pnt_lst->head; + while(pnt) { + unsigned u; /* Local index variable */ + + /* Verify that the point is within the block */ + for(u = 0; u < space->extent.rank; u++) + if(pnt->pnt[u] < start[u] || pnt->pnt[u] > end[u]) + break; + + /* Check if point was within block for all dimensions */ + if(u == space->extent.rank) + HGOTO_DONE(TRUE) + + /* Advance to next point */ + pnt = pnt->next; + } /* end while */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5S__point_intersect_block() */ + + +/*-------------------------------------------------------------------------- + NAME H5S__point_adjust_u PURPOSE Adjust a "point" selection by subtracting an offset diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h index 6646f84..0a9d2e7 100644 --- a/src/H5Sprivate.h +++ b/src/H5Sprivate.h @@ -30,15 +30,6 @@ #include "H5Pprivate.h" /* Property lists */ #include "H5Tprivate.h" /* Datatypes */ -/* Flags for H5S_find */ -#define H5S_CONV_PAR_IO_POSSIBLE 0x0001 -/* The storage options are mutually exclusive */ -/* (2-bits reserved for storage type currently) */ -#define H5S_CONV_STORAGE_COMPACT 0x0000 /* i.e. '0' */ -#define H5S_CONV_STORAGE_CONTIGUOUS 0x0002 /* i.e. '1' */ -#define H5S_CONV_STORAGE_CHUNKED 0x0004 /* i.e. '2' */ -#define H5S_CONV_STORAGE_MASK 0x0006 - /* Forward references of package typedefs */ typedef struct H5S_extent_t H5S_extent_t; typedef struct H5S_pnt_node_t H5S_pnt_node_t; @@ -192,6 +183,8 @@ typedef struct H5S_sel_iter_op_t { #endif /* H5S_MODULE */ /* Handle these callbacks in a special way, since they have prologs that need to be executed */ #define H5S_SELECT_COPY(DST,SRC,SHARE) (H5S_select_copy(DST,SRC,SHARE)) +#define H5S_SELECT_SHAPE_SAME(S1,S2) (H5S_select_shape_same(S1,S2)) +#define H5S_SELECT_INTERSECT_BLOCK(S,START,END) (H5S_select_intersect_block(S,START,END)) #define H5S_SELECT_RELEASE(S) (H5S_select_release(S)) #define H5S_SELECT_DESERIALIZE(S,BUF) (H5S_select_deserialize(S,BUF)) @@ -252,6 +245,8 @@ H5_DLL herr_t H5S_get_select_num_elem_non_unlim(const H5S_t *space, H5_DLL herr_t H5S_select_offset(H5S_t *space, const hssize_t *offset); H5_DLL herr_t H5S_select_copy(H5S_t *dst, const H5S_t *src, hbool_t share_selection); H5_DLL htri_t H5S_select_shape_same(const H5S_t *space1, const H5S_t *space2); +H5_DLL htri_t H5S_select_intersect_block(const H5S_t *space, const hsize_t *start, + const hsize_t *end); H5_DLL herr_t H5S_select_construct_projection(const H5S_t *base_space, H5S_t **new_space_ptr, unsigned new_space_rank, const void *buf, void const **adj_buf_ptr, hsize_t element_size); @@ -287,7 +282,6 @@ H5_DLL herr_t H5S_combine_hyperslab(H5S_t *old_space, H5S_seloper_t op, const hsize_t *block, H5S_t **new_space); H5_DLL herr_t H5S_hyper_add_span_element(H5S_t *space, unsigned rank, const hsize_t *coords); -H5_DLL htri_t H5S_hyper_intersect_block(H5S_t *space, const hsize_t *start, const hsize_t *end); H5_DLL herr_t H5S_hyper_adjust_s(H5S_t *space, const hssize_t *offset); H5_DLL htri_t H5S_hyper_normalize_offset(H5S_t *space, hssize_t *old_offset); H5_DLL herr_t H5S_hyper_denormalize_offset(H5S_t *space, const hssize_t *old_offset); diff --git a/src/H5Spublic.h b/src/H5Spublic.h index 23e6846..a04f3c1 100644 --- a/src/H5Spublic.h +++ b/src/H5Spublic.h @@ -143,8 +143,12 @@ H5_DLL H5S_sel_type H5Sget_select_type(hid_t spaceid); H5_DLL hssize_t H5Sget_select_npoints(hid_t spaceid); H5_DLL herr_t H5Sselect_copy(hid_t dst_id, hid_t src_id); H5_DLL htri_t H5Sselect_valid(hid_t spaceid); +H5_DLL herr_t H5Sselect_adjust_u(hid_t spaceid, const hsize_t *offset); H5_DLL herr_t H5Sget_select_bounds(hid_t spaceid, hsize_t start[], hsize_t end[]); +H5_DLL htri_t H5Sselect_shape_same(hid_t space1_id, hid_t space2_id); +H5_DLL htri_t H5Sselect_intersect_block(hid_t space_id, const hsize_t *start, + const hsize_t *end); H5_DLL herr_t H5Soffset_simple(hid_t space_id, const hssize_t *offset); H5_DLL herr_t H5Sselect_all(hid_t spaceid); H5_DLL herr_t H5Sselect_none(hid_t spaceid); @@ -167,6 +171,7 @@ H5_DLL htri_t H5Sget_regular_hyperslab(hid_t spaceid, hsize_t start[], H5_DLL hssize_t H5Sget_select_hyper_nblocks(hid_t spaceid); H5_DLL herr_t H5Sget_select_hyper_blocklist(hid_t spaceid, hsize_t startblock, hsize_t numblocks, hsize_t buf[/*numblocks*/]); +H5_DLL herr_t H5Shyper_adjust_s(hid_t space_id, const hssize_t *offset); /* Operations on dataspace selection iterators */ H5_DLL hid_t H5Ssel_iter_create(hid_t spaceid, size_t elmt_size, unsigned flags); diff --git a/src/H5Sselect.c b/src/H5Sselect.c index c383fed..1a13f2c 100644 --- a/src/H5Sselect.c +++ b/src/H5Sselect.c @@ -250,11 +250,15 @@ H5S_select_copy(H5S_t *dst, const H5S_t *src, hbool_t share_selection) HDassert(dst); HDassert(src); + /* Release the current selection */ + if(H5S_SELECT_RELEASE(dst) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTRELEASE, FAIL, "unable to release selection") + /* Copy regular fields */ dst->select = src->select; /* Perform correct type of copy based on the type of selection */ - if((ret_value = (*src->select.type->copy)(dst,src,share_selection)) < 0) + if((ret_value = (*src->select.type->copy)(dst, src, share_selection)) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCOPY, FAIL, "can't copy selection specific information") done: @@ -281,7 +285,7 @@ done: herr_t H5S_select_release(H5S_t *ds) { - herr_t ret_value = FAIL; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT @@ -967,6 +971,45 @@ H5S_select_adjust_u(H5S_t *space, const hsize_t *offset) FUNC_LEAVE_NOAPI(ret_value) } /* end H5S_select_adjust_u() */ +/*-------------------------------------------------------------------------- + NAME + H5Sselect_adjust_u + PURPOSE + Adjust a selection by subtracting an offset + USAGE + herr_t H5Sselect_adjust_u(space_id, offset) + hid_t space_id; IN: ID of dataspace to adjust + const hsize_t *offset; IN: Offset to subtract + RETURNS + Non-negative on success, negative on failure + DESCRIPTION + Moves a selection by subtracting an offset from it. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +herr_t +H5Sselect_adjust_u(hid_t space_id, const hsize_t *offset) +{ + H5S_t *space; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*h", space_id, offset); + + if(NULL == (space = (H5S_t *)H5I_object_verify(space_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "not a dataspace") + if(NULL == offset) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "NULL offset pointer") + + if(H5S_select_adjust_u(space, offset) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTSET, FAIL, "can't adjust selection"); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Sselect_adjust_u() */ + /*-------------------------------------------------------------------------- NAME @@ -1689,9 +1732,12 @@ H5S_get_select_type(const H5S_t *space) DESCRIPTION Checks to see if the current selection in the dataspaces are the same dimensionality and shape. + This is primarily used for reading the entire selection in one swoop. GLOBAL VARIABLES COMMENTS, BUGS, ASSUMPTIONS + This routine participates in the "Inlining C function pointers" pattern, + don't call it directly, use the appropriate macro defined in H5Sprivate.h. EXAMPLES REVISION LOG --------------------------------------------------------------------------*/ @@ -1945,6 +1991,164 @@ done: /*-------------------------------------------------------------------------- NAME + H5Sselect_shape_same + PURPOSE + Check if two selections are the same shape + USAGE + htri_t H5Sselect_shape_same(space1_id, space2_id) + hid_t space1_id; IN: ID of 1st Dataspace pointer to compare + hid_t space2_id; IN: ID of 2nd Dataspace pointer to compare + RETURNS + TRUE/FALSE/FAIL + DESCRIPTION + Checks to see if the current selection in the dataspaces are the same + dimensionality and shape. + This is primarily used for reading the entire selection in one swoop. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +htri_t +H5Sselect_shape_same(hid_t space1_id, hid_t space2_id) +{ + H5S_t *space1, *space2; /* Dataspaces to compare */ + htri_t ret_value; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("t", "ii", space1_id, space2_id); + + if(NULL == (space1 = (H5S_t *)H5I_object_verify(space1_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "not a dataspace") + if(NULL == (space2 = (H5S_t *)H5I_object_verify(space2_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "not a dataspace") + + if((ret_value = H5S_select_shape_same(space1, space2)) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCOMPARE, FAIL, "can't compare selections") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Sselect_shape_same() */ + + +/*-------------------------------------------------------------------------- + NAME + H5S_select_intersect_block + PURPOSE + Check if current selection intersects with a block + USAGE + htri_t H5S_select_intersect_block(space, start, end) + const H5S_t *space; IN: Dataspace to compare + const hsize_t *start; IN: Starting coordinate of block + const hsize_t *end; IN: Opposite ("ending") coordinate of block + RETURNS + TRUE / FALSE / FAIL + DESCRIPTION + Checks to see if the current selection in the dataspace intersects with + the block given. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + Assumes that start & end block bounds are _inclusive_, so start == end + value OK. + + This routine participates in the "Inlining C function pointers" pattern, + don't call it directly, use the appropriate macro defined in H5Sprivate.h. +--------------------------------------------------------------------------*/ +htri_t +H5S_select_intersect_block(const H5S_t *space, const hsize_t *start, + const hsize_t *end) +{ + htri_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Check args */ + HDassert(space); + HDassert(start); + HDassert(end); + + /* If selections aren't "none", compare their bounds */ + if(H5S_SEL_NONE != H5S_GET_SELECT_TYPE(space)) { + hsize_t low[H5S_MAX_RANK]; /* Low bound of selection in dataspace */ + hsize_t high[H5S_MAX_RANK]; /* High bound of selection in dataspace */ + unsigned u; /* Local index variable */ + + /* Get low & high bounds for dataspace selection */ + if(H5S_SELECT_BOUNDS(space, low, high) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get selection bounds for dataspace") + + /* Loop over selection bounds and block, checking for overlap */ + for(u = 0; u < space->extent.rank; u++) + /* If selection bounds & block don't overlap, can leave now */ + if(!H5S_RANGE_OVERLAP(low[u], high[u], start[u], end[u])) + HGOTO_DONE(FALSE) + } /* end if */ + + /* Call selection type's intersect routine */ + if((ret_value = (*space->select.type->intersect_block)(space, start, end)) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCOMPARE, FAIL, "can't intersect block with selection") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5S_select_intersect_block() */ + + +/*-------------------------------------------------------------------------- + NAME + H5Sselect_intersect_block + PURPOSE + Check if current selection intersects with a block + USAGE + htri_t H5Sselect_intersect_block(space_id, start, end) + hid_t space1_id; IN: ID of dataspace pointer to compare + const hsize_t *start; IN: Starting coordinate of block + const hsize_t *end; IN: Opposite ("ending") coordinate of block + RETURNS + TRUE / FALSE / FAIL + DESCRIPTION + Checks to see if the current selection in the dataspace intersects with + the block given. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + Assumes that start & end block bounds are _inclusive_, so start == end + value OK. + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +htri_t +H5Sselect_intersect_block(hid_t space_id, const hsize_t *start, const hsize_t *end) +{ + H5S_t *space; /* Dataspace to query */ + unsigned u; /* Local index value */ + htri_t ret_value = FAIL; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("t", "i*h*h", space_id, start, end); + + /* Check arguments */ + if(NULL == (space = (H5S_t *)H5I_object_verify(space_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "not a dataspace") + if(NULL == start) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "block start array pointer is NULL") + if(NULL == end) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "block end array pointer is NULL") + + /* Range check start & end values */ + for(u = 0; u < space->extent.rank; u++) + if(start[u] > end[u]) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "block start[%u] (%llu) > end[%u] (%llu)", u, (unsigned long long)start[u], u, (unsigned long long)end[u]) + + /* Call internal routine to do comparison */ + if((ret_value = H5S_select_intersect_block(space, start, end)) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCOMPARE, FAIL, "can't compare selection and block") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Sselect_intersect_block() */ + + +/*-------------------------------------------------------------------------- + NAME H5S_select_construct_projection PURPOSE diff --git a/src/H5Stest.c b/src/H5Stest.c index 2c42713..b61b6bf 100644 --- a/src/H5Stest.c +++ b/src/H5Stest.c @@ -66,50 +66,6 @@ /*-------------------------------------------------------------------------- NAME - H5S__select_shape_same_test - PURPOSE - Determine if two dataspace selections are the same shape - USAGE - htri_t H5S__select_shape_same_test(sid1, sid2) - hid_t sid1; IN: 1st dataspace to compare - hid_t sid2; IN: 2nd dataspace to compare - RETURNS - Non-negative TRUE/FALSE on success, negative on failure - DESCRIPTION - Checks to see if the current selection in the dataspaces are the same - dimensionality and shape. - GLOBAL VARIABLES - COMMENTS, BUGS, ASSUMPTIONS - DO NOT USE THIS FUNCTION FOR ANYTHING EXCEPT TESTING - EXAMPLES - REVISION LOG ---------------------------------------------------------------------------*/ -htri_t -H5S__select_shape_same_test(hid_t sid1, hid_t sid2) -{ - H5S_t *space1; /* Pointer to 1st dataspace */ - H5S_t *space2; /* Pointer to 2nd dataspace */ - htri_t ret_value = FAIL; /* Return value */ - - FUNC_ENTER_PACKAGE - - /* Get dataspace structures */ - if(NULL == (space1 = (H5S_t *)H5I_object_verify(sid1, H5I_DATASPACE))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataspace") - if(NULL == (space2 = (H5S_t *)H5I_object_verify(sid2, H5I_DATASPACE))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataspace") - - /* Check if the dataspace selections are the same shape */ - if((ret_value = H5S_select_shape_same(space1, space2)) < 0) - HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCOMPARE, FAIL, "unable to compare dataspace selections") - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5S__select_shape_same_test() */ - - -/*-------------------------------------------------------------------------- - NAME H5S__get_rebuild_status_test PURPOSE Determine the status of the diminfo_valid field (whether we know the @@ -1740,6 +1740,7 @@ H5Tcopy(hid_t obj_id) case H5I_GROUP: case H5I_DATASPACE: case H5I_ATTR: + case H5I_MAP: case H5I_VFL: case H5I_VOL: case H5I_GENPROP_CLS: diff --git a/src/H5VLcallback.c b/src/H5VLcallback.c index 696ccab..0131f0e 100644 --- a/src/H5VLcallback.c +++ b/src/H5VLcallback.c @@ -6562,3 +6562,114 @@ done: FUNC_LEAVE_API_NOINIT(ret_value) } /* end H5VLrequest_free() */ + +/*------------------------------------------------------------------------- + * Function: H5VL__optional + * + * Purpose: Optional operation specific to connectors. + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +static herr_t +H5VL__optional(void *obj, const H5VL_class_t *cls, hid_t dxpl_id, + void **req, va_list arguments) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Check if the corresponding VOL callback exists */ + if(NULL == cls->optional) + HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'optional' method") + + /* Call the corresponding VOL callback */ + if((ret_value = (cls->optional)(obj, dxpl_id, req, arguments)) < 0) + HGOTO_ERROR(H5E_VOL, H5E_CANTOPERATE, ret_value, "unable to execute optional callback") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5VL__optional() */ + + +/*------------------------------------------------------------------------- + * Function: H5VL_optional + * + * Purpose: Optional operation specific to connectors. + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +herr_t +H5VL_optional(const H5VL_object_t *vol_obj, hid_t dxpl_id, + void **req, ...) +{ + va_list arguments; /* Argument list passed from the API call */ + hbool_t arg_started = FALSE; /* Whether the va_list has been started */ + hbool_t vol_wrapper_set = FALSE; /* Whether the VOL object wrapping context was set up */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Set wrapper info in API context */ + if(H5VL_set_vol_wrapper(vol_obj->data, vol_obj->connector) < 0) + HGOTO_ERROR(H5E_VOL, H5E_CANTSET, FAIL, "can't set VOL wrapper info") + vol_wrapper_set = TRUE; + + /* Call the corresponding internal VOL routine */ + HDva_start(arguments, req); + arg_started = TRUE; + if((ret_value = H5VL__optional(vol_obj->data, vol_obj->connector->cls, dxpl_id, req, arguments)) < 0) + HGOTO_ERROR(H5E_VOL, H5E_CANTOPERATE, FAIL, "unable to execute optional callback") + +done: + /* End access to the va_list, if we started it */ + if(arg_started) + HDva_end(arguments); + + /* Reset object wrapping info in API context */ + if(vol_wrapper_set && H5VL_reset_vol_wrapper() < 0) + HDONE_ERROR(H5E_VOL, H5E_CANTRESET, ret_value, "can't reset VOL wrapper info") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5VL_optional() */ + + +/*------------------------------------------------------------------------- + * Function: H5VLoptional + * + * Purpose: Performs an optional connector-specific operation + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +herr_t +H5VLoptional(void *obj, hid_t connector_id, hid_t dxpl_id, void **req, + va_list arguments) +{ + H5VL_class_t *cls; /* VOL connector's class struct */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API_NOINIT + H5TRACE5("e", "*xii**xx", obj, connector_id, dxpl_id, req, arguments); + + /* Check args and get class pointer */ + if(NULL == obj) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid object") + if(NULL == (cls = (H5VL_class_t *)H5I_object_verify(connector_id, H5I_VOL))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a VOL connector ID") + + /* Call the corresponding internal VOL routine */ + if((ret_value = H5VL__optional(obj, cls, dxpl_id, req, arguments)) < 0) + HGOTO_ERROR(H5E_VOL, H5E_CANTOPERATE, ret_value, "unable to execute optional callback") + +done: + FUNC_LEAVE_API_NOINIT(ret_value) +} /* end H5VLoptional() */ + diff --git a/src/H5VLconnector.h b/src/H5VLconnector.h index 9de518f..98bc521 100644 --- a/src/H5VLconnector.h +++ b/src/H5VLconnector.h @@ -314,7 +314,7 @@ typedef struct H5VL_group_class_t { /* H5L routines */ typedef struct H5VL_link_class_t { herr_t (*create)(H5VL_link_create_type_t create_type, void *obj, const H5VL_loc_params_t *loc_params, - hid_t lcpl_id, hid_t lapl_id, hid_t dxpl_id, void **req, va_list argumenmts); + hid_t lcpl_id, hid_t lapl_id, hid_t dxpl_id, void **req, va_list arguments); herr_t (*copy)(void *src_obj, const H5VL_loc_params_t *loc_params1, void *dst_obj, const H5VL_loc_params_t *loc_params2, hid_t lcpl, hid_t lapl, hid_t dxpl_id, void **req); diff --git a/src/H5VLint.c b/src/H5VLint.c index 884b2f6..f9262f4 100644 --- a/src/H5VLint.c +++ b/src/H5VLint.c @@ -552,7 +552,8 @@ H5VL__new_vol_obj(H5I_type_t type, void *object, H5VL_t *vol_connector, hbool_t HDassert(vol_connector); /* Make sure type number is valid */ - if(type != H5I_ATTR && type != H5I_DATASET && type != H5I_DATATYPE && type != H5I_FILE && type != H5I_GROUP) + if(type != H5I_ATTR && type != H5I_DATASET && type != H5I_DATATYPE + && type != H5I_FILE && type != H5I_GROUP && type != H5I_MAP) HGOTO_ERROR(H5E_VOL, H5E_BADVALUE, NULL, "invalid type number") /* Create the new VOL object */ @@ -1408,6 +1409,7 @@ H5VL__object(hid_t id, H5I_type_t obj_type) case H5I_DATASET: case H5I_FILE: case H5I_ATTR: + case H5I_MAP: /* get the object */ if (NULL == (vol_obj = (H5VL_object_t *)H5I_object(id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "invalid identifier") diff --git a/src/H5VLnative.h b/src/H5VLnative.h index 1a3007a..a8d5720 100644 --- a/src/H5VLnative.h +++ b/src/H5VLnative.h @@ -36,8 +36,11 @@ typedef int H5VL_native_dataset_optional_t; #define H5VL_NATIVE_DATASET_FORMAT_CONVERT 0 /* H5Dformat_convert (internal) */ #define H5VL_NATIVE_DATASET_GET_CHUNK_INDEX_TYPE 1 /* H5Dget_chunk_index_type */ #define H5VL_NATIVE_DATASET_GET_CHUNK_STORAGE_SIZE 2 /* H5Dget_chunk_storage_size */ -#define H5VL_NATIVE_DATASET_CHUNK_READ 3 /* H5Dchunk_read */ -#define H5VL_NATIVE_DATASET_CHUNK_WRITE 4 /* H5Dchunk_write */ +#define H5VL_NATIVE_DATASET_GET_NUM_CHUNKS 3 /* H5Dget_num_chunks */ +#define H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_IDX 4 /* H5Dget_chunk_info */ +#define H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_COORD 5 /* H5Dget_chunk_info_by_coord */ +#define H5VL_NATIVE_DATASET_CHUNK_READ 6 /* H5Dchunk_read */ +#define H5VL_NATIVE_DATASET_CHUNK_WRITE 7 /* H5Dchunk_write */ /* Typedef and values for native VOL connector file optional VOL operations */ typedef int H5VL_native_file_optional_t; diff --git a/src/H5VLnative_dataset.c b/src/H5VLnative_dataset.c index 1e3d263..676d859 100644 --- a/src/H5VLnative_dataset.c +++ b/src/H5VLnative_dataset.c @@ -339,7 +339,7 @@ H5VL__native_dataset_specific(void *obj, H5VL_dataset_specific_t specific_type, switch(specific_type) { /* H5Dspecific_space */ case H5VL_DATASET_SET_EXTENT: - { + { /* H5Dset_extent (H5Dextend - deprecated) */ const hsize_t *size = HDva_arg(arguments, const hsize_t *); if(H5D__set_extent(dset, size) < 0) @@ -348,7 +348,7 @@ H5VL__native_dataset_specific(void *obj, H5VL_dataset_specific_t specific_type, } case H5VL_DATASET_FLUSH: - { + { /* H5Dflush */ hid_t dset_id = HDva_arg(arguments, hid_t); /* Flush the dataset */ @@ -359,7 +359,7 @@ H5VL__native_dataset_specific(void *obj, H5VL_dataset_specific_t specific_type, } case H5VL_DATASET_REFRESH: - { + { /* H5Drefresh */ hid_t dset_id = HDva_arg(arguments, hid_t); /* Refresh the dataset */ @@ -399,7 +399,7 @@ H5VL__native_dataset_optional(void *obj, hid_t H5_ATTR_UNUSED dxpl_id, switch(optional_type) { case H5VL_NATIVE_DATASET_FORMAT_CONVERT: - { + { /* H5Dformat_convert */ dset = (H5D_t *)obj; switch(dset->shared->layout.type) { @@ -434,7 +434,7 @@ H5VL__native_dataset_optional(void *obj, hid_t H5_ATTR_UNUSED dxpl_id, } case H5VL_NATIVE_DATASET_GET_CHUNK_INDEX_TYPE: - { + { /* H5Dget_chunk_index_type */ H5D_chunk_index_t *idx_type = HDva_arg(arguments, H5D_chunk_index_t *); dset = (H5D_t *)obj; @@ -450,7 +450,7 @@ H5VL__native_dataset_optional(void *obj, hid_t H5_ATTR_UNUSED dxpl_id, } case H5VL_NATIVE_DATASET_GET_CHUNK_STORAGE_SIZE: - { + { /* H5Dget_chunk_storage_size */ hsize_t *offset = HDva_arg(arguments, hsize_t *); hsize_t *chunk_nbytes = HDva_arg(arguments, hsize_t *); @@ -467,8 +467,91 @@ H5VL__native_dataset_optional(void *obj, hid_t H5_ATTR_UNUSED dxpl_id, break; } + case H5VL_NATIVE_DATASET_GET_NUM_CHUNKS: + { /* H5Dget_num_chunks */ + const H5S_t *space = NULL; + hid_t space_id = HDva_arg(arguments, hid_t); + hsize_t *nchunks = HDva_arg(arguments, hsize_t *); + + dset = (H5D_t *)obj; + HDassert(dset); + HDassert(dset->shared); + HDassert(dset->shared->space); + + /* When default dataspace is given, use the dataset's dataspace */ + if(space_id == H5S_ALL) + space = dset->shared->space; + else /* otherwise, use the given space ID */ + if(NULL == (space = (const H5S_t *)H5I_object_verify(space_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a valid dataspace ID") + + /* Make sure the dataset is chunked */ + if(H5D_CHUNKED != dset->shared->layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset") + + /* Call private function */ + if(H5D__get_num_chunks(dset, space, nchunks) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get number of chunks") + + break; + } + + case H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_IDX: + { /* H5Dget_chunk_info */ + const H5S_t *space = NULL; + hid_t space_id = HDva_arg(arguments, hid_t); + hsize_t chk_index = HDva_arg(arguments, hsize_t); + hsize_t *offset = HDva_arg(arguments, hsize_t *); + unsigned *filter_mask = HDva_arg(arguments, unsigned *); + haddr_t *addr = HDva_arg(arguments, haddr_t *); + hsize_t *size = HDva_arg(arguments, hsize_t *); + + dset = (H5D_t *)obj; + HDassert(dset); + HDassert(dset->shared); + HDassert(dset->shared->space); + + /* When default dataspace is given, use the dataset's dataspace */ + if(space_id == H5S_ALL) + space = dset->shared->space; + else /* otherwise, use the given space ID */ + if(NULL == (space = (const H5S_t *)H5I_object_verify(space_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a valid dataspace ID") + + /* Make sure the dataset is chunked */ + if(H5D_CHUNKED != dset->shared->layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset") + + /* Call private function */ + if(H5D__get_chunk_info(dset, space, chk_index, offset, filter_mask, addr, size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk info by index") + break; + } + + case H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_COORD: + { /* H5Dget_chunk_info_by_coord */ + hsize_t *offset = HDva_arg(arguments, hsize_t *); + unsigned *filter_mask = HDva_arg(arguments, unsigned *); + haddr_t *addr = HDva_arg(arguments, haddr_t *); + hsize_t *size = HDva_arg(arguments, hsize_t *); + + dset = (H5D_t *)obj; + HDassert(dset); + HDassert(dset->shared); + + /* Make sure the dataset is chunked */ + if(H5D_CHUNKED != dset->shared->layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset") + + /* Call private function */ + if(H5D__get_chunk_info_by_coord(dset, offset, filter_mask, addr, size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk info by its logical coordinates") + + break; + } + case H5VL_NATIVE_DATASET_CHUNK_READ: - { + { /* H5Dread_chunk */ const hsize_t *offset = HDva_arg(arguments, hsize_t *); uint32_t *filters = HDva_arg(arguments, uint32_t *); void *buf = HDva_arg(arguments, void *); @@ -496,7 +579,7 @@ H5VL__native_dataset_optional(void *obj, hid_t H5_ATTR_UNUSED dxpl_id, } case H5VL_NATIVE_DATASET_CHUNK_WRITE: - { + { /* H5Dwrite_chunk */ uint32_t filters = HDva_arg(arguments, uint32_t); const hsize_t *offset = HDva_arg(arguments, const hsize_t *); uint32_t data_size_32 = HDva_arg(arguments, uint32_t); diff --git a/src/H5VLnative_file.c b/src/H5VLnative_file.c index eeaade6..0ac70e3 100644 --- a/src/H5VLnative_file.c +++ b/src/H5VLnative_file.c @@ -562,11 +562,12 @@ H5VL__native_file_optional(void *obj, hid_t H5_ATTR_UNUSED dxpl_id, void H5_ATTR case H5VL_NATIVE_FILE_GET_FILE_ID: { H5I_type_t type = (H5I_type_t)HDva_arg(arguments, int); /* enum work-around */ + hbool_t app_ref = (hbool_t)HDva_arg(arguments, int); hid_t *file_id = HDva_arg(arguments, hid_t *); if(NULL == (f = H5F__get_file(obj, type))) HGOTO_ERROR(H5E_FILE, H5E_BADTYPE, FAIL, "not a file or file object") - if((*file_id = H5F__get_file_id(f)) < 0) + if((*file_id = H5F__get_file_id(f, app_ref)) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get file ID") break; } diff --git a/src/H5VLnative_link.c b/src/H5VLnative_link.c index 4709eb3..5d5935d 100644 --- a/src/H5VLnative_link.c +++ b/src/H5VLnative_link.c @@ -58,15 +58,15 @@ H5VL__native_link_create(H5VL_link_create_type_t create_type, void *obj, H5G_loc_t cur_loc; H5G_loc_t link_loc; void *cur_obj = HDva_arg(arguments, void *); - H5VL_loc_params_t cur_params = HDva_arg(arguments, H5VL_loc_params_t); + H5VL_loc_params_t *cur_params = HDva_arg(arguments, H5VL_loc_params_t *); - if(NULL != cur_obj && H5G_loc_real(cur_obj, cur_params.obj_type, &cur_loc) < 0) + if(NULL != cur_obj && H5G_loc_real(cur_obj, cur_params->obj_type, &cur_loc) < 0) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file or file object") if(NULL != obj && H5G_loc_real(obj, loc_params->obj_type, &link_loc) < 0) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file or file object") /* H5Lcreate_hard */ - if(H5VL_OBJECT_BY_NAME == cur_params.type) { + if(H5VL_OBJECT_BY_NAME == cur_params->type) { H5G_loc_t *cur_loc_p, *link_loc_p; /* Set up current & new location pointers */ @@ -80,7 +80,7 @@ H5VL__native_link_create(H5VL_link_create_type_t create_type, void *obj, HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "source and destination should be in the same file.") /* Create the link */ - if((ret_value = H5L_create_hard(cur_loc_p, cur_params.loc_data.loc_by_name.name, + if((ret_value = H5L_create_hard(cur_loc_p, cur_params->loc_data.loc_by_name.name, link_loc_p, loc_params->loc_data.loc_by_name.name, lcpl_id)) < 0) HGOTO_ERROR(H5E_LINK, H5E_CANTINIT, FAIL, "unable to create link") } /* end if */ diff --git a/src/H5VLpassthru.c b/src/H5VLpassthru.c index eecdac2..24070fa 100644 --- a/src/H5VLpassthru.c +++ b/src/H5VLpassthru.c @@ -2131,11 +2131,11 @@ H5VL_pass_through_link_create(H5VL_link_create_type_t create_type, void *obj, /* Fix up the link target object for hard link creation */ if(H5VL_LINK_CREATE_HARD == create_type) { void *cur_obj; - H5VL_loc_params_t cur_params; + H5VL_loc_params_t *cur_params; /* Retrieve the object & loc params for the link target */ cur_obj = va_arg(arguments, void *); - cur_params = va_arg(arguments, H5VL_loc_params_t); + cur_params = va_arg(arguments, H5VL_loc_params_t *); /* If it's a non-NULL pointer, find the 'under object' and re-set the property */ if(cur_obj) { diff --git a/src/H5VLprivate.h b/src/H5VLprivate.h index 91fa762..1752b0c 100644 --- a/src/H5VLprivate.h +++ b/src/H5VLprivate.h @@ -193,5 +193,8 @@ H5_DLL herr_t H5VL_request_specific(const H5VL_object_t *vol_obj, H5VL_request_s H5_DLL herr_t H5VL_request_optional(const H5VL_object_t *vol_obj, ...); H5_DLL herr_t H5VL_request_free(const H5VL_object_t *vol_obj); +/* Generic functions */ +H5_DLL herr_t H5VL_optional(const H5VL_object_t *vol_obj, hid_t dxpl_id,void **req, ...); + #endif /* _H5VLprivate_H */ diff --git a/src/H5err.txt b/src/H5err.txt index d4edfba..9fec521 100644 --- a/src/H5err.txt +++ b/src/H5err.txt @@ -78,6 +78,7 @@ MAJOR, H5E_FARRAY, Fixed Array MAJOR, H5E_PLUGIN, Plugin for dynamically loaded library MAJOR, H5E_PAGEBUF, Page Buffering MAJOR, H5E_CONTEXT, API Context +MAJOR, H5E_MAP, Map MAJOR, H5E_NONE_MAJOR, No error # Sections (for grouping minor errors) diff --git a/src/H5make_libsettings.c b/src/H5make_libsettings.c index da7c8d9..fd67184 100644 --- a/src/H5make_libsettings.c +++ b/src/H5make_libsettings.c @@ -35,12 +35,15 @@ static const char *FileHeader = "\n\ #include <stdio.h> #include <time.h> #include "H5private.h" +/* Do NOT use HDfprintf in this file as it is not linked with the library, + * which contains the H5system.c file in which the function is defined. + */ #define LIBSETTINGSFNAME "libhdf5.settings" FILE *rawoutstream = NULL; - + /*------------------------------------------------------------------------- * Function: insert_libhdf5_settings * @@ -105,7 +108,7 @@ insert_libhdf5_settings(FILE *flibinfo) #endif } /* insert_libhdf5_settings() */ - + /*------------------------------------------------------------------------- * Function: make_libinfo * @@ -123,7 +126,7 @@ make_libinfo(void) insert_libhdf5_settings(rawoutstream); } - + /*------------------------------------------------------------------------- * Function: print_header * @@ -229,7 +232,7 @@ information about the library build configuration\n"; fprintf(rawoutstream, "\n */\n\n"); } - + /*------------------------------------------------------------------------- * Function: print_footer * @@ -244,7 +247,7 @@ print_footer(void) /* nothing */ } - + /*------------------------------------------------------------------------- * Function: main * diff --git a/src/H5mpi.c b/src/H5mpi.c new file mode 100644 index 0000000..d48790b --- /dev/null +++ b/src/H5mpi.c @@ -0,0 +1,396 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: Common MPI routines + * + */ + +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5MMprivate.h" /* Memory Management */ + + +#ifdef H5_HAVE_PARALLEL + +/*------------------------------------------------------------------------- + * Function: H5_mpi_comm_dup + * + * Purpose: Duplicate an MPI communicator. + * + * Does not duplicate MPI_COMM_NULL. Instead, comm_new will + * be set to MPI_COMM_NULL directly. + * + * The new communicator is returned via the comm_new pointer. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5_mpi_comm_dup(MPI_Comm comm, MPI_Comm *comm_new) +{ + herr_t ret_value = SUCCEED; + MPI_Comm comm_dup = MPI_COMM_NULL; + int mpi_code; + + FUNC_ENTER_NOAPI(FAIL) + + /* Check arguments */ + if (!comm_new) + HGOTO_ERROR(H5E_INTERNAL, H5E_BADVALUE, FAIL, "comm_new cannot be NULL") + + /* Handle MPI_COMM_NULL separately */ + if (MPI_COMM_NULL == comm) { + /* Don't duplicate MPI_COMM_NULL since that's an error in MPI */ + comm_dup = MPI_COMM_NULL; + } + else { + + /* Duplicate the MPI communicator */ + if (MPI_SUCCESS != (mpi_code = MPI_Comm_dup(comm, &comm_dup))) + HMPI_GOTO_ERROR(FAIL, "MPI_Comm_dup failed", mpi_code) + + /* Set MPI_ERRORS_RETURN on comm_dup so that MPI failures are not fatal, + * and return codes can be checked and handled. + */ + if (MPI_SUCCESS != (mpi_code = MPI_Comm_set_errhandler(comm_dup, MPI_ERRORS_RETURN))) + HMPI_GOTO_ERROR(FAIL, "MPI_Errhandler_set failed", mpi_code) + + } + + /* Copy the new communicator to the return argument */ + *comm_new = comm_dup; + +done: + if (FAIL == ret_value) { + /* need to free anything created here */ + if (MPI_COMM_NULL != comm_dup) + MPI_Comm_free(&comm_dup); + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5_mpi_comm_dup() */ + + +/*------------------------------------------------------------------------- + * Function: H5_mpi_info_dup + * + * Purpose: Duplicate an MPI info. + * + * If the info object is MPI_INFO_NULL, no duplicate + * is made but the same value assigned to the new info object + * handle. + * + * The new info is returned via the info_new pointer. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5_mpi_info_dup(MPI_Info info, MPI_Info *info_new) +{ + herr_t ret_value = SUCCEED; + MPI_Info info_dup = MPI_INFO_NULL; + int mpi_code; + + FUNC_ENTER_NOAPI(FAIL) + + /* Check arguments */ + if (!info_new) + HGOTO_ERROR(H5E_INTERNAL, H5E_BADVALUE, FAIL, "info_new cannot be NULL") + + /* Duplicate the MPI info */ + if (info == MPI_INFO_NULL) { + /* Don't duplicate MPI_INFO_NULL. Just copy it. */ + info_dup = MPI_INFO_NULL; + } + else { + /* Duplicate the info */ + if (MPI_SUCCESS != (mpi_code = MPI_Info_dup(info, &info_dup))) + HMPI_GOTO_ERROR(FAIL, "MPI_Info_dup failed", mpi_code) + } + + /* Copy the new info to the return argument */ + *info_new = info_dup; + +done: + if (FAIL == ret_value) { + /* need to free anything created here */ + if (MPI_INFO_NULL != info_dup) + MPI_Info_free(&info_dup); + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5_mpi_info_dup() */ + + +/*------------------------------------------------------------------------- + * Function: H5_mpi_comm_free + * + * Purpose: Free an MPI communicator. + * + * If comm is MPI_COMM_NULL this call does nothing. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5_mpi_comm_free(MPI_Comm *comm) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + /* Check arguments */ + if (!comm) + HGOTO_ERROR(H5E_INTERNAL, H5E_BADVALUE, FAIL, "comm pointer cannot be NULL") + + /* Free the communicator */ + if (MPI_COMM_NULL != *comm) + MPI_Comm_free(comm); + + *comm = MPI_COMM_NULL; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* End H5_mpi_comm_free() */ + +/*------------------------------------------------------------------------- + * Function: H5_mpi_info_free + * + * Purpose: Free the MPI info. + * + * If info is MPI_INFO_NULL this call does nothing. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5_mpi_info_free(MPI_Info *info) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + /* Check arguments */ + if (!info) + HGOTO_ERROR(H5E_INTERNAL, H5E_BADVALUE, FAIL, "info pointer cannot be NULL") + + /* Free the info */ + if (MPI_INFO_NULL != *info) + MPI_Info_free(info); + + *info = MPI_INFO_NULL; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* End H5_mpi_info_free() */ + + +/*------------------------------------------------------------------------- + * Function: H5_mpi_comm_cmp + * + * Purpose: Compares two MPI communicators. + * + * Note that passing MPI_COMM_NULL to this function will not + * throw errors, unlike MPI_Comm_compare(). + * + * We consider MPI communicators to be the "same" when the + * groups are identical. We don't care about the context + * since that will always be different as we call MPI_Comm_dup + * when we store the communicator in the fapl. + * + * The out parameter is a value like strcmp. The value is + * undefined when the return value is FAIL. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5_mpi_comm_cmp(MPI_Comm comm1, MPI_Comm comm2, int *result) +{ + int mpi_code; + int mpi_result = MPI_IDENT; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + /* Check arguments */ + if (!result) + HGOTO_ERROR(H5E_INTERNAL, H5E_BADVALUE, FAIL, "result cannot be NULL") + + /* Set out parameter to something reasonable in case something goes wrong */ + *result = 0; + + /* Can't pass MPI_COMM_NULL to MPI_Comm_compare() so we have to handle + * it in special cases. + * + * MPI_Comm can either be an integer type or a pointer. We cast them + * to intptr_t so we can compare them with < and > when needed. + */ + if (MPI_COMM_NULL == comm1 && MPI_COMM_NULL == comm2) { + /* Special case of both communicators being MPI_COMM_NULL */ + *result = 0; + } + else if (MPI_COMM_NULL == comm1 || MPI_COMM_NULL == comm2) { + + /* Special case of one communicator being MPI_COMM_NULL */ + *result = (intptr_t)comm1 < (intptr_t)comm2 ? -1 : 1; + } + else { + + /* Normal communicator compare */ + + /* Compare the MPI communicators */ + if (MPI_SUCCESS != (mpi_code = MPI_Comm_compare(comm1, comm2, &mpi_result))) + HMPI_GOTO_ERROR(FAIL, "MPI_Comm_compare failed", mpi_code) + + /* Set the result */ + if (MPI_IDENT == mpi_result || MPI_CONGRUENT == mpi_result) + *result = 0; + else + *result = (intptr_t)comm1 < (intptr_t)comm2 ? -1 : 1; + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5_mpi_comm_cmp() */ + + +/*------------------------------------------------------------------------- + * Function: H5_mpi_info_cmp + * + * Purpose: Compares two MPI info objects. + * + * For our purposes, two mpi info objects are the "same" if + * they contain the same key-value pairs or are both + * MPI_INFO_NULL. + * + * The out parameter is a value like strcmp. The value is + * undefined when the return value is FAIL. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5_mpi_info_cmp(MPI_Info info1, MPI_Info info2, int *result) +{ + hbool_t same = FALSE; + char *key = NULL; + char *value1 = NULL; + char *value2 = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + /* Check arguments */ + if (!result) + HGOTO_ERROR(H5E_INTERNAL, H5E_BADVALUE, FAIL, "result cannot be NULL") + + /* Check for MPI_INFO_NULL */ + if (MPI_INFO_NULL == info1 && MPI_INFO_NULL == info2) { + /* Special case of both info objects being MPI_INFO_NULL */ + same = TRUE; + } + else if (MPI_INFO_NULL == info1 || MPI_INFO_NULL == info2) { + + /* Special case of one info object being MPI_INFO_NULL */ + same = FALSE; + } + else { + int mpi_code; + int nkeys_1; + int nkeys_2; + + /* Check if the number of keys is the same */ + if (MPI_SUCCESS != (mpi_code = MPI_Info_get_nkeys(info1, &nkeys_1))) + HMPI_GOTO_ERROR(FAIL, "MPI_Info_get_nkeys failed", mpi_code) + if (MPI_SUCCESS != (mpi_code = MPI_Info_get_nkeys(info2, &nkeys_2))) + HMPI_GOTO_ERROR(FAIL, "MPI_Info_get_nkeys failed", mpi_code) + + if (nkeys_1 != nkeys_2) + same = FALSE; + else if (0 == nkeys_1 && 0 == nkeys_2) + same = TRUE; + else { + int i; + int flag1 = -1; + int flag2 = -1; + + /* Allocate buffers for iteration */ + if (NULL == (key = (char *)H5MM_malloc(MPI_MAX_INFO_KEY * sizeof(char)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + if (NULL == (value1 = (char *)H5MM_malloc(MPI_MAX_INFO_VAL * sizeof(char)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + if (NULL == (value2 = (char *)H5MM_malloc(MPI_MAX_INFO_VAL * sizeof(char)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + + /* Iterate over the keys, comparing them */ + for (i = 0; i < nkeys_1; i++) { + + same = TRUE; + + /* Memset the buffers to zero */ + HDmemset(key, 0, MPI_MAX_INFO_KEY); + HDmemset(value1, 0, MPI_MAX_INFO_VAL); + HDmemset(value2, 0, MPI_MAX_INFO_VAL); + + /* Get the nth key */ + if (MPI_SUCCESS != (mpi_code = MPI_Info_get_nthkey(info1, i, key))) + HMPI_GOTO_ERROR(FAIL, "MPI_Info_get_nthkey failed", mpi_code) + + /* Get the values */ + if (MPI_SUCCESS != (mpi_code = MPI_Info_get(info1, key, MPI_MAX_INFO_VAL, value1, &flag1))) + HMPI_GOTO_ERROR(FAIL, "MPI_Info_get failed", mpi_code) + if (MPI_SUCCESS != (mpi_code = MPI_Info_get(info2, key, MPI_MAX_INFO_VAL, value2, &flag2))) + HMPI_GOTO_ERROR(FAIL, "MPI_Info_get failed", mpi_code) + + /* Compare values and flags */ + if (!flag1 || !flag2 || HDmemcmp(value1, value2, MPI_MAX_INFO_VAL)) { + same = FALSE; + break; + } + + } /* end for */ + } /* end else */ + } /* end else */ + + /* Set the output value + * + * MPI_Info can either be an integer type or a pointer. We cast them + * to intptr_t so we can compare them with < and > when needed. + */ + if (same) + *result = 0; + else + *result = (intptr_t)info1 < (intptr_t)info2 ? -1 : 1; + +done: + if (key) + H5MM_xfree(key); + if (value1) + H5MM_xfree(value1); + if (value2) + H5MM_xfree(value2); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5_mpi_info_cmp() */ + +#endif /* H5_HAVE_PARALLEL */ + diff --git a/src/H5private.h b/src/H5private.h index 86cd615..bb2a2eb 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -317,8 +317,12 @@ # define H5_ATTR_NORETURN __attribute__((noreturn)) # define H5_ATTR_CONST __attribute__((const)) # define H5_ATTR_PURE __attribute__((pure)) +#if defined(__GNUC__) && __GNUC__ >= 7 && !defined(__INTEL_COMPILER) # define H5_ATTR_FALLTHROUGH __attribute__((fallthrough)); #else +# define H5_ATTR_FALLTHROUGH /*void*/ +#endif +#else # define H5_ATTR_FORMAT(X,Y,Z) /*void*/ # define H5_ATTR_UNUSED /*void*/ # define H5_ATTR_NORETURN /*void*/ @@ -1717,6 +1721,7 @@ typedef enum { H5_PKG_HG, /* Global heaps */ H5_PKG_HL, /* Local heaps */ H5_PKG_I, /* IDs */ + H5_PKG_M, /* Maps */ H5_PKG_MF, /* File memory management */ H5_PKG_MM, /* Core memory management */ H5_PKG_O, /* Object headers */ @@ -2633,6 +2638,8 @@ H5_DLL int H5G_term_package(void); H5_DLL int H5G_top_term_package(void); H5_DLL int H5I_term_package(void); H5_DLL int H5L_term_package(void); +H5_DLL int H5M_term_package(void); +H5_DLL int H5M_top_term_package(void); H5_DLL int H5P_term_package(void); H5_DLL int H5PL_term_package(void); H5_DLL int H5R_term_package(void); @@ -2661,6 +2668,16 @@ H5_DLL double H5_get_time(void); H5_DLL herr_t H5_build_extpath(const char *name, char **extpath /*out*/); H5_DLL herr_t H5_combine_path(const char *path1, const char *path2, char **full_name /*out*/); +#ifdef H5_HAVE_PARALLEL +/* Generic MPI functions */ +H5_DLL herr_t H5_mpi_comm_dup(MPI_Comm comm, MPI_Comm *comm_new); +H5_DLL herr_t H5_mpi_info_dup(MPI_Info info, MPI_Info *info_new); +H5_DLL herr_t H5_mpi_comm_free(MPI_Comm *comm); +H5_DLL herr_t H5_mpi_info_free(MPI_Info *info); +H5_DLL herr_t H5_mpi_comm_cmp(MPI_Comm comm1, MPI_Comm comm2, int *result); +H5_DLL herr_t H5_mpi_info_cmp(MPI_Info info1, MPI_Info info2, int *result); +#endif /* H5_HAVE_PARALLEL */ + /* Functions for debugging */ H5_DLL herr_t H5_buffer_dump(FILE *stream, int indent, const uint8_t *buf, const uint8_t *marker, size_t buf_offset, size_t buf_size); diff --git a/src/H5public.h b/src/H5public.h index a7e21f6..b79fcfb 100644 --- a/src/H5public.h +++ b/src/H5public.h @@ -57,6 +57,9 @@ # include <stddef.h> #endif #ifdef H5_HAVE_PARALLEL +/* Don't link against MPI C++ bindings */ +# define MPICH_SKIP_MPICXX 1 +# define OMPI_SKIP_MPICXX 1 # include <mpi.h> #ifndef MPI_FILE_NULL /*MPIO may be defined in mpi.h already */ # include <mpio.h> diff --git a/src/H5system.c b/src/H5system.c index 1d47d13..384360d 100644 --- a/src/H5system.c +++ b/src/H5system.c @@ -631,7 +631,7 @@ Pflock(int fd, int operation) { flk.l_pid = 0; /* not used with set */ /* Lock or unlock */ - if(HDfcntl(fd, F_SETLK, flk) < 0) + if(HDfcntl(fd, F_SETLK, &flk) < 0) return -1; return 0; diff --git a/src/H5trace.c b/src/H5trace.c index c5d14f5..de761f8 100644 --- a/src/H5trace.c +++ b/src/H5trace.c @@ -1359,6 +1359,10 @@ H5_trace(const double *returning, const char *func, const char *type, ...) HDfprintf(out, "%ld (attr)", (long)obj); break; + case H5I_MAP: + HDfprintf(out, "%ld (map)", (long)obj); + break; + case H5I_VFL: HDfprintf(out, "%ld (file driver)", (long)obj); break; @@ -1544,6 +1548,10 @@ H5_trace(const double *returning, const char *func, const char *type, ...) HDfprintf(out, "H5I_ATTR"); break; + case H5I_MAP: + HDfprintf(out, "H5I_MAP"); + break; + case H5I_VFL: HDfprintf(out, "H5I_VFL"); break; @@ -1799,8 +1807,12 @@ H5_trace(const double *returning, const char *func, const char *type, ...) HDfprintf(out, "H5O_TYPE_NAMED_DATATYPE"); break; + case H5O_TYPE_MAP: + HDfprintf(out, "H5O_TYPE_MAP"); + break; + case H5O_TYPE_NTYPES: - HDfprintf(out, "H5O_TYPE_TYPES"); + HDfprintf(out, "H5O_TYPE_NTYPES"); break; default: diff --git a/src/H5win32defs.h b/src/H5win32defs.h index 2ae2575..29533dd 100644 --- a/src/H5win32defs.h +++ b/src/H5win32defs.h @@ -52,7 +52,11 @@ typedef __int64 h5_stat_size_t; /* Note that the variadic HDopen macro is using a VC++ extension * where the comma is dropped if nothing is passed to the ellipsis. */ +#ifndef H5_HAVE_MINGW #define HDopen(S,F,...) Wopen_utf8(S,F,__VA_ARGS__) +#else +#define HDopen(S,F,...) Wopen_utf8(S,F,##__VA_ARGS__) +#endif #define HDread(F,M,Z) _read(F,M,Z) #define HDremove(S) Wremove_utf8(S) #define HDrmdir(S) _rmdir(S) @@ -65,13 +69,6 @@ typedef __int64 h5_stat_size_t; #define HDtzset() _tzset() #define HDunlink(S) _unlink(S) #define HDwrite(F,M,Z) _write(F,M,Z) -#if (_MSC_VER < 1800) -/* va_copy() does not exist on pre-2013 Visual Studio. Since va_lists are - * just pointers into the stack in those CRTs, the usual work-around - * is to just define the operation as a pointer copy. - */ -#define HDva_copy(D,S) ((D) = (S)) -#endif /* MSC_VER < 1800 */ #ifdef H5_HAVE_VISUAL_STUDIO @@ -82,6 +79,11 @@ typedef __int64 h5_stat_size_t; #ifndef H5_HAVE_STRTOULL #define HDstrtoull(S,R,N) _strtoui64(S,R,N) #endif /* H5_HAVE_STRTOULL */ + /* va_copy() does not exist on pre-2013 Visual Studio. Since va_lists are + * just pointers into the stack in those CRTs, the usual work-around + * is to just define the operation as a pointer copy. + */ + #define HDva_copy(D,S) ((D) = (S)) #endif /* MSC_VER < 1800 */ /* @@ -128,7 +130,7 @@ extern "C" { H5_DLL int Wnanosleep(const struct timespec *req, struct timespec *rem); H5_DLL herr_t H5_expand_windows_env_vars(char **env_var); H5_DLL const wchar_t *H5_get_utf16_str(const char *s); - H5_DLL int Wopen_utf8(const char *path, int oflag, ...); + H5_DLL int Wopen_utf8(const char *path, int oflag, ...); H5_DLL int Wremove_utf8(const char *path); /* Round functions only needed for VS2012 and earlier. diff --git a/src/Makefile.am b/src/Makefile.am index 0eaae1a..ccee69b 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -62,8 +62,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5Fsfile.c H5Fspace.c H5Fsuper.c H5Fsuper_cache.c H5Ftest.c \ H5FA.c H5FAcache.c H5FAdbg.c H5FAdblock.c H5FAdblkpage.c H5FAhdr.c \ H5FAint.c H5FAstat.c H5FAtest.c \ - H5FD.c H5FDcore.c \ - H5FDfamily.c H5FDint.c H5FDlog.c \ + H5FD.c H5FDcore.c H5FDfamily.c H5FDhdfs.c H5FDint.c H5FDlog.c \ H5FDmulti.c H5FDsec2.c H5FDspace.c H5FDstdio.c H5FDtest.c \ H5FL.c H5FO.c H5FS.c H5FScache.c H5FSdbg.c H5FSint.c H5FSsection.c \ H5FSstat.c H5FStest.c \ @@ -78,6 +77,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5HG.c H5HGcache.c H5HGdbg.c H5HGquery.c \ H5HL.c H5HLcache.c H5HLdbg.c H5HLint.c H5HLprfx.c H5HLdblk.c\ H5HP.c H5I.c H5Itest.c H5L.c H5Lexternal.c H5lib_settings.c \ + H5M.c \ H5MF.c H5MFaggr.c H5MFdbg.c H5MFsection.c \ H5MM.c H5MP.c H5MPtest.c \ H5O.c H5Odeprec.c H5Oainfo.c H5Oalloc.c H5Oattr.c \ @@ -95,8 +95,9 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5P.c H5Pacpl.c H5Pdapl.c H5Pdcpl.c \ H5Pdeprec.c H5Pdxpl.c H5Pencdec.c \ H5Pfapl.c H5Pfcpl.c H5Pfmpl.c \ - H5Pgcpl.c H5Pint.c \ - H5Plapl.c H5Plcpl.c H5Pocpl.c H5Pocpypl.c H5Pstrcpl.c H5Ptest.c \ + H5Pgcpl.c H5Pint.c H5Plapl.c H5Plcpl.c \ + H5Pmapl.c H5Pmcpl.c H5Pocpl.c H5Pocpypl.c H5Pstrcpl.c \ + H5Ptest.c \ H5PB.c \ H5PL.c H5PLint.c H5PLpath.c H5PLplugin_cache.c \ H5R.c H5Rint.c H5Rdeprec.c \ @@ -124,7 +125,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ # Only compile parallel sources if necessary if BUILD_PARALLEL_CONDITIONAL - libhdf5_la_SOURCES += H5ACmpio.c H5Cmpio.c H5Dmpio.c H5Fmpi.c H5FDmpi.c H5FDmpio.c H5Smpio.c + libhdf5_la_SOURCES += H5mpi.c H5ACmpio.c H5Cmpio.c H5Dmpio.c H5Fmpi.c H5FDmpi.c H5FDmpio.c H5Smpio.c endif # Only compile the direct VFD if necessary @@ -132,16 +133,21 @@ if DIRECT_VFD_CONDITIONAL libhdf5_la_SOURCES += H5FDdirect.c endif +# Only compile the read-only S3 VFD if necessary +if ROS3_VFD_CONDITIONAL + libhdf5_la_SOURCES += H5FDros3.c H5FDs3comms.c +endif + # Public headers include_HEADERS = hdf5.h H5api_adpt.h H5overflow.h H5pubconf.h H5public.h H5version.h \ H5Apublic.h H5ACpublic.h \ H5Cpublic.h H5Dpublic.h \ H5Epubgen.h H5Epublic.h H5ESpublic.h H5Fpublic.h \ H5FDpublic.h H5FDcore.h H5FDdirect.h \ - H5FDfamily.h H5FDlog.h H5FDmpi.h H5FDmpio.h \ - H5FDmulti.h H5FDsec2.h H5FDstdio.h H5FDwindows.h \ + H5FDfamily.h H5FDhdfs.h H5FDlog.h H5FDmpi.h H5FDmpio.h \ + H5FDmulti.h H5FDros3.h H5FDsec2.h H5FDstdio.h H5FDwindows.h \ H5Gpublic.h H5Ipublic.h H5Lpublic.h \ - H5MMpublic.h H5Opublic.h H5Ppublic.h \ + H5Mpublic.h H5MMpublic.h H5Opublic.h H5Ppublic.h \ H5PLextern.h H5PLpublic.h \ H5Rpublic.h H5Spublic.h H5Tpublic.h \ H5VLconnector.h H5VLconnector_passthru.h \ @@ -29,6 +29,7 @@ #include "H5Gpublic.h" /* Groups */ #include "H5Ipublic.h" /* ID management */ #include "H5Lpublic.h" /* Links */ +#include "H5Mpublic.h" /* Maps */ #include "H5MMpublic.h" /* Memory management */ #include "H5Opublic.h" /* Object headers */ #include "H5Ppublic.h" /* Property lists */ @@ -40,16 +41,18 @@ #include "H5Zpublic.h" /* Data filters */ /* Predefined file drivers */ -#include "H5FDcore.h" /* Files stored entirely in memory */ -#include "H5FDdirect.h" /* Linux direct I/O */ -#include "H5FDfamily.h" /* File families */ +#include "H5FDcore.h" /* Files stored entirely in memory */ +#include "H5FDdirect.h" /* Linux direct I/O */ +#include "H5FDfamily.h" /* File families */ +#include "H5FDhdfs.h" /* Hadoop HDFS */ #include "H5FDlog.h" /* sec2 driver with I/O logging (for debugging) */ -#include "H5FDmpi.h" /* MPI-based file drivers */ -#include "H5FDmulti.h" /* Usage-partitioned file family */ -#include "H5FDsec2.h" /* POSIX unbuffered file I/O */ -#include "H5FDstdio.h" /* Standard C buffered I/O */ +#include "H5FDmpi.h" /* MPI-based file drivers */ +#include "H5FDmulti.h" /* Usage-partitioned file family */ +#include "H5FDros3.h" /* R/O S3 "file" I/O */ +#include "H5FDsec2.h" /* POSIX unbuffered file I/O */ +#include "H5FDstdio.h" /* Standard C buffered I/O */ #ifdef H5_HAVE_WINDOWS -#include "H5FDwindows.h" /* Win32 I/O */ +#include "H5FDwindows.h" /* Win32 I/O */ #endif /* Virtual object layer (VOL) connectors */ diff --git a/src/libhdf5.settings.in b/src/libhdf5.settings.in index f856ebc..1591bed 100644 --- a/src/libhdf5.settings.in +++ b/src/libhdf5.settings.in @@ -78,7 +78,10 @@ Parallel Filtered Dataset Writes: @PARALLEL_FILTERED_WRITES@ With deprecated public symbols: @DEPRECATED_SYMBOLS@ I/O filters (external): @EXTERNAL_FILTERS@ MPE: @MPE@ + Map (H5M) API: @MAP_API@ Direct VFD: @DIRECT_VFD@ + (Read-Only) S3 VFD: @ROS3_VFD@ + (Read-Only) HDFS VFD: @HAVE_LIBHDFS@ dmalloc: @HAVE_DMALLOC@ Packages w/ extra debug output: @INTERNAL_DEBUG_OUTPUT@ API tracing: @TRACE_API@ |