diff options
| author | Dana Robinson <43805+derobins@users.noreply.github.com> | 2023-05-02 05:25:48 (GMT) |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-05-02 05:25:48 (GMT) |
| commit | 6eb021b68a8b46bbc89844713f014e6ba729ca7d (patch) | |
| tree | 25a971cb952841ac796db4f72a4be850ac2ce437 | |
| parent | da46fdbf50984636aeac936386068939a3760fe4 (diff) | |
| download | hdf5-6eb021b68a8b46bbc89844713f014e6ba729ca7d.zip hdf5-6eb021b68a8b46bbc89844713f014e6ba729ca7d.tar.gz hdf5-6eb021b68a8b46bbc89844713f014e6ba729ca7d.tar.bz2 | |
Sync with develop (#2871)
80 files changed, 10272 insertions, 853 deletions
@@ -37,6 +37,7 @@ $Source = ""; "H5D_alloc_time_t" => "Da", "H5D_append_cb_t" => "DA", "H5FD_mpio_collective_opt_t" => "Dc", + "H5D_selection_io_mode_t" => "DC", "H5D_fill_time_t" => "Df", "H5D_fill_value_t" => "DF", "H5D_gather_func_t" => "Dg", diff --git a/config/cmake/HDFLibMacros.cmake b/config/cmake/HDFLibMacros.cmake index b3985d5..282fe0c 100644 --- a/config/cmake/HDFLibMacros.cmake +++ b/config/cmake/HDFLibMacros.cmake @@ -88,7 +88,7 @@ macro (EXTERNAL_SZIP_LIBRARY compress_type encoding) -DBUILD_SHARED_LIBS:BOOL=OFF -DSZIP_PACKAGE_EXT:STRING=${HDF_PACKAGE_EXT} -DSZIP_EXTERNALLY_CONFIGURED:BOOL=OFF - -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} + -DCMAKE_BUILD_TYPE:STRING=${HDF_CFG_NAME} -DCMAKE_DEBUG_POSTFIX:STRING=${CMAKE_DEBUG_POSTFIX} -DCMAKE_INSTALL_PREFIX:PATH=${CMAKE_INSTALL_PREFIX} -DCMAKE_RUNTIME_OUTPUT_DIRECTORY:PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY} @@ -110,7 +110,7 @@ macro (EXTERNAL_SZIP_LIBRARY compress_type encoding) -DBUILD_SHARED_LIBS:BOOL=OFF -DSZIP_PACKAGE_EXT:STRING=${HDF_PACKAGE_EXT} -DSZIP_EXTERNALLY_CONFIGURED:BOOL=OFF - -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} + -DCMAKE_BUILD_TYPE:STRING=${HDF_CFG_NAME} -DCMAKE_DEBUG_POSTFIX:STRING=${CMAKE_DEBUG_POSTFIX} -DCMAKE_INSTALL_PREFIX:PATH=${CMAKE_INSTALL_PREFIX} -DCMAKE_RUNTIME_OUTPUT_DIRECTORY:PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY} @@ -166,7 +166,7 @@ macro (EXTERNAL_ZLIB_LIBRARY compress_type) -DBUILD_SHARED_LIBS:BOOL=OFF -DZLIB_PACKAGE_EXT:STRING=${HDF_PACKAGE_EXT} -DZLIB_EXTERNALLY_CONFIGURED:BOOL=OFF - -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} + -DCMAKE_BUILD_TYPE:STRING=${HDF_CFG_NAME} -DCMAKE_DEBUG_POSTFIX:STRING=${CMAKE_DEBUG_POSTFIX} -DCMAKE_INSTALL_PREFIX:PATH=${CMAKE_INSTALL_PREFIX} -DCMAKE_RUNTIME_OUTPUT_DIRECTORY:PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY} @@ -187,7 +187,7 @@ macro (EXTERNAL_ZLIB_LIBRARY compress_type) -DBUILD_SHARED_LIBS:BOOL=OFF -DZLIB_PACKAGE_EXT:STRING=${HDF_PACKAGE_EXT} -DZLIB_EXTERNALLY_CONFIGURED:BOOL=OFF - -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} + -DCMAKE_BUILD_TYPE:STRING=${HDF_CFG_NAME} -DCMAKE_DEBUG_POSTFIX:STRING=${CMAKE_DEBUG_POSTFIX} -DCMAKE_INSTALL_PREFIX:PATH=${CMAKE_INSTALL_PREFIX} -DCMAKE_RUNTIME_OUTPUT_DIRECTORY:PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY} diff --git a/config/cmake/HDFMacros.cmake b/config/cmake/HDFMacros.cmake index 5ac7316..e0e1220 100644 --- a/config/cmake/HDFMacros.cmake +++ b/config/cmake/HDFMacros.cmake @@ -14,26 +14,23 @@ macro (SET_HDF_BUILD_TYPE) get_property (_isMultiConfig GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) if (_isMultiConfig) - set (HDF_CFG_NAME ${CMAKE_BUILD_TYPE}) - set (HDF_BUILD_TYPE ${CMAKE_CFG_INTDIR}) + # HDF_CFG_BUILD_TYPE is used in the Fortran install commands for the build location of the .mod files set (HDF_CFG_BUILD_TYPE \${CMAKE_INSTALL_CONFIG_NAME}) + if (CMAKE_BUILD_TYPE) + # set the default to the specified command line define + set (HDF_CFG_NAME ${CMAKE_BUILD_TYPE}) + else () + # set the default to the MultiConfig variable + set (HDF_CFG_NAME ${CMAKE_CFG_INTDIR}) + endif () else () set (HDF_CFG_BUILD_TYPE ".") if (CMAKE_BUILD_TYPE) set (HDF_CFG_NAME ${CMAKE_BUILD_TYPE}) - set (HDF_BUILD_TYPE ${CMAKE_BUILD_TYPE}) else () set (HDF_CFG_NAME "Release") - set (HDF_BUILD_TYPE "Release") endif () endif () - if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) - message (VERBOSE "Setting build type to 'RelWithDebInfo' as none was specified.") - set (CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "Choose the type of build." FORCE) - # Set the possible values of build type for cmake-gui - set_property (CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" - "MinSizeRel" "RelWithDebInfo" "Developer") - endif () endmacro () #------------------------------------------------------------------------------- @@ -175,8 +172,8 @@ macro (HDF_IMPORT_SET_LIB_OPTIONS libtarget libname libtype libversion) ) else () set_target_properties (${libtarget} PROPERTIES - IMPORTED_IMPLIB "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${HDF_BUILD_TYPE}/${CMAKE_IMPORT_LIBRARY_PREFIX}${IMPORT_LIB_NAME}${CMAKE_IMPORT_LIBRARY_SUFFIX}" - IMPORTED_LOCATION "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${HDF_BUILD_TYPE}/${CMAKE_IMPORT_LIBRARY_PREFIX}${IMPORT_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}" + IMPORTED_IMPLIB "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${HDF_CFG_NAME}/${CMAKE_IMPORT_LIBRARY_PREFIX}${IMPORT_LIB_NAME}${CMAKE_IMPORT_LIBRARY_SUFFIX}" + IMPORTED_LOCATION "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${HDF_CFG_NAME}/${CMAKE_IMPORT_LIBRARY_PREFIX}${IMPORT_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}" ) endif () else () @@ -201,7 +198,7 @@ macro (HDF_IMPORT_SET_LIB_OPTIONS libtarget libname libtype libversion) else () if (WIN32 AND NOT MINGW) set_target_properties (${libtarget} PROPERTIES - IMPORTED_LOCATION "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${HDF_BUILD_TYPE}/${IMPORT_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" + IMPORTED_LOCATION "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${HDF_CFG_NAME}/${IMPORT_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" IMPORTED_LINK_INTERFACE_LANGUAGES "C" ) else () @@ -455,7 +452,7 @@ macro (HDF_DIR_PATHS package_prefix) ) get_property(_isMultiConfig GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) if(_isMultiConfig) - set (CMAKE_TEST_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${CMAKE_BUILD_TYPE}) + set (CMAKE_TEST_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${HDF_CFG_NAME}) set (CMAKE_PDB_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all pdb files." ) diff --git a/config/cmake/libhdf5.settings.cmake.in b/config/cmake/libhdf5.settings.cmake.in index f2cf6c0..00ae980 100644 --- a/config/cmake/libhdf5.settings.cmake.in +++ b/config/cmake/libhdf5.settings.cmake.in @@ -13,7 +13,7 @@ General Information: Compiling Options: ------------------ - Build Mode: @CMAKE_BUILD_TYPE@ + Build Mode: @HDF_CFG_NAME@ Debugging Symbols: @HDF5_ENABLE_SYMBOLS@ Asserts: @HDF5_ENABLE_ASSERTS@ Profiling: @HDF5_ENABLE_PROFILING@ diff --git a/configure.ac b/configure.ac index 1cc487a..c94f2b9 100644 --- a/configure.ac +++ b/configure.ac @@ -1155,7 +1155,7 @@ AC_SUBST([HDF5_TOOLS]) ## Default is to build tests and tools HDF5_TOOLS=yes -AC_MSG_CHECKING([if building tools is disabled]) +AC_MSG_CHECKING([if building tools is enabled]) AC_ARG_ENABLE([tools], [AS_HELP_STRING([--enable-tools], diff --git a/doxygen/aliases b/doxygen/aliases index 27090e6..96977f3 100644 --- a/doxygen/aliases +++ b/doxygen/aliases @@ -383,4 +383,5 @@ ALIASES += fortran_obsolete="Obsolete API, use the Fortran 2003 version instead. ALIASES += fortran_file="Pointer to filename the async subroutine is being called from, filename must be null character terminated" ALIASES += fortran_func="Pointer to function name the async subroutine is being called in, func must be null character terminated" ALIASES += fortran_line="Line number the async subroutine is being called at" +ALIASES += fortran_plist_id="Property list identifier" diff --git a/doxygen/dox/LearnBasics2.dox b/doxygen/dox/LearnBasics2.dox index 6f94c7f..87bbe87 100644 --- a/doxygen/dox/LearnBasics2.dox +++ b/doxygen/dox/LearnBasics2.dox @@ -788,7 +788,7 @@ The function #H5Tarray_create creates a new array datatype object. Parameters sp \li the dimension permutation of the array, i.e., whether the elements of the array are listed in C or FORTRAN order. <h4>Working with existing array datatypes</h4> -When working with existing arrays, one must first determine the the rank, or number of dimensions, of the array. +When working with existing arrays, one must first determine the rank, or number of dimensions, of the array. The function #H5Tget_array_dims returns the rank of a specified array datatype. diff --git a/doxygen/dox/LearnBasics3.dox b/doxygen/dox/LearnBasics3.dox index 2fe0f52..06afacd 100644 --- a/doxygen/dox/LearnBasics3.dox +++ b/doxygen/dox/LearnBasics3.dox @@ -210,7 +210,7 @@ For details on compiling an HDF5 application: an extendible array dataset, pass in #H5P_DATASET_CREATE for the property list class. \li The #H5Pset_chunk call modifies a Dataset Creation Property List instance to store a chunked layout dataset and sets the size of the chunks used. -\li To extend an unlimited dimension dataset use the the #H5Dset_extent call. Please be aware that +\li To extend an unlimited dimension dataset use the #H5Dset_extent call. Please be aware that after this call, the dataset's dataspace must be refreshed with #H5Dget_space before more data can be accessed. \li The #H5Pget_chunk call retrieves the size of chunks for the raw data of a chunked layout dataset. \li Once there is no longer a need for a Property List instance, it should be closed with the #H5Pclose call. diff --git a/doxygen/dox/ViewTools.dox b/doxygen/dox/ViewTools.dox index 2212d4b..66b2def 100644 --- a/doxygen/dox/ViewTools.dox +++ b/doxygen/dox/ViewTools.dox @@ -465,7 +465,7 @@ example <code style="background-color:whitesmoke;">h5_crtgrpar.c</code>. To disp \endcode \subsubsection subsubsecViewToolsViewDset_h5dumpEx5 Example 5 -The <code style="background-color:whitesmoke;">-p</code> option is used to examine the the dataset filters, storage layout, and fill value properties of a dataset. +The <code style="background-color:whitesmoke;">-p</code> option is used to examine the dataset filters, storage layout, and fill value properties of a dataset. This option can be useful for checking how well compression works, or even for analyzing performance and dataset size issues related to chunking. (The smaller the chunk size, the more chunks that HDF5 diff --git a/doxygen/examples/H5.format.1.0.html b/doxygen/examples/H5.format.1.0.html index 4eb0548..26d0421 100644 --- a/doxygen/examples/H5.format.1.0.html +++ b/doxygen/examples/H5.format.1.0.html @@ -1488,7 +1488,7 @@ Elena> "Free-space object" </tr> <tr valign=top> - <td>Object Size</td> <td>This is the size of the the fields + <td>Object Size</td> <td>This is the size of the fields above plus the object data stored for the object. The actual storage size is rounded up to a multiple of eight.</td> diff --git a/doxygen/examples/H5.format.1.1.html b/doxygen/examples/H5.format.1.1.html index 9d03a76..3af50d6 100644 --- a/doxygen/examples/H5.format.1.1.html +++ b/doxygen/examples/H5.format.1.1.html @@ -6091,7 +6091,7 @@ TABLE.list TD { border:none; } <td>Used by the library before version 1.6.1. In this version, the Flags field is used to indicate whether the actual message is stored in the global heap (never implemented). The Pointer field - either contains the the header message address in the global heap + either contains the header message address in the global heap (never implemented) or the address of the shared object header.</td> </tr> </table> diff --git a/doxygen/examples/ImageSpec.html b/doxygen/examples/ImageSpec.html index 1b700ff..130d86e 100644 --- a/doxygen/examples/ImageSpec.html +++ b/doxygen/examples/ImageSpec.html @@ -851,7 +851,7 @@ This defines the color model that the entries in the palette data set represent. "RGB"</dt> <dd> -Each color index contains a triplet where the the first value defines the +Each color index contains a triplet where the first value defines the red component, second defines the green component, and the third the blue component.</dd> @@ -859,7 +859,7 @@ component.</dd> "CMY"</dt> <dd> -Each color index contains a triplet where the the first value defines the +Each color index contains a triplet where the first value defines the cyan component, second defines the magenta component, and the third the yellow component.</dd> @@ -867,7 +867,7 @@ yellow component.</dd> "CMYK"</dt> <dd> -Each color index contains a quadruplet where the the first value defines +Each color index contains a quadruplet where the first value defines the cyan component, second defines the magenta component, the third the yellow component, and the forth the black component.</dd> @@ -875,7 +875,7 @@ yellow component, and the forth the black component.</dd> "YCbCr"</dt> <dd> -Class Y encoding model. Each color index contains a triplet where the the +Class Y encoding model. Each color index contains a triplet where the first value defines the luminance, second defines the Cb Chromonance, and the third the Cr Chromonance.</dd> @@ -884,14 +884,14 @@ the third the Cr Chromonance.</dd> <dd> Composite encoding color model. Each color index contains a triplet where -the the first value defines the luminance component, second defines the +the first value defines the luminance component, second defines the chromonance component, and the third the value component.</dd> <dt> "HSV"</dt> <dd> -Each color index contains a triplet where the the first value defines the +Each color index contains a triplet where the first value defines the hue component, second defines the saturation component, and the third the value component. The hue component defines the hue spectrum with a low value representing magenta/red progressing to a high value which would diff --git a/doxygen/examples/tables/propertyLists.dox b/doxygen/examples/tables/propertyLists.dox index 375fd50..a260c9d 100644 --- a/doxygen/examples/tables/propertyLists.dox +++ b/doxygen/examples/tables/propertyLists.dox @@ -245,7 +245,7 @@ file access property list.</td> </tr> <tr> <td>#H5Pset_page_buffer_size/#H5Pget_page_buffer_size</td> -<td>Set/get the the maximum size for the page buffer.</td> +<td>Set/get the maximum size for the page buffer.</td> </tr> <tr> <td>#H5Pset_sieve_buf_size/#H5Pget_sieve_buf_size</td> @@ -694,6 +694,18 @@ of the library for reading or writing the actual data.</td> <td>Gets local and global causes that broke collective I/O on the last parallel I/O call.</td> </tr> <tr> +<td>#H5Pset_selection_io/#H5Pget_selection_io</td> +<td>Sets/gets the selection I/O mode.</td> +</tr> +<tr> +<td>#H5Pget_no_selection_io_cause</td> +<td>Gets the cause for not performing selection or vector I/O on the last parallel I/O call.</td> +</tr> +<tr> +<td>#H5Pset_modify_write_buf/#H5Pget_modify_write_buf</td> +<td>Sets/gets a flag allowing the library to modify the contents of the write buffer.</td> +</tr> +<tr> <td>H5Pset_preserve/H5Pget_preserve</td> <td>No longer available, deprecated as it no longer has any effect.</td> </tr> @@ -952,4 +964,3 @@ encoding for object names.</td> //! [acpl_table] * */ -
\ No newline at end of file diff --git a/doxygen/hdf5_navtree_hacks.js b/doxygen/hdf5_navtree_hacks.js index dda8984..804701f 100644 --- a/doxygen/hdf5_navtree_hacks.js +++ b/doxygen/hdf5_navtree_hacks.js @@ -141,7 +141,7 @@ function initNavTree(toroot,relpath) $(window).on("load", showRoot); } -// return false if the the node has no children at all, or has only section/subsection children +// return false if the node has no children at all, or has only section/subsection children function checkChildrenData(node) { if (!(typeof(node.childrenData)==='string')) { for (var i in node.childrenData) { diff --git a/fortran/src/H5Af.c b/fortran/src/H5Af.c index ba3b62d..0fdd8fd 100644 --- a/fortran/src/H5Af.c +++ b/fortran/src/H5Af.c @@ -359,7 +359,7 @@ done: * loc_id - Object identifier * OUTPUTS * - * corder_valid - Indicates whether the the creation order data is valid for this attribute + * corder_valid - Indicates whether the creation order data is valid for this attribute * corder - Is a positive integer containing the creation order of the attribute * cset - Indicates the character set used for the attribute’s name * data_size - indicates the size, in the number of characters, of the attribute @@ -427,7 +427,7 @@ done: * lapl_id - Link access property list * OUTPUTS * - * corder_valid - Indicates whether the the creation order data is valid for this attribute + * corder_valid - Indicates whether the creation order data is valid for this attribute * corder - Is a positive integer containing the creation order of the attribute * cset - Indicates the character set used for the attribute’s name * data_size - indicates the size, in the number of characters, of the attribute @@ -493,7 +493,7 @@ done: * lapl_id - Link access property list * OUTPUTS * - * corder_valid - Indicates whether the the creation order data is valid for this attribute + * corder_valid - Indicates whether the creation order data is valid for this attribute * corder - Is a positive integer containing the creation order of the attribute * cset - Indicates the character set used for the attribute’s name * data_size - indicates the size, in the number of characters, of the attribute diff --git a/fortran/src/H5Lf.c b/fortran/src/H5Lf.c index 63bed99..c7ce511 100644 --- a/fortran/src/H5Lf.c +++ b/fortran/src/H5Lf.c @@ -232,7 +232,7 @@ done: * lapl_id - Link access property list * OUTPUTS * - * corder_valid - Indicates whether the the creation order data is valid for this attribute + * corder_valid - Indicates whether the creation order data is valid for this attribute * corder - Is a positive integer containing the creation order of the attribute * cset - Indicates the character set used for the attribute’s name * data_size - indicates the size, in the number of characters, of the attribute diff --git a/fortran/src/H5Pff.F90 b/fortran/src/H5Pff.F90 index 75cd323..49917dd 100644 --- a/fortran/src/H5Pff.F90 +++ b/fortran/src/H5Pff.F90 @@ -609,6 +609,140 @@ CONTAINS !> !! \ingroup FH5P !! +!! \brief Sets the selection I/O mode +!! +!! \param plist_id \fortran_plist_id +!! \param selection_io_mode The selection I/O mode +!! \param hdferr \fortran_error +!! +!! See C API: @ref H5Pset_selection_io() +!! + SUBROUTINE h5pset_selection_io_f(plist_id, selection_io_mode, hdferr) + + IMPLICIT NONE + + INTEGER(HID_T), INTENT(IN) :: plist_id + INTEGER, INTENT(IN) :: selection_io_mode + INTEGER, INTENT(OUT) :: hdferr + + INTERFACE + INTEGER(C_INT) FUNCTION H5Pset_selection_io(plist_id, selection_io_mode) BIND(C, NAME='H5Pset_selection_io') + IMPORT :: HID_T, C_INT + IMPLICIT NONE + INTEGER(HID_T), VALUE :: plist_id + INTEGER(C_INT), VALUE :: selection_io_mode + END FUNCTION H5Pset_selection_io + END INTERFACE + + hdferr = INT(H5Pset_selection_io(plist_id, INT(selection_io_mode, C_INT))) + + END SUBROUTINE h5pset_selection_io_f + +!> +!! \ingroup FH5P +!! +!! \brief Retrieves the selection I/O mode +!! +!! \param plist_id \fortran_plist_id +!! \param selection_io_mode The selection I/O mode +!! \param hdferr \fortran_error +!! +!! See C API: @ref H5Pget_selection_io() +!! + SUBROUTINE h5pget_selection_io_f(plist_id, selection_io_mode, hdferr) + + IMPLICIT NONE + + INTEGER(HID_T), INTENT(IN) :: plist_id + INTEGER, INTENT(OUT) :: selection_io_mode + INTEGER, INTENT(OUT) :: hdferr + + INTEGER(C_INT) :: c_selection_io_mode + + INTERFACE + INTEGER(C_INT) FUNCTION H5Pget_selection_io(plist_id, selection_io_mode) BIND(C, NAME='H5Pget_selection_io') + IMPORT :: HID_T, C_INT + IMPLICIT NONE + INTEGER(HID_T), VALUE :: plist_id + INTEGER(C_INT) :: selection_io_mode + END FUNCTION H5Pget_selection_io + END INTERFACE + + hdferr = INT(H5Pget_selection_io(plist_id, c_selection_io_mode)) + selection_io_mode = INT(c_selection_io_mode) + + END SUBROUTINE h5pget_selection_io_f + +!> +!! \ingroup FH5P +!! +!! \brief Allows the library to modify the contents of the write buffer +!! +!! \param plist_id \fortran_plist_id +!! \param modify_write_buf Whether the library can modify the contents of the write buffer +!! \param hdferr \fortran_error +!! +!! See C API: @ref H5Pset_modify_write_buf() +!! + SUBROUTINE h5pset_modify_write_buf_f(plist_id, modify_write_buf, hdferr) + + IMPLICIT NONE + + INTEGER(HID_T), INTENT(IN) :: plist_id + LOGICAL, INTENT(IN) :: modify_write_buf + INTEGER, INTENT(OUT) :: hdferr + + INTERFACE + INTEGER(C_INT) FUNCTION H5Pset_modify_write_buf(plist_id, modify_write_buf) BIND(C, NAME='H5Pset_modify_write_buf') + IMPORT :: HID_T, C_INT, C_BOOL + IMPLICIT NONE + INTEGER(HID_T), VALUE :: plist_id + LOGICAL(C_BOOL), VALUE :: modify_write_buf + END FUNCTION H5Pset_modify_write_buf + END INTERFACE + + hdferr = INT(H5Pset_modify_write_buf(plist_id, LOGICAL(modify_write_buf, C_BOOL))) + + END SUBROUTINE h5pset_modify_write_buf_f + +!> +!! \ingroup FH5P +!! +!! \brief Retrieves the "modify write buffer" property +!! +!! \param plist_id \fortran_plist_id +!! \param modify_write_buf Whether the library can modify the contents of the write buffer +!! \param hdferr \fortran_error +!! +!! See C API: @ref H5Pget_modify_write_buf() +!! + SUBROUTINE h5pget_modify_write_buf_f(plist_id, modify_write_buf, hdferr) + + IMPLICIT NONE + + INTEGER(HID_T), INTENT(IN) :: plist_id + LOGICAL, INTENT(OUT) :: modify_write_buf + INTEGER, INTENT(OUT) :: hdferr + + LOGICAL(C_BOOL) :: c_modify_write_buf + + INTERFACE + INTEGER(C_INT) FUNCTION H5Pget_modify_write_buf(plist_id, modify_write_buf) BIND(C, NAME='H5Pget_modify_write_buf') + IMPORT :: HID_T, C_INT, C_BOOL + IMPLICIT NONE + INTEGER(HID_T), VALUE :: plist_id + LOGICAL(C_BOOL) :: modify_write_buf + END FUNCTION H5Pget_modify_write_buf + END INTERFACE + + hdferr = INT(H5Pget_modify_write_buf(plist_id, c_modify_write_buf)) + modify_write_buf = LOGICAL(c_modify_write_buf) + + END SUBROUTINE h5pget_modify_write_buf_f + +!> +!! \ingroup FH5P +!! !! \brief Sets the byte size of the offsets and lengths used to address objects in an HDF5 file. !! !! \param prp_id File creation property list identifier. diff --git a/fortran/src/H5Sf.c b/fortran/src/H5Sf.c index d04b639..00b97f8 100644 --- a/fortran/src/H5Sf.c +++ b/fortran/src/H5Sf.c @@ -176,7 +176,7 @@ h5scopy_c(hid_t_f *space_id, hid_t_f *new_space_id) * h5sget_select_hyper_nblocks_c * PURPOSE * Call H5SH5Sget_select_hyper_nblocks to - * get the the number of hyperslab blocks in + * get the number of hyperslab blocks in * the current dataspace selection if successful * INPUTS * space_id - identifier of the dataspace @@ -215,7 +215,7 @@ h5sget_select_hyper_nblocks_c(hid_t_f *space_id, hssize_t_f *num_blocks) * h5sget_select_elem_npoints_c * PURPOSE * Call H5Sget_select_elem_npoints to - * get the the number of element points in + * get the number of element points in * the current dataspace selection if successful * INPUTS * space_id - identifier of the dataspace diff --git a/fortran/src/H5Tf.c b/fortran/src/H5Tf.c index 11f62b7..a74f5d1 100644 --- a/fortran/src/H5Tf.c +++ b/fortran/src/H5Tf.c @@ -1373,7 +1373,7 @@ DONE: * type_id - identifier of the dataspace * member_no - Number of the field whose offset is requested * OUTPUTS - * offset - byte offset of the the beginning of the field of + * offset - byte offset of the beginning of the field of * a compound datatype * RETURNS * always 0 diff --git a/fortran/src/H5_f.c b/fortran/src/H5_f.c index 3e1b65d..a9f2d96 100644 --- a/fortran/src/H5_f.c +++ b/fortran/src/H5_f.c @@ -467,6 +467,9 @@ h5init_flags_c(int_f *h5d_flags, size_t_f *h5d_size_flags, int_f *h5e_flags, hid h5d_flags[26] = (int_f)H5D_VDS_FIRST_MISSING; h5d_flags[27] = (int_f)H5D_VDS_LAST_AVAILABLE; h5d_flags[28] = (int_f)H5D_VIRTUAL; + h5d_flags[29] = (int_f)H5D_SELECTION_IO_MODE_DEFAULT; + h5d_flags[30] = (int_f)H5D_SELECTION_IO_MODE_OFF; + h5d_flags[31] = (int_f)H5D_SELECTION_IO_MODE_ON; /* * H5E flags diff --git a/fortran/src/H5_ff.F90 b/fortran/src/H5_ff.F90 index f952cac..651c96d 100644 --- a/fortran/src/H5_ff.F90 +++ b/fortran/src/H5_ff.F90 @@ -77,7 +77,7 @@ MODULE H5LIB ! ! H5D flags declaration ! - INTEGER, PARAMETER :: H5D_FLAGS_LEN = 29 + INTEGER, PARAMETER :: H5D_FLAGS_LEN = 32 INTEGER, DIMENSION(1:H5D_FLAGS_LEN) :: H5D_flags INTEGER, PARAMETER :: H5D_SIZE_FLAGS_LEN = 2 INTEGER(SIZE_T), DIMENSION(1:H5D_SIZE_FLAGS_LEN) :: H5D_size_flags @@ -394,35 +394,38 @@ CONTAINS ! ! H5D flags ! - H5D_COMPACT_F = H5D_flags(1) - H5D_CONTIGUOUS_F = H5D_flags(2) - H5D_CHUNKED_F = H5D_flags(3) - H5D_ALLOC_TIME_ERROR_F = H5D_flags(4) - H5D_ALLOC_TIME_DEFAULT_F = H5D_flags(5) - H5D_ALLOC_TIME_EARLY_F = H5D_flags(6) - H5D_ALLOC_TIME_LATE_F = H5D_flags(7) - H5D_ALLOC_TIME_INCR_F = H5D_flags(8) - H5D_SPACE_STS_ERROR_F = H5D_flags(9) - H5D_SPACE_STS_NOT_ALLOCATED_F = H5D_flags(10) - H5D_SPACE_STS_PART_ALLOCATED_F = H5D_flags(11) - H5D_SPACE_STS_ALLOCATED_F = H5D_flags(12) - H5D_FILL_TIME_ERROR_F = H5D_flags(13) - H5D_FILL_TIME_ALLOC_F = H5D_flags(14) - H5D_FILL_TIME_NEVER_F = H5D_flags(15) - H5D_FILL_VALUE_ERROR_F = H5D_flags(16) - H5D_FILL_VALUE_UNDEFINED_F = H5D_flags(17) - H5D_FILL_VALUE_DEFAULT_F = H5D_flags(18) - H5D_FILL_VALUE_USER_DEFINED_F = H5D_flags(19) - H5D_CHUNK_CACHE_W0_DFLT_F = H5D_flags(20) - H5D_MPIO_NO_COLLECTIVE_F = H5D_flags(21) - H5D_MPIO_CHUNK_INDEPENDENT_F = H5D_flags(22) - H5D_MPIO_CHUNK_COLLECTIVE_F = H5D_flags(23) - H5D_MPIO_CHUNK_MIXED_F = H5D_flags(24) - H5D_MPIO_CONTIG_COLLECTIVE_F = H5D_flags(25) - H5D_VDS_ERROR_F = H5D_flags(26) - H5D_VDS_FIRST_MISSING_F = H5D_flags(27) - H5D_VDS_LAST_AVAILABLE_F = H5D_flags(28) - H5D_VIRTUAL_F = H5D_flags(29) + H5D_COMPACT_F = H5D_flags(1) + H5D_CONTIGUOUS_F = H5D_flags(2) + H5D_CHUNKED_F = H5D_flags(3) + H5D_ALLOC_TIME_ERROR_F = H5D_flags(4) + H5D_ALLOC_TIME_DEFAULT_F = H5D_flags(5) + H5D_ALLOC_TIME_EARLY_F = H5D_flags(6) + H5D_ALLOC_TIME_LATE_F = H5D_flags(7) + H5D_ALLOC_TIME_INCR_F = H5D_flags(8) + H5D_SPACE_STS_ERROR_F = H5D_flags(9) + H5D_SPACE_STS_NOT_ALLOCATED_F = H5D_flags(10) + H5D_SPACE_STS_PART_ALLOCATED_F = H5D_flags(11) + H5D_SPACE_STS_ALLOCATED_F = H5D_flags(12) + H5D_FILL_TIME_ERROR_F = H5D_flags(13) + H5D_FILL_TIME_ALLOC_F = H5D_flags(14) + H5D_FILL_TIME_NEVER_F = H5D_flags(15) + H5D_FILL_VALUE_ERROR_F = H5D_flags(16) + H5D_FILL_VALUE_UNDEFINED_F = H5D_flags(17) + H5D_FILL_VALUE_DEFAULT_F = H5D_flags(18) + H5D_FILL_VALUE_USER_DEFINED_F = H5D_flags(19) + H5D_CHUNK_CACHE_W0_DFLT_F = H5D_flags(20) + H5D_MPIO_NO_COLLECTIVE_F = H5D_flags(21) + H5D_MPIO_CHUNK_INDEPENDENT_F = H5D_flags(22) + H5D_MPIO_CHUNK_COLLECTIVE_F = H5D_flags(23) + H5D_MPIO_CHUNK_MIXED_F = H5D_flags(24) + H5D_MPIO_CONTIG_COLLECTIVE_F = H5D_flags(25) + H5D_VDS_ERROR_F = H5D_flags(26) + H5D_VDS_FIRST_MISSING_F = H5D_flags(27) + H5D_VDS_LAST_AVAILABLE_F = H5D_flags(28) + H5D_VIRTUAL_F = H5D_flags(29) + H5D_SELECTION_IO_MODE_DEFAULT_F = H5D_flags(30) + H5D_SELECTION_IO_MODE_OFF_F = H5D_flags(31) + H5D_SELECTION_IO_MODE_ON_F = H5D_flags(32) H5D_CHUNK_CACHE_NSLOTS_DFLT_F = H5D_size_flags(1) H5D_CHUNK_CACHE_NBYTES_DFLT_F = H5D_size_flags(2) diff --git a/fortran/src/H5f90global.F90 b/fortran/src/H5f90global.F90 index 984cae9..c37e22d 100644 --- a/fortran/src/H5f90global.F90 +++ b/fortran/src/H5f90global.F90 @@ -330,6 +330,9 @@ MODULE H5GLOBAL !DEC$ATTRIBUTES DLLEXPORT :: H5D_VDS_FIRST_MISSING_F !DEC$ATTRIBUTES DLLEXPORT :: H5D_VDS_LAST_AVAILABLE_F !DEC$ATTRIBUTES DLLEXPORT :: H5D_VIRTUAL_F + !DEC$ATTRIBUTES DLLEXPORT :: H5D_SELECTION_IO_MODE_DEFAULT_F + !DEC$ATTRIBUTES DLLEXPORT :: H5D_SELECTION_IO_MODE_OFF_F + !DEC$ATTRIBUTES DLLEXPORT :: H5D_SELECTION_IO_MODE_ON_F !DEC$endif !> \addtogroup FH5D !> @{ @@ -375,6 +378,9 @@ MODULE H5GLOBAL INTEGER :: H5D_VDS_FIRST_MISSING_F !< H5D_VDS_FIRST_MISSING INTEGER :: H5D_VDS_LAST_AVAILABLE_F !< H5D_VDS_LAST_AVAILABLE INTEGER :: H5D_VIRTUAL_F !< H5D_VIRTUAL + INTEGER :: H5D_SELECTION_IO_MODE_DEFAULT_F !< H5D_SELECTION_IO_MODE_DEFAULT_F + INTEGER :: H5D_SELECTION_IO_MODE_OFF_F !< H5D_SELECTION_IO_MODE_OFF_F + INTEGER :: H5D_SELECTION_IO_MODE_ON_F !< H5D_SELECTION_IO_MODE_ON_F ! ! H5E flags declaration ! diff --git a/fortran/src/hdf5_fortrandll.def.in b/fortran/src/hdf5_fortrandll.def.in index e55be46..4719633 100644 --- a/fortran/src/hdf5_fortrandll.def.in +++ b/fortran/src/hdf5_fortrandll.def.in @@ -391,6 +391,10 @@ H5P_mp_H5PSET_VOL_F H5P_mp_H5PGET_VOL_ID_F H5P_mp_H5PSET_FILE_LOCKING_F H5P_mp_H5PGET_FILE_LOCKING_F +H5P_mp_H5PSET_SELECTION_IO_F +H5P_mp_H5PGET_SELECTION_IO_F +H5P_mp_H5PSET_MODIFY_WRITE_BUF_F +H5P_mp_H5PGET_MODIFY_WRITE_BUF_F ; Parallel @H5_NOPAREXP@H5P_mp_H5PSET_FAPL_MPIO_F @H5_NOPAREXP@H5P_mp_H5PGET_FAPL_MPIO_F diff --git a/fortran/test/H5_test_buildiface.F90 b/fortran/test/H5_test_buildiface.F90 index 0ea3852..ca945db 100644 --- a/fortran/test/H5_test_buildiface.F90 +++ b/fortran/test/H5_test_buildiface.F90 @@ -133,14 +133,25 @@ PROGRAM H5_test_buildiface WRITE(11,'(A)') '!DEC$endif' ! Subroutine API - WRITE(11,'(A)') ' SUBROUTINE verify_integer_kind_'//TRIM(ADJUSTL(chr2))//'(string,value,correct_value,total_error)' + WRITE(11,'(A)') ' SUBROUTINE verify_integer_kind_'//TRIM(ADJUSTL(chr2))//'(string,value,correct_value,total_error,chck_eq)' WRITE(11,'(A)') ' IMPLICIT NONE' WRITE(11,'(A)') ' CHARACTER(LEN=*) :: string' WRITE(11,'(A)') ' INTEGER(KIND='//TRIM(ADJUSTL(chr2))//') :: value, correct_value' WRITE(11,'(A)') ' INTEGER :: total_error' - WRITE(11,'(A)') ' IF (value .NE. correct_value) THEN' - WRITE(11,'(A)') ' total_error=total_error+1' - WRITE(11,'(A)') ' WRITE(*,*) "ERROR: INCORRECT INTEGER VALIDATION ", string' + WRITE(11,'(A)') ' LOGICAL, OPTIONAL :: chck_eq' + WRITE(11,'(A)') ' LOGICAL :: chck_eq_opt' + WRITE(11,'(A)') ' chck_eq_opt = .TRUE.' + WRITE(11,'(A)') ' IF(PRESENT(chck_eq)) chck_eq_opt = chck_eq' + WRITE(11,'(A)') ' IF(chck_eq_opt .EQV. .TRUE.)THEN' + WRITE(11,'(A)') ' IF (value .NE. correct_value) THEN' + WRITE(11,'(A)') ' total_error=total_error+1' + WRITE(11,'(A)') ' WRITE(*,*) "ERROR: INCORRECT INTEGER VALIDATION ", string' + WRITE(11,'(A)') ' ENDIF' + WRITE(11,'(A)') ' ELSE' + WRITE(11,'(A)') ' IF (value .EQ. correct_value) THEN' + WRITE(11,'(A)') ' total_error=total_error+1' + WRITE(11,'(A)') ' WRITE(*,*) "ERROR: INCORRECT INTEGER VALIDATION ", string' + WRITE(11,'(A)') ' ENDIF' WRITE(11,'(A)') ' ENDIF' WRITE(11,'(A)') ' END SUBROUTINE verify_integer_kind_'//TRIM(ADJUSTL(chr2)) ENDDO @@ -157,14 +168,25 @@ PROGRAM H5_test_buildiface WRITE(11,'(A)') '!DEC$endif' ! Subroutine API - WRITE(11,'(A)') ' SUBROUTINE verify_real_kind_'//TRIM(ADJUSTL(chr2))//'(string,value,correct_value,total_error)' + WRITE(11,'(A)') ' SUBROUTINE verify_real_kind_'//TRIM(ADJUSTL(chr2))//'(string,value,correct_value,total_error,chck_eq)' WRITE(11,'(A)') ' IMPLICIT NONE' WRITE(11,'(A)') ' CHARACTER(LEN=*) :: string' WRITE(11,'(A)') ' REAL(KIND='//TRIM(ADJUSTL(chr2))//') :: value, correct_value' WRITE(11,'(A)') ' INTEGER :: total_error' - WRITE(11,'(A)') ' IF (.NOT.real_eq_kind_'//TRIM(ADJUSTL(chr2))//'( value, correct_value) ) THEN' - WRITE(11,'(A)') ' total_error=total_error+1' - WRITE(11,'(A)') ' WRITE(*,*) "ERROR: INCORRECT REAL VALIDATION ", string' + WRITE(11,'(A)') ' LOGICAL, OPTIONAL :: chck_eq' + WRITE(11,'(A)') ' LOGICAL :: chck_eq_opt' + WRITE(11,'(A)') ' chck_eq_opt = .TRUE.' + WRITE(11,'(A)') ' IF(PRESENT(chck_eq)) chck_eq_opt = chck_eq' + WRITE(11,'(A)') ' IF(chck_eq_opt .EQV. .TRUE.)THEN' + WRITE(11,'(A)') ' IF (.NOT.real_eq_kind_'//TRIM(ADJUSTL(chr2))//'( value, correct_value) ) THEN' + WRITE(11,'(A)') ' total_error=total_error+1' + WRITE(11,'(A)') ' WRITE(*,*) "ERROR: INCORRECT REAL VALIDATION ", string' + WRITE(11,'(A)') ' ENDIF' + WRITE(11,'(A)') ' ELSE' + WRITE(11,'(A)') ' IF (real_eq_kind_'//TRIM(ADJUSTL(chr2))//'( value, correct_value) ) THEN' + WRITE(11,'(A)') ' total_error=total_error+1' + WRITE(11,'(A)') ' WRITE(*,*) "ERROR: INCORRECT REAL VALIDATION ", string' + WRITE(11,'(A)') ' ENDIF' WRITE(11,'(A)') ' ENDIF' WRITE(11,'(A)') ' END SUBROUTINE verify_real_kind_'//TRIM(ADJUSTL(chr2)) @@ -228,14 +250,25 @@ PROGRAM H5_test_buildiface WRITE(11,'(A)') '!DEC$endif' ! Subroutine API - WRITE(11,'(A)') ' SUBROUTINE verify_character(string,value,correct_value,total_error)' + WRITE(11,'(A)') ' SUBROUTINE verify_character(string,value,correct_value,total_error,chck_eq)' WRITE(11,'(A)') ' IMPLICIT NONE' WRITE(11,'(A)') ' CHARACTER*(*) :: string' WRITE(11,'(A)') ' CHARACTER*(*) :: value, correct_value' WRITE(11,'(A)') ' INTEGER :: total_error' - WRITE(11,'(A)') ' IF (TRIM(value) .NE. TRIM(correct_value)) THEN' - WRITE(11,'(A)') ' total_error = total_error + 1' - WRITE(11,'(A)') ' WRITE(*,*) "ERROR: INCORRECT VALIDATION ", string' + WRITE(11,'(A)') ' LOGICAL, OPTIONAL :: chck_eq' + WRITE(11,'(A)') ' LOGICAL :: chck_eq_opt' + WRITE(11,'(A)') ' chck_eq_opt = .TRUE.' + WRITE(11,'(A)') ' IF(PRESENT(chck_eq)) chck_eq_opt = chck_eq' + WRITE(11,'(A)') ' IF(chck_eq_opt .EQV. .TRUE.)THEN' + WRITE(11,'(A)') ' IF (TRIM(value) .NE. TRIM(correct_value)) THEN' + WRITE(11,'(A)') ' total_error = total_error + 1' + WRITE(11,'(A)') ' WRITE(*,*) "ERROR: INCORRECT VALIDATION ", string' + WRITE(11,'(A)') ' ENDIF' + WRITE(11,'(A)') ' ELSE' + WRITE(11,'(A)') ' IF (TRIM(value) .EQ. TRIM(correct_value)) THEN' + WRITE(11,'(A)') ' total_error = total_error + 1' + WRITE(11,'(A)') ' WRITE(*,*) "ERROR: INCORRECT VALIDATION ", string' + WRITE(11,'(A)') ' ENDIF' WRITE(11,'(A)') ' ENDIF' WRITE(11,'(A)') ' END SUBROUTINE verify_character' @@ -248,16 +281,26 @@ PROGRAM H5_test_buildiface WRITE(11,'(A)') '!DEC$attributes dllexport :: verify_logical' WRITE(11,'(A)') '!DEC$endif' ! Subroutine API - WRITE(11,'(A)') ' SUBROUTINE verify_logical(string,value,correct_value,total_error)' + WRITE(11,'(A)') ' SUBROUTINE verify_logical(string,value,correct_value,total_error,chck_eq)' WRITE(11,'(A)') ' IMPLICIT NONE' WRITE(11,'(A)') ' CHARACTER(LEN=*) :: string' WRITE(11,'(A)') ' LOGICAL :: value, correct_value' WRITE(11,'(A)') ' INTEGER :: total_error' - WRITE(11,'(A)') ' IF (value .NEQV. correct_value) THEN' - WRITE(11,'(A)') ' total_error = total_error + 1' - WRITE(11,'(A)') ' WRITE(*,*) "ERROR: INCORRECT VALIDATION ", string' + WRITE(11,'(A)') ' LOGICAL, OPTIONAL :: chck_eq' + WRITE(11,'(A)') ' LOGICAL :: chck_eq_opt' + WRITE(11,'(A)') ' chck_eq_opt = .TRUE.' + WRITE(11,'(A)') ' IF(PRESENT(chck_eq)) chck_eq_opt = chck_eq' + WRITE(11,'(A)') ' IF(chck_eq_opt .EQV. .TRUE.)THEN' + WRITE(11,'(A)') ' IF (value .NEQV. correct_value) THEN' + WRITE(11,'(A)') ' total_error = total_error + 1' + WRITE(11,'(A)') ' WRITE(*,*) "ERROR: INCORRECT VALIDATION ", string' + WRITE(11,'(A)') ' ENDIF' + WRITE(11,'(A)') ' ELSE' + WRITE(11,'(A)') ' IF (value .EQV. correct_value) THEN' + WRITE(11,'(A)') ' total_error = total_error + 1' + WRITE(11,'(A)') ' WRITE(*,*) "ERROR: INCORRECT VALIDATION ", string' + WRITE(11,'(A)') ' ENDIF' WRITE(11,'(A)') ' ENDIF' - WRITE(11,'(A)') ' END SUBROUTINE verify_logical' WRITE(11,'(A)') "END MODULE TH5_MISC_gen" diff --git a/fortran/test/fortranlib_test.F90 b/fortran/test/fortranlib_test.F90 index d7fca7d..ec0dcec 100644 --- a/fortran/test/fortranlib_test.F90 +++ b/fortran/test/fortranlib_test.F90 @@ -193,7 +193,7 @@ PROGRAM fortranlibtest ret_total_error = 0 CALL external_test(cleanup, ret_total_error) - CALL write_test_status(ret_total_error, ' External dataset test', total_error) + CALL write_test_status(ret_total_error, ' External dataset and Selection IO test', total_error) ret_total_error = 0 CALL multi_file_test(cleanup, ret_total_error) @@ -207,6 +207,10 @@ PROGRAM fortranlibtest CALL test_misc_properties(cleanup, ret_total_error) CALL write_test_status(ret_total_error, ' Miscellaneous properties', total_error) + ret_total_error = 0 + CALL test_in_place_conversion(cleanup, ret_total_error) + CALL write_test_status(ret_total_error, ' Test in-place conversion', total_error) + ! ! '=========================================' ! 'Testing ATTRIBUTE interface ' diff --git a/fortran/test/tH5A_1_8.F90 b/fortran/test/tH5A_1_8.F90 index d43279e..03e26ec 100644 --- a/fortran/test/tH5A_1_8.F90 +++ b/fortran/test/tH5A_1_8.F90 @@ -234,7 +234,7 @@ SUBROUTINE test_attr_corder_create_compact(fcpl,fapl, total_error) INTEGER(HID_T) :: attr !String Attribute identifier INTEGER(HSIZE_T), DIMENSION(7) :: data_dims - LOGICAL :: f_corder_valid ! Indicates whether the the creation order data is valid for this attribute + LOGICAL :: f_corder_valid ! Indicates whether the creation order data is valid for this attribute INTEGER :: corder ! Is a positive integer containing the creation order of the attribute INTEGER :: cset ! Indicates the character set used for the attribute’s name INTEGER(HSIZE_T) :: data_size ! indicates the size, in the number of characters @@ -411,7 +411,7 @@ SUBROUTINE test_attr_null_space(fcpl, fapl, total_error) INTEGER(HSIZE_T) :: storage_size ! attributes storage requirements - LOGICAL :: f_corder_valid ! Indicates whether the the creation order data is valid for this attribute + LOGICAL :: f_corder_valid ! Indicates whether the creation order data is valid for this attribute INTEGER :: corder ! Is a positive integer containing the creation order of the attribute INTEGER :: cset ! Indicates the character set used for the attribute’s name INTEGER(HSIZE_T) :: data_size ! indicates the size, in the number of characters @@ -753,7 +753,7 @@ SUBROUTINE test_attr_info_by_idx(new_format, fcpl, fapl, total_error) INTEGER(HID_T) :: attr !String Attribute identifier INTEGER(HSIZE_T), DIMENSION(7) :: data_dims - LOGICAL :: f_corder_valid ! Indicates whether the the creation order data is valid for this attribute + LOGICAL :: f_corder_valid ! Indicates whether the creation order data is valid for this attribute INTEGER :: corder ! Is a positive integer containing the creation order of the attribute INTEGER :: cset ! Indicates the character set used for the attribute’s name INTEGER(HSIZE_T) :: data_size ! indicates the size, in the number of characters @@ -934,7 +934,7 @@ SUBROUTINE attr_info_by_idx_check(obj_id, attrname, n, use_index, total_error ) CHARACTER(LEN=*) :: attrname INTEGER(HSIZE_T) :: n LOGICAL :: use_index - LOGICAL :: f_corder_valid ! Indicates whether the the creation order data is valid for this attribute + LOGICAL :: f_corder_valid ! Indicates whether the creation order data is valid for this attribute INTEGER :: corder ! Is a positive integer containing the creation order of the attribute INTEGER :: cset ! Indicates the character set used for the attribute’s name INTEGER(HSIZE_T) :: data_size ! indicates the size, in the number of characters @@ -1397,7 +1397,7 @@ SUBROUTINE test_attr_delete_by_idx(new_format, fcpl, fapl, total_error) INTEGER(HID_T) :: attr !String Attribute identifier INTEGER(HSIZE_T), DIMENSION(7) :: data_dims - LOGICAL :: f_corder_valid ! Indicates whether the the creation order data is valid for this attribute + LOGICAL :: f_corder_valid ! Indicates whether the creation order data is valid for this attribute INTEGER :: corder ! Is a positive integer containing the creation order of the attribute INTEGER :: cset ! Indicates the character set used for the attribute’s name INTEGER(HSIZE_T) :: data_size ! indicates the size, in the number of characters @@ -2690,7 +2690,7 @@ SUBROUTINE attr_open_check(fid, dsetname, obj_id, max_attrs, total_error ) INTEGER :: u CHARACTER (LEN=8) :: attrname INTEGER :: error - LOGICAL :: f_corder_valid ! Indicates whether the the creation order data is valid for this attribute + LOGICAL :: f_corder_valid ! Indicates whether the creation order data is valid for this attribute INTEGER :: corder ! Is a positive integer containing the creation order of the attribute INTEGER :: cset ! Indicates the character set used for the attribute’s name INTEGER(HSIZE_T) :: data_size ! indicates the size, in the number of characters diff --git a/fortran/test/tH5P.F90 b/fortran/test/tH5P.F90 index 3db5b28..37ecdac 100644 --- a/fortran/test/tH5P.F90 +++ b/fortran/test/tH5P.F90 @@ -34,8 +34,8 @@ SUBROUTINE external_test(cleanup, total_error) ! This subroutine tests following functionalities: ! h5pset_external_f, h5pget_external_count_f, -! h5pget_external_f - +! h5pget_external_f, h5pget_selection_io_f +! h5pSet_selection_io_f IMPLICIT NONE LOGICAL, INTENT(IN) :: cleanup @@ -62,6 +62,7 @@ SUBROUTINE external_test(cleanup, total_error) INTEGER(SIZE_T) :: namesize INTEGER(HSIZE_T) :: size, buf_size INTEGER :: idx + INTEGER :: selection_io_mode buf_size = 4*1024*1024 @@ -77,6 +78,44 @@ SUBROUTINE external_test(cleanup, total_error) CALL h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) CALL check("h5pcreate_f", error, total_error) + + ! Check default Selection IO state + CALL h5pget_selection_io_f(plist_id, selection_io_mode, error) + CALL check("h5pget_selection_io_f", error, total_error) + CALL VERIFY("h5pget_selection_io_f", selection_io_mode, H5D_SELECTION_IO_MODE_DEFAULT_F, total_error) + + ! Turn off Section IO + CALL h5pset_selection_io_f(plist_id, H5D_SELECTION_IO_MODE_OFF_F, error) + CALL check("h5pset_selection_io_f", error, total_error) + + CALL h5pget_selection_io_f(plist_id, selection_io_mode, error) + CALL check("h5pget_selection_io_f", error, total_error) + CALL VERIFY("h5pget_selection_io_f", selection_io_mode, H5D_SELECTION_IO_MODE_OFF_F, total_error) + + ! Turn on Section IO + CALL h5pset_selection_io_f(plist_id, H5D_SELECTION_IO_MODE_ON_F, error) + CALL check("h5pset_selection_io_f", error, total_error) + + CALL h5pget_selection_io_f(plist_id, selection_io_mode, error) + CALL check("h5pget_selection_io_f", error, total_error) + CALL VERIFY("h5pget_selection_io_f", selection_io_mode, H5D_SELECTION_IO_MODE_ON_F, total_error) + + ! Turn off Section IO + CALL h5pset_selection_io_f(plist_id, H5D_SELECTION_IO_MODE_OFF_F, error) + CALL check("h5pset_selection_io_f", error, total_error) + + CALL h5pget_selection_io_f(plist_id, selection_io_mode, error) + CALL check("h5pget_selection_io_f", error, total_error) + CALL VERIFY("h5pget_selection_io_f", selection_io_mode, H5D_SELECTION_IO_MODE_OFF_F, total_error) + + ! Change back to the default + CALL h5pset_selection_io_f(plist_id, H5D_SELECTION_IO_MODE_DEFAULT_F, error) + CALL check("h5pset_selection_io_f", error, total_error) + + CALL h5pget_selection_io_f(plist_id, selection_io_mode, error) + CALL check("h5pget_selection_io_f", error, total_error) + CALL VERIFY("h5pget_selection_io_f", selection_io_mode, H5D_SELECTION_IO_MODE_DEFAULT_F, total_error) + CALL h5pset_buffer_f(plist_id, buf_size, error) CALL check("h5pset_buffer_f", error, total_error) CALL h5pget_buffer_f(plist_id, size, error) @@ -796,4 +835,125 @@ SUBROUTINE test_misc_properties(cleanup, total_error) END SUBROUTINE test_misc_properties +!------------------------------------------------------------------------- +! Function: test_in_place_conversion +! +! Purpose: single dataset reader/write, smaller mem type, no background buffer +! -- create dataset with H5T_NATIVE_DOUBLE +! -- write dataset with H5T_NATIVE_REAL +! -- read dataset with H5T_NATIVE_REAL +! +! Tests APIs: +! h5pset_modify_write_buf_f, h5pget_modify_write_buf_f +! +! Return: Success: 0 +! Failure: >0 +! +!------------------------------------------------------------------------- +! +SUBROUTINE test_in_place_conversion(cleanup, total_error) + + IMPLICIT NONE + LOGICAL, INTENT(IN) :: cleanup + INTEGER, INTENT(INOUT) :: total_error + + CHARACTER(LEN=12), PARAMETER :: filename = "inplace_conv" + CHARACTER(LEN=80) :: fix_filename + CHARACTER(LEN=4), PARAMETER :: dsetname = "dset" + + INTEGER(HID_T) :: file_id ! File identifier + INTEGER(HID_T) :: dset_id ! Dataset identifier + INTEGER(HID_T) :: dspace_id ! Dataspace identifier + INTEGER(HID_T) :: plist_id + LOGICAL :: modify_write_buf + INTEGER :: error !error code + + INTEGER, PARAMETER :: array_len = 10 + INTEGER(HSIZE_T), DIMENSION(1) :: dims = (/array_len/) ! Dataset dimensions + INTEGER :: rank = 1 ! Dataset rank + + REAL(KIND=Fortran_DOUBLE), DIMENSION(1:array_len), TARGET :: wbuf_d + REAL(KIND=Fortran_DOUBLE), DIMENSION(1:array_len) :: wbuf_d_org + REAL(KIND=Fortran_REAL) , DIMENSION(1:array_len), TARGET :: rbuf + INTEGER :: i + TYPE(C_PTR) :: f_ptr + + ! create the data + DO i = 1, array_len + wbuf_d(i) = 1_Fortran_DOUBLE + 0.123456789123456_Fortran_DOUBLE + wbuf_d_org(i) = wbuf_d(i) + ENDDO + + ! + !Create file "inplace_conv.h5" using default properties. + ! + CALL h5_fixname_f(filename, fix_filename, H5P_DEFAULT_F, error) + IF (error .NE. 0) STOP "Cannot modify filename" + + CALL h5fcreate_f(fix_filename, H5F_ACC_TRUNC_F, file_id, error) + CALL check("h5fcreate_f",error,total_error) + ! + ! Create the dataspace. + ! + CALL h5screate_simple_f(rank, dims, dspace_id, error) + CALL check("h5screate_simple_f", error, total_error) + + ! Create dataset transfer property list + CALL h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + CALL check("h5pcreate_f", error, total_error) + + CALL h5pset_selection_io_f(plist_id, H5D_SELECTION_IO_MODE_ON_F, error) + CALL check("h5pset_selection_io_f", error, total_error) + + CALL h5pget_modify_write_buf_f(plist_id, modify_write_buf, error) + CALL check("h5pget_modify_write_buf_f", error, total_error) + CALL VERIFY("h5pget_modify_write_buf_f", modify_write_buf, .FALSE., total_error) + + ! Set to modify the write buffer + CALL h5pset_modify_write_buf_f(plist_id, .TRUE., error) + CALL check("h5pset_modify_write_buf_f", error, total_error) + + CALL h5pget_modify_write_buf_f(plist_id, modify_write_buf, error) + CALL check("h5pget_modify_write_buf_f", error, total_error) + CALL VERIFY("h5pget_modify_write_buf_f", modify_write_buf, .TRUE., total_error) + + CALL h5dcreate_f(file_id, dsetname, H5T_NATIVE_REAL, dspace_id, dset_id, error) + CALL check("h5dcreate_f", error, total_error) + + f_ptr = C_LOC(wbuf_d) + CALL h5dwrite_f(dset_id, H5T_NATIVE_DOUBLE, f_ptr, error, H5S_ALL_F, H5S_ALL_F, xfer_prp=plist_id) + CALL check("h5dwrite_f", error, total_error) + + ! Should not be equal for in-place buffer use + CALL VERIFY("h5dwrite_f -- in-place", wbuf_d(1), wbuf_d_org(1), total_error, .FALSE.) + + f_ptr = C_LOC(rbuf) + CALL h5dread_f(dset_id, H5T_NATIVE_REAL, f_ptr, error) + CALL check("h5dread_f", error, total_error) + + DO i = 1, array_len + CALL VERIFY("h5dwrite_f -- in-place", rbuf(i), REAL(wbuf_d_org(i), Fortran_REAL), total_error) + ENDDO + + ! + ! End access to the dataset and release resources used by it. + ! + CALL h5dclose_f(dset_id, error) + CALL check("h5dclose_f", error, total_error) + ! + ! Terminate access to the data space. + ! + CALL h5sclose_f(dspace_id, error) + CALL check("h5sclose_f", error, total_error) + + ! + ! Close the file. + ! + CALL h5fclose_f(file_id, error) + CALL check("h5fclose_f", error, total_error) + CALL h5pclose_f(plist_id, error) + CALL check("h5pclose_f", error, total_error) + +END SUBROUTINE test_in_place_conversion + END MODULE TH5P diff --git a/hl/test/test_ds.c b/hl/test/test_ds.c index b74b62a..70af31a 100644 --- a/hl/test/test_ds.c +++ b/hl/test/test_ds.c @@ -3673,7 +3673,7 @@ out: * Function: match_dim_scale * * Purpose: example operator function used by H5DSiterate_scales, used - * to verify the the DSID scale size matches the dataset DIM size + * to verify the DSID scale size matches the dataset DIM size * * Return: * The return values from an operator are: diff --git a/hl/test/test_dset_append.c b/hl/test/test_dset_append.c index f8d38c2..c4afe89 100644 --- a/hl/test/test_dset_append.c +++ b/hl/test/test_dset_append.c @@ -1265,7 +1265,7 @@ main(void) /* * The following tests illustrate the scenarios when H5DOappend does not work with extensible array * indexing: - * - when the the dataset has 1 unlimited dimension and the other dimension is fixed but extendible + * - when the dataset has 1 unlimited dimension and the other dimension is fixed but extendible * - the dataset expands along 1 dimension and then expands along the other dimension */ flush_ct = 0; /* Reset flush counter */ diff --git a/release_docs/HISTORY-1_0-1_8_0.txt b/release_docs/HISTORY-1_0-1_8_0.txt index 3669f4d..333d0dc 100644 --- a/release_docs/HISTORY-1_0-1_8_0.txt +++ b/release_docs/HISTORY-1_0-1_8_0.txt @@ -1407,7 +1407,7 @@ Known Problems for your installation. Observe that the basic idea is to insert the script as the first item - on the command line which executes the the test. The script then + on the command line which executes the test. The script then executes the test and filters out the offending text before passing it on. diff --git a/release_docs/HISTORY-1_8_0-1_10_0.txt b/release_docs/HISTORY-1_8_0-1_10_0.txt index 7b84fbc..cc42d3b 100644 --- a/release_docs/HISTORY-1_8_0-1_10_0.txt +++ b/release_docs/HISTORY-1_8_0-1_10_0.txt @@ -791,7 +791,7 @@ Bug Fixes since HDF5-1.8.0 release and the external filters are being built, the CMAKE_BUILD_TYPE define must be set to the same value as the configuration (-DCMAKE_BUILD_TYPE:STRING=Release if using -C Release). This is needed - by the the szip and zlib filter build commands. (ADB - HDFFV-8695) + by the szip and zlib filter build commands. (ADB - HDFFV-8695) - CMake: Remove use of XLATE_UTILITY program. (ADB - 2014/03/28 HDFFV-8640) - CMake: Added missing quotes in setting the CMAKE_EXE_LINKER_FLAGS for the MPI option. (ADB - 2014/02/27 HDFFV-8674) @@ -1698,7 +1698,7 @@ Known Problems for your installation. Observe that the basic idea is to insert the script as the first item - on the command line which executes the the test. The script then + on the command line which executes the test. The script then executes the test and filters out the offending text before passing it on. diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 8220786..01e1235 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -105,6 +105,16 @@ New Features machine's node-local storage while placing the subfiling configuration file on a file system readable by all machine nodes. + - Added H5Pset_selection_io(), H5Pget_selection_io(), and + H5Pget_no_selection_io_cause() API functions to manage the selection I/O + feature. This can be used to enable collective I/O with type conversion, + or it can be used with custom VFDs that support vector or selection I/O. + + - Added H5Pset_modify_write_buf() and H5Pget_modify_write_buf() API + functions to allow the library to modify the contents of write buffers, in + order to avoid malloc/memcpy. Currently only used for type conversion + with selection I/O. + Parallel Library: ----------------- @@ -113,9 +123,11 @@ New Features Fortran Library: ---------------- - - Added Fortran async APIs + - Fortran async APIs H5A, H5D, H5ES, H5G, H5F, H5L and H5O were added. - H5A, H5D, H5ES, H5G, H5F, H5L and H5O async APIs were added. + - Added Fortran APIs: + h5pset_selection_io_f, h5pget_selection_io_f + h5pset_modify_write_buf_f, h5pget_modify_write_buf_f C++ Library: ------------ @@ -82,8 +82,6 @@ hbool_t H5_libinit_g = FALSE; /* Library hasn't been initialized */ hbool_t H5_libterm_g = FALSE; /* Library isn't being shutdown */ #endif -hbool_t H5_use_selection_io_g = FALSE; - char H5_lib_vers_info_g[] = H5_VERS_INFO; static hbool_t H5_dont_atexit_g = FALSE; H5_debug_t H5_debug_g; /* debugging info */ @@ -141,8 +139,7 @@ herr_t H5_init_library(void) { size_t i; - char *env_use_select_io = NULL; - herr_t ret_value = SUCCEED; + herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI(FAIL) @@ -274,14 +271,6 @@ H5_init_library(void) } /* clang-format on */ - /* Check for HDF5_USE_SELECTION_IO env variable */ - env_use_select_io = HDgetenv("HDF5_USE_SELECTION_IO"); - if (NULL != env_use_select_io && HDstrcmp(env_use_select_io, "") && HDstrcmp(env_use_select_io, "0") && - HDstrcmp(env_use_select_io, "no") && HDstrcmp(env_use_select_io, "No") && - HDstrcmp(env_use_select_io, "NO") && HDstrcmp(env_use_select_io, "false") && - HDstrcmp(env_use_select_io, "False") && HDstrcmp(env_use_select_io, "FALSE")) - H5_use_selection_io_g = TRUE; - /* Debugging? */ H5__debug_mask("-all"); H5__debug_mask(HDgetenv("HDF5_DEBUG")); @@ -94,14 +94,12 @@ H5CX_RETRIEVE_PROP_COMMON(PL, DEF_PL, PROP_NAME, PROP_FIELD) \ } /* end if */ -#ifdef H5_HAVE_PARALLEL /* Macro for the duplicated code to retrieve possibly set properties from a property list */ #define H5CX_RETRIEVE_PROP_VALID_SET(PL, DEF_PL, PROP_NAME, PROP_FIELD) \ /* Check if the value has been retrieved already */ \ if (!((*head)->ctx.H5_GLUE(PROP_FIELD, _valid) || (*head)->ctx.H5_GLUE(PROP_FIELD, _set))) { \ H5CX_RETRIEVE_PROP_COMMON(PL, DEF_PL, PROP_NAME, PROP_FIELD) \ - } /* end if */ -#endif /* H5_HAVE_PARALLEL */ + } /* end if */ #if defined(H5_HAVE_PARALLEL) && defined(H5_HAVE_INSTRUMENTED_LIBRARY) /* Macro for the duplicated code to test and set properties for a property list */ @@ -127,7 +125,6 @@ } #endif /* defined(H5_HAVE_PARALLEL) && defined(H5_HAVE_INSTRUMENTED_LIBRARY) */ -#ifdef H5_HAVE_PARALLEL /* Macro for the duplicated code to test and set properties for a property list */ #define H5CX_SET_PROP(PROP_NAME, PROP_FIELD) \ if ((*head)->ctx.H5_GLUE(PROP_FIELD, _set)) { \ @@ -137,8 +134,7 @@ /* Set the property */ \ if (H5P_set((*head)->ctx.dxpl, PROP_NAME, &(*head)->ctx.PROP_FIELD) < 0) \ HGOTO_ERROR(H5E_CONTEXT, H5E_CANTSET, NULL, "error setting data xfer property") \ - } /* end if */ -#endif /* H5_HAVE_PARALLEL */ + } /* end if */ /******************/ /* Local Typedefs */ @@ -242,16 +238,20 @@ typedef struct H5CX_t { unsigned mpio_chunk_opt_ratio; /* Collective chunk ratio (H5D_XFER_MPIO_CHUNK_OPT_RATIO_NAME) */ hbool_t mpio_chunk_opt_ratio_valid; /* Whether collective chunk ratio is valid */ #endif /* H5_HAVE_PARALLEL */ - H5Z_EDC_t err_detect; /* Error detection info (H5D_XFER_EDC_NAME) */ - hbool_t err_detect_valid; /* Whether error detection info is valid */ - H5Z_cb_t filter_cb; /* Filter callback function (H5D_XFER_FILTER_CB_NAME) */ - hbool_t filter_cb_valid; /* Whether filter callback function is valid */ - H5Z_data_xform_t *data_transform; /* Data transform info (H5D_XFER_XFORM_NAME) */ - hbool_t data_transform_valid; /* Whether data transform info is valid */ - H5T_vlen_alloc_info_t vl_alloc_info; /* VL datatype alloc info (H5D_XFER_VLEN_*_NAME) */ - hbool_t vl_alloc_info_valid; /* Whether VL datatype alloc info is valid */ - H5T_conv_cb_t dt_conv_cb; /* Datatype conversion struct (H5D_XFER_CONV_CB_NAME) */ - hbool_t dt_conv_cb_valid; /* Whether datatype conversion struct is valid */ + H5Z_EDC_t err_detect; /* Error detection info (H5D_XFER_EDC_NAME) */ + hbool_t err_detect_valid; /* Whether error detection info is valid */ + H5Z_cb_t filter_cb; /* Filter callback function (H5D_XFER_FILTER_CB_NAME) */ + hbool_t filter_cb_valid; /* Whether filter callback function is valid */ + H5Z_data_xform_t *data_transform; /* Data transform info (H5D_XFER_XFORM_NAME) */ + hbool_t data_transform_valid; /* Whether data transform info is valid */ + H5T_vlen_alloc_info_t vl_alloc_info; /* VL datatype alloc info (H5D_XFER_VLEN_*_NAME) */ + hbool_t vl_alloc_info_valid; /* Whether VL datatype alloc info is valid */ + H5T_conv_cb_t dt_conv_cb; /* Datatype conversion struct (H5D_XFER_CONV_CB_NAME) */ + hbool_t dt_conv_cb_valid; /* Whether datatype conversion struct is valid */ + H5D_selection_io_mode_t selection_io_mode; /* Selection I/O mode (H5D_XFER_SELECTION_IO_MODE_NAME) */ + hbool_t selection_io_mode_valid; /* Whether selection I/O mode is valid */ + hbool_t modify_write_buf; /* Whether the library can modify write buffers */ + hbool_t modify_write_buf_valid; /* Whether the modify_write_buf field is valid */ /* Return-only DXPL properties to return to application */ #ifdef H5_HAVE_PARALLEL @@ -297,9 +297,13 @@ typedef struct H5CX_t { hbool_t mpio_coll_rank0_bcast; /* Instrumented "collective chunk multi ratio ind" value (H5D_XFER_COLL_CHUNK_MULTI_RATIO_IND_NAME) */ hbool_t - mpio_coll_rank0_bcast_set; /* Whether instrumented "collective chunk multi ratio ind" value is set */ -#endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ -#endif /* H5_HAVE_PARALLEL */ + mpio_coll_rank0_bcast_set; /* Whether instrumented "collective chunk multi ratio ind" value is set */ +#endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ +#endif /* H5_HAVE_PARALLEL */ + uint32_t no_selection_io_cause; /* Reason for not performing selection I/O + (H5D_XFER_NO_SELECTION_IO_CAUSE_NAME) */ + hbool_t no_selection_io_cause_set; /* Whether reason for not performing selection I/O is set */ + hbool_t no_selection_io_cause_valid; /* Whether reason for not performing selection I/O is valid */ /* Cached LCPL properties */ H5T_cset_t encoding; /* Link name character encoding */ @@ -370,15 +374,19 @@ typedef struct H5CX_dxpl_cache_t { uint32_t mpio_global_no_coll_cause; /* Global reason for breaking collective I/O (H5D_MPIO_GLOBAL_NO_COLLECTIVE_CAUSE_NAME) */ H5FD_mpio_chunk_opt_t - mpio_chunk_opt_mode; /* Collective chunk option (H5D_XFER_MPIO_CHUNK_OPT_HARD_NAME) */ - unsigned mpio_chunk_opt_num; /* Collective chunk threshold (H5D_XFER_MPIO_CHUNK_OPT_NUM_NAME) */ - unsigned mpio_chunk_opt_ratio; /* Collective chunk ratio (H5D_XFER_MPIO_CHUNK_OPT_RATIO_NAME) */ -#endif /* H5_HAVE_PARALLEL */ - H5Z_EDC_t err_detect; /* Error detection info (H5D_XFER_EDC_NAME) */ - H5Z_cb_t filter_cb; /* Filter callback function (H5D_XFER_FILTER_CB_NAME) */ - H5Z_data_xform_t *data_transform; /* Data transform info (H5D_XFER_XFORM_NAME) */ - H5T_vlen_alloc_info_t vl_alloc_info; /* VL datatype alloc info (H5D_XFER_VLEN_*_NAME) */ - H5T_conv_cb_t dt_conv_cb; /* Datatype conversion struct (H5D_XFER_CONV_CB_NAME) */ + mpio_chunk_opt_mode; /* Collective chunk option (H5D_XFER_MPIO_CHUNK_OPT_HARD_NAME) */ + unsigned mpio_chunk_opt_num; /* Collective chunk threshold (H5D_XFER_MPIO_CHUNK_OPT_NUM_NAME) */ + unsigned mpio_chunk_opt_ratio; /* Collective chunk ratio (H5D_XFER_MPIO_CHUNK_OPT_RATIO_NAME) */ +#endif /* H5_HAVE_PARALLEL */ + H5Z_EDC_t err_detect; /* Error detection info (H5D_XFER_EDC_NAME) */ + H5Z_cb_t filter_cb; /* Filter callback function (H5D_XFER_FILTER_CB_NAME) */ + H5Z_data_xform_t *data_transform; /* Data transform info (H5D_XFER_XFORM_NAME) */ + H5T_vlen_alloc_info_t vl_alloc_info; /* VL datatype alloc info (H5D_XFER_VLEN_*_NAME) */ + H5T_conv_cb_t dt_conv_cb; /* Datatype conversion struct (H5D_XFER_CONV_CB_NAME) */ + H5D_selection_io_mode_t selection_io_mode; /* Selection I/O mode (H5D_XFER_SELECTION_IO_MODE_NAME) */ + uint32_t no_selection_io_cause; /* Reasons for not performing selection I/O + (H5D_XFER_NO_SELECTION_IO_CAUSE_NAME) */ + hbool_t modify_write_buf; /* Whether the library can modify write buffers */ } H5CX_dxpl_cache_t; /* Typedef for cached default link creation property list information */ @@ -566,6 +574,19 @@ H5CX_init(void) if (H5P_get(dx_plist, H5D_XFER_CONV_CB_NAME, &H5CX_def_dxpl_cache.dt_conv_cb) < 0) HGOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "Can't retrieve datatype conversion exception callback") + /* Get the selection I/O mode */ + if (H5P_get(dx_plist, H5D_XFER_SELECTION_IO_MODE_NAME, &H5CX_def_dxpl_cache.selection_io_mode) < 0) + HGOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "Can't retrieve parallel transfer method") + + /* Get the local & global reasons for breaking selection I/O values */ + if (H5P_get(dx_plist, H5D_XFER_NO_SELECTION_IO_CAUSE_NAME, &H5CX_def_dxpl_cache.no_selection_io_cause) < + 0) + HGOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "Can't retrieve cause for no selection I/O") + + /* Get the modify write buffer property */ + if (H5P_get(dx_plist, H5D_XFER_MODIFY_WRITE_BUF_NAME, &H5CX_def_dxpl_cache.modify_write_buf) < 0) + HGOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "Can't retrieve modify write buffer property") + /* Reset the "default LCPL cache" information */ HDmemset(&H5CX_def_lcpl_cache, 0, sizeof(H5CX_lcpl_cache_t)); @@ -2564,6 +2585,111 @@ done: } /* end H5CX_get_dt_conv_cb() */ /*------------------------------------------------------------------------- + * Function: H5CX_get_selection_io_mode + * + * Purpose: Retrieves the selection I/O mode for the current API call context. + * + * Return: Non-negative on success / Negative on failure + * + * Programmer: Vailin Choi + * March 5, 2023 + * + *------------------------------------------------------------------------- + */ +herr_t +H5CX_get_selection_io_mode(H5D_selection_io_mode_t *selection_io_mode) +{ + H5CX_node_t **head = NULL; /* Pointer to head of API context list */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity check */ + HDassert(selection_io_mode); + head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ + HDassert(head && *head); + HDassert(H5P_DEFAULT != (*head)->ctx.dxpl_id); + + H5CX_RETRIEVE_PROP_VALID(dxpl, H5P_DATASET_XFER_DEFAULT, H5D_XFER_SELECTION_IO_MODE_NAME, + selection_io_mode) + + /* Get the value */ + *selection_io_mode = (*head)->ctx.selection_io_mode; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5CX_get_selection_io_mode() */ + +/*------------------------------------------------------------------------- + * Function: H5CX_get_no_selection_io_cause + * + * Purpose: Retrieves the cause for not performing selection I/O + * for the current API call context. + * + * Return: Non-negative on success / Negative on failure + * + * Programmer: Vailin Choi + * April 15, 2023 + * + *------------------------------------------------------------------------- + */ +herr_t +H5CX_get_no_selection_io_cause(uint32_t *no_selection_io_cause) +{ + H5CX_node_t **head = NULL; /* Pointer to head of API context list */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity check */ + HDassert(no_selection_io_cause); + head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ + HDassert(head && *head); + HDassert(H5P_DEFAULT != (*head)->ctx.dxpl_id); + + H5CX_RETRIEVE_PROP_VALID_SET(dxpl, H5P_DATASET_XFER_DEFAULT, H5D_XFER_NO_SELECTION_IO_CAUSE_NAME, + no_selection_io_cause) + + /* Get the value */ + *no_selection_io_cause = (*head)->ctx.no_selection_io_cause; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5CX_get_no_selection_io_cause() */ + +/*------------------------------------------------------------------------- + * Function: H5CX_get_modify_write_buf + * + * Purpose: Retrieves the modify write buffer property for the current API call context. + * + * Return: Non-negative on success / Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5CX_get_modify_write_buf(hbool_t *modify_write_buf) +{ + H5CX_node_t **head = NULL; /* Pointer to head of API context list */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity check */ + HDassert(modify_write_buf); + head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ + HDassert(head && *head); + HDassert(H5P_DEFAULT != (*head)->ctx.dxpl_id); + + H5CX_RETRIEVE_PROP_VALID(dxpl, H5P_DATASET_XFER_DEFAULT, H5D_XFER_MODIFY_WRITE_BUF_NAME, modify_write_buf) + + /* Get the value */ + *modify_write_buf = (*head)->ctx.modify_write_buf; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5CX_get_selection_io_mode() */ + +/*------------------------------------------------------------------------- * Function: H5CX_get_encoding * * Purpose: Retrieves the character encoding for the current API call context. @@ -3544,6 +3670,41 @@ done: #endif /* H5_HAVE_PARALLEL */ /*------------------------------------------------------------------------- + * Function: H5CX_set_no_selecction_io_cause + * + * Purpose: Sets the reason for not performing selection I/O for + * the current API call context. + * + * Return: <none> + * + * Programmer: Vailin Choi + * April 15, 2023 + * + *------------------------------------------------------------------------- + */ +void +H5CX_set_no_selection_io_cause(uint32_t no_selection_io_cause) +{ + H5CX_node_t **head = NULL; /* Pointer to head of API context list */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Sanity checks */ + head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ + HDassert(head && *head); + HDassert((*head)->ctx.dxpl_id != H5P_DEFAULT); + + /* If we're using the default DXPL, don't modify it */ + if ((*head)->ctx.dxpl_id != H5P_DATASET_XFER_DEFAULT) { + /* Cache the value for later, marking it to set in DXPL when context popped */ + (*head)->ctx.no_selection_io_cause = no_selection_io_cause; + (*head)->ctx.no_selection_io_cause_set = TRUE; + } /* end if */ + + FUNC_LEAVE_NOAPI_VOID +} /* end H5CX_set_no_selectiion_io_cause() */ + +/*------------------------------------------------------------------------- * Function: H5CX_get_ohdr_flags * * Purpose: Retrieves the object header flags for the current API call context. @@ -3596,11 +3757,7 @@ H5CX__pop_common(hbool_t update_dxpl_props) H5CX_node_t **head = NULL; /* Pointer to head of API context list */ H5CX_node_t *ret_value = NULL; /* Return value */ -#ifdef H5_HAVE_PARALLEL FUNC_ENTER_PACKAGE -#else - FUNC_ENTER_PACKAGE_NOERR -#endif /* Sanity check */ head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ @@ -3608,6 +3765,7 @@ H5CX__pop_common(hbool_t update_dxpl_props) /* Check for cached DXPL properties to return to application */ if (update_dxpl_props) { + H5CX_SET_PROP(H5D_XFER_NO_SELECTION_IO_CAUSE_NAME, no_selection_io_cause) #ifdef H5_HAVE_PARALLEL H5CX_SET_PROP(H5D_MPIO_ACTUAL_CHUNK_OPT_MODE_NAME, mpio_actual_chunk_opt) H5CX_SET_PROP(H5D_MPIO_ACTUAL_IO_MODE_NAME, mpio_actual_io_mode) @@ -3629,9 +3787,7 @@ H5CX__pop_common(hbool_t update_dxpl_props) ret_value = (*head); (*head) = (*head)->next; -#ifdef H5_HAVE_PARALLEL done: -#endif /* H5_HAVE_PARALLEL */ FUNC_LEAVE_NOAPI(ret_value) } /* end H5CX__pop_common() */ diff --git a/src/H5CXprivate.h b/src/H5CXprivate.h index 4c034c1..f0bec20 100644 --- a/src/H5CXprivate.h +++ b/src/H5CXprivate.h @@ -115,6 +115,9 @@ H5_DLL herr_t H5CX_get_filter_cb(H5Z_cb_t *filter_cb); H5_DLL herr_t H5CX_get_data_transform(H5Z_data_xform_t **data_transform); H5_DLL herr_t H5CX_get_vlen_alloc_info(H5T_vlen_alloc_info_t *vl_alloc_info); H5_DLL herr_t H5CX_get_dt_conv_cb(H5T_conv_cb_t *cb_struct); +H5_DLL herr_t H5CX_get_selection_io_mode(H5D_selection_io_mode_t *selection_io_mode); +H5_DLL herr_t H5CX_get_no_selection_io_cause(uint32_t *no_selection_io_cause); +H5_DLL herr_t H5CX_get_modify_write_buf(hbool_t *modify_write_buf); /* "Getter" routines for LCPL properties cached in API context */ H5_DLL herr_t H5CX_get_encoding(H5T_cset_t *encoding); @@ -158,6 +161,9 @@ H5_DLL herr_t H5CX_set_nlinks(size_t nlinks); H5_DLL herr_t H5CX_init(void); /* "Setter" routines for cached DXPL properties that must be returned to application */ + +H5_DLL void H5CX_set_no_selection_io_cause(uint32_t no_selection_io_cause); + #ifdef H5_HAVE_PARALLEL H5_DLL void H5CX_set_mpio_actual_chunk_opt(H5D_mpio_actual_chunk_opt_mode_t chunk_opt); H5_DLL void H5CX_set_mpio_actual_io_mode(H5D_mpio_actual_io_mode_t actual_io_mode); diff --git a/src/H5Dbtree.c b/src/H5Dbtree.c index 2937b50..a9dfad6 100644 --- a/src/H5Dbtree.c +++ b/src/H5Dbtree.c @@ -10,13 +10,9 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -/* Programmer: Robb Matzke - * Wednesday, October 8, 1997 - * - * Purpose: v1 B-tree indexed (chunked) I/O functions. The chunks are +/* Purpose: v1 B-tree indexed (chunked) I/O functions. The chunks are * given a multi-dimensional index which is used as a lookup key * in a B-tree that maps chunk index to disk address. - * */ /****************/ @@ -627,15 +623,11 @@ done: } /* end H5D__btree_remove() */ /*------------------------------------------------------------------------- - * Function: H5D__btree_decode_key + * Function: H5D__btree_decode_key * - * Purpose: Decodes a raw key into a native key for the B-tree - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Robb Matzke - * Friday, October 10, 1997 + * Purpose: Decodes a raw key into a native key for the B-tree * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t @@ -644,33 +636,33 @@ H5D__btree_decode_key(const H5B_shared_t *shared, const uint8_t *raw, void *_key const H5O_layout_chunk_t *layout; /* Chunk layout description */ H5D_btree_key_t *key = (H5D_btree_key_t *)_key; /* Pointer to decoded key */ hsize_t tmp_offset; /* Temporary coordinate offset, from file */ - unsigned u; /* Local index variable */ - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE - /* check args */ HDassert(shared); HDassert(raw); HDassert(key); layout = (const H5O_layout_chunk_t *)shared->udata; HDassert(layout); - HDassert(layout->ndims > 0 && layout->ndims <= H5O_LAYOUT_NDIMS); - /* decode */ + if (layout->ndims > H5O_LAYOUT_NDIMS) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "bad number of dimensions") + UINT32DECODE(raw, key->nbytes); UINT32DECODE(raw, key->filter_mask); - for (u = 0; u < layout->ndims; u++) { + for (unsigned u = 0; u < layout->ndims; u++) { if (layout->dim[u] == 0) HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "chunk size must be > 0, dim = %u ", u) /* Retrieve coordinate offset */ UINT64DECODE(raw, tmp_offset); - HDassert(0 == (tmp_offset % layout->dim[u])); + if (0 != (tmp_offset % layout->dim[u])) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "bad coordinate offset") /* Convert to a scaled offset */ key->scaled[u] = tmp_offset / layout->dim[u]; - } /* end for */ + } done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index 830560d..0ab4da1 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -10,31 +10,28 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -/* Programmer: Quincey Koziol - * Thursday, April 24, 2008 - * - * Purpose: Abstract indexed (chunked) I/O functions. The logical - * multi-dimensional dataspace is regularly partitioned into - * same-sized "chunks", the first of which is aligned with the - * logical origin. The chunks are indexed by different methods, - * that map a chunk index to disk address. Each chunk can be - * compressed independently and the chunks may move around in the - * file as their storage requirements change. - * - * Cache: Disk I/O is performed in units of chunks and H5MF_alloc() - * contains code to optionally align chunks on disk block - * boundaries for performance. - * - * The chunk cache is an extendible hash indexed by a function - * of storage B-tree address and chunk N-dimensional offset - * within the dataset. Collisions are not resolved -- one of - * the two chunks competing for the hash slot must be preempted - * from the cache. All entries in the hash also participate in - * a doubly-linked list and entries are penalized by moving them - * toward the front of the list. When a new chunk is about to - * be added to the cache the heap is pruned by preempting - * entries near the front of the list to make room for the new - * entry which is added to the end of the list. +/* Purpose: Abstract indexed (chunked) I/O functions. The logical + * multi-dimensional dataspace is regularly partitioned into + * same-sized "chunks", the first of which is aligned with the + * logical origin. The chunks are indexed by different methods, + * that map a chunk index to disk address. Each chunk can be + * compressed independently and the chunks may move around in the + * file as their storage requirements change. + * + * Cache: Disk I/O is performed in units of chunks and H5MF_alloc() + * contains code to optionally align chunks on disk block + * boundaries for performance. + * + * The chunk cache is an extendible hash indexed by a function + * of storage B-tree address and chunk N-dimensional offset + * within the dataset. Collisions are not resolved -- one of + * the two chunks competing for the hash slot must be preempted + * from the cache. All entries in the hash also participate in + * a doubly-linked list and entries are penalized by moving them + * toward the front of the list. When a new chunk is about to + * be added to the cache the heap is pruned by preempting + * entries near the front of the list to make room for the new + * entry which is added to the end of the list. */ /****************/ @@ -289,21 +286,21 @@ static ssize_t H5D__nonexistent_readvv(const H5D_io_info_t *io_info, const H5D_d static int H5D__chunk_format_convert_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata); /* Helper routines */ -static herr_t H5D__chunk_set_info_real(H5O_layout_chunk_t *layout, unsigned ndims, const hsize_t *curr_dims, - const hsize_t *max_dims); -static herr_t H5D__chunk_cinfo_cache_reset(H5D_chunk_cached_t *last); -static herr_t H5D__chunk_cinfo_cache_update(H5D_chunk_cached_t *last, const H5D_chunk_ud_t *udata); -static hbool_t H5D__chunk_cinfo_cache_found(const H5D_chunk_cached_t *last, H5D_chunk_ud_t *udata); -static herr_t H5D__create_piece_map_single(H5D_dset_io_info_t *di, H5D_io_info_t *io_info); -static herr_t H5D__create_piece_file_map_all(H5D_dset_io_info_t *di, H5D_io_info_t *io_info); -static herr_t H5D__create_piece_file_map_hyper(H5D_dset_io_info_t *di, H5D_io_info_t *io_info); -static herr_t H5D__create_piece_mem_map_1d(const H5D_dset_io_info_t *di); -static herr_t H5D__create_piece_mem_map_hyper(const H5D_dset_io_info_t *di); -static herr_t H5D__piece_file_cb(void *elem, const H5T_t *type, unsigned ndims, const hsize_t *coords, +static herr_t H5D__chunk_set_info_real(H5O_layout_chunk_t *layout, unsigned ndims, const hsize_t *curr_dims, + const hsize_t *max_dims); +static herr_t H5D__chunk_cinfo_cache_reset(H5D_chunk_cached_t *last); +static herr_t H5D__chunk_cinfo_cache_update(H5D_chunk_cached_t *last, const H5D_chunk_ud_t *udata); +static hbool_t H5D__chunk_cinfo_cache_found(const H5D_chunk_cached_t *last, H5D_chunk_ud_t *udata); +static herr_t H5D__create_piece_map_single(H5D_dset_io_info_t *di, H5D_io_info_t *io_info); +static herr_t H5D__create_piece_file_map_all(H5D_dset_io_info_t *di, H5D_io_info_t *io_info); +static herr_t H5D__create_piece_file_map_hyper(H5D_dset_io_info_t *di, H5D_io_info_t *io_info); +static herr_t H5D__create_piece_mem_map_1d(const H5D_dset_io_info_t *di); +static herr_t H5D__create_piece_mem_map_hyper(const H5D_dset_io_info_t *di); +static herr_t H5D__piece_file_cb(void *elem, const H5T_t *type, unsigned ndims, const hsize_t *coords, + void *_opdata); +static herr_t H5D__piece_mem_cb(void *elem, const H5T_t *type, unsigned ndims, const hsize_t *coords, void *_opdata); -static herr_t H5D__piece_mem_cb(void *elem, const H5T_t *type, unsigned ndims, const hsize_t *coords, - void *_opdata); -static htri_t H5D__chunk_may_use_select_io(const H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset_info); +static herr_t H5D__chunk_may_use_select_io(H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset_info); static unsigned H5D__chunk_hash_val(const H5D_shared_t *shared, const hsize_t *scaled); static herr_t H5D__chunk_flush_entry(const H5D_t *dset, H5D_rdcc_ent_t *ent, hbool_t reset); static herr_t H5D__chunk_cache_evict(const H5D_t *dset, H5D_rdcc_ent_t *ent, hbool_t flush); @@ -670,31 +667,30 @@ done: /*------------------------------------------------------------------------- * Function: H5D__chunk_set_info_real * - * Purpose: Internal routine to set the information about chunks for a dataset - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * Tuesday, June 30, 2009 + * Purpose: Internal routine to set the information about chunks for a dataset * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t H5D__chunk_set_info_real(H5O_layout_chunk_t *layout, unsigned ndims, const hsize_t *curr_dims, const hsize_t *max_dims) { - unsigned u; /* Local index variable */ - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE - /* Sanity checks */ HDassert(layout); - HDassert(ndims > 0); HDassert(curr_dims); + /* Can happen when corrupt files are parsed */ + if (ndims == 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "number of dimensions cannot be zero") + /* Compute the # of chunks in dataset dimensions */ - for (u = 0, layout->nchunks = 1, layout->max_nchunks = 1; u < ndims; u++) { + layout->nchunks = 1; + layout->max_nchunks = 1; + for (unsigned u = 0; u < ndims; u++) { /* Round up to the next integer # of chunks, to accommodate partial chunks */ layout->chunks[u] = ((curr_dims[u] + layout->dim[u]) - 1) / layout->dim[u]; if (H5S_UNLIMITED == max_dims[u]) @@ -710,7 +706,7 @@ H5D__chunk_set_info_real(H5O_layout_chunk_t *layout, unsigned ndims, const hsize /* Accumulate the # of chunks */ layout->nchunks *= layout->chunks[u]; layout->max_nchunks *= layout->max_chunks[u]; - } /* end for */ + } /* Get the "down" sizes for each dimension */ H5VM_array_down(ndims, layout->chunks, layout->down_chunks); @@ -1062,9 +1058,8 @@ H5D__chunk_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo) htri_t file_space_normalized = FALSE; /* File dataspace was normalized */ unsigned f_ndims; /* The number of dimensions of the file's dataspace */ int sm_ndims; /* The number of dimensions of the memory buffer's dataspace (signed) */ - htri_t use_selection_io = FALSE; /* Whether to use selection I/O */ - unsigned u; /* Local index variable */ - herr_t ret_value = SUCCEED; /* Return value */ + unsigned u; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -1114,10 +1109,29 @@ H5D__chunk_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo) /* Check if we're performing selection I/O and save the result if it hasn't * been disabled already */ - if (io_info->use_select_io) { - if ((use_selection_io = H5D__chunk_may_use_select_io(io_info, dinfo)) < 0) + if (io_info->use_select_io != H5D_SELECTION_IO_MODE_OFF) + if (H5D__chunk_may_use_select_io(io_info, dinfo) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't check if selection I/O is possible") - io_info->use_select_io = (hbool_t)use_selection_io; + + /* Calculate type conversion buffer size if necessary. Currently only implemented for selection I/O. */ + if (io_info->use_select_io != H5D_SELECTION_IO_MODE_OFF && + !(dinfo->type_info.is_xform_noop && dinfo->type_info.is_conv_noop)) { + H5SL_node_t *chunk_node; /* Current node in chunk skip list */ + + /* Iterate through nodes in chunk skip list */ + chunk_node = H5D_CHUNK_GET_FIRST_NODE(dinfo); + while (chunk_node) { + H5D_piece_info_t *piece_info; /* Chunk information */ + + /* Get the actual chunk information from the skip list node */ + piece_info = H5D_CHUNK_GET_NODE_INFO(dinfo, chunk_node); + + /* Handle type conversion buffer */ + H5D_INIT_PIECE_TCONV(io_info, dinfo, piece_info) + + /* Advance to next chunk in list */ + chunk_node = H5D_CHUNK_GET_NEXT_NODE(dinfo, chunk_node); + } } done: @@ -1571,6 +1585,10 @@ H5D__create_piece_map_single(H5D_dset_io_info_t *di, H5D_io_info_t *io_info) /* Indicate that the chunk's memory dataspace is shared */ piece_info->mspace_shared = TRUE; + /* Initialize in-place type conversion info. Start with it disabled. */ + piece_info->in_place_tconv = FALSE; + piece_info->buf_off = 0; + /* make connection to related dset info from this piece_info */ piece_info->dset_info = di; @@ -1700,6 +1718,10 @@ H5D__create_piece_file_map_all(H5D_dset_io_info_t *di, H5D_io_info_t *io_info) /* make connection to related dset info from this piece_info */ new_piece_info->dset_info = di; + /* Initialize in-place type conversion info. Start with it disabled. */ + new_piece_info->in_place_tconv = FALSE; + new_piece_info->buf_off = 0; + /* Insert the new chunk into the skip list */ if (H5SL_insert(fm->dset_sel_pieces, new_piece_info, &new_piece_info->index) < 0) { H5D__free_piece_info(new_piece_info, NULL, NULL); @@ -1896,6 +1918,10 @@ H5D__create_piece_file_map_hyper(H5D_dset_io_info_t *dinfo, H5D_io_info_t *io_in /* make connection to related dset info from this piece_info */ new_piece_info->dset_info = dinfo; + /* Initialize in-place type conversion info. Start with it disabled. */ + new_piece_info->in_place_tconv = FALSE; + new_piece_info->buf_off = 0; + /* Add piece to global piece_count */ io_info->piece_count++; @@ -2271,6 +2297,10 @@ H5D__piece_file_cb(void H5_ATTR_UNUSED *elem, const H5T_t H5_ATTR_UNUSED *type, H5MM_memcpy(piece_info->scaled, scaled, sizeof(hsize_t) * fm->f_ndims); piece_info->scaled[fm->f_ndims] = 0; + /* Initialize in-place type conversion info. Start with it disabled. */ + piece_info->in_place_tconv = FALSE; + piece_info->buf_off = 0; + /* Make connection to related dset info from this piece_info */ piece_info->dset_info = dinfo; @@ -2547,11 +2577,11 @@ done: * *------------------------------------------------------------------------- */ -static htri_t -H5D__chunk_may_use_select_io(const H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset_info) +static herr_t +H5D__chunk_may_use_select_io(H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset_info) { - const H5D_t *dataset = NULL; /* Local pointer to dataset info */ - htri_t ret_value = FAIL; /* Return value */ + const H5D_t *dataset = NULL; /* Local pointer to dataset info */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -2562,20 +2592,22 @@ H5D__chunk_may_use_select_io(const H5D_io_info_t *io_info, const H5D_dset_io_inf dataset = dset_info->dset; HDassert(dataset); - /* Don't use selection I/O if it's globally disabled, there is a type - * conversion, or if there are filters on the dataset (for now) */ - if (dset_info->io_ops.single_read != H5D__select_read || dataset->shared->dcpl_cache.pline.nused > 0) - ret_value = FALSE; + /* Don't use selection I/O if there are filters on the dataset (for now) */ + if (dataset->shared->dcpl_cache.pline.nused > 0) { + io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; + io_info->no_selection_io_cause |= H5D_SEL_IO_DATASET_FILTER; + } else { hbool_t page_buf_enabled; - HDassert(dset_info->io_ops.single_write == H5D__select_write); - /* Check if the page buffer is enabled */ if (H5PB_enabled(io_info->f_sh, H5FD_MEM_DRAW, &page_buf_enabled) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't check if page buffer is enabled") - if (page_buf_enabled) - ret_value = FALSE; + if (page_buf_enabled) { + /* Note that page buffer is disabled in parallel */ + io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; + io_info->no_selection_io_cause |= H5D_SEL_IO_PAGE_BUFFER; + } else { /* Check if chunks in this dataset may be cached, if so don't use * selection I/O (for now). Note that chunks temporarily cached for @@ -2586,16 +2618,14 @@ H5D__chunk_may_use_select_io(const H5D_io_info_t *io_info, const H5D_dset_io_inf * must bypass the chunk-cache scheme because other MPI processes * could be writing to other elements in the same chunk. */ - if (io_info->using_mpi_vfd && (H5F_ACC_RDWR & H5F_INTENT(dataset->oloc.file))) - ret_value = TRUE; - else { + if (!(io_info->using_mpi_vfd && (H5F_ACC_RDWR & H5F_INTENT(dataset->oloc.file)))) { #endif /* H5_HAVE_PARALLEL */ /* Check if the chunk is too large to keep in the cache */ H5_CHECK_OVERFLOW(dataset->shared->layout.u.chunk.size, uint32_t, size_t); - if ((size_t)dataset->shared->layout.u.chunk.size > dataset->shared->cache.chunk.nbytes_max) - ret_value = TRUE; - else - ret_value = FALSE; + if ((size_t)dataset->shared->layout.u.chunk.size <= dataset->shared->cache.chunk.nbytes_max) { + io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; + io_info->no_selection_io_cause |= H5D_SEL_IO_CHUNK_CACHE; + } #ifdef H5_HAVE_PARALLEL } /* end else */ #endif /* H5_HAVE_PARALLEL */ @@ -2668,7 +2698,7 @@ H5D__chunk_read(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info) } /* Different blocks depending on whether we're using selection I/O */ - if (io_info->use_select_io) { + if (io_info->use_select_io == H5D_SELECTION_IO_MODE_ON) { size_t num_chunks; size_t element_sizes[2] = {dset_info->type_info.src_type_size, 0}; void *bufs[2] = {dset_info->buf.vp, NULL}; @@ -2749,6 +2779,8 @@ H5D__chunk_read(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info) io_info->addrs[io_info->pieces_added] = udata.chunk_block.offset; io_info->element_sizes[io_info->pieces_added] = element_sizes[0]; io_info->rbufs[io_info->pieces_added] = bufs[0]; + if (io_info->sel_pieces) + io_info->sel_pieces[io_info->pieces_added] = chunk_info; io_info->pieces_added++; } } /* end if */ @@ -2766,7 +2798,7 @@ H5D__chunk_read(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info) chunk_node = H5D_CHUNK_GET_NEXT_NODE(dset_info, chunk_node); } /* end while */ - /* Only perform I/O if not performing multi dataset I/O, otherwise the + /* Only perform I/O if not performing multi dataset I/O or type conversion, otherwise the * higher level will handle it after all datasets have been processed */ if (H5D_LAYOUT_CB_PERFORM_IO(io_info)) { /* Issue selection I/O call (we can skip the page buffer because we've @@ -2787,7 +2819,13 @@ H5D__chunk_read(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info) chunk_addrs = H5MM_xfree(chunk_addrs); } /* end if */ } /* end if */ - } /* end if */ + +#ifdef H5_HAVE_PARALLEL + /* Report that collective chunk I/O was used (will only be set on the DXPL if collective I/O was + * requested) */ + io_info->actual_io_mode |= H5D_MPIO_CHUNK_COLLECTIVE; +#endif /* H5_HAVE_PARALLEL */ + } /* end if */ else { H5D_io_info_t ctg_io_info; /* Contiguous I/O info object */ H5D_storage_t ctg_store; /* Chunk storage information as contiguous dataset */ @@ -2981,7 +3019,7 @@ H5D__chunk_write(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info) cpt_store.compact.dirty = &cpt_dirty; /* Different blocks depending on whether we're using selection I/O */ - if (io_info->use_select_io) { + if (io_info->use_select_io == H5D_SELECTION_IO_MODE_ON) { size_t num_chunks; size_t element_sizes[2] = {dset_info->type_info.dst_type_size, 0}; const void *bufs[2] = {dset_info->buf.cvp, NULL}; @@ -3139,6 +3177,8 @@ H5D__chunk_write(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info) io_info->addrs[io_info->pieces_added] = udata.chunk_block.offset; io_info->element_sizes[io_info->pieces_added] = element_sizes[0]; io_info->wbufs[io_info->pieces_added] = bufs[0]; + if (io_info->sel_pieces) + io_info->sel_pieces[io_info->pieces_added] = chunk_info; io_info->pieces_added++; } } /* end else */ @@ -3147,7 +3187,7 @@ H5D__chunk_write(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info) chunk_node = H5D_CHUNK_GET_NEXT_NODE(dset_info, chunk_node); } /* end while */ - /* Only perform I/O if not performing multi dataset I/O, otherwise the + /* Only perform I/O if not performing multi dataset I/O or type conversion, otherwise the * higher level will handle it after all datasets have been processed */ if (H5D_LAYOUT_CB_PERFORM_IO(io_info)) { /* Issue selection I/O call (we can skip the page buffer because we've @@ -3168,7 +3208,13 @@ H5D__chunk_write(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info) chunk_addrs = H5MM_xfree(chunk_addrs); } /* end if */ } /* end if */ - } /* end if */ + +#ifdef H5_HAVE_PARALLEL + /* Report that collective chunk I/O was used (will only be set on the DXPL if collective I/O was + * requested) */ + io_info->actual_io_mode |= H5D_MPIO_CHUNK_COLLECTIVE; +#endif /* H5_HAVE_PARALLEL */ + } /* end if */ else { /* Iterate through nodes in chunk skip list */ chunk_node = H5D_CHUNK_GET_FIRST_NODE(dset_info); diff --git a/src/H5Dcompact.c b/src/H5Dcompact.c index 9567c60..5f45da4 100644 --- a/src/H5Dcompact.c +++ b/src/H5Dcompact.c @@ -252,7 +252,8 @@ H5D__compact_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo) dinfo->store->compact.dirty = &dinfo->dset->shared->layout.storage.u.compact.dirty; /* Disable selection I/O */ - io_info->use_select_io = FALSE; + io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; + io_info->no_selection_io_cause |= H5D_SEL_IO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5D__compact_io_init() */ diff --git a/src/H5Dcontig.c b/src/H5Dcontig.c index 3eddfff..c8c7ee7 100644 --- a/src/H5Dcontig.c +++ b/src/H5Dcontig.c @@ -106,7 +106,7 @@ static herr_t H5D__contig_io_term(H5D_io_info_t *io_info, H5D_dset_io_info_t *d /* Helper routines */ static herr_t H5D__contig_write_one(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info, hsize_t offset, size_t size); -static htri_t H5D__contig_may_use_select_io(const H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset_info, +static herr_t H5D__contig_may_use_select_io(H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset_info, H5D_io_op_type_t op_type); /*********************/ @@ -586,8 +586,7 @@ H5D__contig_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo) int sf_ndims; /* The number of dimensions of the file dataspace (signed) */ - htri_t use_selection_io = FALSE; /* Whether to use selection I/O */ - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -666,6 +665,16 @@ H5D__contig_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo) /* get dset file address for piece */ new_piece_info->faddr = dinfo->dset->shared->layout.storage.u.contig.addr; + /* Initialize in-place type conversion info. Start with it disabled. */ + new_piece_info->in_place_tconv = FALSE; + new_piece_info->buf_off = 0; + + /* Calculate type conversion buffer size and check for in-place conversion if necessary. Currently + * only implemented for selection I/O. */ + if (io_info->use_select_io != H5D_SELECTION_IO_MODE_OFF && + !(dinfo->type_info.is_xform_noop && dinfo->type_info.is_conv_noop)) + H5D_INIT_PIECE_TCONV(io_info, dinfo, new_piece_info) + /* Save piece to dataset info struct so it is freed at the end of the * operation */ dinfo->layout_io_info.contig_piece_info = new_piece_info; @@ -676,11 +685,9 @@ H5D__contig_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo) /* Check if we're performing selection I/O if it hasn't been disabled * already */ - if (io_info->use_select_io) { - if ((use_selection_io = H5D__contig_may_use_select_io(io_info, dinfo, H5D_IO_OP_READ)) < 0) + if (io_info->use_select_io != H5D_SELECTION_IO_MODE_OFF) + if (H5D__contig_may_use_select_io(io_info, dinfo, io_info->op_type) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't check if selection I/O is possible") - io_info->use_select_io = (hbool_t)use_selection_io; - } done: if (ret_value < 0) { @@ -740,12 +747,12 @@ H5D__contig_mdio_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo) * *------------------------------------------------------------------------- */ -static htri_t -H5D__contig_may_use_select_io(const H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset_info, +static herr_t +H5D__contig_may_use_select_io(H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset_info, H5D_io_op_type_t op_type) { - const H5D_t *dataset = NULL; /* Local pointer to dataset info */ - htri_t ret_value = FAIL; /* Return value */ + const H5D_t *dataset = NULL; /* Local pointer to dataset info */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -757,27 +764,33 @@ H5D__contig_may_use_select_io(const H5D_io_info_t *io_info, const H5D_dset_io_in dataset = dset_info->dset; - /* Don't use selection I/O if it's globally disabled, if there is a type - * conversion, or if it's not a contiguous dataset, or if the sieve buffer - * exists (write) or is dirty (read) */ - if (dset_info->io_ops.single_read != H5D__select_read || - dset_info->layout_ops.readvv != H5D__contig_readvv || - (op_type == H5D_IO_OP_READ && dataset->shared->cache.contig.sieve_dirty) || - (op_type == H5D_IO_OP_WRITE && dataset->shared->cache.contig.sieve_buf)) - ret_value = FALSE; + /* None of the reasons this function might disable selection I/O are relevant to parallel, so no need to + * update no_selection_io_cause since we're only keeping track of the reason for no selection I/O in + * parallel (for now) */ + + /* Don't use selection I/O if it's globally disabled, if it's not a contiguous dataset, or if the sieve + * buffer exists (write) or is dirty (read) */ + if (dset_info->layout_ops.readvv != H5D__contig_readvv) { + io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; + io_info->no_selection_io_cause |= H5D_SEL_IO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; + } + else if ((op_type == H5D_IO_OP_READ && dataset->shared->cache.contig.sieve_dirty) || + (op_type == H5D_IO_OP_WRITE && dataset->shared->cache.contig.sieve_buf)) { + io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; + io_info->no_selection_io_cause |= H5D_SEL_IO_CONTIGUOUS_SIEVE_BUFFER; + } else { hbool_t page_buf_enabled; - HDassert(dset_info->io_ops.single_write == H5D__select_write); HDassert(dset_info->layout_ops.writevv == H5D__contig_writevv); /* Check if the page buffer is enabled */ if (H5PB_enabled(io_info->f_sh, H5FD_MEM_DRAW, &page_buf_enabled) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't check if page buffer is enabled") - if (page_buf_enabled) - ret_value = FALSE; - else - ret_value = TRUE; + if (page_buf_enabled) { + io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; + io_info->no_selection_io_cause |= H5D_SEL_IO_PAGE_BUFFER; + } } /* end else */ done: @@ -810,9 +823,9 @@ H5D__contig_read(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo) HDassert(dinfo->mem_space); HDassert(dinfo->file_space); - if (io_info->use_select_io) { - /* Only perform I/O if not performing multi dataset I/O with selection - * I/O, otherwise the higher level will handle it after all datasets + if (io_info->use_select_io == H5D_SELECTION_IO_MODE_ON) { + /* Only perform I/O if not performing multi dataset I/O or type conversion, + * otherwise the higher level will handle it after all datasets * have been processed */ if (H5D_LAYOUT_CB_PERFORM_IO(io_info)) { size_t dst_type_size = dinfo->type_info.dst_type_size; @@ -841,10 +854,17 @@ H5D__contig_read(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo) io_info->addrs[io_info->pieces_added] = dinfo->store->contig.dset_addr; io_info->element_sizes[io_info->pieces_added] = dinfo->type_info.src_type_size; io_info->rbufs[io_info->pieces_added] = dinfo->buf.vp; + if (io_info->sel_pieces) + io_info->sel_pieces[io_info->pieces_added] = dinfo->layout_io_info.contig_piece_info; io_info->pieces_added++; } } - } /* end if */ + +#ifdef H5_HAVE_PARALLEL + /* Report that collective contiguous I/O was used */ + io_info->actual_io_mode |= H5D_MPIO_CONTIGUOUS_COLLECTIVE; +#endif /* H5_HAVE_PARALLEL */ + } /* end if */ else /* Read data through legacy (non-selection I/O) pathway */ if ((dinfo->io_ops.single_read)(io_info, dinfo) < 0) @@ -880,9 +900,9 @@ H5D__contig_write(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo) HDassert(dinfo->mem_space); HDassert(dinfo->file_space); - if (io_info->use_select_io) { - /* Only perform I/O if not performing multi dataset I/O with selection - * I/O, otherwise the higher level will handle it after all datasets + if (io_info->use_select_io == H5D_SELECTION_IO_MODE_ON) { + /* Only perform I/O if not performing multi dataset I/O or type conversion, + * otherwise the higher level will handle it after all datasets * have been processed */ if (H5D_LAYOUT_CB_PERFORM_IO(io_info)) { size_t dst_type_size = dinfo->type_info.dst_type_size; @@ -911,10 +931,17 @@ H5D__contig_write(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo) io_info->addrs[io_info->pieces_added] = dinfo->store->contig.dset_addr; io_info->element_sizes[io_info->pieces_added] = dinfo->type_info.dst_type_size; io_info->wbufs[io_info->pieces_added] = dinfo->buf.cvp; + if (io_info->sel_pieces) + io_info->sel_pieces[io_info->pieces_added] = dinfo->layout_io_info.contig_piece_info; io_info->pieces_added++; } } - } /* end if */ + +#ifdef H5_HAVE_PARALLEL + /* Report that collective contiguous I/O was used */ + io_info->actual_io_mode |= H5D_MPIO_CONTIGUOUS_COLLECTIVE; +#endif /* H5_HAVE_PARALLEL */ + } /* end if */ else /* Write data through legacy (non-selection I/O) pathway */ if ((dinfo->io_ops.single_write)(io_info, dinfo) < 0) diff --git a/src/H5Defl.c b/src/H5Defl.c index faa5e4d..7509df7 100644 --- a/src/H5Defl.c +++ b/src/H5Defl.c @@ -213,7 +213,8 @@ H5D__efl_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo) H5MM_memcpy(&dinfo->store->efl, &(dinfo->dset->shared->dcpl_cache.efl), sizeof(H5O_efl_t)); /* Disable selection I/O */ - io_info->use_select_io = FALSE; + io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; + io_info->no_selection_io_cause |= H5D_SEL_IO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5D__efl_io_init() */ diff --git a/src/H5Dio.c b/src/H5Dio.c index 5ec0efe..1cd5835 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -43,14 +43,16 @@ /********************/ /* Setup/teardown routines */ -static herr_t H5D__ioinfo_init(size_t count, H5D_dset_io_info_t *dset_info, H5D_io_info_t *io_info); +static herr_t H5D__ioinfo_init(size_t count, H5D_io_op_type_t op_type, H5D_dset_io_info_t *dset_info, + H5D_io_info_t *io_info); static herr_t H5D__dset_ioinfo_init(H5D_t *dset, H5D_dset_io_info_t *dset_info, H5D_storage_t *store); static herr_t H5D__typeinfo_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info, hid_t mem_type_id); static herr_t H5D__typeinfo_init_phase2(H5D_io_info_t *io_info); +static herr_t H5D__typeinfo_init_phase3(H5D_io_info_t *io_info); #ifdef H5_HAVE_PARALLEL static herr_t H5D__ioinfo_adjust(H5D_io_info_t *io_info); #endif /* H5_HAVE_PARALLEL */ -static herr_t H5D__typeinfo_term(H5D_io_info_t *io_info, size_t type_info_init); +static herr_t H5D__typeinfo_term(H5D_io_info_t *io_info); /*********************/ /* Package Variables */ @@ -77,7 +79,6 @@ herr_t H5D__read(size_t count, H5D_dset_io_info_t *dset_info) { H5D_io_info_t io_info; /* Dataset I/O info for multi dsets */ - size_t type_info_init = 0; /* Number of datatype info structs that have been initialized */ H5S_t *orig_mem_space_local; /* Local buffer for orig_mem_space */ H5S_t **orig_mem_space = NULL; /* If not NULL, ptr to an array of dataspaces */ /* containing the original memory spaces contained */ @@ -107,9 +108,8 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info) FUNC_ENTER_NOAPI(FAIL) /* Init io_info */ - if (H5D__ioinfo_init(count, dset_info, &io_info) < 0) + if (H5D__ioinfo_init(count, H5D_IO_OP_READ, dset_info, &io_info) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize I/O info") - io_info.op_type = H5D_IO_OP_READ; /* Allocate store buffer if necessary */ if (count > 1) @@ -148,7 +148,6 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info) /* Set up datatype info for operation */ if (H5D__typeinfo_init(&io_info, &(dset_info[i]), dset_info[i].mem_type_id) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to set up type info") - type_info_init++; /* Make certain that the number of elements in each selection is the same, and cache nelmts in * dset_info */ @@ -285,7 +284,6 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info) } } /* end of for loop */ - HDassert(type_info_init == count); HDassert(io_op_init + io_skipped == count); /* If no datasets have I/O, we're done */ @@ -297,10 +295,17 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to set up type info (second phase)") #ifdef H5_HAVE_PARALLEL - /* Adjust I/O info for any parallel I/O */ + /* Adjust I/O info for any parallel or selection I/O */ if (H5D__ioinfo_adjust(&io_info) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to adjust I/O info for parallel I/O") -#endif /*H5_HAVE_PARALLEL*/ + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "unable to adjust I/O info for parallel or selection I/O") +#endif /* H5_HAVE_PARALLEL */ + + /* Perform third phase of type info initialization */ + if (H5D__typeinfo_init_phase3(&io_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to set up type info (third phase)") + + H5CX_set_no_selection_io_cause(io_info.no_selection_io_cause); /* If multi dataset I/O callback is not provided, perform read IO via * single-dset path with looping */ @@ -339,6 +344,7 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info) else { haddr_t prev_tag = HADDR_UNDEF; + /* Allocate selection I/O parameter arrays if necessary */ if (!H5D_LAYOUT_CB_PERFORM_IO(&io_info) && io_info.piece_count > 0) { if (NULL == (io_info.mem_spaces = H5MM_malloc(io_info.piece_count * sizeof(H5S_t *)))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, @@ -354,6 +360,11 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info) if (NULL == (io_info.rbufs = H5MM_malloc(io_info.piece_count * sizeof(void *)))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "memory allocation failed for read buffer list") + if (io_info.max_tconv_type_size > 0) + if (NULL == + (io_info.sel_pieces = H5MM_malloc(io_info.piece_count * sizeof(io_info.sel_pieces[0])))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "unable to allocate array of selected pieces") } /* Loop with serial & single-dset read IO path */ @@ -373,15 +384,39 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info) H5AC_tag(prev_tag, NULL); } - /* Make final multi dataset selection I/O call if we are using both - * features - in this case the multi_read callbacks did not perform the - * actual I/O */ - H5_CHECK_OVERFLOW(io_info.pieces_added, size_t, uint32_t) - if (!H5D_LAYOUT_CB_PERFORM_IO(&io_info)) - if (H5F_shared_select_read(io_info.f_sh, H5FD_MEM_DRAW, (uint32_t)io_info.pieces_added, - io_info.mem_spaces, io_info.file_spaces, io_info.addrs, - io_info.element_sizes, io_info.rbufs) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "selection read failed") + /* Make final selection I/O call if the multi_read callbacks did not perform the actual I/O + * (if using selection I/O and either multi dataset or type conversion) */ + if (!H5D_LAYOUT_CB_PERFORM_IO(&io_info)) { + /* Check for type conversion */ + if (io_info.max_tconv_type_size > 0) { + /* Type conversion pathway */ + if (H5D__scatgath_read_select(&io_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "type conversion selection read failed") + } + else { + /* Call selection I/O directly */ + H5_CHECK_OVERFLOW(io_info.pieces_added, size_t, uint32_t) + if (H5F_shared_select_read(io_info.f_sh, H5FD_MEM_DRAW, (uint32_t)io_info.pieces_added, + io_info.mem_spaces, io_info.file_spaces, io_info.addrs, + io_info.element_sizes, io_info.rbufs) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "selection read failed") + } + } + +#ifdef H5_HAVE_PARALLEL + /* Report the actual I/O mode to the application if appropriate */ + if (io_info.using_mpi_vfd) { + H5FD_mpio_xfer_t xfer_mode; /* Parallel transfer for this request */ + + /* Get the parallel I/O transfer mode */ + if (H5CX_get_io_xfer_mode(&xfer_mode) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get MPI-I/O transfer mode") + + /* Only report the collective I/O mode if we're actually performing collective I/O */ + if (xfer_mode == H5FD_MPIO_COLLECTIVE) + H5CX_set_mpio_actual_io_mode(io_info.actual_io_mode); + } +#endif /* H5_HAVE_PARALLEL */ } done: @@ -392,7 +427,7 @@ done: HDONE_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to shut down I/O op info") /* Shut down datatype info for operation */ - if (H5D__typeinfo_term(&io_info, type_info_init) < 0) + if (H5D__typeinfo_term(&io_info) < 0) HDONE_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to shut down type info") /* Discard projected mem spaces and restore originals */ @@ -443,7 +478,6 @@ herr_t H5D__write(size_t count, H5D_dset_io_info_t *dset_info) { H5D_io_info_t io_info; /* Dataset I/O info for multi dsets */ - size_t type_info_init = 0; /* Number of datatype info structs that have been initialized */ H5S_t *orig_mem_space_local; /* Local buffer for orig_mem_space */ H5S_t **orig_mem_space = NULL; /* If not NULL, ptr to an array of dataspaces */ /* containing the original memory spaces contained */ @@ -471,9 +505,8 @@ H5D__write(size_t count, H5D_dset_io_info_t *dset_info) FUNC_ENTER_NOAPI(FAIL) /* Init io_info */ - if (H5D__ioinfo_init(count, dset_info, &io_info) < 0) + if (H5D__ioinfo_init(count, H5D_IO_OP_WRITE, dset_info, &io_info) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize I/O info") - io_info.op_type = H5D_IO_OP_WRITE; /* Allocate store buffer if necessary */ if (count > 1) @@ -509,9 +542,8 @@ H5D__write(size_t count, H5D_dset_io_info_t *dset_info) /* Set up datatype info for operation */ if (H5D__typeinfo_init(&io_info, &(dset_info[i]), dset_info[i].mem_type_id) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to set up type info") - type_info_init++; - /* Various MPI based checks */ + /* Various MPI based checks */ #ifdef H5_HAVE_PARALLEL if (H5F_HAS_FEATURE(dset_info[i].dset->oloc.file, H5FD_FEAT_HAS_MPI)) { /* If MPI based VFD is used, no VL or region reference datatype support yet. */ @@ -660,7 +692,6 @@ H5D__write(size_t count, H5D_dset_io_info_t *dset_info) H5AC_tag(prev_tag, NULL); } /* end of for loop */ - HDassert(type_info_init == count); HDassert(io_op_init == count); /* Perform second phase of type info initialization */ @@ -668,10 +699,17 @@ H5D__write(size_t count, H5D_dset_io_info_t *dset_info) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to set up type info (second phase)") #ifdef H5_HAVE_PARALLEL - /* Adjust I/O info for any parallel I/O */ + /* Adjust I/O info for any parallel or selection I/O */ if (H5D__ioinfo_adjust(&io_info) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to adjust I/O info for parallel I/O") -#endif /*H5_HAVE_PARALLEL*/ + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "unable to adjust I/O info for parallel or selection I/O") +#endif /* H5_HAVE_PARALLEL */ + + /* Perform third phase of type info initialization */ + if (H5D__typeinfo_init_phase3(&io_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to set up type info (third phase)") + + H5CX_set_no_selection_io_cause(io_info.no_selection_io_cause); /* If multi dataset I/O callback is not provided, perform write IO via * single-dset path with looping */ @@ -710,6 +748,7 @@ H5D__write(size_t count, H5D_dset_io_info_t *dset_info) else { haddr_t prev_tag = HADDR_UNDEF; + /* Allocate selection I/O parameter arrays if necessary */ if (!H5D_LAYOUT_CB_PERFORM_IO(&io_info) && io_info.piece_count > 0) { if (NULL == (io_info.mem_spaces = H5MM_malloc(io_info.piece_count * sizeof(H5S_t *)))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, @@ -724,7 +763,12 @@ H5D__write(size_t count, H5D_dset_io_info_t *dset_info) "memory allocation failed for element size list") if (NULL == (io_info.wbufs = H5MM_malloc(io_info.piece_count * sizeof(const void *)))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, - "memory allocation failed for read buffer list") + "memory allocation failed for write buffer list") + if (io_info.max_tconv_type_size > 0) + if (NULL == + (io_info.sel_pieces = H5MM_malloc(io_info.piece_count * sizeof(io_info.sel_pieces[0])))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "unable to allocate array of selected pieces") } /* loop with serial & single-dset write IO path */ @@ -742,15 +786,45 @@ H5D__write(size_t count, H5D_dset_io_info_t *dset_info) H5AC_tag(prev_tag, NULL); } - /* Make final multi dataset selection I/O call if we are using both - * features - in this case the multi_write callbacks did not perform the - * actual I/O */ - H5_CHECK_OVERFLOW(io_info.pieces_added, size_t, uint32_t) - if (!H5D_LAYOUT_CB_PERFORM_IO(&io_info)) - if (H5F_shared_select_write(io_info.f_sh, H5FD_MEM_DRAW, (uint32_t)io_info.pieces_added, - io_info.mem_spaces, io_info.file_spaces, io_info.addrs, - io_info.element_sizes, io_info.wbufs) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "selection write failed") + /* Make final selection I/O call if the multi_write callbacks did not perform the actual I/O + * (if using selection I/O and either multi dataset or type conversion) */ + if (!H5D_LAYOUT_CB_PERFORM_IO(&io_info)) { + /* Check for type conversion */ + if (io_info.max_tconv_type_size > 0) { + /* Type conversion pathway */ + if (H5D__scatgath_write_select(&io_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "type conversion selection write failed") + } + else { + /* Call selection I/O directly */ + H5_CHECK_OVERFLOW(io_info.pieces_added, size_t, uint32_t) + if (H5F_shared_select_write(io_info.f_sh, H5FD_MEM_DRAW, (uint32_t)io_info.pieces_added, + io_info.mem_spaces, io_info.file_spaces, io_info.addrs, + io_info.element_sizes, io_info.wbufs) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "selection write failed") + } + } + +#ifdef H5_HAVE_PARALLEL + /* Report the actual I/O mode to the application if appropriate */ + if (io_info.using_mpi_vfd) { + H5FD_mpio_xfer_t xfer_mode; /* Parallel transfer for this request */ + + /* Get the parallel I/O transfer mode */ + if (H5CX_get_io_xfer_mode(&xfer_mode) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get MPI-I/O transfer mode") + + /* Only report the collective I/O mode if we're actually performing collective I/O */ + if (xfer_mode == H5FD_MPIO_COLLECTIVE) { + H5CX_set_mpio_actual_io_mode(io_info.actual_io_mode); + + /* If we did selection I/O, report that we used "link chunk" mode, since that's the most + * analogous to what selection I/O does */ + if (io_info.use_select_io == H5D_SELECTION_IO_MODE_ON) + H5CX_set_mpio_actual_chunk_opt(H5D_MPIO_LINK_CHUNK); + } + } +#endif /* H5_HAVE_PARALLEL */ } #ifdef OLD_WAY @@ -781,7 +855,7 @@ done: } /* Shut down datatype info for operation */ - if (H5D__typeinfo_term(&io_info, type_info_init) < 0) + if (H5D__typeinfo_term(&io_info) < 0) HDONE_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to shut down type info") /* Discard projected mem spaces and restore originals */ @@ -802,6 +876,13 @@ done: /* Free global piece array */ H5MM_xfree(io_info.sel_pieces); + /* Free selection I/O arrays */ + H5MM_xfree(io_info.mem_spaces); + H5MM_xfree(io_info.file_spaces); + H5MM_xfree(io_info.addrs); + H5MM_xfree(io_info.element_sizes); + H5MM_xfree(io_info.wbufs); + /* Free store array if it was allocated */ if (store != &store_local) H5MM_free(store); @@ -819,8 +900,11 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5D__ioinfo_init(size_t count, H5D_dset_io_info_t *dset_info, H5D_io_info_t *io_info) +H5D__ioinfo_init(size_t count, H5D_io_op_type_t op_type, H5D_dset_io_info_t *dset_info, + H5D_io_info_t *io_info) { + H5D_selection_io_mode_t selection_io_mode; + FUNC_ENTER_PACKAGE_NOERR /* check args */ @@ -833,8 +917,9 @@ H5D__ioinfo_init(size_t count, H5D_dset_io_info_t *dset_info, H5D_io_info_t *io_ HDmemset(io_info, 0, sizeof(*io_info)); /* Set up simple fields */ - io_info->f_sh = count > 0 ? H5F_SHARED(dset_info[0].dset->oloc.file) : NULL; - io_info->count = count; + io_info->op_type = op_type; + io_info->f_sh = count > 0 ? H5F_SHARED(dset_info[0].dset->oloc.file) : NULL; + io_info->count = count; /* Start without multi-dataset I/O ops. If we're not using the collective * I/O path then we will call the single dataset callbacks in a loop. */ @@ -842,16 +927,31 @@ H5D__ioinfo_init(size_t count, H5D_dset_io_info_t *dset_info, H5D_io_info_t *io_ /* Use provided dset_info */ io_info->dsets_info = dset_info; - /* Start with selection I/O on if the global is on, layout callback will - * turn it off if appropriate */ - io_info->use_select_io = H5_use_selection_io_g; + /* Start with selection I/O mode from property list. If enabled, layout callback will turn it off if it + * is not supported by the layout. Handling of H5D_SELECTION_IO_MODE_AUTO occurs in H5D__ioinfo_adjust. + */ + H5CX_get_selection_io_mode(&selection_io_mode); + io_info->use_select_io = selection_io_mode; + + /* Record no selection I/O cause if it was disabled by the API */ + if (selection_io_mode == H5D_SELECTION_IO_MODE_OFF) + io_info->no_selection_io_cause = H5D_SEL_IO_DISABLE_BY_API; #ifdef H5_HAVE_PARALLEL + /* Determine if the file was opened with an MPI VFD */ if (count > 0) io_info->using_mpi_vfd = H5F_HAS_FEATURE(dset_info[0].dset->oloc.file, H5FD_FEAT_HAS_MPI); #endif /* H5_HAVE_PARALLEL */ + /* Check if we could potentially use in-place type conversion */ + if (op_type == H5D_IO_OP_READ) + /* Always on for read (modulo other restrictions that are handled in layout callbacks) */ + io_info->may_use_in_place_tconv = TRUE; + else + /* Only enable in-place type conversion if we're allowed to modify the write buffer */ + H5CX_get_modify_write_buf(&io_info->may_use_in_place_tconv); + FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5D__ioinfo_init() */ @@ -998,9 +1098,9 @@ H5D__typeinfo_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info, hid_t /* Check if the datatypes are compound subsets of one another */ type_info->cmpd_subset = H5T_path_compound_subset(type_info->tpath); - /* Update io_info->max_type_size */ - io_info->max_type_size = - MAX3(io_info->max_type_size, type_info->src_type_size, type_info->dst_type_size); + /* Update io_info->max_tconv_type_size */ + io_info->max_tconv_type_size = + MAX3(io_info->max_tconv_type_size, type_info->src_type_size, type_info->dst_type_size); /* Check if we need a background buffer */ if ((io_info->op_type == H5D_IO_OP_WRITE) && H5T_detect_class(dset->shared->type, H5T_VLEN, FALSE)) @@ -1025,8 +1125,12 @@ done: /*------------------------------------------------------------------------- * Function: H5D__typeinfo_init_phase2 * - * Purpose: Finish initializing type info for all datasets after - * calculating the max type size across all datasets. + * Purpose: Continue initializing type info for all datasets after + * calculating the max type size across all datasets, and + * before final determination of collective/independent in + * H5D__ioinfo_adjust(). Currently just checks to see if + * selection I/O can be used with type conversion, and sets + * no_collective_cause flags related to selection I/O. * * Return: Non-negative on success/Negative on failure * @@ -1042,89 +1146,65 @@ H5D__typeinfo_init_phase2(H5D_io_info_t *io_info) /* check args */ HDassert(io_info); - /* Check if we need to allocate a shared type conversion buffer */ - if (io_info->max_type_size) { - void *tconv_buf; /* Temporary conversion buffer pointer */ - void *bkgr_buf; /* Background conversion buffer pointer */ + /* If selection I/O mode is default (auto), enable it here if the VFD supports it (it will be turned off + * later if something else conflicts), otherwise disable it. If we're using the MPIO VFD, the automatic + * selection will happen in H5D__mpio_opt_possible() inside H5D__ioinfo_adjust(). */ +#ifdef H5_HAVE_PARALLEL + if (!io_info->using_mpi_vfd) +#endif /* H5_HAVE_PARALLEL */ + if (io_info->use_select_io == H5D_SELECTION_IO_MODE_DEFAULT) { + if (H5F_has_vector_select_io(io_info->dsets_info[0].dset->oloc.file, + io_info->op_type == H5D_IO_OP_WRITE)) + io_info->use_select_io = H5D_SELECTION_IO_MODE_ON; + else { + io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; + io_info->no_selection_io_cause |= H5D_SEL_IO_DEFAULT_OFF; + } + } + + /* If we're doing type conversion and we might be doing selection I/O, check if the buffers are large + * enough to handle the whole I/O */ + if (io_info->max_tconv_type_size && io_info->use_select_io != H5D_SELECTION_IO_MODE_OFF) { size_t max_temp_buf; /* Maximum temporary buffer size */ - size_t target_size; /* Desired buffer size */ size_t i; /* Local index variable */ - /* Get info from API context */ - if (H5CX_get_max_temp_buf(&max_temp_buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't retrieve max. temp. buf size") - if (H5CX_get_tconv_buf(&tconv_buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't retrieve temp. conversion buffer pointer") - if (H5CX_get_bkgr_buf(&bkgr_buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't retrieve background conversion buffer pointer") - - /* Set up datatype conversion/background buffers */ - target_size = max_temp_buf; - - /* If the buffer is too small to hold even one element (in the dataset with the largest , try to make - * it bigger */ - if (target_size < io_info->max_type_size) { - hbool_t default_buffer_info; /* Whether the buffer information are the defaults */ - - /* Detect if we have all default settings for buffers */ - default_buffer_info = - (hbool_t)((H5D_TEMP_BUF_SIZE == max_temp_buf) && (NULL == tconv_buf) && (NULL == bkgr_buf)); + /* Collective I/O, conversion buffer must be large enough for entire I/O (for now) */ - /* Check if we are using the default buffer info */ - if (default_buffer_info) - /* OK to get bigger for library default settings */ - target_size = io_info->max_type_size; - else - /* Don't get bigger than the application has requested */ - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "temporary buffer max size is too small") - } /* end if */ - - /* Get a temporary buffer for type conversion unless the app has already - * supplied one through the xfer properties. Instead of allocating a - * buffer which is the exact size, we allocate the target size. This - * buffer is shared among all datasets in the operation, unlike for the - * background buffer, where each dataset gets its own. */ - if (NULL == (io_info->tconv_buf = (uint8_t *)tconv_buf)) { - /* Allocate temporary buffer */ - if (NULL == (io_info->tconv_buf = H5FL_BLK_MALLOC(type_conv, target_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for type conversion") - io_info->tconv_buf_allocated = TRUE; - } /* end if */ - - /* Don't use API provided background buffer if there's more than one dataset, since each - * dataset needs its own */ - if (io_info->count > 1) - bkgr_buf = NULL; - - /* Iterate over datasets */ + /* Calculate size of background buffer (tconv buf size was calculated in layout io_init callbacks) + */ for (i = 0; i < io_info->count; i++) { H5D_type_info_t *type_info = &io_info->dsets_info[i].type_info; - /* Compute the number of elements that will fit into buffer */ - type_info->request_nelmts = target_size / MAX(type_info->src_type_size, type_info->dst_type_size); - ; - - /* Sanity check elements in temporary buffer */ - if (type_info->request_nelmts == 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "temporary buffer max size is too small") + /* Check for background buffer */ + if (type_info->need_bkg) { + /* Add size of this dataset's background buffer to the global background buffer size + */ + io_info->bkg_buf_size += io_info->dsets_info[i].nelmts * type_info->dst_type_size; - /* Allocate background buffer if necessary */ - if (type_info->need_bkg && NULL == (type_info->bkg_buf = (uint8_t *)bkgr_buf)) { - size_t bkg_size; /* Desired background buffer size */ + /* Check if we need to fill the background buffer with the destination contents */ + if (type_info->need_bkg == H5T_BKG_YES) + io_info->must_fill_bkg = TRUE; + } + } - /* Compute the background buffer size */ - /* (don't try to use buffers smaller than the default size) */ - bkg_size = type_info->request_nelmts * type_info->dst_type_size; - if (bkg_size < max_temp_buf) - bkg_size = max_temp_buf; + /* Get max temp buffer size from API context */ + if (H5CX_get_max_temp_buf(&max_temp_buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't retrieve max. temp. buf size") - /* Allocate background buffer */ - /* (Need calloc()-like call since memory needs to be initialized) */ - if (NULL == (type_info->bkg_buf = H5FL_BLK_CALLOC(type_conv, bkg_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, - "memory allocation failed for background conversion") - type_info->bkg_buf_allocated = TRUE; - } /* end if */ + /* Check if the needed type conversion or background buffer size is too big */ + if (io_info->tconv_buf_size > max_temp_buf) { + io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; + io_info->no_selection_io_cause |= H5D_SEL_IO_TCONV_BUF_TOO_SMALL; + io_info->tconv_buf_size = 0; + io_info->bkg_buf_size = 0; + io_info->must_fill_bkg = FALSE; + } + if (io_info->bkg_buf_size > max_temp_buf) { + io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; + io_info->no_selection_io_cause |= H5D_SEL_IO_BKG_BUF_TOO_SMALL; + io_info->tconv_buf_size = 0; + io_info->bkg_buf_size = 0; + io_info->must_fill_bkg = FALSE; } } @@ -1133,15 +1213,13 @@ done: } /* end H5D__typeinfo_init_phase2() */ #ifdef H5_HAVE_PARALLEL - /*------------------------------------------------------------------------- - * Function: H5D__ioinfo_adjust + * Function: H5D__ioinfo_adjust * - * Purpose: Adjust operation's I/O info for any parallel I/O + * Purpose: Adjust operation's I/O info for any parallel I/O, also + * handle decision on selection I/O even in serial case * - * This was derived from H5D__ioinfo_adjust for multi-dset work. - * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -1193,7 +1271,7 @@ H5D__ioinfo_adjust(H5D_io_info_t *io_info) * selection I/O is to be used - in this case the file driver will * handle collective I/O */ /* Check for selection/vector support in file driver? -NAF */ - if (!io_info->use_select_io) { + if (io_info->use_select_io == H5D_SELECTION_IO_MODE_OFF) { io_info->md_io_ops.multi_read_md = H5D__collective_read; io_info->md_io_ops.multi_write_md = H5D__collective_write; io_info->md_io_ops.single_read_md = H5D__mpio_select_read; @@ -1270,6 +1348,137 @@ done: #endif /* H5_HAVE_PARALLEL */ /*------------------------------------------------------------------------- + * Function: H5D__typeinfo_init_phase3 + * + * Purpose: Finish initializing type info for all datasets after + * calculating the max type size across all datasets. And + * after final collective/independent determination in + * H5D__ioinfo_adjust(). + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__typeinfo_init_phase3(H5D_io_info_t *io_info) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* check args */ + HDassert(io_info); + + /* Check if we need to allocate a shared type conversion buffer */ + if (io_info->max_tconv_type_size) { + void *tconv_buf; /* Temporary conversion buffer pointer */ + void *bkgr_buf; /* Background conversion buffer pointer */ + + /* Get provided buffers from API context */ + if (H5CX_get_tconv_buf(&tconv_buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't retrieve temp. conversion buffer pointer") + if (H5CX_get_bkgr_buf(&bkgr_buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't retrieve background conversion buffer pointer") + + /* Check if we're doing selection I/O */ + if (io_info->use_select_io == H5D_SELECTION_IO_MODE_ON) { + /* Selection I/O, conversion buffers must be large enough for entire I/O (for now) */ + + /* Allocate global type conversion buffer (if any, could be none if datasets in this + * I/O have 0 elements selected) */ + /* Allocating large buffers here will blow out all other type conversion buffers + * on the free list. Should we change this to a regular malloc? Would require + * keeping track of which version of free() to call. -NAF */ + if (io_info->tconv_buf_size > 0) { + if (NULL == (io_info->tconv_buf = H5FL_BLK_MALLOC(type_conv, io_info->tconv_buf_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, + "memory allocation failed for type conversion") + io_info->tconv_buf_allocated = TRUE; + } + + /* Allocate global background buffer (if any) */ + if (io_info->bkg_buf_size > 0) { + if (NULL == (io_info->bkg_buf = H5FL_BLK_MALLOC(type_conv, io_info->bkg_buf_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, + "memory allocation failed for type conversion") + io_info->bkg_buf_allocated = TRUE; + } + } + else { + /* No selection I/O, only need to make sure it's big enough for one element */ + size_t max_temp_buf; /* Maximum temporary buffer size */ + size_t target_size; /* Desired buffer size */ + size_t i; + + /* Make sure selection I/O is disabled (DEFAULT should have been handled by now) */ + HDassert(io_info->use_select_io == H5D_SELECTION_IO_MODE_OFF); + + /* Get max buffer size from API context */ + if (H5CX_get_max_temp_buf(&max_temp_buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't retrieve max. temp. buf size") + + /* Set up datatype conversion/background buffers */ + target_size = max_temp_buf; + + /* If the buffer is too small to hold even one element (in the dataset with the largest , try to + * make it bigger */ + if (target_size < io_info->max_tconv_type_size) { + hbool_t default_buffer_info; /* Whether the buffer information are the defaults */ + + /* Detect if we have all default settings for buffers */ + default_buffer_info = (hbool_t)((H5D_TEMP_BUF_SIZE == max_temp_buf) && (NULL == tconv_buf) && + (NULL == bkgr_buf)); + + /* Check if we are using the default buffer info */ + if (default_buffer_info) + /* OK to get bigger for library default settings */ + target_size = io_info->max_tconv_type_size; + else + /* Don't get bigger than the application has requested */ + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "temporary buffer max size is too small") + } /* end if */ + + /* Get a temporary buffer for type conversion unless the app has already + * supplied one through the xfer properties. Instead of allocating a + * buffer which is the exact size, we allocate the target size. This + * buffer is shared among all datasets in the operation. */ + if (NULL == (io_info->tconv_buf = (uint8_t *)tconv_buf)) { + /* Allocate temporary buffer */ + if (NULL == (io_info->tconv_buf = H5FL_BLK_MALLOC(type_conv, target_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, + "memory allocation failed for type conversion") + io_info->tconv_buf_allocated = TRUE; + } /* end if */ + + /* Iterate over datasets */ + for (i = 0; i < io_info->count; i++) { + H5D_type_info_t *type_info = &io_info->dsets_info[i].type_info; + + /* Compute the number of elements that will fit into buffer */ + type_info->request_nelmts = + target_size / MAX(type_info->src_type_size, type_info->dst_type_size); + + /* Check if we need a background buffer and one hasn't been allocated yet */ + if (type_info->need_bkg && (NULL == io_info->bkg_buf) && + (NULL == (io_info->bkg_buf = (uint8_t *)bkgr_buf))) { + /* Allocate background buffer with the same size as the type conversion buffer. We can do + * this since the number of elements that fit in the type conversion buffer will never be + * larger than the number that could fit in a background buffer of equal size, since the + * tconv element size is max(src, dst) and the bkg element size is dst */ + if (NULL == (io_info->bkg_buf = H5FL_BLK_MALLOC(type_conv, target_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, + "memory allocation failed for background conversion") + io_info->bkg_buf_allocated = TRUE; + } + } + } + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__typeinfo_init_phase3() */ + +/*------------------------------------------------------------------------- * Function: H5D__typeinfo_term * * Purpose: Common logic for terminating a type info object @@ -1282,10 +1491,8 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5D__typeinfo_term(H5D_io_info_t *io_info, size_t type_info_init) +H5D__typeinfo_term(H5D_io_info_t *io_info) { - size_t i; - FUNC_ENTER_PACKAGE_NOERR /* Check for releasing datatype conversion & background buffers */ @@ -1293,11 +1500,10 @@ H5D__typeinfo_term(H5D_io_info_t *io_info, size_t type_info_init) HDassert(io_info->tconv_buf); (void)H5FL_BLK_FREE(type_conv, io_info->tconv_buf); } /* end if */ - for (i = 0; i < type_info_init; i++) - if (io_info->dsets_info[i].type_info.bkg_buf_allocated) { - HDassert(io_info->dsets_info[i].type_info.bkg_buf); - (void)H5FL_BLK_FREE(type_conv, io_info->dsets_info[i].type_info.bkg_buf); - } /* end if */ + if (io_info->bkg_buf_allocated) { + HDassert(io_info->bkg_buf); + (void)H5FL_BLK_FREE(type_conv, io_info->bkg_buf); + } /* end if */ FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5D__typeinfo_term() */ diff --git a/src/H5Dmpio.c b/src/H5Dmpio.c index 8e413c1..77edfc4 100644 --- a/src/H5Dmpio.c +++ b/src/H5Dmpio.c @@ -626,13 +626,17 @@ H5D__mpio_opt_possible(H5D_io_info_t *io_info) if (!H5FD_mpi_opt_types_g) local_cause[0] |= H5D_MPIO_MPI_OPT_TYPES_ENV_VAR_DISABLED; - /* Don't allow collective operations if datatype conversions need to happen */ - if (!type_info->is_conv_noop) - local_cause[0] |= H5D_MPIO_DATATYPE_CONVERSION; - - /* Don't allow collective operations if data transform operations should occur */ - if (!type_info->is_xform_noop) - local_cause[0] |= H5D_MPIO_DATA_TRANSFORMS; + /* Datatype conversions and transformations are allowed with selection I/O. If the selection I/O mode + * is auto (default), disable collective for now and re-enable later if we can */ + if (io_info->use_select_io != H5D_SELECTION_IO_MODE_ON) { + /* Don't allow collective operations if datatype conversions need to happen */ + if (!type_info->is_conv_noop) + local_cause[0] |= H5D_MPIO_DATATYPE_CONVERSION; + + /* Don't allow collective operations if data transform operations should occur */ + if (!type_info->is_xform_noop) + local_cause[0] |= H5D_MPIO_DATA_TRANSFORMS; + } /* Check whether these are both simple or scalar dataspaces */ if (!((H5S_SIMPLE == H5S_GET_EXTENT_TYPE(mem_space) || @@ -662,6 +666,15 @@ H5D__mpio_opt_possible(H5D_io_info_t *io_info) local_cause[0] |= H5D_MPIO_PARALLEL_FILTERED_WRITES_DISABLED; #endif + /* Check if we would be able to perform collective if we could use selection I/O. If so add reasons + * for not using selection I/O to local_cause[0] */ + if ((io_info->use_select_io == H5D_SELECTION_IO_MODE_OFF) && local_cause[0] && + !(local_cause[0] & + ~((unsigned)H5D_MPIO_DATATYPE_CONVERSION | (unsigned)H5D_MPIO_DATA_TRANSFORMS))) { + HDassert(io_info->no_selection_io_cause & H5D_MPIO_NO_SELECTION_IO_CAUSES); + local_cause[0] |= H5D_MPIO_NO_SELECTION_IO; + } + /* Check if we are able to do a MPI_Bcast of the data from one rank * instead of having all the processes involved in the collective I/O call. */ @@ -722,6 +735,25 @@ H5D__mpio_opt_possible(H5D_io_info_t *io_info) HMPI_GOTO_ERROR(FAIL, "MPI_Allreduce failed", mpi_code) } /* end else */ + /* If the selection I/O mode is default (auto), decide here whether it should be on or off */ + if (io_info->use_select_io == H5D_SELECTION_IO_MODE_DEFAULT) { + /* If the only reason(s) we've disabled collective are type conversions and/or transforms, enable + * selection I/O and re-enable collective I/O since it's supported by selection I/O */ + if (global_cause[0] && !(global_cause[0] & ~((unsigned)H5D_MPIO_DATATYPE_CONVERSION | + (unsigned)H5D_MPIO_DATA_TRANSFORMS))) { + HDassert(!(local_cause[0] & + ~((unsigned)H5D_MPIO_DATATYPE_CONVERSION | (unsigned)H5D_MPIO_DATA_TRANSFORMS))); + local_cause[0] = 0; + global_cause[0] = 0; + io_info->use_select_io = H5D_SELECTION_IO_MODE_ON; + } + else { + /* Otherwise, there's currently no benefit to selection I/O, so leave it off */ + io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; + io_info->no_selection_io_cause |= H5D_SEL_IO_DEFAULT_OFF; + } + } + /* Set the local & global values of no-collective-cause in the API context */ H5CX_set_mpio_local_no_coll_cause(local_cause[0]); H5CX_set_mpio_global_no_coll_cause(global_cause[0]); @@ -774,7 +806,7 @@ H5D__mpio_get_no_coll_cause_strings(char *local_cause, size_t local_cause_len, c * Use compile-time assertion so this routine is updated * when any new "no collective cause" values are added */ - HDcompile_assert(H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE == (H5D_mpio_no_collective_cause_t)256); + HDcompile_assert(H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE == (H5D_mpio_no_collective_cause_t)0x200); /* Initialize output buffers */ if (local_cause) @@ -827,6 +859,11 @@ H5D__mpio_get_no_coll_cause_strings(char *local_cause, size_t local_cause_len, c case H5D_MPIO_ERROR_WHILE_CHECKING_COLLECTIVE_POSSIBLE: cause_str = "an error occurred while checking if collective I/O was possible"; break; + case H5D_MPIO_NO_SELECTION_IO: + cause_str = "collective I/O may be supported by selection or vector I/O but that feature was " + "not possible (see causes via H5Pget_no_selection_io_cause())"; + break; + case H5D_MPIO_COLLECTIVE: case H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE: default: @@ -1438,13 +1475,8 @@ H5D__link_piece_collective_io(H5D_io_info_t *io_info, int H5_ATTR_UNUSED mpi_ran HDassert(io_info->dsets_info[i].dset->shared->dcpl_cache.pline.nused == 0); if (io_info->dsets_info[i].layout->type == H5D_CHUNKED) actual_io_mode |= H5D_MPIO_CHUNK_COLLECTIVE; - else if (io_info->dsets_info[i].layout->type == H5D_CONTIGUOUS) { + else if (io_info->dsets_info[i].layout->type == H5D_CONTIGUOUS) actual_io_mode |= H5D_MPIO_CONTIGUOUS_COLLECTIVE; - - /* if only single-dset */ - if (1 == io_info->count) - actual_chunk_opt_mode = H5D_MPIO_NO_CHUNK_OPTIMIZATION; - } else HGOTO_ERROR(H5E_IO, H5E_UNSUPPORTED, FAIL, "unsupported storage layout") } diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 0906028..4a23fe7 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -67,7 +67,49 @@ #define H5D_BT2_MERGE_PERC 40 /* Macro to determine if the layout I/O callback should perform I/O */ -#define H5D_LAYOUT_CB_PERFORM_IO(IO_INFO) (!(IO_INFO)->use_select_io || (IO_INFO)->count == 1) +#define H5D_LAYOUT_CB_PERFORM_IO(IO_INFO) \ + (((IO_INFO)->use_select_io == H5D_SELECTION_IO_MODE_OFF) || \ + ((IO_INFO)->count == 1 && (IO_INFO)->max_tconv_type_size == 0)) + +/* Macro to check if in-place type conversion will be used for a piece and add it to the global type + * conversion size if it won't be used */ +#define H5D_INIT_PIECE_TCONV(IO_INFO, DINFO, PIECE_INFO) \ + { \ + /* Check for potential in-place conversion */ \ + if ((IO_INFO)->may_use_in_place_tconv) { \ + size_t mem_type_size = ((IO_INFO)->op_type == H5D_IO_OP_READ) \ + ? (DINFO)->type_info.dst_type_size \ + : (DINFO)->type_info.src_type_size; \ + size_t file_type_size = ((IO_INFO)->op_type == H5D_IO_OP_READ) \ + ? (DINFO)->type_info.src_type_size \ + : (DINFO)->type_info.dst_type_size; \ + \ + /* Make sure the memory type is not smaller than the file type, otherwise the memory buffer \ + * won't be big enough to serve as the type conversion buffer */ \ + if (mem_type_size >= file_type_size) { \ + hbool_t is_contig; \ + hsize_t sel_off; \ + \ + /* Check if the space is contiguous */ \ + if (H5S_select_contig_block((PIECE_INFO)->mspace, &is_contig, &sel_off, NULL) < 0) \ + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't check if dataspace is contiguous") \ + \ + /* If the first sequence includes all the elements selected in this piece, it it contiguous \ + */ \ + if (is_contig) { \ + H5_CHECK_OVERFLOW(sel_off, hsize_t, size_t); \ + (PIECE_INFO)->in_place_tconv = TRUE; \ + (PIECE_INFO)->buf_off = (size_t)sel_off * mem_type_size; \ + } \ + } \ + } \ + \ + /* If we're not using in-place type conversion, add this piece to global type conversion buffer \ + * size. This will only be used if we must allocate a type conversion buffer for the entire I/O. */ \ + if (!(PIECE_INFO)->in_place_tconv) \ + (IO_INFO)->tconv_buf_size += (PIECE_INFO)->piece_points * MAX((DINFO)->type_info.src_type_size, \ + (DINFO)->type_info.dst_type_size); \ + } /****************************/ /* Package Private Typedefs */ @@ -83,15 +125,13 @@ typedef struct H5D_type_info_t { hid_t dst_type_id; /* Destination datatype ID */ /* Computed/derived values */ - size_t src_type_size; /* Size of source type */ - size_t dst_type_size; /* Size of destination type */ - hbool_t is_conv_noop; /* Whether the type conversion is a NOOP */ - hbool_t is_xform_noop; /* Whether the data transform is a NOOP */ - const H5T_subset_info_t *cmpd_subset; /* Info related to the compound subset conversion functions */ - H5T_bkg_t need_bkg; /* Type of background buf needed */ - size_t request_nelmts; /* Requested strip mine */ - uint8_t *bkg_buf; /* Background buffer */ - hbool_t bkg_buf_allocated; /* Whether the background buffer was allocated */ + size_t src_type_size; /* Size of source type */ + size_t dst_type_size; /* Size of destination type */ + hbool_t is_conv_noop; /* Whether the type conversion is a NOOP */ + hbool_t is_xform_noop; /* Whether the data transform is a NOOP */ + const H5T_subset_info_t *cmpd_subset; /* Info related to the compound subset conversion functions */ + H5T_bkg_t need_bkg; /* Type of background buf needed */ + size_t request_nelmts; /* Requested strip mine */ } H5D_type_info_t; /* Forward declaration of structs used below */ @@ -209,9 +249,11 @@ typedef struct H5D_piece_info_t { hsize_t piece_points; /* Number of elements selected in piece */ hsize_t scaled[H5O_LAYOUT_NDIMS]; /* Scaled coordinates of chunk (in file dataset's dataspace) */ H5S_t *fspace; /* Dataspace describing chunk & selection in it */ - unsigned fspace_shared; /* Indicate that the file space for a chunk is shared and shouldn't be freed */ - H5S_t *mspace; /* Dataspace describing selection in memory corresponding to this chunk */ - unsigned mspace_shared; /* Indicate that the memory space for a chunk is shared and shouldn't be freed */ + unsigned fspace_shared; /* Indicate that the file space for a chunk is shared and shouldn't be freed */ + H5S_t *mspace; /* Dataspace describing selection in memory corresponding to this chunk */ + unsigned mspace_shared; /* Indicate that the memory space for a chunk is shared and shouldn't be freed */ + hbool_t in_place_tconv; /* Whether to perform type conversion in-place */ + size_t buf_off; /* Buffer offset for in-place type conversion */ struct H5D_dset_io_info_t *dset_info; /* Pointer to dset_info */ } H5D_piece_info_t; @@ -263,10 +305,23 @@ typedef struct H5D_io_info_t { const void **wbufs; /* Array of write buffers */ haddr_t store_faddr; /* lowest file addr for read/write */ H5_flexible_const_ptr_t base_maddr; /* starting mem address */ - hbool_t use_select_io; /* Whether to use selection I/O */ + H5D_selection_io_mode_t use_select_io; /* Whether to use selection I/O */ uint8_t *tconv_buf; /* Datatype conv buffer */ hbool_t tconv_buf_allocated; /* Whether the type conversion buffer was allocated */ - size_t max_type_size; /* Largest of all source and destination type sizes */ + size_t tconv_buf_size; /* Size of type conversion buffer */ + uint8_t *bkg_buf; /* Background buffer */ + hbool_t bkg_buf_allocated; /* Whether the background buffer was allocated */ + size_t bkg_buf_size; /* Size of background buffer */ + size_t max_tconv_type_size; /* Largest of all source and destination type sizes involved in type + conversion */ + hbool_t + must_fill_bkg; /* Whether any datasets need a background buffer filled with destination contents */ + hbool_t may_use_in_place_tconv; /* Whether datasets in this I/O could potentially use in-place type + conversion if the type sizes are compatible with it */ +#ifdef H5_HAVE_PARALLEL + H5D_mpio_actual_io_mode_t actual_io_mode; /* Actual type of collective or independent I/O */ +#endif /* H5_HAVE_PARALLEL */ + unsigned no_selection_io_cause; /* "No selection I/O cause" flags */ } H5D_io_info_t; /* Created to pass both at once for callback func */ @@ -620,12 +675,14 @@ H5_DLL herr_t H5D__select_write(const H5D_io_info_t *io_info, const H5D_dset_io_ H5_DLL herr_t H5D_select_io_mem(void *dst_buf, H5S_t *dst_space, const void *src_buf, H5S_t *src_space, size_t elmt_size, size_t nelmts); -/* Functions that perform scatter-gather serial I/O operations */ +/* Functions that perform scatter-gather I/O operations */ H5_DLL herr_t H5D__scatter_mem(const void *_tscat_buf, H5S_sel_iter_t *iter, size_t nelmts, void *_buf); H5_DLL size_t H5D__gather_mem(const void *_buf, H5S_sel_iter_t *iter, size_t nelmts, void *_tgath_buf /*out*/); H5_DLL herr_t H5D__scatgath_read(const H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset_info); H5_DLL herr_t H5D__scatgath_write(const H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset_info); +H5_DLL herr_t H5D__scatgath_read_select(H5D_io_info_t *io_info); +H5_DLL herr_t H5D__scatgath_write_select(H5D_io_info_t *io_info); /* Functions that operate on dataset's layout information */ H5_DLL herr_t H5D__layout_set_io_ops(const H5D_t *dataset); diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h index 357945b..8265ac2 100644 --- a/src/H5Dprivate.h +++ b/src/H5Dprivate.h @@ -78,12 +78,15 @@ #define H5D_MPIO_LOCAL_NO_COLLECTIVE_CAUSE_NAME \ "local_no_collective_cause" /* cause of broken collective I/O in each process */ #define H5D_MPIO_GLOBAL_NO_COLLECTIVE_CAUSE_NAME \ - "global_no_collective_cause" /* cause of broken collective I/O in all processes */ -#define H5D_XFER_EDC_NAME "err_detect" /* EDC */ -#define H5D_XFER_FILTER_CB_NAME "filter_cb" /* Filter callback function */ -#define H5D_XFER_CONV_CB_NAME "type_conv_cb" /* Type conversion callback function */ -#define H5D_XFER_XFORM_NAME "data_transform" /* Data transform */ -#define H5D_XFER_DSET_IO_SEL_NAME "dset_io_selection" /* Dataset I/O selection */ + "global_no_collective_cause" /* cause of broken collective I/O in all processes */ +#define H5D_XFER_EDC_NAME "err_detect" /* EDC */ +#define H5D_XFER_FILTER_CB_NAME "filter_cb" /* Filter callback function */ +#define H5D_XFER_CONV_CB_NAME "type_conv_cb" /* Type conversion callback function */ +#define H5D_XFER_XFORM_NAME "data_transform" /* Data transform */ +#define H5D_XFER_DSET_IO_SEL_NAME "dset_io_selection" /* Dataset I/O selection */ +#define H5D_XFER_SELECTION_IO_MODE_NAME "selection_io_mode" /* Selection I/O mode */ +#define H5D_XFER_NO_SELECTION_IO_CAUSE_NAME "no_selection_io_cause" /* Cause for no selection I/O */ +#define H5D_XFER_MODIFY_WRITE_BUF_NAME "modify_write_buf" /* Modify write buffers */ #ifdef H5_HAVE_INSTRUMENTED_LIBRARY /* Collective chunk instrumentation properties */ #define H5D_XFER_COLL_CHUNK_LINK_HARD_NAME "coll_chunk_link_hard" diff --git a/src/H5Dscatgath.c b/src/H5Dscatgath.c index 2547bdc..802544c 100644 --- a/src/H5Dscatgath.c +++ b/src/H5Dscatgath.c @@ -31,6 +31,11 @@ /* Local Macros */ /****************/ +/* Macro to determine if we're using H5D__compound_opt_read() */ +#define H5D__SCATGATH_USE_CMPD_OPT_READ(DSET_INFO, PIECE_INFO) \ + ((DSET_INFO)->type_info.cmpd_subset && H5T_SUBSET_FALSE != (DSET_INFO)->type_info.cmpd_subset->subset && \ + !(PIECE_INFO)->in_place_tconv) + /******************/ /* Local Typedefs */ /******************/ @@ -529,7 +534,7 @@ H5D__scatgath_read(const H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset_ } /* end if */ else { if (H5T_BKG_YES == dset_info->type_info.need_bkg) { - n = H5D__gather_mem(buf, bkg_iter, smine_nelmts, dset_info->type_info.bkg_buf /*out*/); + n = H5D__gather_mem(buf, bkg_iter, smine_nelmts, io_info->bkg_buf /*out*/); if (n != smine_nelmts) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "mem gather failed") } /* end if */ @@ -539,7 +544,7 @@ H5D__scatgath_read(const H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset_ */ if (H5T_convert(dset_info->type_info.tpath, dset_info->type_info.src_type_id, dset_info->type_info.dst_type_id, smine_nelmts, (size_t)0, (size_t)0, - io_info->tconv_buf, dset_info->type_info.bkg_buf) < 0) + io_info->tconv_buf, io_info->bkg_buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "datatype conversion failed") /* Do the data transform after the conversion (since we're using type mem_type) */ @@ -672,8 +677,7 @@ H5D__scatgath_write(const H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset } /* end if */ else { if (H5T_BKG_YES == dset_info->type_info.need_bkg) { - n = H5D__gather_file(io_info, dset_info, bkg_iter, smine_nelmts, - dset_info->type_info.bkg_buf /*out*/); + n = H5D__gather_file(io_info, dset_info, bkg_iter, smine_nelmts, io_info->bkg_buf /*out*/); if (n != smine_nelmts) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file gather failed") } /* end if */ @@ -697,7 +701,7 @@ H5D__scatgath_write(const H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset */ if (H5T_convert(dset_info->type_info.tpath, dset_info->type_info.src_type_id, dset_info->type_info.dst_type_id, smine_nelmts, (size_t)0, (size_t)0, - io_info->tconv_buf, dset_info->type_info.bkg_buf) < 0) + io_info->tconv_buf, io_info->bkg_buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "datatype conversion failed") } /* end else */ @@ -727,6 +731,559 @@ done: } /* end H5D__scatgath_write() */ /*------------------------------------------------------------------------- + * Function: H5D__scatgath_read_select + * + * Purpose: Perform scatter/gather read from a list of dataset pieces + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__scatgath_read_select(H5D_io_info_t *io_info) +{ + H5S_t **tmp_mem_spaces = NULL; /* Memory spaces to use for read from disk */ + H5S_sel_iter_t *mem_iter = NULL; /* Memory selection iteration info */ + hbool_t mem_iter_init = FALSE; /* Memory selection iteration info has been initialized */ + void **tmp_bufs = NULL; /* Buffers to use for read from disk */ + void *tmp_bkg_buf = NULL; /* Temporary background buffer pointer */ + size_t tconv_bytes_used = 0; /* Number of bytes used so far in conversion buffer */ + size_t bkg_bytes_used = 0; /* Number of bytes used so far in background buffer */ + size_t i; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Sanity check */ + HDassert(io_info); + HDassert(io_info->count > 0); + HDassert(io_info->mem_spaces || io_info->pieces_added == 0); + HDassert(io_info->file_spaces || io_info->pieces_added == 0); + HDassert(io_info->addrs || io_info->pieces_added == 0); + HDassert(io_info->element_sizes || io_info->pieces_added == 0); + HDassert(io_info->rbufs || io_info->pieces_added == 0); + + /* Allocate list of buffers (within the tconv buf) */ + if (NULL == (tmp_bufs = H5MM_malloc(io_info->pieces_added * sizeof(void *)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "memory allocation failed for temporary buffer list") + + /* Allocate the iterator */ + if (NULL == (mem_iter = H5FL_MALLOC(H5S_sel_iter_t))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't allocate memory iterator") + + /* Allocate list of block memory spaces */ + /*!FIXME delay doing this until we find the first mem space that is non-contiguous or doesn't start at 0 + */ + if (NULL == (tmp_mem_spaces = H5MM_malloc(io_info->pieces_added * sizeof(H5S_t *)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "memory allocation failed for temporary memory space list") + + /* Build read operation to tconv buffer */ + for (i = 0; i < io_info->pieces_added; i++) { + H5D_dset_io_info_t *dset_info = io_info->sel_pieces[i]->dset_info; + + HDassert(io_info->sel_pieces[i]->piece_points > 0); + + /* Check if this piece is involved in type conversion */ + if (dset_info->type_info.is_xform_noop && dset_info->type_info.is_conv_noop) { + /* No type conversion, just copy the mem space and buffer */ + tmp_mem_spaces[i] = io_info->mem_spaces[i]; + tmp_bufs[i] = io_info->rbufs[i]; + } + else { + /* Create block memory space */ + if (NULL == + (tmp_mem_spaces[i] = H5S_create_simple(1, &io_info->sel_pieces[i]->piece_points, NULL))) { + HDmemset(&tmp_mem_spaces[i], 0, (io_info->pieces_added - i) * sizeof(tmp_mem_spaces[0])); + HGOTO_ERROR(H5E_DATASET, H5E_CANTCREATE, FAIL, "unable to create simple memory dataspace") + } + + /* Check for in-place type conversion */ + if (io_info->sel_pieces[i]->in_place_tconv) + /* Set buffer to point to read buffer + offset */ + tmp_bufs[i] = (uint8_t *)(io_info->rbufs[i]) + io_info->sel_pieces[i]->buf_off; + else { + /* Set buffer to point into type conversion buffer */ + tmp_bufs[i] = io_info->tconv_buf + tconv_bytes_used; + tconv_bytes_used += + io_info->sel_pieces[i]->piece_points * + MAX(dset_info->type_info.src_type_size, dset_info->type_info.dst_type_size); + HDassert(tconv_bytes_used <= io_info->tconv_buf_size); + } + + /* Fill background buffer here unless we will use H5D__compound_opt_read(). Must do this before + * the read so the read buffer doesn't get wiped out if we're using in-place type conversion */ + if (!H5D__SCATGATH_USE_CMPD_OPT_READ(dset_info, io_info->sel_pieces[i])) { + /* Check for background buffer */ + if (dset_info->type_info.need_bkg) { + HDassert(io_info->bkg_buf); + + /* Calculate background buffer position */ + tmp_bkg_buf = io_info->bkg_buf + bkg_bytes_used; + bkg_bytes_used += + io_info->sel_pieces[i]->piece_points * dset_info->type_info.dst_type_size; + HDassert(bkg_bytes_used <= io_info->bkg_buf_size); + + /* Gather data from read buffer to background buffer if necessary */ + if (H5T_BKG_YES == dset_info->type_info.need_bkg) { + /* Initialize memory iterator */ + HDassert(!mem_iter_init); + if (H5S_select_iter_init(mem_iter, io_info->mem_spaces[i], + dset_info->type_info.dst_type_size, 0) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "unable to initialize memory selection information") + mem_iter_init = TRUE; /* Memory selection iteration info has been initialized */ + + if ((size_t)io_info->sel_pieces[i]->piece_points != + H5D__gather_mem(io_info->rbufs[i], mem_iter, + (size_t)io_info->sel_pieces[i]->piece_points, + tmp_bkg_buf /*out*/)) + HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "mem gather failed") + + /* Reset selection iterator */ + HDassert(mem_iter_init); + if (H5S_SELECT_ITER_RELEASE(mem_iter) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't release selection iterator") + mem_iter_init = FALSE; + } + } + } + } + } + + /* Read data from all pieces */ + H5_CHECK_OVERFLOW(io_info->pieces_added, size_t, uint32_t) + if (H5F_shared_select_read(io_info->f_sh, H5FD_MEM_DRAW, (uint32_t)io_info->pieces_added, tmp_mem_spaces, + io_info->file_spaces, io_info->addrs, io_info->element_sizes, tmp_bufs) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "selection read failed") + + /* Reset bkg_bytes_used */ + bkg_bytes_used = 0; + + /* Perform type conversion and scatter data to memory buffers for datasets that need this */ + for (i = 0; i < io_info->pieces_added; i++) { + H5D_dset_io_info_t *dset_info = io_info->sel_pieces[i]->dset_info; + + HDassert(tmp_mem_spaces[i]); + + /* Check if this piece is involved in type conversion */ + if (tmp_mem_spaces[i] != io_info->mem_spaces[i]) { + H5_CHECK_OVERFLOW(io_info->sel_pieces[i]->piece_points, hsize_t, size_t); + + /* Initialize memory iterator */ + HDassert(!mem_iter_init); + if (H5S_select_iter_init(mem_iter, io_info->mem_spaces[i], dset_info->type_info.dst_type_size, + 0) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "unable to initialize memory selection information") + mem_iter_init = TRUE; /* Memory selection iteration info has been initialized */ + + /* If the source and destination are compound types and subset of each other + * and no conversion is needed, copy the data directly into user's buffer and + * bypass the rest of steps. + */ + if (H5D__SCATGATH_USE_CMPD_OPT_READ(dset_info, io_info->sel_pieces[i])) { + if (H5D__compound_opt_read((size_t)io_info->sel_pieces[i]->piece_points, mem_iter, + &dset_info->type_info, tmp_bufs[i], io_info->rbufs[i] /*out*/) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "datatype conversion failed") + } + else { + /* Check for background buffer */ + if (dset_info->type_info.need_bkg) { + HDassert(io_info->bkg_buf); + + /* Calculate background buffer position */ + tmp_bkg_buf = io_info->bkg_buf + bkg_bytes_used; + bkg_bytes_used += + io_info->sel_pieces[i]->piece_points * dset_info->type_info.dst_type_size; + HDassert(bkg_bytes_used <= io_info->bkg_buf_size); + } + + /* + * Perform datatype conversion. + */ + if (H5T_convert(dset_info->type_info.tpath, dset_info->type_info.src_type_id, + dset_info->type_info.dst_type_id, + (size_t)io_info->sel_pieces[i]->piece_points, (size_t)0, (size_t)0, + tmp_bufs[i], tmp_bkg_buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "datatype conversion failed") + + /* Do the data transform after the conversion (since we're using type mem_type) */ + if (!dset_info->type_info.is_xform_noop) { + H5Z_data_xform_t *data_transform; /* Data transform info */ + + /* Retrieve info from API context */ + if (H5CX_get_data_transform(&data_transform) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get data transform info") + + if (H5Z_xform_eval(data_transform, tmp_bufs[i], + (size_t)io_info->sel_pieces[i]->piece_points, + dset_info->type_info.mem_type) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "Error performing data transform") + } + + /* Scatter the data into memory if this was not an in-place conversion */ + if (!io_info->sel_pieces[i]->in_place_tconv) + if (H5D__scatter_mem(tmp_bufs[i], mem_iter, (size_t)io_info->sel_pieces[i]->piece_points, + io_info->rbufs[i] /*out*/) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "scatter failed") + } + + /* Release selection iterator */ + HDassert(mem_iter_init); + if (H5S_SELECT_ITER_RELEASE(mem_iter) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't release selection iterator") + mem_iter_init = FALSE; + } + } + +done: + /* Release and free selection iterator */ + if (mem_iter_init && H5S_SELECT_ITER_RELEASE(mem_iter) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't release selection iterator") + if (mem_iter) + mem_iter = H5FL_FREE(H5S_sel_iter_t, mem_iter); + + /* Free tmp_bufs */ + H5MM_free(tmp_bufs); + tmp_bufs = NULL; + + /* Clear and free tmp_mem_spaces */ + if (tmp_mem_spaces) { + for (i = 0; i < io_info->pieces_added; i++) + if (tmp_mem_spaces[i] != io_info->mem_spaces[i] && tmp_mem_spaces[i] && + H5S_close(tmp_mem_spaces[i]) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "Can't close dataspace") + H5MM_free(tmp_mem_spaces); + tmp_mem_spaces = NULL; + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__scatgath_read_select() */ + +/*------------------------------------------------------------------------- + * Function: H5D__scatgath_write_select + * + * Purpose: Perform scatter/gather write to a list of dataset pieces. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__scatgath_write_select(H5D_io_info_t *io_info) +{ + H5S_t **write_mem_spaces = NULL; /* Memory spaces to use for write to disk */ + size_t spaces_added = 0; /* Number of spaces added to write_mem_spaces */ + H5S_sel_iter_t *mem_iter = NULL; /* Memory selection iteration info */ + hbool_t mem_iter_init = FALSE; /* Memory selection iteration info has been initialized */ + const void **write_bufs = NULL; /* Buffers to use for write to disk */ + size_t tconv_bytes_used = 0; /* Number of bytes used so far in conversion buffer */ + size_t bkg_bytes_used = 0; /* Number of bytes used so far in background buffer */ + H5S_t **bkg_mem_spaces = NULL; /* Array of memory spaces for read to background buffer */ + H5S_t **bkg_file_spaces = NULL; /* Array of file spaces for read to background buffer */ + haddr_t *bkg_addrs = NULL; /* Array of file addresses for read to background buffer */ + size_t *bkg_element_sizes = NULL; /* Array of element sizes for read to background buffer */ + void **bkg_bufs = NULL; /* Array background buffers for read of existing file contents */ + size_t bkg_pieces = 0; /* Number of pieces that need to read the background data from disk */ + size_t i; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Sanity check */ + HDassert(io_info); + HDassert(io_info->count > 0); + HDassert(io_info->mem_spaces || io_info->pieces_added == 0); + HDassert(io_info->file_spaces || io_info->pieces_added == 0); + HDassert(io_info->addrs || io_info->pieces_added == 0); + HDassert(io_info->element_sizes || io_info->pieces_added == 0); + HDassert(io_info->wbufs || io_info->pieces_added == 0); + + /* Allocate list of buffers (within the tconv buf) */ + if (NULL == (write_bufs = (const void **)H5MM_malloc(io_info->pieces_added * sizeof(const void *)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "memory allocation failed for temporary buffer list") + + /* Allocate the iterator */ + if (NULL == (mem_iter = H5FL_MALLOC(H5S_sel_iter_t))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't allocate memory iterator") + + /* Allocate list of block memory spaces */ + /*!FIXME delay doing this until we find the first mem space that is non-contiguous or doesn't start at 0 + */ + if (NULL == (write_mem_spaces = H5MM_malloc(io_info->pieces_added * sizeof(H5S_t *)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "memory allocation failed for temporary memory space list") + + /* Build operations to read data to background buffer and to write data */ + for (i = 0; i < io_info->pieces_added; i++) { + H5D_dset_io_info_t *dset_info = io_info->sel_pieces[i]->dset_info; + + HDassert(io_info->sel_pieces[i]->piece_points > 0); + + /* Check if this piece is involved in type conversion */ + if (dset_info->type_info.is_xform_noop && dset_info->type_info.is_conv_noop) { + /* No type conversion, just copy the mem space and buffer */ + write_mem_spaces[i] = io_info->mem_spaces[i]; + spaces_added++; + write_bufs[i] = io_info->wbufs[i]; + } + else { + void *tmp_write_buf; /* To sidestep const warnings */ + void *tmp_bkg_buf = NULL; + + H5_CHECK_OVERFLOW(io_info->sel_pieces[i]->piece_points, hsize_t, size_t); + + /* Initialize memory iterator */ + HDassert(!mem_iter_init); + if (H5S_select_iter_init(mem_iter, io_info->mem_spaces[i], dset_info->type_info.src_type_size, + 0) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "unable to initialize memory selection information") + mem_iter_init = TRUE; /* Memory selection iteration info has been initialized */ + + /* Create block memory space */ + if (NULL == + (write_mem_spaces[i] = H5S_create_simple(1, &io_info->sel_pieces[i]->piece_points, NULL))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCREATE, FAIL, "unable to create simple memory dataspace") + spaces_added++; + + /* Check for in-place type conversion */ + if (io_info->sel_pieces[i]->in_place_tconv) { + H5_flexible_const_ptr_t flex_buf; + + /* Set buffer to point to write buffer + offset */ + /* Use cast to union to twiddle away const. OK because if we're doing this it means the user + * explicitly allowed us to modify this buffer via H5Pset_modify_write_buf(). */ + flex_buf.cvp = io_info->wbufs[i]; + tmp_write_buf = (uint8_t *)flex_buf.vp + io_info->sel_pieces[i]->buf_off; + } + else { + /* Set buffer to point into type conversion buffer */ + tmp_write_buf = io_info->tconv_buf + tconv_bytes_used; + tconv_bytes_used += + io_info->sel_pieces[i]->piece_points * + MAX(dset_info->type_info.src_type_size, dset_info->type_info.dst_type_size); + HDassert(tconv_bytes_used <= io_info->tconv_buf_size); + + /* Gather data from application buffer into the datatype conversion buffer */ + if ((size_t)io_info->sel_pieces[i]->piece_points != + H5D__gather_mem(io_info->wbufs[i], mem_iter, (size_t)io_info->sel_pieces[i]->piece_points, + tmp_write_buf /*out*/)) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "mem gather failed") + } + + /* Set buffer for writing to disk (from type conversion buffer) */ + write_bufs[i] = (const void *)tmp_write_buf; + + /* If the source and destination are compound types and the destination is a subset of + * the source and no conversion is needed, copy the data directly into the type + * conversion buffer and bypass the rest of steps. If the source is a subset of the + * destination, the optimization is done in conversion function H5T_conv_struct_opt to + * protect the background data. + */ + if (dset_info->type_info.cmpd_subset && + H5T_SUBSET_DST == dset_info->type_info.cmpd_subset->subset && + dset_info->type_info.dst_type_size == dset_info->type_info.cmpd_subset->copy_size && + !io_info->sel_pieces[i]->in_place_tconv) { + if (H5D__compound_opt_write((size_t)io_info->sel_pieces[i]->piece_points, + &dset_info->type_info, tmp_write_buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "datatype conversion failed") + + /* No background buffer necessary, prevent this element from being considered in the second + * loop */ + /* Add this to H5Tconv.c? -NAF */ + dset_info->type_info.need_bkg = H5T_BKG_NO; + } /* end if */ + else { + /* Check for background buffer */ + if (dset_info->type_info.need_bkg) { + HDassert(io_info->bkg_buf); + + /* Calculate background buffer position */ + tmp_bkg_buf = io_info->bkg_buf + bkg_bytes_used; + bkg_bytes_used += + io_info->sel_pieces[i]->piece_points * dset_info->type_info.dst_type_size; + HDassert(bkg_bytes_used <= io_info->bkg_buf_size); + } + + /* Set up background buffer read operation if necessary */ + if (H5T_BKG_YES == dset_info->type_info.need_bkg) { + HDassert(io_info->must_fill_bkg); + + /* Allocate arrays of parameters for selection read to background buffer if necessary */ + if (!bkg_mem_spaces) { + HDassert(!bkg_file_spaces && !bkg_addrs && !bkg_element_sizes && !bkg_bufs); + if (NULL == (bkg_mem_spaces = H5MM_malloc(io_info->pieces_added * sizeof(H5S_t *)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "memory allocation failed for memory space list") + if (NULL == (bkg_file_spaces = H5MM_malloc(io_info->pieces_added * sizeof(H5S_t *)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "memory allocation failed for file space list") + if (NULL == (bkg_addrs = H5MM_malloc(io_info->pieces_added * sizeof(haddr_t)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "memory allocation failed for piece address list") + if (NULL == (bkg_element_sizes = H5MM_malloc(io_info->pieces_added * sizeof(size_t)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "memory allocation failed for element size list") + if (NULL == (bkg_bufs = H5MM_malloc(io_info->pieces_added * sizeof(const void *)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "memory allocation failed for write buffer list") + } + + /* Use same (block) memory space, file space, address, and element size as write operation + */ + HDassert(bkg_mem_spaces && bkg_file_spaces && bkg_addrs && bkg_element_sizes && bkg_bufs); + bkg_mem_spaces[bkg_pieces] = write_mem_spaces[i]; + bkg_file_spaces[bkg_pieces] = io_info->file_spaces[i]; + bkg_addrs[bkg_pieces] = io_info->addrs[i]; + bkg_element_sizes[bkg_pieces] = io_info->element_sizes[i]; + + /* Use previously calculated background buffer position */ + bkg_bufs[bkg_pieces] = tmp_bkg_buf; + + /* Add piece */ + bkg_pieces++; + } + else { + /* Perform type conversion here to avoid second loop if no dsets use the background buffer + */ + /* Do the data transform before the type conversion (since + * transforms must be done in the memory type). */ + if (!dset_info->type_info.is_xform_noop) { + H5Z_data_xform_t *data_transform; /* Data transform info */ + + /* Retrieve info from API context */ + if (H5CX_get_data_transform(&data_transform) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get data transform info") + + if (H5Z_xform_eval(data_transform, tmp_write_buf, + (size_t)io_info->sel_pieces[i]->piece_points, + dset_info->type_info.mem_type) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "Error performing data transform") + } + + /* + * Perform datatype conversion. + */ + if (H5T_convert(dset_info->type_info.tpath, dset_info->type_info.src_type_id, + dset_info->type_info.dst_type_id, + (size_t)io_info->sel_pieces[i]->piece_points, (size_t)0, (size_t)0, + tmp_write_buf, tmp_bkg_buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "datatype conversion failed") + } + } + + /* Release selection iterator */ + HDassert(mem_iter_init); + if (H5S_SELECT_ITER_RELEASE(mem_iter) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't release selection iterator") + mem_iter_init = FALSE; + } + } + + HDassert(spaces_added == io_info->pieces_added); + + /* Gather data to background buffer if necessary */ + if (io_info->must_fill_bkg) { + size_t j = 0; /* Index into array of background buffers */ + + /* Read data */ + H5_CHECK_OVERFLOW(bkg_pieces, size_t, uint32_t) + if (H5F_shared_select_read(io_info->f_sh, H5FD_MEM_DRAW, (uint32_t)bkg_pieces, bkg_mem_spaces, + bkg_file_spaces, bkg_addrs, bkg_element_sizes, bkg_bufs) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "selection read to background buffer failed") + + /* Perform type conversion on pieces with background buffers that were just read */ + for (i = 0; i < io_info->pieces_added; i++) { + H5D_dset_io_info_t *dset_info = io_info->sel_pieces[i]->dset_info; + + if (H5T_BKG_YES == dset_info->type_info.need_bkg) { + /* Non-const write_buf[i]. Use pointer math here to avoid const warnings. When + * there's a background buffer write_buf[i] always points inside the non-const tconv + * buf so this is OK. */ + void *tmp_write_buf = + (void *)((uint8_t *)io_info->tconv_buf + + ((const uint8_t *)write_bufs[i] - (const uint8_t *)io_info->tconv_buf)); + + /* Do the data transform before the type conversion (since + * transforms must be done in the memory type). */ + if (!dset_info->type_info.is_xform_noop) { + H5Z_data_xform_t *data_transform; /* Data transform info */ + + /* Retrieve info from API context */ + if (H5CX_get_data_transform(&data_transform) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get data transform info") + + if (H5Z_xform_eval(data_transform, tmp_write_buf, + (size_t)io_info->sel_pieces[i]->piece_points, + dset_info->type_info.mem_type) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "Error performing data transform") + } + + /* + * Perform datatype conversion. + */ + HDassert(j < bkg_pieces); + if (H5T_convert(dset_info->type_info.tpath, dset_info->type_info.src_type_id, + dset_info->type_info.dst_type_id, + (size_t)io_info->sel_pieces[i]->piece_points, (size_t)0, (size_t)0, + tmp_write_buf, bkg_bufs[j]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "datatype conversion failed") + + /* Advance to next background buffer */ + j++; + } + } + + HDassert(j == bkg_pieces); + } + + /* Write data to disk */ + H5_CHECK_OVERFLOW(io_info->pieces_added, size_t, uint32_t) + if (H5F_shared_select_write(io_info->f_sh, H5FD_MEM_DRAW, (uint32_t)io_info->pieces_added, + write_mem_spaces, io_info->file_spaces, io_info->addrs, + io_info->element_sizes, write_bufs) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "selection write failed") + +done: + /* Release and free selection iterator */ + if (mem_iter_init && H5S_SELECT_ITER_RELEASE(mem_iter) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't release selection iterator") + if (mem_iter) + mem_iter = H5FL_FREE(H5S_sel_iter_t, mem_iter); + + /* Free write_bufs */ + H5MM_free(write_bufs); + write_bufs = NULL; + + /* Clear and free write_mem_spaces */ + if (write_mem_spaces) { + for (i = 0; i < spaces_added; i++) { + HDassert(write_mem_spaces[i]); + if (write_mem_spaces[i] != io_info->mem_spaces[i] && H5S_close(write_mem_spaces[i]) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "Can't close dataspace") + } + H5MM_free(write_mem_spaces); + write_mem_spaces = NULL; + } + + /* Free bakcground buffer parameter arrays */ + H5MM_free(bkg_mem_spaces); + bkg_mem_spaces = NULL; + H5MM_free(bkg_file_spaces); + bkg_file_spaces = NULL; + H5MM_free(bkg_addrs); + bkg_addrs = NULL; + H5MM_free(bkg_element_sizes); + bkg_element_sizes = NULL; + H5MM_free(bkg_bufs); + bkg_bufs = NULL; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__scatgath_write_select() */ + +/*------------------------------------------------------------------------- * Function: H5D__compound_opt_read * * Purpose: A special optimization case when the source and diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index c40642f..6695c02 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -2384,7 +2384,8 @@ H5D__virtual_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t H5_ATTR_UNUSED * FUNC_ENTER_PACKAGE_NOERR /* Disable selection I/O */ - io_info->use_select_io = FALSE; + io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; + io_info->no_selection_io_cause |= H5D_SEL_IO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5D__virtual_io_init() */ diff --git a/src/H5FDint.c b/src/H5FDint.c index c5b8713..6d90aae 100644 --- a/src/H5FDint.c +++ b/src/H5FDint.c @@ -469,6 +469,7 @@ H5FD_read_vector(H5FD_t *file, uint32_t count, H5FD_mem_t types[], haddr_t addrs */ extend_sizes = FALSE; extend_types = FALSE; + uint32_t no_selection_io_cause; for (i = 0; i < count; i++) { @@ -505,6 +506,11 @@ H5FD_read_vector(H5FD_t *file, uint32_t count, H5FD_mem_t types[], haddr_t addrs if ((file->cls->read)(file, type, dxpl_id, addrs[i], size, bufs[i]) < 0) HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "driver read request failed") } + + /* Add H5D_SEL_IO_NO_VECTOR_OR_SELECTION_IO_CB to no selection I/O cause */ + H5CX_get_no_selection_io_cause(&no_selection_io_cause); + no_selection_io_cause |= H5D_SEL_IO_NO_VECTOR_OR_SELECTION_IO_CB; + H5CX_set_no_selection_io_cause(no_selection_io_cause); } done: @@ -669,6 +675,7 @@ H5FD_write_vector(H5FD_t *file, uint32_t count, H5FD_mem_t types[], haddr_t addr */ extend_sizes = FALSE; extend_types = FALSE; + uint32_t no_selection_io_cause; for (i = 0; i < count; i++) { @@ -705,6 +712,11 @@ H5FD_write_vector(H5FD_t *file, uint32_t count, H5FD_mem_t types[], haddr_t addr if ((file->cls->write)(file, type, dxpl_id, addrs[i], size, bufs[i]) < 0) HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "driver write request failed") } + + /* Add H5D_SEL_IO_NO_VECTOR_OR_SELECTION_IO_CB to no selection I/O cause */ + H5CX_get_no_selection_io_cause(&no_selection_io_cause); + no_selection_io_cause |= H5D_SEL_IO_NO_VECTOR_OR_SELECTION_IO_CB; + H5CX_set_no_selection_io_cause(no_selection_io_cause); } done: @@ -991,6 +1003,14 @@ H5FD__read_selection_translate(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, uin 0) HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "driver read vector request failed") } + else { + uint32_t no_selection_io_cause; + + /* Add H5D_SEL_IO_NO_VECTOR_OR_SELECTION_IO_CB to no selection I/O cause */ + H5CX_get_no_selection_io_cause(&no_selection_io_cause); + no_selection_io_cause |= H5D_SEL_IO_NO_VECTOR_OR_SELECTION_IO_CB; + H5CX_set_no_selection_io_cause(no_selection_io_cause); + } done: /* Terminate and free iterators */ @@ -1630,6 +1650,14 @@ H5FD__write_selection_translate(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, ui 0) HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "driver write vector request failed") } + else { + uint32_t no_selection_io_cause; + + /* Add H5D_SEL_IO_NO_VECTOR_OR_SELECTION_IO_CB to no selection I/O cause */ + H5CX_get_no_selection_io_cause(&no_selection_io_cause); + no_selection_io_cause |= H5D_SEL_IO_NO_VECTOR_OR_SELECTION_IO_CB; + H5CX_set_no_selection_io_cause(no_selection_io_cause); + } done: /* Terminate and free iterators */ diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index f3e463d..40c6cff 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -906,6 +906,7 @@ H5_DLL haddr_t H5F_shared_get_eoa(const H5F_shared_t *f_sh, H5FD_mem_t type); H5_DLL haddr_t H5F_get_eoa(const H5F_t *f, H5FD_mem_t type); H5_DLL herr_t H5F_shared_get_file_driver(const H5F_shared_t *f_sh, H5FD_t **file_handle); H5_DLL herr_t H5F_get_vfd_handle(const H5F_t *file, hid_t fapl, void **file_handle); +H5_DLL hbool_t H5F_has_vector_select_io(const H5F_t *f, hbool_t is_write); /* File mounting routines */ H5_DLL herr_t H5F_mount(const struct H5G_loc_t *loc, const char *name, H5F_t *child, hid_t plist_id); diff --git a/src/H5Fquery.c b/src/H5Fquery.c index 469df58..72b173f 100644 --- a/src/H5Fquery.c +++ b/src/H5Fquery.c @@ -1372,3 +1372,30 @@ H5F_get_file_locking(const H5F_t *f) FUNC_LEAVE_NOAPI(f->shared->use_file_locking) } /* end H5F_get_file_locking */ + +/*------------------------------------------------------------------------- + * Function: H5F_has_vector_select_io + * + * Purpose: Determine if vector or selection I/O is supported by this file + * + * Return: TRUE/FALSE + * + *------------------------------------------------------------------------- + */ +hbool_t +H5F_has_vector_select_io(const H5F_t *f, hbool_t is_write) +{ + hbool_t ret_value; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(f->shared); + + if (is_write) + ret_value = (f->shared->lf->cls->write_vector != NULL || f->shared->lf->cls->write_selection != NULL); + else + ret_value = (f->shared->lf->cls->read_vector != NULL || f->shared->lf->cls->read_selection != NULL); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F_has_vector_select_io */ diff --git a/src/H5Ocache.c b/src/H5Ocache.c index 42d8f35..72261fa 100644 --- a/src/H5Ocache.c +++ b/src/H5Ocache.c @@ -13,10 +13,8 @@ /*------------------------------------------------------------------------- * * Created: H5Ocache.c - * Sep 28 2005 - * Quincey Koziol * - * Purpose: Object header metadata cache virtual functions. + * Purpose: Object header metadata cache virtual functions * *------------------------------------------------------------------------- */ @@ -30,13 +28,13 @@ /***********/ /* Headers */ /***********/ -#include "H5private.h" /* Generic Functions */ -#include "H5Eprivate.h" /* Error handling */ -#include "H5FLprivate.h" /* Free lists */ -#include "H5MFprivate.h" /* File memory management */ -#include "H5MMprivate.h" /* Memory management */ -#include "H5Opkg.h" /* Object headers */ -#include "H5WBprivate.h" /* Wrapped Buffers */ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free lists */ +#include "H5MFprivate.h" /* File memory management */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5Opkg.h" /* Object headers */ +#include "H5WBprivate.h" /* Wrapped Buffers */ /****************/ /* Local Macros */ @@ -74,7 +72,7 @@ static herr_t H5O__cache_chk_notify(H5AC_notify_action_t action, void *_thing); static herr_t H5O__cache_chk_free_icr(void *thing); /* Prefix routines */ -static herr_t H5O__prefix_deserialize(const uint8_t *image, H5O_cache_ud_t *udata); +static herr_t H5O__prefix_deserialize(const uint8_t *image, size_t len, H5O_cache_ud_t *udata); /* Chunk routines */ static herr_t H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t chunk_size, const uint8_t *image, @@ -144,15 +142,10 @@ H5FL_SEQ_DEFINE(H5O_cont_t); /*------------------------------------------------------------------------- * Function: H5O__cache_get_initial_load_size() * - * Purpose: Tell the metadata cache how much data to read from file in - * the first speculative read for the object header. - * - * Return: Success: SUCCEED - * Failure: FAIL - * - * Programmer: John Mainzer - * 7/28/14 + * Purpose: Tell the metadata cache how much data to read from file in + * the first speculative read for the object header. * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t @@ -160,7 +153,6 @@ H5O__cache_get_initial_load_size(void H5_ATTR_UNUSED *_udata, size_t *image_len) { FUNC_ENTER_PACKAGE_NOERR - /* Check arguments */ HDassert(image_len); /* Set the image length size */ @@ -172,33 +164,26 @@ H5O__cache_get_initial_load_size(void H5_ATTR_UNUSED *_udata, size_t *image_len) /*------------------------------------------------------------------------- * Function: H5O__cache_get_final_load_size() * - * Purpose: Tell the metadata cache the final size of an object header. - * - * Return: Success: SUCCEED - * Failure: FAIL - * - * Programmer: Quincey Koziol - * November 18, 2016 + * Purpose: Tell the metadata cache the final size of an object header. * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t -H5O__cache_get_final_load_size(const void *image, size_t H5_ATTR_NDEBUG_UNUSED image_len, void *_udata, - size_t *actual_len) +H5O__cache_get_final_load_size(const void *image, size_t image_len, void *_udata, size_t *actual_len) { H5O_cache_ud_t *udata = (H5O_cache_ud_t *)_udata; /* User data for callback */ - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE - /* Check arguments */ HDassert(image); HDassert(udata); HDassert(actual_len); HDassert(*actual_len == image_len); /* Deserialize the object header prefix */ - if (H5O__prefix_deserialize((const uint8_t *)image, udata) < 0) + if (H5O__prefix_deserialize((const uint8_t *)image, image_len, udata) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTDECODE, FAIL, "can't deserialize object header prefix") /* Sanity check */ @@ -219,10 +204,6 @@ done: * * Return: Success: TRUE/FALSE * Failure: Negative - * - * Programmer: Vailin Choi - * Aug 2015 - * *------------------------------------------------------------------------- */ static htri_t @@ -230,11 +211,10 @@ H5O__cache_verify_chksum(const void *_image, size_t len, void *_udata) { const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ H5O_cache_ud_t *udata = (H5O_cache_ud_t *)_udata; /* User data for callback */ - htri_t ret_value = TRUE; /* Return value */ + htri_t ret_value = TRUE; FUNC_ENTER_PACKAGE_NOERR - /* Check arguments */ HDassert(image); HDassert(udata); HDassert(udata->oh); @@ -257,8 +237,8 @@ H5O__cache_verify_chksum(const void *_image, size_t len, void *_udata) in H5O__prefix_deserialize() */ udata->free_oh = TRUE; ret_value = FALSE; - } /* end if */ - } /* end if */ + } + } else HDassert(!(udata->common.file_intent & H5F_ACC_SWMR_WRITE)); @@ -268,21 +248,18 @@ H5O__cache_verify_chksum(const void *_image, size_t len, void *_udata) /*------------------------------------------------------------------------- * Function: H5O__cache_deserialize * - * Purpose: Attempt to deserialize the object header contained in the - * supplied buffer, load the data into an instance of H5O_t, and - * return a pointer to the new instance. + * Purpose: Attempt to deserialize the object header contained in the + * supplied buffer, load the data into an instance of H5O_t, and + * return a pointer to the new instance. * - * Note that the object header is read with with a speculative read. - * If the initial read is too small, make note of this fact and return - * without error. H5C__load_entry() will note the size discrepancy - * and retry the deserialize operation with the correct size read. + * Note that the object header is read with with a speculative + * read. If the initial read is too small, make note of this fact + * and return without error. H5C__load_entry() will note the + * size discrepancy and retry the deserialize operation with + * the correct size read. * * Return: Success: Pointer to in core representation * Failure: NULL - * - * Programmer: John Mainzer - * 7/28/14 - * *------------------------------------------------------------------------- */ static void * @@ -290,11 +267,10 @@ H5O__cache_deserialize(const void *image, size_t len, void *_udata, hbool_t *dir { H5O_t *oh = NULL; /* Object header read in */ H5O_cache_ud_t *udata = (H5O_cache_ud_t *)_udata; /* User data for callback */ - void *ret_value = NULL; /* Return value */ + void *ret_value = NULL; FUNC_ENTER_PACKAGE - /* Check arguments */ HDassert(image); HDassert(len > 0); HDassert(udata); @@ -302,19 +278,18 @@ H5O__cache_deserialize(const void *image, size_t len, void *_udata, hbool_t *dir HDassert(udata->common.cont_msg_info); HDassert(dirty); - /* Check for partially deserialized object header */ - /* (Object header prefix will be deserialized if the object header came - * through the 'get_final_load_size' callback and not deserialized if - * the object header is coming from a cache image - QAK, 2016/12/14) + /* Check for partially deserialized object header + * + * The Object header prefix will be deserialized if the object header came + * through the 'get_final_load_size' callback and not deserialized if + * the object header is coming from a cache image. */ if (NULL == udata->oh) { /* Deserialize the object header prefix */ - if (H5O__prefix_deserialize((const uint8_t *)image, udata) < 0) + if (H5O__prefix_deserialize((const uint8_t *)image, len, udata) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTDECODE, NULL, "can't deserialize object header prefix") - - /* Sanity check */ HDassert(udata->oh); - } /* end if */ + } /* Retrieve partially deserialized object header from user data */ oh = udata->oh; @@ -327,7 +302,7 @@ H5O__cache_deserialize(const void *image, size_t len, void *_udata, hbool_t *dir /* Create virtual entry, for use as proxy */ if (NULL == (oh->proxy = H5AC_proxy_entry_create())) HGOTO_ERROR(H5E_OHDR, H5E_CANTCREATE, NULL, "can't create object header proxy") - } /* end if */ + } else oh->proxy = NULL; @@ -354,16 +329,11 @@ done: /*------------------------------------------------------------------------- * Function: H5O__cache_image_len * - * Purpose: Compute the size in bytes of the specified instance of - * H5O_t on disk, and return it in *image_len. On failure, - * the value of *image_len is undefined. - * - * Return: Success: SUCCEED - * Failure: FAIL - * - * Programmer: John Mainzer - * 7/28/14 + * Purpose: Compute the size in bytes of the specified instance of + * H5O_t on disk, and return it in *image_len. On failure, + * the value of *image_len is undefined. * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t @@ -373,7 +343,6 @@ H5O__cache_image_len(const void *_thing, size_t *image_len) FUNC_ENTER_PACKAGE_NOERR - /* Check arguments */ HDassert(oh); HDassert(oh->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(oh->cache_info.type == H5AC_OHDR); @@ -388,15 +357,10 @@ H5O__cache_image_len(const void *_thing, size_t *image_len) /*------------------------------------------------------------------------- * Function: H5O__cache_serialize * - * Purpose: Serialize the contents of the supplied object header, and - * load this data into the supplied buffer. - * - * Return: Success: SUCCEED - * Failure: FAIL - * - * Programmer: John Mainzer - * 7/28/14 + * Purpose: Serialize the contents of the supplied object header, and + * load this data into the supplied buffer. * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t @@ -404,11 +368,10 @@ H5O__cache_serialize(const H5F_t *f, void *image, size_t len, void *_thing) { H5O_t *oh = (H5O_t *)_thing; /* Object header to encode */ uint8_t *chunk_image; /* Pointer to object header prefix buffer */ - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE - /* Check arguments */ HDassert(f); HDassert(image); HDassert(oh); @@ -451,13 +414,13 @@ H5O__cache_serialize(const H5F_t *f, void *image, size_t len, void *_thing) UINT32ENCODE(chunk_image, oh->mtime); UINT32ENCODE(chunk_image, oh->ctime); UINT32ENCODE(chunk_image, oh->btime); - } /* end if */ + } /* Attribute fields */ if (oh->flags & H5O_HDR_ATTR_STORE_PHASE_CHANGE) { UINT16ENCODE(chunk_image, oh->max_compact); UINT16ENCODE(chunk_image, oh->min_dense); - } /* end if */ + } /* First chunk size */ switch (oh->flags & H5O_HDR_CHUNK0_SIZE) { @@ -483,8 +446,8 @@ H5O__cache_serialize(const H5F_t *f, void *image, size_t len, void *_thing) default: HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad size for chunk 0") - } /* end switch */ - } /* end if */ + } + } else { /* Version */ *chunk_image++ = oh->version; @@ -509,7 +472,7 @@ H5O__cache_serialize(const H5F_t *f, void *image, size_t len, void *_thing) /* Zero to alignment */ HDmemset(chunk_image, 0, (size_t)(H5O_SIZEOF_HDR(oh) - 12)); chunk_image += (size_t)(H5O_SIZEOF_HDR(oh) - 12); - } /* end else */ + } HDassert((size_t)(chunk_image - oh->chunk[0].image) == (size_t)(H5O_SIZEOF_HDR(oh) - H5O_SIZEOF_CHKSUM_OH(oh))); @@ -533,24 +496,17 @@ done: * * Purpose: Handle cache action notifications * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * Jul 23 2016 - * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t H5O__cache_notify(H5AC_notify_action_t action, void *_thing) { H5O_t *oh = (H5O_t *)_thing; - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE - /* - * Check arguments. - */ HDassert(oh); switch (action) { @@ -563,12 +519,12 @@ H5O__cache_notify(H5AC_notify_action_t action, void *_thing) /* Register the object header as a parent of the virtual entry */ if (H5AC_proxy_entry_add_parent(oh->proxy, oh) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTSET, FAIL, "can't add object header as parent of proxy") - } /* end if */ + } break; case H5AC_NOTIFY_ACTION_AFTER_FLUSH: case H5AC_NOTIFY_ACTION_ENTRY_DIRTIED: - /* do nothing */ + /* Do nothing */ break; case H5AC_NOTIFY_ACTION_ENTRY_CLEANED: { @@ -578,17 +534,17 @@ H5O__cache_notify(H5AC_notify_action_t action, void *_thing) for (u = 0; u < oh->nmesgs; u++) if (oh->mesg[u].chunkno == 0) oh->mesg[u].dirty = FALSE; -#ifndef NDEBUG +#ifdef H5O_DEBUG /* Reset the number of messages dirtied by decoding */ oh->ndecode_dirtied = 0; -#endif /* NDEBUG */ +#endif } break; case H5AC_NOTIFY_ACTION_CHILD_DIRTIED: case H5AC_NOTIFY_ACTION_CHILD_CLEANED: case H5AC_NOTIFY_ACTION_CHILD_UNSERIALIZED: case H5AC_NOTIFY_ACTION_CHILD_SERIALIZED: - /* do nothing */ + /* Do nothing */ break; case H5AC_NOTIFY_ACTION_BEFORE_EVICT: @@ -596,12 +552,12 @@ H5O__cache_notify(H5AC_notify_action_t action, void *_thing) /* Unregister the object header as a parent of the virtual entry */ if (H5AC_proxy_entry_remove_parent(oh->proxy, oh) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTSET, FAIL, "can't remove object header as parent of proxy") - } /* end if */ + } break; default: HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "unknown action from metadata cache") - } /* end switch */ + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -610,29 +566,23 @@ done: /*------------------------------------------------------------------------- * Function: H5O__cache_free_icr * - * Purpose: Free the in core representation of the supplied object header. - * - * Note: The metadata cache sets the object's cache_info.magic to - * H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC before calling a free_icr - * callback (checked in assert). + * Purpose: Free the in core representation of the supplied object header. * - * Return: Success: SUCCEED - * Failure: FAIL - * - * Programmer: John Mainzer - * 7/28/14 + * Note: The metadata cache sets the object's cache_info.magic to + * H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC before calling a free_icr + * callback (checked in assert). * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t H5O__cache_free_icr(void *_thing) { H5O_t *oh = (H5O_t *)_thing; /* Object header to destroy */ - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE - /* Check arguments */ HDassert(oh); HDassert(oh->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC); HDassert(oh->cache_info.type == H5AC_OHDR); @@ -648,16 +598,11 @@ done: /*------------------------------------------------------------------------- * Function: H5O__cache_chk_get_initial_load_size() * - * Purpose: Tell the metadata cache how large the on disk image of the - * chunk proxy is, so it can load the image into a buffer for the - * deserialize call. - * - * Return: Success: SUCCEED - * Failure: FAIL - * - * Programmer: John Mainzer - * 7/28/14 + * Purpose: Tell the metadata cache how large the on disk image of the + * chunk proxy is, so it can load the image into a buffer for the + * deserialize call. * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t @@ -667,7 +612,6 @@ H5O__cache_chk_get_initial_load_size(void *_udata, size_t *image_len) FUNC_ENTER_PACKAGE_NOERR - /* Check arguments */ HDassert(udata); HDassert(udata->oh); HDassert(image_len); @@ -686,10 +630,6 @@ H5O__cache_chk_get_initial_load_size(void *_udata, size_t *image_len) * * Return: Success: TRUE/FALSE * Failure: Negative - * - * Programmer: Vailin Choi - * Aug 2015 - * *------------------------------------------------------------------------- */ static htri_t @@ -697,11 +637,10 @@ H5O__cache_chk_verify_chksum(const void *_image, size_t len, void *_udata) { const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ H5O_chk_cache_ud_t *udata = (H5O_chk_cache_ud_t *)_udata; /* User data for callback */ - htri_t ret_value = TRUE; /* Return value */ + htri_t ret_value = TRUE; FUNC_ENTER_PACKAGE_NOERR - /* Check arguments */ HDassert(image); /* There is no checksum for version 1 */ @@ -714,7 +653,7 @@ H5O__cache_chk_verify_chksum(const void *_image, size_t len, void *_udata) if (stored_chksum != computed_chksum) ret_value = FALSE; - } /* end if */ + } FUNC_LEAVE_NOAPI(ret_value) } /* end H5O__cache_chk_verify_chksum() */ @@ -722,16 +661,12 @@ H5O__cache_chk_verify_chksum(const void *_image, size_t len, void *_udata) /*------------------------------------------------------------------------- * Function: H5O__cache_chk_deserialize * - * Purpose: Attempt to deserialize the object header continuation chunk - * contained in the supplied buffer, load the data into an instance - * of H5O_chunk_proxy_t, and return a pointer to the new instance. + * Purpose: Attempt to deserialize the object header continuation chunk + * contained in the supplied buffer, load the data into an instance + * of H5O_chunk_proxy_t, and return a pointer to the new instance. * * Return: Success: Pointer to in core representation * Failure: NULL - * - * Programmer: John Mainzer - * 7/28/14 - * *------------------------------------------------------------------------- */ static void * @@ -739,11 +674,10 @@ H5O__cache_chk_deserialize(const void *image, size_t len, void *_udata, hbool_t { H5O_chunk_proxy_t *chk_proxy = NULL; /* Chunk proxy object */ H5O_chk_cache_ud_t *udata = (H5O_chk_cache_ud_t *)_udata; /* User data for callback */ - void *ret_value = NULL; /* Return value */ + void *ret_value = NULL; FUNC_ENTER_PACKAGE - /* Check arguments */ HDassert(image); HDassert(len > 0); HDassert(udata); @@ -757,7 +691,6 @@ H5O__cache_chk_deserialize(const void *image, size_t len, void *_udata, hbool_t /* Check if we are still decoding the object header */ /* (as opposed to bringing a piece of it back from the file) */ if (udata->decoding) { - /* Sanity check */ HDassert(udata->common.f); HDassert(udata->common.cont_msg_info); @@ -768,7 +701,7 @@ H5O__cache_chk_deserialize(const void *image, size_t len, void *_udata, hbool_t /* Set the chunk number for the chunk proxy */ H5_CHECKED_ASSIGN(chk_proxy->chunkno, unsigned, udata->oh->nchunks - 1, size_t); - } /* end if */ + } else { /* Sanity check */ HDassert(udata->chunkno < udata->oh->nchunks); @@ -781,7 +714,7 @@ H5O__cache_chk_deserialize(const void *image, size_t len, void *_udata, hbool_t */ HDassert(0 == HDmemcmp(image, udata->oh->chunk[chk_proxy->chunkno].image, udata->oh->chunk[chk_proxy->chunkno].size)); - } /* end else */ + } /* Increment reference count of object header */ if (H5O__inc_rc(udata->oh) < 0) @@ -802,15 +735,10 @@ done: /*------------------------------------------------------------------------- * Function: H5O__cache_chk_image_len * - * Purpose: Return the on disk image size of a object header chunk to the - * metadata cache via the image_len. - * - * Return: Success: SUCCEED - * Failure: FAIL - * - * Programmer: John Mainzer - * 7/28/14 + * Purpose: Return the on disk image size of a object header chunk to the + * metadata cache via the image_len. * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t @@ -820,7 +748,6 @@ H5O__cache_chk_image_len(const void *_thing, size_t *image_len) FUNC_ENTER_PACKAGE_NOERR - /* Check arguments */ HDassert(chk_proxy); HDassert(chk_proxy->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(chk_proxy->cache_info.type == H5AC_OHDR_CHK); @@ -835,28 +762,22 @@ H5O__cache_chk_image_len(const void *_thing, size_t *image_len) /*------------------------------------------------------------------------- * Function: H5O__cache_chk_serialize * - * Purpose: Given a pointer to an instance of an object header chunk and an - * appropriately sized buffer, serialize the contents of the - * instance for writing to disk, and copy the serialized data - * into the buffer. - * - * Return: Success: SUCCEED - * Failure: FAIL - * - * Programmer: John Mainzer - * 7/28/14 + * Purpose: Given a pointer to an instance of an object header chunk and an + * appropriately sized buffer, serialize the contents of the + * instance for writing to disk, and copy the serialized data + * into the buffer. * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t H5O__cache_chk_serialize(const H5F_t *f, void *image, size_t len, void *_thing) { H5O_chunk_proxy_t *chk_proxy = (H5O_chunk_proxy_t *)_thing; /* Object header chunk to serialize */ - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE - /* Check arguments */ HDassert(f); HDassert(image); HDassert(chk_proxy); @@ -883,24 +804,17 @@ done: * * Purpose: Handle cache action notifications * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Neil Fortner - * Mar 20 2012 - * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t H5O__cache_chk_notify(H5AC_notify_action_t action, void *_thing) { H5O_chunk_proxy_t *chk_proxy = (H5O_chunk_proxy_t *)_thing; - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE - /* - * Check arguments. - */ HDassert(chk_proxy); HDassert(chk_proxy->oh); @@ -922,13 +836,13 @@ H5O__cache_chk_notify(H5AC_notify_action_t action, void *_thing) */ if (H5AC_create_flush_dependency(chk_proxy->fd_parent, chk_proxy) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTDEPEND, FAIL, "unable to create flush dependency") - } /* end if */ + } /* Add flush dependency on object header */ { if (H5AC_create_flush_dependency(chk_proxy->oh, chk_proxy) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTDEPEND, FAIL, "unable to create flush dependency") - } /* end if */ + } /* Add flush dependency on object header proxy, if proxy exists */ { @@ -940,12 +854,12 @@ H5O__cache_chk_notify(H5AC_notify_action_t action, void *_thing) HGOTO_ERROR(H5E_OHDR, H5E_CANTSET, FAIL, "can't add object header chunk as parent of proxy") } - } /* end if */ + } break; case H5AC_NOTIFY_ACTION_AFTER_FLUSH: case H5AC_NOTIFY_ACTION_ENTRY_DIRTIED: - /* do nothing */ + /* Do nothing */ break; case H5AC_NOTIFY_ACTION_ENTRY_CLEANED: { @@ -961,7 +875,7 @@ H5O__cache_chk_notify(H5AC_notify_action_t action, void *_thing) case H5AC_NOTIFY_ACTION_CHILD_CLEANED: case H5AC_NOTIFY_ACTION_CHILD_UNSERIALIZED: case H5AC_NOTIFY_ACTION_CHILD_SERIALIZED: - /* do nothing */ + /* Do nothing */ break; case H5AC_NOTIFY_ACTION_BEFORE_EVICT: @@ -978,7 +892,7 @@ H5O__cache_chk_notify(H5AC_notify_action_t action, void *_thing) if (H5AC_destroy_flush_dependency(chk_proxy->fd_parent, chk_proxy) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTUNDEPEND, FAIL, "unable to destroy flush dependency") chk_proxy->fd_parent = NULL; - } /* end if */ + } /* Unregister the object header as a parent of the virtual entry */ if (H5AC_destroy_flush_dependency(chk_proxy->oh, chk_proxy) < 0) @@ -988,16 +902,12 @@ H5O__cache_chk_notify(H5AC_notify_action_t action, void *_thing) if (H5AC_proxy_entry_remove_parent(chk_proxy->oh->proxy, chk_proxy) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTSET, FAIL, "can't remove object header chunk as parent of proxy") - } /* end if */ + } break; default: -#ifdef NDEBUG HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "unknown action from metadata cache") -#else /* NDEBUG */ - HDassert(0 && "Unknown action?!?"); -#endif /* NDEBUG */ - } /* end switch */ + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -1006,30 +916,24 @@ done: /*------------------------------------------------------------------------- * Function: H5O__cache_chk_free_icr * - * Purpose: Free the in core memory associated with the supplied object - * header continuation chunk. - * - * Note: The metadata cache sets the object's cache_info.magic to - * H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC before calling a free_icr - * callback (checked in assert). - * - * Return: Success: SUCCEED - * Failure: FAIL + * Purpose: Free the in core memory associated with the supplied object + * header continuation chunk. * - * Programmer: John Mainzer - * 7/28/14 + * Note: The metadata cache sets the object's cache_info.magic to + * H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC before calling a free_icr + * callback (checked in assert). * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t H5O__cache_chk_free_icr(void *_thing) { H5O_chunk_proxy_t *chk_proxy = (H5O_chunk_proxy_t *)_thing; /* Object header chunk proxy to release */ - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE - /* Check arguments */ HDassert(chk_proxy); HDassert(chk_proxy->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC); HDassert(chk_proxy->cache_info.type == H5AC_OHDR_CHK); @@ -1045,26 +949,20 @@ done: /*------------------------------------------------------------------------- * Function: H5O__add_cont_msg * - * Purpose: Add information from a continuation message to the list of + * Purpose: Add information from a continuation message to the list of * continuation messages in the object header * - * Return: Success: SUCCEED - * Failure: FAIL - * - * Programmer: Quincey Koziol - * July 12, 2008 - * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t H5O__add_cont_msg(H5O_cont_msgs_t *cont_msg_info, const H5O_cont_t *cont) { - size_t contno; /* Continuation message index */ - herr_t ret_value = SUCCEED; /* Return value */ + size_t contno; /* Continuation message index */ + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE - /* Check arguments */ HDassert(cont_msg_info); HDassert(cont); @@ -1077,7 +975,7 @@ H5O__add_cont_msg(H5O_cont_msgs_t *cont_msg_info, const H5O_cont_t *cont) HGOTO_ERROR(H5E_OHDR, H5E_NOSPACE, FAIL, "memory allocation failed") cont_msg_info->alloc_nmsgs = na; cont_msg_info->msgs = x; - } /* end if */ + } /* Init the continuation message info */ contno = cont_msg_info->nmsgs++; @@ -1092,26 +990,21 @@ done: /*------------------------------------------------------------------------- * Function: H5O__prefix_deserialize() * - * Purpose: Deserialize an object header prefix - * - * Return: Success: SUCCEED - * Failure: FAIL - * - * Programmer: Quincey Koziol - * December 14, 2016 + * Purpose: Deserialize an object header prefix * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t -H5O__prefix_deserialize(const uint8_t *_image, H5O_cache_ud_t *udata) +H5O__prefix_deserialize(const uint8_t *_image, size_t len, H5O_cache_ud_t *udata) { const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + const uint8_t *p_end = image + len - 1; /* End of image buffer */ H5O_t *oh = NULL; /* Object header read in */ - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE - /* Check arguments */ HDassert(image); HDassert(udata); @@ -1125,16 +1018,23 @@ H5O__prefix_deserialize(const uint8_t *_image, H5O_cache_ud_t *udata) /* Check for presence of magic number */ /* (indicates version 2 or later) */ + if (H5_IS_BUFFER_OVERFLOW(image, H5_SIZEOF_MAGIC, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); if (!HDmemcmp(image, H5O_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { - /* Magic number */ + + /* Magic number (bounds checked above) */ image += H5_SIZEOF_MAGIC; /* Version */ + if (H5_IS_BUFFER_OVERFLOW(image, 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); oh->version = *image++; if (H5O_VERSION_2 != oh->version) HGOTO_ERROR(H5E_OHDR, H5E_VERSION, FAIL, "bad object header version number") /* Flags */ + if (H5_IS_BUFFER_OVERFLOW(image, 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); oh->flags = *image++; if (oh->flags & ~H5O_HDR_ALL_FLAGS) HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "unknown object header status flag(s)") @@ -1144,7 +1044,10 @@ H5O__prefix_deserialize(const uint8_t *_image, H5O_cache_ud_t *udata) /* Time fields */ if (oh->flags & H5O_HDR_STORE_TIMES) { - uint32_t tmp; /* Temporary value */ + uint32_t tmp; + + if (H5_IS_BUFFER_OVERFLOW(image, 4 + 4 + 4 + 4, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); UINT32DECODE(image, tmp); oh->atime = (time_t)tmp; @@ -1154,48 +1057,61 @@ H5O__prefix_deserialize(const uint8_t *_image, H5O_cache_ud_t *udata) oh->ctime = (time_t)tmp; UINT32DECODE(image, tmp); oh->btime = (time_t)tmp; - } /* end if */ + } else oh->atime = oh->mtime = oh->ctime = oh->btime = 0; /* Attribute fields */ if (oh->flags & H5O_HDR_ATTR_STORE_PHASE_CHANGE) { + if (H5_IS_BUFFER_OVERFLOW(image, 2 + 2, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); + UINT16DECODE(image, oh->max_compact); UINT16DECODE(image, oh->min_dense); if (oh->max_compact < oh->min_dense) HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad object header attribute phase change values") - } /* end if */ + } else { oh->max_compact = H5O_CRT_ATTR_MAX_COMPACT_DEF; oh->min_dense = H5O_CRT_ATTR_MIN_DENSE_DEF; - } /* end else */ + } /* First chunk size */ switch (oh->flags & H5O_HDR_CHUNK0_SIZE) { case 0: /* 1 byte size */ + if (H5_IS_BUFFER_OVERFLOW(image, 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); udata->chunk0_size = *image++; break; case 1: /* 2 byte size */ + if (H5_IS_BUFFER_OVERFLOW(image, 2, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); UINT16DECODE(image, udata->chunk0_size); break; case 2: /* 4 byte size */ + if (H5_IS_BUFFER_OVERFLOW(image, 4, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); UINT32DECODE(image, udata->chunk0_size); break; case 3: /* 8 byte size */ + if (H5_IS_BUFFER_OVERFLOW(image, 8, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); UINT64DECODE(image, udata->chunk0_size); break; default: HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad size for chunk 0") - } /* end switch */ + } if (udata->chunk0_size > 0 && udata->chunk0_size < H5O_SIZEOF_MSGHDR_OH(oh)) HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad object header chunk size") - } /* end if */ + } else { /* Version */ + if (H5_IS_BUFFER_OVERFLOW(image, 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); oh->version = *image++; if (H5O_VERSION_1 != oh->version) HGOTO_ERROR(H5E_OHDR, H5E_VERSION, FAIL, "bad object header version number") @@ -1204,12 +1120,18 @@ H5O__prefix_deserialize(const uint8_t *_image, H5O_cache_ud_t *udata) oh->flags = H5O_CRT_OHDR_FLAGS_DEF; /* Reserved */ + if (H5_IS_BUFFER_OVERFLOW(image, 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); image++; /* Number of messages */ + if (H5_IS_BUFFER_OVERFLOW(image, 2, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); UINT16DECODE(image, udata->v1_pfx_nmesgs); /* Link count */ + if (H5_IS_BUFFER_OVERFLOW(image, 4, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); UINT32DECODE(image, oh->nlink); /* Reset unused time fields */ @@ -1220,21 +1142,27 @@ H5O__prefix_deserialize(const uint8_t *_image, H5O_cache_ud_t *udata) oh->min_dense = 0; /* First chunk size */ + if (H5_IS_BUFFER_OVERFLOW(image, 4, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); UINT32DECODE(image, udata->chunk0_size); if ((udata->v1_pfx_nmesgs > 0 && udata->chunk0_size < H5O_SIZEOF_MSGHDR_OH(oh)) || (udata->v1_pfx_nmesgs == 0 && udata->chunk0_size > 0)) HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad object header chunk size") /* Reserved, in version 1 (for 8-byte alignment padding) */ + if (H5_IS_BUFFER_OVERFLOW(image, 4, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); image += 4; - } /* end else */ + } /* Verify object header prefix length */ - HDassert((size_t)(image - _image) == (size_t)(H5O_SIZEOF_HDR(oh) - H5O_SIZEOF_CHKSUM_OH(oh))); + if ((size_t)(image - _image) != (size_t)(H5O_SIZEOF_HDR(oh) - H5O_SIZEOF_CHKSUM_OH(oh))) + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad object header prefix length") /* If udata->oh is to be freed (see H5O__cache_verify_chksum), - save the pointer to udata->oh and free it later after setting - udata->oh with the new object header */ + * save the pointer to udata->oh and free it later after setting + * udata->oh with the new object header + */ if (udata->free_oh) { H5O_t *saved_oh = udata->oh; HDassert(udata->oh); @@ -1263,34 +1191,27 @@ done: /*------------------------------------------------------------------------- * Function: H5O__chunk_deserialize * - * Purpose: Deserialize a chunk for an object header - * - * Return: Success: SUCCEED - * Failure: FAIL - * - * Programmer: Quincey Koziol - * July 12, 2008 + * Purpose: Deserialize a chunk for an object header * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t chunk_size, const uint8_t *image, size_t len, H5O_common_cache_ud_t *udata, hbool_t *dirty) { - const uint8_t *chunk_image; /* Pointer into buffer to decode */ + const uint8_t *chunk_image = NULL; /* Pointer into buffer to decode */ + const uint8_t *p_end = NULL; /* End of image buffer */ uint8_t *eom_ptr; /* Pointer to end of messages for a chunk */ unsigned merged_null_msgs = 0; /* Number of null messages merged together */ unsigned chunkno; /* Current chunk's index */ -#ifndef NDEBUG - unsigned nullcnt; /* Count of null messages (for sanity checking gaps in chunks) */ -#endif /* NDEBUG */ - hbool_t mesgs_modified = + unsigned nullcnt; /* Count of null messages (for sanity checking gaps in chunks) */ + hbool_t mesgs_modified = FALSE; /* Whether any messages were modified when the object header was deserialized */ - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE - /* Check arguments */ HDassert(oh); HDassert(H5F_addr_defined(addr)); HDassert(image); @@ -1307,7 +1228,7 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t chunk_size, const uint8_t HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "memory allocation failed") oh->alloc_nchunks = na; oh->chunk = x; - } /* end if */ + } /* Init the chunk data info */ chunkno = (unsigned)oh->nchunks++; @@ -1329,24 +1250,31 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t chunk_size, const uint8_t /* Point into chunk image to decode */ chunk_image = oh->chunk[chunkno].image; + p_end = chunk_image + oh->chunk[chunkno].size - 1; + + /* Skip over [already decoded] prefix in special case of chunk 0 */ + if (chunkno == 0) { + size_t skip = (size_t)(H5O_SIZEOF_HDR(oh) - H5O_SIZEOF_CHKSUM_OH(oh)); + + if (H5_IS_BUFFER_OVERFLOW(chunk_image, skip, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); + chunk_image += skip; + } - /* Handle chunk 0 as special case */ - if (chunkno == 0) - /* Skip over [already decoded] prefix */ - chunk_image += (size_t)(H5O_SIZEOF_HDR(oh) - H5O_SIZEOF_CHKSUM_OH(oh)); /* Check for magic # on chunks > 0 in later versions of the format */ else if (chunkno > 0 && oh->version > H5O_VERSION_1) { /* Magic number */ - if (HDmemcmp(chunk_image, H5O_CHK_MAGIC, (size_t)H5_SIZEOF_MAGIC) != 0) + if (H5_IS_BUFFER_OVERFLOW(chunk_image, H5_SIZEOF_MAGIC, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); + if (HDmemcmp(chunk_image, H5O_CHK_MAGIC, H5_SIZEOF_MAGIC) != 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, FAIL, "wrong object header chunk signature") chunk_image += H5_SIZEOF_MAGIC; - } /* end if */ + } /* Decode messages from this chunk */ eom_ptr = oh->chunk[chunkno].image + (oh->chunk[chunkno].size - H5O_SIZEOF_CHKSUM_OH(oh)); -#ifndef NDEBUG nullcnt = 0; -#endif /* NDEBUG */ + while (chunk_image < eom_ptr) { size_t mesg_size; /* Size of message read in */ unsigned id; /* ID (type) of current message */ @@ -1356,17 +1284,27 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t chunk_size, const uint8_t /* Decode message prefix info */ /* Version # */ - if (oh->version == H5O_VERSION_1) + if (oh->version == H5O_VERSION_1) { + if (H5_IS_BUFFER_OVERFLOW(chunk_image, 2, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); UINT16DECODE(chunk_image, id) - else + } + else { + if (H5_IS_BUFFER_OVERFLOW(chunk_image, 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); id = *chunk_image++; + } /* Message size */ + if (H5_IS_BUFFER_OVERFLOW(chunk_image, 2, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); UINT16DECODE(chunk_image, mesg_size); if (mesg_size != H5O_ALIGN_OH(oh, mesg_size)) HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, FAIL, "message not aligned") /* Message flags */ + if (H5_IS_BUFFER_OVERFLOW(chunk_image, 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); flags = *chunk_image++; if (flags & ~H5O_MSG_FLAG_BITS) HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, FAIL, "unknown flag for message") @@ -1381,13 +1319,20 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t chunk_size, const uint8_t * knows about */ /* Reserved bytes/creation index */ - if (oh->version == H5O_VERSION_1) - chunk_image += 3; /*reserved*/ + if (oh->version == H5O_VERSION_1) { + /* Reserved bytes */ + if (H5_IS_BUFFER_OVERFLOW(chunk_image, 3, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); + chunk_image += 3; + } else { /* Only decode creation index if they are being tracked */ - if (oh->flags & H5O_HDR_ATTR_CRT_ORDER_TRACKED) + if (oh->flags & H5O_HDR_ATTR_CRT_ORDER_TRACKED) { + if (H5_IS_BUFFER_OVERFLOW(chunk_image, 2, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); UINT16DECODE(chunk_image, crt_idx); - } /* end else */ + } + } /* Try to detect invalidly formatted object header message that * extends past end of chunk. @@ -1395,11 +1340,9 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t chunk_size, const uint8_t if (chunk_image + mesg_size > eom_ptr) HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, FAIL, "corrupt object header") -#ifndef NDEBUG /* Increment count of null messages */ if (H5O_NULL_ID == id) nullcnt++; -#endif /* NDEBUG */ /* Check for combining two adjacent 'null' messages */ if ((udata->file_intent & H5F_ACC_RDWR) && H5O_NULL_ID == id && oh->nmesgs > 0 && @@ -1412,7 +1355,7 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t chunk_size, const uint8_t oh->mesg[mesgno].raw_size += (size_t)H5O_SIZEOF_MSGHDR_OH(oh) + mesg_size; oh->mesg[mesgno].dirty = TRUE; merged_null_msgs++; - } /* end if */ + } else { H5O_mesg_t *mesg; /* Pointer to new message */ unsigned ioflags = 0; /* Flags for decode routine */ @@ -1476,20 +1419,20 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t chunk_size, const uint8_t /* This is a bit aggressive, since the application may * never change anything about the object (metadata or * raw data), but we can sort out the finer details - * when/if we start using the flag - QAK + * when/if we start using the flag. */ /* Also, it's possible that this functionality may not * get invoked if the object header is brought into * the metadata cache in some other "weird" way, like - * using H5Ocopy() - QAK + * using H5Ocopy(). */ mesg->flags |= H5O_MSG_FLAG_WAS_UNKNOWN; /* Mark the message and chunk as dirty */ mesg->dirty = TRUE; mesgs_modified = TRUE; - } /* end if */ - } /* end if */ + } + } else { /* Check for message of unshareable class marked as "shareable" */ @@ -1500,7 +1443,7 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t chunk_size, const uint8_t /* Set message class for "known" messages */ mesg->type = H5O_msg_class_g[id]; - } /* end else */ + } /* Do some inspection/interpretation of new messages from this chunk */ /* (detect continuation messages, ref. count messages, etc.) */ @@ -1522,7 +1465,7 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t chunk_size, const uint8_t /* Add to continuation messages left to interpret */ if (H5O__add_cont_msg(udata->cont_msg_info, cont) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTSET, FAIL, "can't add continuation message") - } /* end if */ + } /* Check if message is a ref. count message */ else if (H5O_REFCOUNT_ID == id) { H5O_refcount_t *refcount; @@ -1542,24 +1485,24 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t chunk_size, const uint8_t if (!refcount) HGOTO_ERROR(H5E_OHDR, H5E_CANTSET, FAIL, "can't decode refcount") oh->nlink = *refcount; - } /* end if */ + } /* Check if message is a link message */ else if (H5O_LINK_ID == id) { /* Increment the count of link messages */ oh->link_msgs_seen++; - } /* end if */ + } /* Check if message is an attribute message */ else if (H5O_ATTR_ID == id) { /* Increment the count of attribute messages */ oh->attr_msgs_seen++; - } /* end if */ + } /* Mark the message & chunk as dirty if the message was changed by decoding */ if ((ioflags & H5O_DECODEIO_DIRTY) && (udata->file_intent & H5F_ACC_RDWR)) { mesg->dirty = TRUE; mesgs_modified = TRUE; - } /* end if */ - } /* end else */ + } + } /* Advance decode pointer past message */ chunk_image += mesg_size; @@ -1567,18 +1510,20 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t chunk_size, const uint8_t /* Check for 'gap' at end of chunk */ if ((eom_ptr - chunk_image) > 0 && (eom_ptr - chunk_image) < H5O_SIZEOF_MSGHDR_OH(oh)) { /* Gaps can only occur in later versions of the format */ - HDassert(oh->version > H5O_VERSION_1); + if (oh->version == H5O_VERSION_1) + HGOTO_ERROR(H5E_OHDR, H5E_BADMESG, FAIL, "gap found in early version of file format"); /* Gaps should only occur in chunks with no null messages */ - HDassert(nullcnt == 0); + if (nullcnt != 0) + HGOTO_ERROR(H5E_OHDR, H5E_BADMESG, FAIL, "gap in chunk with no null messages"); /* Set gap information for chunk */ oh->chunk[chunkno].gap = (size_t)(eom_ptr - chunk_image); /* Increment location in chunk */ chunk_image += oh->chunk[chunkno].gap; - } /* end if */ - } /* end while */ + } + } /* Check for correct checksum on chunks, in later versions of the format */ if (oh->version > H5O_VERSION_1) { @@ -1587,11 +1532,14 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t chunk_size, const uint8_t /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ + if (H5_IS_BUFFER_OVERFLOW(chunk_image, 4, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); UINT32DECODE(chunk_image, stored_chksum); - } /* end if */ + } - /* Sanity check */ - HDassert(chunk_image == oh->chunk[chunkno].image + oh->chunk[chunkno].size); + /* Size check */ + if (chunk_image != oh->chunk[chunkno].image + oh->chunk[chunkno].size) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "object header image size mismatch"); /* Mark the chunk dirty if we've modified messages */ if (mesgs_modified) @@ -1601,7 +1549,7 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t chunk_size, const uint8_t if (merged_null_msgs > 0) { udata->merged_null_msgs += merged_null_msgs; *dirty = TRUE; - } /* end if */ + } done: if (ret_value < 0 && udata->cont_msg_info->msgs) { @@ -1612,28 +1560,22 @@ done: } /* H5O__chunk_deserialize() */ /*------------------------------------------------------------------------- - * Function: H5O__chunk_serialize + * Function: H5O__chunk_serialize * - * Purpose: Serialize a chunk for an object header - * - * Return: Success: SUCCEED - * Failure: FAIL - * - * Programmer: Quincey Koziol - * July 12, 2008 + * Purpose: Serialize a chunk for an object header * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t H5O__chunk_serialize(const H5F_t *f, H5O_t *oh, unsigned chunkno) { - H5O_mesg_t *curr_msg; /* Pointer to current message being operated on */ - unsigned u; /* Local index variable */ - herr_t ret_value = SUCCEED; /* Return value */ + H5O_mesg_t *curr_msg; /* Pointer to current message being operated on */ + unsigned u; /* Local index variable */ + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE - /* Check arguments */ HDassert(f); HDassert(oh); @@ -1673,7 +1615,7 @@ H5O__chunk_serialize(const H5F_t *f, H5O_t *oh, unsigned chunkno) /* Metadata checksum */ chunk_image = oh->chunk[chunkno].image + (oh->chunk[chunkno].size - H5O_SIZEOF_CHKSUM); UINT32ENCODE(chunk_image, metadata_chksum); - } /* end if */ + } done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5Odtype.c b/src/H5Odtype.c index b6e1b90..977e4b1 100644 --- a/src/H5Odtype.c +++ b/src/H5Odtype.c @@ -336,18 +336,31 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t for (dt->shared->u.compnd.nmembs = 0; dt->shared->u.compnd.nmembs < nmembs; dt->shared->u.compnd.nmembs++) { - size_t actual_name_length; /* Actual length of name */ - size_t max = (size_t)(p_end - *pp + 1); /* Max possible name length */ - unsigned ndims = 0; /* Number of dimensions of the array field */ - htri_t can_upgrade; /* Whether we can upgrade this type's version */ - hsize_t dim[H5O_LAYOUT_NDIMS]; /* Dimensions of the array */ - H5T_t *array_dt; /* Temporary pointer to the array datatype */ - H5T_t *temp_type; /* Temporary pointer to the field's datatype */ + size_t actual_name_length = 0; /* Actual length of name */ + unsigned ndims = 0; /* Number of dimensions of the array field */ + htri_t can_upgrade; /* Whether we can upgrade this type's version */ + hsize_t dim[H5O_LAYOUT_NDIMS]; /* Dimensions of the array */ + H5T_t *array_dt; /* Temporary pointer to the array datatype */ + H5T_t *temp_type; /* Temporary pointer to the field's datatype */ /* Get the length of the field name */ - actual_name_length = HDstrnlen((const char *)*pp, max); - if (actual_name_length == max) - HGOTO_ERROR(H5E_OHDR, H5E_NOSPACE, FAIL, "field name not null terminated") + if (!skip) { + /* There is a realistic buffer end, so check bounds */ + + size_t max = (size_t)(p_end - *pp + 1); /* Max possible name length */ + + actual_name_length = HDstrnlen((const char *)*pp, max); + if (actual_name_length == max) + HGOTO_ERROR(H5E_OHDR, H5E_NOSPACE, FAIL, "field name not null terminated") + } + else { + /* The buffer end can't be determined when it's an unbounded buffer + * passed via H5Tdecode(), so don't bounds check and hope for + * the best. + */ + actual_name_length = HDstrlen((const char *)*pp); + } + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, actual_name_length, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); @@ -624,12 +637,25 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t for (dt->shared->u.enumer.nmembs = 0; dt->shared->u.enumer.nmembs < nmembs; dt->shared->u.enumer.nmembs++) { - size_t actual_name_length; /* Actual length of name */ - size_t max = (size_t)(p_end - *pp + 1); /* Max possible name length */ + size_t actual_name_length = 0; /* Actual length of name */ - actual_name_length = HDstrnlen((const char *)*pp, max); - if (actual_name_length == max) - HGOTO_ERROR(H5E_OHDR, H5E_NOSPACE, FAIL, "enum name not null terminated") + /* Get the length of the enum name */ + if (!skip) { + /* There is a realistic buffer end, so check bounds */ + + size_t max = (size_t)(p_end - *pp + 1); /* Max possible name length */ + + actual_name_length = HDstrnlen((const char *)*pp, max); + if (actual_name_length == max) + HGOTO_ERROR(H5E_OHDR, H5E_NOSPACE, FAIL, "enum name not null terminated") + } + else { + /* The buffer end can't be determined when it's an unbounded buffer + * passed via H5Tdecode(), so don't bounds check and hope for + * the best. + */ + actual_name_length = HDstrlen((const char *)*pp); + } if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, actual_name_length, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); diff --git a/src/H5Pdxpl.c b/src/H5Pdxpl.c index ecf0372..942d6f2 100644 --- a/src/H5Pdxpl.c +++ b/src/H5Pdxpl.c @@ -168,6 +168,19 @@ #define H5D_XFER_DSET_IO_SEL_ENC H5P__dxfr_edc_enc #define H5D_XFER_DSET_IO_SEL_DEC H5P__dxfr_edc_dec #endif /* QAK */ +/* Definition for selection I/O mode property */ +#define H5D_XFER_SELECTION_IO_MODE_SIZE sizeof(H5D_selection_io_mode_t) +#define H5D_XFER_SELECTION_IO_MODE_DEF H5D_SELECTION_IO_MODE_DEFAULT +#define H5D_XFER_SELECTION_IO_MODE_ENC H5P__dxfr_selection_io_mode_enc +#define H5D_XFER_SELECTION_IO_MODE_DEC H5P__dxfr_selection_io_mode_dec +/* Definitions for cause of no selection I/O property */ +#define H5D_XFER_NO_SELECTION_IO_CAUSE_SIZE sizeof(uint32_t) +#define H5D_XFER_NO_SELECTION_IO_CAUSE_DEF 0 +/* Definitions for modify write buffer property */ +#define H5D_XFER_MODIFY_WRITE_BUF_SIZE sizeof(hbool_t) +#define H5D_XFER_MODIFY_WRITE_BUF_DEF FALSE +#define H5D_XFER_MODIFY_WRITE_BUF_ENC H5P__dxfr_modify_write_buf_enc +#define H5D_XFER_MODIFY_WRITE_BUF_DEC H5P__dxfr_modify_write_buf_dec /******************/ /* Local Typedefs */ @@ -208,6 +221,10 @@ static herr_t H5P__dxfr_xform_close(const char *name, size_t size, void *value); static herr_t H5P__dxfr_dset_io_hyp_sel_copy(const char *name, size_t size, void *value); static int H5P__dxfr_dset_io_hyp_sel_cmp(const void *value1, const void *value2, size_t size); static herr_t H5P__dxfr_dset_io_hyp_sel_close(const char *name, size_t size, void *value); +static herr_t H5P__dxfr_selection_io_mode_enc(const void *value, void **pp, size_t *size); +static herr_t H5P__dxfr_selection_io_mode_dec(const void **pp, void *value); +static herr_t H5P__dxfr_modify_write_buf_enc(const void *value, void **pp, size_t *size); +static herr_t H5P__dxfr_modify_write_buf_dec(const void **pp, void *value); /*********************/ /* Package Variables */ @@ -277,6 +294,9 @@ static const H5T_conv_cb_t H5D_def_conv_cb_g = static const void *H5D_def_xfer_xform_g = H5D_XFER_XFORM_DEF; /* Default value for data transform */ static const H5S_t *H5D_def_dset_io_sel_g = H5D_XFER_DSET_IO_SEL_DEF; /* Default value for dataset I/O selection */ +static const H5D_selection_io_mode_t H5D_def_selection_io_mode_g = H5D_XFER_SELECTION_IO_MODE_DEF; +static const uint32_t H5D_def_no_selection_io_cause_g = H5D_XFER_NO_SELECTION_IO_CAUSE_DEF; +static const hbool_t H5D_def_modify_write_buf_g = H5D_XFER_MODIFY_WRITE_BUF_DEF; /*------------------------------------------------------------------------- * Function: H5P__dxfr_reg_prop @@ -441,6 +461,24 @@ H5P__dxfr_reg_prop(H5P_genclass_t *pclass) H5D_XFER_DSET_IO_SEL_CLOSE) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + if (H5P__register_real(pclass, H5D_XFER_SELECTION_IO_MODE_NAME, H5D_XFER_SELECTION_IO_MODE_SIZE, + &H5D_def_selection_io_mode_g, NULL, NULL, NULL, H5D_XFER_SELECTION_IO_MODE_ENC, + H5D_XFER_SELECTION_IO_MODE_DEC, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + + /* Register the cause of no selection I/O property */ + /* (Note: this property should not have an encode/decode callback) */ + if (H5P__register_real(pclass, H5D_XFER_NO_SELECTION_IO_CAUSE_NAME, H5D_XFER_NO_SELECTION_IO_CAUSE_SIZE, + &H5D_def_no_selection_io_cause_g, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + + /* Register the modify write buffer property */ + if (H5P__register_real(pclass, H5D_XFER_MODIFY_WRITE_BUF_NAME, H5D_XFER_MODIFY_WRITE_BUF_SIZE, + &H5D_def_modify_write_buf_g, NULL, NULL, NULL, H5D_XFER_MODIFY_WRITE_BUF_ENC, + H5D_XFER_MODIFY_WRITE_BUF_DEC, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P__dxfr_reg_prop() */ @@ -2259,6 +2297,78 @@ done: } /* end H5P__dxfr_dset_io_hyp_sel_close() */ /*------------------------------------------------------------------------- + * Function: H5P__dxfr_selection_io_mode_enc + * + * Purpose: Callback routine which is called whenever the selection + * I/O mode property in the dataset transfer property list + * is encoded. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Vailin Choi + * Feb 2023 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__dxfr_selection_io_mode_enc(const void *value, void **_pp, size_t *size) +{ + const H5D_selection_io_mode_t *select_io_mode = + (const H5D_selection_io_mode_t *)value; /* Create local alias for values */ + uint8_t **pp = (uint8_t **)_pp; + + FUNC_ENTER_PACKAGE_NOERR + + /* Sanity check */ + HDassert(select_io_mode); + HDassert(size); + + if (NULL != *pp) + /* Encode selection I/O mode property */ + *(*pp)++ = (uint8_t)*select_io_mode; + + /* Size of selection I/O mode property */ + (*size)++; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P__dxfr_selection_io_mode_enc() */ + +/*------------------------------------------------------------------------- + * Function: H5P__dxfr_selection_io_mode_dec + * + * Purpose: Callback routine which is called whenever the selection + * I/O mode property in the dataset transfer property list + * is decoded. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Vailin Choi + * Feb 2023 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__dxfr_selection_io_mode_dec(const void **_pp, void *_value) +{ + H5D_selection_io_mode_t *select_io_mode = (H5D_selection_io_mode_t *)_value; /* Selection I/O mode */ + const uint8_t **pp = (const uint8_t **)_pp; + + FUNC_ENTER_PACKAGE_NOERR + + /* Sanity checks */ + HDassert(pp); + HDassert(*pp); + HDassert(select_io_mode); + + /* Decode selection I/O mode property */ + *select_io_mode = (H5D_selection_io_mode_t) * (*pp)++; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P__dxfr_selection_io_dec() */ + +/*------------------------------------------------------------------------- * Function: H5Pset_dataset_io_hyperslab_selection * * Purpose: H5Pset_dataset_io_hyperslab_selection() is designed to be used @@ -2386,3 +2496,248 @@ done: FUNC_LEAVE_API(ret_value) } /* end H5Pset_dataset_io_hyperslab_selection() */ + +/*------------------------------------------------------------------------- + * Function: H5Pset_selection_io + * + * Purpose: To set the selection I/O mode in the dataset + * transfer property list. + * + * Note: The library may not perform selection I/O as it asks for if + * the layout callback determines that it is not feasible to do so. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Vailin Choi + * March 5, 2023 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_selection_io(hid_t plist_id, H5D_selection_io_mode_t selection_io_mode) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "iDC", plist_id, selection_io_mode); + + /* Check arguments */ + if (plist_id == H5P_DEFAULT) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "can't set values in default property list") + + if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER))) + HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a dxpl") + + /* Set the selection I/O mode */ + if (H5P_set(plist, H5D_XFER_SELECTION_IO_MODE_NAME, &selection_io_mode) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set value") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_selection_io() */ + +/*------------------------------------------------------------------------- + * Function: H5Pget_selection_io + * + * Purpose: To retrieve the selection I/O mode that is set in + * the dataset transfer property list. + * + * Note: The library may not perform selection I/O as it asks for if + * the layout callback determines that it is not feasible to do so. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Vailin Choi + * March 5, 2023 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_selection_io(hid_t plist_id, H5D_selection_io_mode_t *selection_io_mode /*out*/) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "ix", plist_id, selection_io_mode); + + /* Check arguments */ + if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER))) + HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a dxpl") + + /* Get the selection I/O mode */ + if (selection_io_mode) + if (H5P_get(plist, H5D_XFER_SELECTION_IO_MODE_NAME, selection_io_mode) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get value") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_selection_io() */ + +/*------------------------------------------------------------------------- + * Function: H5Pget_no_selection_io_cause + * + * Purpose: Retrieves causes for not performing selection I/O + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi + * April 17, 2023 + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_no_selection_io_cause(hid_t plist_id, uint32_t *no_selection_io_cause /*out*/) +{ + H5P_genplist_t *plist; + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "ix", plist_id, no_selection_io_cause); + + /* Get the plist structure */ + if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER))) + HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID") + + /* Return values */ + if (no_selection_io_cause) + if (H5P_get(plist, H5D_XFER_NO_SELECTION_IO_CAUSE_NAME, no_selection_io_cause) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get no_selection_io_cause value") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_no_selection_io_cause() */ + +/*------------------------------------------------------------------------- + * Function: H5P__dxfr_modify_write_buf_enc + * + * Purpose: Callback routine which is called whenever the modify write + * buffer property in the dataset transfer property list is + * encoded. + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__dxfr_modify_write_buf_enc(const void *value, void **_pp /*out*/, size_t *size /*out*/) +{ + const hbool_t *modify_write_buf = (const hbool_t *)value; /* Create local alias for values */ + uint8_t **pp = (uint8_t **)_pp; + + FUNC_ENTER_PACKAGE_NOERR + + /* Sanity check */ + HDassert(modify_write_buf); + HDassert(size); + + if (NULL != *pp) + /* Encode modify write buf property. Use "!!" so we always get 0 or 1 */ + *(*pp)++ = (uint8_t)(!!(*modify_write_buf)); + + /* Size of modify write buf property */ + (*size)++; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P__dxfr_modify_write_buf_enc() */ + +/*------------------------------------------------------------------------- + * Function: H5P__dxfr_modify_write_buf_dec + * + * Purpose: Callback routine which is called whenever the modify write + * buffer property in the dataset transfer property list is + * decoded. + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__dxfr_modify_write_buf_dec(const void **_pp, void *_value /*out*/) +{ + hbool_t *modify_write_buf = (hbool_t *)_value; /* Modify write buffer */ + const uint8_t **pp = (const uint8_t **)_pp; + + FUNC_ENTER_PACKAGE_NOERR + + /* Sanity checks */ + HDassert(pp); + HDassert(*pp); + HDassert(modify_write_buf); + + /* Decode selection I/O mode property */ + *modify_write_buf = (hbool_t) * (*pp)++; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P__dxfr_modify_write_buf_dec() */ + +/*------------------------------------------------------------------------- + * Function: H5Pset_modify_write_buf + * + * Purpose: Allows the library to modify the contents of the write + * buffer + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_modify_write_buf(hid_t plist_id, hbool_t modify_write_buf) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "ib", plist_id, modify_write_buf); + + /* Check arguments */ + if (plist_id == H5P_DEFAULT) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "can't set values in default property list") + + if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER))) + HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a dxpl") + + /* Set the selection I/O mode */ + if (H5P_set(plist, H5D_XFER_MODIFY_WRITE_BUF_NAME, &modify_write_buf) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set value") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_modify_write_buf() */ + +/*------------------------------------------------------------------------- + * Function: H5Pget_modify_write_buf + * + * Purpose: Retrieves the "modify write buffer" property + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_modify_write_buf(hid_t plist_id, hbool_t *modify_write_buf /*out*/) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "ix", plist_id, modify_write_buf); + + /* Check arguments */ + if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER))) + HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a dxpl") + + /* Get the selection I/O mode */ + if (modify_write_buf) + if (H5P_get(plist, H5D_XFER_MODIFY_WRITE_BUF_NAME, modify_write_buf) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get value") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_modify_write_buf() */ diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index bb5d421..a08119d 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -328,7 +328,6 @@ typedef enum H5D_mpio_actual_io_mode_t { H5D_MPIO_CHUNK_MIXED = 0x1 | 0x2, /**< HDF5 performed one the chunk collective optimization schemes and some chunks were accessed independently, some collectively. */ - /** \internal The contiguous case is separate from the bit field. */ H5D_MPIO_CONTIGUOUS_COLLECTIVE = 0x4 /**< Collective I/O was performed on a contiguous dataset */ } H5D_mpio_actual_io_mode_t; @@ -344,7 +343,8 @@ typedef enum H5D_mpio_no_collective_cause_t { H5D_MPIO_SET_INDEPENDENT = 0x01, /**< Collective I/O was not performed because independent I/O was requested */ H5D_MPIO_DATATYPE_CONVERSION = 0x02, - /**< Collective I/O was not performed because datatype conversions were required */ + /**< Collective I/O was not performed because datatype conversions were required and selection I/O was not + possible (see below) */ H5D_MPIO_DATA_TRANSFORMS = 0x04, /**< Collective I/O was not performed because data transforms needed to be applied */ H5D_MPIO_MPI_OPT_TYPES_ENV_VAR_DISABLED = 0x08, @@ -357,11 +357,72 @@ typedef enum H5D_mpio_no_collective_cause_t { /**< Collective I/O was not performed because parallel filtered writes are disabled */ H5D_MPIO_ERROR_WHILE_CHECKING_COLLECTIVE_POSSIBLE = 0x80, /**< Error */ - H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE = 0x100 + H5D_MPIO_NO_SELECTION_IO = 0x100, + /**< Collective I/O would be supported by selection or vector I/O but that feature was disabled + (see causes via H5Pget_no_selection_io_cause()) */ + H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE = 0x200 /**< Sentinel */ } H5D_mpio_no_collective_cause_t; //! <!-- [H5D_mpio_no_collective_cause_t_snip] --> +/** + * Causes for H5Pget_no_selection_io_cause() property + */ +#define H5D_SEL_IO_DISABLE_BY_API \ + (0x0001u) /**< Selection I/O was not performed because \ + the feature was disabled by the API */ +#define H5D_SEL_IO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET \ + (0x0002u) /**< Selection I/O was not performed because the \ + dataset was neither contiguous nor chunked */ +#define H5D_SEL_IO_CONTIGUOUS_SIEVE_BUFFER \ + (0x0004u) /**< Selection I/O was not performed because of \ + sieve buffer for contiguous dataset */ +#define H5D_SEL_IO_NO_VECTOR_OR_SELECTION_IO_CB \ + (0x0008u) /**< Selection I/O was not performed because the VFD \ + does not have vector or selection I/O callback */ +#define H5D_SEL_IO_PAGE_BUFFER \ + (0x0010u) /**< Selection I/O was not performed because of \ + page buffer */ +#define H5D_SEL_IO_DATASET_FILTER \ + (0x0020u) /**< Selection I/O was not performed because of \ + dataset filters */ +#define H5D_SEL_IO_CHUNK_CACHE \ + (0x0040u) /**< Selection I/O was not performed because of \ + chunk cache */ +#define H5D_SEL_IO_TCONV_BUF_TOO_SMALL \ + (0x0080u) /**< Selection I/O was not performed because the \ + type conversion buffer is too small */ +#define H5D_SEL_IO_BKG_BUF_TOO_SMALL \ + (0x0100u) /**< Selection I/O was not performed because the \ + type conversion background buffer is too small */ +#define H5D_SEL_IO_DEFAULT_OFF \ + (0x0200u) /**< Selection I/O was not performed because the \ + selection I/O mode is DEFAULT and the library \ + chose it to be off for this case */ + +/* Causes for H5D_MPIO_NO_SELECTION_IO */ +#define H5D_MPIO_NO_SELECTION_IO_CAUSES \ + (H5D_SEL_IO_DISABLE_BY_API | H5D_SEL_IO_TCONV_BUF_TOO_SMALL | H5D_SEL_IO_BKG_BUF_TOO_SMALL | \ + H5D_SEL_IO_DATASET_FILTER | H5D_SEL_IO_CHUNK_CACHE) + +//! <!--[H5D_selection_io_mode_t_snip] --> +/** + * Selection I/O mode property + * + * \details The default value, #H5D_SELECTION_IO_MODE_DEFAULT, + * indicates selection I/O can be ON or OFF as + * determined by library internal. + */ +typedef enum H5D_selection_io_mode_t { + H5D_SELECTION_IO_MODE_DEFAULT = 0, + /**< Default selection I/O mode. */ + H5D_SELECTION_IO_MODE_OFF, + /**< Selection I/O is off. */ + H5D_SELECTION_IO_MODE_ON + /**< Selection I/O is on. */ +} H5D_selection_io_mode_t; +//! <!--[H5D_selection_io_mode_t_snip] --> + /********************/ /* Public Variables */ /********************/ @@ -8205,6 +8266,191 @@ H5_DLL herr_t H5Pset_dataset_io_hyperslab_selection(hid_t plist_id, unsigned ran const hsize_t count[], const hsize_t block[]); /** + * + * \ingroup DXPL + * + * \brief Sets the selection I/O mode + * + * \dxpl_id{plist_id} + * \param[in] selection_io_mode The selection I/O mode to be set + * + * \return \herr_t + * + * \details H5Pset_selection_io() sets the selection I/O mode + * \p selection_io_mode in the dataset transfer property + * list \p plist_id. + * + * This can be used to enable collective I/O with type conversion, or + * with custom VFDs that support vector or selection I/O. + * + * Values that can be set in \p selection_io_mode: + * \snippet this H5D_selection_io_mode_t_snip + * \click4more + * + * \note The library may not perform selection I/O as it asks for if the + * layout callback determines that it is not feasible to do so. Please + * refer to H5Pget_no_selection_io_cause() for details. + * + * When used with type conversion, selection I/O requires the type + * conversion buffer (and the background buffer if applicable) be large + * enough to hold the entirety of the data involved in the I/O. For + * read operations, the library will use the application's read buffer + * as the type conversion buffer if the memory type is not smaller than + * the file type, eliminating the need for a separate type conversion + * buffer (a background buffer may still be required). For write + * operations, the library will similarly use the write buffer as a + * type conversion buffer, but only if H5Pset_modify_write_buf() is + * used to allow the library to modify the contents of the write + * buffer. + * + * \since 1.14.1 + * + */ +H5_DLL herr_t H5Pset_selection_io(hid_t plist_id, H5D_selection_io_mode_t selection_io_mode); + +/** + * + * \ingroup DXPL + * + * \brief Retrieves the selection I/O mode + * + * \dxpl_id{plist_id} + * \param[out] selection_io_mode The selection I/O mode + * + * \return \herr_t + * + * \details H5Pget_selection_io() queries the selection I/O mode set in + * in the dataset transfer property list \p plist_id. + * + * Values returned in \p selection_io_mode: + * \snippet this H5D_selection_io_mode_t_snip + * \click4more + * + * \note The library may not perform selection I/O as it asks for if the + * layout callback determines that it is not feasible to do so. Please + * refer to H5Pget_no_selection_io_cause() for details. + * + * \since 1.14.1 + * + */ +H5_DLL herr_t H5Pget_selection_io(hid_t plist_id, H5D_selection_io_mode_t *selection_io_mode); + +/** + * \ingroup DXPL + * + * \brief Retrieves the cause for not performing selection or vector I/O on the + * last parallel I/O call + * + * \dxpl_id{plist_id} + * \param[out] no_selection_io_cause A bitwise set value indicating the relevant + * causes that prevented selection I/O from + * being performed + * \return \herr_t + * + * \par Motivation: + * A user can request selection I/O to be performed via a data transfer + * property list (DXPL). This can be used to enable collective I/O with + * type conversion, or with custom VFDs that support vector or selection + * I/O. However, there are conditions that can cause HDF5 to forgo + * selection or vector I/O and perform legacy (scalar) I/O instead. + * + * \details H5Pget_no_selection_io_cause() can be used to determine whether + * selection or vector I/O was applied for the last preceding I/O call. + * If selection or vector I/O was not used, this function retrieves the + * cause(s) that prevent selection or vector I/O to be performed on + * that I/O call. The properties retrieved by this function are set + * before I/O takes place and are retained even when I/O fails. + * + * If a selection I/O request falls back to vector I/O, that is not + * considered "breaking" selection I/O by this function, since vector + * I/O still passes all information to the file driver in a single + * callback. + * + * Valid values returned in \p no_selection_io_cause are listed + * as follows. If there are multiple causes, it is a bitwise OR of + * the relevant causes. + * + * - #H5D_SEL_IO_DISABLE_BY_API + * Selection I/O was not performed because the feature was disabled by the API + * - #H5D_SEL_IO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET + * Selection I/O was not performed because the dataset was neither contiguous nor chunked + * - #H5D_SEL_IO_CONTIGUOUS_SIEVE_BUFFER + * Selection I/O was not performed because of sieve buffer for contiguous dataset + * - #H5D_SEL_IO_NO_VECTOR_OR_SELECTION_IO_CB + * Selection I/O was not performed because the VFD does not have vector or selection I/O callback + * - #H5D_SEL_IO_PAGE_BUFFER + * Selection I/O was not performed because of page buffer + * - #H5D_SEL_IO_DATASET_FILTER + * Selection I/O was not performed because of dataset filters + * - #H5D_SEL_IO_CHUNK_CACHE + * Selection I/O was not performed because of chunk cache + * - #H5D_SEL_IO_TCONV_BUF_TOO_SMALL + * Selection I/O was not performed because the type conversion buffer is too small + * - #H5D_SEL_IO_BKG_BUF_TOO_SMALL + * Selection I/O was not performed because the type conversion background buffer is too small + * - #H5D_SEL_IO_DEFAULT_OFF + * Selection I/O was not performed because the selection I/O mode is DEFAULT and the library chose it + * to be off for this case + * + * \since 1.14.1 + * + */ +H5_DLL herr_t H5Pget_no_selection_io_cause(hid_t plist_id, uint32_t *no_selection_io_cause); + +/** + * + * \ingroup DXPL + * + * \brief Allows the library to modify the contents of the write buffer + * + * \dxpl_id{plist_id} + * \param[in] modify_write_buf Whether the library can modify the contents of the write buffer + * + * \return \herr_t + * + * \details H5Pset_modify_write_buf() sets whether the library is allowed to + * modify the contents of write buffers passed to HDF5 API routines + * that are passed the dataset transfer property list \p plist_id. The + * default value for modify_write_buf is FALSE. + * + * This function can be used to allow the library to perform in-place + * type conversion on write operations to save memory space. This is + * currently only used for selection I/O operations, which are used for + * collective I/O with type conversion. After making an API call with + * this parameter set to TRUE, the contents of the write buffer are + * undefined. + * + * \note When modify_write_buf is set to TRUE the library may violate the + * const qualifier on the API parameter for the write buffer. + * + * \since 1.14.1 + * + */ +H5_DLL herr_t H5Pset_modify_write_buf(hid_t plist_id, hbool_t modify_write_buf); + +/** + * + * \ingroup DXPL + * + * \brief Retrieves the "modify write buffer" property + * + * \dxpl_id{plist_id} + * \param[out] modify_write_buf Whether the library can modify the contents of the write buffer + * + * \return \herr_t + * + * \details H5Pget_modify_write_buf() gets the "modify write buffer" property + * from the dataset transfer property list \p plist_id. This property + * determines whether the library is allowed to modify the contents of + * write buffers passed to HDF5 API routines that are passed + * \p plist_id. The default value for modify_write_buf is FALSE. + * + * \since 1.14.1 + * + */ +H5_DLL herr_t H5Pget_modify_write_buf(hid_t plist_id, hbool_t *modify_write_buf); + +/** * \ingroup LCPL * * \brief Determines whether property is set to enable creating missing diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h index 19127b3..4303eee 100644 --- a/src/H5Sprivate.h +++ b/src/H5Sprivate.h @@ -260,6 +260,7 @@ H5_DLL herr_t H5S_select_project_simple(const H5S_t *space, H5S_t *new_space, H5_DLL herr_t H5S_select_project_intersection(H5S_t *src_space, H5S_t *dst_space, H5S_t *src_intersect_space, H5S_t **new_space_ptr, hbool_t share_space); H5_DLL herr_t H5S_select_subtract(H5S_t *space, H5S_t *subtract_space); +H5_DLL herr_t H5S_select_contig_block(H5S_t *space, hbool_t *is_contig, hsize_t *off, size_t *len); /* Operations on all selections */ H5_DLL herr_t H5S_select_all(H5S_t *space, hbool_t rel_prev); diff --git a/src/H5Spublic.h b/src/H5Spublic.h index 871a8e7..bd5a82c 100644 --- a/src/H5Spublic.h +++ b/src/H5Spublic.h @@ -864,9 +864,9 @@ H5_DLL hid_t H5Ssel_iter_create(hid_t spaceid, size_t elmt_size, unsigned flags) * * \space_id{sel_iter_id} * \param[in] maxseq Maximum number of sequences to retrieve - * \param[in] maxbytes Maximum number of bytes to retrieve in sequences + * \param[in] maxelmts Maximum number of elements to retrieve in sequences * \param[out] nseq Number of sequences retrieved - * \param[out] nbytes Number of bytes retrieved, in all sequences + * \param[out] nelmts Number of elements retrieved, in all sequences * \param[out] off Array of sequence offsets * \param[out] len Array of sequence lengths * @@ -883,9 +883,9 @@ H5_DLL hid_t H5Ssel_iter_create(hid_t spaceid, size_t elmt_size, unsigned flags) * #H5S_SEL_ITER_GET_SEQ_LIST_SORTED flag is passed to * H5Ssel_iter_create() for a point selection. * - * \p maxseq and \p maxbytes specify the most sequences or bytes + * \p maxseq and \p maxelmts specify the most sequences or elements * possible to place into the \p off and \p len arrays. \p nseq and - * \p nbytes return the actual number of sequences and bytes put + * \p nelmts return the actual number of sequences and elements put * into the arrays. * * Each call to H5Ssel_iter_get_seq_list() will retrieve the next @@ -897,13 +897,13 @@ H5_DLL hid_t H5Ssel_iter_create(hid_t spaceid, size_t elmt_size, unsigned flags) * the iterator was created from (which can be retrieved with * H5Sget_select_npoints(). When there are no further sequences of * elements to retrieve, calls to this routine will set \p nseq - * and \p nbytes to zero. + * and \p nelmts to zero. * * \since 1.12.0 * */ -H5_DLL herr_t H5Ssel_iter_get_seq_list(hid_t sel_iter_id, size_t maxseq, size_t maxbytes, size_t *nseq, - size_t *nbytes, hsize_t *off, size_t *len); +H5_DLL herr_t H5Ssel_iter_get_seq_list(hid_t sel_iter_id, size_t maxseq, size_t maxelmts, size_t *nseq, + size_t *nelmts, hsize_t *off, size_t *len); /** * \ingroup H5S * diff --git a/src/H5Sselect.c b/src/H5Sselect.c index 9d13cf2..02889f7 100644 --- a/src/H5Sselect.c +++ b/src/H5Sselect.c @@ -2865,9 +2865,9 @@ done: herr_t H5Ssel_iter_get_seq_list(sel_iter_id, maxseq, maxbytes, nseq, nbytes, off, len) hid_t sel_iter_id; IN: ID of the dataspace selection iterator to retrieve sequence from size_t maxseq; IN: Max. # of sequences to retrieve - size_t maxbytes; IN: Max. # of bytes to retrieve in sequences + size_t maxelmts; IN: Max. # of elements to retrieve in sequences size_t *nseq; OUT: # of sequences retrieved - size_t *nbytes; OUT: # of bytes retrieved, in all sequences + size_t *nelmts; OUT: # of elements retrieved, in all sequences hsize_t *off; OUT: Array of sequence offsets size_t *len; OUT: Array of sequence lengths RETURNS @@ -2882,8 +2882,8 @@ done: selections is "in order selected", unless the H5S_SEL_ITER_GET_SEQ_LIST_SORTED flag is passed to H5Sset_iter_create for a point selection. - MAXSEQ and MAXBYTES specify the most sequences or bytes possible to - place into the OFF and LEN arrays. *NSEQ and *NBYTES return the actual + MAXSEQ and MAXELMTS specify the most sequences or bytes possible to + place into the OFF and LEN arrays. *NSEQ and *NELMTS return the actual number of sequences and bytes put into the arrays. Each call to H5Ssel_iter_get_seq_list() will retrieve the next set @@ -2894,7 +2894,7 @@ done: of elements selected in the dataspace the iterator was created from (which can be retrieved with H5Sget_select_npoints). When there are no further sequences of elements to retrieve, calls to this routine will - set *NSEQ and *NBYTES to zero. + set *NSEQ and *NELMTS to zero. PROGRAMMER Quincey Koziol - February 11, 2019 GLOBAL VARIABLES @@ -2903,21 +2903,21 @@ done: REVISION LOG --------------------------------------------------------------------------*/ herr_t -H5Ssel_iter_get_seq_list(hid_t sel_iter_id, size_t maxseq, size_t maxbytes, size_t *nseq /*out*/, - size_t *nbytes /*out*/, hsize_t *off /*out*/, size_t *len /*out*/) +H5Ssel_iter_get_seq_list(hid_t sel_iter_id, size_t maxseq, size_t maxelmts, size_t *nseq /*out*/, + size_t *nelmts /*out*/, hsize_t *off /*out*/, size_t *len /*out*/) { H5S_sel_iter_t *sel_iter; /* Dataspace selection iterator to operate on */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE7("e", "izzxxxx", sel_iter_id, maxseq, maxbytes, nseq, nbytes, off, len); + H5TRACE7("e", "izzxxxx", sel_iter_id, maxseq, maxelmts, nseq, nelmts, off, len); /* Check args */ if (NULL == (sel_iter = (H5S_sel_iter_t *)H5I_object_verify(sel_iter_id, H5I_SPACE_SEL_ITER))) HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "not a dataspace selection iterator") if (NULL == nseq) HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "'nseq' pointer is NULL") - if (NULL == nbytes) + if (NULL == nelmts) HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "'nbytes' pointer is NULL") if (NULL == off) HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "offset array pointer is NULL") @@ -2925,12 +2925,12 @@ H5Ssel_iter_get_seq_list(hid_t sel_iter_id, size_t maxseq, size_t maxbytes, size HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "length array pointer is NULL") /* Get the sequences of bytes */ - if (maxseq > 0 && maxbytes > 0 && sel_iter->elmt_left > 0) { - if (H5S_SELECT_ITER_GET_SEQ_LIST(sel_iter, maxseq, maxbytes, nseq, nbytes, off, len) < 0) + if (maxseq > 0 && maxelmts > 0 && sel_iter->elmt_left > 0) { + if (H5S_SELECT_ITER_GET_SEQ_LIST(sel_iter, maxseq, maxelmts, nseq, nelmts, off, len) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "sequence length generation failed") } /* end if */ else - *nseq = *nbytes = 0; + *nseq = *nelmts = 0; done: FUNC_LEAVE_API(ret_value) @@ -2938,6 +2938,83 @@ done: /*-------------------------------------------------------------------------- NAME + H5S_select_contig_block + + PURPOSE + Determines if a selection is a single contiguous block, and returns the + offset and length (in elements) if it is + + USAGE + herr_t H5S_select_contig_block(space, is_contig, off, len) + H5S_t *space; IN: Selection to check + hbool_t *is_contig; OUT: Whether the selection is contiguous + hsize_t *off; OUT: Offset of selection + size_t *len; OUT: Length of selection + + RETURNS + Non-negative on success/Negative on failure. + + DESCRIPTION + Determines if a selection is a single contiguous block, and returns the + offset and length (in elements) if it is. + + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +herr_t +H5S_select_contig_block(H5S_t *space, hbool_t *is_contig, hsize_t *off, size_t *len) +{ + H5S_sel_iter_t *iter = NULL; /* Selection iterator */ + hbool_t iter_init = FALSE; /* Selection iteration info has been initialized */ + size_t nseq_tmp; + size_t nelem_tmp; + hsize_t sel_off; + size_t sel_len; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(space); + + /* Allocate and initialize the iterator */ + if (NULL == (iter = H5FL_MALLOC(H5S_sel_iter_t))) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate iterator") + if (H5S_select_iter_init(iter, space, 1, 0) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to initialize memory selection information") + iter_init = TRUE; + + /* Get list of sequences for selection, to check if it is contiguous */ + if (H5S_SELECT_ITER_GET_SEQ_LIST(iter, (size_t)1, (size_t)-1, &nseq_tmp, &nelem_tmp, &sel_off, &sel_len) < + 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTNEXT, FAIL, "sequence length generation failed") + + /* If the first sequence includes all the elements selected in this piece, it it contiguous */ + H5_CHECK_OVERFLOW(space->select.num_elem, hsize_t, size_t); + if (sel_len == (size_t)space->select.num_elem) { + if (is_contig) + *is_contig = TRUE; + if (off) + *off = sel_off; + if (len) + *len = sel_len; + } + else if (is_contig) + *is_contig = FALSE; + +done: + if (iter_init && H5S_SELECT_ITER_RELEASE(iter) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't release selection iterator") + if (iter) + iter = H5FL_FREE(H5S_sel_iter_t, iter); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5S_select_contig_block() */ + +/*-------------------------------------------------------------------------- + NAME H5Ssel_iter_reset PURPOSE Resets a dataspace selection iterator back to an initial state. diff --git a/src/H5private.h b/src/H5private.h index 70aed8f..eac2cba 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -1965,14 +1965,6 @@ extern hbool_t H5_libterm_g; /* Is the library being shutdown? */ #endif /* H5_HAVE_THREADSAFE */ -/* Extern global to determine if we should use selection I/O if available (this - * variable should be removed once selection I/O performs as well as the - * previous scalar I/O implementation - * - * NOTE: Must be exposed via H5_DLLVAR so parallel tests pass on Windows. - */ -H5_DLLVAR hbool_t H5_use_selection_io_g; - #ifdef H5_HAVE_CODESTACK /* Include required function stack header */ diff --git a/src/H5trace.c b/src/H5trace.c index 03eaf11..71b66a8 100644 --- a/src/H5trace.c +++ b/src/H5trace.c @@ -673,6 +673,31 @@ H5_trace_args(H5RS_str_t *rs, const char *type, va_list ap) } /* end block */ break; + case 'C': /* H5D_selection_io_mode_t */ + { + H5D_selection_io_mode_t selection_io_mode = + (H5D_selection_io_mode_t)HDva_arg(ap, int); + + switch (selection_io_mode) { + case H5D_SELECTION_IO_MODE_DEFAULT: + H5RS_acat(rs, "H5D_SELECTION_IO_MODE_DEFAULT"); + break; + + case H5D_SELECTION_IO_MODE_OFF: + H5RS_acat(rs, "H5D_SELECTION_IO_MODE_OFF"); + break; + + case H5D_SELECTION_IO_MODE_ON: + H5RS_acat(rs, "H5D_SELECTION_IO_MODE_ON"); + break; + + default: + H5RS_asprintf_cat(rs, "%ld", (long)selection_io_mode); + break; + } /* end switch */ + } /* end block */ + break; + case 'f': /* H5D_fill_time_t */ { H5D_fill_time_t fill_time = (H5D_fill_time_t)HDva_arg(ap, int); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index da98f15..c3365b7 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -337,6 +337,7 @@ set (H5_TESTS page_buffer dtypes dsets + select_io_dset chunk_info # compression lib link cmpd_dset mdset diff --git a/test/Makefile.am b/test/Makefile.am index bad52c8..291907c 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -65,7 +65,7 @@ TEST_PROG= testhdf5 \ accum hyperslab istore bittests dt_arith page_buffer \ dtypes dsets chunk_info cmpd_dset mdset cmpd_dtransform filter_fail extend direct_chunk \ external efc objcopy objcopy_ref links unlink twriteorder big mtime \ - fillval mount \ + fillval mount select_io_dset\ flush1 flush2 app_ref enum set_extent ttsafe enc_dec_plist \ enc_dec_plist_cross_platform getname vfd ros3 s3comms hdfs ntypes \ dangle dtransform reserved cross_read freespace mf vds file_image \ diff --git a/test/enc_dec_plist.c b/test/enc_dec_plist.c index fd4ae5e..5b75178 100644 --- a/test/enc_dec_plist.c +++ b/test/enc_dec_plist.c @@ -367,6 +367,12 @@ main(void) if ((H5Pset_data_transform(dxpl, c_to_f)) < 0) FAIL_STACK_ERROR; + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_ON) < 0) + FAIL_STACK_ERROR; + + if (H5Pset_modify_write_buf(dxpl, TRUE) < 0) + FAIL_STACK_ERROR; + /* Test encoding & decoding property list */ if (test_encode_decode(dxpl, low, high) < 0) FAIL_PUTS_ERROR("DXPL encoding/decoding failed\n"); diff --git a/test/gen_plist.c b/test/gen_plist.c index afe8494..5302dc7 100644 --- a/test/gen_plist.c +++ b/test/gen_plist.c @@ -183,6 +183,11 @@ main(void) assert(ret > 0); if ((ret = H5Pset_data_transform(dxpl1, c_to_f)) < 0) assert(ret > 0); + if ((ret = H5Pset_selection_io(dxpl1, H5D_SELECTION_IO_MODE_ON)) < 0) + assert(ret > 0); + + if ((ret = H5Pset_modify_write_buf(dxpl1, TRUE)) < 0) + assert(ret > 0); if ((ret = encode_plist(dxpl1, little_endian, word_length, "testfiles/plist_files/dxpl_")) < 0) assert(ret > 0); diff --git a/test/select_io_dset.c b/test/select_io_dset.c new file mode 100644 index 0000000..0724365 --- /dev/null +++ b/test/select_io_dset.c @@ -0,0 +1,3280 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Programmer: + * + * Purpose: Tests selection IO for the dataset interface (H5D) + */ + +#include "testhdf5.h" +#include "H5srcdir.h" + +const char *FILENAME[] = {"select_io", /* 0 */ + NULL}; + +#define FILENAME_BUF_SIZE 1024 + +/* + * Test configurations + */ +typedef enum { + TEST_NO_TYPE_CONV, /* no type conversion (null case) */ + TEST_NO_SIZE_CHANGE_NO_BKG, /* no size change, no bkg buffer */ + TEST_LARGER_MEM_NO_BKG, /* larger memory type, no bkg buffer */ + TEST_SMALLER_MEM_NO_BKG, /* smaller memory type, no bkg buffer */ + TEST_CMPD_WITH_BKG, /* compound types with bkg buffer */ + TEST_MULTI_CONV_NO_BKG, /* multi dataset test 1 */ + TEST_MULTI_CONV_BKG, /* multi dataset test 2 */ + TEST_MULTI_CONV_SIZE_CHANGE, /* multi dataset test 3 */ + TEST_MULTI_ALL, /* multi dataset test 4 */ + TEST_SELECT_NTESTS +} test_select_config_t; + +#define DSET_SELECT_DIM 100 +#define DSET_SELECT_CHUNK_DIM 10 + +#define MULTI_NUM_DSETS 3 +#define MULTI_MIN_DSETS 3 +#define DSET_NAME_LEN 64 + +/* Compound type */ +typedef struct s1_t { + int a; + int b; + int c; + int d; +} s1_t; + +/* + * Variation of s1 with: + * --no conversion for 2 member types + * --1 larger mem type, + * --1 smaller mem type + */ +typedef struct s2_t { + int a; + long b; + int c; + short d; +} s2_t; + +/* Variation of s1: reverse of s1_t */ +typedef struct s3_t { + int d; + int c; + int b; + int a; +} s3_t; + +/* Variations of s1: only 2 members in s1_t */ +typedef struct s4_t { + unsigned int b; + unsigned int d; +} s4_t; + +/* Defines for test_multi_dsets_all() */ +typedef enum { + DSET_WITH_NO_CONV, /* Dataset with no type conversion */ + DSET_WITH_CONV_AND_NO_BKG, /* Dataset with type conversion but no background buffer */ + DSET_WITH_CONV_AND_BKG, /* Dataset with type conversion and background buffer */ + DSET_NTTYPES +} multi_dset_type_t; + +/* Test setting A and B */ +#define SETTING_A 1 +#define SETTING_B 2 + +/* Definitions of the test modes for test_get_no_selection_io_cause() */ +#define TEST_DISABLE_BY_API 0x001 +#define TEST_DATATYPE_CONVERSION 0x002 +#define TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET 0x004 +#define TEST_CONTIGUOUS_SIEVE_BUFFER 0x008 +#define TEST_NO_VECTOR_OR_SELECTION_IO_CB 0x010 +#define TEST_PAGE_BUFFER 0x020 +#define TEST_DATASET_FILTER 0x040 +#define TEST_CHUNK_CACHE 0x080 +#define TEST_TCONV_BUF_TOO_SMALL 0x100 +#define TEST_IN_PLACE_TCONV 0x200 + +/* + * Case 1: single dataset read/write, no type conversion (null case) + * --create dataset with H5T_NATIVE_INT + * --write/read dataset with H5T_NATIVE_INT + */ +static herr_t +test_no_type_conv(hid_t fid, unsigned chunked, unsigned dtrans, unsigned mwbuf) +{ + int i; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t ntrans_dxpl = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + int wbuf[DSET_SELECT_DIM]; + int wbuf_bak[DSET_SELECT_DIM]; + int trans_wbuf[DSET_SELECT_DIM]; + int rbuf[DSET_SELECT_DIM]; + char dset_name[DSET_NAME_LEN]; + const char *expr = "2*x"; + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + FAIL_STACK_ERROR; + } + + /* Generate dataset name */ + HDsnprintf(dset_name, sizeof(dset_name), "no_tconv_%s_%s_%s", chunked ? "chunked" : "contig", + dtrans ? "xform" : "noxform", mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create dataset */ + if ((did = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* Initialize data */ + for (i = 0; i < DSET_SELECT_DIM; i++) { + wbuf[i] = i; + trans_wbuf[i] = 2 * wbuf[i]; + } + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + FAIL_STACK_ERROR; + + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_ON) < 0) + FAIL_STACK_ERROR; + + /* Set modify write buffer if requested */ + if (mwbuf) + if (H5Pset_modify_write_buf(dxpl, TRUE) < 0) + FAIL_STACK_ERROR; + + if ((ntrans_dxpl = H5Pcopy(dxpl)) < 0) + FAIL_STACK_ERROR; + + /* Set data transform */ + if (dtrans) + if (H5Pset_data_transform(dxpl, expr) < 0) + FAIL_STACK_ERROR; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(wbuf_bak, wbuf, sizeof(wbuf)); + + /* Write data to the dataset with/without data transform */ + if (H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, dxpl, wbuf) < 0) + FAIL_STACK_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(wbuf, wbuf_bak, sizeof(wbuf)); + + /* Read data from the dataset without data transform set in dxpl */ + if (H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, ntrans_dxpl, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify data or transformed data read */ + for (i = 0; i < DSET_SELECT_DIM; i++) + if (rbuf[i] != (dtrans ? trans_wbuf[i] : wbuf[i])) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" At index %d\n", i); + TEST_ERROR; + } + + if (dtrans) { + + /* Read the data from the dataset with data transform set in dxpl */ + if (H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, dxpl, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify data read is transformed a second time */ + for (i = 0; i < DSET_SELECT_DIM; i++) + if (rbuf[i] != (2 * trans_wbuf[i])) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" At index %d\n", i); + TEST_ERROR; + } + } + + if (H5Sclose(sid) < 0) + FAIL_STACK_ERROR; + if (H5Dclose(did) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dxpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(ntrans_dxpl) < 0) + FAIL_STACK_ERROR; + + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Sclose(sid); + H5Dclose(did); + H5Pclose(dcpl); + H5Pclose(dxpl); + H5Pclose(ntrans_dxpl); + } + H5E_END_TRY; + + return FAIL; + +} /* test_no_type_conv() */ + +/* + * Case 2: single dataset read/write, no size change, no background buffer + * --create dataset with H5T_STD_I32BE + * --write/read dataset with H5T_STD_I32LE + * --read again with H5T_STD_I32BE + */ +static herr_t +test_no_size_change_no_bkg(hid_t fid, unsigned chunked, unsigned mwbuf) +{ + int i; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + char *wbuf = NULL; + char *wbuf_bak = NULL; + char *rbuf = NULL; + char dset_name[DSET_NAME_LEN]; + + if ((wbuf = (char *)HDmalloc((size_t)(4 * DSET_SELECT_DIM))) == NULL) + FAIL_STACK_ERROR; + if (mwbuf && (wbuf_bak = (char *)HDmalloc((size_t)(4 * DSET_SELECT_DIM))) == NULL) + FAIL_STACK_ERROR; + if ((rbuf = (char *)HDmalloc((size_t)(4 * DSET_SELECT_DIM))) == NULL) + FAIL_STACK_ERROR; + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + FAIL_STACK_ERROR; + + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_ON) < 0) + FAIL_STACK_ERROR; + + /* Set modify write buffer if requested */ + if (mwbuf) + if (H5Pset_modify_write_buf(dxpl, TRUE) < 0) + FAIL_STACK_ERROR; + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + FAIL_STACK_ERROR; + } + + /* Generate dataset name */ + HDsnprintf(dset_name, sizeof(dset_name), "no_size_change_%s_%s", chunked ? "chunked" : "contig", + mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create 1d dataset */ + if ((did = H5Dcreate2(fid, dset_name, H5T_STD_I32BE, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* Initialize data */ + for (i = 0; i < DSET_SELECT_DIM; i++) { + wbuf[i * 4 + 3] = 0x1; + wbuf[i * 4 + 2] = 0x2; + wbuf[i * 4 + 1] = 0x3; + wbuf[i * 4 + 0] = 0x4; + } + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(wbuf_bak, wbuf, (size_t)(4 * DSET_SELECT_DIM)); + + /* Write the data to the dataset with little endian */ + if (H5Dwrite(did, H5T_STD_I32LE, H5S_ALL, H5S_ALL, dxpl, wbuf) < 0) + FAIL_STACK_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(wbuf, wbuf_bak, (size_t)(4 * DSET_SELECT_DIM)); + + /* Read the data from the dataset with little endian */ + if (H5Dread(did, H5T_STD_I32LE, H5S_ALL, H5S_ALL, dxpl, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify data read little endian */ + for (i = 0; i < DSET_SELECT_DIM; i++) + if (rbuf[4 * i + 0] != wbuf[4 * i + 0] || rbuf[4 * i + 1] != wbuf[4 * i + 1] || + rbuf[4 * i + 2] != wbuf[4 * i + 2] || rbuf[4 * i + 3] != wbuf[4 * i + 3]) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" At index %d\n", i); + TEST_ERROR; + } + + /* Read the data from the dataset with big endian */ + if (H5Dread(did, H5T_STD_I32BE, H5S_ALL, H5S_ALL, dxpl, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify data read in big endian */ + for (i = 0; i < DSET_SELECT_DIM; i++) + if (rbuf[4 * i + 0] != wbuf[4 * i + 3] || rbuf[4 * i + 1] != wbuf[4 * i + 2] || + rbuf[4 * i + 2] != wbuf[4 * i + 1] || rbuf[4 * i + 3] != wbuf[4 * i + 0]) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" At index %d\n", i); + TEST_ERROR; + } + + if (H5Sclose(sid) < 0) + FAIL_STACK_ERROR; + if (H5Dclose(did) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dxpl) < 0) + FAIL_STACK_ERROR; + + HDfree(wbuf); + HDfree(wbuf_bak); + HDfree(rbuf); + + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Sclose(sid); + H5Dclose(did); + H5Pclose(dcpl); + H5Pclose(dxpl); + } + H5E_END_TRY; + + if (wbuf) + HDfree(wbuf); + if (wbuf_bak) + HDfree(wbuf_bak); + if (wbuf) + HDfree(rbuf); + + return FAIL; + +} /* test_no_size_change_no_bkg() */ + +/* + * Case 3: single dataset read/write, larger mem type, no background buffer + * --create dataset with H5T_NATIVE_INT + * --write dataset with H5T_NATIVE_LONG + * --read dataset with H5T_NATIVE_LLONG + * + */ +static herr_t +test_larger_mem_type_no_bkg(hid_t fid, unsigned chunked, unsigned dtrans, unsigned mwbuf) +{ + int i; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t ntrans_dxpl = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + long wbuf[DSET_SELECT_DIM]; + long wbuf_bak[DSET_SELECT_DIM]; + long trans_wbuf[DSET_SELECT_DIM]; + long long rbuf[DSET_SELECT_DIM]; + char dset_name[DSET_NAME_LEN]; + const char *expr = "5 * (10 - x)"; + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + FAIL_STACK_ERROR; + } + + /* Generate dataset name */ + HDsnprintf(dset_name, sizeof(dset_name), "larger_no_bkg_%s_%s_%s", chunked ? "chunked" : "contig", + dtrans ? "xform" : "noxform", mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create dataset */ + if ((did = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* Initialize data */ + for (i = 0; i < DSET_SELECT_DIM; i++) { + wbuf[i] = i; + trans_wbuf[i] = 5 * (10 - wbuf[i]); + } + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + FAIL_STACK_ERROR; + + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_ON) < 0) + FAIL_STACK_ERROR; + + /* Set modify write buffer if requested */ + if (mwbuf) + if (H5Pset_modify_write_buf(dxpl, TRUE) < 0) + FAIL_STACK_ERROR; + + if ((ntrans_dxpl = H5Pcopy(dxpl)) < 0) + FAIL_STACK_ERROR; + + /* Set data transform */ + if (dtrans) + if (H5Pset_data_transform(dxpl, expr) < 0) + FAIL_STACK_ERROR; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(wbuf_bak, wbuf, sizeof(wbuf)); + + /* Write data to the dataset with/without data transform set in dxpl */ + if (H5Dwrite(did, H5T_NATIVE_LONG, H5S_ALL, H5S_ALL, dxpl, wbuf) < 0) + FAIL_STACK_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(wbuf, wbuf_bak, sizeof(wbuf)); + + /* Read the data from the dataset without data transform in dxpl */ + if (H5Dread(did, H5T_NATIVE_LLONG, H5S_ALL, H5S_ALL, ntrans_dxpl, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify data or transformed data read */ + for (i = 0; i < DSET_SELECT_DIM; i++) + if (rbuf[i] != (long long)(dtrans ? trans_wbuf[i] : wbuf[i])) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" At index %d\n", i); + TEST_ERROR; + } + + if (dtrans) { + + /* Read data from the dataset with data transform set in dxpl */ + if (H5Dread(did, H5T_NATIVE_LLONG, H5S_ALL, H5S_ALL, dxpl, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify data read is transformed a second time */ + for (i = 0; i < DSET_SELECT_DIM; i++) + if (rbuf[i] != (long long)(5 * (10 - trans_wbuf[i]))) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" At index %d\n", i); + TEST_ERROR; + } + } + + if (H5Sclose(sid) < 0) + FAIL_STACK_ERROR; + if (H5Dclose(did) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dxpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(ntrans_dxpl) < 0) + FAIL_STACK_ERROR; + + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Sclose(sid); + H5Dclose(did); + H5Pclose(dcpl); + H5Pclose(dxpl); + H5Pclose(ntrans_dxpl); + } + H5E_END_TRY; + + return FAIL; + +} /* test_larger_mem_type_no_bkg() */ + +/* + * Case 4: single dataset reader/write, smaller mem type, no background buffer + * --create dataset with H5T_NATIVE_INT + * --write dataset with H5T_NATIVE_SHORT + * --read dataset with H5T_NATIVE_SHORT + */ +static herr_t +test_smaller_mem_type_no_bkg(hid_t fid, unsigned chunked, unsigned dtrans, unsigned mwbuf) +{ + int i; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t ntrans_dxpl = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + short wbuf[DSET_SELECT_DIM]; + int wbuf_bak[DSET_SELECT_DIM]; + short trans_wbuf[DSET_SELECT_DIM]; + short rbuf[DSET_SELECT_DIM]; + char dset_name[DSET_NAME_LEN]; + const char *expr = "2 * (10 + x)"; + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + FAIL_STACK_ERROR; + } + + /* Generate dataset name */ + HDsnprintf(dset_name, sizeof(dset_name), "smaller_no_bkg_%s_%s_%s", chunked ? "chunked" : "contig", + dtrans ? "xform" : "noxform", mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create 1d chunked dataset with/without data transform */ + if ((did = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* Initialize data */ + for (i = 0; i < DSET_SELECT_DIM; i++) { + wbuf[i] = (short)i; + trans_wbuf[i] = (short)(2 * (10 + wbuf[i])); + } + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + FAIL_STACK_ERROR; + + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_ON) < 0) + FAIL_STACK_ERROR; + + /* Set modify write buffer if requested */ + if (mwbuf) + if (H5Pset_modify_write_buf(dxpl, TRUE) < 0) + FAIL_STACK_ERROR; + + if ((ntrans_dxpl = H5Pcopy(dxpl)) < 0) + FAIL_STACK_ERROR; + + /* Set data transform */ + if (dtrans) { + if (H5Pset_data_transform(dxpl, expr) < 0) + FAIL_STACK_ERROR; + } + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(wbuf_bak, wbuf, sizeof(wbuf)); + + /* Write data to the dataset with/without data transform in dxpl */ + if (H5Dwrite(did, H5T_NATIVE_SHORT, H5S_ALL, H5S_ALL, dxpl, wbuf) < 0) + FAIL_STACK_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(wbuf, wbuf_bak, sizeof(wbuf)); + + /* Read data from the dataset without data transform in dxpl */ + if (H5Dread(did, H5T_NATIVE_SHORT, H5S_ALL, H5S_ALL, ntrans_dxpl, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify data or transformed data read */ + for (i = 0; i < DSET_SELECT_DIM; i++) + if (rbuf[i] != (dtrans ? trans_wbuf[i] : wbuf[i])) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" At index %d\n", i); + TEST_ERROR; + } + + if (dtrans) { + + /* Read data from the dataset with data transform set in dxpl */ + if (H5Dread(did, H5T_NATIVE_SHORT, H5S_ALL, H5S_ALL, dxpl, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify data read is transformed a second time */ + for (i = 0; i < DSET_SELECT_DIM; i++) + if (rbuf[i] != (2 * (10 + trans_wbuf[i]))) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" At index %d\n", i); + TEST_ERROR; + } + } + + if (H5Sclose(sid) < 0) + FAIL_STACK_ERROR; + if (H5Dclose(did) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dxpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(ntrans_dxpl) < 0) + FAIL_STACK_ERROR; + + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Sclose(sid); + H5Dclose(did); + H5Pclose(dcpl); + H5Pclose(dxpl); + H5Pclose(ntrans_dxpl); + } + H5E_END_TRY; + + return FAIL; + +} /* test_smaller_mem_type_no_bkg() */ + +/* + * Case 5: single dataset reade/write, compound types with background buffer + * + * (a) Initialize compound buffer in memory with unique values + * Write all compound fields to disk + * Verify values read + * (b) Update all fields of the compound type in memory write buffer with new unique values + * Write some but not all all compound fields to disk + * Read the entire compound type + * Verify the fields have the correct (old or new) values + * (c) Update all fields of the compound type in memory read buffer with new unique values + * Read some but not all the compound fields to memory + * Verify the fields have the correct (old, middle or new) values + * (d) Set up a different compound type which has: + * --no conversion for member types + * --a field with larger mem type + * --a field with smaller mem type + * Write this compound type to disk + * Read the entire compound type + * Verify the values read + * + */ +static herr_t +test_cmpd_with_bkg(hid_t fid, unsigned chunked, unsigned mwbuf) +{ + int i; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t s1_tid = H5I_INVALID_HID; + hid_t s2_tid = H5I_INVALID_HID; + hid_t ss_ac_tid = H5I_INVALID_HID; + hid_t ss_bc_tid = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + s1_t *s1_wbuf = NULL; + s1_t *s1_wbuf_bak = NULL; + s1_t *s1_rbuf = NULL; + s2_t *s2_wbuf = NULL; + s2_t *s2_wbuf_bak = NULL; + s2_t *s2_rbuf = NULL; + char dset_name[DSET_NAME_LEN]; + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + FAIL_STACK_ERROR; + + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_ON) < 0) + FAIL_STACK_ERROR; + + /* Set modify write buffer if requested */ + if (mwbuf) + if (H5Pset_modify_write_buf(dxpl, TRUE) < 0) + FAIL_STACK_ERROR; + + /* Allocate buffers for datasets */ + if (NULL == (s1_wbuf = (s1_t *)HDmalloc(sizeof(s1_t) * DSET_SELECT_DIM))) + FAIL_STACK_ERROR; + if (mwbuf && NULL == (s1_wbuf_bak = (s1_t *)HDmalloc(sizeof(s1_t) * DSET_SELECT_DIM))) + FAIL_STACK_ERROR; + if (NULL == (s1_rbuf = (s1_t *)HDmalloc(sizeof(s1_t) * DSET_SELECT_DIM))) + FAIL_STACK_ERROR; + if (NULL == (s2_wbuf = (s2_t *)HDmalloc(sizeof(s2_t) * DSET_SELECT_DIM))) + FAIL_STACK_ERROR; + if (mwbuf && NULL == (s2_wbuf_bak = (s2_t *)HDmalloc(sizeof(s2_t) * DSET_SELECT_DIM))) + FAIL_STACK_ERROR; + if (NULL == (s2_rbuf = (s2_t *)HDmalloc(sizeof(s2_t) * DSET_SELECT_DIM))) + FAIL_STACK_ERROR; + + /* Create the memory data type */ + if ((s1_tid = H5Tcreate(H5T_COMPOUND, sizeof(s1_t))) < 0) + FAIL_STACK_ERROR; + + if (H5Tinsert(s1_tid, "a", HOFFSET(s1_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "b", HOFFSET(s1_t, b), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "c", HOFFSET(s1_t, c), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "d", HOFFSET(s1_t, d), H5T_NATIVE_INT) < 0) + FAIL_STACK_ERROR; + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + FAIL_STACK_ERROR; + } + + /* Case 5(a) */ + + /* Generate dataset name */ + HDsnprintf(dset_name, sizeof(dset_name), "cmpd_with_bkg_%s_%s", chunked ? "chunked" : "contig", + mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create 1d dataset */ + if ((did = H5Dcreate2(fid, dset_name, s1_tid, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* Initialize data */ + for (i = 0; i < DSET_SELECT_DIM; i++) { + s1_wbuf[i].a = 4 * i; + s1_wbuf[i].b = (4 * i) + 1; + s1_wbuf[i].c = (4 * i) + 2; + s1_wbuf[i].d = (4 * i) + 3; + } + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(s1_wbuf_bak, s1_wbuf, sizeof(s1_t) * DSET_SELECT_DIM); + + /* Write all the data to the dataset */ + if (H5Dwrite(did, s1_tid, H5S_ALL, H5S_ALL, dxpl, s1_wbuf) < 0) + FAIL_STACK_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(s1_wbuf, s1_wbuf_bak, sizeof(s1_t) * DSET_SELECT_DIM); + + /* Read all the data from the dataset */ + if (H5Dread(did, s1_tid, H5S_ALL, H5S_ALL, dxpl, s1_rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify data read */ + for (i = 0; i < DSET_SELECT_DIM; i++) { + if (s1_wbuf[i].a != s1_rbuf[i].a || s1_wbuf[i].b != s1_rbuf[i].b || s1_wbuf[i].c != s1_rbuf[i].c || + s1_wbuf[i].d != s1_rbuf[i].d) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" At index %d\n", i); + TEST_ERROR; + } + } + + /* Case 5(b) */ + + /* Update s1_wbuf with unique values */ + for (i = 0; i < DSET_SELECT_DIM; i++) { + s1_wbuf[i].a = (4 * i) + DSET_SELECT_DIM; + s1_wbuf[i].b = (4 * i) + DSET_SELECT_DIM + 1; + s1_wbuf[i].c = (4 * i) + DSET_SELECT_DIM + 2; + s1_wbuf[i].d = (4 * i) + DSET_SELECT_DIM + 3; + } + + /* Create a compound type same size as s1_t */ + if ((ss_ac_tid = H5Tcreate(H5T_COMPOUND, sizeof(s1_t))) < 0) + FAIL_STACK_ERROR; + + /* but contains only subset members of s1_t */ + if (H5Tinsert(ss_ac_tid, "a", HOFFSET(s1_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(ss_ac_tid, "c", HOFFSET(s1_t, c), H5T_NATIVE_INT) < 0) + FAIL_STACK_ERROR; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(s1_wbuf_bak, s1_wbuf, sizeof(s1_t) * DSET_SELECT_DIM); + + /* Write s1_wbuf to the dataset with only subset members in ss_tid */ + if (H5Dwrite(did, ss_ac_tid, H5S_ALL, H5S_ALL, dxpl, s1_wbuf) < 0) + FAIL_STACK_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(s1_wbuf, s1_wbuf_bak, sizeof(s1_t) * DSET_SELECT_DIM); + + /* Read the whole compound back */ + if (H5Dread(did, ss_ac_tid, H5S_ALL, H5S_ALL, dxpl, s1_rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify the compound fields have the correct (old or new) values */ + for (i = 0; i < DSET_SELECT_DIM; i++) { + if (s1_rbuf[i].a != s1_wbuf[i].a || s1_rbuf[i].b != ((4 * i) + 1) || s1_rbuf[i].c != s1_wbuf[i].c || + s1_rbuf[i].d != ((4 * i) + 3)) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" At index %d\n", i); + TEST_ERROR; + } + } + + /* Case 5(c) */ + + /* Update s1_rbuf with new unique values */ + for (i = 0; i < DSET_SELECT_DIM; i++) { + s1_rbuf[i].a = (4 * i) + (2 * DSET_SELECT_DIM); + s1_rbuf[i].b = (4 * i) + (2 * DSET_SELECT_DIM) + 1; + s1_rbuf[i].c = (4 * i) + (2 * DSET_SELECT_DIM) + 2; + s1_rbuf[i].d = (4 * i) + (2 * DSET_SELECT_DIM) + 3; + } + + /* Create a compound type same size as s1_t */ + if ((ss_bc_tid = H5Tcreate(H5T_COMPOUND, sizeof(s1_t))) < 0) + FAIL_STACK_ERROR; + + /* but contains only subset members of s1_t */ + if (H5Tinsert(ss_bc_tid, "b", HOFFSET(s1_t, b), H5T_NATIVE_INT) < 0 || + H5Tinsert(ss_bc_tid, "c", HOFFSET(s1_t, c), H5T_NATIVE_INT) < 0) + FAIL_STACK_ERROR; + + /* Read the dataset: will read only what is set in */ + if (H5Dread(did, ss_bc_tid, H5S_ALL, H5S_ALL, dxpl, s1_rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify data read */ + for (i = 0; i < DSET_SELECT_DIM; i++) { + if (s1_rbuf[i].a != ((4 * i) + (2 * DSET_SELECT_DIM)) || s1_rbuf[i].b != ((4 * i) + 1) || + s1_rbuf[i].c != ((4 * i) + DSET_SELECT_DIM + 2) || + s1_rbuf[i].d != ((4 * i) + (2 * DSET_SELECT_DIM) + 3)) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" At index %d\n", i); + TEST_ERROR; + } + } + + /* Case 5(d) */ + + /* Create s2_t compound type with: + * --no conversion for 2 member types, + * --1 larger mem type + * --1 smaller mem type + */ + if ((s2_tid = H5Tcreate(H5T_COMPOUND, sizeof(s2_t))) < 0) + FAIL_STACK_ERROR; + + if (H5Tinsert(s2_tid, "a", HOFFSET(s2_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(s2_tid, "b", HOFFSET(s2_t, b), H5T_NATIVE_LONG) < 0 || + H5Tinsert(s2_tid, "c", HOFFSET(s2_t, c), H5T_NATIVE_INT) < 0 || + H5Tinsert(s2_tid, "d", HOFFSET(s2_t, d), H5T_NATIVE_SHORT) < 0) + FAIL_STACK_ERROR; + + /* Update s2_wbuf with unique values */ + for (i = 0; i < DSET_SELECT_DIM; i++) { + s2_wbuf[i].a = 8 * i; + s2_wbuf[i].b = (long)((8 * i) + 1); + s2_wbuf[i].c = (8 * i) + 2; + s2_wbuf[i].d = (short)((8 * i) + 3); + } + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(s2_wbuf_bak, s2_wbuf, sizeof(s2_t) * DSET_SELECT_DIM); + + if (H5Dwrite(did, s2_tid, H5S_ALL, H5S_ALL, dxpl, s2_wbuf) < 0) + FAIL_STACK_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(s2_wbuf, s2_wbuf_bak, sizeof(s2_t) * DSET_SELECT_DIM); + + /* Read it back */ + if (H5Dread(did, s2_tid, H5S_ALL, H5S_ALL, dxpl, s2_rbuf) < 0) { + goto error; + } + + /* Verify data read */ + for (i = 0; i < DSET_SELECT_DIM; i++) { + if (s2_wbuf[i].a != s2_rbuf[i].a || s2_wbuf[i].b != s2_rbuf[i].b || s2_wbuf[i].c != s2_rbuf[i].c || + s2_wbuf[i].d != s2_rbuf[i].d) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" At index %d\n", i); + TEST_ERROR; + } + } + + if (H5Sclose(sid) < 0) + FAIL_STACK_ERROR; + if (H5Tclose(s1_tid) < 0) + FAIL_STACK_ERROR; + if (H5Tclose(s2_tid) < 0) + FAIL_STACK_ERROR; + if (H5Tclose(ss_ac_tid) < 0) + FAIL_STACK_ERROR; + if (H5Tclose(ss_bc_tid) < 0) + FAIL_STACK_ERROR; + if (H5Dclose(did) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dxpl) < 0) + FAIL_STACK_ERROR; + + /* Release buffers */ + HDfree(s1_wbuf); + HDfree(s1_wbuf_bak); + HDfree(s1_rbuf); + HDfree(s2_wbuf); + HDfree(s2_wbuf_bak); + HDfree(s2_rbuf); + + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Sclose(sid); + H5Tclose(s1_tid); + H5Tclose(s2_tid); + H5Tclose(ss_ac_tid); + H5Tclose(ss_bc_tid); + H5Dclose(did); + H5Pclose(dcpl); + H5Pclose(dxpl); + } + H5E_END_TRY; + + if (s1_wbuf) + HDfree(s1_wbuf); + if (s1_wbuf_bak) + HDfree(s1_wbuf_bak); + if (s1_rbuf) + HDfree(s1_rbuf); + if (s2_wbuf) + HDfree(s2_wbuf); + if (s2_wbuf_bak) + HDfree(s2_wbuf_bak); + if (s2_rbuf) + HDfree(s2_rbuf); + return FAIL; + +} /* test_cmpd_with_bkg() */ + +/* + * Test 1 for multi-dataset: + * --Datasets with/without type conversion+smaller/larger mem type+no background buffer + * + * Create datasets: randomized H5T_NATIVE_INT or H5T_NATIVE_LONG + * + * Case a--setting for multi write/read to ndsets: + * Datatype for all datasets: H5T_NATIVE_INT + * + * Case b--setting for multi write/read to ndsets: + * Datatype for all datasets: H5T_NATIVE_LONG + */ +static herr_t +test_multi_dsets_no_bkg(hid_t fid, unsigned chunked, unsigned dtrans, unsigned mwbuf) +{ + size_t ndsets; + int i, j; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t ntrans_dxpl = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + + hid_t file_sids[MULTI_NUM_DSETS]; + hid_t mem_sids[MULTI_NUM_DSETS]; + hid_t mem_tids[MULTI_NUM_DSETS]; + + char dset_names[MULTI_NUM_DSETS][DSET_NAME_LEN]; + hid_t dset_dids[MULTI_NUM_DSETS]; + + size_t buf_size; + int *total_wbuf = NULL; + int *total_wbuf_bak = NULL; + int *total_trans_wbuf = NULL; + int *total_rbuf = NULL; + long *total_lwbuf = NULL; + long *total_lwbuf_bak = NULL; + long *total_trans_lwbuf = NULL; + long *total_lrbuf = NULL; + + int *wbufi[MULTI_NUM_DSETS]; + int *trans_wbufi[MULTI_NUM_DSETS]; + int *rbufi[MULTI_NUM_DSETS]; + + long *lwbufi[MULTI_NUM_DSETS]; + long *trans_lwbufi[MULTI_NUM_DSETS]; + long *lrbufi[MULTI_NUM_DSETS]; + + const void *wbufs[MULTI_NUM_DSETS]; + void *rbufs[MULTI_NUM_DSETS]; + const char *expr = "2*x"; + + ndsets = MAX(MULTI_MIN_DSETS, MULTI_NUM_DSETS); + + dims[0] = DSET_SELECT_DIM; + + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + FAIL_STACK_ERROR; + } + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + FAIL_STACK_ERROR; + + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_ON) < 0) + FAIL_STACK_ERROR; + + /* Set modify write buffer if requested */ + if (mwbuf) + if (H5Pset_modify_write_buf(dxpl, TRUE) < 0) + FAIL_STACK_ERROR; + + if ((ntrans_dxpl = H5Pcopy(dxpl)) < 0) + FAIL_STACK_ERROR; + + /* Set data transform */ + if (dtrans) + if (H5Pset_data_transform(dxpl, expr) < 0) + FAIL_STACK_ERROR; + + /* Set up file space ids, mem space ids, and dataset ids */ + for (i = 0; i < (int)ndsets; i++) { + if ((file_sids[i] = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + + if ((mem_sids[i] = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + + /* Generate dataset name */ + HDsnprintf(dset_names[i], sizeof(dset_names[i]), "multi_dset%d_%s_%s_%s", i, + chunked ? "chunked" : "contig", dtrans ? "xform" : "noxform", mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create ith dataset */ + if ((dset_dids[i] = + H5Dcreate2(fid, dset_names[i], ((HDrandom() % 2) ? H5T_NATIVE_LONG : H5T_NATIVE_INT), + file_sids[i], H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + } + + buf_size = ndsets * DSET_SELECT_DIM * sizeof(int); + + /* Allocate buffers for all datasets */ + if (NULL == (total_wbuf = (int *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (mwbuf && NULL == (total_wbuf_bak = (int *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (NULL == (total_trans_wbuf = (int *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (NULL == (total_rbuf = (int *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + + buf_size = ndsets * DSET_SELECT_DIM * sizeof(long); + + if (NULL == (total_lwbuf = (long *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (mwbuf && NULL == (total_lwbuf_bak = (long *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (NULL == (total_trans_lwbuf = (long *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (NULL == (total_lrbuf = (long *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + + /* Initialize buffer indices */ + for (i = 0; i < (int)ndsets; i++) { + wbufi[i] = total_wbuf + (i * DSET_SELECT_DIM); + trans_wbufi[i] = total_trans_wbuf + (i * DSET_SELECT_DIM); + rbufi[i] = total_rbuf + (i * DSET_SELECT_DIM); + + wbufs[i] = wbufi[i]; + rbufs[i] = rbufi[i]; + } + + /* Initialize the buffer data */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < DSET_SELECT_DIM; j++) { + wbufi[i][j] = (int)j; + trans_wbufi[i][j] = 2 * wbufi[i][j]; + } + + /* Case a */ + + /* Datatype setting for multi write/read */ + for (i = 0; i < (int)ndsets; i++) + mem_tids[i] = H5T_NATIVE_INT; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_wbuf_bak, total_wbuf, ndsets * DSET_SELECT_DIM * sizeof(int)); + + /* Write data to the dataset with/without data transform */ + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_wbuf, total_wbuf_bak, ndsets * DSET_SELECT_DIM * sizeof(int)); + + /* Read data from the dataset (if dtrans, without data transform set in dxpl) */ + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, ntrans_dxpl, rbufs) < 0) + TEST_ERROR; + + /* Verify */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < DSET_SELECT_DIM; j++) + if (rbufi[i][j] != (dtrans ? trans_wbufi[i][j] : wbufi[i][j])) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + TEST_ERROR; + } + + if (dtrans) { + + /* Read the data from the dataset with data transform set in dxpl */ + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + TEST_ERROR; + + /* Verify */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < DSET_SELECT_DIM; j++) + if (rbufi[i][j] != (2 * trans_wbufi[i][j])) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + TEST_ERROR; + } + } + + /* Case b */ + + /* Datatype setting for multi write/read */ + for (i = 0; i < (int)ndsets; i++) + mem_tids[i] = H5T_NATIVE_LONG; + + /* Initialize buffer indices */ + for (i = 0; i < (int)ndsets; i++) { + lwbufi[i] = total_lwbuf + (i * DSET_SELECT_DIM); + trans_lwbufi[i] = total_trans_lwbuf + (i * DSET_SELECT_DIM); + lrbufi[i] = total_lrbuf + (i * DSET_SELECT_DIM); + wbufs[i] = lwbufi[i]; + rbufs[i] = lrbufi[i]; + } + + /* Initialize the buffer data */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < DSET_SELECT_DIM; j++) { + lwbufi[i][j] = (long)j + DSET_SELECT_DIM; + trans_lwbufi[i][j] = 2 * lwbufi[i][j]; + } + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_lwbuf_bak, total_lwbuf, ndsets * DSET_SELECT_DIM * sizeof(long)); + + /* Write data to the dataset with/without data transform */ + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_lwbuf, total_lwbuf_bak, ndsets * DSET_SELECT_DIM * sizeof(long)); + + /* Read data from the dataset (if dtrans, with data transform again in dxpl) */ + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + TEST_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + for (j = 0; j < DSET_SELECT_DIM; j++) { + if (lrbufi[i][j] != (dtrans ? (2 * trans_lwbufi[i][j]) : lwbufi[i][j])) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + TEST_ERROR; + } + } + } + + if (H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dxpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(ntrans_dxpl) < 0) + FAIL_STACK_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + if (H5Sclose(file_sids[i]) < 0) + FAIL_STACK_ERROR; + if (H5Sclose(mem_sids[i]) < 0) + FAIL_STACK_ERROR; + if (H5Dclose(dset_dids[i]) < 0) + FAIL_STACK_ERROR; + } + + HDfree(total_wbuf); + HDfree(total_wbuf_bak); + HDfree(total_rbuf); + HDfree(total_trans_wbuf); + HDfree(total_lwbuf); + HDfree(total_lwbuf_bak); + HDfree(total_lrbuf); + HDfree(total_trans_lwbuf); + + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + H5Pclose(dcpl); + H5Pclose(dxpl); + H5Pclose(ntrans_dxpl); + for (i = 0; i < (int)ndsets; i++) { + H5Sclose(file_sids[i]); + H5Sclose(mem_sids[i]); + H5Dclose(dset_dids[i]); + } + H5E_END_TRY; + + if (total_wbuf) + HDfree(total_wbuf); + if (total_wbuf_bak) + HDfree(total_wbuf_bak); + if (total_trans_wbuf) + HDfree(total_trans_wbuf); + if (total_rbuf) + HDfree(total_rbuf); + if (total_lwbuf) + HDfree(total_lwbuf); + if (total_lwbuf_bak) + HDfree(total_lwbuf_bak); + if (total_lrbuf) + HDfree(total_lrbuf); + if (total_trans_lwbuf) + HDfree(total_lrbuf); + + return FAIL; + +} /* test_multi_dsets_no_bkg() */ + +/* + * Test 2 for multi-dataset: + * + * Datasets with compound types+background buffer + * + * Create datasets with the same compound type + * (a) Initialize compound buffer in memory with unique values + * All datasets: + * --Write all compound fields to disk + * --Read the entire compound type for all datasets + * --Verify values read + * (b) Update all fields of the compound type in memory write buffer with new unique values + * dset0: + * --Write some but not all all compound fields to disk + * --Read and verify the fields have the correct (old(a) or new) values + * Remaining datasets: + * --Untouched + * --Read and verify the fields have the correct old(a) values + * (c) Update all fields of the compound type in memory read buffer with new unique values + * Randomized <mm> dataset: + * --Read some but not all the compound fields to memory + * --Verify the fields have the correct (old(a) or new) values + * dset0: + * --Untouched + * --Read and verify the fields have the correct (old(a) or middle(b)) values + * Remaining datasets: + * --Untouched + * --Read and verify the fields have the correct old(a) values + * (d) Set up a different compound type which has: + * --no type conversion for 2 member types + * --a field with larger mem type + * --a field with smaller mem type + * All datasets: + * --Write the compound fields to disk + * --Read the entire compound type + * --Verify values read + */ +static herr_t +test_multi_dsets_cmpd_with_bkg(hid_t fid, unsigned chunked, unsigned mwbuf) +{ + size_t ndsets; + int i, j, mm; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + + hid_t file_sids[MULTI_NUM_DSETS]; + hid_t mem_sids[MULTI_NUM_DSETS]; + hid_t mem_tids[MULTI_NUM_DSETS]; + + hid_t s1_tid = H5I_INVALID_HID; + hid_t ss_ac_tid = H5I_INVALID_HID; + hid_t ss_bc_tid = H5I_INVALID_HID; + hid_t s2_tid = H5I_INVALID_HID; + + char dset_names[MULTI_NUM_DSETS][DSET_NAME_LEN]; + hid_t dset_dids[MULTI_NUM_DSETS]; + + size_t buf_size; + size_t s2_buf_size; + + s1_t *total_wbuf = NULL; + s1_t *total_wbuf_bak = NULL; + s1_t *total_rbuf = NULL; + + s2_t *s2_total_wbuf = NULL; + s2_t *s2_total_wbuf_bak = NULL; + s2_t *s2_total_rbuf = NULL; + + s1_t *wbufi[MULTI_NUM_DSETS]; + s1_t *rbufi[MULTI_NUM_DSETS]; + + s2_t *s2_wbufi[MULTI_NUM_DSETS]; + s2_t *s2_rbufi[MULTI_NUM_DSETS]; + + const void *wbufs[MULTI_NUM_DSETS]; + void *rbufs[MULTI_NUM_DSETS]; + + ndsets = MAX(MULTI_MIN_DSETS, MULTI_NUM_DSETS); + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + FAIL_STACK_ERROR; + + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_ON) < 0) + FAIL_STACK_ERROR; + + /* Set modify write buffer if requested */ + if (mwbuf) + if (H5Pset_modify_write_buf(dxpl, TRUE) < 0) + FAIL_STACK_ERROR; + + dims[0] = DSET_SELECT_DIM; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + FAIL_STACK_ERROR; + } + + /* Create the memory data type */ + if ((s1_tid = H5Tcreate(H5T_COMPOUND, sizeof(s1_t))) < 0) + FAIL_STACK_ERROR; + + if (H5Tinsert(s1_tid, "a", HOFFSET(s1_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "b", HOFFSET(s1_t, b), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "c", HOFFSET(s1_t, c), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "d", HOFFSET(s1_t, d), H5T_NATIVE_INT) < 0) + FAIL_STACK_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + if ((file_sids[i] = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + if ((mem_sids[i] = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + + /* Generate dataset name */ + HDsnprintf(dset_names[i], sizeof(dset_names[i]), "multi_cmpd_dset%d_%s_%s", i, + chunked ? "chunked" : "contig", mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create ith dataset */ + if ((dset_dids[i] = + H5Dcreate2(fid, dset_names[i], s1_tid, file_sids[i], H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + } + + buf_size = ndsets * DSET_SELECT_DIM * sizeof(s1_t); + s2_buf_size = ndsets * DSET_SELECT_DIM * sizeof(s2_t); + + /* Allocate buffers */ + if (NULL == (total_wbuf = (s1_t *)HDmalloc(buf_size))) + TEST_ERROR; + if (mwbuf && NULL == (total_wbuf_bak = (s1_t *)HDmalloc(buf_size))) + TEST_ERROR; + if (NULL == (total_rbuf = (s1_t *)HDmalloc(buf_size))) + TEST_ERROR; + + if (NULL == (s2_total_wbuf = (s2_t *)HDmalloc(s2_buf_size))) + TEST_ERROR; + if (mwbuf && NULL == (s2_total_wbuf_bak = (s2_t *)HDmalloc(s2_buf_size))) + TEST_ERROR; + if (NULL == (s2_total_rbuf = (s2_t *)HDmalloc(s2_buf_size))) + TEST_ERROR; + + /* Initialize buffer indices */ + for (i = 0; i < (int)ndsets; i++) { + wbufi[i] = total_wbuf + (i * DSET_SELECT_DIM); + rbufi[i] = total_rbuf + (i * DSET_SELECT_DIM); + + wbufs[i] = wbufi[i]; + rbufs[i] = rbufi[i]; + } + + /* Case a */ + + /* Initialize the buffer data for all the datasets */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < DSET_SELECT_DIM; j++) { + wbufi[i][j].a = (4 * j); + wbufi[i][j].b = (4 * j) + 1; + wbufi[i][j].c = (4 * j) + 2; + wbufi[i][j].d = (4 * j) + 3; + } + + /* Datatype setting for multi write */ + for (i = 0; i < (int)ndsets; i++) + mem_tids[i] = s1_tid; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_wbuf_bak, total_wbuf, buf_size); + + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_wbuf, total_wbuf_bak, buf_size); + + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + TEST_ERROR; + + /* Verify data read */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < DSET_SELECT_DIM; j++) { + if (wbufi[i][j].a != rbufi[i][j].a || wbufi[i][j].b != rbufi[i][j].b || + wbufi[i][j].c != rbufi[i][j].c || wbufi[i][j].d != rbufi[i][j].d) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + TEST_ERROR; + } + } + + /* Case b */ + + /* Update data in wbufi for dset0 with unique values */ + for (j = 0; j < DSET_SELECT_DIM; j++) { + wbufi[0][j].a = (4 * j) + DSET_SELECT_DIM; + wbufi[0][j].b = (4 * j) + DSET_SELECT_DIM + 1; + wbufi[0][j].c = (4 * j) + DSET_SELECT_DIM + 2; + wbufi[0][j].d = (4 * j) + DSET_SELECT_DIM + 3; + } + + /* Create a compound type same size as s1_t */ + if ((ss_ac_tid = H5Tcreate(H5T_COMPOUND, sizeof(s1_t))) < 0) + FAIL_STACK_ERROR; + + /* but contains only subset members of s1_t */ + if (H5Tinsert(ss_ac_tid, "a", HOFFSET(s1_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(ss_ac_tid, "c", HOFFSET(s1_t, c), H5T_NATIVE_INT) < 0) + FAIL_STACK_ERROR; + + /* Untouched memory and file spaces for other datasets */ + for (i = 0; i < (int)ndsets; i++) { + if (i == 0) + continue; + + if (H5Sselect_none(mem_sids[i]) < 0) + TEST_ERROR; + if (H5Sselect_none(file_sids[i]) < 0) + TEST_ERROR; + } + + /* Datatype setting for write to dset0 */ + mem_tids[0] = ss_ac_tid; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_wbuf_bak, total_wbuf, buf_size); + + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_wbuf, total_wbuf_bak, buf_size); + + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + TEST_ERROR; + + /* Verify data read */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < DSET_SELECT_DIM; j++) + if (i == 0) { /* dset0 */ + if (wbufi[i][j].a != rbufi[i][j].a || ((4 * (int)j) + 1) != rbufi[i][j].b || + wbufi[i][j].c != rbufi[i][j].c || ((4 * (int)j) + 3) != rbufi[i][j].d) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + TEST_ERROR; + } + } + else { /* other datasets */ + for (j = 0; j < DSET_SELECT_DIM; j++) + if ((4 * (int)j) != rbufi[i][j].a || ((4 * (int)j) + 1) != rbufi[i][j].b || + ((4 * (int)j) + 2) != rbufi[i][j].c || ((4 * (int)j) + 3) != rbufi[i][j].d) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + TEST_ERROR; + } + } + + /* Case c */ + mm = HDrandom() % (int)ndsets; + if (!mm) + mm++; + + /* Update data in rbufi for dset1 with new unique values */ + for (j = 0; j < DSET_SELECT_DIM; j++) { + rbufi[mm][j].a = (4 * j) + (2 * DSET_SELECT_DIM); + rbufi[mm][j].b = (4 * j) + (2 * DSET_SELECT_DIM) + 1; + rbufi[mm][j].c = (4 * j) + (2 * DSET_SELECT_DIM) + 2; + rbufi[mm][j].d = (4 * j) + (2 * DSET_SELECT_DIM) + 3; + } + + /* Create a compound type same size as s1_t */ + if ((ss_bc_tid = H5Tcreate(H5T_COMPOUND, sizeof(s1_t))) < 0) + FAIL_STACK_ERROR; + + /* but contains only subset members of s1_t */ + if (H5Tinsert(ss_bc_tid, "b", HOFFSET(s1_t, b), H5T_NATIVE_INT) < 0 || + H5Tinsert(ss_bc_tid, "c", HOFFSET(s1_t, c), H5T_NATIVE_INT) < 0) + FAIL_STACK_ERROR; + + /* Reset memory and file space for <mm> dataset */ + if (H5Sselect_all(mem_sids[mm]) < 0) + FAIL_STACK_ERROR; + if (H5Sselect_all(file_sids[mm]) < 0) + FAIL_STACK_ERROR; + + /* Untouched memory and file space for other datasets */ + for (i = 0; i < (int)ndsets; i++) { + if (i == 0 || i == mm) + continue; + if (H5Sselect_none(mem_sids[i]) < 0) + TEST_ERROR; + if (H5Sselect_none(file_sids[i]) < 0) + TEST_ERROR; + } + + /* Datatype setting for read from <mm> dataset */ + mem_tids[mm] = ss_bc_tid; + + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + TEST_ERROR; + + /* Verify data read */ + /* dset0 */ + for (j = 0; j < DSET_SELECT_DIM; j++) + if (wbufi[0][j].a != rbufi[0][j].a || ((4 * (int)j) + 1) != rbufi[0][j].b || + wbufi[0][j].c != rbufi[0][j].c || ((4 * (int)j) + 3) != rbufi[0][j].d) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset0 at index %d\n", j); + TEST_ERROR; + } + + /* <mm> dset */ + for (j = 0; j < DSET_SELECT_DIM; j++) + if (rbufi[mm][j].a != ((4 * (int)j) + (2 * DSET_SELECT_DIM)) || + rbufi[mm][j].b != ((4 * (int)j) + 1) || rbufi[mm][j].c != ((4 * (int)j) + 2) || + rbufi[mm][j].d != ((4 * (int)j) + (2 * DSET_SELECT_DIM) + 3)) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset1 at index %d\n", j); + TEST_ERROR; + } + + /* other datasets */ + for (i = 0; i < (int)ndsets; i++) { + if (i == 0 || i == mm) + continue; + + for (j = 0; j < DSET_SELECT_DIM; j++) + if (rbufi[i][j].a != (4 * (int)j) || rbufi[i][j].b != ((4 * (int)j) + 1) || + rbufi[i][j].c != ((4 * (int)j) + 2) || rbufi[i][j].d != ((4 * (int)j) + 3)) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + TEST_ERROR; + } + } + + /* Case d */ + + /* Create s2_t compound type with: + * --no conversion for 2 member types, + * --1 larger mem type + * --1 smaller mem type + */ + if ((s2_tid = H5Tcreate(H5T_COMPOUND, sizeof(s2_t))) < 0) + FAIL_STACK_ERROR; + + if (H5Tinsert(s2_tid, "a", HOFFSET(s2_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(s2_tid, "b", HOFFSET(s2_t, b), H5T_NATIVE_LONG) < 0 || + H5Tinsert(s2_tid, "c", HOFFSET(s2_t, c), H5T_NATIVE_INT) < 0 || + H5Tinsert(s2_tid, "d", HOFFSET(s2_t, d), H5T_NATIVE_SHORT) < 0) + FAIL_STACK_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + s2_wbufi[i] = s2_total_wbuf + (i * DSET_SELECT_DIM); + s2_rbufi[i] = s2_total_rbuf + (i * DSET_SELECT_DIM); + + wbufs[i] = s2_wbufi[i]; + rbufs[i] = s2_rbufi[i]; + + mem_tids[i] = s2_tid; + + if (H5Sselect_all(mem_sids[i]) < 0) + TEST_ERROR; + if (H5Sselect_all(file_sids[i]) < 0) + TEST_ERROR; + } + + /* Initialize the buffer data for all the datasets */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < DSET_SELECT_DIM; j++) { + s2_wbufi[i][j].a = 8 * j; + s2_wbufi[i][j].b = (long)((8 * j) + 1); + s2_wbufi[i][j].c = (8 * j) + 2; + s2_wbufi[i][j].d = (short)((8 * j) + 3); + } + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(s2_total_wbuf_bak, s2_total_wbuf, s2_buf_size); + + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(s2_total_wbuf, s2_total_wbuf_bak, s2_buf_size); + + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + TEST_ERROR; + + /* Verify data read */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < DSET_SELECT_DIM; j++) + if (s2_wbufi[i][j].a != s2_rbufi[i][j].a || s2_wbufi[i][j].b != s2_rbufi[i][j].b || + s2_wbufi[i][j].c != s2_rbufi[i][j].c || s2_wbufi[i][j].d != s2_rbufi[i][j].d) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + TEST_ERROR; + } + + if (H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dxpl) < 0) + FAIL_STACK_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + if (H5Sclose(file_sids[i]) < 0) + FAIL_STACK_ERROR; + if (H5Sclose(mem_sids[i]) < 0) + FAIL_STACK_ERROR; + if (H5Dclose(dset_dids[i]) < 0) + FAIL_STACK_ERROR; + } + + HDfree(total_wbuf); + HDfree(total_wbuf_bak); + HDfree(total_rbuf); + HDfree(s2_total_wbuf); + HDfree(s2_total_wbuf_bak); + HDfree(s2_total_rbuf); + + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + H5Pclose(dcpl); + H5Pclose(dxpl); + for (i = 0; i < (int)ndsets; i++) { + H5Sclose(file_sids[i]); + H5Sclose(mem_sids[i]); + H5Dclose(dset_dids[i]); + } + H5E_END_TRY; + + if (total_wbuf) + HDfree(total_wbuf); + if (total_wbuf_bak) + HDfree(total_wbuf_bak); + if (total_rbuf) + HDfree(total_rbuf); + if (s2_total_wbuf) + HDfree(s2_total_wbuf); + if (s2_total_wbuf_bak) + HDfree(s2_total_wbuf_bak); + if (s2_total_rbuf) + HDfree(s2_total_rbuf); + + return FAIL; + +} /* test_multi_dsets_cmpd_with_bkg() */ + +/* + * Test 3 for multi-dataset: + * --Datasets with/without type conv+size change+no background buffer + * + * Create dset0: H5T_STD_I32BE + * Create other dateasets: randomized H5T_STD_I64LE or H5T_STD_I16LE + * + * Case a--setting for write/read to ndsets: + * Datatype for all datasets: H5T_STD_I32BE + * + * Case b--setting for write/read to ndsets + * Datatype for all datasets: H5T_STD_I64BE + * + * Case c--setting for write/read to ndsets + * Datatype for all datasets: H5T_STD_I16BE + */ +static herr_t +test_multi_dsets_size_change_no_bkg(hid_t fid, unsigned chunked, unsigned mwbuf) +{ + size_t ndsets; + int i, j; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + + hid_t file_sids[MULTI_NUM_DSETS]; + hid_t mem_sids[MULTI_NUM_DSETS]; + hid_t mem_tids[MULTI_NUM_DSETS]; + + char dset_names[MULTI_NUM_DSETS][DSET_NAME_LEN]; + hid_t dset_dids[MULTI_NUM_DSETS]; + + size_t buf_size, ss; + uint8_t *total_wbuf = NULL; + uint8_t *total_wbuf_bak = NULL; + uint8_t *total_rbuf = NULL; + uint8_t *total_lwbuf = NULL; + uint8_t *total_lwbuf_bak = NULL; + uint8_t *total_lrbuf = NULL; + uint8_t *total_swbuf = NULL; + uint8_t *total_swbuf_bak = NULL; + uint8_t *total_srbuf = NULL; + + uint8_t *wbufi[MULTI_NUM_DSETS]; + uint8_t *rbufi[MULTI_NUM_DSETS]; + uint8_t *lwbufi[MULTI_NUM_DSETS]; + uint8_t *lrbufi[MULTI_NUM_DSETS]; + uint8_t *swbufi[MULTI_NUM_DSETS]; + uint8_t *srbufi[MULTI_NUM_DSETS]; + + const void *wbufs[MULTI_NUM_DSETS]; + void *rbufs[MULTI_NUM_DSETS]; + + ndsets = MAX(MULTI_MIN_DSETS, MULTI_NUM_DSETS); + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + FAIL_STACK_ERROR; + + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_ON) < 0) + FAIL_STACK_ERROR; + + /* Set modify write buffer if requested */ + if (mwbuf) + if (H5Pset_modify_write_buf(dxpl, TRUE) < 0) + FAIL_STACK_ERROR; + + dims[0] = DSET_SELECT_DIM; + + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + FAIL_STACK_ERROR; + } + + /* Set up file space ids, mem space ids, and dataset ids */ + for (i = 0; i < (int)ndsets; i++) { + if ((file_sids[i] = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + + if ((mem_sids[i] = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + + /* Generate dataset name */ + HDsnprintf(dset_names[i], sizeof(dset_names[i]), "multi_size_dset%d_%s_%s", i, + chunked ? "chunked" : "contig", mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create ith dataset */ + if ((dset_dids[i] = H5Dcreate2(fid, dset_names[i], H5T_STD_I32BE, file_sids[i], H5P_DEFAULT, dcpl, + H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + } + + /* Case a */ + + ss = H5Tget_size(H5T_STD_I32BE); + buf_size = ndsets * ss * DSET_SELECT_DIM; + + /* Allocate buffers for all datasets */ + if (NULL == (total_wbuf = (uint8_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (NULL == (total_wbuf_bak = (uint8_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (NULL == (total_rbuf = (uint8_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + + /* Initialize buffer indices */ + for (i = 0; i < (int)ndsets; i++) { + wbufi[i] = total_wbuf + (i * (int)ss * DSET_SELECT_DIM); + rbufi[i] = total_rbuf + (i * (int)ss * DSET_SELECT_DIM); + + wbufs[i] = wbufi[i]; + rbufs[i] = rbufi[i]; + } + + /* Initialize the buffer data: big endian */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < DSET_SELECT_DIM; j++) { + wbufi[i][j * (int)ss + 0] = 0x1; + wbufi[i][j * (int)ss + 1] = 0x2; + wbufi[i][j * (int)ss + 2] = 0x3; + wbufi[i][j * (int)ss + 3] = (uint8_t)(0x4 + j); + } + + /* Datatype setting for multi write/read */ + for (i = 0; i < (int)ndsets; i++) + mem_tids[i] = H5T_STD_I32BE; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_wbuf_bak, total_wbuf, buf_size); + + /* Write data to the dataset */ + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_wbuf, total_wbuf_bak, buf_size); + + /* Read data from the dataset */ + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + TEST_ERROR; + + /* Verify */ + for (i = 0; i < (int)ndsets; i++) + /* Only compare when it's at least the size of H5T_STD_I32BE */ + if (H5Tget_size(H5Dget_type(dset_dids[i])) >= ss) { + for (j = 0; j < DSET_SELECT_DIM; j++) + if (rbufi[i][(int)ss * j + 0] != wbufi[i][(int)ss * j + 0] || + rbufi[i][(int)ss * j + 1] != wbufi[i][(int)ss * j + 1] || + rbufi[i][(int)ss * j + 2] != wbufi[i][(int)ss * j + 2] || + rbufi[i][(int)ss * j + 3] != wbufi[i][(int)ss * j + 3]) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + TEST_ERROR; + } + } + + /* Case b */ + + ss = H5Tget_size(H5T_STD_I64BE); + buf_size = ndsets * (ss * DSET_SELECT_DIM); + + /* Allocate buffers for all datasets */ + if (NULL == (total_lwbuf = (uint8_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (NULL == (total_lwbuf_bak = (uint8_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (NULL == (total_lrbuf = (uint8_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + + /* Initialize buffer indices */ + for (i = 0; i < (int)ndsets; i++) { + lwbufi[i] = total_lwbuf + (i * (int)ss * DSET_SELECT_DIM); + lrbufi[i] = total_lrbuf + (i * (int)ss * DSET_SELECT_DIM); + + wbufs[i] = lwbufi[i]; + rbufs[i] = lrbufi[i]; + } + + /* Initialize the buffer data: big endian */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < DSET_SELECT_DIM; j++) { + lwbufi[i][j * (int)ss + 0] = 0x1; + lwbufi[i][j * (int)ss + 1] = 0x2; + lwbufi[i][j * (int)ss + 2] = 0x3; + lwbufi[i][j * (int)ss + 3] = 0x4; + lwbufi[i][j * (int)ss + 4] = 0x5; + lwbufi[i][j * (int)ss + 5] = 0x6; + lwbufi[i][j * (int)ss + 6] = 0x7; + lwbufi[i][j * (int)ss + 7] = (uint8_t)(0x8 + j); + } + + /* Datatype setting for multi write/read */ + for (i = 0; i < (int)ndsets; i++) + mem_tids[i] = H5T_STD_I64BE; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_lwbuf_bak, total_lwbuf, buf_size); + + /* Write data to the dataset */ + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_lwbuf, total_lwbuf_bak, buf_size); + + /* Read data from the dataset */ + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + TEST_ERROR; + + /* Verify */ + for (i = 0; i < (int)ndsets; i++) + /* Only compare when it's the size of H5T_STD_I64BE */ + if (H5Tget_size(H5Dget_type(dset_dids[i])) >= ss) { + for (j = 0; j < DSET_SELECT_DIM; j++) + if (lrbufi[i][(int)ss * j + 0] != lwbufi[i][(int)ss * j + 0] || + lrbufi[i][(int)ss * j + 1] != lwbufi[i][(int)ss * j + 1] || + lrbufi[i][(int)ss * j + 2] != lwbufi[i][(int)ss * j + 2] || + lrbufi[i][(int)ss * j + 3] != lwbufi[i][(int)ss * j + 3] || + lrbufi[i][(int)ss * j + 4] != lwbufi[i][(int)ss * j + 4] || + lrbufi[i][(int)ss * j + 5] != lwbufi[i][(int)ss * j + 5] || + lrbufi[i][(int)ss * j + 6] != lwbufi[i][(int)ss * j + 6] || + lrbufi[i][(int)ss * j + 7] != lwbufi[i][(int)ss * j + 7]) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + TEST_ERROR; + } + } + + /* Case c */ + + ss = H5Tget_size(H5T_STD_I16BE); + buf_size = ndsets * (ss * DSET_SELECT_DIM); + + /* Allocate buffers for all datasets */ + if (NULL == (total_swbuf = (uint8_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (NULL == (total_swbuf_bak = (uint8_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (NULL == (total_srbuf = (uint8_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + + /* Initialize buffer indices */ + for (i = 0; i < (int)ndsets; i++) { + swbufi[i] = total_swbuf + (i * (int)ss * DSET_SELECT_DIM); + srbufi[i] = total_srbuf + (i * (int)ss * DSET_SELECT_DIM); + + wbufs[i] = swbufi[i]; + rbufs[i] = srbufi[i]; + } + + /* Initialize the buffer data: big endian */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < DSET_SELECT_DIM; j++) { + swbufi[i][j * (int)ss + 0] = 0x1; + swbufi[i][j * (int)ss + 1] = (uint8_t)(0x2 + j); + } + + /* Datatype setting for multi write/read */ + for (i = 0; i < (int)ndsets; i++) + mem_tids[i] = H5T_STD_I16BE; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_swbuf_bak, total_swbuf, buf_size); + + /* Write data to the dataset */ + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_swbuf, total_swbuf_bak, buf_size); + + /* Read data from the dataset */ + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + TEST_ERROR; + + /* Verify */ + for (i = 0; i < (int)ndsets; i++) + /* Can compare for all cases */ + for (j = 0; j < DSET_SELECT_DIM; j++) + if (srbufi[i][(int)ss * j + 0] != swbufi[i][(int)ss * j + 0] || + srbufi[i][(int)ss * j + 1] != swbufi[i][(int)ss * j + 1]) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + TEST_ERROR; + } + + if (H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dxpl) < 0) + FAIL_STACK_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + if (H5Sclose(file_sids[i]) < 0) + FAIL_STACK_ERROR; + if (H5Sclose(mem_sids[i]) < 0) + FAIL_STACK_ERROR; + if (H5Dclose(dset_dids[i]) < 0) + FAIL_STACK_ERROR; + } + + HDfree(total_wbuf); + HDfree(total_wbuf_bak); + HDfree(total_rbuf); + HDfree(total_lwbuf); + HDfree(total_lwbuf_bak); + HDfree(total_lrbuf); + HDfree(total_swbuf); + HDfree(total_swbuf_bak); + HDfree(total_srbuf); + + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + H5Pclose(dcpl); + H5Pclose(dxpl); + for (i = 0; i < (int)ndsets; i++) { + H5Sclose(file_sids[i]); + H5Sclose(mem_sids[i]); + H5Dclose(dset_dids[i]); + } + H5E_END_TRY; + + if (total_wbuf) + HDfree(total_wbuf); + if (total_wbuf_bak) + HDfree(total_wbuf_bak); + if (total_rbuf) + HDfree(total_rbuf); + if (total_lwbuf) + HDfree(total_lwbuf); + if (total_lwbuf_bak) + HDfree(total_lwbuf_bak); + if (total_lrbuf) + HDfree(total_lrbuf); + if (total_swbuf) + HDfree(total_swbuf); + if (total_swbuf_bak) + HDfree(total_swbuf_bak); + if (total_srbuf) + HDfree(total_srbuf); + + return FAIL; + +} /* test_multi_dsets_size_change_no_bkg() */ + +/* + * Test 4 for multi-dataset: + * + * Repeat the following test for niter times to ensure the + * random combinations of all dataset types are hit. + * + * Create randomized contiguous or chunked datasets with: + * --DSET_WITH_NO_CONV: + * --with no type conversion + * --dataset with H5T_NATIVE_INT + * --DSET_WITH_CONV_AND_NO_BKG: + * --type conversion without background buffer + * --dataset with H5T_NATIVE_LONG + * --DSET_WITH_CONV_AND_BKG: + * --type conversion with background buffer + * --dataset with compound type s1_t + * + * Do H5Dwrite_multi() and H5Dread_multi() for the above datasets: + * Setting A: + * --DSET_WITH_NO_CONV: + * --write: mem_tids[] = H5T_NATIVE_INT + * --read: r_mem_tids[] = H5T_NATIVE_INT + * --DSET_WITH_CONV_AND_NO_BKG: + * --write: mem_tids[] = H5T_NATIVE_ULONG + * --read: r_mem_tids[] = H5T_NATIVE_LONG + * --DSET_WITH_CONV_AND_BKG: + * --write: mem_tids[] = s1_tid; + * --read: r_mem_tids[i] = s3_tid; + * + * Setting B: + * --DSET_WITH_NO_CONV: + * --write: mem_tids[] = H5T_NATIVE_INT + * --read: r_mem_tids[] = H5T_NATIVE_INT + * --DSET_WITH_CONV_AND_NO_BKG: + * --write: mem_tids[] = H5T_NATIVE_LONG; + * --read: r_mem_tids[] = H5T_NATIVE_SHORT; + * --DSET_WITH_CONV_AND_BKG: + * --write: mem_tids[] = s4_tid; + * --read: r_mem_tids[i] = s1_tid; + * + * Verify the result read as below: + * Setting A: + * --DSET_WITH_NO_CONV: + * --verify data read in rbufi1[i][j] is same as wbufi1[i][j] + * --DSET_WITH_CONV_AND_NO_BKG: + * --verify data read in l_rbufi2[i][j] is all LONG_MAX + * --DSET_WITH_CONV_AND_BKG: + * --verify all fields read in s3_rbufi3[i][j] is the + * reverse of s1_wbufi3[i][j] + * Setting B: + * --DSET_WITH_NO_CONV: + * --verify data read in rbufi1[i][j] is same as wbufi1[i][j] + * --DSET_WITH_CONV_AND_NO_BKG: + * --verify data read in s_rbufi2[i][j] is all SHRT_MAX + * --DSET_WITH_CONV_AND_BKG: + * --verify fields read in s1_rbufi3[i][j] is as follows: + * --fields 'a' and 'c' are as s1_wbufi3[i][j].a and s1_wbufi3[i][j].c + * --fields 'b' and 'd' are (DSET_SELECT_DIM + j + start[0]) + * + */ +static herr_t +test_multi_dsets_all(int niter, hid_t fid, unsigned chunked, unsigned mwbuf) +{ + size_t ndsets; + int i, j, mm; + int s, n; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + + hid_t file_sids[MULTI_NUM_DSETS]; + hid_t mem_sids[MULTI_NUM_DSETS]; + hid_t mem_tids[MULTI_NUM_DSETS]; + hid_t r_mem_tids[MULTI_NUM_DSETS]; + + multi_dset_type_t dset_types[MULTI_NUM_DSETS]; + + hid_t s1_tid = H5I_INVALID_HID; + hid_t s3_tid = H5I_INVALID_HID; + hid_t s4_tid = H5I_INVALID_HID; + + char dset_names[MULTI_NUM_DSETS][DSET_NAME_LEN]; + hid_t dset_dids[MULTI_NUM_DSETS]; + + size_t buf_size; + + int *total_wbuf1 = NULL; + int *total_wbuf1_bak = NULL; + int *total_rbuf1 = NULL; + + int *wbufi1[MULTI_NUM_DSETS]; + int *rbufi1[MULTI_NUM_DSETS]; + + unsigned long *ul_total_wbuf2 = NULL; + unsigned long *ul_total_wbuf2_bak = NULL; + long *l_total_rbuf2 = NULL; + unsigned long *ul_wbufi2[MULTI_NUM_DSETS]; + long *l_rbufi2[MULTI_NUM_DSETS]; + + long *l_total_wbuf2 = NULL; + long *l_total_wbuf2_bak = NULL; + short *s_total_rbuf2 = NULL; + long *l_wbufi2[MULTI_NUM_DSETS]; + short *s_rbufi2[MULTI_NUM_DSETS]; + + s1_t *s1_total_wbuf3 = NULL; + s1_t *s1_total_wbuf3_bak = NULL; + s3_t *s3_total_rbuf3 = NULL; + s1_t *s1_wbufi3[MULTI_NUM_DSETS]; + s3_t *s3_rbufi3[MULTI_NUM_DSETS]; + + s4_t *s4_total_wbuf3 = NULL; + s4_t *s4_total_wbuf3_bak = NULL; + s1_t *s1_total_rbuf3 = NULL; + s4_t *s4_wbufi3[MULTI_NUM_DSETS]; + s1_t *s1_rbufi3[MULTI_NUM_DSETS]; + + const void *wbufs[MULTI_NUM_DSETS]; + void *rbufs[MULTI_NUM_DSETS]; + + /* for n niter to ensure that all randomized dset_types with multi_dset_type_t will be covered */ + for (n = 0; n < niter; n++) { + + /* Set up the number of datasets for testing */ + ndsets = MAX(MULTI_MIN_DSETS, MULTI_NUM_DSETS); + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + FAIL_STACK_ERROR; + + /* Enable selection I/O */ + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_ON) < 0) + FAIL_STACK_ERROR; + + /* Set modify write buffer if requested */ + if (mwbuf) + if (H5Pset_modify_write_buf(dxpl, TRUE) < 0) + FAIL_STACK_ERROR; + + /* Set dataset layout: contiguous or chunked */ + dims[0] = DSET_SELECT_DIM; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + FAIL_STACK_ERROR; + } + + /* Create compound data type: s1_t */ + if ((s1_tid = H5Tcreate(H5T_COMPOUND, sizeof(s1_t))) < 0) + FAIL_STACK_ERROR; + + if (H5Tinsert(s1_tid, "a", HOFFSET(s1_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "b", HOFFSET(s1_t, b), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "c", HOFFSET(s1_t, c), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "d", HOFFSET(s1_t, d), H5T_NATIVE_INT) < 0) + FAIL_STACK_ERROR; + + /* Create compound data type: s3_t */ + if ((s3_tid = H5Tcreate(H5T_COMPOUND, sizeof(s3_t))) < 0) + FAIL_STACK_ERROR; + + if (H5Tinsert(s3_tid, "a", HOFFSET(s3_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(s3_tid, "b", HOFFSET(s3_t, b), H5T_NATIVE_INT) < 0 || + H5Tinsert(s3_tid, "c", HOFFSET(s3_t, c), H5T_NATIVE_INT) < 0 || + H5Tinsert(s3_tid, "d", HOFFSET(s3_t, d), H5T_NATIVE_INT) < 0) + FAIL_STACK_ERROR; + + /* Create compound data type: s4_t */ + if ((s4_tid = H5Tcreate(H5T_COMPOUND, sizeof(s4_t))) < 0) + FAIL_STACK_ERROR; + + if (H5Tinsert(s4_tid, "b", HOFFSET(s4_t, b), H5T_NATIVE_UINT) < 0 || + H5Tinsert(s4_tid, "d", HOFFSET(s4_t, d), H5T_NATIVE_UINT) < 0) + FAIL_STACK_ERROR; + + /* Create dataset for i ndsets */ + for (i = 0; i < (int)ndsets; i++) { + + /* File space ids */ + if ((file_sids[i] = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + + /* Memory space ids */ + if ((mem_sids[i] = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + + mm = HDrandom() % (int)ndsets; + if (mm == 0) { + dset_types[i] = DSET_WITH_NO_CONV; + HDsnprintf(dset_names[i], sizeof(dset_names[i]), "multi_all_nconv_dset%d_%s_%s", i, + chunked ? "chunked" : "contig", mwbuf ? "mwbuf" : "nomwbuf"); + if ((dset_dids[i] = H5Dcreate2(fid, dset_names[i], H5T_NATIVE_INT, file_sids[i], H5P_DEFAULT, + dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + } + else if (mm == 1) { + dset_types[i] = DSET_WITH_CONV_AND_NO_BKG; + HDsnprintf(dset_names[i], sizeof(dset_names[i]), "multi_all_conv_nbkg_dset%d_%s_%s", i, + chunked ? "chunked" : "contig", mwbuf ? "mwbuf" : "nomwbuf"); + if ((dset_dids[i] = H5Dcreate2(fid, dset_names[i], H5T_NATIVE_LONG, file_sids[i], H5P_DEFAULT, + dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + } + else { + dset_types[i] = DSET_WITH_CONV_AND_BKG; + HDsnprintf(dset_names[i], sizeof(dset_names[i]), "multi_all_conv_bkg_dset%d_%s_%s", i, + chunked ? "chunked" : "contig", mwbuf ? "mwbuf" : "nomwbuf"); + if ((dset_dids[i] = H5Dcreate2(fid, dset_names[i], s1_tid, file_sids[i], H5P_DEFAULT, dcpl, + H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + } + + } /* end for i ndsets */ + + /* Allocate buffers for all datasets */ + + /* DSET_WITH_NO_CONV */ + buf_size = ndsets * DSET_SELECT_DIM * sizeof(int); + if (NULL == (total_wbuf1 = (int *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (mwbuf && NULL == (total_wbuf1_bak = (int *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (NULL == (total_rbuf1 = (int *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + + /* DSET_WITH_CONV_AND_NO_BKG */ + buf_size = ndsets * DSET_SELECT_DIM * sizeof(unsigned long); + if (NULL == (ul_total_wbuf2 = (unsigned long *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (mwbuf && NULL == (ul_total_wbuf2_bak = (unsigned long *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + buf_size = ndsets * DSET_SELECT_DIM * sizeof(long); + if (NULL == (l_total_rbuf2 = (long *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + + buf_size = ndsets * DSET_SELECT_DIM * sizeof(long); + if (NULL == (l_total_wbuf2 = (long *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (mwbuf && NULL == (l_total_wbuf2_bak = (long *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + buf_size = ndsets * DSET_SELECT_DIM * sizeof(short); + if (NULL == (s_total_rbuf2 = (short *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + + /* DSET_WITH_CONV_AND_BKG */ + buf_size = ndsets * DSET_SELECT_DIM * sizeof(s1_t); + if (NULL == (s1_total_wbuf3 = (s1_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (mwbuf && NULL == (s1_total_wbuf3_bak = (s1_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + buf_size = ndsets * DSET_SELECT_DIM * sizeof(s3_t); + if (NULL == (s3_total_rbuf3 = (s3_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + + buf_size = ndsets * DSET_SELECT_DIM * sizeof(s4_t); + if (NULL == (s4_total_wbuf3 = (s4_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (mwbuf && NULL == (s4_total_wbuf3_bak = (s4_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + buf_size = ndsets * DSET_SELECT_DIM * sizeof(s1_t); + if (NULL == (s1_total_rbuf3 = (s1_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + + /* Test with s settings for ndsets */ + for (s = SETTING_A; s <= SETTING_B; s++) { + + /* for i ndsets */ + for (i = 0; i < (int)ndsets; i++) { + + switch (dset_types[i]) { + + case DSET_WITH_NO_CONV: + /* Initialize buffer indices */ + wbufi1[i] = total_wbuf1 + (i * DSET_SELECT_DIM); + rbufi1[i] = total_rbuf1 + (i * DSET_SELECT_DIM); + + wbufs[i] = wbufi1[i]; + rbufs[i] = rbufi1[i]; + + /* Initialize the buffer data */ + for (j = 0; j < DSET_SELECT_DIM; j++) + wbufi1[i][j] = (int)j; + + /* Same for all cases */ + mem_tids[i] = H5T_NATIVE_INT; + r_mem_tids[i] = H5T_NATIVE_INT; + + break; + + case DSET_WITH_CONV_AND_NO_BKG: + + if (s == SETTING_A) { + /* Initialize buffer indices */ + ul_wbufi2[i] = ul_total_wbuf2 + (i * DSET_SELECT_DIM); + l_rbufi2[i] = l_total_rbuf2 + (i * DSET_SELECT_DIM); + + wbufs[i] = ul_wbufi2[i]; + rbufs[i] = l_rbufi2[i]; + + for (j = 0; j < DSET_SELECT_DIM; j++) + ul_wbufi2[i][j] = ULONG_MAX - (unsigned long)j; + + mem_tids[i] = H5T_NATIVE_ULONG; + r_mem_tids[i] = H5T_NATIVE_LONG; + } + else if (s == SETTING_B) { + /* Initialize buffer indices */ + l_wbufi2[i] = l_total_wbuf2 + (i * DSET_SELECT_DIM); + s_rbufi2[i] = s_total_rbuf2 + (i * DSET_SELECT_DIM); + + wbufs[i] = l_wbufi2[i]; + rbufs[i] = s_rbufi2[i]; + + /* Initialize the buffer data */ + for (j = 0; j < DSET_SELECT_DIM; j++) + l_wbufi2[i][j] = LONG_MAX - (long)j; + + mem_tids[i] = H5T_NATIVE_LONG; + r_mem_tids[i] = H5T_NATIVE_SHORT; + } + + break; + + case DSET_WITH_CONV_AND_BKG: + + if (s == SETTING_A) { + /* Initialize buffer indices */ + s1_wbufi3[i] = s1_total_wbuf3 + (i * DSET_SELECT_DIM); + s3_rbufi3[i] = s3_total_rbuf3 + (i * DSET_SELECT_DIM); + + wbufs[i] = s1_wbufi3[i]; + rbufs[i] = s3_rbufi3[i]; + + /* Initialize buffer data for s1_t */ + for (j = 0; j < DSET_SELECT_DIM; j++) { + s1_wbufi3[i][j].a = (4 * j); + s1_wbufi3[i][j].b = (4 * j) + 1; + s1_wbufi3[i][j].c = (4 * j) + 2; + s1_wbufi3[i][j].d = (4 * j) + 3; + } + mem_tids[i] = s1_tid; + r_mem_tids[i] = s3_tid; + } + else if (s == SETTING_B) { + /* Initialize buffer indices */ + s4_wbufi3[i] = s4_total_wbuf3 + (i * DSET_SELECT_DIM); + s1_rbufi3[i] = s1_total_rbuf3 + (i * DSET_SELECT_DIM); + + wbufs[i] = s4_wbufi3[i]; + rbufs[i] = s1_rbufi3[i]; + + /* Initialize buffer data for s4_t */ + for (j = 0; j < DSET_SELECT_DIM; j++) { + s4_wbufi3[i][j].b = DSET_SELECT_DIM + (unsigned int)j; + s4_wbufi3[i][j].d = DSET_SELECT_DIM + (unsigned int)j; + } + mem_tids[i] = s4_tid; + r_mem_tids[i] = s1_tid; + } + + break; + + case DSET_NTTYPES: + default: + TEST_ERROR; + + } /* end switch dset_types */ + + } /* end for i ndsets */ + + /* Copy wbufs if the library will be modifying them */ + if (mwbuf) { + HDmemcpy(total_wbuf1_bak, total_wbuf1, ndsets * DSET_SELECT_DIM * sizeof(int)); + HDmemcpy(ul_total_wbuf2_bak, ul_total_wbuf2, + ndsets * DSET_SELECT_DIM * sizeof(unsigned long)); + HDmemcpy(l_total_wbuf2_bak, l_total_wbuf2, ndsets * DSET_SELECT_DIM * sizeof(long)); + HDmemcpy(s1_total_wbuf3_bak, s1_total_wbuf3, ndsets * DSET_SELECT_DIM * sizeof(s1_t)); + HDmemcpy(s4_total_wbuf3_bak, s4_total_wbuf3, ndsets * DSET_SELECT_DIM * sizeof(s4_t)); + } + + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + TEST_ERROR; + + /* Restore wbufs from backup if the library modified them */ + if (mwbuf) { + HDmemcpy(total_wbuf1, total_wbuf1_bak, ndsets * DSET_SELECT_DIM * sizeof(int)); + HDmemcpy(ul_total_wbuf2, ul_total_wbuf2_bak, + ndsets * DSET_SELECT_DIM * sizeof(unsigned long)); + HDmemcpy(l_total_wbuf2, l_total_wbuf2_bak, ndsets * DSET_SELECT_DIM * sizeof(long)); + HDmemcpy(s1_total_wbuf3, s1_total_wbuf3_bak, ndsets * DSET_SELECT_DIM * sizeof(s1_t)); + HDmemcpy(s4_total_wbuf3, s4_total_wbuf3_bak, ndsets * DSET_SELECT_DIM * sizeof(s4_t)); + } + + if (H5Dread_multi(ndsets, dset_dids, r_mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + TEST_ERROR; + + /* Verify result read */ + /* for i ndsets */ + for (i = 0; i < (int)ndsets; i++) { + switch (dset_types[i]) { + + case DSET_WITH_NO_CONV: + for (j = 0; j < DSET_SELECT_DIM; j++) + if (rbufi1[i][j] != wbufi1[i][j]) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + TEST_ERROR; + } + + break; + + case DSET_WITH_CONV_AND_NO_BKG: + if (s == SETTING_A) { + for (j = 0; j < DSET_SELECT_DIM; j++) + if (l_rbufi2[i][j] != LONG_MAX) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + TEST_ERROR; + } + } + else if (s == SETTING_B) { + for (j = 0; j < DSET_SELECT_DIM; j++) + if (s_rbufi2[i][j] != SHRT_MAX) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + } + } + + break; + + case DSET_WITH_CONV_AND_BKG: + if (s == SETTING_A) { + for (j = 0; j < DSET_SELECT_DIM; j++) + if (s3_rbufi3[i][j].a != s1_wbufi3[i][j].a || + s3_rbufi3[i][j].b != s1_wbufi3[i][j].b || + s3_rbufi3[i][j].c != s1_wbufi3[i][j].c || + s3_rbufi3[i][j].d != s1_wbufi3[i][j].d) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + } + } + else if (s == SETTING_B) { + for (j = 0; j < DSET_SELECT_DIM; j++) + if (s1_rbufi3[i][j].a != s1_wbufi3[i][j].a || + s1_rbufi3[i][j].b != (DSET_SELECT_DIM + j) || + s1_rbufi3[i][j].c != s1_wbufi3[i][j].c || + s1_rbufi3[i][j].d != (DSET_SELECT_DIM + j)) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + } + } + + break; + + case DSET_NTTYPES: + default: + TEST_ERROR; + + } /* end switch dset_types */ + + } /* end for i ndsets */ + + } /* end for s settings */ + + /* Closing */ + if (H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dxpl) < 0) + FAIL_STACK_ERROR; + + if (H5Tclose(s1_tid) < 0) + FAIL_STACK_ERROR; + if (H5Tclose(s3_tid) < 0) + FAIL_STACK_ERROR; + if (H5Tclose(s4_tid) < 0) + FAIL_STACK_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + if (H5Sclose(file_sids[i]) < 0) + FAIL_STACK_ERROR; + if (H5Dclose(dset_dids[i]) < 0) + FAIL_STACK_ERROR; + /* Don't delete the last set of datasets */ + if ((n + 1) != niter) + if (H5Ldelete(fid, dset_names[i], H5P_DEFAULT) < 0) + FAIL_STACK_ERROR; + } + + /* Freeing */ + HDfree(total_wbuf1); + total_wbuf1 = NULL; + HDfree(total_wbuf1_bak); + total_wbuf1_bak = NULL; + HDfree(total_rbuf1); + total_rbuf1 = NULL; + + HDfree(ul_total_wbuf2); + ul_total_wbuf2 = NULL; + HDfree(ul_total_wbuf2_bak); + ul_total_wbuf2_bak = NULL; + HDfree(l_total_rbuf2); + l_total_rbuf2 = NULL; + HDfree(l_total_wbuf2); + l_total_wbuf2 = NULL; + HDfree(l_total_wbuf2_bak); + l_total_wbuf2_bak = NULL; + HDfree(s_total_rbuf2); + s_total_rbuf2 = NULL; + + HDfree(s1_total_wbuf3); + s1_total_wbuf3 = NULL; + HDfree(s1_total_wbuf3_bak); + s1_total_wbuf3_bak = NULL; + HDfree(s3_total_rbuf3); + s3_total_rbuf3 = NULL; + HDfree(s4_total_wbuf3); + s4_total_wbuf3 = NULL; + HDfree(s4_total_wbuf3_bak); + s4_total_wbuf3_bak = NULL; + HDfree(s1_total_rbuf3); + s1_total_rbuf3 = NULL; + + } /* end for n niter */ + + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + H5Pclose(dcpl); + H5Pclose(dxpl); + H5Tclose(s1_tid); + H5Tclose(s3_tid); + H5Tclose(s4_tid); + for (i = 0; i < (int)ndsets; i++) { + H5Sclose(file_sids[i]); + H5Sclose(mem_sids[i]); + H5Dclose(dset_dids[i]); + } + H5E_END_TRY; + + if (total_wbuf1) + HDfree(total_wbuf1); + if (total_wbuf1_bak) + HDfree(total_wbuf1_bak); + if (total_rbuf1) + HDfree(total_rbuf1); + + if (ul_total_wbuf2) + HDfree(ul_total_wbuf2); + if (ul_total_wbuf2_bak) + HDfree(ul_total_wbuf2_bak); + if (l_total_rbuf2) + HDfree(l_total_rbuf2); + if (l_total_wbuf2) + HDfree(l_total_wbuf2); + if (l_total_wbuf2_bak) + HDfree(l_total_wbuf2_bak); + if (s_total_rbuf2) + HDfree(s_total_rbuf2); + + if (s1_total_wbuf3) + HDfree(s1_total_wbuf3); + if (s1_total_wbuf3_bak) + HDfree(s1_total_wbuf3_bak); + if (s3_total_rbuf3) + HDfree(s3_total_rbuf3); + if (s4_total_wbuf3) + HDfree(s4_total_wbuf3); + if (s4_total_wbuf3_bak) + HDfree(s4_total_wbuf3_bak); + if (s1_total_rbuf3) + HDfree(s1_total_rbuf3); + + return FAIL; + +} /* test_multi_dsets_all() */ + +/* + * Verify H5Pset/get_selection_io API works as expected + */ +static herr_t +test_set_get_select_io_mode(hid_t fid) +{ + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + int i; + long wbuf[DSET_SELECT_DIM]; + H5D_selection_io_mode_t selection_io_mode; + + HDprintf("\n"); + TESTING("H5Pget/set_selection_io_mode()"); + + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + TEST_ERROR; + + /* default case */ + if (H5Pget_selection_io(dxpl, &selection_io_mode) < 0) + TEST_ERROR; + + if (selection_io_mode != H5D_SELECTION_IO_MODE_DEFAULT) + TEST_ERROR; + + /* Disable case */ + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_OFF) < 0) + TEST_ERROR; + + if (H5Pget_selection_io(dxpl, &selection_io_mode) < 0) + TEST_ERROR; + + if (selection_io_mode != H5D_SELECTION_IO_MODE_OFF) + TEST_ERROR; + + /* Enable case */ + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_ON) < 0) + TEST_ERROR; + + if (H5Pget_selection_io(dxpl, &selection_io_mode) < 0) + TEST_ERROR; + + if (selection_io_mode != H5D_SELECTION_IO_MODE_ON) + TEST_ERROR; + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + FAIL_STACK_ERROR; + + if ((did = H5Dcreate2(fid, "test_chk_dset", H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* Initialize data */ + for (i = 0; i < DSET_SELECT_DIM; i++) + wbuf[i] = i; + + /* May change the selection io actually performed */ + if (H5Dwrite(did, H5T_NATIVE_LONG, H5S_ALL, H5S_ALL, dxpl, wbuf) < 0) + FAIL_STACK_ERROR; + + if (H5Pget_selection_io(dxpl, &selection_io_mode) < 0) + TEST_ERROR; + + /* Should still be enabled */ + if (selection_io_mode != H5D_SELECTION_IO_MODE_ON) + TEST_ERROR; + + if (H5Dclose(did) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dxpl) < 0) + FAIL_STACK_ERROR; + if (H5Sclose(sid) < 0) + FAIL_STACK_ERROR; + + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Sclose(sid); + H5Dclose(did); + H5Pclose(dcpl); + H5Pclose(dxpl); + } + H5E_END_TRY; + + return FAIL; +} /* test_set_get_select_io_mode() */ + +/* + * To test with various test_mode that no selelction I/O is performed + * + * Note: It's the responsibility of the tester to feed proper combination + * of test_mode as needed. + */ +static herr_t +test_no_selection_io_cause_mode(const char *filename, hid_t fapl, uint32_t test_mode) +{ + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t fid = H5I_INVALID_HID; + hid_t fcpl = H5I_INVALID_HID; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + hbool_t is_chunked = FALSE; + hid_t tid = H5T_NATIVE_INT; + uint32_t no_selection_io_cause_write = 0; + uint32_t no_selection_io_cause_read = 0; + uint32_t no_selection_io_cause_write_expected = 0; + uint32_t no_selection_io_cause_read_expected = 0; + int wbuf[DSET_SELECT_DIM]; + int rbuf[DSET_SELECT_DIM]; + int i; + + /* Check for (currently) incompatible combinations */ + if (test_mode & TEST_PAGE_BUFFER) { + char *env_h5_drvr = NULL; + + /* The split and multi driver are not compatible with page buffering. No message since the other + * cases aren't skipped. */ + env_h5_drvr = HDgetenv(HDF5_DRIVER); + if (env_h5_drvr && (!HDstrcmp(env_h5_drvr, "split") || !HDstrcmp(env_h5_drvr, "multi"))) + return 0; + } + + if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR; + + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + FAIL_STACK_ERROR; + + /* Enable page buffering to trigger H5D_PAGE_BUFFER */ + if (test_mode & TEST_PAGE_BUFFER) { + if (H5Pset_page_buffer_size(fapl, 4096, 0, 0) < 0) + FAIL_STACK_ERROR; + if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + FAIL_STACK_ERROR; + } + else { + /* Not page buffer test, reset to default */ + if (H5Pset_page_buffer_size(fapl, 0, 0, 0) < 0) + FAIL_STACK_ERROR; + if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, 0, (hsize_t)1) < 0) + FAIL_STACK_ERROR; + } + + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR; + + /* If default mode, 1st write will trigger cb, 2nd write will trigger sieve */ + /* If on mode, will trigger nothing because the on mode path is different */ + /* Need 2 writes */ + if (test_mode & TEST_CONTIGUOUS_SIEVE_BUFFER) { + no_selection_io_cause_write_expected |= H5D_SEL_IO_CONTIGUOUS_SIEVE_BUFFER; + no_selection_io_cause_read_expected |= H5D_SEL_IO_CONTIGUOUS_SIEVE_BUFFER; + } + + if (test_mode & TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET) { + if (H5Pset_layout(dcpl, H5D_COMPACT) < 0) + FAIL_STACK_ERROR; + no_selection_io_cause_write_expected |= H5D_SEL_IO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; + no_selection_io_cause_read_expected |= H5D_SEL_IO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; + } + + if (test_mode == TEST_DATASET_FILTER) { + if (H5Pset_deflate(dcpl, 9) < 0) + FAIL_STACK_ERROR; + is_chunked = TRUE; + no_selection_io_cause_write_expected |= H5D_SEL_IO_DATASET_FILTER; + no_selection_io_cause_read_expected |= H5D_SEL_IO_DATASET_FILTER; + } + + if (test_mode == TEST_CHUNK_CACHE) { + is_chunked = TRUE; + no_selection_io_cause_write_expected |= H5D_SEL_IO_CHUNK_CACHE; + no_selection_io_cause_read_expected |= H5D_SEL_IO_CHUNK_CACHE; + } + + if (test_mode == TEST_DISABLE_BY_API) { + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_OFF) < 0) + FAIL_STACK_ERROR; + no_selection_io_cause_write_expected |= H5D_SEL_IO_DISABLE_BY_API; + no_selection_io_cause_read_expected |= H5D_SEL_IO_DISABLE_BY_API; + } + + if (test_mode & TEST_NO_VECTOR_OR_SELECTION_IO_CB) { + no_selection_io_cause_write_expected |= H5D_SEL_IO_DEFAULT_OFF; + no_selection_io_cause_read_expected |= H5D_SEL_IO_DEFAULT_OFF; + } + + /* Datatype conversion */ + if (test_mode & TEST_DATATYPE_CONVERSION) { + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_ON) < 0) + FAIL_STACK_ERROR; + tid = H5T_NATIVE_UINT; + + /* If we're testing a too small tconv buffer, set the buffer to be too small */ + if (test_mode & TEST_TCONV_BUF_TOO_SMALL) { + if (H5Pset_buffer(dxpl, sizeof(int), NULL, NULL) < 0) + FAIL_STACK_ERROR; + + /* If we're using in-place type conversion sel io will succeed and only switch to scalar at the + * VFL */ + if (test_mode & TEST_IN_PLACE_TCONV) { + if (H5Pset_modify_write_buf(dxpl, TRUE) < 0) + FAIL_STACK_ERROR; + no_selection_io_cause_write_expected |= H5D_SEL_IO_NO_VECTOR_OR_SELECTION_IO_CB; + } + else + no_selection_io_cause_write_expected |= H5D_SEL_IO_TCONV_BUF_TOO_SMALL; + + /* In-place type conversion for read doesn't require modify_write_buf */ + no_selection_io_cause_read_expected |= H5D_SEL_IO_NO_VECTOR_OR_SELECTION_IO_CB; + } + else { + /* sel io will succeed and only switch to scalar at the VFL */ + no_selection_io_cause_write_expected |= H5D_SEL_IO_NO_VECTOR_OR_SELECTION_IO_CB; + no_selection_io_cause_read_expected |= H5D_SEL_IO_NO_VECTOR_OR_SELECTION_IO_CB; + } + } + + if (test_mode & TEST_PAGE_BUFFER) { + no_selection_io_cause_write_expected |= H5D_SEL_IO_PAGE_BUFFER; + no_selection_io_cause_read_expected |= H5D_SEL_IO_PAGE_BUFFER; + } + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + + if (is_chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + FAIL_STACK_ERROR; + } + + if ((did = H5Dcreate2(fid, "no_selection_io_cause", H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, + H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* Initialize data */ + for (i = 0; i < DSET_SELECT_DIM; i++) + wbuf[i] = i; + + if (H5Dwrite(did, tid, H5S_ALL, H5S_ALL, dxpl, wbuf) < 0) + FAIL_STACK_ERROR; + + if (test_mode & TEST_CONTIGUOUS_SIEVE_BUFFER) { + if (H5Dwrite(did, tid, H5S_ALL, H5S_ALL, dxpl, wbuf) < 0) + FAIL_STACK_ERROR; + } + + if (H5Pget_no_selection_io_cause(dxpl, &no_selection_io_cause_write) < 0) + TEST_ERROR; + + /* Verify causes of no selection I/O for write are as expected */ + if (no_selection_io_cause_write != no_selection_io_cause_write_expected) + TEST_ERROR; + + /* Flush to clear the sieve buf */ + if (test_mode & TEST_NO_VECTOR_OR_SELECTION_IO_CB || test_mode & TEST_DATATYPE_CONVERSION || + test_mode & TEST_PAGE_BUFFER) { + + if (H5Dflush(did) < 0) + FAIL_STACK_ERROR; + } + + if (H5Dread(did, tid, H5S_ALL, H5S_ALL, dxpl, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify causes of no selection I/O for write is as expected */ + if (H5Pget_no_selection_io_cause(dxpl, &no_selection_io_cause_read) < 0) + TEST_ERROR; + + /* Verify causes of no selection I/O for read are as expected */ + if (no_selection_io_cause_read != no_selection_io_cause_read_expected) + TEST_ERROR; + + if (H5Dclose(did) < 0) + FAIL_STACK_ERROR; + if (H5Sclose(sid) < 0) + FAIL_STACK_ERROR; + + if (H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dxpl) < 0) + FAIL_STACK_ERROR; + + if (H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + + if (H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Pclose(fcpl); + H5Pclose(dcpl); + H5Pclose(dxpl); + H5Dclose(did); + H5Dclose(sid); + H5Fclose(fid); + } + H5E_END_TRY; + + return FAIL; +} /* test_no_selection_io_cause_mode() */ + +/* + * Test for causes of not performing selection I/O + */ +static herr_t +test_get_no_selection_io_cause(const char *filename, hid_t fapl) +{ + + int errs = 0; + + HDprintf("\n"); + TESTING("H5Pget_no_selection_io_cause()"); + + errs += test_no_selection_io_cause_mode(filename, fapl, TEST_DISABLE_BY_API); + errs += test_no_selection_io_cause_mode(filename, fapl, TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET); + errs += test_no_selection_io_cause_mode(filename, fapl, TEST_CONTIGUOUS_SIEVE_BUFFER); + errs += test_no_selection_io_cause_mode(filename, fapl, TEST_DATASET_FILTER); + errs += test_no_selection_io_cause_mode(filename, fapl, TEST_CHUNK_CACHE); + errs += test_no_selection_io_cause_mode(filename, fapl, TEST_NO_VECTOR_OR_SELECTION_IO_CB); + errs += test_no_selection_io_cause_mode(filename, fapl, TEST_DATATYPE_CONVERSION); + errs += + test_no_selection_io_cause_mode(filename, fapl, TEST_DATATYPE_CONVERSION | TEST_TCONV_BUF_TOO_SMALL); + errs += test_no_selection_io_cause_mode( + filename, fapl, TEST_DATATYPE_CONVERSION | TEST_TCONV_BUF_TOO_SMALL | TEST_IN_PLACE_TCONV); +#ifndef H5_HAVE_PARALLEL + errs += test_no_selection_io_cause_mode(filename, fapl, TEST_PAGE_BUFFER); +#endif + + if (errs) { + HDprintf(" FAILED\n"); + return FAIL; + } + else { + PASSED(); + return SUCCEED; + } +} + +/*------------------------------------------------------------------------- + * Function: main + * + * Purpose: Test cases for selection I/O + * + * Return: EXIT_SUCCESS/EXIT_FAILURE + * + * Programmer: + * + *------------------------------------------------------------------------- + */ +int +main(void) +{ + int nerrors = 0; + char filename[FILENAME_BUF_SIZE]; + hid_t fapl = H5I_INVALID_HID; + hid_t fid = H5I_INVALID_HID; + int test_select_config; + unsigned chunked; + unsigned dtrans; + unsigned mwbuf; + + /* Testing setup */ + h5_reset(); + fapl = h5_fileaccess(); + + h5_fixname(FILENAME[0], fapl, filename, sizeof filename); + + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + TEST_ERROR; + + /* Test with contiguous or chunked dataset */ + for (chunked = FALSE; chunked <= TRUE; chunked++) { + + /* Data transforms only apply to integer or floating-point datasets */ + /* therefore, not all tests are run with data transform */ + for (dtrans = FALSE; dtrans <= TRUE; dtrans++) { + + /* Test with and without modify_write_buf turned on */ + for (mwbuf = FALSE; mwbuf <= TRUE; mwbuf++) { + /* Print configuration message */ + printf("Testing for selection I/O "); + if (chunked) + printf("with chunked dataset, "); + else + printf("with contiguous dataset, "); + if (dtrans) + printf("data transform, "); + else + printf("without data transform, "); + if (mwbuf) + printf("and with modifying write buffers\n"); + else + printf("and without modifying write buffers\n"); + + for (test_select_config = (int)TEST_NO_TYPE_CONV; + test_select_config < (int)TEST_SELECT_NTESTS; test_select_config++) { + + switch (test_select_config) { + case TEST_NO_TYPE_CONV: /* case 1 */ + TESTING_2("No type conversion (null case)"); + + nerrors += (test_no_type_conv(fid, chunked, dtrans, mwbuf) < 0 ? 1 : 0); + + break; + + case TEST_NO_SIZE_CHANGE_NO_BKG: /* case 2 */ + TESTING_2("No size change, no background buffer"); + + /* Data transforms does not apply to the dataset datatype for this test */ + if (dtrans) + SKIPPED(); + else + nerrors += (test_no_size_change_no_bkg(fid, chunked, mwbuf) < 0 ? 1 : 0); + + break; + + case TEST_LARGER_MEM_NO_BKG: /* case 3 */ + TESTING_2("Larger memory type, no background buffer"); + + nerrors += (test_larger_mem_type_no_bkg(fid, chunked, dtrans, mwbuf) < 0 ? 1 : 0); + + break; + + case TEST_SMALLER_MEM_NO_BKG: /* case 4 */ + TESTING_2("Smaller memory type, no background buffer"); + + nerrors += + (test_smaller_mem_type_no_bkg(fid, chunked, dtrans, mwbuf) < 0 ? 1 : 0); + + break; + + case TEST_CMPD_WITH_BKG: /* case 5 */ + TESTING_2("Compound types with background buffer"); + + /* Data transforms does not apply to the dataset datatype for this test */ + if (dtrans) + SKIPPED(); + else + nerrors += (test_cmpd_with_bkg(fid, chunked, mwbuf) < 0 ? 1 : 0); + + break; + + case TEST_MULTI_CONV_NO_BKG: /* case 6 */ + TESTING_2("multi-datasets: type conv + no bkg buffer"); + + nerrors += test_multi_dsets_no_bkg(fid, chunked, dtrans, mwbuf); + break; + + case TEST_MULTI_CONV_BKG: /* case 7 */ + TESTING_2("multi-datasets: type conv + bkg buffer"); + + /* Data transforms does not apply to the dataset datatype for this test */ + if (dtrans) + SKIPPED(); + else + nerrors += test_multi_dsets_cmpd_with_bkg(fid, chunked, mwbuf); + + break; + + case TEST_MULTI_CONV_SIZE_CHANGE: /* case 8 */ + TESTING_2("multi-datasets: type conv + size change + no bkg buffer"); + + /* Data transforms does not apply to the dataset datatype for this test */ + if (dtrans) + SKIPPED(); + else + nerrors += test_multi_dsets_size_change_no_bkg(fid, chunked, mwbuf); + + break; + + case TEST_MULTI_ALL: /* case 9 */ + TESTING_2("multi-datasets: no conv + conv without bkg + conv with bkg"); + + /* Data transforms does not apply to the dataset datatype for this test */ + if (dtrans) + SKIPPED(); + else + nerrors += test_multi_dsets_all(10, fid, chunked, mwbuf); + + break; + + case TEST_SELECT_NTESTS: + default: + TEST_ERROR; + + } /* end switch */ + } /* end for test_select_config */ + + } /* end mwbuf */ + + } /* end dtrans */ + + } /* end chunked */ + + nerrors += test_set_get_select_io_mode(fid); + + if (H5Fclose(fid) < 0) + TEST_ERROR; + + /* Use own file */ + nerrors += test_get_no_selection_io_cause(filename, fapl); + + if (nerrors) + goto error; + + printf("\n===================================\n"); + HDprintf("All selection I/O dataset tests passed.\n"); + printf("===================================\n"); + + h5_cleanup(FILENAME, fapl); + + HDexit(EXIT_SUCCESS); + +error: + nerrors = MAX(1, nerrors); + HDprintf("***** %d SELECTION I/O DATASET TEST%s FAILED! *****\n", nerrors, 1 == nerrors ? "" : "S"); + HDexit(EXIT_FAILURE); + +} /* end main() */ diff --git a/test/testfiles/plist_files/def_dxpl_32be b/test/testfiles/plist_files/def_dxpl_32be Binary files differindex b13f456..77ed5d1 100644 --- a/test/testfiles/plist_files/def_dxpl_32be +++ b/test/testfiles/plist_files/def_dxpl_32be diff --git a/test/testfiles/plist_files/def_dxpl_32le b/test/testfiles/plist_files/def_dxpl_32le Binary files differindex b13f456..77ed5d1 100644 --- a/test/testfiles/plist_files/def_dxpl_32le +++ b/test/testfiles/plist_files/def_dxpl_32le diff --git a/test/testfiles/plist_files/def_dxpl_64be b/test/testfiles/plist_files/def_dxpl_64be Binary files differindex b13f456..77ed5d1 100644 --- a/test/testfiles/plist_files/def_dxpl_64be +++ b/test/testfiles/plist_files/def_dxpl_64be diff --git a/test/testfiles/plist_files/def_dxpl_64le b/test/testfiles/plist_files/def_dxpl_64le Binary files differindex b13f456..77ed5d1 100644 --- a/test/testfiles/plist_files/def_dxpl_64le +++ b/test/testfiles/plist_files/def_dxpl_64le diff --git a/test/testfiles/plist_files/dxpl_32be b/test/testfiles/plist_files/dxpl_32be Binary files differindex 5ff2ea0..b6ff37d 100644 --- a/test/testfiles/plist_files/dxpl_32be +++ b/test/testfiles/plist_files/dxpl_32be diff --git a/test/testfiles/plist_files/dxpl_32le b/test/testfiles/plist_files/dxpl_32le Binary files differindex 5ff2ea0..b6ff37d 100644 --- a/test/testfiles/plist_files/dxpl_32le +++ b/test/testfiles/plist_files/dxpl_32le diff --git a/test/testfiles/plist_files/dxpl_64be b/test/testfiles/plist_files/dxpl_64be Binary files differindex 5ff2ea0..b6ff37d 100644 --- a/test/testfiles/plist_files/dxpl_64be +++ b/test/testfiles/plist_files/dxpl_64be diff --git a/test/testfiles/plist_files/dxpl_64le b/test/testfiles/plist_files/dxpl_64le Binary files differindex 5ff2ea0..b6ff37d 100644 --- a/test/testfiles/plist_files/dxpl_64le +++ b/test/testfiles/plist_files/dxpl_64le diff --git a/test/tselect.c b/test/tselect.c index 3dd739c..d67e5bd 100644 --- a/test/tselect.c +++ b/test/tselect.c @@ -2280,7 +2280,7 @@ test_select_hyper_contig_dr__run_test(int test_num, const uint16_t *cube_buf, co * H5Sselect_shape_same() views as being of the same shape. * * Start by writing small_rank D slices from the in memory large cube, to - * the the on disk small cube dataset. After each write, read the small + * the on disk small cube dataset. After each write, read the small * cube dataset back from disk, and verify that it contains the expected * data. Verify that H5Sselect_shape_same() returns true on the * memory and file selections. @@ -3309,7 +3309,7 @@ test_select_hyper_checker_board_dr__run_test(int test_num, const uint16_t *cube_ * H5Sselect_shape_same() views as being of the same shape. * * Start by writing small_rank D slices from the in memory large cube, to - * the the on disk small cube dataset. After each write, read the small + * the on disk small cube dataset. After each write, read the small * cube dataset back from disk, and verify that it contains the expected * data. Verify that H5Sselect_shape_same() returns true on the * memory and file selections. @@ -15840,7 +15840,7 @@ test_hyper_io_1d(void) hsize_t chunk_dims[1]; /* Chunk dimension size */ hsize_t offset[1]; /* Starting offset for hyperslab */ hsize_t stride[1]; /* Distance between blocks in the hyperslab selection */ - hsize_t count[1]; /* # of blocks in the the hyperslab selection */ + hsize_t count[1]; /* # of blocks in the hyperslab selection */ hsize_t block[1]; /* Size of block in the hyperslab selection */ unsigned int wdata[CHUNKSZ]; /* Data to be written */ unsigned int rdata[NUM_ELEMENTS / 10]; /* Data to be read */ diff --git a/testpar/CMakeLists.txt b/testpar/CMakeLists.txt index d876a21..fb66e76 100644 --- a/testpar/CMakeLists.txt +++ b/testpar/CMakeLists.txt @@ -8,6 +8,7 @@ project (HDF5_TEST_PAR C) set (testphdf5_SOURCES ${HDF5_TEST_PAR_SOURCE_DIR}/testphdf5.c ${HDF5_TEST_PAR_SOURCE_DIR}/t_dset.c + ${HDF5_TEST_PAR_SOURCE_DIR}/t_select_io_dset.c ${HDF5_TEST_PAR_SOURCE_DIR}/t_file.c ${HDF5_TEST_PAR_SOURCE_DIR}/t_file_image.c ${HDF5_TEST_PAR_SOURCE_DIR}/t_mdset.c @@ -96,6 +97,7 @@ set (H5P_TESTS t_prestart t_init_term t_pmulti_dset + t_select_io_dset t_shapesame t_filters_parallel t_subfiling_vfd diff --git a/testpar/Makefile.am b/testpar/Makefile.am index 539750a..59d47e1 100644 --- a/testpar/Makefile.am +++ b/testpar/Makefile.am @@ -33,7 +33,7 @@ check_SCRIPTS = $(TEST_SCRIPT_PARA) # Test programs. These are our main targets. # -TEST_PROG_PARA=t_mpi t_bigio testphdf5 t_cache t_cache_image t_pread t_pshutdown t_prestart t_init_term t_pmulti_dset t_shapesame t_filters_parallel t_2Gio t_vfd +TEST_PROG_PARA=t_mpi t_bigio testphdf5 t_cache t_cache_image t_pread t_pshutdown t_prestart t_init_term t_pmulti_dset t_select_io_dset t_shapesame t_filters_parallel t_2Gio t_vfd if SUBFILING_VFD_CONDITIONAL TEST_PROG_PARA += t_subfiling_vfd diff --git a/testpar/t_2Gio.c b/testpar/t_2Gio.c index d62fb55..8b67dd9 100644 --- a/testpar/t_2Gio.c +++ b/testpar/t_2Gio.c @@ -3868,6 +3868,10 @@ test_no_collective_cause_mode(int selection_mode) uint32_t no_collective_cause_global_expected = 0; // hsize_t coord[NELM][MAX_RANK]; + uint32_t no_selection_io_cause_write = 0; + uint32_t no_selection_io_cause_read = 0; + uint32_t no_selection_io_cause_expected = 0; + const char *filename; const char *test_name; hbool_t is_chunked = 1; @@ -3968,27 +3972,50 @@ test_no_collective_cause_mode(int selection_mode) dataset = H5Dcreate2(fid, "nocolcause", data_type, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); VRFY((dataset >= 0), "H5Dcreate2() dataset succeeded"); + /* Set up the dxpl for the write */ + dxpl_write = H5Pcreate(H5P_DATASET_XFER); + VRFY((dxpl_write >= 0), "H5Pcreate(H5P_DATASET_XFER) succeeded"); + /* * Set expected causes and some tweaks based on the type of test */ if (selection_mode & TEST_DATATYPE_CONVERSION) { test_name = "Broken Collective I/O - Datatype Conversion"; - no_collective_cause_local_expected |= H5D_MPIO_DATATYPE_CONVERSION; - no_collective_cause_global_expected |= H5D_MPIO_DATATYPE_CONVERSION; + /* set different sign to trigger type conversion */ data_type = H5T_NATIVE_UINT; + + /* Disable selection I/O since datatype conversion is supported in collective with selection I/O */ + ret = H5Pset_selection_io(dxpl_write, H5D_SELECTION_IO_MODE_OFF); + VRFY((ret >= 0), "H5Pset_selection_io succeeded"); + + no_collective_cause_local_expected |= H5D_MPIO_DATATYPE_CONVERSION | H5D_MPIO_NO_SELECTION_IO; + no_collective_cause_global_expected |= H5D_MPIO_DATATYPE_CONVERSION | H5D_MPIO_NO_SELECTION_IO; + no_selection_io_cause_expected |= H5D_SEL_IO_DISABLE_BY_API; } if (selection_mode & TEST_DATA_TRANSFORMS) { test_name = "Broken Collective I/O - DATA Transforms"; - no_collective_cause_local_expected |= H5D_MPIO_DATA_TRANSFORMS; - no_collective_cause_global_expected |= H5D_MPIO_DATA_TRANSFORMS; + + /* Set transform */ + ret = H5Pset_data_transform(dxpl_write, "x+1"); + VRFY((ret >= 0), "H5Pset_data_transform succeeded"); + + /* Disable selection I/O since data transforms are supported in collective with selection I/O */ + ret = H5Pset_selection_io(dxpl_write, H5D_SELECTION_IO_MODE_OFF); + VRFY((ret >= 0), "H5Pset_selection_io succeeded"); + + no_collective_cause_local_expected |= H5D_MPIO_DATA_TRANSFORMS | H5D_MPIO_NO_SELECTION_IO; + no_collective_cause_global_expected |= H5D_MPIO_DATA_TRANSFORMS | H5D_MPIO_NO_SELECTION_IO; + no_selection_io_cause_expected |= H5D_SEL_IO_DISABLE_BY_API; } if (selection_mode & TEST_NOT_SIMPLE_OR_SCALAR_DATASPACES) { test_name = "Broken Collective I/O - No Simple or Scalar DataSpace"; no_collective_cause_local_expected |= H5D_MPIO_NOT_SIMPLE_OR_SCALAR_DATASPACES; no_collective_cause_global_expected |= H5D_MPIO_NOT_SIMPLE_OR_SCALAR_DATASPACES; + no_collective_cause_local_expected &= ~(unsigned)H5D_MPIO_NO_SELECTION_IO; + no_collective_cause_global_expected &= ~(unsigned)H5D_MPIO_NO_SELECTION_IO; } if (selection_mode & TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET_COMPACT || @@ -3996,6 +4023,8 @@ test_no_collective_cause_mode(int selection_mode) test_name = "Broken Collective I/O - No CONTI or CHUNKED Dataset"; no_collective_cause_local_expected |= H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; no_collective_cause_global_expected |= H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; + no_collective_cause_local_expected &= ~(unsigned)H5D_MPIO_NO_SELECTION_IO; + no_collective_cause_global_expected &= ~(unsigned)H5D_MPIO_NO_SELECTION_IO; } if (selection_mode & TEST_COLLECTIVE) { @@ -4008,6 +4037,8 @@ test_no_collective_cause_mode(int selection_mode) test_name = "Broken Collective I/O - Independent"; no_collective_cause_local_expected = H5D_MPIO_SET_INDEPENDENT; no_collective_cause_global_expected = H5D_MPIO_SET_INDEPENDENT; + no_collective_cause_local_expected &= ~(unsigned)H5D_MPIO_NO_SELECTION_IO; + no_collective_cause_global_expected &= ~(unsigned)H5D_MPIO_NO_SELECTION_IO; /* switch to independent io */ is_independent = 1; } @@ -4037,10 +4068,6 @@ test_no_collective_cause_mode(int selection_mode) for (i = 0; i < length; i++) buffer[i] = i; - /* Set up the dxpl for the write */ - dxpl_write = H5Pcreate(H5P_DATASET_XFER); - VRFY((dxpl_write >= 0), "H5Pcreate(H5P_DATASET_XFER) succeeded"); - if (is_independent) { /* Set Independent I/O */ ret = H5Pset_dxpl_mpio(dxpl_write, H5FD_MPIO_INDEPENDENT); @@ -4052,11 +4079,6 @@ test_no_collective_cause_mode(int selection_mode) VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); } - if (selection_mode & TEST_DATA_TRANSFORMS) { - ret = H5Pset_data_transform(dxpl_write, "x+1"); - VRFY((ret >= 0), "H5Pset_data_transform succeeded"); - } - /*--------------------- * Test Write access *---------------------*/ @@ -4072,6 +4094,20 @@ test_no_collective_cause_mode(int selection_mode) &no_collective_cause_global_write); VRFY((ret >= 0), "retrieving no collective cause succeeded"); + ret = H5Pget_no_selection_io_cause(dxpl_write, &no_selection_io_cause_write); + VRFY((ret >= 0), "retrieving no selection io cause succeeded"); + + if (no_collective_cause_local_write & H5D_MPIO_NO_SELECTION_IO) { + VRFY((no_selection_io_cause_write == no_selection_io_cause_expected), + "H5D_MPIO_NO_SELECTION_IO for write is as expected"); + } + + if (no_collective_cause_global_write & H5D_MPIO_NO_SELECTION_IO) { + + VRFY((no_selection_io_cause_write == no_selection_io_cause_expected), + "H5D_MPIO_NO_SELECTION_IO for write is as expected"); + } + /*--------------------- * Test Read access *---------------------*/ @@ -4092,6 +4128,21 @@ test_no_collective_cause_mode(int selection_mode) &no_collective_cause_global_read); VRFY((ret >= 0), "retrieving no collective cause succeeded"); + ret = H5Pget_no_selection_io_cause(dxpl_read, &no_selection_io_cause_read); + VRFY((ret >= 0), "retrieving no selection io cause succeeded"); + + if (no_collective_cause_local_read & H5D_MPIO_NO_SELECTION_IO) { + + VRFY((no_selection_io_cause_read == no_selection_io_cause_expected), + "H5D_MPIO_NO_SELECTION_IO for read is as expected"); + } + + if (no_collective_cause_global_read & H5D_MPIO_NO_SELECTION_IO) { + + VRFY((no_selection_io_cause_read == no_selection_io_cause_expected), + "H5D_MPIO_NO_SELECTION_IO for read is as expected"); + } + /* Check write vs read */ VRFY((no_collective_cause_local_read == no_collective_cause_local_write), "reading and writing are the same for local cause of Broken Collective I/O"); diff --git a/testpar/t_coll_chunk.c b/testpar/t_coll_chunk.c index 5f853e3..6636ffa 100644 --- a/testpar/t_coll_chunk.c +++ b/testpar/t_coll_chunk.c @@ -566,7 +566,8 @@ coll_chunktest(const char *filename, int chunk_factor, int select_factor, int ap hsize_t start[RANK], count[RANK], stride[RANK], block[RANK]; #ifdef H5_HAVE_INSTRUMENTED_LIBRARY - unsigned prop_value; + unsigned prop_value; + H5D_selection_io_mode_t selection_io_mode; #endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ int mpi_size, mpi_rank; @@ -804,7 +805,11 @@ coll_chunktest(const char *filename, int chunk_factor, int select_factor, int ap /* Only check chunk optimization mode if selection I/O is not being used - * selection I/O bypasses this IO mode decision - it's effectively always * multi chunk currently */ - if (facc_type == FACC_MPIO && !H5_use_selection_io_g) { + + status = H5Pget_selection_io(xfer_plist, &selection_io_mode); + VRFY((status >= 0), "testing property list get succeeded"); + + if (facc_type == FACC_MPIO && (selection_io_mode != H5D_SELECTION_IO_MODE_ON)) { switch (api_option) { case API_LINK_HARD: status = H5Pget(xfer_plist, H5D_XFER_COLL_CHUNK_LINK_HARD_NAME, &prop_value); diff --git a/testpar/t_dset.c b/testpar/t_dset.c index 34c4d97..d144235 100644 --- a/testpar/t_dset.c +++ b/testpar/t_dset.c @@ -3348,13 +3348,27 @@ test_actual_io_mode(int selection_mode) void actual_io_mode_tests(void) { - int mpi_size = -1; + H5D_selection_io_mode_t selection_io_mode; + hid_t dxpl_id = H5I_INVALID_HID; + herr_t ret; + int mpi_size = -1; + int mpi_rank = -1; + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); /* Only run these tests if selection I/O is not being used - selection I/O * bypasses this IO mode decision - it's effectively always multi chunk * currently */ - if (!H5_use_selection_io_g) { + + dxpl_id = H5Pcreate(H5P_DATASET_XFER); + VRFY((dxpl_id >= 0), "H5Pcreate(H5P_DATASET_XFER) succeeded"); + ret = H5Pget_selection_io(dxpl_id, &selection_io_mode); + VRFY((ret >= 0), "retrieving selection io mode succeeded"); + ret = H5Pclose(dxpl_id); + VRFY((ret >= 0), "H5Pclose succeeded"); + + if (selection_io_mode != H5D_SELECTION_IO_MODE_ON) { test_actual_io_mode(TEST_ACTUAL_IO_NO_COLLECTIVE); /* @@ -3438,6 +3452,10 @@ test_no_collective_cause_mode(int selection_mode) uint32_t no_collective_cause_global_read = 0; uint32_t no_collective_cause_global_expected = 0; + uint32_t no_selection_io_cause_write = 0; + uint32_t no_selection_io_cause_read = 0; + uint32_t no_selection_io_cause_expected = 0; + const char *filename; const char *test_name; hbool_t is_chunked = 1; @@ -3538,27 +3556,50 @@ test_no_collective_cause_mode(int selection_mode) dataset = H5Dcreate2(fid, "nocolcause", data_type, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); VRFY((dataset >= 0), "H5Dcreate2() dataset succeeded"); + /* Set up the dxpl for the write */ + dxpl_write = H5Pcreate(H5P_DATASET_XFER); + VRFY((dxpl_write >= 0), "H5Pcreate(H5P_DATASET_XFER) succeeded"); + /* * Set expected causes and some tweaks based on the type of test */ if (selection_mode & TEST_DATATYPE_CONVERSION) { test_name = "Broken Collective I/O - Datatype Conversion"; - no_collective_cause_local_expected |= H5D_MPIO_DATATYPE_CONVERSION; - no_collective_cause_global_expected |= H5D_MPIO_DATATYPE_CONVERSION; + /* set different sign to trigger type conversion */ data_type = H5T_NATIVE_UINT; + + /* Disable selection I/O since datatype conversion is supported in collective with selection I/O */ + ret = H5Pset_selection_io(dxpl_write, H5D_SELECTION_IO_MODE_OFF); + VRFY((ret >= 0), "H5Pset_selection_io succeeded"); + + no_collective_cause_local_expected |= H5D_MPIO_DATATYPE_CONVERSION | H5D_MPIO_NO_SELECTION_IO; + no_collective_cause_global_expected |= H5D_MPIO_DATATYPE_CONVERSION | H5D_MPIO_NO_SELECTION_IO; + no_selection_io_cause_expected |= H5D_SEL_IO_DISABLE_BY_API; } if (selection_mode & TEST_DATA_TRANSFORMS) { test_name = "Broken Collective I/O - DATA Transforms"; - no_collective_cause_local_expected |= H5D_MPIO_DATA_TRANSFORMS; - no_collective_cause_global_expected |= H5D_MPIO_DATA_TRANSFORMS; + + /* Set transform */ + ret = H5Pset_data_transform(dxpl_write, "x+1"); + VRFY((ret >= 0), "H5Pset_data_transform succeeded"); + + /* Disable selection I/O since data transforms are supported in collective with selection I/O */ + ret = H5Pset_selection_io(dxpl_write, H5D_SELECTION_IO_MODE_OFF); + VRFY((ret >= 0), "H5Pset_selection_io succeeded"); + + no_collective_cause_local_expected |= H5D_MPIO_DATA_TRANSFORMS | H5D_MPIO_NO_SELECTION_IO; + no_collective_cause_global_expected |= H5D_MPIO_DATA_TRANSFORMS | H5D_MPIO_NO_SELECTION_IO; + no_selection_io_cause_expected |= H5D_SEL_IO_DISABLE_BY_API; } if (selection_mode & TEST_NOT_SIMPLE_OR_SCALAR_DATASPACES) { test_name = "Broken Collective I/O - No Simple or Scalar DataSpace"; no_collective_cause_local_expected |= H5D_MPIO_NOT_SIMPLE_OR_SCALAR_DATASPACES; no_collective_cause_global_expected |= H5D_MPIO_NOT_SIMPLE_OR_SCALAR_DATASPACES; + no_collective_cause_local_expected &= ~(unsigned)H5D_MPIO_NO_SELECTION_IO; + no_collective_cause_global_expected &= ~(unsigned)H5D_MPIO_NO_SELECTION_IO; } if (selection_mode & TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET_COMPACT || @@ -3566,6 +3607,8 @@ test_no_collective_cause_mode(int selection_mode) test_name = "Broken Collective I/O - No CONTI or CHUNKED Dataset"; no_collective_cause_local_expected |= H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; no_collective_cause_global_expected |= H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; + no_collective_cause_local_expected &= ~(unsigned)H5D_MPIO_NO_SELECTION_IO; + no_collective_cause_global_expected &= ~(unsigned)H5D_MPIO_NO_SELECTION_IO; } if (selection_mode & TEST_COLLECTIVE) { @@ -3578,6 +3621,8 @@ test_no_collective_cause_mode(int selection_mode) test_name = "Broken Collective I/O - Independent"; no_collective_cause_local_expected = H5D_MPIO_SET_INDEPENDENT; no_collective_cause_global_expected = H5D_MPIO_SET_INDEPENDENT; + no_collective_cause_local_expected &= ~(unsigned)H5D_MPIO_NO_SELECTION_IO; + no_collective_cause_global_expected &= ~(unsigned)H5D_MPIO_NO_SELECTION_IO; /* switch to independent io */ is_independent = 1; } @@ -3607,10 +3652,6 @@ test_no_collective_cause_mode(int selection_mode) for (i = 0; i < length; i++) buffer[i] = i; - /* Set up the dxpl for the write */ - dxpl_write = H5Pcreate(H5P_DATASET_XFER); - VRFY((dxpl_write >= 0), "H5Pcreate(H5P_DATASET_XFER) succeeded"); - if (is_independent) { /* Set Independent I/O */ ret = H5Pset_dxpl_mpio(dxpl_write, H5FD_MPIO_INDEPENDENT); @@ -3642,6 +3683,20 @@ test_no_collective_cause_mode(int selection_mode) &no_collective_cause_global_write); VRFY((ret >= 0), "retrieving no collective cause succeeded"); + ret = H5Pget_no_selection_io_cause(dxpl_write, &no_selection_io_cause_write); + VRFY((ret >= 0), "retrieving no selection io cause succeeded"); + + if (no_collective_cause_local_write & H5D_MPIO_NO_SELECTION_IO) { + VRFY((no_selection_io_cause_write == no_selection_io_cause_expected), + "H5D_MPIO_NO_SELECTION_IO for write is as expected"); + } + + if (no_collective_cause_global_write & H5D_MPIO_NO_SELECTION_IO) { + + VRFY((no_selection_io_cause_write == no_selection_io_cause_expected), + "H5D_MPIO_NO_SELECTION_IO for write is as expected"); + } + /*--------------------- * Test Read access *---------------------*/ @@ -3662,6 +3717,21 @@ test_no_collective_cause_mode(int selection_mode) &no_collective_cause_global_read); VRFY((ret >= 0), "retrieving no collective cause succeeded"); + ret = H5Pget_no_selection_io_cause(dxpl_read, &no_selection_io_cause_read); + VRFY((ret >= 0), "retrieving no selection io cause succeeded"); + + if (no_collective_cause_local_read & H5D_MPIO_NO_SELECTION_IO) { + + VRFY((no_selection_io_cause_read == no_selection_io_cause_expected), + "H5D_MPIO_NO_SELECTION_IO for read is as expected"); + } + + if (no_collective_cause_global_read & H5D_MPIO_NO_SELECTION_IO) { + + VRFY((no_selection_io_cause_read == no_selection_io_cause_expected), + "H5D_MPIO_NO_SELECTION_IO for read is as expected"); + } + /* Check write vs read */ VRFY((no_collective_cause_local_read == no_collective_cause_local_write), "reading and writing are the same for local cause of Broken Collective I/O"); diff --git a/testpar/t_select_io_dset.c b/testpar/t_select_io_dset.c new file mode 100644 index 0000000..10b29e4 --- /dev/null +++ b/testpar/t_select_io_dset.c @@ -0,0 +1,3786 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * + * Purpose: Test selection I/O + */ + +#include "h5test.h" +#include "testpar.h" + +#define FILENAME "pselect_io.h5" + +/* MPI variables */ +int mpi_size; +int mpi_rank; + +/* Number of errors */ +int nerrors = 0; +int curr_nerrors = 0; + +#define P_TEST_ERROR \ + do { \ + nerrors++; \ + H5_FAILED(); \ + AT(); \ + } while (0) + +#define CHECK_PASSED() \ + do { \ + int err_result = (nerrors > curr_nerrors); \ + \ + MPI_Allreduce(MPI_IN_PLACE, &err_result, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); \ + \ + if (MAINPROCESS) { \ + if (err_result == 0) \ + PASSED(); \ + else \ + HDputs(" ***TEST FAILED***"); \ + } \ + } while (0) + +/* + * Test configurations + */ +typedef enum { + TEST_NO_TYPE_CONV, /* no type conversion (null case) */ + TEST_NO_SIZE_CHANGE_NO_BKG, /* no size change, no bkg buffer */ + TEST_LARGER_MEM_NO_BKG, /* larger memory type, no bkg buffer */ + TEST_SMALLER_MEM_NO_BKG, /* smaller memory type, no bkg buffer */ + TEST_CMPD_WITH_BKG, /* compound types with bkg buffer */ + TEST_TYPE_CONV_SEL_EMPTY, /* some processes have null/empty selections and with type conversion */ + TEST_MULTI_CONV_NO_BKG, /* multi dataset test 1 */ + TEST_MULTI_CONV_BKG, /* multi dataset test 2 */ + TEST_MULTI_CONV_SIZE_CHANGE, /* multi dataset test 3 */ + TEST_MULTI_CONV_SEL_EMPTY, /* multi dataset test 4 */ + TEST_MULTI_ALL, /* multi dataset test 5 */ + TEST_SELECT_NTESTS +} test_select_config_t; + +#define DSET_SELECT_DIM 100 +#define DSET_SELECT_CHUNK_DIM 10 + +#define MULTI_NUM_DSETS 3 +#define MULTI_MIN_DSETS 3 +#define DSET_NAME_LEN 64 + +/* Compound type */ +typedef struct s1_t { + int a; + int b; + int c; + int d; +} s1_t; + +/* + * Variation of s1 with: + * --no conversion for 2 member types + * --1 larger mem type, + * --1 smaller mem type + */ +typedef struct s2_t { + int a; + long long b; + int c; + short d; +} s2_t; + +/* Variation of s1: reverse of s1_t */ +typedef struct s3_t { + int d; + int c; + int b; + int a; +} s3_t; + +/* Variations of s1: only 2 members in s1_t */ +typedef struct s4_t { + unsigned int b; + unsigned int d; +} s4_t; + +/* Defines for test_multi_dsets_all() */ +typedef enum { + DSET_WITH_NO_CONV, /* Dataset with no type conversion */ + DSET_WITH_CONV_AND_NO_BKG, /* Dataset with type conversion but no background buffer */ + DSET_WITH_CONV_AND_BKG, /* Dataset with type conversion and background buffer */ + DSET_NTTYPES +} multi_dset_type_t; + +/* Test setting A and B */ +#define SETTING_A 1 +#define SETTING_B 2 + +/* Definitions of the test modes for test_get_no_selection_io_cause() */ +#define TEST_DISABLE_BY_API 0x001 +#define TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET 0x002 +#define TEST_DATATYPE_CONVERSION 0x004 +#define TEST_TCONV_BUF_TOO_SMALL 0x008 +#define TEST_IN_PLACE_TCONV 0x010 + +/* + * Helper routine to set dxpl + * --selection I/O mode + * --type of I/O + * --type of collective I/O + */ +static void +set_dxpl(hid_t dxpl, H5D_selection_io_mode_t select_io_mode, H5FD_mpio_xfer_t mpio_type, + H5FD_mpio_collective_opt_t mpio_coll_opt, unsigned mwbuf) +{ + if (H5Pset_selection_io(dxpl, select_io_mode) < 0) + P_TEST_ERROR; + + if (H5Pset_dxpl_mpio(dxpl, mpio_type) < 0) + P_TEST_ERROR; + + if (H5Pset_dxpl_mpio_collective_opt(dxpl, mpio_coll_opt) < 0) + P_TEST_ERROR; + + if (mwbuf) + if (H5Pset_modify_write_buf(dxpl, TRUE) < 0) + P_TEST_ERROR; + +} /* set_dxpl() */ + +/* + * Helper routine to check actual I/O mode on a dxpl + */ +static void +check_io_mode(hid_t dxpl, unsigned chunked) +{ + H5D_mpio_actual_io_mode_t actual_io_mode = H5D_MPIO_NO_COLLECTIVE; + + if (H5Pget_mpio_actual_io_mode(dxpl, &actual_io_mode) < 0) + P_TEST_ERROR; + + if (chunked) { + if (actual_io_mode != H5D_MPIO_CHUNK_COLLECTIVE) { + nerrors++; + if (MAINPROCESS) + HDprintf("\n Failed: Incorrect I/O mode (expected chunked, returned %u)", + (unsigned)actual_io_mode); + } + } + else if (actual_io_mode != H5D_MPIO_CONTIGUOUS_COLLECTIVE) { + nerrors++; + if (MAINPROCESS) + HDprintf("\n Failed: Incorrect I/O mode (expected contiguous, returned %u)", + (unsigned)actual_io_mode); + } + +} /* check_io_mode() */ + +/* + * Case 1: single dataset read/write, no type conversion (null case) + */ +static void +test_no_type_conv(hid_t fid, unsigned chunked, unsigned dtrans, unsigned mwbuf) +{ + int i; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t ntrans_dxpl = H5I_INVALID_HID; + hid_t fspace_id = H5I_INVALID_HID; + hid_t mspace_id = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + hsize_t start[1], stride[1], count[1], block[1]; + int wbuf[DSET_SELECT_DIM]; + int wbuf_bak[DSET_SELECT_DIM]; + int trans_wbuf[DSET_SELECT_DIM]; + int rbuf[DSET_SELECT_DIM]; + char dset_name[DSET_NAME_LEN]; + const char *expr = "2*x"; + + curr_nerrors = nerrors; + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + P_TEST_ERROR; + + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + P_TEST_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + P_TEST_ERROR; + } + + /* Generate dataset name */ + HDsnprintf(dset_name, sizeof(dset_name), "no_tconv_%s_%s_%s", chunked ? "chunked" : "contig", + dtrans ? "xform" : "noxform", mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create dataset */ + if ((did = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + P_TEST_ERROR; + + /* Each process takes x number of elements */ + block[0] = dims[0] / (hsize_t)mpi_size; + stride[0] = block[0]; + count[0] = 1; + start[0] = (hsize_t)mpi_rank * block[0]; + + /* Initialize data */ + for (i = 0; i < (int)block[0]; i++) { + wbuf[i] = i + (int)start[0]; + trans_wbuf[i] = 2 * wbuf[i]; + } + + /* Create a memory dataspace */ + if ((mspace_id = H5Screate_simple(1, block, NULL)) < 0) + P_TEST_ERROR; + + /* Create a file dataspace */ + if ((fspace_id = H5Dget_space(did)) < 0) + P_TEST_ERROR; + + if (H5Sselect_hyperslab(fspace_id, H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + P_TEST_ERROR; + + /* Set selection I/O mode, type of I/O and type of collective I/O */ + set_dxpl(dxpl, H5D_SELECTION_IO_MODE_ON, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, mwbuf); + + if ((ntrans_dxpl = H5Pcopy(dxpl)) < 0) + P_TEST_ERROR; + + /* Set data transform */ + if (dtrans) + if (H5Pset_data_transform(dxpl, expr) < 0) + P_TEST_ERROR; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(wbuf_bak, wbuf, sizeof(wbuf)); + + /* Write data to the dataset with/without data transform */ + if (H5Dwrite(did, H5T_NATIVE_INT, mspace_id, fspace_id, dxpl, wbuf) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(wbuf, wbuf_bak, sizeof(wbuf)); + + check_io_mode(dxpl, chunked); + + /* Read data from the dataset (if dtrans, without data transform set in dxpl) */ + if (H5Dread(did, H5T_NATIVE_INT, mspace_id, fspace_id, ntrans_dxpl, rbuf) < 0) + P_TEST_ERROR; + + /* Verify data read (if dtrans, verify data is transformed) */ + for (i = 0; i < (int)block[0]; i++) + if (rbuf[i] != (dtrans ? trans_wbuf[i] : wbuf[i])) { + nerrors++; + HDprintf("\n Error in first data verification:\n"); + HDprintf(" At index %d: %d, %d\n", i + (int)start[0], dtrans ? trans_wbuf[i] : wbuf[i], + rbuf[i]); + break; + } + + if (dtrans) { + + /* Read the data from the dataset with data transform set in dxpl */ + if (H5Dread(did, H5T_NATIVE_INT, mspace_id, fspace_id, dxpl, rbuf) < 0) + P_TEST_ERROR; + + /* Verify data read is transformed a second time */ + for (i = 0; i < (int)block[0]; i++) + if (rbuf[i] != (2 * trans_wbuf[i])) { + nerrors++; + HDprintf("\n Error in second data verification:.\n"); + HDprintf(" At index %d: %d, %d\n", i + (int)start[0], 2 * trans_wbuf[i], rbuf[i]); + break; + } + } + + if (H5Sclose(mspace_id) < 0) + P_TEST_ERROR; + if (H5Sclose(fspace_id) < 0) + P_TEST_ERROR; + + if (H5Sclose(sid) < 0) + P_TEST_ERROR; + if (H5Dclose(did) < 0) + P_TEST_ERROR; + if (H5Pclose(dxpl) < 0) + P_TEST_ERROR; + if (H5Pclose(ntrans_dxpl) < 0) + P_TEST_ERROR; + + CHECK_PASSED(); + + return; +} /* test_no_type_conv() */ + +/* + * Case 2: single dataset read/write, no size change, no background buffer + */ +static void +test_no_size_change_no_bkg(hid_t fid, unsigned chunked, unsigned mwbuf) +{ + int i; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t fspace_id = H5I_INVALID_HID; + hid_t mspace_id = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + hsize_t start[1], stride[1], count[1], block[1]; + char *wbuf = NULL; + char *wbuf_bak = NULL; + char *rbuf = NULL; + char dset_name[DSET_NAME_LEN]; + + curr_nerrors = nerrors; + + if ((wbuf = (char *)HDmalloc((size_t)(4 * DSET_SELECT_DIM))) == NULL) + P_TEST_ERROR; + if (mwbuf && (wbuf_bak = (char *)HDmalloc((size_t)(4 * DSET_SELECT_DIM))) == NULL) + P_TEST_ERROR; + if ((rbuf = (char *)HDmalloc((size_t)(4 * DSET_SELECT_DIM))) == NULL) + P_TEST_ERROR; + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + P_TEST_ERROR; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + P_TEST_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + P_TEST_ERROR; + } + + /* Generate dataset name */ + HDsnprintf(dset_name, sizeof(dset_name), "no_size_change_%s_%s", chunked ? "chunked" : "contig", + mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create 1d dataset */ + if ((did = H5Dcreate2(fid, dset_name, H5T_STD_I32BE, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + P_TEST_ERROR; + + /* Each process takes x number of elements */ + block[0] = dims[0] / (hsize_t)mpi_size; + stride[0] = block[0]; + count[0] = 1; + start[0] = (hsize_t)mpi_rank * block[0]; + + /* Initialize data */ + for (i = 0; i < (int)block[0]; i++) { + wbuf[i * 4 + 3] = 0x1; + wbuf[i * 4 + 2] = 0x2; + wbuf[i * 4 + 1] = 0x3; + wbuf[i * 4 + 0] = 0x4; + } + + /* Create a memory dataspace independently */ + if ((mspace_id = H5Screate_simple(1, block, NULL)) < 0) + P_TEST_ERROR; + + /* Create a file dataspace independently */ + if ((fspace_id = H5Dget_space(did)) < 0) + P_TEST_ERROR; + + if (H5Sselect_hyperslab(fspace_id, H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + P_TEST_ERROR; + + /* Set selection I/O mode, type of I/O and type of collective I/O */ + set_dxpl(dxpl, H5D_SELECTION_IO_MODE_ON, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, mwbuf); + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(wbuf_bak, wbuf, (size_t)(4 * DSET_SELECT_DIM)); + + /* Write the data to the dataset with little endian */ + if (H5Dwrite(did, H5T_STD_I32LE, mspace_id, fspace_id, dxpl, wbuf) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(wbuf, wbuf_bak, (size_t)(4 * DSET_SELECT_DIM)); + + check_io_mode(dxpl, chunked); + + /* Read the data from the dataset with little endian */ + if (H5Dread(did, H5T_STD_I32LE, mspace_id, fspace_id, dxpl, rbuf) < 0) + P_TEST_ERROR; + + /* Verify data read */ + for (i = 0; i < (int)block[0]; i++) { + if (rbuf[4 * i + 0] != wbuf[4 * i + 0] || rbuf[4 * i + 1] != wbuf[4 * i + 1] || + rbuf[4 * i + 2] != wbuf[4 * i + 2] || rbuf[4 * i + 3] != wbuf[4 * i + 3]) { + nerrors++; + HDprintf("\n Error in data verification:\n"); + HDprintf("\n Error in data verification at index %d\n", i + (int)start[0]); + break; + } + } + + /* Read the data from the dataset with big endian */ + if (H5Dread(did, H5T_STD_I32BE, mspace_id, fspace_id, dxpl, rbuf) < 0) + P_TEST_ERROR; + + /* Verify data read */ + for (i = 0; i < (int)block[0]; i++) { + if (rbuf[4 * i + 0] != wbuf[4 * i + 3] || rbuf[4 * i + 1] != wbuf[4 * i + 2] || + rbuf[4 * i + 2] != wbuf[4 * i + 1] || rbuf[4 * i + 3] != wbuf[4 * i + 0]) { + nerrors++; + HDprintf("\n Error in data verification at index %d\n", i + (int)start[0]); + break; + } + } + + if (H5Sclose(mspace_id) < 0) + P_TEST_ERROR; + if (H5Sclose(fspace_id) < 0) + P_TEST_ERROR; + + if (H5Sclose(sid) < 0) + P_TEST_ERROR; + if (H5Dclose(did) < 0) + P_TEST_ERROR; + + if (wbuf) + HDfree(wbuf); + + if (wbuf_bak) + HDfree(wbuf_bak); + + if (rbuf) + HDfree(rbuf); + + CHECK_PASSED(); + + return; +} /* test_no_size_change_no_bkg() */ + +/* + * Case 3: single dataset read/write, larger mem type, no background buffer + */ +static void +test_larger_mem_type_no_bkg(hid_t fid, unsigned chunked, unsigned dtrans, unsigned mwbuf) +{ + int i; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t ntrans_dxpl = H5I_INVALID_HID; + hid_t fspace_id = H5I_INVALID_HID; + hid_t mspace_id = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + hsize_t start[1], stride[1], count[1], block[1]; + long wbuf[DSET_SELECT_DIM]; + long wbuf_bak[DSET_SELECT_DIM]; + long trans_wbuf[DSET_SELECT_DIM]; + long long rbuf[DSET_SELECT_DIM]; + char dset_name[DSET_NAME_LEN]; + const char *expr = "100 - x"; + + curr_nerrors = nerrors; + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + P_TEST_ERROR; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + P_TEST_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + P_TEST_ERROR; + } + + /* Generate dataset name */ + HDsnprintf(dset_name, sizeof(dset_name), "larger_no_bkg_%s_%s_%s", chunked ? "chunked" : "contig", + dtrans ? "xform" : "noxform", mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create 1d chunked dataset with/without data transform */ + if ((did = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + P_TEST_ERROR; + + /* Each process takes x number of elements */ + block[0] = dims[0] / (hsize_t)mpi_size; + stride[0] = block[0]; + count[0] = 1; + start[0] = (hsize_t)mpi_rank * block[0]; + + /* Initialize data */ + for (i = 0; i < (int)block[0]; i++) { + wbuf[i] = i + (int)start[0]; + trans_wbuf[i] = 100 - wbuf[i]; + } + + /* Create a memory dataspace */ + if ((mspace_id = H5Screate_simple(1, block, NULL)) < 0) + P_TEST_ERROR; + + /* Create a file dataspace */ + if ((fspace_id = H5Dget_space(did)) < 0) + P_TEST_ERROR; + + if (H5Sselect_hyperslab(fspace_id, H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + P_TEST_ERROR; + + /* Set selection I/O mode, type of I/O and type of collective I/O */ + set_dxpl(dxpl, H5D_SELECTION_IO_MODE_ON, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, mwbuf); + + if ((ntrans_dxpl = H5Pcopy(dxpl)) < 0) + P_TEST_ERROR; + + /* Set data transform */ + if (dtrans) + if (H5Pset_data_transform(dxpl, expr) < 0) + P_TEST_ERROR; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(wbuf_bak, wbuf, sizeof(wbuf)); + + /* Write data to the dataset with/without data transform set in dxpl */ + if (H5Dwrite(did, H5T_NATIVE_LONG, mspace_id, fspace_id, dxpl, wbuf) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(wbuf, wbuf_bak, sizeof(wbuf)); + + check_io_mode(dxpl, chunked); + + /* Read data from the dataset (if dtrans, without data transform set in dxpl) */ + if (H5Dread(did, H5T_NATIVE_LLONG, mspace_id, fspace_id, ntrans_dxpl, rbuf) < 0) + P_TEST_ERROR; + + /* Verify data read (if dtrans, verify data is transformed) */ + for (i = 0; i < (int)block[0]; i++) + if (rbuf[i] != (long long)(dtrans ? trans_wbuf[i] : wbuf[i])) { + nerrors++; + HDprintf("\n Error in first data verification:\n"); + HDprintf(" At index %d: %lld, %lld\n", i + (int)start[0], + (long long)(dtrans ? trans_wbuf[i] : wbuf[i]), rbuf[i]); + break; + } + + if (dtrans) { + + /* Read data from the dataset with data transform set in dxpl */ + if (H5Dread(did, H5T_NATIVE_LLONG, mspace_id, fspace_id, dxpl, rbuf) < 0) + P_TEST_ERROR; + + /* Verify data read is transformed a second time */ + for (i = 0; i < (int)block[0]; i++) + if (rbuf[i] != (long long)(100 - trans_wbuf[i])) { + nerrors++; + HDprintf("\n Error in second data verification:.\n"); + HDprintf(" At index %d: %lld, %lld\n", i + (int)start[0], + (long long)(100 - trans_wbuf[i]), rbuf[i]); + break; + } + } + if (H5Sclose(mspace_id) < 0) + P_TEST_ERROR; + if (H5Sclose(fspace_id) < 0) + P_TEST_ERROR; + if (H5Sclose(sid) < 0) + P_TEST_ERROR; + if (H5Dclose(did) < 0) + P_TEST_ERROR; + if (H5Pclose(dxpl) < 0) + P_TEST_ERROR; + if (H5Pclose(ntrans_dxpl) < 0) + P_TEST_ERROR; + + CHECK_PASSED(); + + return; + +} /* test_larger_mem_type_no_bkg() */ + +/* + * Case 4: single dataset reader/write, smaller mem type, no background buffer + */ +static void +test_smaller_mem_type_no_bkg(hid_t fid, unsigned chunked, unsigned dtrans, unsigned mwbuf) +{ + int i; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t ntrans_dxpl = H5I_INVALID_HID; + hid_t fspace_id = H5I_INVALID_HID; + hid_t mspace_id = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + hsize_t start[1], stride[1], count[1], block[1]; + short wbuf[DSET_SELECT_DIM]; + int wbuf_bak[DSET_SELECT_DIM]; + short trans_wbuf[DSET_SELECT_DIM]; + short rbuf[DSET_SELECT_DIM]; + char dset_name[DSET_NAME_LEN]; + const char *expr = "2 * (10 + x)"; + + curr_nerrors = nerrors; + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + P_TEST_ERROR; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + P_TEST_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + P_TEST_ERROR; + } + + /* Generate dataset name */ + HDsnprintf(dset_name, sizeof(dset_name), "smaller_no_bkg_%s_%s_%s", chunked ? "chunked" : "contig", + dtrans ? "xform" : "noxform", mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create 1d chunked dataset with/without data transform */ + if ((did = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + P_TEST_ERROR; + + /* Each process takes x number of elements */ + block[0] = dims[0] / (hsize_t)mpi_size; + stride[0] = block[0]; + count[0] = 1; + start[0] = (hsize_t)mpi_rank * block[0]; + + /* Initialize data */ + for (i = 0; i < (int)block[0]; i++) { + wbuf[i] = (short)(i + (int)start[0]); + trans_wbuf[i] = (short)(2 * (10 + wbuf[i])); + } + + /* Create a memory dataspace */ + if ((mspace_id = H5Screate_simple(1, block, NULL)) < 0) + P_TEST_ERROR; + + /* Create a file dataspace */ + if ((fspace_id = H5Dget_space(did)) < 0) + P_TEST_ERROR; + + if (H5Sselect_hyperslab(fspace_id, H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + P_TEST_ERROR; + + /* Set selection I/O mode, type of I/O and type of collective I/O */ + set_dxpl(dxpl, H5D_SELECTION_IO_MODE_ON, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, mwbuf); + + if ((ntrans_dxpl = H5Pcopy(dxpl)) < 0) + P_TEST_ERROR; + + /* Set data transform */ + if (dtrans) { + if (H5Pset_data_transform(dxpl, expr) < 0) + P_TEST_ERROR; + } + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(wbuf_bak, wbuf, sizeof(wbuf)); + + /* Write data to the dataset with/without data transform in dxpl */ + if (H5Dwrite(did, H5T_NATIVE_SHORT, mspace_id, fspace_id, dxpl, wbuf) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(wbuf, wbuf_bak, sizeof(wbuf)); + + check_io_mode(dxpl, chunked); + + /* Read data from the dataset (if dtrans, without data transform set in dxpl) */ + if (H5Dread(did, H5T_NATIVE_SHORT, mspace_id, fspace_id, ntrans_dxpl, rbuf) < 0) + P_TEST_ERROR; + + /* Verify data read (if dtrans, verify data is transformed) */ + for (i = 0; i < (int)block[0]; i++) + if (rbuf[i] != (dtrans ? trans_wbuf[i] : wbuf[i])) { + nerrors++; + HDprintf("\n Error in first data verification:\n"); + HDprintf(" At index %d: %d, %d\n", i + (int)start[0], wbuf[i], rbuf[i]); + break; + } + + if (dtrans) { + + /* Read data from the dataset with data transform set in dxpl */ + if (H5Dread(did, H5T_NATIVE_SHORT, mspace_id, fspace_id, dxpl, rbuf) < 0) + P_TEST_ERROR; + + /* Verify data read is transformed a second time */ + for (i = 0; i < (int)block[0]; i++) + if (rbuf[i] != (2 * (10 + trans_wbuf[i]))) { + nerrors++; + HDprintf("\n Error in second data verification:.\n"); + HDprintf(" At index %d: %d, %d\n", i + (int)start[0], (2 * (10 - trans_wbuf[i])), + rbuf[i]); + break; + } + } + + if (H5Sclose(mspace_id) < 0) + P_TEST_ERROR; + if (H5Sclose(fspace_id) < 0) + P_TEST_ERROR; + + if (H5Sclose(sid) < 0) + P_TEST_ERROR; + if (H5Dclose(did) < 0) + P_TEST_ERROR; + if (H5Pclose(dxpl) < 0) + P_TEST_ERROR; + if (H5Pclose(ntrans_dxpl) < 0) + P_TEST_ERROR; + + CHECK_PASSED(); + + return; + +} /* test_smaller_mem_type_no_bkg() */ + +/* + * Case 5: single dataset reade/write, compound types with background buffer + * + * (a) Initialize compound buffer in memory with unique values + * Write all compound fields to disk + * Verify values read + * (b) Update all fields of the compound type in memory write buffer with new unique values + * Write some but not all all compound fields to disk + * Read the entire compound type + * Verify the fields have the correct (old or new) values + * (c) Update all fields of the compound type in memory read buffer with new unique values + * Read some but not all the compound fields to memory + * Verify the fields have the correct (old, middle or new) values + * (d) Set up a different compound type which has: + * --no conversion for member types + * --a field with larger mem type + * --a field with smaller mem type + * Write this compound type to disk + * Read the entire compound type + * Verify the values read + */ +static void +test_cmpd_with_bkg(hid_t fid, unsigned chunked, unsigned mwbuf) +{ + int i; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t s1_tid = H5I_INVALID_HID; + hid_t s2_tid = H5I_INVALID_HID; + hid_t ss_ac_tid = H5I_INVALID_HID; + hid_t ss_bc_tid = H5I_INVALID_HID; + hid_t fspace_id = H5I_INVALID_HID; + hid_t mspace_id = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + hsize_t start[1], stride[1], count[1], block[1]; + s1_t *s1_wbuf = NULL; + s1_t *s1_wbuf_bak = NULL; + s1_t *s1_rbuf = NULL; + s2_t *s2_wbuf = NULL; + s2_t *s2_wbuf_bak = NULL; + s2_t *s2_rbuf = NULL; + char dset_name[DSET_NAME_LEN]; + + curr_nerrors = nerrors; + + /* Allocate buffers for datasets */ + if (NULL == (s1_wbuf = (s1_t *)HDmalloc(sizeof(s1_t) * DSET_SELECT_DIM))) + P_TEST_ERROR; + if (mwbuf && NULL == (s1_wbuf_bak = (s1_t *)HDmalloc(sizeof(s1_t) * DSET_SELECT_DIM))) + P_TEST_ERROR; + if (NULL == (s1_rbuf = (s1_t *)HDmalloc(sizeof(s1_t) * DSET_SELECT_DIM))) + P_TEST_ERROR; + if (NULL == (s2_wbuf = (s2_t *)HDmalloc(sizeof(s2_t) * DSET_SELECT_DIM))) + P_TEST_ERROR; + if (mwbuf && NULL == (s2_wbuf_bak = (s2_t *)HDmalloc(sizeof(s2_t) * DSET_SELECT_DIM))) + P_TEST_ERROR; + if (NULL == (s2_rbuf = (s2_t *)HDmalloc(sizeof(s2_t) * DSET_SELECT_DIM))) + P_TEST_ERROR; + + /* Create the memory data type */ + if ((s1_tid = H5Tcreate(H5T_COMPOUND, sizeof(s1_t))) < 0) + P_TEST_ERROR; + + if (H5Tinsert(s1_tid, "a", HOFFSET(s1_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "b", HOFFSET(s1_t, b), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "c", HOFFSET(s1_t, c), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "d", HOFFSET(s1_t, d), H5T_NATIVE_INT) < 0) + P_TEST_ERROR; + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + P_TEST_ERROR; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + P_TEST_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + P_TEST_ERROR; + } + + /* Case 5(a) */ + + /* Generate dataset name */ + HDsnprintf(dset_name, sizeof(dset_name), "cmpd_with_bkg_%s_%s", chunked ? "chunked" : "contig", + mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create 1d dataset */ + if ((did = H5Dcreate2(fid, dset_name, s1_tid, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + P_TEST_ERROR; + + /* Each process takes x number of elements */ + block[0] = dims[0] / (hsize_t)mpi_size; + stride[0] = block[0]; + count[0] = 1; + start[0] = (hsize_t)mpi_rank * block[0]; + + /* Initialize data */ + for (i = 0; i < (int)block[0]; i++) { + s1_wbuf[i].a = 4 * (i + (int)start[0]); + s1_wbuf[i].b = 4 * (i + (int)start[0]) + 1; + s1_wbuf[i].c = 4 * (i + (int)start[0]) + 2; + s1_wbuf[i].d = 4 * (i + (int)start[0]) + 3; + } + + /* Create a memory dataspace */ + if ((mspace_id = H5Screate_simple(1, block, NULL)) < 0) + P_TEST_ERROR; + + /* Create a file dataspace */ + if ((fspace_id = H5Dget_space(did)) < 0) + P_TEST_ERROR; + + if (H5Sselect_hyperslab(fspace_id, H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + P_TEST_ERROR; + + /* Set selection I/O mode, type of I/O and type of collective I/O */ + set_dxpl(dxpl, H5D_SELECTION_IO_MODE_ON, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, mwbuf); + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(s1_wbuf_bak, s1_wbuf, sizeof(s1_t) * DSET_SELECT_DIM); + + /* Write all the data to the dataset */ + if (H5Dwrite(did, s1_tid, mspace_id, fspace_id, dxpl, s1_wbuf) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(s1_wbuf, s1_wbuf_bak, sizeof(s1_t) * DSET_SELECT_DIM); + + check_io_mode(dxpl, chunked); + + /* Read all the data from the dataset */ + HDmemset(s1_rbuf, 0, sizeof(s1_t) * DSET_SELECT_DIM); + if (H5Dread(did, s1_tid, mspace_id, fspace_id, dxpl, s1_rbuf) < 0) + P_TEST_ERROR; + + /* Verify data read */ + for (i = 0; i < (int)block[0]; i++) + if (s1_wbuf[i].a != s1_rbuf[i].a || s1_wbuf[i].b != s1_rbuf[i].b || s1_wbuf[i].c != s1_rbuf[i].c || + s1_wbuf[i].d != s1_rbuf[i].d) { + nerrors++; + HDprintf("\n Error in 1st data verification:\n"); + HDprintf(" At index %d: %d/%d, %d/%d, %d/%d, %d/%d\n", i + (int)start[0], s1_wbuf[i].a, + s1_rbuf[i].a, s1_wbuf[i].b, s1_rbuf[i].b, s1_wbuf[i].c, s1_rbuf[i].c, s1_wbuf[i].d, + s1_rbuf[i].d); + break; + } + + /* Case 5(b) */ + + /* Update s1_wbuf with unique values */ + for (i = 0; i < (int)block[0]; i++) { + s1_wbuf[i].a = 4 * (i + (int)start[0]) + DSET_SELECT_DIM; + s1_wbuf[i].b = 4 * (i + (int)start[0]) + DSET_SELECT_DIM + 1; + s1_wbuf[i].c = 4 * (i + (int)start[0]) + DSET_SELECT_DIM + 2; + s1_wbuf[i].d = 4 * (i + (int)start[0]) + DSET_SELECT_DIM + 3; + } + + /* Create a compound type same size as s1_t */ + if ((ss_ac_tid = H5Tcreate(H5T_COMPOUND, sizeof(s1_t))) < 0) + P_TEST_ERROR; + + /* but contains only subset members of s1_t */ + if (H5Tinsert(ss_ac_tid, "a", HOFFSET(s1_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(ss_ac_tid, "c", HOFFSET(s1_t, c), H5T_NATIVE_INT) < 0) + P_TEST_ERROR; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(s1_wbuf_bak, s1_wbuf, sizeof(s1_t) * DSET_SELECT_DIM); + + /* Write s1_wbuf to the dataset but with only subset members in ss_tid */ + if (H5Dwrite(did, ss_ac_tid, mspace_id, fspace_id, dxpl, s1_wbuf) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(s1_wbuf, s1_wbuf_bak, sizeof(s1_t) * DSET_SELECT_DIM); + + /* Read the whole compound back */ + HDmemset(s1_rbuf, 0, sizeof(s1_t) * DSET_SELECT_DIM); + if (H5Dread(did, s1_tid, mspace_id, fspace_id, dxpl, s1_rbuf) < 0) + P_TEST_ERROR; + + /* Verify the compound fields have the correct (old or new) values */ + for (i = 0; i < (int)block[0]; i++) + if (s1_rbuf[i].a != s1_wbuf[i].a || s1_rbuf[i].b != (4 * (i + (int)start[0]) + 1) || + s1_rbuf[i].c != s1_wbuf[i].c || s1_rbuf[i].d != (4 * (i + (int)start[0]) + 3)) { + nerrors++; + HDprintf("\n Error in 2nd data verification:\n"); + HDprintf(" At index %d: %d/%d, %d/%d, %d/%d, %d/%d\n", i + (int)start[0], s1_wbuf[i].a, + s1_rbuf[i].a, (4 * (i + (int)start[0]) + 1), s1_rbuf[i].b, s1_wbuf[i].c, s1_rbuf[i].c, + (4 * (i + (int)start[0]) + 3), s1_rbuf[i].d); + break; + } + + /* Case 5(c) */ + + /* Update s1_rbuf with new unique values */ + for (i = 0; i < (int)block[0]; i++) { + s1_rbuf[i].a = (4 * (i + (int)start[0])) + (2 * DSET_SELECT_DIM); + s1_rbuf[i].b = (4 * (i + (int)start[0])) + (2 * DSET_SELECT_DIM) + 1; + s1_rbuf[i].c = (4 * (i + (int)start[0])) + (2 * DSET_SELECT_DIM) + 2; + s1_rbuf[i].d = (4 * (i + (int)start[0])) + (2 * DSET_SELECT_DIM) + 3; + } + + /* Create a compound type same size as s1_t */ + if ((ss_bc_tid = H5Tcreate(H5T_COMPOUND, sizeof(s1_t))) < 0) + P_TEST_ERROR; + + /* but contains only subset members of s1_t */ + if (H5Tinsert(ss_bc_tid, "b", HOFFSET(s1_t, b), H5T_NATIVE_INT) < 0 || + H5Tinsert(ss_bc_tid, "c", HOFFSET(s1_t, c), H5T_NATIVE_INT) < 0) + P_TEST_ERROR; + + /* Read the dataset: will read only what is set in ss_bc_tid */ + if (H5Dread(did, ss_bc_tid, mspace_id, fspace_id, dxpl, s1_rbuf) < 0) + P_TEST_ERROR; + + /* Verify data read */ + for (i = 0; i < (int)block[0]; i++) + if (s1_rbuf[i].a != ((4 * (i + (int)start[0])) + (2 * DSET_SELECT_DIM)) || + s1_rbuf[i].b != (4 * (i + (int)start[0]) + 1) || + s1_rbuf[i].c != (4 * (i + (int)start[0]) + DSET_SELECT_DIM + 2) || + s1_rbuf[i].d != ((4 * (i + (int)start[0])) + (2 * DSET_SELECT_DIM) + 3)) { + nerrors++; + HDprintf("\n Error in 3rd data verification:\n"); + HDprintf(" At index %d: %d/%d, %d/%d, %d/%d, %d/%d\n", i + (int)start[0], + ((4 * (i + (int)start[0])) + (2 * DSET_SELECT_DIM)), s1_rbuf[i].a, + (4 * (i + (int)start[0]) + 1), s1_rbuf[i].b, + (4 * (i + (int)start[0]) + DSET_SELECT_DIM + 2), s1_rbuf[i].c, + ((4 * (i + (int)start[0])) + (2 * DSET_SELECT_DIM) + 3), s1_rbuf[i].d); + break; + } + + /* Case 5(d) */ + + /* Create s2_t compound type with: + * --no conversion for 2 member types, + * --1 larger mem type + * --1 smaller mem type + */ + if ((s2_tid = H5Tcreate(H5T_COMPOUND, sizeof(s2_t))) < 0) + P_TEST_ERROR; + + if (H5Tinsert(s2_tid, "a", HOFFSET(s2_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(s2_tid, "b", HOFFSET(s2_t, b), H5T_NATIVE_LLONG) < 0 || + H5Tinsert(s2_tid, "c", HOFFSET(s2_t, c), H5T_NATIVE_INT) < 0 || + H5Tinsert(s2_tid, "d", HOFFSET(s2_t, d), H5T_NATIVE_SHORT) < 0) + P_TEST_ERROR; + + /* Update s2_wbuf with unique values */ + for (i = 0; i < (int)block[0]; i++) { + s2_wbuf[i].a = (8 * (i + (int)start[0])); + s2_wbuf[i].b = (long long)(8 * (i + (int)start[0]) + 1); + s2_wbuf[i].c = (8 * (i + (int)start[0]) + 2); + s2_wbuf[i].d = (short)(8 * (i + (int)start[0]) + 3); + } + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(s2_wbuf_bak, s2_wbuf, sizeof(s2_t) * DSET_SELECT_DIM); + + if (H5Dwrite(did, s2_tid, mspace_id, fspace_id, dxpl, s2_wbuf) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(s2_wbuf, s2_wbuf_bak, sizeof(s2_t) * DSET_SELECT_DIM); + + /* Read it back */ + HDmemset(s2_rbuf, 0, sizeof(s2_t) * DSET_SELECT_DIM); + if (H5Dread(did, s2_tid, mspace_id, fspace_id, dxpl, s2_rbuf) < 0) + P_TEST_ERROR; + + /* Verify data read */ + for (i = 0; i < (int)block[0]; i++) + if (s2_wbuf[i].a != s2_rbuf[i].a || s2_wbuf[i].b != s2_rbuf[i].b || s2_wbuf[i].c != s2_rbuf[i].c || + s2_wbuf[i].d != s2_rbuf[i].d) { + nerrors++; + HDprintf("\n Error in 4th data verification:\n"); + HDprintf(" At index %d: %d/%d, %lld/%lld, %d/%d, %d/%d\n", i + (int)start[0], s2_wbuf[i].a, + s2_rbuf[i].a, s2_wbuf[i].b, s2_rbuf[i].b, s2_wbuf[i].c, s2_rbuf[i].c, s2_wbuf[i].d, + s2_rbuf[i].d); + break; + } + + if (H5Sclose(mspace_id) < 0) + P_TEST_ERROR; + if (H5Sclose(fspace_id) < 0) + P_TEST_ERROR; + + if (H5Sclose(sid) < 0) + P_TEST_ERROR; + if (H5Tclose(s1_tid) < 0) + P_TEST_ERROR; + if (H5Tclose(s2_tid) < 0) + P_TEST_ERROR; + if (H5Tclose(ss_ac_tid) < 0) + P_TEST_ERROR; + if (H5Tclose(ss_bc_tid) < 0) + P_TEST_ERROR; + if (H5Dclose(did) < 0) + P_TEST_ERROR; + + /* Release buffers */ + HDfree(s1_wbuf); + HDfree(s1_wbuf_bak); + HDfree(s1_rbuf); + HDfree(s2_wbuf); + HDfree(s2_wbuf_bak); + HDfree(s2_rbuf); + + CHECK_PASSED(); + + return; + +} /* test_cmpd_with_bkg() */ + +/* + * Case 6: Type conversions + some processes have null/empty selections in datasets + */ +static void +test_type_conv_sel_empty(hid_t fid, unsigned chunked, unsigned dtrans, unsigned mwbuf) +{ + int i; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t ntrans_dxpl = H5I_INVALID_HID; + hid_t fspace_id = H5I_INVALID_HID; + hid_t mspace_id = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + hsize_t start[1], stride[1], count[1], block[1]; + + long lwbuf[DSET_SELECT_DIM]; + long lwbuf_bak[DSET_SELECT_DIM]; + long trans_lwbuf[DSET_SELECT_DIM]; + long lrbuf[DSET_SELECT_DIM]; + short srbuf[DSET_SELECT_DIM]; + short swbuf[DSET_SELECT_DIM]; + short swbuf_bak[DSET_SELECT_DIM]; + short trans_swbuf[DSET_SELECT_DIM]; + long long llrbuf[DSET_SELECT_DIM]; + char dset_name[DSET_NAME_LEN]; + + const char *expr = "2*x"; + + curr_nerrors = nerrors; + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + P_TEST_ERROR; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + P_TEST_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + P_TEST_ERROR; + } + + /* Generate dataset name */ + HDsnprintf(dset_name, sizeof(dset_name), "tconv_sel_empty_%s_%s_%s", chunked ? "chunked" : "contig", + dtrans ? "xform" : "noxform", mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create dataset */ + if ((did = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + P_TEST_ERROR; + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + P_TEST_ERROR; + + /* Set selection I/O mode, type of I/O and type of collective I/O */ + set_dxpl(dxpl, H5D_SELECTION_IO_MODE_ON, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, mwbuf); + + if ((ntrans_dxpl = H5Pcopy(dxpl)) < 0) + P_TEST_ERROR; + + /* Set data transform */ + if (dtrans) { + if (H5Pset_data_transform(dxpl, expr) < 0) + P_TEST_ERROR; + } + + /* Each process takes x number of elements */ + block[0] = dims[0] / (hsize_t)mpi_size; + stride[0] = block[0]; + count[0] = 1; + start[0] = (hsize_t)mpi_rank * block[0]; + + /* Initialize data */ + for (i = 0; i < (int)block[0]; i++) { + lwbuf[i] = i + (int)start[0]; + trans_lwbuf[i] = 2 * lwbuf[i]; + } + + /* Case 6(a) process 0: hyperslab; other processes: select none */ + + /* Create a file dataspace */ + if ((fspace_id = H5Dget_space(did)) < 0) + P_TEST_ERROR; + if (MAINPROCESS) { + if (H5Sselect_hyperslab(fspace_id, H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + } + else { + if (H5Sselect_none(fspace_id) < 0) + P_TEST_ERROR; + } + + /* Create a memory dataspace */ + if ((mspace_id = H5Screate_simple(1, block, NULL)) < 0) + P_TEST_ERROR; + if (mpi_rank) { + if (H5Sselect_none(mspace_id) < 0) + P_TEST_ERROR; + } + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(lwbuf_bak, lwbuf, sizeof(lwbuf)); + + /* Write data to the dataset with/without data transform in dxpl */ + if (H5Dwrite(did, H5T_NATIVE_LONG, mspace_id, fspace_id, dxpl, lwbuf) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(lwbuf, lwbuf_bak, sizeof(lwbuf)); + + check_io_mode(dxpl, chunked); + + /* Read the data from the dataset: type conversion int-->long */ + /* If dtrans, without data transform set in dxpl */ + if (H5Dread(did, H5T_NATIVE_LONG, mspace_id, fspace_id, ntrans_dxpl, lrbuf) < 0) + P_TEST_ERROR; + + if (MAINPROCESS) { + for (i = 0; i < (int)block[0]; i++) + if (lrbuf[i] != (dtrans ? trans_lwbuf[i] : lwbuf[i])) { + nerrors++; + HDprintf("\n Error in first data verification:\n"); + HDprintf(" At index %d: %ld, %ld\n", i + (int)start[0], + dtrans ? trans_lwbuf[i] : lwbuf[i], lrbuf[i]); + break; + } + } + + if (H5Sclose(mspace_id) < 0) + P_TEST_ERROR; + if (H5Sclose(fspace_id) < 0) + P_TEST_ERROR; + + /* Case 6(b) process 0: get 0 row; other processes: hyperslab */ + + block[0] = mpi_rank ? (dims[0] / (hsize_t)mpi_size) : 0; + stride[0] = mpi_rank ? block[0] : 1; + count[0] = 1; + start[0] = mpi_rank ? ((hsize_t)mpi_rank * block[0]) : 0; + + /* Create a file dataspace */ + if ((fspace_id = H5Dget_space(did)) < 0) + P_TEST_ERROR; + + if (H5Sselect_hyperslab(fspace_id, H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + + /* Create a memory dataspace */ + if ((mspace_id = H5Screate_simple(1, block, NULL)) < 0) + P_TEST_ERROR; + + /* need to make memory space to match for process 0 */ + if (MAINPROCESS) { + if (H5Sselect_hyperslab(mspace_id, H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + } + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(lwbuf_bak, lwbuf, sizeof(lwbuf)); + + /* Write data to the dataset with/without data transform */ + if (H5Dwrite(did, H5T_NATIVE_LONG, mspace_id, fspace_id, dxpl, lwbuf) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(lwbuf, lwbuf_bak, sizeof(lwbuf)); + + /* Read the data from the dataset: type conversion int-->short */ + /* If dtrans, without data transform set in dxpl */ + if (H5Dread(did, H5T_NATIVE_SHORT, mspace_id, fspace_id, ntrans_dxpl, srbuf) < 0) + P_TEST_ERROR; + + if (mpi_rank) { + for (i = 0; i < (int)block[0]; i++) + if (srbuf[i] != (short)(dtrans ? trans_lwbuf[i] : lwbuf[i])) { + HDprintf("\n Error in second data verification:\n"); + HDprintf(" At index %d: %d, %d\n", i + (int)start[0], + (short)(dtrans ? trans_lwbuf[i] : lwbuf[i]), srbuf[i]); + break; + } + } + + if (H5Sclose(mspace_id) < 0) + P_TEST_ERROR; + if (H5Sclose(fspace_id) < 0) + P_TEST_ERROR; + + /* Case 6(c) process 0: select none; other processes: select all */ + + /* Initialize data */ + block[0] = DSET_SELECT_DIM; + for (i = 0; i < (int)block[0]; i++) { + swbuf[i] = (short)(i + DSET_SELECT_DIM); + trans_swbuf[i] = (short)(2 * swbuf[i]); + } + + /* Create a file dataspace */ + if ((fspace_id = H5Dget_space(did)) < 0) + P_TEST_ERROR; + if (MAINPROCESS) { + if (H5Sselect_none(fspace_id) < 0) + P_TEST_ERROR; + } + else { + if (H5Sselect_all(fspace_id) < 0) + P_TEST_ERROR; + } + + /* Create a memory dataspace */ + if ((mspace_id = H5Screate_simple(1, block, NULL)) < 0) + P_TEST_ERROR; + if (MAINPROCESS) { + if (H5Sselect_none(mspace_id) < 0) + P_TEST_ERROR; + } + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(swbuf_bak, swbuf, sizeof(swbuf)); + + /* Write data to the dataset with/without data transform */ + if (H5Dwrite(did, H5T_NATIVE_SHORT, mspace_id, fspace_id, dxpl, swbuf) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(swbuf, swbuf_bak, sizeof(swbuf)); + + /* Read the data from the dataset: type conversion int-->llong */ + /* If dtrans, without data transform set in dxpl */ + if (H5Dread(did, H5T_NATIVE_LLONG, mspace_id, fspace_id, ntrans_dxpl, llrbuf) < 0) + P_TEST_ERROR; + + if (mpi_rank) { + for (i = 0; i < (int)block[0]; i++) + if (llrbuf[i] != (long long)(dtrans ? trans_swbuf[i] : swbuf[i])) { + HDprintf("\n Error in third data verification:\n"); + HDprintf(" At index %d: %lld, %lld\n", i + (int)start[0], + (long long)(dtrans ? trans_swbuf[i] : swbuf[i]), llrbuf[i]); + break; + } + } + + if (H5Sclose(mspace_id) < 0) + P_TEST_ERROR; + if (H5Sclose(fspace_id) < 0) + P_TEST_ERROR; + + if (H5Sclose(sid) < 0) + P_TEST_ERROR; + if (H5Dclose(did) < 0) + P_TEST_ERROR; + if (H5Pclose(dxpl) < 0) + P_TEST_ERROR; + if (H5Pclose(ntrans_dxpl) < 0) + P_TEST_ERROR; + + CHECK_PASSED(); + + return; + +} /* test_type_conv_sel_empty() */ + +/* + * Test 1 for multi-dataset: + * --Datasets with/without type conversion+smaller/larger mem type+no background buffer + * + * Create datasets: randomized H5T_NATIVE_INT or H5T_NATIVE_LONG + * + * Case a--setting for multi write/read to ndsets: + * Datatype for all datasets: H5T_NATIVE_INT + * + * Case b--setting for multi write/read to ndsets: + * Datatype for all datasets: H5T_NATIVE_LONG + */ +static void +test_multi_dsets_no_bkg(hid_t fid, unsigned chunked, unsigned dtrans, unsigned mwbuf) +{ + size_t ndsets; + int i, j; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t ntrans_dxpl = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + hsize_t start[1], stride[1], count[1], block[1]; + + hid_t file_sids[MULTI_NUM_DSETS]; + hid_t mem_sids[MULTI_NUM_DSETS]; + hid_t mem_tids[MULTI_NUM_DSETS]; + + char dset_names[MULTI_NUM_DSETS][DSET_NAME_LEN]; + hid_t dset_dids[MULTI_NUM_DSETS]; + + size_t buf_size; + + int *total_wbuf = NULL; + int *total_wbuf_bak = NULL; + int *total_trans_wbuf = NULL; + int *total_rbuf = NULL; + + long *total_lwbuf = NULL; + long *total_lwbuf_bak = NULL; + long *total_trans_lwbuf = NULL; + long *total_lrbuf = NULL; + + int *wbufi[MULTI_NUM_DSETS]; + int *trans_wbufi[MULTI_NUM_DSETS]; + int *rbufi[MULTI_NUM_DSETS]; + + long *lwbufi[MULTI_NUM_DSETS]; + long *trans_lwbufi[MULTI_NUM_DSETS]; + long *lrbufi[MULTI_NUM_DSETS]; + + const void *wbufs[MULTI_NUM_DSETS]; + void *rbufs[MULTI_NUM_DSETS]; + const char *expr = "2*x"; + + curr_nerrors = nerrors; + + ndsets = MAX(MULTI_MIN_DSETS, MULTI_NUM_DSETS); + + dims[0] = DSET_SELECT_DIM; + + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + P_TEST_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + P_TEST_ERROR; + } + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + P_TEST_ERROR; + + /* Set selection I/O mode, type of I/O and type of collective I/O */ + set_dxpl(dxpl, H5D_SELECTION_IO_MODE_ON, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, mwbuf); + + if ((ntrans_dxpl = H5Pcopy(dxpl)) < 0) + P_TEST_ERROR; + + /* Set data transform */ + if (dtrans) + if (H5Pset_data_transform(dxpl, expr) < 0) + P_TEST_ERROR; + + /* Set up file space ids and dataset ids */ + for (i = 0; i < (int)ndsets; i++) { + if ((file_sids[i] = H5Screate_simple(1, dims, NULL)) < 0) + P_TEST_ERROR; + + /* Generate dataset name */ + HDsnprintf(dset_names[i], sizeof(dset_names[i]), "multi_dset%d_%s_%s_%s", i, + chunked ? "chunked" : "contig", dtrans ? "xform" : "noxform", mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create ith dataset */ + if ((dset_dids[i] = + H5Dcreate2(fid, dset_names[i], ((HDrandom() % 2) ? H5T_NATIVE_LONG : H5T_NATIVE_INT), + file_sids[i], H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + P_TEST_ERROR; + } + + /* Each process takes x number of elements */ + block[0] = dims[0] / (hsize_t)mpi_size; + stride[0] = block[0]; + count[0] = 1; + start[0] = (hsize_t)mpi_rank * block[0]; + + for (i = 0; i < (int)ndsets; i++) { + if ((mem_sids[i] = H5Screate_simple(1, block, NULL)) < 0) + P_TEST_ERROR; + + if (H5Sselect_hyperslab(file_sids[i], H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + } + + buf_size = ndsets * DSET_SELECT_DIM * sizeof(int); + + /* Allocate buffers for all datasets */ + if (NULL == (total_wbuf = (int *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (mwbuf && NULL == (total_wbuf_bak = (int *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_trans_wbuf = (int *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_rbuf = (int *)HDmalloc(buf_size))) + P_TEST_ERROR; + + buf_size = ndsets * DSET_SELECT_DIM * sizeof(long); + + if (NULL == (total_lwbuf = (long *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (mwbuf && NULL == (total_lwbuf_bak = (long *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_trans_lwbuf = (long *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_lrbuf = (long *)HDmalloc(buf_size))) + P_TEST_ERROR; + + /* Initialize buffer indices */ + for (i = 0; i < (int)ndsets; i++) { + wbufi[i] = total_wbuf + (i * DSET_SELECT_DIM); + rbufi[i] = total_rbuf + (i * DSET_SELECT_DIM); + trans_wbufi[i] = total_trans_wbuf + (i * DSET_SELECT_DIM); + + wbufs[i] = wbufi[i]; + rbufs[i] = rbufi[i]; + } + + /* Case a */ + + /* Initialize the buffer data */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < (int)block[0]; j++) { + wbufi[i][j] = j + (int)start[0]; + trans_wbufi[i][j] = 2 * wbufi[i][j]; + } + + /* Datatype setting for multi write/read */ + for (i = 0; i < (int)ndsets; i++) + mem_tids[i] = H5T_NATIVE_INT; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_wbuf_bak, total_wbuf, ndsets * DSET_SELECT_DIM * sizeof(int)); + + /* Write data to the dataset with/without data transform */ + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_wbuf, total_wbuf_bak, ndsets * DSET_SELECT_DIM * sizeof(int)); + + check_io_mode(dxpl, chunked); + + /* Read data from the dataset (if dtrans, without data transform set in dxpl) */ + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, ntrans_dxpl, rbufs) < 0) + P_TEST_ERROR; + + /* Verify */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < (int)block[0]; j++) + if (rbufi[i][j] != (dtrans ? trans_wbufi[i][j] : wbufi[i][j])) { + nerrors++; + HDprintf("\n Error in 1st data verification for dset %d:\n", i); + HDprintf(" At index %d: %d, %d\n", j + (int)start[0], + dtrans ? trans_wbufi[i][j] : wbufi[i][j], rbufi[i][j]); + break; + } + + if (dtrans) { + + /* Read the data from the dataset with data transform set in dxpl */ + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + P_TEST_ERROR; + + /* Verify */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < (int)block[0]; j++) + if (rbufi[i][j] != (2 * trans_wbufi[i][j])) { + nerrors++; + HDprintf("\n Error in 1st (with dtrans) data verification for dset %d:\n", i); + HDprintf(" At index %d: %d, %d\n", j + (int)start[0], 2 * trans_wbufi[i][j], + rbufi[i][j]); + break; + } + } + + /* Case b */ + + /* Initialize buffer indices */ + for (i = 0; i < (int)ndsets; i++) { + lwbufi[i] = total_lwbuf + (i * DSET_SELECT_DIM); + trans_lwbufi[i] = total_trans_lwbuf + (i * DSET_SELECT_DIM); + lrbufi[i] = total_lrbuf + (i * DSET_SELECT_DIM); + wbufs[i] = lwbufi[i]; + rbufs[i] = lrbufi[i]; + } + + /* Initialize the buffer data */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < (int)block[0]; j++) { + lwbufi[i][j] = j + (int)start[0] + DSET_SELECT_DIM; + trans_lwbufi[i][j] = 2 * lwbufi[i][j]; + } + + /* Datatype setting for multi write/read */ + for (i = 0; i < (int)ndsets; i++) + mem_tids[i] = H5T_NATIVE_LONG; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_lwbuf_bak, total_lwbuf, ndsets * DSET_SELECT_DIM * sizeof(long)); + + /* Write data to the dataset with/without data transform */ + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_lwbuf, total_lwbuf_bak, ndsets * DSET_SELECT_DIM * sizeof(long)); + + /* Read data from the dataset (if dtrans, with data transform again in dxpl */ + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + P_TEST_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + for (j = 0; j < (int)block[0]; j++) { + if (lrbufi[i][j] != (dtrans ? (2 * trans_lwbufi[i][j]) : lwbufi[i][j])) { + nerrors++; + HDprintf("\n Error in 2nd data verification for dset %d:\n", i); + HDprintf(" At index %d: %ld/%ld\n", j + (int)start[0], + (dtrans ? (2 * trans_lwbufi[i][j]) : lwbufi[i][j]), lrbufi[i][j]); + break; + } + } + } + + if (H5Pclose(dcpl) < 0) + P_TEST_ERROR; + if (H5Pclose(dxpl) < 0) + P_TEST_ERROR; + if (H5Pclose(ntrans_dxpl) < 0) + P_TEST_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + if (H5Sclose(file_sids[i]) < 0) + P_TEST_ERROR; + if (H5Sclose(mem_sids[i]) < 0) + P_TEST_ERROR; + if (H5Dclose(dset_dids[i]) < 0) + P_TEST_ERROR; + } + + HDfree(total_wbuf); + HDfree(total_wbuf_bak); + HDfree(total_rbuf); + HDfree(total_trans_wbuf); + HDfree(total_lwbuf); + HDfree(total_lwbuf_bak); + HDfree(total_trans_lwbuf); + HDfree(total_lrbuf); + + CHECK_PASSED(); + + return; + +} /* test_multi_dsets_no_bkg() */ + +/* + * Test 2 for multi-dataset: + * Datasets with compound types+background buffer + * + * Create datasets with the same compound type + * Case (a) Initialize compound buffer in memory with unique values + * All datasets: + * --Write all compound fields to disk + * --Read the entire compound type for all datasets + * --Verify values read + * Case (b) Update all fields of the compound type in memory write buffer with new unique values + * dset0: + * --Write some but not all all compound fields to disk + * --Read and verify the fields have the correct (old(a) or new) values + * Remaining datasets: + * --Untouched + * --Read and verify the fields have the correct old(a) values + * Case (c) Update all fields of the compound type in memory read buffer with new unique values + * Randomized <mm> dataset: + * --Read some but not all the compound fields to memory + * --Verify the fields have the correct (old(a) or new) values + * dset0: + * --Untouched + * --Read and verify the fields have the correct (old(a) or middle(b)) values + * Remaining datasets: + * --Untouched + * --Read and verify the fields have the correct old(a) values + * Case (d) Set up a different compound type which has: + * --no type conversion for 2 member types + * --a field with larger mem type + * --a field with smaller mem type + * All datasets: + * --Write the compound fields to disk + * --Read the entire compound type + * --Verify values read + */ +static void +test_multi_dsets_cmpd_with_bkg(hid_t fid, unsigned chunked, unsigned mwbuf) +{ + size_t ndsets; + int i, j, mm; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + hsize_t start[1], stride[1], count[1], block[1]; + + hid_t file_sids[MULTI_NUM_DSETS]; + hid_t mem_sids[MULTI_NUM_DSETS]; + hid_t mem_tids[MULTI_NUM_DSETS]; + + hid_t s1_tid = H5I_INVALID_HID; + hid_t ss_ac_tid = H5I_INVALID_HID; + hid_t ss_bc_tid = H5I_INVALID_HID; + hid_t s2_tid = H5I_INVALID_HID; + + char dset_names[MULTI_NUM_DSETS][DSET_NAME_LEN]; + hid_t dset_dids[MULTI_NUM_DSETS]; + + size_t buf_size; + size_t s2_buf_size; + + s1_t *total_wbuf = NULL; + s1_t *total_wbuf_bak = NULL; + s1_t *total_rbuf = NULL; + + s2_t *s2_total_wbuf = NULL; + s2_t *s2_total_wbuf_bak = NULL; + s2_t *s2_total_rbuf = NULL; + + s1_t *wbufi[MULTI_NUM_DSETS]; + s1_t *rbufi[MULTI_NUM_DSETS]; + + s2_t *s2_wbufi[MULTI_NUM_DSETS]; + s2_t *s2_rbufi[MULTI_NUM_DSETS]; + + const void *wbufs[MULTI_NUM_DSETS]; + void *rbufs[MULTI_NUM_DSETS]; + + curr_nerrors = nerrors; + + ndsets = MAX(MULTI_MIN_DSETS, MULTI_NUM_DSETS); + + dims[0] = DSET_SELECT_DIM; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + P_TEST_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + P_TEST_ERROR; + } + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + P_TEST_ERROR; + + /* Set selection I/O mode, type of I/O and type of collective I/O */ + set_dxpl(dxpl, H5D_SELECTION_IO_MODE_ON, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, mwbuf); + + /* Each process takes x number of elements */ + block[0] = dims[0] / (hsize_t)mpi_size; + stride[0] = block[0]; + count[0] = 1; + start[0] = (hsize_t)mpi_rank * block[0]; + + /* Create the memory data type */ + if ((s1_tid = H5Tcreate(H5T_COMPOUND, sizeof(s1_t))) < 0) + P_TEST_ERROR; + + if (H5Tinsert(s1_tid, "a", HOFFSET(s1_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "b", HOFFSET(s1_t, b), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "c", HOFFSET(s1_t, c), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "d", HOFFSET(s1_t, d), H5T_NATIVE_INT) < 0) + P_TEST_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + if ((file_sids[i] = H5Screate_simple(1, dims, NULL)) < 0) + P_TEST_ERROR; + if ((mem_sids[i] = H5Screate_simple(1, block, NULL)) < 0) + P_TEST_ERROR; + + /* Generate dataset name */ + HDsnprintf(dset_names[i], sizeof(dset_names[i]), "multi_cmpd_dset%d_%s_%s", i, + chunked ? "chunked" : "contig", mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create ith dataset */ + if ((dset_dids[i] = + H5Dcreate2(fid, dset_names[i], s1_tid, file_sids[i], H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + P_TEST_ERROR; + + if (H5Sselect_hyperslab(file_sids[i], H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + } + + buf_size = ndsets * DSET_SELECT_DIM * sizeof(s1_t); + s2_buf_size = ndsets * DSET_SELECT_DIM * sizeof(s2_t); + + /* Allocate buffers for all datasets */ + if (NULL == (total_wbuf = (s1_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (mwbuf && NULL == (total_wbuf_bak = (s1_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_rbuf = (s1_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + + if (NULL == (s2_total_wbuf = (s2_t *)HDmalloc(s2_buf_size))) + P_TEST_ERROR; + if (mwbuf && NULL == (s2_total_wbuf_bak = (s2_t *)HDmalloc(s2_buf_size))) + P_TEST_ERROR; + if (NULL == (s2_total_rbuf = (s2_t *)HDmalloc(s2_buf_size))) + P_TEST_ERROR; + + /* Initialize buffer indices */ + for (i = 0; i < (int)ndsets; i++) { + wbufi[i] = total_wbuf + (i * DSET_SELECT_DIM); + rbufi[i] = total_rbuf + (i * DSET_SELECT_DIM); + + wbufs[i] = wbufi[i]; + rbufs[i] = rbufi[i]; + } + + /* Case a */ + + /* Initialize the buffer data for all the datasets */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < (int)block[0]; j++) { + wbufi[i][j].a = 4 * (j + (int)start[0]); + wbufi[i][j].b = 4 * (j + (int)start[0]) + 1; + wbufi[i][j].c = 4 * (j + (int)start[0]) + 2; + wbufi[i][j].d = 4 * (j + (int)start[0]) + 3; + } + + /* Datatype setting for multi write */ + for (i = 0; i < (int)ndsets; i++) + mem_tids[i] = s1_tid; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_wbuf_bak, total_wbuf, buf_size); + + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_wbuf, total_wbuf_bak, buf_size); + + check_io_mode(dxpl, chunked); + + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + P_TEST_ERROR; + + /* Verify data read */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < (int)block[0]; j++) { + if (wbufi[i][j].a != rbufi[i][j].a || wbufi[i][j].b != rbufi[i][j].b || + wbufi[i][j].c != rbufi[i][j].c || wbufi[i][j].d != rbufi[i][j].d) { + nerrors++; + HDprintf("\n Error in 1st data verification for dset %d:\n", i); + HDprintf(" At index %d: %d/%d, %d/%d, %d/%d, %d/%d\n", j + (int)start[0], wbufi[i][j].a, + rbufi[i][j].a, wbufi[i][j].b, rbufi[i][j].b, wbufi[i][j].c, rbufi[i][j].c, + wbufi[i][j].d, rbufi[i][j].d); + + break; + } + } + + /* Case b */ + + /* Update data in wbufi for dset0 with unique values */ + for (j = 0; j < (int)block[0]; j++) { + wbufi[0][j].a = (4 * (j + (int)start[0])) + DSET_SELECT_DIM; + wbufi[0][j].b = (4 * (j + (int)start[0])) + DSET_SELECT_DIM + 1; + wbufi[0][j].c = (4 * (j + (int)start[0])) + DSET_SELECT_DIM + 2; + wbufi[0][j].d = (4 * (j + (int)start[0])) + DSET_SELECT_DIM + 3; + } + + /* Create a compound type same size as s1_t */ + if ((ss_ac_tid = H5Tcreate(H5T_COMPOUND, sizeof(s1_t))) < 0) + P_TEST_ERROR; + + /* but contains only subset members of s1_t */ + if (H5Tinsert(ss_ac_tid, "a", HOFFSET(s1_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(ss_ac_tid, "c", HOFFSET(s1_t, c), H5T_NATIVE_INT) < 0) + P_TEST_ERROR; + + /* Datatype setting for write to dset0 */ + mem_tids[0] = ss_ac_tid; + + /* Untouched memory and file spaces for other datasets */ + for (i = 0; i < (int)ndsets; i++) { + if (i == 0) + continue; + + if (H5Sselect_none(mem_sids[i]) < 0) + P_TEST_ERROR; + if (H5Sselect_none(file_sids[i]) < 0) + P_TEST_ERROR; + } + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_wbuf_bak, total_wbuf, buf_size); + + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_wbuf, total_wbuf_bak, buf_size); + + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + P_TEST_ERROR; + + /* Verify data read */ + for (i = 0; i < (int)ndsets; i++) + if (i == 0) { /* dset0 */ + for (j = 0; j < (int)block[0]; j++) + if (wbufi[i][j].a != rbufi[i][j].a || (4 * (j + (int)start[0]) + 1) != rbufi[i][j].b || + wbufi[i][j].c != rbufi[i][j].c || (4 * (j + (int)start[0]) + 3) != rbufi[i][j].d) { + nerrors++; + HDprintf("\n Error in 2nd data verification for dset %d:\n", i); + HDprintf(" At index %d: %d/%d, %d/%d, %d/%d, %d/%d\n", j + (int)start[0], + wbufi[i][j].a, rbufi[i][j].a, (4 * (j + (int)start[0]) + 1), rbufi[i][j].b, + wbufi[i][j].c, rbufi[i][j].c, (4 * (j + (int)start[0]) + 3), rbufi[i][j].d); + break; + } + } + else { /* other datasets */ + for (j = 0; j < (int)block[0]; j++) + if ((4 * (j + (int)start[0])) != rbufi[i][j].a || + (4 * (j + (int)start[0]) + 1) != rbufi[i][j].b || + (4 * (j + (int)start[0]) + 2) != rbufi[i][j].c || + (4 * (j + (int)start[0]) + 3) != rbufi[i][j].d) { + nerrors++; + HDprintf("\n Error in 2nd data verification for dset %d:\n", i); + HDprintf(" At index %d: %d/%d, %d/%d, %d/%d, %d/%d\n", j + (int)start[0], + (4 * (j + (int)start[0])), rbufi[i][j].a, (4 * (j + (int)start[0]) + 1), + rbufi[i][j].b, (4 * (j + (int)start[0]) + 2), rbufi[i][j].c, + (4 * (j + (int)start[0]) + 3), rbufi[i][j].d); + break; + } + } + + /* Case c */ + mm = HDrandom() % (int)ndsets; + if (!mm) + mm++; + + /* Update data in rbufi for <mm> dset with new unique values */ + for (j = 0; j < (int)block[0]; j++) { + rbufi[mm][j].a = (4 * (j + (int)start[0])) + (2 * DSET_SELECT_DIM); + rbufi[mm][j].b = (4 * (j + (int)start[0])) + (2 * DSET_SELECT_DIM) + 1; + rbufi[mm][j].c = (4 * (j + (int)start[0])) + (2 * DSET_SELECT_DIM) + 2; + rbufi[mm][j].d = (4 * (j + (int)start[0])) + (2 * DSET_SELECT_DIM) + 3; + } + + /* Create a compound type same size as s1_t */ + if ((ss_bc_tid = H5Tcreate(H5T_COMPOUND, sizeof(s1_t))) < 0) + P_TEST_ERROR; + + /* but contains only subset members of s1_t */ + if (H5Tinsert(ss_bc_tid, "b", HOFFSET(s1_t, b), H5T_NATIVE_INT) < 0 || + H5Tinsert(ss_bc_tid, "c", HOFFSET(s1_t, c), H5T_NATIVE_INT) < 0) + P_TEST_ERROR; + + /* Reset memory and file spaces for <mm> dset */ + if (H5Sselect_all(mem_sids[mm]) < 0) + P_TEST_ERROR; + if (H5Sselect_all(file_sids[mm]) < 0) + P_TEST_ERROR; + if (H5Sselect_hyperslab(file_sids[mm], H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + + /* Untouched memory and file space for other datasets */ + for (i = 0; i < (int)ndsets; i++) { + if (i == 0 || i == mm) + continue; + if (H5Sselect_none(mem_sids[i]) < 0) + P_TEST_ERROR; + if (H5Sselect_none(file_sids[i]) < 0) + P_TEST_ERROR; + } + + /* Datatype setting for read from <mm> dataset */ + mem_tids[mm] = ss_bc_tid; + + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + P_TEST_ERROR; + + /* Verify data read */ + /* dset0 */ + for (j = 0; j < (int)block[0]; j++) + if (wbufi[0][j].a != rbufi[0][j].a || ((4 * (j + (int)start[0])) + 1) != rbufi[0][j].b || + wbufi[0][j].c != rbufi[0][j].c || ((4 * (j + (int)start[0])) + 3) != rbufi[0][j].d) { + nerrors++; + HDprintf("\n Error in 3rd data verification for dset0:\n"); + HDprintf(" At index %d: %d/%d, %d/%d, %d/%d, %d/%d\n", j + (int)start[0], wbufi[0][j].a, + rbufi[0][j].a, (4 * (j + (int)start[0]) + 1), rbufi[0][j].b, wbufi[0][j].c, + rbufi[0][j].c, (4 * (j + (int)start[0]) + 3), rbufi[0][j].d); + break; + } + + /* <mm> dset */ + for (j = 0; j < (int)block[0]; j++) + if (rbufi[mm][j].a != ((4 * (j + (int)start[0])) + (2 * DSET_SELECT_DIM)) || + rbufi[mm][j].b != ((4 * (j + (int)start[0])) + 1) || + rbufi[mm][j].c != ((4 * (j + (int)start[0])) + 2) || + rbufi[mm][j].d != ((4 * (j + (int)start[0])) + (2 * DSET_SELECT_DIM) + 3)) { + nerrors++; + HDprintf("\n Error in 3rd data verification for dset %d:\n", mm); + HDprintf(" At index %d: %d/%d, %d/%d, %d/%d, %d/%d\n", j + (int)start[0], + ((4 * (j + (int)start[0])) + (2 * DSET_SELECT_DIM)), rbufi[mm][j].a, + ((4 * (j + (int)start[0])) + 1), rbufi[mm][j].b, ((4 * (j + (int)start[0])) + 2), + rbufi[mm][j].c, ((4 * (j + (int)start[0])) + (2 * DSET_SELECT_DIM) + 3), rbufi[mm][j].d); + break; + } + + /* other datasets */ + for (i = 0; i < (int)ndsets; i++) { + if (i == 0 || i == mm) + continue; + + for (j = 0; j < (int)block[0]; j++) + if (rbufi[i][j].a != ((4 * (j + (int)start[0]))) || + rbufi[i][j].b != ((4 * (j + (int)start[0])) + 1) || + rbufi[i][j].c != ((4 * (j + (int)start[0])) + 2) || + rbufi[i][j].d != ((4 * (j + (int)start[0])) + 3)) { + nerrors++; + HDprintf("\n Error in 3rd data verification for dset %d:\n", i); + HDprintf(" At index %d: %d/%d, %d/%d, %d/%d, %d/%d\n", j + (int)start[0], + ((4 * (j + (int)start[0]))), rbufi[i][j].a, ((4 * (j + (int)start[0])) + 1), + rbufi[i][j].b, ((4 * (j + (int)start[0])) + 2), rbufi[i][j].c, + ((4 * (j + (int)start[0])) + 3), rbufi[i][j].d); + break; + } + } + + /* Case d */ + + /* Create s2_t compound type with: + * --no conversion for 2 member types, + * --1 larger mem type + * --1 smaller mem type + */ + if ((s2_tid = H5Tcreate(H5T_COMPOUND, sizeof(s2_t))) < 0) + P_TEST_ERROR; + + if (H5Tinsert(s2_tid, "a", HOFFSET(s2_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(s2_tid, "b", HOFFSET(s2_t, b), H5T_NATIVE_LLONG) < 0 || + H5Tinsert(s2_tid, "c", HOFFSET(s2_t, c), H5T_NATIVE_INT) < 0 || + H5Tinsert(s2_tid, "d", HOFFSET(s2_t, d), H5T_NATIVE_SHORT) < 0) + P_TEST_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + s2_wbufi[i] = s2_total_wbuf + (i * DSET_SELECT_DIM); + s2_rbufi[i] = s2_total_rbuf + (i * DSET_SELECT_DIM); + + wbufs[i] = s2_wbufi[i]; + rbufs[i] = s2_rbufi[i]; + + mem_tids[i] = s2_tid; + + if (H5Sselect_all(mem_sids[i]) < 0) + P_TEST_ERROR; + if (H5Sselect_all(file_sids[i]) < 0) + P_TEST_ERROR; + if (H5Sselect_hyperslab(file_sids[i], H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + } + + /* Initialize the buffer data for all the datasets */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < (int)block[0]; j++) { + s2_wbufi[i][j].a = 8 * (j + (int)start[0]); + s2_wbufi[i][j].b = (long long)((8 * (j + (int)start[0])) + 1); + s2_wbufi[i][j].c = (8 * (j + (int)start[0])) + 2; + s2_wbufi[i][j].d = (short)((8 * (j + (int)start[0])) + 3); + } + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(s2_total_wbuf_bak, s2_total_wbuf, s2_buf_size); + + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(s2_total_wbuf, s2_total_wbuf_bak, s2_buf_size); + + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + P_TEST_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + for (j = 0; j < (int)block[0]; j++) + if (s2_rbufi[i][j].a != s2_wbufi[i][j].a || s2_rbufi[i][j].b != s2_wbufi[i][j].b || + s2_rbufi[i][j].c != s2_wbufi[i][j].c || s2_rbufi[i][j].d != s2_wbufi[i][j].d) { + nerrors++; + HDprintf("\n Error in 3rd data verification for dset %d:\n", i); + HDprintf(" At index %d: %d/%d, %lld/%lld, %d/%d, %d/%d\n", j + (int)start[0], + s2_wbufi[i][j].a, s2_rbufi[i][j].a, s2_wbufi[i][j].b, s2_rbufi[i][j].b, + s2_wbufi[i][j].c, s2_rbufi[i][j].c, s2_wbufi[i][j].d, s2_rbufi[i][j].d); + break; + } + } + + if (H5Pclose(dcpl) < 0) + P_TEST_ERROR; + + if (H5Pclose(dxpl) < 0) + P_TEST_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + if (H5Sclose(file_sids[i]) < 0) + P_TEST_ERROR; + if (H5Sclose(mem_sids[i]) < 0) + P_TEST_ERROR; + if (H5Dclose(dset_dids[i]) < 0) + P_TEST_ERROR; + } + + HDfree(total_wbuf); + HDfree(total_wbuf_bak); + HDfree(total_rbuf); + HDfree(s2_total_wbuf); + HDfree(s2_total_wbuf_bak); + HDfree(s2_total_rbuf); + + CHECK_PASSED(); + + return; + +} /* test_multi_dsets_cmpd_with_bkg() */ + +/* + * Test 3 for multi-dataset: + * --Datasets with/without type conv+size change+no background buffer + * + * Create dset0: H5T_STD_I32BE + * Create other dateasets: randomized H5T_STD_I64LE or H5T_STD_I16LE + * + * Case a--setting for multi write/read to ndsets: + * Datatype for all datasets: H5T_STD_I32BE + * + * Case b--setting for multi write/read to ndsets + * Datatype for all datasets: H5T_STD_I64BE + * + * Case c--setting for multi write/read to ndsets + * Datatype for all datasets: H5T_STD_I16BE + */ +static void +test_multi_dsets_size_change_no_bkg(hid_t fid, unsigned chunked, unsigned mwbuf) +{ + size_t ndsets; + int i, j; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + hsize_t start[1], stride[1], count[1], block[1]; + + hid_t file_sids[MULTI_NUM_DSETS]; + hid_t mem_sids[MULTI_NUM_DSETS]; + hid_t mem_tids[MULTI_NUM_DSETS]; + + char dset_names[MULTI_NUM_DSETS][DSET_NAME_LEN]; + hid_t dset_dids[MULTI_NUM_DSETS]; + + size_t buf_size, ss; + uint8_t *total_wbuf = NULL; + uint8_t *total_wbuf_bak = NULL; + uint8_t *total_rbuf = NULL; + uint8_t *total_lwbuf = NULL; + uint8_t *total_lwbuf_bak = NULL; + uint8_t *total_lrbuf = NULL; + uint8_t *total_swbuf = NULL; + uint8_t *total_swbuf_bak = NULL; + uint8_t *total_srbuf = NULL; + + uint8_t *wbufi[MULTI_NUM_DSETS]; + uint8_t *rbufi[MULTI_NUM_DSETS]; + uint8_t *lwbufi[MULTI_NUM_DSETS]; + uint8_t *lrbufi[MULTI_NUM_DSETS]; + uint8_t *swbufi[MULTI_NUM_DSETS]; + uint8_t *srbufi[MULTI_NUM_DSETS]; + + const void *wbufs[MULTI_NUM_DSETS]; + void *rbufs[MULTI_NUM_DSETS]; + + curr_nerrors = nerrors; + + ndsets = MAX(MULTI_MIN_DSETS, MULTI_NUM_DSETS); + + dims[0] = DSET_SELECT_DIM; + + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + P_TEST_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + P_TEST_ERROR; + } + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + P_TEST_ERROR; + + /* Set selection I/O mode, type of I/O and type of collective I/O */ + set_dxpl(dxpl, H5D_SELECTION_IO_MODE_ON, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, mwbuf); + + /* Set up file space ids, mem space ids, and dataset ids */ + for (i = 0; i < (int)ndsets; i++) { + if ((file_sids[i] = H5Screate_simple(1, dims, NULL)) < 0) + P_TEST_ERROR; + + /* Generate dataset name */ + HDsnprintf(dset_names[i], sizeof(dset_names[i]), "multi_size_dset%d_%s_%s", i, + chunked ? "chunked" : "contig", mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create ith dataset */ + if ((dset_dids[i] = H5Dcreate2(fid, dset_names[i], H5T_STD_I32BE, file_sids[i], H5P_DEFAULT, dcpl, + H5P_DEFAULT)) < 0) + P_TEST_ERROR; + } + + /* Each process takes x number of elements */ + block[0] = dims[0] / (hsize_t)mpi_size; + stride[0] = block[0]; + count[0] = 1; + start[0] = (hsize_t)mpi_rank * block[0]; + + for (i = 0; i < (int)ndsets; i++) { + if ((mem_sids[i] = H5Screate_simple(1, block, NULL)) < 0) + P_TEST_ERROR; + + if (H5Sselect_hyperslab(file_sids[i], H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + } + + /* Case a */ + + ss = H5Tget_size(H5T_STD_I32BE); + buf_size = ndsets * ss * DSET_SELECT_DIM; + + /* Allocate buffers for all datasets */ + if (NULL == (total_wbuf = (uint8_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_wbuf_bak = (uint8_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_rbuf = (uint8_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + + /* Initialize buffer indices */ + for (i = 0; i < (int)ndsets; i++) { + wbufi[i] = total_wbuf + (i * (int)ss * DSET_SELECT_DIM); + rbufi[i] = total_rbuf + (i * (int)ss * DSET_SELECT_DIM); + + wbufs[i] = wbufi[i]; + rbufs[i] = rbufi[i]; + } + + /* Initialize the buffer data: big endian */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < (int)block[0]; j++) { + wbufi[i][j * (int)ss + 0] = 0x1; + wbufi[i][j * (int)ss + 1] = 0x2; + wbufi[i][j * (int)ss + 2] = 0x3; + wbufi[i][j * (int)ss + 3] = (uint8_t)(0x4 + j + (int)start[0]); + } + + /* Datatype setting for multi write/read */ + for (i = 0; i < (int)ndsets; i++) + mem_tids[i] = H5T_STD_I32BE; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_wbuf_bak, total_wbuf, buf_size); + + /* Write data to the dataset */ + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_wbuf, total_wbuf_bak, buf_size); + + check_io_mode(dxpl, chunked); + + /* Read data from the dataset */ + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + P_TEST_ERROR; + + /* Verify */ + for (i = 0; i < (int)ndsets; i++) + /* Only compare when it's at least the size of H5T_STD_I32BE */ + if (H5Tget_size(H5Dget_type(dset_dids[i])) >= ss) { + for (j = 0; j < (int)block[0]; j++) + if (rbufi[i][(int)ss * j + 0] != wbufi[i][(int)ss * j + 0] || + rbufi[i][(int)ss * j + 1] != wbufi[i][(int)ss * j + 1] || + rbufi[i][(int)ss * j + 2] != wbufi[i][(int)ss * j + 2] || + rbufi[i][(int)ss * j + 3] != wbufi[i][(int)ss * j + 3]) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j + (int)start[0]); + P_TEST_ERROR; + } + } + + /* Case b */ + + ss = H5Tget_size(H5T_STD_I64BE); + buf_size = ndsets * (ss * DSET_SELECT_DIM); + + /* Allocate buffers for all datasets */ + if (NULL == (total_lwbuf = (uint8_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_lwbuf_bak = (uint8_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_lrbuf = (uint8_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + + /* Initialize buffer indices */ + for (i = 0; i < (int)ndsets; i++) { + lwbufi[i] = total_lwbuf + (i * (int)ss * DSET_SELECT_DIM); + lrbufi[i] = total_lrbuf + (i * (int)ss * DSET_SELECT_DIM); + + wbufs[i] = lwbufi[i]; + rbufs[i] = lrbufi[i]; + } + + /* Initialize the buffer data: big endian */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < (int)block[0]; j++) { + lwbufi[i][j * (int)ss + 0] = 0x1; + lwbufi[i][j * (int)ss + 1] = 0x2; + lwbufi[i][j * (int)ss + 2] = 0x3; + lwbufi[i][j * (int)ss + 3] = 0x4; + lwbufi[i][j * (int)ss + 4] = 0x5; + lwbufi[i][j * (int)ss + 5] = 0x6; + lwbufi[i][j * (int)ss + 6] = 0x7; + lwbufi[i][j * (int)ss + 7] = (uint8_t)(0x8 + j + (int)start[0]); + } + + /* Datatype setting for multi write/read */ + for (i = 0; i < (int)ndsets; i++) + mem_tids[i] = H5T_STD_I64BE; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_lwbuf_bak, total_lwbuf, buf_size); + + /* Write data to the dataset */ + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_lwbuf, total_lwbuf_bak, buf_size); + + /* Read data from the dataset */ + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + P_TEST_ERROR; + + /* Verify */ + for (i = 0; i < (int)ndsets; i++) + /* Only compare when it's the size of H5T_STD_I64BE */ + if (H5Tget_size(H5Dget_type(dset_dids[i])) >= ss) { + for (j = 0; j < (int)block[0]; j++) + if (lrbufi[i][(int)ss * j + 0] != lwbufi[i][(int)ss * j + 0] || + lrbufi[i][(int)ss * j + 1] != lwbufi[i][(int)ss * j + 1] || + lrbufi[i][(int)ss * j + 2] != lwbufi[i][(int)ss * j + 2] || + lrbufi[i][(int)ss * j + 3] != lwbufi[i][(int)ss * j + 3] || + lrbufi[i][(int)ss * j + 4] != lwbufi[i][(int)ss * j + 4] || + lrbufi[i][(int)ss * j + 5] != lwbufi[i][(int)ss * j + 5] || + lrbufi[i][(int)ss * j + 6] != lwbufi[i][(int)ss * j + 6] || + lrbufi[i][(int)ss * j + 7] != lwbufi[i][(int)ss * j + 7]) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j + (int)start[0]); + P_TEST_ERROR; + } + } + + /* Case c */ + + ss = H5Tget_size(H5T_STD_I16BE); + buf_size = ndsets * (ss * DSET_SELECT_DIM); + + /* Allocate buffers for all datasets */ + if (NULL == (total_swbuf = (uint8_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_swbuf_bak = (uint8_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_srbuf = (uint8_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + + /* Initialize buffer indices */ + for (i = 0; i < (int)ndsets; i++) { + swbufi[i] = total_swbuf + (i * (int)ss * DSET_SELECT_DIM); + srbufi[i] = total_srbuf + (i * (int)ss * DSET_SELECT_DIM); + + wbufs[i] = swbufi[i]; + rbufs[i] = srbufi[i]; + } + + /* Initialize the buffer data: big endian */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < (int)block[0]; j++) { + swbufi[i][j * (int)ss + 0] = 0x1; + swbufi[i][j * (int)ss + 1] = (uint8_t)(0x2 + j + (int)start[0]); + } + + /* Datatype setting for multi write/read */ + for (i = 0; i < (int)ndsets; i++) + mem_tids[i] = H5T_STD_I16BE; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_swbuf_bak, total_swbuf, buf_size); + + /* Write data to the dataset */ + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_swbuf, total_swbuf_bak, buf_size); + + /* Read data from the dataset */ + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + P_TEST_ERROR; + + /* Verify */ + for (i = 0; i < (int)ndsets; i++) + /* Can compare for all cases */ + for (j = 0; j < (int)block[0]; j++) + if (srbufi[i][(int)ss * j + 0] != swbufi[i][(int)ss * j + 0] || + srbufi[i][(int)ss * j + 1] != swbufi[i][(int)ss * j + 1]) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j + (int)start[0]); + P_TEST_ERROR; + } + + if (H5Pclose(dcpl) < 0) + P_TEST_ERROR; + if (H5Pclose(dxpl) < 0) + P_TEST_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + if (H5Sclose(file_sids[i]) < 0) + P_TEST_ERROR; + if (H5Sclose(mem_sids[i]) < 0) + P_TEST_ERROR; + if (H5Dclose(dset_dids[i]) < 0) + P_TEST_ERROR; + } + + HDfree(total_wbuf); + HDfree(total_wbuf_bak); + HDfree(total_rbuf); + HDfree(total_lwbuf); + HDfree(total_lwbuf_bak); + HDfree(total_lrbuf); + HDfree(total_swbuf); + HDfree(total_swbuf_bak); + HDfree(total_srbuf); + + CHECK_PASSED(); + + return; + +} /* test_multi_dsets_size_change_no_bkg() */ + +/* + * Test 4 for multi-dataset: + * Datasets with type conversions+some processes have null/empty selections + * + * Create dset0: H5T_NATIVE_INT + * Create other datasets: randomized H5T_NATIVE_LLONG or H5T_NATIVE_SHORT + * Type conversions + some processes have null/empty selections in datasets + * + * Case (a): dset0 + * process 0: hyperslab; other processes: select none + * Case (b): randomized dset <mm> + * process 0: get 0 row; other processes: hyperslab + * Case (c): randomized dset <ll> + * process 0: select none; other processes: select all + * + * Memory datatype for multi write to all datasets: H5T_NATIVE_INT + * --this will not trigger type conversion for case (a) but + * type conversion for cases (b) & (c) + * Memory datatype for multi read to all datasets: H5T_NATIVE_LONG + * --this will trigger type conversion for (a), (b) & (c) + */ +static void +test_multi_dsets_conv_sel_empty(hid_t fid, unsigned chunked, unsigned dtrans, unsigned mwbuf) +{ + size_t ndsets; + int i, j; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t ntrans_dxpl = H5I_INVALID_HID; + + hsize_t dims[1]; + hsize_t cdims[1]; + hsize_t start[1], stride[1], count[1], block[1]; + + hid_t file_sids[MULTI_NUM_DSETS]; + hid_t mem_sids[MULTI_NUM_DSETS]; + hid_t mem_tids[MULTI_NUM_DSETS]; + + char dset_names[MULTI_NUM_DSETS][DSET_NAME_LEN]; + hid_t dset_dids[MULTI_NUM_DSETS]; + + size_t buf_size; + int *total_wbuf = NULL; + int *total_wbuf_bak = NULL; + int *total_trans_wbuf = NULL; + long *total_lrbuf = NULL; + + int *wbufi[MULTI_NUM_DSETS]; + int *trans_wbufi[MULTI_NUM_DSETS]; + long *l_rbufi[MULTI_NUM_DSETS]; + + const void *wbufs[MULTI_NUM_DSETS]; + void *rbufs[MULTI_NUM_DSETS]; + + int save_block0; + int mm, ll; + + const char *expr = "2*x"; + + curr_nerrors = nerrors; + + ndsets = MAX(MULTI_MIN_DSETS, MULTI_NUM_DSETS); + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + P_TEST_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + P_TEST_ERROR; + } + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + P_TEST_ERROR; + + /* Set selection I/O mode, type of I/O and type of collective I/O */ + set_dxpl(dxpl, H5D_SELECTION_IO_MODE_ON, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, mwbuf); + + if ((ntrans_dxpl = H5Pcopy(dxpl)) < 0) + P_TEST_ERROR; + + /* Set data transform */ + if (dtrans) + if (H5Pset_data_transform(dxpl, expr) < 0) + P_TEST_ERROR; + + /* Set up file space ids and dataset ids */ + for (i = 0; i < (int)ndsets; i++) { + if ((file_sids[i] = H5Screate_simple(1, dims, NULL)) < 0) + P_TEST_ERROR; + + /* Generate dataset name */ + HDsnprintf(dset_names[i], sizeof(dset_names[i]), "multi_sel_dset%d_%s_%s_%s", i, + chunked ? "chunked" : "contig", dtrans ? "xform" : "noxform", mwbuf ? "mwbuf" : "nomwbuf"); + + if (i == 0) { + if ((dset_dids[i] = H5Dcreate2(fid, dset_names[i], H5T_NATIVE_INT, file_sids[i], H5P_DEFAULT, + dcpl, H5P_DEFAULT)) < 0) + P_TEST_ERROR; + } + else { + if ((dset_dids[i] = + H5Dcreate2(fid, dset_names[i], ((HDrandom() % 2) ? H5T_NATIVE_LLONG : H5T_NATIVE_SHORT), + file_sids[i], H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + P_TEST_ERROR; + } + } + + buf_size = ndsets * DSET_SELECT_DIM * sizeof(int); + + /* Allocate buffers for all datasets */ + if (NULL == (total_wbuf = (int *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_wbuf_bak = (int *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_trans_wbuf = (int *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_lrbuf = (long *)HDmalloc(ndsets * DSET_SELECT_DIM * sizeof(long)))) + P_TEST_ERROR; + + /* Initialize buffer indices */ + for (i = 0; i < (int)ndsets; i++) { + wbufi[i] = total_wbuf + (i * DSET_SELECT_DIM); + trans_wbufi[i] = total_trans_wbuf + (i * DSET_SELECT_DIM); + + wbufs[i] = wbufi[i]; + } + + /* + * Case (a): dset0 + * process 0: hyperslab; other processes: select none + */ + + /* Each process takes x number of elements */ + block[0] = dims[0] / (hsize_t)mpi_size; + save_block0 = (int)block[0]; + stride[0] = block[0]; + count[0] = 1; + start[0] = (hsize_t)mpi_rank * block[0]; + + /* Get file dataspace */ + if ((file_sids[0] = H5Dget_space(dset_dids[0])) < 0) + P_TEST_ERROR; + + if (MAINPROCESS) { + if (H5Sselect_hyperslab(file_sids[0], H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + } + else { + if (H5Sselect_none(file_sids[0]) < 0) + P_TEST_ERROR; + } + + /* Create memory dataspace */ + if ((mem_sids[0] = H5Screate_simple(1, block, NULL)) < 0) + P_TEST_ERROR; + if (mpi_rank) { + if (H5Sselect_none(mem_sids[0]) < 0) + P_TEST_ERROR; + } + + /* Initialize data for wbufi[0] */ + for (j = 0; j < (int)block[0]; j++) { + wbufi[0][j] = j + (int)start[0]; + trans_wbufi[0][j] = 2 * wbufi[0][j]; + } + + /* + * Case (b): choose a dataset -- dset <mm> + * process 0: get 0 row; other processes: hyperslab + */ + + mm = HDrandom() % (int)ndsets; + if (mm == 0) + mm++; + + block[0] = mpi_rank ? (dims[0] / (hsize_t)mpi_size) : 0; + stride[0] = mpi_rank ? block[0] : 1; + count[0] = 1; + start[0] = mpi_rank ? ((hsize_t)mpi_rank * block[0]) : 0; + + /* Get file dataspace */ + if ((file_sids[mm] = H5Dget_space(dset_dids[mm])) < 0) + P_TEST_ERROR; + + if (H5Sselect_hyperslab(file_sids[mm], H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + + /* Create a memory dataspace */ + if ((mem_sids[mm] = H5Screate_simple(1, block, NULL)) < 0) + P_TEST_ERROR; + + /* need to make memory space to match for process 0 */ + if (MAINPROCESS) { + if (H5Sselect_hyperslab(mem_sids[mm], H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + } + else { + if (H5Sselect_all(mem_sids[mm]) < 0) + P_TEST_ERROR; + } + + /* Initialize data for wbufi[1] */ + for (j = 0; j < (int)block[0]; j++) { + wbufi[mm][j] = j + (int)start[0]; + trans_wbufi[mm][j] = 2 * wbufi[mm][j]; + } + + /* + * Case (c): choose a dataset -- dset <ll> + * process 0: select none; other processes: select all + */ + + ll = mm + 1; + if ((ll % (int)ndsets) == 0) + ll = 1; + + /* Get file dataspace */ + if ((file_sids[ll] = H5Dget_space(dset_dids[ll])) < 0) + P_TEST_ERROR; + if (MAINPROCESS) { + if (H5Sselect_none(file_sids[ll]) < 0) + P_TEST_ERROR; + } + else { + if (H5Sselect_all(file_sids[ll]) < 0) + P_TEST_ERROR; + } + + /* Create a memory dataspace */ + if ((mem_sids[ll] = H5Screate_simple(1, dims, NULL)) < 0) + P_TEST_ERROR; + if (MAINPROCESS) { + if (H5Sselect_none(mem_sids[ll]) < 0) + P_TEST_ERROR; + } + else if (H5Sselect_all(mem_sids[ll]) < 0) + P_TEST_ERROR; + + /* Initialize data for wbufi[ll] */ + for (j = 0; j < (int)dims[0]; j++) { + wbufi[ll][j] = (int)j + DSET_SELECT_DIM; + trans_wbufi[ll][j] = 2 * wbufi[ll][j]; + } + + /* Set up remaining dsets */ + for (i = 0; i < (int)ndsets; i++) { + if (i == 0 || i == mm || i == ll) + continue; + /* Get file dataspace */ + if ((file_sids[i] = H5Dget_space(dset_dids[i])) < 0) + P_TEST_ERROR; + if (H5Sselect_none(file_sids[i]) < 0) + P_TEST_ERROR; + + if ((mem_sids[i] = H5Screate_simple(1, dims, NULL)) < 0) + P_TEST_ERROR; + if (H5Sselect_none(mem_sids[i]) < 0) + P_TEST_ERROR; + } + + /* Set up mem_tids[] for multi write */ + for (i = 0; i < (int)ndsets; i++) + mem_tids[i] = H5T_NATIVE_INT; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_wbuf_bak, total_wbuf, buf_size); + + /* Write data to the dataset with/without data transform */ + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_wbuf, total_wbuf_bak, buf_size); + + check_io_mode(dxpl, chunked); + + /* Initialize buffer indices */ + for (i = 0; i < (int)ndsets; i++) { + l_rbufi[i] = total_lrbuf + (i * DSET_SELECT_DIM); + rbufs[i] = l_rbufi[i]; + } + + /* Set up mem_tids[] for multi read */ + for (i = 0; i < (int)ndsets; i++) + mem_tids[i] = H5T_NATIVE_LONG; + + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, ntrans_dxpl, rbufs) < 0) + P_TEST_ERROR; + + if (MAINPROCESS) { + /* Case a: verify dset0 */ + for (j = 0; j < save_block0; j++) + if (l_rbufi[0][j] != (dtrans ? (long)trans_wbufi[0][j] : (long)wbufi[0][j])) { + nerrors++; + HDprintf(" Verify dset0 at index %d: %ld, %ld\n", j + (int)start[0], + dtrans ? (long)trans_wbufi[0][j] : (long)wbufi[0][j], l_rbufi[0][j]); + break; + } + } + + if (mpi_rank) { + /* Case b: verify dset <mm> */ + for (j = 0; j < (int)block[0]; j++) + if (l_rbufi[mm][j] != (long)(dtrans ? trans_wbufi[mm][j] : wbufi[mm][j])) { + nerrors++; + HDprintf(" Verify dset %d at index %d: %ld, %ld\n", mm, j + (int)start[0], + (long)(dtrans ? trans_wbufi[mm][j] : wbufi[mm][j]), l_rbufi[mm][j]); + break; + } + + /* Case c: verify dset <ll> */ + for (j = 0; j < (int)dims[0]; j++) + if (l_rbufi[ll][j] != (long)(dtrans ? trans_wbufi[ll][j] : wbufi[ll][j])) { + nerrors++; + HDprintf(" Verify dset %d at index %d: %ld, %ld\n", ll, j, + (long)(dtrans ? trans_wbufi[ll][j] : wbufi[ll][j]), l_rbufi[ll][j]); + break; + } + } + + if (H5Pclose(dcpl) < 0) + P_TEST_ERROR; + if (H5Pclose(dxpl) < 0) + P_TEST_ERROR; + if (H5Pclose(ntrans_dxpl) < 0) + P_TEST_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + if (H5Sclose(file_sids[i]) < 0) + P_TEST_ERROR; + if (H5Sclose(mem_sids[i]) < 0) + P_TEST_ERROR; + if (H5Dclose(dset_dids[i]) < 0) + P_TEST_ERROR; + } + + HDfree(total_wbuf); + HDfree(total_wbuf_bak); + HDfree(total_trans_wbuf); + HDfree(total_lrbuf); + + CHECK_PASSED(); + + return; + +} /* test_multi_dsets_conv_sel_empty() */ + +/* + * Test 5 for multi-dataset: + * + * Repeat the following test for niter times to ensure the + * random combinations of all dataset types are hit. + * + * Create randomized contiguous or chunked datasets with: + * --DSET_WITH_NO_CONV: + * --with no type conversion + * --dataset with H5T_NATIVE_INT + * --DSET_WITH_CONV_AND_NO_BKG: + * --type conversion without background buffer + * --dataset with H5T_NATIVE_LONG + * --DSET_WITH_CONV_AND_BKG: + * --type conversion with background buffer + * --dataset with compound type s1_t + * + * Do H5Dwrite_multi() and H5Dread_multi() for the above randomized + * datasets with the settings below: + * Setting A: + * --DSET_WITH_NO_CONV: + * --write: mem_tids[] = H5T_NATIVE_INT + * --read: r_mem_tids[] = H5T_NATIVE_INT + * --DSET_WITH_CONV_AND_NO_BKG: + * --write: mem_tids[] = H5T_NATIVE_ULONG + * --read: r_mem_tids[] = H5T_NATIVE_LONG + * --DSET_WITH_CONV_AND_BKG: + * --write: mem_tids[] = s1_tid; + * --read: r_mem_tids[i] = s3_tid; + * + * Setting B: + * --DSET_WITH_NO_CONV: + * --write: mem_tids[] = H5T_NATIVE_INT + * --read: r_mem_tids[] = H5T_NATIVE_INT + * --DSET_WITH_CONV_AND_NO_BKG: + * --write: mem_tids[] = H5T_NATIVE_LONG; + * --read: r_mem_tids[] = H5T_NATIVE_SHORT; + * --DSET_WITH_CONV_AND_BKG: + * --write: mem_tids[] = s4_tid; + * --read: r_mem_tids[i] = s1_tid; + * + * Verify the result read as below: + * Setting A: + * --DSET_WITH_NO_CONV: + * --verify data read in rbufi1[i][j] is same as wbufi1[i][j] + * --DSET_WITH_CONV_AND_NO_BKG: + * --verify data read in l_rbufi2[i][j] is all LONG_MAX + * --DSET_WITH_CONV_AND_BKG: + * --verify all fields read in s3_rbufi3[i][j] is the + * reverse of s1_wbufi3[i][j] + * Setting B: + * --DSET_WITH_NO_CONV: + * --verify data read in rbufi1[i][j] is same as wbufi1[i][j] + * --DSET_WITH_CONV_AND_NO_BKG: + * --verify data read in s_rbufi2[i][j] is all SHRT_MAX + * --DSET_WITH_CONV_AND_BKG: + * --verify fields read in s1_rbufi3[i][j] is as follows: + * --fields 'a' and 'c' are as s1_wbufi3[i][j].a and s1_wbufi3[i][j].c + * --fields 'b' and 'd' are (DSET_SELECT_DIM + j + start[0]) + */ +static void +test_multi_dsets_all(int niter, hid_t fid, unsigned chunked, unsigned mwbuf) +{ + size_t ndsets; + int i, j, mm; + int s, n; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + + hsize_t dims[1]; + hsize_t cdims[1]; + hsize_t start[1], stride[1], count[1], block[1]; + + hid_t file_sids[MULTI_NUM_DSETS]; + hid_t mem_sids[MULTI_NUM_DSETS]; + hid_t mem_tids[MULTI_NUM_DSETS]; + hid_t r_mem_tids[MULTI_NUM_DSETS]; + + multi_dset_type_t dset_types[MULTI_NUM_DSETS]; + + hid_t s1_tid = H5I_INVALID_HID; + hid_t s3_tid = H5I_INVALID_HID; + hid_t s4_tid = H5I_INVALID_HID; + + char dset_names[MULTI_NUM_DSETS][DSET_NAME_LEN]; + hid_t dset_dids[MULTI_NUM_DSETS]; + + size_t buf_size; + + int *total_wbuf1 = NULL; + int *total_wbuf1_bak = NULL; + int *total_rbuf1 = NULL; + + int *wbufi1[MULTI_NUM_DSETS]; + int *rbufi1[MULTI_NUM_DSETS]; + + unsigned long *ul_total_wbuf2 = NULL; + unsigned long *ul_total_wbuf2_bak = NULL; + long *l_total_rbuf2 = NULL; + unsigned long *ul_wbufi2[MULTI_NUM_DSETS]; + long *l_rbufi2[MULTI_NUM_DSETS]; + + long *l_total_wbuf2 = NULL; + long *l_total_wbuf2_bak = NULL; + short *s_total_rbuf2 = NULL; + long *l_wbufi2[MULTI_NUM_DSETS]; + short *s_rbufi2[MULTI_NUM_DSETS]; + + s1_t *s1_total_wbuf3 = NULL; + s1_t *s1_total_wbuf3_bak = NULL; + s3_t *s3_total_rbuf3 = NULL; + s1_t *s1_wbufi3[MULTI_NUM_DSETS]; + s3_t *s3_rbufi3[MULTI_NUM_DSETS]; + + s4_t *s4_total_wbuf3 = NULL; + s4_t *s4_total_wbuf3_bak = NULL; + s1_t *s1_total_rbuf3 = NULL; + s4_t *s4_wbufi3[MULTI_NUM_DSETS]; + s1_t *s1_rbufi3[MULTI_NUM_DSETS]; + + const void *wbufs[MULTI_NUM_DSETS]; + void *rbufs[MULTI_NUM_DSETS]; + + /* for n niter to ensure that all randomized dset_types with multi_dset_type_t will be covered */ + for (n = 0; n < niter; n++) { + + /* Set up the number of datasets for testing */ + ndsets = MAX(MULTI_MIN_DSETS, MULTI_NUM_DSETS); + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + P_TEST_ERROR; + + /* Set selection I/O mode, type of I/O and type of collective I/O */ + set_dxpl(dxpl, H5D_SELECTION_IO_MODE_ON, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, mwbuf); + + /* Set dataset layout: contiguous or chunked */ + dims[0] = DSET_SELECT_DIM; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + P_TEST_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + P_TEST_ERROR; + } + + /* Each process takes x number of elements */ + block[0] = dims[0] / (hsize_t)mpi_size; + stride[0] = block[0]; + count[0] = 1; + start[0] = (hsize_t)mpi_rank * block[0]; + + /* Create compound data type: s1_t */ + if ((s1_tid = H5Tcreate(H5T_COMPOUND, sizeof(s1_t))) < 0) + P_TEST_ERROR; + + if (H5Tinsert(s1_tid, "a", HOFFSET(s1_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "b", HOFFSET(s1_t, b), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "c", HOFFSET(s1_t, c), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "d", HOFFSET(s1_t, d), H5T_NATIVE_INT) < 0) + P_TEST_ERROR; + + /* Create compound data type: s3_t */ + if ((s3_tid = H5Tcreate(H5T_COMPOUND, sizeof(s3_t))) < 0) + P_TEST_ERROR; + + if (H5Tinsert(s3_tid, "a", HOFFSET(s3_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(s3_tid, "b", HOFFSET(s3_t, b), H5T_NATIVE_INT) < 0 || + H5Tinsert(s3_tid, "c", HOFFSET(s3_t, c), H5T_NATIVE_INT) < 0 || + H5Tinsert(s3_tid, "d", HOFFSET(s3_t, d), H5T_NATIVE_INT) < 0) + P_TEST_ERROR; + + /* Create compound data type: s4_t */ + if ((s4_tid = H5Tcreate(H5T_COMPOUND, sizeof(s4_t))) < 0) + P_TEST_ERROR; + + if (H5Tinsert(s4_tid, "b", HOFFSET(s4_t, b), H5T_NATIVE_UINT) < 0 || + H5Tinsert(s4_tid, "d", HOFFSET(s4_t, d), H5T_NATIVE_UINT) < 0) + P_TEST_ERROR; + + /* Create dataset for i ndsets */ + for (i = 0; i < (int)ndsets; i++) { + + /* File space ids */ + if ((file_sids[i] = H5Screate_simple(1, dims, NULL)) < 0) + P_TEST_ERROR; + + /* Memory space ids */ + if ((mem_sids[i] = H5Screate_simple(1, block, NULL)) < 0) + P_TEST_ERROR; + + mm = HDrandom() % (int)ndsets; + if (mm == 0) { + dset_types[i] = DSET_WITH_NO_CONV; + HDsnprintf(dset_names[i], sizeof(dset_names[i]), "multi_all_nconv_dset%d_%s_%s", i, + chunked ? "chunked" : "contig", mwbuf ? "mwbuf" : "nomwbuf"); + if ((dset_dids[i] = H5Dcreate2(fid, dset_names[i], H5T_NATIVE_INT, file_sids[i], H5P_DEFAULT, + dcpl, H5P_DEFAULT)) < 0) + P_TEST_ERROR; + } + else if (mm == 1) { + dset_types[i] = DSET_WITH_CONV_AND_NO_BKG; + HDsnprintf(dset_names[i], sizeof(dset_names[i]), "multi_all_conv_nbkg_dset%d_%s_%s", i, + chunked ? "chunked" : "contig", mwbuf ? "mwbuf" : "nomwbuf"); + if ((dset_dids[i] = H5Dcreate2(fid, dset_names[i], H5T_NATIVE_LONG, file_sids[i], H5P_DEFAULT, + dcpl, H5P_DEFAULT)) < 0) + P_TEST_ERROR; + } + else { + dset_types[i] = DSET_WITH_CONV_AND_BKG; + HDsnprintf(dset_names[i], sizeof(dset_names[i]), "multi_all_conv_bkg_dset%d_%s_%s", i, + chunked ? "chunked" : "contig", mwbuf ? "mwbuf" : "nomwbuf"); + if ((dset_dids[i] = H5Dcreate2(fid, dset_names[i], s1_tid, file_sids[i], H5P_DEFAULT, dcpl, + H5P_DEFAULT)) < 0) + P_TEST_ERROR; + } + + if (H5Sselect_hyperslab(file_sids[i], H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + + } /* end for i ndsets */ + + /* Allocate buffers for all datasets */ + + /* DSET_WITH_NO_CONV */ + buf_size = ndsets * DSET_SELECT_DIM * sizeof(int); + if (NULL == (total_wbuf1 = (int *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (mwbuf && NULL == (total_wbuf1_bak = (int *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_rbuf1 = (int *)HDmalloc(buf_size))) + P_TEST_ERROR; + + /* DSET_WITH_CONV_AND_NO_BKG */ + buf_size = ndsets * DSET_SELECT_DIM * sizeof(unsigned long); + if (NULL == (ul_total_wbuf2 = (unsigned long *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (mwbuf && NULL == (ul_total_wbuf2_bak = (unsigned long *)HDmalloc(buf_size))) + P_TEST_ERROR; + buf_size = ndsets * DSET_SELECT_DIM * sizeof(long); + if (NULL == (l_total_rbuf2 = (long *)HDmalloc(buf_size))) + P_TEST_ERROR; + + buf_size = ndsets * DSET_SELECT_DIM * sizeof(long); + if (NULL == (l_total_wbuf2 = (long *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (mwbuf && NULL == (l_total_wbuf2_bak = (long *)HDmalloc(buf_size))) + P_TEST_ERROR; + buf_size = ndsets * DSET_SELECT_DIM * sizeof(short); + if (NULL == (s_total_rbuf2 = (short *)HDmalloc(buf_size))) + P_TEST_ERROR; + + /* DSET_WITH_CONV_AND_BKG */ + buf_size = ndsets * DSET_SELECT_DIM * sizeof(s1_t); + if (NULL == (s1_total_wbuf3 = (s1_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (mwbuf && NULL == (s1_total_wbuf3_bak = (s1_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + buf_size = ndsets * DSET_SELECT_DIM * sizeof(s3_t); + if (NULL == (s3_total_rbuf3 = (s3_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + + buf_size = ndsets * DSET_SELECT_DIM * sizeof(s4_t); + if (NULL == (s4_total_wbuf3 = (s4_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (mwbuf && NULL == (s4_total_wbuf3_bak = (s4_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + buf_size = ndsets * DSET_SELECT_DIM * sizeof(s1_t); + if (NULL == (s1_total_rbuf3 = (s1_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + + /* Test with s settings for ndsets */ + for (s = SETTING_A; s <= SETTING_B; s++) { + + /* for i ndsets */ + for (i = 0; i < (int)ndsets; i++) { + + switch (dset_types[i]) { + + case DSET_WITH_NO_CONV: + /* Initialize buffer indices */ + wbufi1[i] = total_wbuf1 + (i * DSET_SELECT_DIM); + rbufi1[i] = total_rbuf1 + (i * DSET_SELECT_DIM); + + wbufs[i] = wbufi1[i]; + rbufs[i] = rbufi1[i]; + + /* Initialize the buffer data */ + for (j = 0; j < (int)block[0]; j++) + wbufi1[i][j] = j + (int)start[0]; + + /* Same for all cases */ + mem_tids[i] = H5T_NATIVE_INT; + r_mem_tids[i] = H5T_NATIVE_INT; + + break; + + case DSET_WITH_CONV_AND_NO_BKG: + if (s == SETTING_A) { + /* Initialize buffer indices */ + ul_wbufi2[i] = ul_total_wbuf2 + (i * DSET_SELECT_DIM); + l_rbufi2[i] = l_total_rbuf2 + (i * DSET_SELECT_DIM); + + wbufs[i] = ul_wbufi2[i]; + rbufs[i] = l_rbufi2[i]; + + for (j = 0; j < (int)block[0]; j++) + ul_wbufi2[i][j] = ULONG_MAX - (unsigned long)(j + (int)start[0]); + + mem_tids[i] = H5T_NATIVE_ULONG; + r_mem_tids[i] = H5T_NATIVE_LONG; + } + else if (s == SETTING_B) { + /* Initialize buffer indices */ + l_wbufi2[i] = l_total_wbuf2 + (i * DSET_SELECT_DIM); + s_rbufi2[i] = s_total_rbuf2 + (i * DSET_SELECT_DIM); + + wbufs[i] = l_wbufi2[i]; + rbufs[i] = s_rbufi2[i]; + + /* Initialize the buffer data */ + for (j = 0; j < (int)block[0]; j++) + l_wbufi2[i][j] = LONG_MAX - (long)(j + (int)start[0]); + + mem_tids[i] = H5T_NATIVE_LONG; + r_mem_tids[i] = H5T_NATIVE_SHORT; + } + + break; + + case DSET_WITH_CONV_AND_BKG: + + if (s == SETTING_A) { + /* Initialize buffer indices */ + s1_wbufi3[i] = s1_total_wbuf3 + (i * DSET_SELECT_DIM); + s3_rbufi3[i] = s3_total_rbuf3 + (i * DSET_SELECT_DIM); + + wbufs[i] = s1_wbufi3[i]; + rbufs[i] = s3_rbufi3[i]; + + /* Initialize buffer data for s1_t */ + for (j = 0; j < (int)block[0]; j++) { + s1_wbufi3[i][j].a = (4 * j + (int)start[0]); + s1_wbufi3[i][j].b = (4 * j + (int)start[0]) + 1; + s1_wbufi3[i][j].c = (4 * j + (int)start[0]) + 2; + s1_wbufi3[i][j].d = (4 * j + (int)start[0]) + 3; + } + mem_tids[i] = s1_tid; + r_mem_tids[i] = s3_tid; + } + else if (s == SETTING_B) { + /* Initialize buffer indices */ + s4_wbufi3[i] = s4_total_wbuf3 + (i * DSET_SELECT_DIM); + s1_rbufi3[i] = s1_total_rbuf3 + (i * DSET_SELECT_DIM); + + wbufs[i] = s4_wbufi3[i]; + rbufs[i] = s1_rbufi3[i]; + + /* Initialize buffer data for s4_t */ + for (j = 0; j < (int)block[0]; j++) { + s4_wbufi3[i][j].b = DSET_SELECT_DIM + (unsigned int)(j + (int)start[0]); + s4_wbufi3[i][j].d = DSET_SELECT_DIM + (unsigned int)(j + (int)start[0]); + } + mem_tids[i] = s4_tid; + r_mem_tids[i] = s1_tid; + } + + break; + + case DSET_NTTYPES: + default: + P_TEST_ERROR; + + } /* end switch dset_types */ + + } /* end for i ndsets */ + + /* Copy wbufs if the library will be modifying them */ + if (mwbuf) { + HDmemcpy(total_wbuf1_bak, total_wbuf1, ndsets * DSET_SELECT_DIM * sizeof(int)); + HDmemcpy(ul_total_wbuf2_bak, ul_total_wbuf2, + ndsets * DSET_SELECT_DIM * sizeof(unsigned long)); + HDmemcpy(l_total_wbuf2_bak, l_total_wbuf2, ndsets * DSET_SELECT_DIM * sizeof(long)); + HDmemcpy(s1_total_wbuf3_bak, s1_total_wbuf3, ndsets * DSET_SELECT_DIM * sizeof(s1_t)); + HDmemcpy(s4_total_wbuf3_bak, s4_total_wbuf3, ndsets * DSET_SELECT_DIM * sizeof(s4_t)); + } + + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + P_TEST_ERROR; + + /* Restore wbufs from backup if the library modified them */ + if (mwbuf) { + HDmemcpy(total_wbuf1, total_wbuf1_bak, ndsets * DSET_SELECT_DIM * sizeof(int)); + HDmemcpy(ul_total_wbuf2, ul_total_wbuf2_bak, + ndsets * DSET_SELECT_DIM * sizeof(unsigned long)); + HDmemcpy(l_total_wbuf2, l_total_wbuf2_bak, ndsets * DSET_SELECT_DIM * sizeof(long)); + HDmemcpy(s1_total_wbuf3, s1_total_wbuf3_bak, ndsets * DSET_SELECT_DIM * sizeof(s1_t)); + HDmemcpy(s4_total_wbuf3, s4_total_wbuf3_bak, ndsets * DSET_SELECT_DIM * sizeof(s4_t)); + } + + if (H5Dread_multi(ndsets, dset_dids, r_mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + P_TEST_ERROR; + + check_io_mode(dxpl, chunked); + + /* Verify result read */ + /* for i ndsets */ + for (i = 0; i < (int)ndsets; i++) { + switch (dset_types[i]) { + + case DSET_WITH_NO_CONV: + for (j = 0; j < (int)block[0]; j++) + if (rbufi1[i][j] != wbufi1[i][j]) { + nerrors++; + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j + (int)start[0]); + break; + } + + break; + + case DSET_WITH_CONV_AND_NO_BKG: + if (s == SETTING_A) { + for (j = 0; j < (int)block[0]; j++) + if (l_rbufi2[i][j] != LONG_MAX) { + nerrors++; + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j + (int)start[0]); + break; + } + } + else if (s == SETTING_B) { + for (j = 0; j < (int)block[0]; j++) + if (s_rbufi2[i][j] != SHRT_MAX) { + nerrors++; + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j + (int)start[0]); + break; + } + } + + break; + + case DSET_WITH_CONV_AND_BKG: + if (s == SETTING_A) { + for (j = 0; j < (int)block[0]; j++) + if (s3_rbufi3[i][j].a != s1_wbufi3[i][j].a || + s3_rbufi3[i][j].b != s1_wbufi3[i][j].b || + s3_rbufi3[i][j].c != s1_wbufi3[i][j].c || + s3_rbufi3[i][j].d != s1_wbufi3[i][j].d) { + nerrors++; + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j + (int)start[0]); + break; + } + } + else if (s == SETTING_B) { + for (j = 0; j < (int)block[0]; j++) + if (s1_rbufi3[i][j].a != s1_wbufi3[i][j].a || + s1_rbufi3[i][j].b != (DSET_SELECT_DIM + j + (int)start[0]) || + s1_rbufi3[i][j].c != s1_wbufi3[i][j].c || + s1_rbufi3[i][j].d != (DSET_SELECT_DIM + j + (int)start[0])) { + nerrors++; + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j + (int)start[0]); + break; + } + } + + break; + + case DSET_NTTYPES: + default: + P_TEST_ERROR; + + } /* end switch dset_types */ + + } /* end for i ndsets */ + + } /* end for s settings */ + + /* Closing */ + if (H5Pclose(dcpl) < 0) + P_TEST_ERROR; + if (H5Pclose(dxpl) < 0) + P_TEST_ERROR; + + if (H5Tclose(s1_tid) < 0) + P_TEST_ERROR; + if (H5Tclose(s3_tid) < 0) + P_TEST_ERROR; + if (H5Tclose(s4_tid) < 0) + P_TEST_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + if (H5Sclose(file_sids[i]) < 0) + P_TEST_ERROR; + if (H5Dclose(dset_dids[i]) < 0) + P_TEST_ERROR; + /* Don't delete the last set of datasets */ + if ((n + 1) != niter) + if (H5Ldelete(fid, dset_names[i], H5P_DEFAULT) < 0) + P_TEST_ERROR; + } + + /* Freeing */ + HDfree(total_wbuf1); + total_wbuf1 = NULL; + HDfree(total_wbuf1_bak); + total_wbuf1_bak = NULL; + HDfree(total_rbuf1); + total_rbuf1 = NULL; + + HDfree(ul_total_wbuf2); + ul_total_wbuf2 = NULL; + HDfree(ul_total_wbuf2_bak); + ul_total_wbuf2_bak = NULL; + HDfree(l_total_rbuf2); + l_total_rbuf2 = NULL; + HDfree(l_total_wbuf2); + l_total_wbuf2 = NULL; + HDfree(l_total_wbuf2_bak); + l_total_wbuf2_bak = NULL; + HDfree(s_total_rbuf2); + s_total_rbuf2 = NULL; + + HDfree(s1_total_wbuf3); + s1_total_wbuf3 = NULL; + HDfree(s1_total_wbuf3_bak); + s1_total_wbuf3_bak = NULL; + HDfree(s3_total_rbuf3); + s3_total_rbuf3 = NULL; + HDfree(s4_total_wbuf3); + s4_total_wbuf3 = NULL; + HDfree(s4_total_wbuf3_bak); + s4_total_wbuf3_bak = NULL; + HDfree(s1_total_rbuf3); + s1_total_rbuf3 = NULL; + + } /* end for n niter */ + + CHECK_PASSED(); + + return; + +} /* test_multi_dsets_all() */ + +/* + * Test with various test_mode that no selection I/O is performed + * + * Note: It is the responsibility of the tester to + * understand and feed proper combination of test_mode + * as needed. + */ +static void +test_no_selection_io_cause_mode(const char *filename, hid_t fapl, uint32_t test_mode) +{ + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t fid = H5I_INVALID_HID; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + hbool_t is_chunked = FALSE; + hid_t tid = H5T_NATIVE_INT; + uint32_t no_selection_io_cause_write = 0; + uint32_t no_selection_io_cause_read = 0; + uint32_t no_selection_io_cause_write_expected = 0; + uint32_t no_selection_io_cause_read_expected = 0; + int wbuf[DSET_SELECT_DIM]; + int rbuf[DSET_SELECT_DIM]; + int i; + + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + P_TEST_ERROR; + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + P_TEST_ERROR; + + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + P_TEST_ERROR; + + if (test_mode & TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET) { + if (H5Pset_layout(dcpl, H5D_COMPACT) < 0) + P_TEST_ERROR; + no_selection_io_cause_write_expected |= H5D_SEL_IO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; + no_selection_io_cause_read_expected |= H5D_SEL_IO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; + } + + if (test_mode == TEST_DISABLE_BY_API) { + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_OFF) < 0) + P_TEST_ERROR; + no_selection_io_cause_write_expected |= H5D_SEL_IO_DISABLE_BY_API; + no_selection_io_cause_read_expected |= H5D_SEL_IO_DISABLE_BY_API; + } + + /* Datatype conversion */ + if (test_mode & TEST_DATATYPE_CONVERSION) { + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_ON) < 0) + P_TEST_ERROR; + tid = H5T_NATIVE_UINT; + + /* If we're testing a too small tconv buffer, set the buffer to be too small */ + if (test_mode & TEST_TCONV_BUF_TOO_SMALL) { + if (H5Pset_buffer(dxpl, sizeof(int), NULL, NULL) < 0) + P_TEST_ERROR; + + /* If we're using in-place type conversion sel io will succeed */ + if (test_mode & TEST_IN_PLACE_TCONV) { + if (H5Pset_modify_write_buf(dxpl, TRUE) < 0) + P_TEST_ERROR; + } + else + no_selection_io_cause_write_expected |= H5D_SEL_IO_TCONV_BUF_TOO_SMALL; + + /* In-place type conversion for read doesn't require modify_write_buf */ + } + + /* If the tconv buf is largge enough sel io will succeed */ + } + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + P_TEST_ERROR; + + if (is_chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + P_TEST_ERROR; + } + + if ((did = H5Dcreate2(fid, "no_selection_io_cause", H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, + H5P_DEFAULT)) < 0) + P_TEST_ERROR; + + /* Initialize data */ + for (i = 0; i < DSET_SELECT_DIM; i++) + wbuf[i] = i; + + if (H5Dwrite(did, tid, H5S_ALL, H5S_ALL, dxpl, wbuf) < 0) + P_TEST_ERROR; + + if (H5Pget_no_selection_io_cause(dxpl, &no_selection_io_cause_write) < 0) + P_TEST_ERROR; + + /* Verify causes of no selection I/O for write are as expected */ + if (no_selection_io_cause_write != no_selection_io_cause_write_expected) + P_TEST_ERROR; + + if (H5Dread(did, tid, H5S_ALL, H5S_ALL, dxpl, rbuf) < 0) + P_TEST_ERROR; + + if (H5Pget_no_selection_io_cause(dxpl, &no_selection_io_cause_read) < 0) + P_TEST_ERROR; + + /* Verify causes of no selection I/O for read are as expected */ + if (no_selection_io_cause_read != no_selection_io_cause_read_expected) + P_TEST_ERROR; + + if (H5Dclose(did) < 0) + P_TEST_ERROR; + + if (H5Pclose(dcpl) < 0) + P_TEST_ERROR; + + if (H5Pclose(dxpl) < 0) + P_TEST_ERROR; + + if (H5Sclose(sid) < 0) + P_TEST_ERROR; + + if (H5Fclose(fid) < 0) + P_TEST_ERROR; + + return; + +} /* test_no_selection_io_cause_mode() */ + +/* + * Test for causes of not performing selection I/O + */ +static void +test_get_no_selection_io_cause(const char *filename, hid_t fapl) +{ + test_no_selection_io_cause_mode(filename, fapl, TEST_DISABLE_BY_API); + test_no_selection_io_cause_mode(filename, fapl, TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET); + test_no_selection_io_cause_mode(filename, fapl, TEST_DATATYPE_CONVERSION); + test_no_selection_io_cause_mode(filename, fapl, TEST_DATATYPE_CONVERSION | TEST_TCONV_BUF_TOO_SMALL); + test_no_selection_io_cause_mode( + filename, fapl, TEST_DATATYPE_CONVERSION | TEST_TCONV_BUF_TOO_SMALL | TEST_IN_PLACE_TCONV); + + CHECK_PASSED(); + + return; +} /* test_get_no_selection_io_cause() */ + +/*------------------------------------------------------------------------- + * Function: main + * + * Purpose: Runs tests with all combinations of configuration + * flags. + * + * Return: Success: 0 + * Failure: 1 + * + *------------------------------------------------------------------------- + */ +int +main(int argc, char *argv[]) +{ + int ret; + hid_t fapl = H5I_INVALID_HID; + hid_t fid = H5I_INVALID_HID; + int test_select_config; + unsigned chunked; + unsigned dtrans; + unsigned mwbuf; + + h5_reset(); + + /* Initialize MPI */ + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + + if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + P_TEST_ERROR; + + /* Set MPIO file driver */ + if (H5Pset_fapl_mpio(fapl, MPI_COMM_WORLD, MPI_INFO_NULL) < 0) + P_TEST_ERROR; + + if ((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + P_TEST_ERROR; + + /* Test with contiguous or chunked dataset */ + for (chunked = FALSE; chunked <= TRUE; chunked++) { + + /* Data transforms only apply to integer or floating-point datasets */ + /* therefore, not all tests are run with data transform */ + for (dtrans = FALSE; dtrans <= TRUE; dtrans++) { + + /* Test with and without modify_write_buf turned on */ + for (mwbuf = FALSE; mwbuf <= TRUE; mwbuf++) { + + if (MAINPROCESS) { + /* Print configuration message */ + printf("Testing for selection I/O "); + if (chunked) + printf("with chunked dataset, "); + else + printf("with contiguous dataset, "); + if (dtrans) + printf("data transform, "); + else + printf("without data transform, "); + if (mwbuf) + printf("and with modifying write buffers\n"); + else + printf("and without modifying write buffers\n"); + } + + for (test_select_config = (int)TEST_NO_TYPE_CONV; + test_select_config < (int)TEST_SELECT_NTESTS; test_select_config++) { + + switch (test_select_config) { + + case TEST_NO_TYPE_CONV: /* case 1 */ + if (MAINPROCESS) + TESTING_2("No type conversion (null case)"); + + test_no_type_conv(fid, chunked, dtrans, mwbuf); + + break; + + case TEST_NO_SIZE_CHANGE_NO_BKG: /* case 2 */ + if (MAINPROCESS) + TESTING_2("No size change, no background buffer"); + + /* Data transforms does not apply to the dataset datatype for this test */ + if (dtrans) { + if (MAINPROCESS) + SKIPPED(); + continue; + } + + test_no_size_change_no_bkg(fid, chunked, mwbuf); + + break; + + case TEST_LARGER_MEM_NO_BKG: /* case 3 */ + if (MAINPROCESS) + TESTING_2("Larger memory type, no background buffer"); + + test_larger_mem_type_no_bkg(fid, chunked, dtrans, mwbuf); + + break; + + case TEST_SMALLER_MEM_NO_BKG: /* case 4 */ + if (MAINPROCESS) + TESTING_2("Smaller memory type, no background buffer"); + + test_smaller_mem_type_no_bkg(fid, chunked, dtrans, mwbuf); + + break; + + case TEST_CMPD_WITH_BKG: /* case 5 */ + if (MAINPROCESS) + TESTING_2("Compound types with background buffer"); + /* Data transforms does not apply to the dataset datatype for this test */ + if (dtrans) { + if (MAINPROCESS) + SKIPPED(); + continue; + } + + test_cmpd_with_bkg(fid, chunked, mwbuf); + + break; + + case TEST_TYPE_CONV_SEL_EMPTY: /* case 6 */ + if (MAINPROCESS) + TESTING_2("Empty selections + Type conversion"); + + test_type_conv_sel_empty(fid, chunked, dtrans, mwbuf); + + break; + + case TEST_MULTI_CONV_NO_BKG: /* case 7 */ + if (MAINPROCESS) + TESTING_2("multi-datasets: type conv + no bkg buffer"); + + test_multi_dsets_no_bkg(fid, chunked, dtrans, mwbuf); + + break; + + case TEST_MULTI_CONV_BKG: /* case 8 */ + if (MAINPROCESS) + TESTING_2("multi-datasets: type conv + bkg buffer"); + + /* Data transforms does not apply to the dataset datatype for this test */ + if (dtrans) { + if (MAINPROCESS) + SKIPPED(); + } + else + test_multi_dsets_cmpd_with_bkg(fid, chunked, mwbuf); + + break; + + case TEST_MULTI_CONV_SIZE_CHANGE: /* case 9 */ + if (MAINPROCESS) + TESTING_2("multi-datasets: type conv + size change + no bkg buffer"); + + /* Data transforms does not apply to the dataset datatype for this test */ + if (dtrans) { + if (MAINPROCESS) + SKIPPED(); + } + else + test_multi_dsets_size_change_no_bkg(fid, chunked, mwbuf); + + break; + + case TEST_MULTI_CONV_SEL_EMPTY: /* case 10 */ + if (MAINPROCESS) + TESTING_2("multi-datasets: type conv + empty selections"); + + test_multi_dsets_conv_sel_empty(fid, chunked, dtrans, mwbuf); + + break; + + case TEST_MULTI_ALL: /* case 11 */ + if (MAINPROCESS) + TESTING_2("multi-datasets: no conv + conv without bkg + conv with bkg"); + + /* Data transforms does not apply to the dataset datatype for this test */ + if (dtrans) { + if (MAINPROCESS) + SKIPPED(); + } + else + test_multi_dsets_all(2, fid, chunked, mwbuf); + + break; + + case TEST_SELECT_NTESTS: + default: + P_TEST_ERROR; + break; + + } /* end switch */ + + } /* end for test_select_config */ + + } /* end mwbuf */ + + } /* end dtrans */ + } /* end chunked */ + + if (H5Fclose(fid) < 0) + P_TEST_ERROR; + + if (MAINPROCESS) { + printf("\n"); + TESTING("Testing for H5Pget_no_selection_io_cause()"); + } + test_get_no_selection_io_cause(FILENAME, fapl); + + /* Barrier to make sure all ranks are done before deleting the file, and + * also to clean up output (make sure PASSED is printed before any of the + * following messages) */ + if (MPI_Barrier(MPI_COMM_WORLD) != MPI_SUCCESS) + P_TEST_ERROR; + + /* Delete file */ + if (H5Fdelete(FILENAME, fapl) < 0) + P_TEST_ERROR; + + if (H5Pclose(fapl) < 0) + P_TEST_ERROR; + + /* Gather errors from all processes */ + MPI_Allreduce(&nerrors, &ret, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); + nerrors = ret; + + if (MAINPROCESS) { + printf("\n===================================\n"); + if (nerrors) + HDprintf("***Parallel selection I/O dataset tests detected %d errors***\n", nerrors); + else + HDprintf("Parallel selection I/O dataset tests finished with no errors\n"); + printf("===================================\n"); + } + + /* close HDF5 library */ + H5close(); + + /* MPI_Finalize must be called AFTER H5close which may use MPI calls */ + MPI_Finalize(); + + /* cannot just return (nerrors) because exit code is limited to 1 byte */ + return (nerrors != 0); +} /* end main() */ diff --git a/testpar/t_subfiling_vfd.c b/testpar/t_subfiling_vfd.c index 85df3bd..f827aa5 100644 --- a/testpar/t_subfiling_vfd.c +++ b/testpar/t_subfiling_vfd.c @@ -425,7 +425,10 @@ test_stripe_sizes(void) VRFY(tmp_filename, "HDmalloc succeeded"); dxpl_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((dxpl_id >= 0), "DCPL creation succeeded"); + VRFY((dxpl_id >= 0), "DXPL creation succeeded"); + + /* Set selection I/O mode on DXPL */ + VRFY((H5Pset_selection_io(dxpl_id, H5D_SELECTION_IO_MODE_ON) >= 0), "H5Pset_selection_io succeeded"); for (size_t i = 0; i < SUBF_NITER; i++) { H5FD_subfiling_params_t cfg; @@ -1011,12 +1014,19 @@ test_read_different_stripe_size(void) hid_t file_id = H5I_INVALID_HID; hid_t fapl_id = H5I_INVALID_HID; hid_t dset_id = H5I_INVALID_HID; + hid_t dxpl_id = H5I_INVALID_HID; hid_t fspace_id = H5I_INVALID_HID; char *tmp_filename = NULL; void *buf = NULL; curr_nerrors = nerrors; + dxpl_id = H5Pcreate(H5P_DATASET_XFER); + VRFY((dxpl_id >= 0), "DXPL creation succeeded"); + + /* Set selection I/O mode on DXPL */ + VRFY((H5Pset_selection_io(dxpl_id, H5D_SELECTION_IO_MODE_ON) >= 0), "H5Pset_selection_io succeeded"); + if (MAINPROCESS) TESTING_2("file re-opening with different stripe size"); @@ -1066,7 +1076,7 @@ test_read_different_stripe_size(void) for (size_t i = 0; i < count[0]; i++) ((SUBF_C_TYPE *)buf)[i] = (SUBF_C_TYPE)((size_t)mpi_rank + i); - VRFY((H5Dwrite(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, fspace_id, H5P_DEFAULT, buf) >= 0), + VRFY((H5Dwrite(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, fspace_id, dxpl_id, buf) >= 0), "Dataset write succeeded"); HDfree(buf); @@ -1133,7 +1143,7 @@ test_read_different_stripe_size(void) buf = HDcalloc(1, count[0] * sizeof(SUBF_C_TYPE)); VRFY(buf, "HDcalloc succeeded"); - VRFY((H5Dread(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, fspace_id, H5P_DEFAULT, buf) >= 0), + VRFY((H5Dread(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, fspace_id, dxpl_id, buf) >= 0), "Dataset read succeeded"); for (size_t i = 0; i < count[0]; i++) { @@ -1185,6 +1195,7 @@ test_read_different_stripe_size(void) } H5E_END_TRY; + VRFY((H5Pclose(dxpl_id) >= 0), "DXPL close succeeded"); VRFY((H5Pclose(fapl_id) >= 0), "FAPL close succeeded"); HDfree(tmp_filename); @@ -1214,11 +1225,18 @@ test_subfiling_precreate_rank_0(void) hid_t file_id = H5I_INVALID_HID; hid_t fapl_id = H5I_INVALID_HID; hid_t dset_id = H5I_INVALID_HID; + hid_t dxpl_id = H5I_INVALID_HID; hid_t fspace_id = H5I_INVALID_HID; void *buf = NULL; curr_nerrors = nerrors; + dxpl_id = H5Pcreate(H5P_DATASET_XFER); + VRFY((dxpl_id >= 0), "DXPL creation succeeded"); + + /* Set selection I/O mode on DXPL */ + VRFY((H5Pset_selection_io(dxpl_id, H5D_SELECTION_IO_MODE_ON) >= 0), "H5Pset_selection_io succeeded"); + if (MAINPROCESS) TESTING_2("file pre-creation on rank 0"); @@ -1278,7 +1296,7 @@ test_subfiling_precreate_rank_0(void) for (size_t i = 0; i < dset_dims[0]; i++) ((SUBF_C_TYPE *)buf)[i] = (SUBF_C_TYPE)((i / n_elements_per_rank) + (i % n_elements_per_rank)); - VRFY((H5Dwrite(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, fspace_id, H5P_DEFAULT, buf) >= 0), + VRFY((H5Dwrite(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, fspace_id, dxpl_id, buf) >= 0), "Dataset write succeeded"); HDfree(buf); @@ -1357,7 +1375,7 @@ test_subfiling_precreate_rank_0(void) buf = HDcalloc(1, count[0] * sizeof(SUBF_C_TYPE)); VRFY(buf, "HDcalloc succeeded"); - VRFY((H5Dread(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, fspace_id, H5P_DEFAULT, buf) >= 0), + VRFY((H5Dread(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, fspace_id, dxpl_id, buf) >= 0), "Dataset read succeeded"); for (size_t i = 0; i < n_elements_per_rank; i++) { @@ -1380,6 +1398,7 @@ test_subfiling_precreate_rank_0(void) H5E_END_TRY; VRFY((H5Pclose(fapl_id) >= 0), "FAPL close succeeded"); + VRFY((H5Pclose(dxpl_id) >= 0), "DXPL close succeeded"); CHECK_PASSED(); } @@ -1405,11 +1424,18 @@ test_subfiling_write_many_read_one(void) hid_t file_id = H5I_INVALID_HID; hid_t fapl_id = H5I_INVALID_HID; hid_t dset_id = H5I_INVALID_HID; + hid_t dxpl_id = H5I_INVALID_HID; hid_t fspace_id = H5I_INVALID_HID; void *buf = NULL; curr_nerrors = nerrors; + dxpl_id = H5Pcreate(H5P_DATASET_XFER); + VRFY((dxpl_id >= 0), "DXPL creation succeeded"); + + /* Set selection I/O mode on DXPL */ + VRFY((H5Pset_selection_io(dxpl_id, H5D_SELECTION_IO_MODE_ON) >= 0), "H5Pset_selection_io succeeded"); + if (MAINPROCESS) TESTING_2("reading back file with single MPI rank"); @@ -1461,7 +1487,7 @@ test_subfiling_write_many_read_one(void) for (size_t i = 0; i < count[0]; i++) ((SUBF_C_TYPE *)buf)[i] = (SUBF_C_TYPE)((size_t)mpi_rank + i); - VRFY((H5Dwrite(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, fspace_id, H5P_DEFAULT, buf) >= 0), + VRFY((H5Dwrite(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, fspace_id, dxpl_id, buf) >= 0), "Dataset write succeeded"); HDfree(buf); @@ -1486,7 +1512,7 @@ test_subfiling_write_many_read_one(void) buf = HDcalloc(1, target_size); VRFY(buf, "HDcalloc succeeded"); - VRFY((H5Dread(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, H5S_ALL, H5P_DEFAULT, buf) >= 0), + VRFY((H5Dread(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, H5S_ALL, dxpl_id, buf) >= 0), "Dataset read succeeded"); for (size_t i = 0; i < (size_t)mpi_size; i++) { @@ -1516,6 +1542,7 @@ test_subfiling_write_many_read_one(void) VRFY((mpi_code_g == MPI_SUCCESS), "MPI_Barrier succeeded"); VRFY((H5Sclose(fspace_id) >= 0), "File dataspace close succeeded"); + VRFY((H5Pclose(dxpl_id) >= 0), "DXPL close succeeded"); CHECK_PASSED(); } @@ -1543,11 +1570,18 @@ test_subfiling_write_many_read_few(void) hid_t file_id = H5I_INVALID_HID; hid_t fapl_id = H5I_INVALID_HID; hid_t dset_id = H5I_INVALID_HID; + hid_t dxpl_id = H5I_INVALID_HID; hid_t fspace_id = H5I_INVALID_HID; void *buf = NULL; curr_nerrors = nerrors; + dxpl_id = H5Pcreate(H5P_DATASET_XFER); + VRFY((dxpl_id >= 0), "DXPL creation succeeded"); + + /* Set selection I/O mode on DXPL */ + VRFY((H5Pset_selection_io(dxpl_id, H5D_SELECTION_IO_MODE_ON) >= 0), "H5Pset_selection_io succeeded"); + if (MAINPROCESS) TESTING_2("reading back file with fewer MPI ranks than written with"); @@ -1609,7 +1643,7 @@ test_subfiling_write_many_read_few(void) for (size_t i = 0; i < count[0]; i++) ((SUBF_C_TYPE *)buf)[i] = (SUBF_C_TYPE)((size_t)mpi_rank + i); - VRFY((H5Dwrite(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, fspace_id, H5P_DEFAULT, buf) >= 0), + VRFY((H5Dwrite(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, fspace_id, dxpl_id, buf) >= 0), "Dataset write succeeded"); HDfree(buf); @@ -1664,7 +1698,7 @@ test_subfiling_write_many_read_few(void) buf = HDcalloc(1, target_size); VRFY(buf, "HDcalloc succeeded"); - VRFY((H5Dread(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, H5S_ALL, H5P_DEFAULT, buf) >= 0), + VRFY((H5Dread(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, H5S_ALL, dxpl_id, buf) >= 0), "Dataset read succeeded"); for (size_t i = 0; i < (size_t)mpi_size; i++) { @@ -1699,6 +1733,7 @@ test_subfiling_write_many_read_few(void) VRFY((mpi_code_g == MPI_SUCCESS), "MPI_Barrier succeeded"); VRFY((H5Sclose(fspace_id) >= 0), "File dataspace close succeeded"); + VRFY((H5Pclose(dxpl_id) >= 0), "DXPL close succeeded"); CHECK_PASSED(); } @@ -1727,6 +1762,7 @@ test_subfiling_h5fuse(void) hid_t file_id = H5I_INVALID_HID; hid_t fapl_id = H5I_INVALID_HID; hid_t dset_id = H5I_INVALID_HID; + hid_t dxpl_id = H5I_INVALID_HID; hid_t fspace_id = H5I_INVALID_HID; void *buf = NULL; int skip_test = 0; @@ -1734,6 +1770,12 @@ test_subfiling_h5fuse(void) curr_nerrors = nerrors; + dxpl_id = H5Pcreate(H5P_DATASET_XFER); + VRFY((dxpl_id >= 0), "DXPL creation succeeded"); + + /* Set selection I/O mode on DXPL */ + VRFY((H5Pset_selection_io(dxpl_id, H5D_SELECTION_IO_MODE_ON) >= 0), "H5Pset_selection_io succeeded"); + if (MAINPROCESS) TESTING_2("h5fuse utility"); @@ -1826,7 +1868,7 @@ test_subfiling_h5fuse(void) for (size_t i = 0; i < count[0]; i++) ((SUBF_C_TYPE *)buf)[i] = (SUBF_C_TYPE)((size_t)mpi_rank + i); - VRFY((H5Dwrite(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, fspace_id, H5P_DEFAULT, buf) >= 0), + VRFY((H5Dwrite(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, fspace_id, dxpl_id, buf) >= 0), "Dataset write succeeded"); HDfree(buf); @@ -1899,7 +1941,7 @@ test_subfiling_h5fuse(void) buf = HDcalloc(1, target_size); VRFY(buf, "HDcalloc succeeded"); - VRFY((H5Dread(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, H5S_ALL, H5P_DEFAULT, buf) >= 0), + VRFY((H5Dread(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, H5S_ALL, dxpl_id, buf) >= 0), "Dataset read succeeded"); for (size_t i = 0; i < (size_t)mpi_size; i++) { @@ -1969,6 +2011,7 @@ test_subfiling_h5fuse(void) } VRFY((H5Pclose(fapl_id) >= 0), "FAPL close succeeded"); + VRFY((H5Pclose(dxpl_id) >= 0), "DXPL close succeeded"); mpi_code_g = MPI_Barrier(comm_g); VRFY((mpi_code_g == MPI_SUCCESS), "MPI_Barrier succeeded"); @@ -2132,9 +2175,6 @@ main(int argc, char **argv) H5open(); - /* Enable selection I/O using internal temporary workaround */ - H5_use_selection_io_g = TRUE; - if (MAINPROCESS) { HDprintf("Testing Subfiling VFD functionality\n"); } diff --git a/tools/lib/h5tools_dump.c b/tools/lib/h5tools_dump.c index 435ca87..57a1712 100644 --- a/tools/lib/h5tools_dump.c +++ b/tools/lib/h5tools_dump.c @@ -3747,7 +3747,7 @@ h5tools_dump_dcpl(FILE *stream, const h5tool_format_t *info, h5tools_context_t * /*------------------------------------------------------------------------- * Function: dump_comment * - * Purpose: prints the comment for the the object name + * Purpose: prints the comment for the object name * * Return: void *------------------------------------------------------------------------- |