diff options
| author | Neil Fortner <fortnern@gmail.com> | 2023-04-28 23:58:25 (GMT) |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-04-28 23:58:25 (GMT) |
| commit | 3236fb79cedcbe8fed2f421db1ae15f630b5b90f (patch) | |
| tree | b4751d6d020d057f4a43f3c89186418cdd1ac0cf | |
| parent | 4497feb5756d76479fb36fef999692c0909f8340 (diff) | |
| download | hdf5-3236fb79cedcbe8fed2f421db1ae15f630b5b90f.zip hdf5-3236fb79cedcbe8fed2f421db1ae15f630b5b90f.tar.gz hdf5-3236fb79cedcbe8fed2f421db1ae15f630b5b90f.tar.bz2 | |
Implement selection I/O with type conversion (#2823)
Initial implementation of selection I/O with type conversion. Allows
Parallel collective I/O with type conversion, as long as selection I/O
is enabled.
46 files changed, 9466 insertions, 378 deletions
@@ -37,6 +37,7 @@ $Source = ""; "H5D_alloc_time_t" => "Da", "H5D_append_cb_t" => "DA", "H5FD_mpio_collective_opt_t" => "Dc", + "H5D_selection_io_mode_t" => "DC", "H5D_fill_time_t" => "Df", "H5D_fill_value_t" => "DF", "H5D_gather_func_t" => "Dg", diff --git a/doxygen/examples/tables/propertyLists.dox b/doxygen/examples/tables/propertyLists.dox index 375fd50..4e20197 100644 --- a/doxygen/examples/tables/propertyLists.dox +++ b/doxygen/examples/tables/propertyLists.dox @@ -694,6 +694,18 @@ of the library for reading or writing the actual data.</td> <td>Gets local and global causes that broke collective I/O on the last parallel I/O call.</td> </tr> <tr> +<td>#H5Pset_selection_io/#H5Pget_selection_io</td> +<td>Sets/gets the selection I/O mode.</td> +</tr> +<tr> +<td>#H5Pget_no_selection_io_cause</td> +<td>Gets the cause for not performing selection or vector I/O on the last parallel I/O call.</td> +</tr> +<tr> +<td>#H5Pset_modify_write_buf/#H5Pget_modify_write_buf</td> +<td>Sets/gets a flag allowing the library to modify the contents of the write buffer.</td> +</tr> +<tr> <td>H5Pset_preserve/H5Pget_preserve</td> <td>No longer available, deprecated as it no longer has any effect.</td> </tr> @@ -952,4 +964,3 @@ encoding for object names.</td> //! [acpl_table] * */ -
\ No newline at end of file diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index ad51ff1..89a821c 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -105,6 +105,16 @@ New Features machine's node-local storage while placing the subfiling configuration file on a file system readable by all machine nodes. + - Added H5Pset_selection_io(), H5Pget_selection_io(), and + H5Pget_no_selection_io_cause() API functions to manage the selection I/O + feature. This can be used to enable collective I/O with type conversion, + or it can be used with custom VFDs that support vector or selection I/O. + + - Added H5Pset_modify_write_buf() and H5Pget_modify_write_buf() API + functions to allow the library to modify the contents of write buffers, in + order to avoid malloc/memcpy. Currently only used for type conversion + with selection I/O. + Parallel Library: ----------------- @@ -82,8 +82,6 @@ hbool_t H5_libinit_g = FALSE; /* Library hasn't been initialized */ hbool_t H5_libterm_g = FALSE; /* Library isn't being shutdown */ #endif -hbool_t H5_use_selection_io_g = FALSE; - char H5_lib_vers_info_g[] = H5_VERS_INFO; static hbool_t H5_dont_atexit_g = FALSE; H5_debug_t H5_debug_g; /* debugging info */ @@ -141,8 +139,7 @@ herr_t H5_init_library(void) { size_t i; - char *env_use_select_io = NULL; - herr_t ret_value = SUCCEED; + herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI(FAIL) @@ -274,14 +271,6 @@ H5_init_library(void) } /* clang-format on */ - /* Check for HDF5_USE_SELECTION_IO env variable */ - env_use_select_io = HDgetenv("HDF5_USE_SELECTION_IO"); - if (NULL != env_use_select_io && HDstrcmp(env_use_select_io, "") && HDstrcmp(env_use_select_io, "0") && - HDstrcmp(env_use_select_io, "no") && HDstrcmp(env_use_select_io, "No") && - HDstrcmp(env_use_select_io, "NO") && HDstrcmp(env_use_select_io, "false") && - HDstrcmp(env_use_select_io, "False") && HDstrcmp(env_use_select_io, "FALSE")) - H5_use_selection_io_g = TRUE; - /* Debugging? */ H5__debug_mask("-all"); H5__debug_mask(HDgetenv("HDF5_DEBUG")); @@ -94,14 +94,12 @@ H5CX_RETRIEVE_PROP_COMMON(PL, DEF_PL, PROP_NAME, PROP_FIELD) \ } /* end if */ -#ifdef H5_HAVE_PARALLEL /* Macro for the duplicated code to retrieve possibly set properties from a property list */ #define H5CX_RETRIEVE_PROP_VALID_SET(PL, DEF_PL, PROP_NAME, PROP_FIELD) \ /* Check if the value has been retrieved already */ \ if (!((*head)->ctx.H5_GLUE(PROP_FIELD, _valid) || (*head)->ctx.H5_GLUE(PROP_FIELD, _set))) { \ H5CX_RETRIEVE_PROP_COMMON(PL, DEF_PL, PROP_NAME, PROP_FIELD) \ - } /* end if */ -#endif /* H5_HAVE_PARALLEL */ + } /* end if */ #if defined(H5_HAVE_PARALLEL) && defined(H5_HAVE_INSTRUMENTED_LIBRARY) /* Macro for the duplicated code to test and set properties for a property list */ @@ -127,7 +125,6 @@ } #endif /* defined(H5_HAVE_PARALLEL) && defined(H5_HAVE_INSTRUMENTED_LIBRARY) */ -#ifdef H5_HAVE_PARALLEL /* Macro for the duplicated code to test and set properties for a property list */ #define H5CX_SET_PROP(PROP_NAME, PROP_FIELD) \ if ((*head)->ctx.H5_GLUE(PROP_FIELD, _set)) { \ @@ -137,8 +134,7 @@ /* Set the property */ \ if (H5P_set((*head)->ctx.dxpl, PROP_NAME, &(*head)->ctx.PROP_FIELD) < 0) \ HGOTO_ERROR(H5E_CONTEXT, H5E_CANTSET, NULL, "error setting data xfer property") \ - } /* end if */ -#endif /* H5_HAVE_PARALLEL */ + } /* end if */ /******************/ /* Local Typedefs */ @@ -242,16 +238,20 @@ typedef struct H5CX_t { unsigned mpio_chunk_opt_ratio; /* Collective chunk ratio (H5D_XFER_MPIO_CHUNK_OPT_RATIO_NAME) */ hbool_t mpio_chunk_opt_ratio_valid; /* Whether collective chunk ratio is valid */ #endif /* H5_HAVE_PARALLEL */ - H5Z_EDC_t err_detect; /* Error detection info (H5D_XFER_EDC_NAME) */ - hbool_t err_detect_valid; /* Whether error detection info is valid */ - H5Z_cb_t filter_cb; /* Filter callback function (H5D_XFER_FILTER_CB_NAME) */ - hbool_t filter_cb_valid; /* Whether filter callback function is valid */ - H5Z_data_xform_t *data_transform; /* Data transform info (H5D_XFER_XFORM_NAME) */ - hbool_t data_transform_valid; /* Whether data transform info is valid */ - H5T_vlen_alloc_info_t vl_alloc_info; /* VL datatype alloc info (H5D_XFER_VLEN_*_NAME) */ - hbool_t vl_alloc_info_valid; /* Whether VL datatype alloc info is valid */ - H5T_conv_cb_t dt_conv_cb; /* Datatype conversion struct (H5D_XFER_CONV_CB_NAME) */ - hbool_t dt_conv_cb_valid; /* Whether datatype conversion struct is valid */ + H5Z_EDC_t err_detect; /* Error detection info (H5D_XFER_EDC_NAME) */ + hbool_t err_detect_valid; /* Whether error detection info is valid */ + H5Z_cb_t filter_cb; /* Filter callback function (H5D_XFER_FILTER_CB_NAME) */ + hbool_t filter_cb_valid; /* Whether filter callback function is valid */ + H5Z_data_xform_t *data_transform; /* Data transform info (H5D_XFER_XFORM_NAME) */ + hbool_t data_transform_valid; /* Whether data transform info is valid */ + H5T_vlen_alloc_info_t vl_alloc_info; /* VL datatype alloc info (H5D_XFER_VLEN_*_NAME) */ + hbool_t vl_alloc_info_valid; /* Whether VL datatype alloc info is valid */ + H5T_conv_cb_t dt_conv_cb; /* Datatype conversion struct (H5D_XFER_CONV_CB_NAME) */ + hbool_t dt_conv_cb_valid; /* Whether datatype conversion struct is valid */ + H5D_selection_io_mode_t selection_io_mode; /* Selection I/O mode (H5D_XFER_SELECTION_IO_MODE_NAME) */ + hbool_t selection_io_mode_valid; /* Whether selection I/O mode is valid */ + hbool_t modify_write_buf; /* Whether the library can modify write buffers */ + hbool_t modify_write_buf_valid; /* Whether the modify_write_buf field is valid */ /* Return-only DXPL properties to return to application */ #ifdef H5_HAVE_PARALLEL @@ -297,9 +297,13 @@ typedef struct H5CX_t { hbool_t mpio_coll_rank0_bcast; /* Instrumented "collective chunk multi ratio ind" value (H5D_XFER_COLL_CHUNK_MULTI_RATIO_IND_NAME) */ hbool_t - mpio_coll_rank0_bcast_set; /* Whether instrumented "collective chunk multi ratio ind" value is set */ -#endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ -#endif /* H5_HAVE_PARALLEL */ + mpio_coll_rank0_bcast_set; /* Whether instrumented "collective chunk multi ratio ind" value is set */ +#endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ +#endif /* H5_HAVE_PARALLEL */ + uint32_t no_selection_io_cause; /* Reason for not performing selection I/O + (H5D_XFER_NO_SELECTION_IO_CAUSE_NAME) */ + hbool_t no_selection_io_cause_set; /* Whether reason for not performing selection I/O is set */ + hbool_t no_selection_io_cause_valid; /* Whether reason for not performing selection I/O is valid */ /* Cached LCPL properties */ H5T_cset_t encoding; /* Link name character encoding */ @@ -370,15 +374,19 @@ typedef struct H5CX_dxpl_cache_t { uint32_t mpio_global_no_coll_cause; /* Global reason for breaking collective I/O (H5D_MPIO_GLOBAL_NO_COLLECTIVE_CAUSE_NAME) */ H5FD_mpio_chunk_opt_t - mpio_chunk_opt_mode; /* Collective chunk option (H5D_XFER_MPIO_CHUNK_OPT_HARD_NAME) */ - unsigned mpio_chunk_opt_num; /* Collective chunk threshold (H5D_XFER_MPIO_CHUNK_OPT_NUM_NAME) */ - unsigned mpio_chunk_opt_ratio; /* Collective chunk ratio (H5D_XFER_MPIO_CHUNK_OPT_RATIO_NAME) */ -#endif /* H5_HAVE_PARALLEL */ - H5Z_EDC_t err_detect; /* Error detection info (H5D_XFER_EDC_NAME) */ - H5Z_cb_t filter_cb; /* Filter callback function (H5D_XFER_FILTER_CB_NAME) */ - H5Z_data_xform_t *data_transform; /* Data transform info (H5D_XFER_XFORM_NAME) */ - H5T_vlen_alloc_info_t vl_alloc_info; /* VL datatype alloc info (H5D_XFER_VLEN_*_NAME) */ - H5T_conv_cb_t dt_conv_cb; /* Datatype conversion struct (H5D_XFER_CONV_CB_NAME) */ + mpio_chunk_opt_mode; /* Collective chunk option (H5D_XFER_MPIO_CHUNK_OPT_HARD_NAME) */ + unsigned mpio_chunk_opt_num; /* Collective chunk threshold (H5D_XFER_MPIO_CHUNK_OPT_NUM_NAME) */ + unsigned mpio_chunk_opt_ratio; /* Collective chunk ratio (H5D_XFER_MPIO_CHUNK_OPT_RATIO_NAME) */ +#endif /* H5_HAVE_PARALLEL */ + H5Z_EDC_t err_detect; /* Error detection info (H5D_XFER_EDC_NAME) */ + H5Z_cb_t filter_cb; /* Filter callback function (H5D_XFER_FILTER_CB_NAME) */ + H5Z_data_xform_t *data_transform; /* Data transform info (H5D_XFER_XFORM_NAME) */ + H5T_vlen_alloc_info_t vl_alloc_info; /* VL datatype alloc info (H5D_XFER_VLEN_*_NAME) */ + H5T_conv_cb_t dt_conv_cb; /* Datatype conversion struct (H5D_XFER_CONV_CB_NAME) */ + H5D_selection_io_mode_t selection_io_mode; /* Selection I/O mode (H5D_XFER_SELECTION_IO_MODE_NAME) */ + uint32_t no_selection_io_cause; /* Reasons for not performing selection I/O + (H5D_XFER_NO_SELECTION_IO_CAUSE_NAME) */ + hbool_t modify_write_buf; /* Whether the library can modify write buffers */ } H5CX_dxpl_cache_t; /* Typedef for cached default link creation property list information */ @@ -566,6 +574,19 @@ H5CX_init(void) if (H5P_get(dx_plist, H5D_XFER_CONV_CB_NAME, &H5CX_def_dxpl_cache.dt_conv_cb) < 0) HGOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "Can't retrieve datatype conversion exception callback") + /* Get the selection I/O mode */ + if (H5P_get(dx_plist, H5D_XFER_SELECTION_IO_MODE_NAME, &H5CX_def_dxpl_cache.selection_io_mode) < 0) + HGOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "Can't retrieve parallel transfer method") + + /* Get the local & global reasons for breaking selection I/O values */ + if (H5P_get(dx_plist, H5D_XFER_NO_SELECTION_IO_CAUSE_NAME, &H5CX_def_dxpl_cache.no_selection_io_cause) < + 0) + HGOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "Can't retrieve cause for no selection I/O") + + /* Get the modify write buffer property */ + if (H5P_get(dx_plist, H5D_XFER_MODIFY_WRITE_BUF_NAME, &H5CX_def_dxpl_cache.modify_write_buf) < 0) + HGOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "Can't retrieve modify write buffer property") + /* Reset the "default LCPL cache" information */ HDmemset(&H5CX_def_lcpl_cache, 0, sizeof(H5CX_lcpl_cache_t)); @@ -2564,6 +2585,111 @@ done: } /* end H5CX_get_dt_conv_cb() */ /*------------------------------------------------------------------------- + * Function: H5CX_get_selection_io_mode + * + * Purpose: Retrieves the selection I/O mode for the current API call context. + * + * Return: Non-negative on success / Negative on failure + * + * Programmer: Vailin Choi + * March 5, 2023 + * + *------------------------------------------------------------------------- + */ +herr_t +H5CX_get_selection_io_mode(H5D_selection_io_mode_t *selection_io_mode) +{ + H5CX_node_t **head = NULL; /* Pointer to head of API context list */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity check */ + HDassert(selection_io_mode); + head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ + HDassert(head && *head); + HDassert(H5P_DEFAULT != (*head)->ctx.dxpl_id); + + H5CX_RETRIEVE_PROP_VALID(dxpl, H5P_DATASET_XFER_DEFAULT, H5D_XFER_SELECTION_IO_MODE_NAME, + selection_io_mode) + + /* Get the value */ + *selection_io_mode = (*head)->ctx.selection_io_mode; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5CX_get_selection_io_mode() */ + +/*------------------------------------------------------------------------- + * Function: H5CX_get_no_selection_io_cause + * + * Purpose: Retrieves the cause for not performing selection I/O + * for the current API call context. + * + * Return: Non-negative on success / Negative on failure + * + * Programmer: Vailin Choi + * April 15, 2023 + * + *------------------------------------------------------------------------- + */ +herr_t +H5CX_get_no_selection_io_cause(uint32_t *no_selection_io_cause) +{ + H5CX_node_t **head = NULL; /* Pointer to head of API context list */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity check */ + HDassert(no_selection_io_cause); + head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ + HDassert(head && *head); + HDassert(H5P_DEFAULT != (*head)->ctx.dxpl_id); + + H5CX_RETRIEVE_PROP_VALID_SET(dxpl, H5P_DATASET_XFER_DEFAULT, H5D_XFER_NO_SELECTION_IO_CAUSE_NAME, + no_selection_io_cause) + + /* Get the value */ + *no_selection_io_cause = (*head)->ctx.no_selection_io_cause; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5CX_get_no_selection_io_cause() */ + +/*------------------------------------------------------------------------- + * Function: H5CX_get_modify_write_buf + * + * Purpose: Retrieves the modify write buffer property for the current API call context. + * + * Return: Non-negative on success / Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5CX_get_modify_write_buf(hbool_t *modify_write_buf) +{ + H5CX_node_t **head = NULL; /* Pointer to head of API context list */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity check */ + HDassert(modify_write_buf); + head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ + HDassert(head && *head); + HDassert(H5P_DEFAULT != (*head)->ctx.dxpl_id); + + H5CX_RETRIEVE_PROP_VALID(dxpl, H5P_DATASET_XFER_DEFAULT, H5D_XFER_MODIFY_WRITE_BUF_NAME, modify_write_buf) + + /* Get the value */ + *modify_write_buf = (*head)->ctx.modify_write_buf; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5CX_get_selection_io_mode() */ + +/*------------------------------------------------------------------------- * Function: H5CX_get_encoding * * Purpose: Retrieves the character encoding for the current API call context. @@ -3544,6 +3670,41 @@ done: #endif /* H5_HAVE_PARALLEL */ /*------------------------------------------------------------------------- + * Function: H5CX_set_no_selecction_io_cause + * + * Purpose: Sets the reason for not performing selection I/O for + * the current API call context. + * + * Return: <none> + * + * Programmer: Vailin Choi + * April 15, 2023 + * + *------------------------------------------------------------------------- + */ +void +H5CX_set_no_selection_io_cause(uint32_t no_selection_io_cause) +{ + H5CX_node_t **head = NULL; /* Pointer to head of API context list */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Sanity checks */ + head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ + HDassert(head && *head); + HDassert((*head)->ctx.dxpl_id != H5P_DEFAULT); + + /* If we're using the default DXPL, don't modify it */ + if ((*head)->ctx.dxpl_id != H5P_DATASET_XFER_DEFAULT) { + /* Cache the value for later, marking it to set in DXPL when context popped */ + (*head)->ctx.no_selection_io_cause = no_selection_io_cause; + (*head)->ctx.no_selection_io_cause_set = TRUE; + } /* end if */ + + FUNC_LEAVE_NOAPI_VOID +} /* end H5CX_set_no_selectiion_io_cause() */ + +/*------------------------------------------------------------------------- * Function: H5CX_get_ohdr_flags * * Purpose: Retrieves the object header flags for the current API call context. @@ -3596,11 +3757,7 @@ H5CX__pop_common(hbool_t update_dxpl_props) H5CX_node_t **head = NULL; /* Pointer to head of API context list */ H5CX_node_t *ret_value = NULL; /* Return value */ -#ifdef H5_HAVE_PARALLEL FUNC_ENTER_PACKAGE -#else - FUNC_ENTER_PACKAGE_NOERR -#endif /* Sanity check */ head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ @@ -3608,6 +3765,7 @@ H5CX__pop_common(hbool_t update_dxpl_props) /* Check for cached DXPL properties to return to application */ if (update_dxpl_props) { + H5CX_SET_PROP(H5D_XFER_NO_SELECTION_IO_CAUSE_NAME, no_selection_io_cause) #ifdef H5_HAVE_PARALLEL H5CX_SET_PROP(H5D_MPIO_ACTUAL_CHUNK_OPT_MODE_NAME, mpio_actual_chunk_opt) H5CX_SET_PROP(H5D_MPIO_ACTUAL_IO_MODE_NAME, mpio_actual_io_mode) @@ -3629,9 +3787,7 @@ H5CX__pop_common(hbool_t update_dxpl_props) ret_value = (*head); (*head) = (*head)->next; -#ifdef H5_HAVE_PARALLEL done: -#endif /* H5_HAVE_PARALLEL */ FUNC_LEAVE_NOAPI(ret_value) } /* end H5CX__pop_common() */ diff --git a/src/H5CXprivate.h b/src/H5CXprivate.h index 4c034c1..f0bec20 100644 --- a/src/H5CXprivate.h +++ b/src/H5CXprivate.h @@ -115,6 +115,9 @@ H5_DLL herr_t H5CX_get_filter_cb(H5Z_cb_t *filter_cb); H5_DLL herr_t H5CX_get_data_transform(H5Z_data_xform_t **data_transform); H5_DLL herr_t H5CX_get_vlen_alloc_info(H5T_vlen_alloc_info_t *vl_alloc_info); H5_DLL herr_t H5CX_get_dt_conv_cb(H5T_conv_cb_t *cb_struct); +H5_DLL herr_t H5CX_get_selection_io_mode(H5D_selection_io_mode_t *selection_io_mode); +H5_DLL herr_t H5CX_get_no_selection_io_cause(uint32_t *no_selection_io_cause); +H5_DLL herr_t H5CX_get_modify_write_buf(hbool_t *modify_write_buf); /* "Getter" routines for LCPL properties cached in API context */ H5_DLL herr_t H5CX_get_encoding(H5T_cset_t *encoding); @@ -158,6 +161,9 @@ H5_DLL herr_t H5CX_set_nlinks(size_t nlinks); H5_DLL herr_t H5CX_init(void); /* "Setter" routines for cached DXPL properties that must be returned to application */ + +H5_DLL void H5CX_set_no_selection_io_cause(uint32_t no_selection_io_cause); + #ifdef H5_HAVE_PARALLEL H5_DLL void H5CX_set_mpio_actual_chunk_opt(H5D_mpio_actual_chunk_opt_mode_t chunk_opt); H5_DLL void H5CX_set_mpio_actual_io_mode(H5D_mpio_actual_io_mode_t actual_io_mode); diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index 830560d..59577c3 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -289,21 +289,21 @@ static ssize_t H5D__nonexistent_readvv(const H5D_io_info_t *io_info, const H5D_d static int H5D__chunk_format_convert_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata); /* Helper routines */ -static herr_t H5D__chunk_set_info_real(H5O_layout_chunk_t *layout, unsigned ndims, const hsize_t *curr_dims, - const hsize_t *max_dims); -static herr_t H5D__chunk_cinfo_cache_reset(H5D_chunk_cached_t *last); -static herr_t H5D__chunk_cinfo_cache_update(H5D_chunk_cached_t *last, const H5D_chunk_ud_t *udata); -static hbool_t H5D__chunk_cinfo_cache_found(const H5D_chunk_cached_t *last, H5D_chunk_ud_t *udata); -static herr_t H5D__create_piece_map_single(H5D_dset_io_info_t *di, H5D_io_info_t *io_info); -static herr_t H5D__create_piece_file_map_all(H5D_dset_io_info_t *di, H5D_io_info_t *io_info); -static herr_t H5D__create_piece_file_map_hyper(H5D_dset_io_info_t *di, H5D_io_info_t *io_info); -static herr_t H5D__create_piece_mem_map_1d(const H5D_dset_io_info_t *di); -static herr_t H5D__create_piece_mem_map_hyper(const H5D_dset_io_info_t *di); -static herr_t H5D__piece_file_cb(void *elem, const H5T_t *type, unsigned ndims, const hsize_t *coords, +static herr_t H5D__chunk_set_info_real(H5O_layout_chunk_t *layout, unsigned ndims, const hsize_t *curr_dims, + const hsize_t *max_dims); +static herr_t H5D__chunk_cinfo_cache_reset(H5D_chunk_cached_t *last); +static herr_t H5D__chunk_cinfo_cache_update(H5D_chunk_cached_t *last, const H5D_chunk_ud_t *udata); +static hbool_t H5D__chunk_cinfo_cache_found(const H5D_chunk_cached_t *last, H5D_chunk_ud_t *udata); +static herr_t H5D__create_piece_map_single(H5D_dset_io_info_t *di, H5D_io_info_t *io_info); +static herr_t H5D__create_piece_file_map_all(H5D_dset_io_info_t *di, H5D_io_info_t *io_info); +static herr_t H5D__create_piece_file_map_hyper(H5D_dset_io_info_t *di, H5D_io_info_t *io_info); +static herr_t H5D__create_piece_mem_map_1d(const H5D_dset_io_info_t *di); +static herr_t H5D__create_piece_mem_map_hyper(const H5D_dset_io_info_t *di); +static herr_t H5D__piece_file_cb(void *elem, const H5T_t *type, unsigned ndims, const hsize_t *coords, + void *_opdata); +static herr_t H5D__piece_mem_cb(void *elem, const H5T_t *type, unsigned ndims, const hsize_t *coords, void *_opdata); -static herr_t H5D__piece_mem_cb(void *elem, const H5T_t *type, unsigned ndims, const hsize_t *coords, - void *_opdata); -static htri_t H5D__chunk_may_use_select_io(const H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset_info); +static herr_t H5D__chunk_may_use_select_io(H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset_info); static unsigned H5D__chunk_hash_val(const H5D_shared_t *shared, const hsize_t *scaled); static herr_t H5D__chunk_flush_entry(const H5D_t *dset, H5D_rdcc_ent_t *ent, hbool_t reset); static herr_t H5D__chunk_cache_evict(const H5D_t *dset, H5D_rdcc_ent_t *ent, hbool_t flush); @@ -1062,9 +1062,8 @@ H5D__chunk_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo) htri_t file_space_normalized = FALSE; /* File dataspace was normalized */ unsigned f_ndims; /* The number of dimensions of the file's dataspace */ int sm_ndims; /* The number of dimensions of the memory buffer's dataspace (signed) */ - htri_t use_selection_io = FALSE; /* Whether to use selection I/O */ - unsigned u; /* Local index variable */ - herr_t ret_value = SUCCEED; /* Return value */ + unsigned u; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -1114,10 +1113,29 @@ H5D__chunk_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo) /* Check if we're performing selection I/O and save the result if it hasn't * been disabled already */ - if (io_info->use_select_io) { - if ((use_selection_io = H5D__chunk_may_use_select_io(io_info, dinfo)) < 0) + if (io_info->use_select_io != H5D_SELECTION_IO_MODE_OFF) + if (H5D__chunk_may_use_select_io(io_info, dinfo) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't check if selection I/O is possible") - io_info->use_select_io = (hbool_t)use_selection_io; + + /* Calculate type conversion buffer size if necessary. Currently only implemented for selection I/O. */ + if (io_info->use_select_io != H5D_SELECTION_IO_MODE_OFF && + !(dinfo->type_info.is_xform_noop && dinfo->type_info.is_conv_noop)) { + H5SL_node_t *chunk_node; /* Current node in chunk skip list */ + + /* Iterate through nodes in chunk skip list */ + chunk_node = H5D_CHUNK_GET_FIRST_NODE(dinfo); + while (chunk_node) { + H5D_piece_info_t *piece_info; /* Chunk information */ + + /* Get the actual chunk information from the skip list node */ + piece_info = H5D_CHUNK_GET_NODE_INFO(dinfo, chunk_node); + + /* Handle type conversion buffer */ + H5D_INIT_PIECE_TCONV(io_info, dinfo, piece_info) + + /* Advance to next chunk in list */ + chunk_node = H5D_CHUNK_GET_NEXT_NODE(dinfo, chunk_node); + } } done: @@ -1571,6 +1589,10 @@ H5D__create_piece_map_single(H5D_dset_io_info_t *di, H5D_io_info_t *io_info) /* Indicate that the chunk's memory dataspace is shared */ piece_info->mspace_shared = TRUE; + /* Initialize in-place type conversion info. Start with it disabled. */ + piece_info->in_place_tconv = FALSE; + piece_info->buf_off = 0; + /* make connection to related dset info from this piece_info */ piece_info->dset_info = di; @@ -1700,6 +1722,10 @@ H5D__create_piece_file_map_all(H5D_dset_io_info_t *di, H5D_io_info_t *io_info) /* make connection to related dset info from this piece_info */ new_piece_info->dset_info = di; + /* Initialize in-place type conversion info. Start with it disabled. */ + new_piece_info->in_place_tconv = FALSE; + new_piece_info->buf_off = 0; + /* Insert the new chunk into the skip list */ if (H5SL_insert(fm->dset_sel_pieces, new_piece_info, &new_piece_info->index) < 0) { H5D__free_piece_info(new_piece_info, NULL, NULL); @@ -1896,6 +1922,10 @@ H5D__create_piece_file_map_hyper(H5D_dset_io_info_t *dinfo, H5D_io_info_t *io_in /* make connection to related dset info from this piece_info */ new_piece_info->dset_info = dinfo; + /* Initialize in-place type conversion info. Start with it disabled. */ + new_piece_info->in_place_tconv = FALSE; + new_piece_info->buf_off = 0; + /* Add piece to global piece_count */ io_info->piece_count++; @@ -2271,6 +2301,10 @@ H5D__piece_file_cb(void H5_ATTR_UNUSED *elem, const H5T_t H5_ATTR_UNUSED *type, H5MM_memcpy(piece_info->scaled, scaled, sizeof(hsize_t) * fm->f_ndims); piece_info->scaled[fm->f_ndims] = 0; + /* Initialize in-place type conversion info. Start with it disabled. */ + piece_info->in_place_tconv = FALSE; + piece_info->buf_off = 0; + /* Make connection to related dset info from this piece_info */ piece_info->dset_info = dinfo; @@ -2547,11 +2581,11 @@ done: * *------------------------------------------------------------------------- */ -static htri_t -H5D__chunk_may_use_select_io(const H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset_info) +static herr_t +H5D__chunk_may_use_select_io(H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset_info) { - const H5D_t *dataset = NULL; /* Local pointer to dataset info */ - htri_t ret_value = FAIL; /* Return value */ + const H5D_t *dataset = NULL; /* Local pointer to dataset info */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -2562,20 +2596,22 @@ H5D__chunk_may_use_select_io(const H5D_io_info_t *io_info, const H5D_dset_io_inf dataset = dset_info->dset; HDassert(dataset); - /* Don't use selection I/O if it's globally disabled, there is a type - * conversion, or if there are filters on the dataset (for now) */ - if (dset_info->io_ops.single_read != H5D__select_read || dataset->shared->dcpl_cache.pline.nused > 0) - ret_value = FALSE; + /* Don't use selection I/O if there are filters on the dataset (for now) */ + if (dataset->shared->dcpl_cache.pline.nused > 0) { + io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; + io_info->no_selection_io_cause |= H5D_SEL_IO_DATASET_FILTER; + } else { hbool_t page_buf_enabled; - HDassert(dset_info->io_ops.single_write == H5D__select_write); - /* Check if the page buffer is enabled */ if (H5PB_enabled(io_info->f_sh, H5FD_MEM_DRAW, &page_buf_enabled) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't check if page buffer is enabled") - if (page_buf_enabled) - ret_value = FALSE; + if (page_buf_enabled) { + /* Note that page buffer is disabled in parallel */ + io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; + io_info->no_selection_io_cause |= H5D_SEL_IO_PAGE_BUFFER; + } else { /* Check if chunks in this dataset may be cached, if so don't use * selection I/O (for now). Note that chunks temporarily cached for @@ -2586,16 +2622,14 @@ H5D__chunk_may_use_select_io(const H5D_io_info_t *io_info, const H5D_dset_io_inf * must bypass the chunk-cache scheme because other MPI processes * could be writing to other elements in the same chunk. */ - if (io_info->using_mpi_vfd && (H5F_ACC_RDWR & H5F_INTENT(dataset->oloc.file))) - ret_value = TRUE; - else { + if (!(io_info->using_mpi_vfd && (H5F_ACC_RDWR & H5F_INTENT(dataset->oloc.file)))) { #endif /* H5_HAVE_PARALLEL */ /* Check if the chunk is too large to keep in the cache */ H5_CHECK_OVERFLOW(dataset->shared->layout.u.chunk.size, uint32_t, size_t); - if ((size_t)dataset->shared->layout.u.chunk.size > dataset->shared->cache.chunk.nbytes_max) - ret_value = TRUE; - else - ret_value = FALSE; + if ((size_t)dataset->shared->layout.u.chunk.size <= dataset->shared->cache.chunk.nbytes_max) { + io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; + io_info->no_selection_io_cause |= H5D_SEL_IO_CHUNK_CACHE; + } #ifdef H5_HAVE_PARALLEL } /* end else */ #endif /* H5_HAVE_PARALLEL */ @@ -2668,7 +2702,7 @@ H5D__chunk_read(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info) } /* Different blocks depending on whether we're using selection I/O */ - if (io_info->use_select_io) { + if (io_info->use_select_io == H5D_SELECTION_IO_MODE_ON) { size_t num_chunks; size_t element_sizes[2] = {dset_info->type_info.src_type_size, 0}; void *bufs[2] = {dset_info->buf.vp, NULL}; @@ -2749,6 +2783,8 @@ H5D__chunk_read(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info) io_info->addrs[io_info->pieces_added] = udata.chunk_block.offset; io_info->element_sizes[io_info->pieces_added] = element_sizes[0]; io_info->rbufs[io_info->pieces_added] = bufs[0]; + if (io_info->sel_pieces) + io_info->sel_pieces[io_info->pieces_added] = chunk_info; io_info->pieces_added++; } } /* end if */ @@ -2766,7 +2802,7 @@ H5D__chunk_read(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info) chunk_node = H5D_CHUNK_GET_NEXT_NODE(dset_info, chunk_node); } /* end while */ - /* Only perform I/O if not performing multi dataset I/O, otherwise the + /* Only perform I/O if not performing multi dataset I/O or type conversion, otherwise the * higher level will handle it after all datasets have been processed */ if (H5D_LAYOUT_CB_PERFORM_IO(io_info)) { /* Issue selection I/O call (we can skip the page buffer because we've @@ -2787,7 +2823,13 @@ H5D__chunk_read(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info) chunk_addrs = H5MM_xfree(chunk_addrs); } /* end if */ } /* end if */ - } /* end if */ + +#ifdef H5_HAVE_PARALLEL + /* Report that collective chunk I/O was used (will only be set on the DXPL if collective I/O was + * requested) */ + io_info->actual_io_mode |= H5D_MPIO_CHUNK_COLLECTIVE; +#endif /* H5_HAVE_PARALLEL */ + } /* end if */ else { H5D_io_info_t ctg_io_info; /* Contiguous I/O info object */ H5D_storage_t ctg_store; /* Chunk storage information as contiguous dataset */ @@ -2981,7 +3023,7 @@ H5D__chunk_write(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info) cpt_store.compact.dirty = &cpt_dirty; /* Different blocks depending on whether we're using selection I/O */ - if (io_info->use_select_io) { + if (io_info->use_select_io == H5D_SELECTION_IO_MODE_ON) { size_t num_chunks; size_t element_sizes[2] = {dset_info->type_info.dst_type_size, 0}; const void *bufs[2] = {dset_info->buf.cvp, NULL}; @@ -3139,6 +3181,8 @@ H5D__chunk_write(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info) io_info->addrs[io_info->pieces_added] = udata.chunk_block.offset; io_info->element_sizes[io_info->pieces_added] = element_sizes[0]; io_info->wbufs[io_info->pieces_added] = bufs[0]; + if (io_info->sel_pieces) + io_info->sel_pieces[io_info->pieces_added] = chunk_info; io_info->pieces_added++; } } /* end else */ @@ -3147,7 +3191,7 @@ H5D__chunk_write(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info) chunk_node = H5D_CHUNK_GET_NEXT_NODE(dset_info, chunk_node); } /* end while */ - /* Only perform I/O if not performing multi dataset I/O, otherwise the + /* Only perform I/O if not performing multi dataset I/O or type conversion, otherwise the * higher level will handle it after all datasets have been processed */ if (H5D_LAYOUT_CB_PERFORM_IO(io_info)) { /* Issue selection I/O call (we can skip the page buffer because we've @@ -3168,7 +3212,13 @@ H5D__chunk_write(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info) chunk_addrs = H5MM_xfree(chunk_addrs); } /* end if */ } /* end if */ - } /* end if */ + +#ifdef H5_HAVE_PARALLEL + /* Report that collective chunk I/O was used (will only be set on the DXPL if collective I/O was + * requested) */ + io_info->actual_io_mode |= H5D_MPIO_CHUNK_COLLECTIVE; +#endif /* H5_HAVE_PARALLEL */ + } /* end if */ else { /* Iterate through nodes in chunk skip list */ chunk_node = H5D_CHUNK_GET_FIRST_NODE(dset_info); diff --git a/src/H5Dcompact.c b/src/H5Dcompact.c index 9567c60..5f45da4 100644 --- a/src/H5Dcompact.c +++ b/src/H5Dcompact.c @@ -252,7 +252,8 @@ H5D__compact_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo) dinfo->store->compact.dirty = &dinfo->dset->shared->layout.storage.u.compact.dirty; /* Disable selection I/O */ - io_info->use_select_io = FALSE; + io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; + io_info->no_selection_io_cause |= H5D_SEL_IO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5D__compact_io_init() */ diff --git a/src/H5Dcontig.c b/src/H5Dcontig.c index 3eddfff..c8c7ee7 100644 --- a/src/H5Dcontig.c +++ b/src/H5Dcontig.c @@ -106,7 +106,7 @@ static herr_t H5D__contig_io_term(H5D_io_info_t *io_info, H5D_dset_io_info_t *d /* Helper routines */ static herr_t H5D__contig_write_one(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info, hsize_t offset, size_t size); -static htri_t H5D__contig_may_use_select_io(const H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset_info, +static herr_t H5D__contig_may_use_select_io(H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset_info, H5D_io_op_type_t op_type); /*********************/ @@ -586,8 +586,7 @@ H5D__contig_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo) int sf_ndims; /* The number of dimensions of the file dataspace (signed) */ - htri_t use_selection_io = FALSE; /* Whether to use selection I/O */ - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -666,6 +665,16 @@ H5D__contig_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo) /* get dset file address for piece */ new_piece_info->faddr = dinfo->dset->shared->layout.storage.u.contig.addr; + /* Initialize in-place type conversion info. Start with it disabled. */ + new_piece_info->in_place_tconv = FALSE; + new_piece_info->buf_off = 0; + + /* Calculate type conversion buffer size and check for in-place conversion if necessary. Currently + * only implemented for selection I/O. */ + if (io_info->use_select_io != H5D_SELECTION_IO_MODE_OFF && + !(dinfo->type_info.is_xform_noop && dinfo->type_info.is_conv_noop)) + H5D_INIT_PIECE_TCONV(io_info, dinfo, new_piece_info) + /* Save piece to dataset info struct so it is freed at the end of the * operation */ dinfo->layout_io_info.contig_piece_info = new_piece_info; @@ -676,11 +685,9 @@ H5D__contig_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo) /* Check if we're performing selection I/O if it hasn't been disabled * already */ - if (io_info->use_select_io) { - if ((use_selection_io = H5D__contig_may_use_select_io(io_info, dinfo, H5D_IO_OP_READ)) < 0) + if (io_info->use_select_io != H5D_SELECTION_IO_MODE_OFF) + if (H5D__contig_may_use_select_io(io_info, dinfo, io_info->op_type) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't check if selection I/O is possible") - io_info->use_select_io = (hbool_t)use_selection_io; - } done: if (ret_value < 0) { @@ -740,12 +747,12 @@ H5D__contig_mdio_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo) * *------------------------------------------------------------------------- */ -static htri_t -H5D__contig_may_use_select_io(const H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset_info, +static herr_t +H5D__contig_may_use_select_io(H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset_info, H5D_io_op_type_t op_type) { - const H5D_t *dataset = NULL; /* Local pointer to dataset info */ - htri_t ret_value = FAIL; /* Return value */ + const H5D_t *dataset = NULL; /* Local pointer to dataset info */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -757,27 +764,33 @@ H5D__contig_may_use_select_io(const H5D_io_info_t *io_info, const H5D_dset_io_in dataset = dset_info->dset; - /* Don't use selection I/O if it's globally disabled, if there is a type - * conversion, or if it's not a contiguous dataset, or if the sieve buffer - * exists (write) or is dirty (read) */ - if (dset_info->io_ops.single_read != H5D__select_read || - dset_info->layout_ops.readvv != H5D__contig_readvv || - (op_type == H5D_IO_OP_READ && dataset->shared->cache.contig.sieve_dirty) || - (op_type == H5D_IO_OP_WRITE && dataset->shared->cache.contig.sieve_buf)) - ret_value = FALSE; + /* None of the reasons this function might disable selection I/O are relevant to parallel, so no need to + * update no_selection_io_cause since we're only keeping track of the reason for no selection I/O in + * parallel (for now) */ + + /* Don't use selection I/O if it's globally disabled, if it's not a contiguous dataset, or if the sieve + * buffer exists (write) or is dirty (read) */ + if (dset_info->layout_ops.readvv != H5D__contig_readvv) { + io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; + io_info->no_selection_io_cause |= H5D_SEL_IO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; + } + else if ((op_type == H5D_IO_OP_READ && dataset->shared->cache.contig.sieve_dirty) || + (op_type == H5D_IO_OP_WRITE && dataset->shared->cache.contig.sieve_buf)) { + io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; + io_info->no_selection_io_cause |= H5D_SEL_IO_CONTIGUOUS_SIEVE_BUFFER; + } else { hbool_t page_buf_enabled; - HDassert(dset_info->io_ops.single_write == H5D__select_write); HDassert(dset_info->layout_ops.writevv == H5D__contig_writevv); /* Check if the page buffer is enabled */ if (H5PB_enabled(io_info->f_sh, H5FD_MEM_DRAW, &page_buf_enabled) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't check if page buffer is enabled") - if (page_buf_enabled) - ret_value = FALSE; - else - ret_value = TRUE; + if (page_buf_enabled) { + io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; + io_info->no_selection_io_cause |= H5D_SEL_IO_PAGE_BUFFER; + } } /* end else */ done: @@ -810,9 +823,9 @@ H5D__contig_read(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo) HDassert(dinfo->mem_space); HDassert(dinfo->file_space); - if (io_info->use_select_io) { - /* Only perform I/O if not performing multi dataset I/O with selection - * I/O, otherwise the higher level will handle it after all datasets + if (io_info->use_select_io == H5D_SELECTION_IO_MODE_ON) { + /* Only perform I/O if not performing multi dataset I/O or type conversion, + * otherwise the higher level will handle it after all datasets * have been processed */ if (H5D_LAYOUT_CB_PERFORM_IO(io_info)) { size_t dst_type_size = dinfo->type_info.dst_type_size; @@ -841,10 +854,17 @@ H5D__contig_read(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo) io_info->addrs[io_info->pieces_added] = dinfo->store->contig.dset_addr; io_info->element_sizes[io_info->pieces_added] = dinfo->type_info.src_type_size; io_info->rbufs[io_info->pieces_added] = dinfo->buf.vp; + if (io_info->sel_pieces) + io_info->sel_pieces[io_info->pieces_added] = dinfo->layout_io_info.contig_piece_info; io_info->pieces_added++; } } - } /* end if */ + +#ifdef H5_HAVE_PARALLEL + /* Report that collective contiguous I/O was used */ + io_info->actual_io_mode |= H5D_MPIO_CONTIGUOUS_COLLECTIVE; +#endif /* H5_HAVE_PARALLEL */ + } /* end if */ else /* Read data through legacy (non-selection I/O) pathway */ if ((dinfo->io_ops.single_read)(io_info, dinfo) < 0) @@ -880,9 +900,9 @@ H5D__contig_write(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo) HDassert(dinfo->mem_space); HDassert(dinfo->file_space); - if (io_info->use_select_io) { - /* Only perform I/O if not performing multi dataset I/O with selection - * I/O, otherwise the higher level will handle it after all datasets + if (io_info->use_select_io == H5D_SELECTION_IO_MODE_ON) { + /* Only perform I/O if not performing multi dataset I/O or type conversion, + * otherwise the higher level will handle it after all datasets * have been processed */ if (H5D_LAYOUT_CB_PERFORM_IO(io_info)) { size_t dst_type_size = dinfo->type_info.dst_type_size; @@ -911,10 +931,17 @@ H5D__contig_write(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo) io_info->addrs[io_info->pieces_added] = dinfo->store->contig.dset_addr; io_info->element_sizes[io_info->pieces_added] = dinfo->type_info.dst_type_size; io_info->wbufs[io_info->pieces_added] = dinfo->buf.cvp; + if (io_info->sel_pieces) + io_info->sel_pieces[io_info->pieces_added] = dinfo->layout_io_info.contig_piece_info; io_info->pieces_added++; } } - } /* end if */ + +#ifdef H5_HAVE_PARALLEL + /* Report that collective contiguous I/O was used */ + io_info->actual_io_mode |= H5D_MPIO_CONTIGUOUS_COLLECTIVE; +#endif /* H5_HAVE_PARALLEL */ + } /* end if */ else /* Write data through legacy (non-selection I/O) pathway */ if ((dinfo->io_ops.single_write)(io_info, dinfo) < 0) diff --git a/src/H5Defl.c b/src/H5Defl.c index faa5e4d..7509df7 100644 --- a/src/H5Defl.c +++ b/src/H5Defl.c @@ -213,7 +213,8 @@ H5D__efl_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo) H5MM_memcpy(&dinfo->store->efl, &(dinfo->dset->shared->dcpl_cache.efl), sizeof(H5O_efl_t)); /* Disable selection I/O */ - io_info->use_select_io = FALSE; + io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; + io_info->no_selection_io_cause |= H5D_SEL_IO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5D__efl_io_init() */ diff --git a/src/H5Dio.c b/src/H5Dio.c index 5ec0efe..f6f743c 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -43,14 +43,16 @@ /********************/ /* Setup/teardown routines */ -static herr_t H5D__ioinfo_init(size_t count, H5D_dset_io_info_t *dset_info, H5D_io_info_t *io_info); +static herr_t H5D__ioinfo_init(size_t count, H5D_io_op_type_t op_type, H5D_dset_io_info_t *dset_info, + H5D_io_info_t *io_info); static herr_t H5D__dset_ioinfo_init(H5D_t *dset, H5D_dset_io_info_t *dset_info, H5D_storage_t *store); static herr_t H5D__typeinfo_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info, hid_t mem_type_id); static herr_t H5D__typeinfo_init_phase2(H5D_io_info_t *io_info); +static herr_t H5D__typeinfo_init_phase3(H5D_io_info_t *io_info); #ifdef H5_HAVE_PARALLEL static herr_t H5D__ioinfo_adjust(H5D_io_info_t *io_info); #endif /* H5_HAVE_PARALLEL */ -static herr_t H5D__typeinfo_term(H5D_io_info_t *io_info, size_t type_info_init); +static herr_t H5D__typeinfo_term(H5D_io_info_t *io_info); /*********************/ /* Package Variables */ @@ -77,7 +79,6 @@ herr_t H5D__read(size_t count, H5D_dset_io_info_t *dset_info) { H5D_io_info_t io_info; /* Dataset I/O info for multi dsets */ - size_t type_info_init = 0; /* Number of datatype info structs that have been initialized */ H5S_t *orig_mem_space_local; /* Local buffer for orig_mem_space */ H5S_t **orig_mem_space = NULL; /* If not NULL, ptr to an array of dataspaces */ /* containing the original memory spaces contained */ @@ -107,9 +108,8 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info) FUNC_ENTER_NOAPI(FAIL) /* Init io_info */ - if (H5D__ioinfo_init(count, dset_info, &io_info) < 0) + if (H5D__ioinfo_init(count, H5D_IO_OP_READ, dset_info, &io_info) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize I/O info") - io_info.op_type = H5D_IO_OP_READ; /* Allocate store buffer if necessary */ if (count > 1) @@ -148,7 +148,6 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info) /* Set up datatype info for operation */ if (H5D__typeinfo_init(&io_info, &(dset_info[i]), dset_info[i].mem_type_id) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to set up type info") - type_info_init++; /* Make certain that the number of elements in each selection is the same, and cache nelmts in * dset_info */ @@ -285,7 +284,6 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info) } } /* end of for loop */ - HDassert(type_info_init == count); HDassert(io_op_init + io_skipped == count); /* If no datasets have I/O, we're done */ @@ -297,10 +295,17 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to set up type info (second phase)") #ifdef H5_HAVE_PARALLEL - /* Adjust I/O info for any parallel I/O */ + /* Adjust I/O info for any parallel or selection I/O */ if (H5D__ioinfo_adjust(&io_info) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to adjust I/O info for parallel I/O") -#endif /*H5_HAVE_PARALLEL*/ + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "unable to adjust I/O info for parallel or selection I/O") +#endif /* H5_HAVE_PARALLEL */ + + /* Perform third phase of type info initialization */ + if (H5D__typeinfo_init_phase3(&io_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to set up type info (third phase)") + + H5CX_set_no_selection_io_cause(io_info.no_selection_io_cause); /* If multi dataset I/O callback is not provided, perform read IO via * single-dset path with looping */ @@ -339,6 +344,7 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info) else { haddr_t prev_tag = HADDR_UNDEF; + /* Allocate selection I/O parameter arrays if necessary */ if (!H5D_LAYOUT_CB_PERFORM_IO(&io_info) && io_info.piece_count > 0) { if (NULL == (io_info.mem_spaces = H5MM_malloc(io_info.piece_count * sizeof(H5S_t *)))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, @@ -354,6 +360,11 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info) if (NULL == (io_info.rbufs = H5MM_malloc(io_info.piece_count * sizeof(void *)))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "memory allocation failed for read buffer list") + if (io_info.max_tconv_type_size > 0) + if (NULL == + (io_info.sel_pieces = H5MM_malloc(io_info.piece_count * sizeof(io_info.sel_pieces[0])))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "unable to allocate array of selected pieces") } /* Loop with serial & single-dset read IO path */ @@ -373,15 +384,39 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info) H5AC_tag(prev_tag, NULL); } - /* Make final multi dataset selection I/O call if we are using both - * features - in this case the multi_read callbacks did not perform the - * actual I/O */ - H5_CHECK_OVERFLOW(io_info.pieces_added, size_t, uint32_t) - if (!H5D_LAYOUT_CB_PERFORM_IO(&io_info)) - if (H5F_shared_select_read(io_info.f_sh, H5FD_MEM_DRAW, (uint32_t)io_info.pieces_added, - io_info.mem_spaces, io_info.file_spaces, io_info.addrs, - io_info.element_sizes, io_info.rbufs) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "selection read failed") + /* Make final selection I/O call if the multi_read callbacks did not perform the actual I/O + * (if using selection I/O and either multi dataset or type conversion) */ + if (!H5D_LAYOUT_CB_PERFORM_IO(&io_info)) { + /* Check for type conversion */ + if (io_info.max_tconv_type_size > 0) { + /* Type conversion pathway */ + if (H5D__scatgath_read_select(&io_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "type conversion selection read failed") + } + else { + /* Call selection I/O directly */ + H5_CHECK_OVERFLOW(io_info.pieces_added, size_t, uint32_t) + if (H5F_shared_select_read(io_info.f_sh, H5FD_MEM_DRAW, (uint32_t)io_info.pieces_added, + io_info.mem_spaces, io_info.file_spaces, io_info.addrs, + io_info.element_sizes, io_info.rbufs) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "selection read failed") + } + } + +#ifdef H5_HAVE_PARALLEL + /* Report the actual I/O mode to the application if appropriate */ + if (io_info.using_mpi_vfd) { + H5FD_mpio_xfer_t xfer_mode; /* Parallel transfer for this request */ + + /* Get the parallel I/O transfer mode */ + if (H5CX_get_io_xfer_mode(&xfer_mode) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get MPI-I/O transfer mode") + + /* Only report the collective I/O mode if we're actually performing collective I/O */ + if (xfer_mode == H5FD_MPIO_COLLECTIVE) + H5CX_set_mpio_actual_io_mode(io_info.actual_io_mode); + } +#endif /* H5_HAVE_PARALLEL */ } done: @@ -392,7 +427,7 @@ done: HDONE_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to shut down I/O op info") /* Shut down datatype info for operation */ - if (H5D__typeinfo_term(&io_info, type_info_init) < 0) + if (H5D__typeinfo_term(&io_info) < 0) HDONE_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to shut down type info") /* Discard projected mem spaces and restore originals */ @@ -443,7 +478,6 @@ herr_t H5D__write(size_t count, H5D_dset_io_info_t *dset_info) { H5D_io_info_t io_info; /* Dataset I/O info for multi dsets */ - size_t type_info_init = 0; /* Number of datatype info structs that have been initialized */ H5S_t *orig_mem_space_local; /* Local buffer for orig_mem_space */ H5S_t **orig_mem_space = NULL; /* If not NULL, ptr to an array of dataspaces */ /* containing the original memory spaces contained */ @@ -471,9 +505,8 @@ H5D__write(size_t count, H5D_dset_io_info_t *dset_info) FUNC_ENTER_NOAPI(FAIL) /* Init io_info */ - if (H5D__ioinfo_init(count, dset_info, &io_info) < 0) + if (H5D__ioinfo_init(count, H5D_IO_OP_WRITE, dset_info, &io_info) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize I/O info") - io_info.op_type = H5D_IO_OP_WRITE; /* Allocate store buffer if necessary */ if (count > 1) @@ -509,9 +542,8 @@ H5D__write(size_t count, H5D_dset_io_info_t *dset_info) /* Set up datatype info for operation */ if (H5D__typeinfo_init(&io_info, &(dset_info[i]), dset_info[i].mem_type_id) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to set up type info") - type_info_init++; - /* Various MPI based checks */ + /* Various MPI based checks */ #ifdef H5_HAVE_PARALLEL if (H5F_HAS_FEATURE(dset_info[i].dset->oloc.file, H5FD_FEAT_HAS_MPI)) { /* If MPI based VFD is used, no VL or region reference datatype support yet. */ @@ -660,7 +692,6 @@ H5D__write(size_t count, H5D_dset_io_info_t *dset_info) H5AC_tag(prev_tag, NULL); } /* end of for loop */ - HDassert(type_info_init == count); HDassert(io_op_init == count); /* Perform second phase of type info initialization */ @@ -668,10 +699,17 @@ H5D__write(size_t count, H5D_dset_io_info_t *dset_info) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to set up type info (second phase)") #ifdef H5_HAVE_PARALLEL - /* Adjust I/O info for any parallel I/O */ + /* Adjust I/O info for any parallel or selection I/O */ if (H5D__ioinfo_adjust(&io_info) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to adjust I/O info for parallel I/O") -#endif /*H5_HAVE_PARALLEL*/ + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "unable to adjust I/O info for parallel or selection I/O") +#endif /* H5_HAVE_PARALLEL */ + + /* Perform third phase of type info initialization */ + if (H5D__typeinfo_init_phase3(&io_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to set up type info (third phase)") + + H5CX_set_no_selection_io_cause(io_info.no_selection_io_cause); /* If multi dataset I/O callback is not provided, perform write IO via * single-dset path with looping */ @@ -710,6 +748,7 @@ H5D__write(size_t count, H5D_dset_io_info_t *dset_info) else { haddr_t prev_tag = HADDR_UNDEF; + /* Allocate selection I/O parameter arrays if necessary */ if (!H5D_LAYOUT_CB_PERFORM_IO(&io_info) && io_info.piece_count > 0) { if (NULL == (io_info.mem_spaces = H5MM_malloc(io_info.piece_count * sizeof(H5S_t *)))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, @@ -724,7 +763,12 @@ H5D__write(size_t count, H5D_dset_io_info_t *dset_info) "memory allocation failed for element size list") if (NULL == (io_info.wbufs = H5MM_malloc(io_info.piece_count * sizeof(const void *)))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, - "memory allocation failed for read buffer list") + "memory allocation failed for write buffer list") + if (io_info.max_tconv_type_size > 0) + if (NULL == + (io_info.sel_pieces = H5MM_malloc(io_info.piece_count * sizeof(io_info.sel_pieces[0])))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "unable to allocate array of selected pieces") } /* loop with serial & single-dset write IO path */ @@ -742,15 +786,45 @@ H5D__write(size_t count, H5D_dset_io_info_t *dset_info) H5AC_tag(prev_tag, NULL); } - /* Make final multi dataset selection I/O call if we are using both - * features - in this case the multi_write callbacks did not perform the - * actual I/O */ - H5_CHECK_OVERFLOW(io_info.pieces_added, size_t, uint32_t) - if (!H5D_LAYOUT_CB_PERFORM_IO(&io_info)) - if (H5F_shared_select_write(io_info.f_sh, H5FD_MEM_DRAW, (uint32_t)io_info.pieces_added, - io_info.mem_spaces, io_info.file_spaces, io_info.addrs, - io_info.element_sizes, io_info.wbufs) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "selection write failed") + /* Make final selection I/O call if the multi_write callbacks did not perform the actual I/O + * (if using selection I/O and either multi dataset or type conversion) */ + if (!H5D_LAYOUT_CB_PERFORM_IO(&io_info)) { + /* Check for type conversion */ + if (io_info.max_tconv_type_size > 0) { + /* Type conversion pathway */ + if (H5D__scatgath_write_select(&io_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "type conversion selection write failed") + } + else { + /* Call selection I/O directly */ + H5_CHECK_OVERFLOW(io_info.pieces_added, size_t, uint32_t) + if (H5F_shared_select_write(io_info.f_sh, H5FD_MEM_DRAW, (uint32_t)io_info.pieces_added, + io_info.mem_spaces, io_info.file_spaces, io_info.addrs, + io_info.element_sizes, io_info.wbufs) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "selection write failed") + } + } + +#ifdef H5_HAVE_PARALLEL + /* Report the actual I/O mode to the application if appropriate */ + if (io_info.using_mpi_vfd) { + H5FD_mpio_xfer_t xfer_mode; /* Parallel transfer for this request */ + + /* Get the parallel I/O transfer mode */ + if (H5CX_get_io_xfer_mode(&xfer_mode) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get MPI-I/O transfer mode") + + /* Only report the collective I/O mode if we're actually performing collective I/O */ + if (xfer_mode == H5FD_MPIO_COLLECTIVE) { + H5CX_set_mpio_actual_io_mode(io_info.actual_io_mode); + + /* If we did selection I/O, report that we used "link chunk" mode, since that's the most + * analogous to what selection I/O does */ + if (io_info.use_select_io == H5D_SELECTION_IO_MODE_ON) + H5CX_set_mpio_actual_chunk_opt(H5D_MPIO_LINK_CHUNK); + } + } +#endif /* H5_HAVE_PARALLEL */ } #ifdef OLD_WAY @@ -781,7 +855,7 @@ done: } /* Shut down datatype info for operation */ - if (H5D__typeinfo_term(&io_info, type_info_init) < 0) + if (H5D__typeinfo_term(&io_info) < 0) HDONE_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to shut down type info") /* Discard projected mem spaces and restore originals */ @@ -819,8 +893,11 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5D__ioinfo_init(size_t count, H5D_dset_io_info_t *dset_info, H5D_io_info_t *io_info) +H5D__ioinfo_init(size_t count, H5D_io_op_type_t op_type, H5D_dset_io_info_t *dset_info, + H5D_io_info_t *io_info) { + H5D_selection_io_mode_t selection_io_mode; + FUNC_ENTER_PACKAGE_NOERR /* check args */ @@ -833,8 +910,9 @@ H5D__ioinfo_init(size_t count, H5D_dset_io_info_t *dset_info, H5D_io_info_t *io_ HDmemset(io_info, 0, sizeof(*io_info)); /* Set up simple fields */ - io_info->f_sh = count > 0 ? H5F_SHARED(dset_info[0].dset->oloc.file) : NULL; - io_info->count = count; + io_info->op_type = op_type; + io_info->f_sh = count > 0 ? H5F_SHARED(dset_info[0].dset->oloc.file) : NULL; + io_info->count = count; /* Start without multi-dataset I/O ops. If we're not using the collective * I/O path then we will call the single dataset callbacks in a loop. */ @@ -842,16 +920,31 @@ H5D__ioinfo_init(size_t count, H5D_dset_io_info_t *dset_info, H5D_io_info_t *io_ /* Use provided dset_info */ io_info->dsets_info = dset_info; - /* Start with selection I/O on if the global is on, layout callback will - * turn it off if appropriate */ - io_info->use_select_io = H5_use_selection_io_g; + /* Start with selection I/O mode from property list. If enabled, layout callback will turn it off if it + * is not supported by the layout. Handling of H5D_SELECTION_IO_MODE_AUTO occurs in H5D__ioinfo_adjust. + */ + H5CX_get_selection_io_mode(&selection_io_mode); + io_info->use_select_io = selection_io_mode; + + /* Record no selection I/O cause if it was disabled by the API */ + if (selection_io_mode == H5D_SELECTION_IO_MODE_OFF) + io_info->no_selection_io_cause = H5D_SEL_IO_DISABLE_BY_API; #ifdef H5_HAVE_PARALLEL + /* Determine if the file was opened with an MPI VFD */ if (count > 0) io_info->using_mpi_vfd = H5F_HAS_FEATURE(dset_info[0].dset->oloc.file, H5FD_FEAT_HAS_MPI); #endif /* H5_HAVE_PARALLEL */ + /* Check if we could potentially use in-place type conversion */ + if (op_type == H5D_IO_OP_READ) + /* Always on for read (modulo other restrictions that are handled in layout callbacks) */ + io_info->may_use_in_place_tconv = TRUE; + else + /* Only enable in-place type conversion if we're allowed to modify the write buffer */ + H5CX_get_modify_write_buf(&io_info->may_use_in_place_tconv); + FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5D__ioinfo_init() */ @@ -998,9 +1091,9 @@ H5D__typeinfo_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info, hid_t /* Check if the datatypes are compound subsets of one another */ type_info->cmpd_subset = H5T_path_compound_subset(type_info->tpath); - /* Update io_info->max_type_size */ - io_info->max_type_size = - MAX3(io_info->max_type_size, type_info->src_type_size, type_info->dst_type_size); + /* Update io_info->max_tconv_type_size */ + io_info->max_tconv_type_size = + MAX3(io_info->max_tconv_type_size, type_info->src_type_size, type_info->dst_type_size); /* Check if we need a background buffer */ if ((io_info->op_type == H5D_IO_OP_WRITE) && H5T_detect_class(dset->shared->type, H5T_VLEN, FALSE)) @@ -1025,8 +1118,12 @@ done: /*------------------------------------------------------------------------- * Function: H5D__typeinfo_init_phase2 * - * Purpose: Finish initializing type info for all datasets after - * calculating the max type size across all datasets. + * Purpose: Continue initializing type info for all datasets after + * calculating the max type size across all datasets, and + * before final determination of collective/independent in + * H5D__ioinfo_adjust(). Currently just checks to see if + * selection I/O can be used with type conversion, and sets + * no_collective_cause flags related to selection I/O. * * Return: Non-negative on success/Negative on failure * @@ -1042,89 +1139,65 @@ H5D__typeinfo_init_phase2(H5D_io_info_t *io_info) /* check args */ HDassert(io_info); - /* Check if we need to allocate a shared type conversion buffer */ - if (io_info->max_type_size) { - void *tconv_buf; /* Temporary conversion buffer pointer */ - void *bkgr_buf; /* Background conversion buffer pointer */ + /* If selection I/O mode is default (auto), enable it here if the VFD supports it (it will be turned off + * later if something else conflicts), otherwise disable it. If we're using the MPIO VFD, the automatic + * selection will happen in H5D__mpio_opt_possible() inside H5D__ioinfo_adjust(). */ +#ifdef H5_HAVE_PARALLEL + if (!io_info->using_mpi_vfd) +#endif /* H5_HAVE_PARALLEL */ + if (io_info->use_select_io == H5D_SELECTION_IO_MODE_DEFAULT) { + if (H5F_has_vector_select_io(io_info->dsets_info[0].dset->oloc.file, + io_info->op_type == H5D_IO_OP_WRITE)) + io_info->use_select_io = H5D_SELECTION_IO_MODE_ON; + else { + io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; + io_info->no_selection_io_cause |= H5D_SEL_IO_DEFAULT_OFF; + } + } + + /* If we're doing type conversion and we might be doing selection I/O, check if the buffers are large + * enough to handle the whole I/O */ + if (io_info->max_tconv_type_size && io_info->use_select_io != H5D_SELECTION_IO_MODE_OFF) { size_t max_temp_buf; /* Maximum temporary buffer size */ - size_t target_size; /* Desired buffer size */ size_t i; /* Local index variable */ - /* Get info from API context */ - if (H5CX_get_max_temp_buf(&max_temp_buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't retrieve max. temp. buf size") - if (H5CX_get_tconv_buf(&tconv_buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't retrieve temp. conversion buffer pointer") - if (H5CX_get_bkgr_buf(&bkgr_buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't retrieve background conversion buffer pointer") - - /* Set up datatype conversion/background buffers */ - target_size = max_temp_buf; - - /* If the buffer is too small to hold even one element (in the dataset with the largest , try to make - * it bigger */ - if (target_size < io_info->max_type_size) { - hbool_t default_buffer_info; /* Whether the buffer information are the defaults */ - - /* Detect if we have all default settings for buffers */ - default_buffer_info = - (hbool_t)((H5D_TEMP_BUF_SIZE == max_temp_buf) && (NULL == tconv_buf) && (NULL == bkgr_buf)); + /* Collective I/O, conversion buffer must be large enough for entire I/O (for now) */ - /* Check if we are using the default buffer info */ - if (default_buffer_info) - /* OK to get bigger for library default settings */ - target_size = io_info->max_type_size; - else - /* Don't get bigger than the application has requested */ - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "temporary buffer max size is too small") - } /* end if */ - - /* Get a temporary buffer for type conversion unless the app has already - * supplied one through the xfer properties. Instead of allocating a - * buffer which is the exact size, we allocate the target size. This - * buffer is shared among all datasets in the operation, unlike for the - * background buffer, where each dataset gets its own. */ - if (NULL == (io_info->tconv_buf = (uint8_t *)tconv_buf)) { - /* Allocate temporary buffer */ - if (NULL == (io_info->tconv_buf = H5FL_BLK_MALLOC(type_conv, target_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for type conversion") - io_info->tconv_buf_allocated = TRUE; - } /* end if */ - - /* Don't use API provided background buffer if there's more than one dataset, since each - * dataset needs its own */ - if (io_info->count > 1) - bkgr_buf = NULL; - - /* Iterate over datasets */ + /* Calculate size of background buffer (tconv buf size was calculated in layout io_init callbacks) + */ for (i = 0; i < io_info->count; i++) { H5D_type_info_t *type_info = &io_info->dsets_info[i].type_info; - /* Compute the number of elements that will fit into buffer */ - type_info->request_nelmts = target_size / MAX(type_info->src_type_size, type_info->dst_type_size); - ; + /* Check for background buffer */ + if (type_info->need_bkg) { + /* Add size of this dataset's background buffer to the global background buffer size + */ + io_info->bkg_buf_size += io_info->dsets_info[i].nelmts * type_info->dst_type_size; - /* Sanity check elements in temporary buffer */ - if (type_info->request_nelmts == 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "temporary buffer max size is too small") - - /* Allocate background buffer if necessary */ - if (type_info->need_bkg && NULL == (type_info->bkg_buf = (uint8_t *)bkgr_buf)) { - size_t bkg_size; /* Desired background buffer size */ + /* Check if we need to fill the background buffer with the destination contents */ + if (type_info->need_bkg == H5T_BKG_YES) + io_info->must_fill_bkg = TRUE; + } + } - /* Compute the background buffer size */ - /* (don't try to use buffers smaller than the default size) */ - bkg_size = type_info->request_nelmts * type_info->dst_type_size; - if (bkg_size < max_temp_buf) - bkg_size = max_temp_buf; + /* Get max temp buffer size from API context */ + if (H5CX_get_max_temp_buf(&max_temp_buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't retrieve max. temp. buf size") - /* Allocate background buffer */ - /* (Need calloc()-like call since memory needs to be initialized) */ - if (NULL == (type_info->bkg_buf = H5FL_BLK_CALLOC(type_conv, bkg_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, - "memory allocation failed for background conversion") - type_info->bkg_buf_allocated = TRUE; - } /* end if */ + /* Check if the needed type conversion or background buffer size is too big */ + if (io_info->tconv_buf_size > max_temp_buf) { + io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; + io_info->no_selection_io_cause |= H5D_SEL_IO_TCONV_BUF_TOO_SMALL; + io_info->tconv_buf_size = 0; + io_info->bkg_buf_size = 0; + io_info->must_fill_bkg = FALSE; + } + if (io_info->bkg_buf_size > max_temp_buf) { + io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; + io_info->no_selection_io_cause |= H5D_SEL_IO_BKG_BUF_TOO_SMALL; + io_info->tconv_buf_size = 0; + io_info->bkg_buf_size = 0; + io_info->must_fill_bkg = FALSE; } } @@ -1133,15 +1206,13 @@ done: } /* end H5D__typeinfo_init_phase2() */ #ifdef H5_HAVE_PARALLEL - /*------------------------------------------------------------------------- - * Function: H5D__ioinfo_adjust - * - * Purpose: Adjust operation's I/O info for any parallel I/O + * Function: H5D__ioinfo_adjust * - * This was derived from H5D__ioinfo_adjust for multi-dset work. + * Purpose: Adjust operation's I/O info for any parallel I/O, also + * handle decision on selection I/O even in serial case * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -1193,7 +1264,7 @@ H5D__ioinfo_adjust(H5D_io_info_t *io_info) * selection I/O is to be used - in this case the file driver will * handle collective I/O */ /* Check for selection/vector support in file driver? -NAF */ - if (!io_info->use_select_io) { + if (io_info->use_select_io == H5D_SELECTION_IO_MODE_OFF) { io_info->md_io_ops.multi_read_md = H5D__collective_read; io_info->md_io_ops.multi_write_md = H5D__collective_write; io_info->md_io_ops.single_read_md = H5D__mpio_select_read; @@ -1270,6 +1341,137 @@ done: #endif /* H5_HAVE_PARALLEL */ /*------------------------------------------------------------------------- + * Function: H5D__typeinfo_init_phase3 + * + * Purpose: Finish initializing type info for all datasets after + * calculating the max type size across all datasets. And + * after final collective/independent determination in + * H5D__ioinfo_adjust(). + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__typeinfo_init_phase3(H5D_io_info_t *io_info) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* check args */ + HDassert(io_info); + + /* Check if we need to allocate a shared type conversion buffer */ + if (io_info->max_tconv_type_size) { + void *tconv_buf; /* Temporary conversion buffer pointer */ + void *bkgr_buf; /* Background conversion buffer pointer */ + + /* Get provided buffers from API context */ + if (H5CX_get_tconv_buf(&tconv_buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't retrieve temp. conversion buffer pointer") + if (H5CX_get_bkgr_buf(&bkgr_buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't retrieve background conversion buffer pointer") + + /* Check if we're doing selection I/O */ + if (io_info->use_select_io == H5D_SELECTION_IO_MODE_ON) { + /* Selection I/O, conversion buffers must be large enough for entire I/O (for now) */ + + /* Allocate global type conversion buffer (if any, could be none if datasets in this + * I/O have 0 elements selected) */ + /* Allocating large buffers here will blow out all other type conversion buffers + * on the free list. Should we change this to a regular malloc? Would require + * keeping track of which version of free() to call. -NAF */ + if (io_info->tconv_buf_size > 0) { + if (NULL == (io_info->tconv_buf = H5FL_BLK_MALLOC(type_conv, io_info->tconv_buf_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, + "memory allocation failed for type conversion") + io_info->tconv_buf_allocated = TRUE; + } + + /* Allocate global background buffer (if any) */ + if (io_info->bkg_buf_size > 0) { + if (NULL == (io_info->bkg_buf = H5FL_BLK_MALLOC(type_conv, io_info->bkg_buf_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, + "memory allocation failed for type conversion") + io_info->bkg_buf_allocated = TRUE; + } + } + else { + /* No selection I/O, only need to make sure it's big enough for one element */ + size_t max_temp_buf; /* Maximum temporary buffer size */ + size_t target_size; /* Desired buffer size */ + size_t i; + + /* Make sure selection I/O is disabled (DEFAULT should have been handled by now) */ + HDassert(io_info->use_select_io == H5D_SELECTION_IO_MODE_OFF); + + /* Get max buffer size from API context */ + if (H5CX_get_max_temp_buf(&max_temp_buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't retrieve max. temp. buf size") + + /* Set up datatype conversion/background buffers */ + target_size = max_temp_buf; + + /* If the buffer is too small to hold even one element (in the dataset with the largest , try to + * make it bigger */ + if (target_size < io_info->max_tconv_type_size) { + hbool_t default_buffer_info; /* Whether the buffer information are the defaults */ + + /* Detect if we have all default settings for buffers */ + default_buffer_info = (hbool_t)((H5D_TEMP_BUF_SIZE == max_temp_buf) && (NULL == tconv_buf) && + (NULL == bkgr_buf)); + + /* Check if we are using the default buffer info */ + if (default_buffer_info) + /* OK to get bigger for library default settings */ + target_size = io_info->max_tconv_type_size; + else + /* Don't get bigger than the application has requested */ + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "temporary buffer max size is too small") + } /* end if */ + + /* Get a temporary buffer for type conversion unless the app has already + * supplied one through the xfer properties. Instead of allocating a + * buffer which is the exact size, we allocate the target size. This + * buffer is shared among all datasets in the operation. */ + if (NULL == (io_info->tconv_buf = (uint8_t *)tconv_buf)) { + /* Allocate temporary buffer */ + if (NULL == (io_info->tconv_buf = H5FL_BLK_MALLOC(type_conv, target_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, + "memory allocation failed for type conversion") + io_info->tconv_buf_allocated = TRUE; + } /* end if */ + + /* Iterate over datasets */ + for (i = 0; i < io_info->count; i++) { + H5D_type_info_t *type_info = &io_info->dsets_info[i].type_info; + + /* Compute the number of elements that will fit into buffer */ + type_info->request_nelmts = + target_size / MAX(type_info->src_type_size, type_info->dst_type_size); + + /* Check if we need a background buffer and one hasn't been allocated yet */ + if (type_info->need_bkg && (NULL == io_info->bkg_buf) && + (NULL == (io_info->bkg_buf = (uint8_t *)bkgr_buf))) { + /* Allocate background buffer with the same size as the type conversion buffer. We can do + * this since the number of elements that fit in the type conversion buffer will never be + * larger than the number that could fit in a background buffer of equal size, since the + * tconv element size is max(src, dst) and the bkg element size is dst */ + if (NULL == (io_info->bkg_buf = H5FL_BLK_MALLOC(type_conv, target_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, + "memory allocation failed for background conversion") + io_info->bkg_buf_allocated = TRUE; + } + } + } + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__typeinfo_init_phase3() */ + +/*------------------------------------------------------------------------- * Function: H5D__typeinfo_term * * Purpose: Common logic for terminating a type info object @@ -1282,10 +1484,8 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5D__typeinfo_term(H5D_io_info_t *io_info, size_t type_info_init) +H5D__typeinfo_term(H5D_io_info_t *io_info) { - size_t i; - FUNC_ENTER_PACKAGE_NOERR /* Check for releasing datatype conversion & background buffers */ @@ -1293,11 +1493,10 @@ H5D__typeinfo_term(H5D_io_info_t *io_info, size_t type_info_init) HDassert(io_info->tconv_buf); (void)H5FL_BLK_FREE(type_conv, io_info->tconv_buf); } /* end if */ - for (i = 0; i < type_info_init; i++) - if (io_info->dsets_info[i].type_info.bkg_buf_allocated) { - HDassert(io_info->dsets_info[i].type_info.bkg_buf); - (void)H5FL_BLK_FREE(type_conv, io_info->dsets_info[i].type_info.bkg_buf); - } /* end if */ + if (io_info->bkg_buf_allocated) { + HDassert(io_info->bkg_buf); + (void)H5FL_BLK_FREE(type_conv, io_info->bkg_buf); + } /* end if */ FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5D__typeinfo_term() */ diff --git a/src/H5Dmpio.c b/src/H5Dmpio.c index 8e413c1..77edfc4 100644 --- a/src/H5Dmpio.c +++ b/src/H5Dmpio.c @@ -626,13 +626,17 @@ H5D__mpio_opt_possible(H5D_io_info_t *io_info) if (!H5FD_mpi_opt_types_g) local_cause[0] |= H5D_MPIO_MPI_OPT_TYPES_ENV_VAR_DISABLED; - /* Don't allow collective operations if datatype conversions need to happen */ - if (!type_info->is_conv_noop) - local_cause[0] |= H5D_MPIO_DATATYPE_CONVERSION; - - /* Don't allow collective operations if data transform operations should occur */ - if (!type_info->is_xform_noop) - local_cause[0] |= H5D_MPIO_DATA_TRANSFORMS; + /* Datatype conversions and transformations are allowed with selection I/O. If the selection I/O mode + * is auto (default), disable collective for now and re-enable later if we can */ + if (io_info->use_select_io != H5D_SELECTION_IO_MODE_ON) { + /* Don't allow collective operations if datatype conversions need to happen */ + if (!type_info->is_conv_noop) + local_cause[0] |= H5D_MPIO_DATATYPE_CONVERSION; + + /* Don't allow collective operations if data transform operations should occur */ + if (!type_info->is_xform_noop) + local_cause[0] |= H5D_MPIO_DATA_TRANSFORMS; + } /* Check whether these are both simple or scalar dataspaces */ if (!((H5S_SIMPLE == H5S_GET_EXTENT_TYPE(mem_space) || @@ -662,6 +666,15 @@ H5D__mpio_opt_possible(H5D_io_info_t *io_info) local_cause[0] |= H5D_MPIO_PARALLEL_FILTERED_WRITES_DISABLED; #endif + /* Check if we would be able to perform collective if we could use selection I/O. If so add reasons + * for not using selection I/O to local_cause[0] */ + if ((io_info->use_select_io == H5D_SELECTION_IO_MODE_OFF) && local_cause[0] && + !(local_cause[0] & + ~((unsigned)H5D_MPIO_DATATYPE_CONVERSION | (unsigned)H5D_MPIO_DATA_TRANSFORMS))) { + HDassert(io_info->no_selection_io_cause & H5D_MPIO_NO_SELECTION_IO_CAUSES); + local_cause[0] |= H5D_MPIO_NO_SELECTION_IO; + } + /* Check if we are able to do a MPI_Bcast of the data from one rank * instead of having all the processes involved in the collective I/O call. */ @@ -722,6 +735,25 @@ H5D__mpio_opt_possible(H5D_io_info_t *io_info) HMPI_GOTO_ERROR(FAIL, "MPI_Allreduce failed", mpi_code) } /* end else */ + /* If the selection I/O mode is default (auto), decide here whether it should be on or off */ + if (io_info->use_select_io == H5D_SELECTION_IO_MODE_DEFAULT) { + /* If the only reason(s) we've disabled collective are type conversions and/or transforms, enable + * selection I/O and re-enable collective I/O since it's supported by selection I/O */ + if (global_cause[0] && !(global_cause[0] & ~((unsigned)H5D_MPIO_DATATYPE_CONVERSION | + (unsigned)H5D_MPIO_DATA_TRANSFORMS))) { + HDassert(!(local_cause[0] & + ~((unsigned)H5D_MPIO_DATATYPE_CONVERSION | (unsigned)H5D_MPIO_DATA_TRANSFORMS))); + local_cause[0] = 0; + global_cause[0] = 0; + io_info->use_select_io = H5D_SELECTION_IO_MODE_ON; + } + else { + /* Otherwise, there's currently no benefit to selection I/O, so leave it off */ + io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; + io_info->no_selection_io_cause |= H5D_SEL_IO_DEFAULT_OFF; + } + } + /* Set the local & global values of no-collective-cause in the API context */ H5CX_set_mpio_local_no_coll_cause(local_cause[0]); H5CX_set_mpio_global_no_coll_cause(global_cause[0]); @@ -774,7 +806,7 @@ H5D__mpio_get_no_coll_cause_strings(char *local_cause, size_t local_cause_len, c * Use compile-time assertion so this routine is updated * when any new "no collective cause" values are added */ - HDcompile_assert(H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE == (H5D_mpio_no_collective_cause_t)256); + HDcompile_assert(H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE == (H5D_mpio_no_collective_cause_t)0x200); /* Initialize output buffers */ if (local_cause) @@ -827,6 +859,11 @@ H5D__mpio_get_no_coll_cause_strings(char *local_cause, size_t local_cause_len, c case H5D_MPIO_ERROR_WHILE_CHECKING_COLLECTIVE_POSSIBLE: cause_str = "an error occurred while checking if collective I/O was possible"; break; + case H5D_MPIO_NO_SELECTION_IO: + cause_str = "collective I/O may be supported by selection or vector I/O but that feature was " + "not possible (see causes via H5Pget_no_selection_io_cause())"; + break; + case H5D_MPIO_COLLECTIVE: case H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE: default: @@ -1438,13 +1475,8 @@ H5D__link_piece_collective_io(H5D_io_info_t *io_info, int H5_ATTR_UNUSED mpi_ran HDassert(io_info->dsets_info[i].dset->shared->dcpl_cache.pline.nused == 0); if (io_info->dsets_info[i].layout->type == H5D_CHUNKED) actual_io_mode |= H5D_MPIO_CHUNK_COLLECTIVE; - else if (io_info->dsets_info[i].layout->type == H5D_CONTIGUOUS) { + else if (io_info->dsets_info[i].layout->type == H5D_CONTIGUOUS) actual_io_mode |= H5D_MPIO_CONTIGUOUS_COLLECTIVE; - - /* if only single-dset */ - if (1 == io_info->count) - actual_chunk_opt_mode = H5D_MPIO_NO_CHUNK_OPTIMIZATION; - } else HGOTO_ERROR(H5E_IO, H5E_UNSUPPORTED, FAIL, "unsupported storage layout") } diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 0906028..4a23fe7 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -67,7 +67,49 @@ #define H5D_BT2_MERGE_PERC 40 /* Macro to determine if the layout I/O callback should perform I/O */ -#define H5D_LAYOUT_CB_PERFORM_IO(IO_INFO) (!(IO_INFO)->use_select_io || (IO_INFO)->count == 1) +#define H5D_LAYOUT_CB_PERFORM_IO(IO_INFO) \ + (((IO_INFO)->use_select_io == H5D_SELECTION_IO_MODE_OFF) || \ + ((IO_INFO)->count == 1 && (IO_INFO)->max_tconv_type_size == 0)) + +/* Macro to check if in-place type conversion will be used for a piece and add it to the global type + * conversion size if it won't be used */ +#define H5D_INIT_PIECE_TCONV(IO_INFO, DINFO, PIECE_INFO) \ + { \ + /* Check for potential in-place conversion */ \ + if ((IO_INFO)->may_use_in_place_tconv) { \ + size_t mem_type_size = ((IO_INFO)->op_type == H5D_IO_OP_READ) \ + ? (DINFO)->type_info.dst_type_size \ + : (DINFO)->type_info.src_type_size; \ + size_t file_type_size = ((IO_INFO)->op_type == H5D_IO_OP_READ) \ + ? (DINFO)->type_info.src_type_size \ + : (DINFO)->type_info.dst_type_size; \ + \ + /* Make sure the memory type is not smaller than the file type, otherwise the memory buffer \ + * won't be big enough to serve as the type conversion buffer */ \ + if (mem_type_size >= file_type_size) { \ + hbool_t is_contig; \ + hsize_t sel_off; \ + \ + /* Check if the space is contiguous */ \ + if (H5S_select_contig_block((PIECE_INFO)->mspace, &is_contig, &sel_off, NULL) < 0) \ + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't check if dataspace is contiguous") \ + \ + /* If the first sequence includes all the elements selected in this piece, it it contiguous \ + */ \ + if (is_contig) { \ + H5_CHECK_OVERFLOW(sel_off, hsize_t, size_t); \ + (PIECE_INFO)->in_place_tconv = TRUE; \ + (PIECE_INFO)->buf_off = (size_t)sel_off * mem_type_size; \ + } \ + } \ + } \ + \ + /* If we're not using in-place type conversion, add this piece to global type conversion buffer \ + * size. This will only be used if we must allocate a type conversion buffer for the entire I/O. */ \ + if (!(PIECE_INFO)->in_place_tconv) \ + (IO_INFO)->tconv_buf_size += (PIECE_INFO)->piece_points * MAX((DINFO)->type_info.src_type_size, \ + (DINFO)->type_info.dst_type_size); \ + } /****************************/ /* Package Private Typedefs */ @@ -83,15 +125,13 @@ typedef struct H5D_type_info_t { hid_t dst_type_id; /* Destination datatype ID */ /* Computed/derived values */ - size_t src_type_size; /* Size of source type */ - size_t dst_type_size; /* Size of destination type */ - hbool_t is_conv_noop; /* Whether the type conversion is a NOOP */ - hbool_t is_xform_noop; /* Whether the data transform is a NOOP */ - const H5T_subset_info_t *cmpd_subset; /* Info related to the compound subset conversion functions */ - H5T_bkg_t need_bkg; /* Type of background buf needed */ - size_t request_nelmts; /* Requested strip mine */ - uint8_t *bkg_buf; /* Background buffer */ - hbool_t bkg_buf_allocated; /* Whether the background buffer was allocated */ + size_t src_type_size; /* Size of source type */ + size_t dst_type_size; /* Size of destination type */ + hbool_t is_conv_noop; /* Whether the type conversion is a NOOP */ + hbool_t is_xform_noop; /* Whether the data transform is a NOOP */ + const H5T_subset_info_t *cmpd_subset; /* Info related to the compound subset conversion functions */ + H5T_bkg_t need_bkg; /* Type of background buf needed */ + size_t request_nelmts; /* Requested strip mine */ } H5D_type_info_t; /* Forward declaration of structs used below */ @@ -209,9 +249,11 @@ typedef struct H5D_piece_info_t { hsize_t piece_points; /* Number of elements selected in piece */ hsize_t scaled[H5O_LAYOUT_NDIMS]; /* Scaled coordinates of chunk (in file dataset's dataspace) */ H5S_t *fspace; /* Dataspace describing chunk & selection in it */ - unsigned fspace_shared; /* Indicate that the file space for a chunk is shared and shouldn't be freed */ - H5S_t *mspace; /* Dataspace describing selection in memory corresponding to this chunk */ - unsigned mspace_shared; /* Indicate that the memory space for a chunk is shared and shouldn't be freed */ + unsigned fspace_shared; /* Indicate that the file space for a chunk is shared and shouldn't be freed */ + H5S_t *mspace; /* Dataspace describing selection in memory corresponding to this chunk */ + unsigned mspace_shared; /* Indicate that the memory space for a chunk is shared and shouldn't be freed */ + hbool_t in_place_tconv; /* Whether to perform type conversion in-place */ + size_t buf_off; /* Buffer offset for in-place type conversion */ struct H5D_dset_io_info_t *dset_info; /* Pointer to dset_info */ } H5D_piece_info_t; @@ -263,10 +305,23 @@ typedef struct H5D_io_info_t { const void **wbufs; /* Array of write buffers */ haddr_t store_faddr; /* lowest file addr for read/write */ H5_flexible_const_ptr_t base_maddr; /* starting mem address */ - hbool_t use_select_io; /* Whether to use selection I/O */ + H5D_selection_io_mode_t use_select_io; /* Whether to use selection I/O */ uint8_t *tconv_buf; /* Datatype conv buffer */ hbool_t tconv_buf_allocated; /* Whether the type conversion buffer was allocated */ - size_t max_type_size; /* Largest of all source and destination type sizes */ + size_t tconv_buf_size; /* Size of type conversion buffer */ + uint8_t *bkg_buf; /* Background buffer */ + hbool_t bkg_buf_allocated; /* Whether the background buffer was allocated */ + size_t bkg_buf_size; /* Size of background buffer */ + size_t max_tconv_type_size; /* Largest of all source and destination type sizes involved in type + conversion */ + hbool_t + must_fill_bkg; /* Whether any datasets need a background buffer filled with destination contents */ + hbool_t may_use_in_place_tconv; /* Whether datasets in this I/O could potentially use in-place type + conversion if the type sizes are compatible with it */ +#ifdef H5_HAVE_PARALLEL + H5D_mpio_actual_io_mode_t actual_io_mode; /* Actual type of collective or independent I/O */ +#endif /* H5_HAVE_PARALLEL */ + unsigned no_selection_io_cause; /* "No selection I/O cause" flags */ } H5D_io_info_t; /* Created to pass both at once for callback func */ @@ -620,12 +675,14 @@ H5_DLL herr_t H5D__select_write(const H5D_io_info_t *io_info, const H5D_dset_io_ H5_DLL herr_t H5D_select_io_mem(void *dst_buf, H5S_t *dst_space, const void *src_buf, H5S_t *src_space, size_t elmt_size, size_t nelmts); -/* Functions that perform scatter-gather serial I/O operations */ +/* Functions that perform scatter-gather I/O operations */ H5_DLL herr_t H5D__scatter_mem(const void *_tscat_buf, H5S_sel_iter_t *iter, size_t nelmts, void *_buf); H5_DLL size_t H5D__gather_mem(const void *_buf, H5S_sel_iter_t *iter, size_t nelmts, void *_tgath_buf /*out*/); H5_DLL herr_t H5D__scatgath_read(const H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset_info); H5_DLL herr_t H5D__scatgath_write(const H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset_info); +H5_DLL herr_t H5D__scatgath_read_select(H5D_io_info_t *io_info); +H5_DLL herr_t H5D__scatgath_write_select(H5D_io_info_t *io_info); /* Functions that operate on dataset's layout information */ H5_DLL herr_t H5D__layout_set_io_ops(const H5D_t *dataset); diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h index 357945b..8265ac2 100644 --- a/src/H5Dprivate.h +++ b/src/H5Dprivate.h @@ -78,12 +78,15 @@ #define H5D_MPIO_LOCAL_NO_COLLECTIVE_CAUSE_NAME \ "local_no_collective_cause" /* cause of broken collective I/O in each process */ #define H5D_MPIO_GLOBAL_NO_COLLECTIVE_CAUSE_NAME \ - "global_no_collective_cause" /* cause of broken collective I/O in all processes */ -#define H5D_XFER_EDC_NAME "err_detect" /* EDC */ -#define H5D_XFER_FILTER_CB_NAME "filter_cb" /* Filter callback function */ -#define H5D_XFER_CONV_CB_NAME "type_conv_cb" /* Type conversion callback function */ -#define H5D_XFER_XFORM_NAME "data_transform" /* Data transform */ -#define H5D_XFER_DSET_IO_SEL_NAME "dset_io_selection" /* Dataset I/O selection */ + "global_no_collective_cause" /* cause of broken collective I/O in all processes */ +#define H5D_XFER_EDC_NAME "err_detect" /* EDC */ +#define H5D_XFER_FILTER_CB_NAME "filter_cb" /* Filter callback function */ +#define H5D_XFER_CONV_CB_NAME "type_conv_cb" /* Type conversion callback function */ +#define H5D_XFER_XFORM_NAME "data_transform" /* Data transform */ +#define H5D_XFER_DSET_IO_SEL_NAME "dset_io_selection" /* Dataset I/O selection */ +#define H5D_XFER_SELECTION_IO_MODE_NAME "selection_io_mode" /* Selection I/O mode */ +#define H5D_XFER_NO_SELECTION_IO_CAUSE_NAME "no_selection_io_cause" /* Cause for no selection I/O */ +#define H5D_XFER_MODIFY_WRITE_BUF_NAME "modify_write_buf" /* Modify write buffers */ #ifdef H5_HAVE_INSTRUMENTED_LIBRARY /* Collective chunk instrumentation properties */ #define H5D_XFER_COLL_CHUNK_LINK_HARD_NAME "coll_chunk_link_hard" diff --git a/src/H5Dscatgath.c b/src/H5Dscatgath.c index 2547bdc..802544c 100644 --- a/src/H5Dscatgath.c +++ b/src/H5Dscatgath.c @@ -31,6 +31,11 @@ /* Local Macros */ /****************/ +/* Macro to determine if we're using H5D__compound_opt_read() */ +#define H5D__SCATGATH_USE_CMPD_OPT_READ(DSET_INFO, PIECE_INFO) \ + ((DSET_INFO)->type_info.cmpd_subset && H5T_SUBSET_FALSE != (DSET_INFO)->type_info.cmpd_subset->subset && \ + !(PIECE_INFO)->in_place_tconv) + /******************/ /* Local Typedefs */ /******************/ @@ -529,7 +534,7 @@ H5D__scatgath_read(const H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset_ } /* end if */ else { if (H5T_BKG_YES == dset_info->type_info.need_bkg) { - n = H5D__gather_mem(buf, bkg_iter, smine_nelmts, dset_info->type_info.bkg_buf /*out*/); + n = H5D__gather_mem(buf, bkg_iter, smine_nelmts, io_info->bkg_buf /*out*/); if (n != smine_nelmts) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "mem gather failed") } /* end if */ @@ -539,7 +544,7 @@ H5D__scatgath_read(const H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset_ */ if (H5T_convert(dset_info->type_info.tpath, dset_info->type_info.src_type_id, dset_info->type_info.dst_type_id, smine_nelmts, (size_t)0, (size_t)0, - io_info->tconv_buf, dset_info->type_info.bkg_buf) < 0) + io_info->tconv_buf, io_info->bkg_buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "datatype conversion failed") /* Do the data transform after the conversion (since we're using type mem_type) */ @@ -672,8 +677,7 @@ H5D__scatgath_write(const H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset } /* end if */ else { if (H5T_BKG_YES == dset_info->type_info.need_bkg) { - n = H5D__gather_file(io_info, dset_info, bkg_iter, smine_nelmts, - dset_info->type_info.bkg_buf /*out*/); + n = H5D__gather_file(io_info, dset_info, bkg_iter, smine_nelmts, io_info->bkg_buf /*out*/); if (n != smine_nelmts) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file gather failed") } /* end if */ @@ -697,7 +701,7 @@ H5D__scatgath_write(const H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset */ if (H5T_convert(dset_info->type_info.tpath, dset_info->type_info.src_type_id, dset_info->type_info.dst_type_id, smine_nelmts, (size_t)0, (size_t)0, - io_info->tconv_buf, dset_info->type_info.bkg_buf) < 0) + io_info->tconv_buf, io_info->bkg_buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "datatype conversion failed") } /* end else */ @@ -727,6 +731,559 @@ done: } /* end H5D__scatgath_write() */ /*------------------------------------------------------------------------- + * Function: H5D__scatgath_read_select + * + * Purpose: Perform scatter/gather read from a list of dataset pieces + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__scatgath_read_select(H5D_io_info_t *io_info) +{ + H5S_t **tmp_mem_spaces = NULL; /* Memory spaces to use for read from disk */ + H5S_sel_iter_t *mem_iter = NULL; /* Memory selection iteration info */ + hbool_t mem_iter_init = FALSE; /* Memory selection iteration info has been initialized */ + void **tmp_bufs = NULL; /* Buffers to use for read from disk */ + void *tmp_bkg_buf = NULL; /* Temporary background buffer pointer */ + size_t tconv_bytes_used = 0; /* Number of bytes used so far in conversion buffer */ + size_t bkg_bytes_used = 0; /* Number of bytes used so far in background buffer */ + size_t i; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Sanity check */ + HDassert(io_info); + HDassert(io_info->count > 0); + HDassert(io_info->mem_spaces || io_info->pieces_added == 0); + HDassert(io_info->file_spaces || io_info->pieces_added == 0); + HDassert(io_info->addrs || io_info->pieces_added == 0); + HDassert(io_info->element_sizes || io_info->pieces_added == 0); + HDassert(io_info->rbufs || io_info->pieces_added == 0); + + /* Allocate list of buffers (within the tconv buf) */ + if (NULL == (tmp_bufs = H5MM_malloc(io_info->pieces_added * sizeof(void *)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "memory allocation failed for temporary buffer list") + + /* Allocate the iterator */ + if (NULL == (mem_iter = H5FL_MALLOC(H5S_sel_iter_t))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't allocate memory iterator") + + /* Allocate list of block memory spaces */ + /*!FIXME delay doing this until we find the first mem space that is non-contiguous or doesn't start at 0 + */ + if (NULL == (tmp_mem_spaces = H5MM_malloc(io_info->pieces_added * sizeof(H5S_t *)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "memory allocation failed for temporary memory space list") + + /* Build read operation to tconv buffer */ + for (i = 0; i < io_info->pieces_added; i++) { + H5D_dset_io_info_t *dset_info = io_info->sel_pieces[i]->dset_info; + + HDassert(io_info->sel_pieces[i]->piece_points > 0); + + /* Check if this piece is involved in type conversion */ + if (dset_info->type_info.is_xform_noop && dset_info->type_info.is_conv_noop) { + /* No type conversion, just copy the mem space and buffer */ + tmp_mem_spaces[i] = io_info->mem_spaces[i]; + tmp_bufs[i] = io_info->rbufs[i]; + } + else { + /* Create block memory space */ + if (NULL == + (tmp_mem_spaces[i] = H5S_create_simple(1, &io_info->sel_pieces[i]->piece_points, NULL))) { + HDmemset(&tmp_mem_spaces[i], 0, (io_info->pieces_added - i) * sizeof(tmp_mem_spaces[0])); + HGOTO_ERROR(H5E_DATASET, H5E_CANTCREATE, FAIL, "unable to create simple memory dataspace") + } + + /* Check for in-place type conversion */ + if (io_info->sel_pieces[i]->in_place_tconv) + /* Set buffer to point to read buffer + offset */ + tmp_bufs[i] = (uint8_t *)(io_info->rbufs[i]) + io_info->sel_pieces[i]->buf_off; + else { + /* Set buffer to point into type conversion buffer */ + tmp_bufs[i] = io_info->tconv_buf + tconv_bytes_used; + tconv_bytes_used += + io_info->sel_pieces[i]->piece_points * + MAX(dset_info->type_info.src_type_size, dset_info->type_info.dst_type_size); + HDassert(tconv_bytes_used <= io_info->tconv_buf_size); + } + + /* Fill background buffer here unless we will use H5D__compound_opt_read(). Must do this before + * the read so the read buffer doesn't get wiped out if we're using in-place type conversion */ + if (!H5D__SCATGATH_USE_CMPD_OPT_READ(dset_info, io_info->sel_pieces[i])) { + /* Check for background buffer */ + if (dset_info->type_info.need_bkg) { + HDassert(io_info->bkg_buf); + + /* Calculate background buffer position */ + tmp_bkg_buf = io_info->bkg_buf + bkg_bytes_used; + bkg_bytes_used += + io_info->sel_pieces[i]->piece_points * dset_info->type_info.dst_type_size; + HDassert(bkg_bytes_used <= io_info->bkg_buf_size); + + /* Gather data from read buffer to background buffer if necessary */ + if (H5T_BKG_YES == dset_info->type_info.need_bkg) { + /* Initialize memory iterator */ + HDassert(!mem_iter_init); + if (H5S_select_iter_init(mem_iter, io_info->mem_spaces[i], + dset_info->type_info.dst_type_size, 0) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "unable to initialize memory selection information") + mem_iter_init = TRUE; /* Memory selection iteration info has been initialized */ + + if ((size_t)io_info->sel_pieces[i]->piece_points != + H5D__gather_mem(io_info->rbufs[i], mem_iter, + (size_t)io_info->sel_pieces[i]->piece_points, + tmp_bkg_buf /*out*/)) + HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "mem gather failed") + + /* Reset selection iterator */ + HDassert(mem_iter_init); + if (H5S_SELECT_ITER_RELEASE(mem_iter) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't release selection iterator") + mem_iter_init = FALSE; + } + } + } + } + } + + /* Read data from all pieces */ + H5_CHECK_OVERFLOW(io_info->pieces_added, size_t, uint32_t) + if (H5F_shared_select_read(io_info->f_sh, H5FD_MEM_DRAW, (uint32_t)io_info->pieces_added, tmp_mem_spaces, + io_info->file_spaces, io_info->addrs, io_info->element_sizes, tmp_bufs) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "selection read failed") + + /* Reset bkg_bytes_used */ + bkg_bytes_used = 0; + + /* Perform type conversion and scatter data to memory buffers for datasets that need this */ + for (i = 0; i < io_info->pieces_added; i++) { + H5D_dset_io_info_t *dset_info = io_info->sel_pieces[i]->dset_info; + + HDassert(tmp_mem_spaces[i]); + + /* Check if this piece is involved in type conversion */ + if (tmp_mem_spaces[i] != io_info->mem_spaces[i]) { + H5_CHECK_OVERFLOW(io_info->sel_pieces[i]->piece_points, hsize_t, size_t); + + /* Initialize memory iterator */ + HDassert(!mem_iter_init); + if (H5S_select_iter_init(mem_iter, io_info->mem_spaces[i], dset_info->type_info.dst_type_size, + 0) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "unable to initialize memory selection information") + mem_iter_init = TRUE; /* Memory selection iteration info has been initialized */ + + /* If the source and destination are compound types and subset of each other + * and no conversion is needed, copy the data directly into user's buffer and + * bypass the rest of steps. + */ + if (H5D__SCATGATH_USE_CMPD_OPT_READ(dset_info, io_info->sel_pieces[i])) { + if (H5D__compound_opt_read((size_t)io_info->sel_pieces[i]->piece_points, mem_iter, + &dset_info->type_info, tmp_bufs[i], io_info->rbufs[i] /*out*/) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "datatype conversion failed") + } + else { + /* Check for background buffer */ + if (dset_info->type_info.need_bkg) { + HDassert(io_info->bkg_buf); + + /* Calculate background buffer position */ + tmp_bkg_buf = io_info->bkg_buf + bkg_bytes_used; + bkg_bytes_used += + io_info->sel_pieces[i]->piece_points * dset_info->type_info.dst_type_size; + HDassert(bkg_bytes_used <= io_info->bkg_buf_size); + } + + /* + * Perform datatype conversion. + */ + if (H5T_convert(dset_info->type_info.tpath, dset_info->type_info.src_type_id, + dset_info->type_info.dst_type_id, + (size_t)io_info->sel_pieces[i]->piece_points, (size_t)0, (size_t)0, + tmp_bufs[i], tmp_bkg_buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "datatype conversion failed") + + /* Do the data transform after the conversion (since we're using type mem_type) */ + if (!dset_info->type_info.is_xform_noop) { + H5Z_data_xform_t *data_transform; /* Data transform info */ + + /* Retrieve info from API context */ + if (H5CX_get_data_transform(&data_transform) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get data transform info") + + if (H5Z_xform_eval(data_transform, tmp_bufs[i], + (size_t)io_info->sel_pieces[i]->piece_points, + dset_info->type_info.mem_type) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "Error performing data transform") + } + + /* Scatter the data into memory if this was not an in-place conversion */ + if (!io_info->sel_pieces[i]->in_place_tconv) + if (H5D__scatter_mem(tmp_bufs[i], mem_iter, (size_t)io_info->sel_pieces[i]->piece_points, + io_info->rbufs[i] /*out*/) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "scatter failed") + } + + /* Release selection iterator */ + HDassert(mem_iter_init); + if (H5S_SELECT_ITER_RELEASE(mem_iter) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't release selection iterator") + mem_iter_init = FALSE; + } + } + +done: + /* Release and free selection iterator */ + if (mem_iter_init && H5S_SELECT_ITER_RELEASE(mem_iter) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't release selection iterator") + if (mem_iter) + mem_iter = H5FL_FREE(H5S_sel_iter_t, mem_iter); + + /* Free tmp_bufs */ + H5MM_free(tmp_bufs); + tmp_bufs = NULL; + + /* Clear and free tmp_mem_spaces */ + if (tmp_mem_spaces) { + for (i = 0; i < io_info->pieces_added; i++) + if (tmp_mem_spaces[i] != io_info->mem_spaces[i] && tmp_mem_spaces[i] && + H5S_close(tmp_mem_spaces[i]) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "Can't close dataspace") + H5MM_free(tmp_mem_spaces); + tmp_mem_spaces = NULL; + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__scatgath_read_select() */ + +/*------------------------------------------------------------------------- + * Function: H5D__scatgath_write_select + * + * Purpose: Perform scatter/gather write to a list of dataset pieces. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__scatgath_write_select(H5D_io_info_t *io_info) +{ + H5S_t **write_mem_spaces = NULL; /* Memory spaces to use for write to disk */ + size_t spaces_added = 0; /* Number of spaces added to write_mem_spaces */ + H5S_sel_iter_t *mem_iter = NULL; /* Memory selection iteration info */ + hbool_t mem_iter_init = FALSE; /* Memory selection iteration info has been initialized */ + const void **write_bufs = NULL; /* Buffers to use for write to disk */ + size_t tconv_bytes_used = 0; /* Number of bytes used so far in conversion buffer */ + size_t bkg_bytes_used = 0; /* Number of bytes used so far in background buffer */ + H5S_t **bkg_mem_spaces = NULL; /* Array of memory spaces for read to background buffer */ + H5S_t **bkg_file_spaces = NULL; /* Array of file spaces for read to background buffer */ + haddr_t *bkg_addrs = NULL; /* Array of file addresses for read to background buffer */ + size_t *bkg_element_sizes = NULL; /* Array of element sizes for read to background buffer */ + void **bkg_bufs = NULL; /* Array background buffers for read of existing file contents */ + size_t bkg_pieces = 0; /* Number of pieces that need to read the background data from disk */ + size_t i; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Sanity check */ + HDassert(io_info); + HDassert(io_info->count > 0); + HDassert(io_info->mem_spaces || io_info->pieces_added == 0); + HDassert(io_info->file_spaces || io_info->pieces_added == 0); + HDassert(io_info->addrs || io_info->pieces_added == 0); + HDassert(io_info->element_sizes || io_info->pieces_added == 0); + HDassert(io_info->wbufs || io_info->pieces_added == 0); + + /* Allocate list of buffers (within the tconv buf) */ + if (NULL == (write_bufs = (const void **)H5MM_malloc(io_info->pieces_added * sizeof(const void *)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "memory allocation failed for temporary buffer list") + + /* Allocate the iterator */ + if (NULL == (mem_iter = H5FL_MALLOC(H5S_sel_iter_t))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't allocate memory iterator") + + /* Allocate list of block memory spaces */ + /*!FIXME delay doing this until we find the first mem space that is non-contiguous or doesn't start at 0 + */ + if (NULL == (write_mem_spaces = H5MM_malloc(io_info->pieces_added * sizeof(H5S_t *)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "memory allocation failed for temporary memory space list") + + /* Build operations to read data to background buffer and to write data */ + for (i = 0; i < io_info->pieces_added; i++) { + H5D_dset_io_info_t *dset_info = io_info->sel_pieces[i]->dset_info; + + HDassert(io_info->sel_pieces[i]->piece_points > 0); + + /* Check if this piece is involved in type conversion */ + if (dset_info->type_info.is_xform_noop && dset_info->type_info.is_conv_noop) { + /* No type conversion, just copy the mem space and buffer */ + write_mem_spaces[i] = io_info->mem_spaces[i]; + spaces_added++; + write_bufs[i] = io_info->wbufs[i]; + } + else { + void *tmp_write_buf; /* To sidestep const warnings */ + void *tmp_bkg_buf = NULL; + + H5_CHECK_OVERFLOW(io_info->sel_pieces[i]->piece_points, hsize_t, size_t); + + /* Initialize memory iterator */ + HDassert(!mem_iter_init); + if (H5S_select_iter_init(mem_iter, io_info->mem_spaces[i], dset_info->type_info.src_type_size, + 0) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "unable to initialize memory selection information") + mem_iter_init = TRUE; /* Memory selection iteration info has been initialized */ + + /* Create block memory space */ + if (NULL == + (write_mem_spaces[i] = H5S_create_simple(1, &io_info->sel_pieces[i]->piece_points, NULL))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCREATE, FAIL, "unable to create simple memory dataspace") + spaces_added++; + + /* Check for in-place type conversion */ + if (io_info->sel_pieces[i]->in_place_tconv) { + H5_flexible_const_ptr_t flex_buf; + + /* Set buffer to point to write buffer + offset */ + /* Use cast to union to twiddle away const. OK because if we're doing this it means the user + * explicitly allowed us to modify this buffer via H5Pset_modify_write_buf(). */ + flex_buf.cvp = io_info->wbufs[i]; + tmp_write_buf = (uint8_t *)flex_buf.vp + io_info->sel_pieces[i]->buf_off; + } + else { + /* Set buffer to point into type conversion buffer */ + tmp_write_buf = io_info->tconv_buf + tconv_bytes_used; + tconv_bytes_used += + io_info->sel_pieces[i]->piece_points * + MAX(dset_info->type_info.src_type_size, dset_info->type_info.dst_type_size); + HDassert(tconv_bytes_used <= io_info->tconv_buf_size); + + /* Gather data from application buffer into the datatype conversion buffer */ + if ((size_t)io_info->sel_pieces[i]->piece_points != + H5D__gather_mem(io_info->wbufs[i], mem_iter, (size_t)io_info->sel_pieces[i]->piece_points, + tmp_write_buf /*out*/)) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "mem gather failed") + } + + /* Set buffer for writing to disk (from type conversion buffer) */ + write_bufs[i] = (const void *)tmp_write_buf; + + /* If the source and destination are compound types and the destination is a subset of + * the source and no conversion is needed, copy the data directly into the type + * conversion buffer and bypass the rest of steps. If the source is a subset of the + * destination, the optimization is done in conversion function H5T_conv_struct_opt to + * protect the background data. + */ + if (dset_info->type_info.cmpd_subset && + H5T_SUBSET_DST == dset_info->type_info.cmpd_subset->subset && + dset_info->type_info.dst_type_size == dset_info->type_info.cmpd_subset->copy_size && + !io_info->sel_pieces[i]->in_place_tconv) { + if (H5D__compound_opt_write((size_t)io_info->sel_pieces[i]->piece_points, + &dset_info->type_info, tmp_write_buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "datatype conversion failed") + + /* No background buffer necessary, prevent this element from being considered in the second + * loop */ + /* Add this to H5Tconv.c? -NAF */ + dset_info->type_info.need_bkg = H5T_BKG_NO; + } /* end if */ + else { + /* Check for background buffer */ + if (dset_info->type_info.need_bkg) { + HDassert(io_info->bkg_buf); + + /* Calculate background buffer position */ + tmp_bkg_buf = io_info->bkg_buf + bkg_bytes_used; + bkg_bytes_used += + io_info->sel_pieces[i]->piece_points * dset_info->type_info.dst_type_size; + HDassert(bkg_bytes_used <= io_info->bkg_buf_size); + } + + /* Set up background buffer read operation if necessary */ + if (H5T_BKG_YES == dset_info->type_info.need_bkg) { + HDassert(io_info->must_fill_bkg); + + /* Allocate arrays of parameters for selection read to background buffer if necessary */ + if (!bkg_mem_spaces) { + HDassert(!bkg_file_spaces && !bkg_addrs && !bkg_element_sizes && !bkg_bufs); + if (NULL == (bkg_mem_spaces = H5MM_malloc(io_info->pieces_added * sizeof(H5S_t *)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "memory allocation failed for memory space list") + if (NULL == (bkg_file_spaces = H5MM_malloc(io_info->pieces_added * sizeof(H5S_t *)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "memory allocation failed for file space list") + if (NULL == (bkg_addrs = H5MM_malloc(io_info->pieces_added * sizeof(haddr_t)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "memory allocation failed for piece address list") + if (NULL == (bkg_element_sizes = H5MM_malloc(io_info->pieces_added * sizeof(size_t)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "memory allocation failed for element size list") + if (NULL == (bkg_bufs = H5MM_malloc(io_info->pieces_added * sizeof(const void *)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "memory allocation failed for write buffer list") + } + + /* Use same (block) memory space, file space, address, and element size as write operation + */ + HDassert(bkg_mem_spaces && bkg_file_spaces && bkg_addrs && bkg_element_sizes && bkg_bufs); + bkg_mem_spaces[bkg_pieces] = write_mem_spaces[i]; + bkg_file_spaces[bkg_pieces] = io_info->file_spaces[i]; + bkg_addrs[bkg_pieces] = io_info->addrs[i]; + bkg_element_sizes[bkg_pieces] = io_info->element_sizes[i]; + + /* Use previously calculated background buffer position */ + bkg_bufs[bkg_pieces] = tmp_bkg_buf; + + /* Add piece */ + bkg_pieces++; + } + else { + /* Perform type conversion here to avoid second loop if no dsets use the background buffer + */ + /* Do the data transform before the type conversion (since + * transforms must be done in the memory type). */ + if (!dset_info->type_info.is_xform_noop) { + H5Z_data_xform_t *data_transform; /* Data transform info */ + + /* Retrieve info from API context */ + if (H5CX_get_data_transform(&data_transform) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get data transform info") + + if (H5Z_xform_eval(data_transform, tmp_write_buf, + (size_t)io_info->sel_pieces[i]->piece_points, + dset_info->type_info.mem_type) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "Error performing data transform") + } + + /* + * Perform datatype conversion. + */ + if (H5T_convert(dset_info->type_info.tpath, dset_info->type_info.src_type_id, + dset_info->type_info.dst_type_id, + (size_t)io_info->sel_pieces[i]->piece_points, (size_t)0, (size_t)0, + tmp_write_buf, tmp_bkg_buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "datatype conversion failed") + } + } + + /* Release selection iterator */ + HDassert(mem_iter_init); + if (H5S_SELECT_ITER_RELEASE(mem_iter) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't release selection iterator") + mem_iter_init = FALSE; + } + } + + HDassert(spaces_added == io_info->pieces_added); + + /* Gather data to background buffer if necessary */ + if (io_info->must_fill_bkg) { + size_t j = 0; /* Index into array of background buffers */ + + /* Read data */ + H5_CHECK_OVERFLOW(bkg_pieces, size_t, uint32_t) + if (H5F_shared_select_read(io_info->f_sh, H5FD_MEM_DRAW, (uint32_t)bkg_pieces, bkg_mem_spaces, + bkg_file_spaces, bkg_addrs, bkg_element_sizes, bkg_bufs) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "selection read to background buffer failed") + + /* Perform type conversion on pieces with background buffers that were just read */ + for (i = 0; i < io_info->pieces_added; i++) { + H5D_dset_io_info_t *dset_info = io_info->sel_pieces[i]->dset_info; + + if (H5T_BKG_YES == dset_info->type_info.need_bkg) { + /* Non-const write_buf[i]. Use pointer math here to avoid const warnings. When + * there's a background buffer write_buf[i] always points inside the non-const tconv + * buf so this is OK. */ + void *tmp_write_buf = + (void *)((uint8_t *)io_info->tconv_buf + + ((const uint8_t *)write_bufs[i] - (const uint8_t *)io_info->tconv_buf)); + + /* Do the data transform before the type conversion (since + * transforms must be done in the memory type). */ + if (!dset_info->type_info.is_xform_noop) { + H5Z_data_xform_t *data_transform; /* Data transform info */ + + /* Retrieve info from API context */ + if (H5CX_get_data_transform(&data_transform) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get data transform info") + + if (H5Z_xform_eval(data_transform, tmp_write_buf, + (size_t)io_info->sel_pieces[i]->piece_points, + dset_info->type_info.mem_type) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "Error performing data transform") + } + + /* + * Perform datatype conversion. + */ + HDassert(j < bkg_pieces); + if (H5T_convert(dset_info->type_info.tpath, dset_info->type_info.src_type_id, + dset_info->type_info.dst_type_id, + (size_t)io_info->sel_pieces[i]->piece_points, (size_t)0, (size_t)0, + tmp_write_buf, bkg_bufs[j]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "datatype conversion failed") + + /* Advance to next background buffer */ + j++; + } + } + + HDassert(j == bkg_pieces); + } + + /* Write data to disk */ + H5_CHECK_OVERFLOW(io_info->pieces_added, size_t, uint32_t) + if (H5F_shared_select_write(io_info->f_sh, H5FD_MEM_DRAW, (uint32_t)io_info->pieces_added, + write_mem_spaces, io_info->file_spaces, io_info->addrs, + io_info->element_sizes, write_bufs) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "selection write failed") + +done: + /* Release and free selection iterator */ + if (mem_iter_init && H5S_SELECT_ITER_RELEASE(mem_iter) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't release selection iterator") + if (mem_iter) + mem_iter = H5FL_FREE(H5S_sel_iter_t, mem_iter); + + /* Free write_bufs */ + H5MM_free(write_bufs); + write_bufs = NULL; + + /* Clear and free write_mem_spaces */ + if (write_mem_spaces) { + for (i = 0; i < spaces_added; i++) { + HDassert(write_mem_spaces[i]); + if (write_mem_spaces[i] != io_info->mem_spaces[i] && H5S_close(write_mem_spaces[i]) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "Can't close dataspace") + } + H5MM_free(write_mem_spaces); + write_mem_spaces = NULL; + } + + /* Free bakcground buffer parameter arrays */ + H5MM_free(bkg_mem_spaces); + bkg_mem_spaces = NULL; + H5MM_free(bkg_file_spaces); + bkg_file_spaces = NULL; + H5MM_free(bkg_addrs); + bkg_addrs = NULL; + H5MM_free(bkg_element_sizes); + bkg_element_sizes = NULL; + H5MM_free(bkg_bufs); + bkg_bufs = NULL; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__scatgath_write_select() */ + +/*------------------------------------------------------------------------- * Function: H5D__compound_opt_read * * Purpose: A special optimization case when the source and diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index c40642f..6695c02 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -2384,7 +2384,8 @@ H5D__virtual_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t H5_ATTR_UNUSED * FUNC_ENTER_PACKAGE_NOERR /* Disable selection I/O */ - io_info->use_select_io = FALSE; + io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; + io_info->no_selection_io_cause |= H5D_SEL_IO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5D__virtual_io_init() */ diff --git a/src/H5FDint.c b/src/H5FDint.c index c5b8713..6d90aae 100644 --- a/src/H5FDint.c +++ b/src/H5FDint.c @@ -469,6 +469,7 @@ H5FD_read_vector(H5FD_t *file, uint32_t count, H5FD_mem_t types[], haddr_t addrs */ extend_sizes = FALSE; extend_types = FALSE; + uint32_t no_selection_io_cause; for (i = 0; i < count; i++) { @@ -505,6 +506,11 @@ H5FD_read_vector(H5FD_t *file, uint32_t count, H5FD_mem_t types[], haddr_t addrs if ((file->cls->read)(file, type, dxpl_id, addrs[i], size, bufs[i]) < 0) HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "driver read request failed") } + + /* Add H5D_SEL_IO_NO_VECTOR_OR_SELECTION_IO_CB to no selection I/O cause */ + H5CX_get_no_selection_io_cause(&no_selection_io_cause); + no_selection_io_cause |= H5D_SEL_IO_NO_VECTOR_OR_SELECTION_IO_CB; + H5CX_set_no_selection_io_cause(no_selection_io_cause); } done: @@ -669,6 +675,7 @@ H5FD_write_vector(H5FD_t *file, uint32_t count, H5FD_mem_t types[], haddr_t addr */ extend_sizes = FALSE; extend_types = FALSE; + uint32_t no_selection_io_cause; for (i = 0; i < count; i++) { @@ -705,6 +712,11 @@ H5FD_write_vector(H5FD_t *file, uint32_t count, H5FD_mem_t types[], haddr_t addr if ((file->cls->write)(file, type, dxpl_id, addrs[i], size, bufs[i]) < 0) HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "driver write request failed") } + + /* Add H5D_SEL_IO_NO_VECTOR_OR_SELECTION_IO_CB to no selection I/O cause */ + H5CX_get_no_selection_io_cause(&no_selection_io_cause); + no_selection_io_cause |= H5D_SEL_IO_NO_VECTOR_OR_SELECTION_IO_CB; + H5CX_set_no_selection_io_cause(no_selection_io_cause); } done: @@ -991,6 +1003,14 @@ H5FD__read_selection_translate(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, uin 0) HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "driver read vector request failed") } + else { + uint32_t no_selection_io_cause; + + /* Add H5D_SEL_IO_NO_VECTOR_OR_SELECTION_IO_CB to no selection I/O cause */ + H5CX_get_no_selection_io_cause(&no_selection_io_cause); + no_selection_io_cause |= H5D_SEL_IO_NO_VECTOR_OR_SELECTION_IO_CB; + H5CX_set_no_selection_io_cause(no_selection_io_cause); + } done: /* Terminate and free iterators */ @@ -1630,6 +1650,14 @@ H5FD__write_selection_translate(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, ui 0) HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "driver write vector request failed") } + else { + uint32_t no_selection_io_cause; + + /* Add H5D_SEL_IO_NO_VECTOR_OR_SELECTION_IO_CB to no selection I/O cause */ + H5CX_get_no_selection_io_cause(&no_selection_io_cause); + no_selection_io_cause |= H5D_SEL_IO_NO_VECTOR_OR_SELECTION_IO_CB; + H5CX_set_no_selection_io_cause(no_selection_io_cause); + } done: /* Terminate and free iterators */ diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index f3e463d..40c6cff 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -906,6 +906,7 @@ H5_DLL haddr_t H5F_shared_get_eoa(const H5F_shared_t *f_sh, H5FD_mem_t type); H5_DLL haddr_t H5F_get_eoa(const H5F_t *f, H5FD_mem_t type); H5_DLL herr_t H5F_shared_get_file_driver(const H5F_shared_t *f_sh, H5FD_t **file_handle); H5_DLL herr_t H5F_get_vfd_handle(const H5F_t *file, hid_t fapl, void **file_handle); +H5_DLL hbool_t H5F_has_vector_select_io(const H5F_t *f, hbool_t is_write); /* File mounting routines */ H5_DLL herr_t H5F_mount(const struct H5G_loc_t *loc, const char *name, H5F_t *child, hid_t plist_id); diff --git a/src/H5Fquery.c b/src/H5Fquery.c index 469df58..72b173f 100644 --- a/src/H5Fquery.c +++ b/src/H5Fquery.c @@ -1372,3 +1372,30 @@ H5F_get_file_locking(const H5F_t *f) FUNC_LEAVE_NOAPI(f->shared->use_file_locking) } /* end H5F_get_file_locking */ + +/*------------------------------------------------------------------------- + * Function: H5F_has_vector_select_io + * + * Purpose: Determine if vector or selection I/O is supported by this file + * + * Return: TRUE/FALSE + * + *------------------------------------------------------------------------- + */ +hbool_t +H5F_has_vector_select_io(const H5F_t *f, hbool_t is_write) +{ + hbool_t ret_value; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(f->shared); + + if (is_write) + ret_value = (f->shared->lf->cls->write_vector != NULL || f->shared->lf->cls->write_selection != NULL); + else + ret_value = (f->shared->lf->cls->read_vector != NULL || f->shared->lf->cls->read_selection != NULL); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F_has_vector_select_io */ diff --git a/src/H5Pdxpl.c b/src/H5Pdxpl.c index ecf0372..6eef558 100644 --- a/src/H5Pdxpl.c +++ b/src/H5Pdxpl.c @@ -168,6 +168,19 @@ #define H5D_XFER_DSET_IO_SEL_ENC H5P__dxfr_edc_enc #define H5D_XFER_DSET_IO_SEL_DEC H5P__dxfr_edc_dec #endif /* QAK */ +/* Definition for selection I/O mode property */ +#define H5D_XFER_SELECTION_IO_MODE_SIZE sizeof(H5D_selection_io_mode_t) +#define H5D_XFER_SELECTION_IO_MODE_DEF H5D_SELECTION_IO_MODE_DEFAULT +#define H5D_XFER_SELECTION_IO_MODE_ENC H5P__dxfr_selection_io_mode_enc +#define H5D_XFER_SELECTION_IO_MODE_DEC H5P__dxfr_selection_io_mode_dec +/* Definitions for cause of no selection I/O property */ +#define H5D_XFER_NO_SELECTION_IO_CAUSE_SIZE sizeof(uint32_t) +#define H5D_XFER_NO_SELECTION_IO_CAUSE_DEF 0 +/* Definitions for modify write buffer property */ +#define H5D_XFER_MODIFY_WRITE_BUF_SIZE sizeof(hbool_t) +#define H5D_XFER_MODIFY_WRITE_BUF_DEF FALSE +#define H5D_XFER_MODIFY_WRITE_BUF_ENC H5P__dxfr_modify_write_buf_enc +#define H5D_XFER_MODIFY_WRITE_BUF_DEC H5P__dxfr_modify_write_buf_dec /******************/ /* Local Typedefs */ @@ -208,6 +221,10 @@ static herr_t H5P__dxfr_xform_close(const char *name, size_t size, void *value); static herr_t H5P__dxfr_dset_io_hyp_sel_copy(const char *name, size_t size, void *value); static int H5P__dxfr_dset_io_hyp_sel_cmp(const void *value1, const void *value2, size_t size); static herr_t H5P__dxfr_dset_io_hyp_sel_close(const char *name, size_t size, void *value); +static herr_t H5P__dxfr_selection_io_mode_enc(const void *value, void **pp, size_t *size); +static herr_t H5P__dxfr_selection_io_mode_dec(const void **pp, void *value); +static herr_t H5P__dxfr_modify_write_buf_enc(const void *value, void **pp, size_t *size); +static herr_t H5P__dxfr_modify_write_buf_dec(const void **pp, void *value); /*********************/ /* Package Variables */ @@ -277,6 +294,9 @@ static const H5T_conv_cb_t H5D_def_conv_cb_g = static const void *H5D_def_xfer_xform_g = H5D_XFER_XFORM_DEF; /* Default value for data transform */ static const H5S_t *H5D_def_dset_io_sel_g = H5D_XFER_DSET_IO_SEL_DEF; /* Default value for dataset I/O selection */ +static const H5D_selection_io_mode_t H5D_def_selection_io_mode_g = H5D_XFER_SELECTION_IO_MODE_DEF; +static const uint32_t H5D_def_no_selection_io_cause_g = H5D_XFER_NO_SELECTION_IO_CAUSE_DEF; +static const hbool_t H5D_def_modify_write_buf_g = H5D_XFER_MODIFY_WRITE_BUF_DEF; /*------------------------------------------------------------------------- * Function: H5P__dxfr_reg_prop @@ -441,6 +461,24 @@ H5P__dxfr_reg_prop(H5P_genclass_t *pclass) H5D_XFER_DSET_IO_SEL_CLOSE) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + if (H5P__register_real(pclass, H5D_XFER_SELECTION_IO_MODE_NAME, H5D_XFER_SELECTION_IO_MODE_SIZE, + &H5D_def_selection_io_mode_g, NULL, NULL, NULL, H5D_XFER_SELECTION_IO_MODE_ENC, + H5D_XFER_SELECTION_IO_MODE_DEC, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + + /* Register the cause of no selection I/O property */ + /* (Note: this property should not have an encode/decode callback) */ + if (H5P__register_real(pclass, H5D_XFER_NO_SELECTION_IO_CAUSE_NAME, H5D_XFER_NO_SELECTION_IO_CAUSE_SIZE, + &H5D_def_no_selection_io_cause_g, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + + /* Register the modify write buffer property */ + if (H5P__register_real(pclass, H5D_XFER_MODIFY_WRITE_BUF_NAME, H5D_XFER_MODIFY_WRITE_BUF_SIZE, + &H5D_def_modify_write_buf_g, NULL, NULL, NULL, H5D_XFER_MODIFY_WRITE_BUF_ENC, + H5D_XFER_MODIFY_WRITE_BUF_DEC, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P__dxfr_reg_prop() */ @@ -2259,6 +2297,78 @@ done: } /* end H5P__dxfr_dset_io_hyp_sel_close() */ /*------------------------------------------------------------------------- + * Function: H5P__dxfr_selection_io_mode_enc + * + * Purpose: Callback routine which is called whenever the selection + * I/O mode property in the dataset transfer property list + * is encoded. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Vailin Choi + * Feb 2023 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__dxfr_selection_io_mode_enc(const void *value, void **_pp, size_t *size) +{ + const H5D_selection_io_mode_t *select_io_mode = + (const H5D_selection_io_mode_t *)value; /* Create local alias for values */ + uint8_t **pp = (uint8_t **)_pp; + + FUNC_ENTER_PACKAGE_NOERR + + /* Sanity check */ + HDassert(select_io_mode); + HDassert(size); + + if (NULL != *pp) + /* Encode selection I/O mode property */ + *(*pp)++ = (uint8_t)*select_io_mode; + + /* Size of selection I/O mode property */ + (*size)++; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P__dxfr_selection_io_mode_enc() */ + +/*------------------------------------------------------------------------- + * Function: H5P__dxfr_selection_io_mode_dec + * + * Purpose: Callback routine which is called whenever the selection + * I/O mode property in the dataset transfer property list + * is decoded. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Vailin Choi + * Feb 2023 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__dxfr_selection_io_mode_dec(const void **_pp, void *_value) +{ + H5D_selection_io_mode_t *select_io_mode = (H5D_selection_io_mode_t *)_value; /* Selection I/O mode */ + const uint8_t **pp = (const uint8_t **)_pp; + + FUNC_ENTER_PACKAGE_NOERR + + /* Sanity checks */ + HDassert(pp); + HDassert(*pp); + HDassert(select_io_mode); + + /* Decode selection I/O mode property */ + *select_io_mode = (H5D_selection_io_mode_t) * (*pp)++; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P__dxfr_selection_io_dec() */ + +/*------------------------------------------------------------------------- * Function: H5Pset_dataset_io_hyperslab_selection * * Purpose: H5Pset_dataset_io_hyperslab_selection() is designed to be used @@ -2386,3 +2496,248 @@ done: FUNC_LEAVE_API(ret_value) } /* end H5Pset_dataset_io_hyperslab_selection() */ + +/*------------------------------------------------------------------------- + * Function: H5Pset_selection_io + * + * Purpose: To set the selection I/O mode in the dataset + * transfer property list. + * + * Note: The library may not perform selection I/O as it asks for if + * the layout callback determines that it is not feasible to do so. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Vailin Choi + * March 5, 2023 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_selection_io(hid_t plist_id, H5D_selection_io_mode_t selection_io_mode) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "iDC", plist_id, selection_io_mode); + + /* Check arguments */ + if (plist_id == H5P_DEFAULT) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "can't set values in default property list") + + if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER))) + HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a dxpl") + + /* Set the selection I/O mode */ + if (H5P_set(plist, H5D_XFER_SELECTION_IO_MODE_NAME, &selection_io_mode) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set value") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_selection_io() */ + +/*------------------------------------------------------------------------- + * Function: H5Pget_selection_io + * + * Purpose: To retrieve the selection I/O mode that is set in + * the dataset transfer property list. + * + * Note: The library may not perform selection I/O as it asks for if + * the layout callback determines that it is not feasible to do so. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Vailin Choi + * March 5, 2023 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_selection_io(hid_t plist_id, H5D_selection_io_mode_t *selection_io_mode /*out*/) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*DC", plist_id, selection_io_mode); + + /* Check arguments */ + if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER))) + HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a dxpl") + + /* Get the selection I/O mode */ + if (selection_io_mode) + if (H5P_get(plist, H5D_XFER_SELECTION_IO_MODE_NAME, selection_io_mode) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get value") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_selection_io() */ + +/*------------------------------------------------------------------------- + * Function: H5Pget_no_selection_io_cause + * + * Purpose: Retrieves causes for not performing selection I/O + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi + * April 17, 2023 + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_no_selection_io_cause(hid_t plist_id, uint32_t *no_selection_io_cause /*out*/) +{ + H5P_genplist_t *plist; + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "ix", plist_id, no_selection_io_cause); + + /* Get the plist structure */ + if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER))) + HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID") + + /* Return values */ + if (no_selection_io_cause) + if (H5P_get(plist, H5D_XFER_NO_SELECTION_IO_CAUSE_NAME, no_selection_io_cause) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get no_selection_io_cause value") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_no_selection_io_cause() */ + +/*------------------------------------------------------------------------- + * Function: H5P__dxfr_modify_write_buf_enc + * + * Purpose: Callback routine which is called whenever the modify write + * buffer property in the dataset transfer property list is + * encoded. + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__dxfr_modify_write_buf_enc(const void *value, void **_pp /*out*/, size_t *size /*out*/) +{ + const hbool_t *modify_write_buf = (const hbool_t *)value; /* Create local alias for values */ + uint8_t **pp = (uint8_t **)_pp; + + FUNC_ENTER_PACKAGE_NOERR + + /* Sanity check */ + HDassert(modify_write_buf); + HDassert(size); + + if (NULL != *pp) + /* Encode modify write buf property. Use "!!" so we always get 0 or 1 */ + *(*pp)++ = (uint8_t)(!!(*modify_write_buf)); + + /* Size of modify write buf property */ + (*size)++; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P__dxfr_modify_write_buf_enc() */ + +/*------------------------------------------------------------------------- + * Function: H5P__dxfr_modify_write_buf_dec + * + * Purpose: Callback routine which is called whenever the modify write + * buffer property in the dataset transfer property list is + * decoded. + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__dxfr_modify_write_buf_dec(const void **_pp, void *_value /*out*/) +{ + hbool_t *modify_write_buf = (hbool_t *)_value; /* Modify write buffer */ + const uint8_t **pp = (const uint8_t **)_pp; + + FUNC_ENTER_PACKAGE_NOERR + + /* Sanity checks */ + HDassert(pp); + HDassert(*pp); + HDassert(modify_write_buf); + + /* Decode selection I/O mode property */ + *modify_write_buf = (hbool_t) * (*pp)++; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P__dxfr_modify_write_buf_dec() */ + +/*------------------------------------------------------------------------- + * Function: H5Pset_modify_write_buf + * + * Purpose: Allows the library to modify the contents of the write + * buffer + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_modify_write_buf(hid_t plist_id, hbool_t modify_write_buf) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "ib", plist_id, modify_write_buf); + + /* Check arguments */ + if (plist_id == H5P_DEFAULT) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "can't set values in default property list") + + if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER))) + HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a dxpl") + + /* Set the selection I/O mode */ + if (H5P_set(plist, H5D_XFER_MODIFY_WRITE_BUF_NAME, &modify_write_buf) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set value") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_modify_write_buf() */ + +/*------------------------------------------------------------------------- + * Function: H5Pget_modify_write_buf + * + * Purpose: Retrieves the "modify write buffer" property + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_modify_write_buf(hid_t plist_id, hbool_t *modify_write_buf /*out*/) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*b", plist_id, modify_write_buf); + + /* Check arguments */ + if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER))) + HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a dxpl") + + /* Get the selection I/O mode */ + if (modify_write_buf) + if (H5P_get(plist, H5D_XFER_MODIFY_WRITE_BUF_NAME, modify_write_buf) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get value") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_modify_write_buf() */ diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index bb5d421..a08119d 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -328,7 +328,6 @@ typedef enum H5D_mpio_actual_io_mode_t { H5D_MPIO_CHUNK_MIXED = 0x1 | 0x2, /**< HDF5 performed one the chunk collective optimization schemes and some chunks were accessed independently, some collectively. */ - /** \internal The contiguous case is separate from the bit field. */ H5D_MPIO_CONTIGUOUS_COLLECTIVE = 0x4 /**< Collective I/O was performed on a contiguous dataset */ } H5D_mpio_actual_io_mode_t; @@ -344,7 +343,8 @@ typedef enum H5D_mpio_no_collective_cause_t { H5D_MPIO_SET_INDEPENDENT = 0x01, /**< Collective I/O was not performed because independent I/O was requested */ H5D_MPIO_DATATYPE_CONVERSION = 0x02, - /**< Collective I/O was not performed because datatype conversions were required */ + /**< Collective I/O was not performed because datatype conversions were required and selection I/O was not + possible (see below) */ H5D_MPIO_DATA_TRANSFORMS = 0x04, /**< Collective I/O was not performed because data transforms needed to be applied */ H5D_MPIO_MPI_OPT_TYPES_ENV_VAR_DISABLED = 0x08, @@ -357,11 +357,72 @@ typedef enum H5D_mpio_no_collective_cause_t { /**< Collective I/O was not performed because parallel filtered writes are disabled */ H5D_MPIO_ERROR_WHILE_CHECKING_COLLECTIVE_POSSIBLE = 0x80, /**< Error */ - H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE = 0x100 + H5D_MPIO_NO_SELECTION_IO = 0x100, + /**< Collective I/O would be supported by selection or vector I/O but that feature was disabled + (see causes via H5Pget_no_selection_io_cause()) */ + H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE = 0x200 /**< Sentinel */ } H5D_mpio_no_collective_cause_t; //! <!-- [H5D_mpio_no_collective_cause_t_snip] --> +/** + * Causes for H5Pget_no_selection_io_cause() property + */ +#define H5D_SEL_IO_DISABLE_BY_API \ + (0x0001u) /**< Selection I/O was not performed because \ + the feature was disabled by the API */ +#define H5D_SEL_IO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET \ + (0x0002u) /**< Selection I/O was not performed because the \ + dataset was neither contiguous nor chunked */ +#define H5D_SEL_IO_CONTIGUOUS_SIEVE_BUFFER \ + (0x0004u) /**< Selection I/O was not performed because of \ + sieve buffer for contiguous dataset */ +#define H5D_SEL_IO_NO_VECTOR_OR_SELECTION_IO_CB \ + (0x0008u) /**< Selection I/O was not performed because the VFD \ + does not have vector or selection I/O callback */ +#define H5D_SEL_IO_PAGE_BUFFER \ + (0x0010u) /**< Selection I/O was not performed because of \ + page buffer */ +#define H5D_SEL_IO_DATASET_FILTER \ + (0x0020u) /**< Selection I/O was not performed because of \ + dataset filters */ +#define H5D_SEL_IO_CHUNK_CACHE \ + (0x0040u) /**< Selection I/O was not performed because of \ + chunk cache */ +#define H5D_SEL_IO_TCONV_BUF_TOO_SMALL \ + (0x0080u) /**< Selection I/O was not performed because the \ + type conversion buffer is too small */ +#define H5D_SEL_IO_BKG_BUF_TOO_SMALL \ + (0x0100u) /**< Selection I/O was not performed because the \ + type conversion background buffer is too small */ +#define H5D_SEL_IO_DEFAULT_OFF \ + (0x0200u) /**< Selection I/O was not performed because the \ + selection I/O mode is DEFAULT and the library \ + chose it to be off for this case */ + +/* Causes for H5D_MPIO_NO_SELECTION_IO */ +#define H5D_MPIO_NO_SELECTION_IO_CAUSES \ + (H5D_SEL_IO_DISABLE_BY_API | H5D_SEL_IO_TCONV_BUF_TOO_SMALL | H5D_SEL_IO_BKG_BUF_TOO_SMALL | \ + H5D_SEL_IO_DATASET_FILTER | H5D_SEL_IO_CHUNK_CACHE) + +//! <!--[H5D_selection_io_mode_t_snip] --> +/** + * Selection I/O mode property + * + * \details The default value, #H5D_SELECTION_IO_MODE_DEFAULT, + * indicates selection I/O can be ON or OFF as + * determined by library internal. + */ +typedef enum H5D_selection_io_mode_t { + H5D_SELECTION_IO_MODE_DEFAULT = 0, + /**< Default selection I/O mode. */ + H5D_SELECTION_IO_MODE_OFF, + /**< Selection I/O is off. */ + H5D_SELECTION_IO_MODE_ON + /**< Selection I/O is on. */ +} H5D_selection_io_mode_t; +//! <!--[H5D_selection_io_mode_t_snip] --> + /********************/ /* Public Variables */ /********************/ @@ -8205,6 +8266,191 @@ H5_DLL herr_t H5Pset_dataset_io_hyperslab_selection(hid_t plist_id, unsigned ran const hsize_t count[], const hsize_t block[]); /** + * + * \ingroup DXPL + * + * \brief Sets the selection I/O mode + * + * \dxpl_id{plist_id} + * \param[in] selection_io_mode The selection I/O mode to be set + * + * \return \herr_t + * + * \details H5Pset_selection_io() sets the selection I/O mode + * \p selection_io_mode in the dataset transfer property + * list \p plist_id. + * + * This can be used to enable collective I/O with type conversion, or + * with custom VFDs that support vector or selection I/O. + * + * Values that can be set in \p selection_io_mode: + * \snippet this H5D_selection_io_mode_t_snip + * \click4more + * + * \note The library may not perform selection I/O as it asks for if the + * layout callback determines that it is not feasible to do so. Please + * refer to H5Pget_no_selection_io_cause() for details. + * + * When used with type conversion, selection I/O requires the type + * conversion buffer (and the background buffer if applicable) be large + * enough to hold the entirety of the data involved in the I/O. For + * read operations, the library will use the application's read buffer + * as the type conversion buffer if the memory type is not smaller than + * the file type, eliminating the need for a separate type conversion + * buffer (a background buffer may still be required). For write + * operations, the library will similarly use the write buffer as a + * type conversion buffer, but only if H5Pset_modify_write_buf() is + * used to allow the library to modify the contents of the write + * buffer. + * + * \since 1.14.1 + * + */ +H5_DLL herr_t H5Pset_selection_io(hid_t plist_id, H5D_selection_io_mode_t selection_io_mode); + +/** + * + * \ingroup DXPL + * + * \brief Retrieves the selection I/O mode + * + * \dxpl_id{plist_id} + * \param[out] selection_io_mode The selection I/O mode + * + * \return \herr_t + * + * \details H5Pget_selection_io() queries the selection I/O mode set in + * in the dataset transfer property list \p plist_id. + * + * Values returned in \p selection_io_mode: + * \snippet this H5D_selection_io_mode_t_snip + * \click4more + * + * \note The library may not perform selection I/O as it asks for if the + * layout callback determines that it is not feasible to do so. Please + * refer to H5Pget_no_selection_io_cause() for details. + * + * \since 1.14.1 + * + */ +H5_DLL herr_t H5Pget_selection_io(hid_t plist_id, H5D_selection_io_mode_t *selection_io_mode); + +/** + * \ingroup DXPL + * + * \brief Retrieves the cause for not performing selection or vector I/O on the + * last parallel I/O call + * + * \dxpl_id{plist_id} + * \param[out] no_selection_io_cause A bitwise set value indicating the relevant + * causes that prevented selection I/O from + * being performed + * \return \herr_t + * + * \par Motivation: + * A user can request selection I/O to be performed via a data transfer + * property list (DXPL). This can be used to enable collective I/O with + * type conversion, or with custom VFDs that support vector or selection + * I/O. However, there are conditions that can cause HDF5 to forgo + * selection or vector I/O and perform legacy (scalar) I/O instead. + * + * \details H5Pget_no_selection_io_cause() can be used to determine whether + * selection or vector I/O was applied for the last preceding I/O call. + * If selection or vector I/O was not used, this function retrieves the + * cause(s) that prevent selection or vector I/O to be performed on + * that I/O call. The properties retrieved by this function are set + * before I/O takes place and are retained even when I/O fails. + * + * If a selection I/O request falls back to vector I/O, that is not + * considered "breaking" selection I/O by this function, since vector + * I/O still passes all information to the file driver in a single + * callback. + * + * Valid values returned in \p no_selection_io_cause are listed + * as follows. If there are multiple causes, it is a bitwise OR of + * the relevant causes. + * + * - #H5D_SEL_IO_DISABLE_BY_API + * Selection I/O was not performed because the feature was disabled by the API + * - #H5D_SEL_IO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET + * Selection I/O was not performed because the dataset was neither contiguous nor chunked + * - #H5D_SEL_IO_CONTIGUOUS_SIEVE_BUFFER + * Selection I/O was not performed because of sieve buffer for contiguous dataset + * - #H5D_SEL_IO_NO_VECTOR_OR_SELECTION_IO_CB + * Selection I/O was not performed because the VFD does not have vector or selection I/O callback + * - #H5D_SEL_IO_PAGE_BUFFER + * Selection I/O was not performed because of page buffer + * - #H5D_SEL_IO_DATASET_FILTER + * Selection I/O was not performed because of dataset filters + * - #H5D_SEL_IO_CHUNK_CACHE + * Selection I/O was not performed because of chunk cache + * - #H5D_SEL_IO_TCONV_BUF_TOO_SMALL + * Selection I/O was not performed because the type conversion buffer is too small + * - #H5D_SEL_IO_BKG_BUF_TOO_SMALL + * Selection I/O was not performed because the type conversion background buffer is too small + * - #H5D_SEL_IO_DEFAULT_OFF + * Selection I/O was not performed because the selection I/O mode is DEFAULT and the library chose it + * to be off for this case + * + * \since 1.14.1 + * + */ +H5_DLL herr_t H5Pget_no_selection_io_cause(hid_t plist_id, uint32_t *no_selection_io_cause); + +/** + * + * \ingroup DXPL + * + * \brief Allows the library to modify the contents of the write buffer + * + * \dxpl_id{plist_id} + * \param[in] modify_write_buf Whether the library can modify the contents of the write buffer + * + * \return \herr_t + * + * \details H5Pset_modify_write_buf() sets whether the library is allowed to + * modify the contents of write buffers passed to HDF5 API routines + * that are passed the dataset transfer property list \p plist_id. The + * default value for modify_write_buf is FALSE. + * + * This function can be used to allow the library to perform in-place + * type conversion on write operations to save memory space. This is + * currently only used for selection I/O operations, which are used for + * collective I/O with type conversion. After making an API call with + * this parameter set to TRUE, the contents of the write buffer are + * undefined. + * + * \note When modify_write_buf is set to TRUE the library may violate the + * const qualifier on the API parameter for the write buffer. + * + * \since 1.14.1 + * + */ +H5_DLL herr_t H5Pset_modify_write_buf(hid_t plist_id, hbool_t modify_write_buf); + +/** + * + * \ingroup DXPL + * + * \brief Retrieves the "modify write buffer" property + * + * \dxpl_id{plist_id} + * \param[out] modify_write_buf Whether the library can modify the contents of the write buffer + * + * \return \herr_t + * + * \details H5Pget_modify_write_buf() gets the "modify write buffer" property + * from the dataset transfer property list \p plist_id. This property + * determines whether the library is allowed to modify the contents of + * write buffers passed to HDF5 API routines that are passed + * \p plist_id. The default value for modify_write_buf is FALSE. + * + * \since 1.14.1 + * + */ +H5_DLL herr_t H5Pget_modify_write_buf(hid_t plist_id, hbool_t *modify_write_buf); + +/** * \ingroup LCPL * * \brief Determines whether property is set to enable creating missing diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h index 19127b3..4303eee 100644 --- a/src/H5Sprivate.h +++ b/src/H5Sprivate.h @@ -260,6 +260,7 @@ H5_DLL herr_t H5S_select_project_simple(const H5S_t *space, H5S_t *new_space, H5_DLL herr_t H5S_select_project_intersection(H5S_t *src_space, H5S_t *dst_space, H5S_t *src_intersect_space, H5S_t **new_space_ptr, hbool_t share_space); H5_DLL herr_t H5S_select_subtract(H5S_t *space, H5S_t *subtract_space); +H5_DLL herr_t H5S_select_contig_block(H5S_t *space, hbool_t *is_contig, hsize_t *off, size_t *len); /* Operations on all selections */ H5_DLL herr_t H5S_select_all(H5S_t *space, hbool_t rel_prev); diff --git a/src/H5Spublic.h b/src/H5Spublic.h index 871a8e7..bd5a82c 100644 --- a/src/H5Spublic.h +++ b/src/H5Spublic.h @@ -864,9 +864,9 @@ H5_DLL hid_t H5Ssel_iter_create(hid_t spaceid, size_t elmt_size, unsigned flags) * * \space_id{sel_iter_id} * \param[in] maxseq Maximum number of sequences to retrieve - * \param[in] maxbytes Maximum number of bytes to retrieve in sequences + * \param[in] maxelmts Maximum number of elements to retrieve in sequences * \param[out] nseq Number of sequences retrieved - * \param[out] nbytes Number of bytes retrieved, in all sequences + * \param[out] nelmts Number of elements retrieved, in all sequences * \param[out] off Array of sequence offsets * \param[out] len Array of sequence lengths * @@ -883,9 +883,9 @@ H5_DLL hid_t H5Ssel_iter_create(hid_t spaceid, size_t elmt_size, unsigned flags) * #H5S_SEL_ITER_GET_SEQ_LIST_SORTED flag is passed to * H5Ssel_iter_create() for a point selection. * - * \p maxseq and \p maxbytes specify the most sequences or bytes + * \p maxseq and \p maxelmts specify the most sequences or elements * possible to place into the \p off and \p len arrays. \p nseq and - * \p nbytes return the actual number of sequences and bytes put + * \p nelmts return the actual number of sequences and elements put * into the arrays. * * Each call to H5Ssel_iter_get_seq_list() will retrieve the next @@ -897,13 +897,13 @@ H5_DLL hid_t H5Ssel_iter_create(hid_t spaceid, size_t elmt_size, unsigned flags) * the iterator was created from (which can be retrieved with * H5Sget_select_npoints(). When there are no further sequences of * elements to retrieve, calls to this routine will set \p nseq - * and \p nbytes to zero. + * and \p nelmts to zero. * * \since 1.12.0 * */ -H5_DLL herr_t H5Ssel_iter_get_seq_list(hid_t sel_iter_id, size_t maxseq, size_t maxbytes, size_t *nseq, - size_t *nbytes, hsize_t *off, size_t *len); +H5_DLL herr_t H5Ssel_iter_get_seq_list(hid_t sel_iter_id, size_t maxseq, size_t maxelmts, size_t *nseq, + size_t *nelmts, hsize_t *off, size_t *len); /** * \ingroup H5S * diff --git a/src/H5Sselect.c b/src/H5Sselect.c index 9d13cf2..02889f7 100644 --- a/src/H5Sselect.c +++ b/src/H5Sselect.c @@ -2865,9 +2865,9 @@ done: herr_t H5Ssel_iter_get_seq_list(sel_iter_id, maxseq, maxbytes, nseq, nbytes, off, len) hid_t sel_iter_id; IN: ID of the dataspace selection iterator to retrieve sequence from size_t maxseq; IN: Max. # of sequences to retrieve - size_t maxbytes; IN: Max. # of bytes to retrieve in sequences + size_t maxelmts; IN: Max. # of elements to retrieve in sequences size_t *nseq; OUT: # of sequences retrieved - size_t *nbytes; OUT: # of bytes retrieved, in all sequences + size_t *nelmts; OUT: # of elements retrieved, in all sequences hsize_t *off; OUT: Array of sequence offsets size_t *len; OUT: Array of sequence lengths RETURNS @@ -2882,8 +2882,8 @@ done: selections is "in order selected", unless the H5S_SEL_ITER_GET_SEQ_LIST_SORTED flag is passed to H5Sset_iter_create for a point selection. - MAXSEQ and MAXBYTES specify the most sequences or bytes possible to - place into the OFF and LEN arrays. *NSEQ and *NBYTES return the actual + MAXSEQ and MAXELMTS specify the most sequences or bytes possible to + place into the OFF and LEN arrays. *NSEQ and *NELMTS return the actual number of sequences and bytes put into the arrays. Each call to H5Ssel_iter_get_seq_list() will retrieve the next set @@ -2894,7 +2894,7 @@ done: of elements selected in the dataspace the iterator was created from (which can be retrieved with H5Sget_select_npoints). When there are no further sequences of elements to retrieve, calls to this routine will - set *NSEQ and *NBYTES to zero. + set *NSEQ and *NELMTS to zero. PROGRAMMER Quincey Koziol - February 11, 2019 GLOBAL VARIABLES @@ -2903,21 +2903,21 @@ done: REVISION LOG --------------------------------------------------------------------------*/ herr_t -H5Ssel_iter_get_seq_list(hid_t sel_iter_id, size_t maxseq, size_t maxbytes, size_t *nseq /*out*/, - size_t *nbytes /*out*/, hsize_t *off /*out*/, size_t *len /*out*/) +H5Ssel_iter_get_seq_list(hid_t sel_iter_id, size_t maxseq, size_t maxelmts, size_t *nseq /*out*/, + size_t *nelmts /*out*/, hsize_t *off /*out*/, size_t *len /*out*/) { H5S_sel_iter_t *sel_iter; /* Dataspace selection iterator to operate on */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE7("e", "izzxxxx", sel_iter_id, maxseq, maxbytes, nseq, nbytes, off, len); + H5TRACE7("e", "izzxxxx", sel_iter_id, maxseq, maxelmts, nseq, nelmts, off, len); /* Check args */ if (NULL == (sel_iter = (H5S_sel_iter_t *)H5I_object_verify(sel_iter_id, H5I_SPACE_SEL_ITER))) HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "not a dataspace selection iterator") if (NULL == nseq) HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "'nseq' pointer is NULL") - if (NULL == nbytes) + if (NULL == nelmts) HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "'nbytes' pointer is NULL") if (NULL == off) HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "offset array pointer is NULL") @@ -2925,12 +2925,12 @@ H5Ssel_iter_get_seq_list(hid_t sel_iter_id, size_t maxseq, size_t maxbytes, size HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "length array pointer is NULL") /* Get the sequences of bytes */ - if (maxseq > 0 && maxbytes > 0 && sel_iter->elmt_left > 0) { - if (H5S_SELECT_ITER_GET_SEQ_LIST(sel_iter, maxseq, maxbytes, nseq, nbytes, off, len) < 0) + if (maxseq > 0 && maxelmts > 0 && sel_iter->elmt_left > 0) { + if (H5S_SELECT_ITER_GET_SEQ_LIST(sel_iter, maxseq, maxelmts, nseq, nelmts, off, len) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "sequence length generation failed") } /* end if */ else - *nseq = *nbytes = 0; + *nseq = *nelmts = 0; done: FUNC_LEAVE_API(ret_value) @@ -2938,6 +2938,83 @@ done: /*-------------------------------------------------------------------------- NAME + H5S_select_contig_block + + PURPOSE + Determines if a selection is a single contiguous block, and returns the + offset and length (in elements) if it is + + USAGE + herr_t H5S_select_contig_block(space, is_contig, off, len) + H5S_t *space; IN: Selection to check + hbool_t *is_contig; OUT: Whether the selection is contiguous + hsize_t *off; OUT: Offset of selection + size_t *len; OUT: Length of selection + + RETURNS + Non-negative on success/Negative on failure. + + DESCRIPTION + Determines if a selection is a single contiguous block, and returns the + offset and length (in elements) if it is. + + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +herr_t +H5S_select_contig_block(H5S_t *space, hbool_t *is_contig, hsize_t *off, size_t *len) +{ + H5S_sel_iter_t *iter = NULL; /* Selection iterator */ + hbool_t iter_init = FALSE; /* Selection iteration info has been initialized */ + size_t nseq_tmp; + size_t nelem_tmp; + hsize_t sel_off; + size_t sel_len; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(space); + + /* Allocate and initialize the iterator */ + if (NULL == (iter = H5FL_MALLOC(H5S_sel_iter_t))) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate iterator") + if (H5S_select_iter_init(iter, space, 1, 0) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to initialize memory selection information") + iter_init = TRUE; + + /* Get list of sequences for selection, to check if it is contiguous */ + if (H5S_SELECT_ITER_GET_SEQ_LIST(iter, (size_t)1, (size_t)-1, &nseq_tmp, &nelem_tmp, &sel_off, &sel_len) < + 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTNEXT, FAIL, "sequence length generation failed") + + /* If the first sequence includes all the elements selected in this piece, it it contiguous */ + H5_CHECK_OVERFLOW(space->select.num_elem, hsize_t, size_t); + if (sel_len == (size_t)space->select.num_elem) { + if (is_contig) + *is_contig = TRUE; + if (off) + *off = sel_off; + if (len) + *len = sel_len; + } + else if (is_contig) + *is_contig = FALSE; + +done: + if (iter_init && H5S_SELECT_ITER_RELEASE(iter) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't release selection iterator") + if (iter) + iter = H5FL_FREE(H5S_sel_iter_t, iter); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5S_select_contig_block() */ + +/*-------------------------------------------------------------------------- + NAME H5Ssel_iter_reset PURPOSE Resets a dataspace selection iterator back to an initial state. diff --git a/src/H5private.h b/src/H5private.h index 70aed8f..eac2cba 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -1965,14 +1965,6 @@ extern hbool_t H5_libterm_g; /* Is the library being shutdown? */ #endif /* H5_HAVE_THREADSAFE */ -/* Extern global to determine if we should use selection I/O if available (this - * variable should be removed once selection I/O performs as well as the - * previous scalar I/O implementation - * - * NOTE: Must be exposed via H5_DLLVAR so parallel tests pass on Windows. - */ -H5_DLLVAR hbool_t H5_use_selection_io_g; - #ifdef H5_HAVE_CODESTACK /* Include required function stack header */ diff --git a/src/H5trace.c b/src/H5trace.c index 3be5b91..48e94a6 100644 --- a/src/H5trace.c +++ b/src/H5trace.c @@ -673,6 +673,31 @@ H5_trace_args(H5RS_str_t *rs, const char *type, va_list ap) } /* end block */ break; + case 'C': /* H5D_selection_io_mode_t */ + { + H5D_selection_io_mode_t selection_io_mode = + (H5D_selection_io_mode_t)HDva_arg(ap, int); + + switch (selection_io_mode) { + case H5D_SELECTION_IO_MODE_DEFAULT: + H5RS_acat(rs, "H5D_SELECTION_IO_MODE_DEFAULT"); + break; + + case H5D_SELECTION_IO_MODE_OFF: + H5RS_acat(rs, "H5D_SELECTION_IO_MODE_OFF"); + break; + + case H5D_SELECTION_IO_MODE_ON: + H5RS_acat(rs, "H5D_SELECTION_IO_MODE_ON"); + break; + + default: + H5RS_asprintf_cat(rs, "%ld", (long)selection_io_mode); + break; + } /* end switch */ + } /* end block */ + break; + case 'f': /* H5D_fill_time_t */ { H5D_fill_time_t fill_time = (H5D_fill_time_t)HDva_arg(ap, int); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index da98f15..c3365b7 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -337,6 +337,7 @@ set (H5_TESTS page_buffer dtypes dsets + select_io_dset chunk_info # compression lib link cmpd_dset mdset diff --git a/test/Makefile.am b/test/Makefile.am index bad52c8..291907c 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -65,7 +65,7 @@ TEST_PROG= testhdf5 \ accum hyperslab istore bittests dt_arith page_buffer \ dtypes dsets chunk_info cmpd_dset mdset cmpd_dtransform filter_fail extend direct_chunk \ external efc objcopy objcopy_ref links unlink twriteorder big mtime \ - fillval mount \ + fillval mount select_io_dset\ flush1 flush2 app_ref enum set_extent ttsafe enc_dec_plist \ enc_dec_plist_cross_platform getname vfd ros3 s3comms hdfs ntypes \ dangle dtransform reserved cross_read freespace mf vds file_image \ diff --git a/test/enc_dec_plist.c b/test/enc_dec_plist.c index fd4ae5e..5b75178 100644 --- a/test/enc_dec_plist.c +++ b/test/enc_dec_plist.c @@ -367,6 +367,12 @@ main(void) if ((H5Pset_data_transform(dxpl, c_to_f)) < 0) FAIL_STACK_ERROR; + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_ON) < 0) + FAIL_STACK_ERROR; + + if (H5Pset_modify_write_buf(dxpl, TRUE) < 0) + FAIL_STACK_ERROR; + /* Test encoding & decoding property list */ if (test_encode_decode(dxpl, low, high) < 0) FAIL_PUTS_ERROR("DXPL encoding/decoding failed\n"); diff --git a/test/gen_plist.c b/test/gen_plist.c index afe8494..5302dc7 100644 --- a/test/gen_plist.c +++ b/test/gen_plist.c @@ -183,6 +183,11 @@ main(void) assert(ret > 0); if ((ret = H5Pset_data_transform(dxpl1, c_to_f)) < 0) assert(ret > 0); + if ((ret = H5Pset_selection_io(dxpl1, H5D_SELECTION_IO_MODE_ON)) < 0) + assert(ret > 0); + + if ((ret = H5Pset_modify_write_buf(dxpl1, TRUE)) < 0) + assert(ret > 0); if ((ret = encode_plist(dxpl1, little_endian, word_length, "testfiles/plist_files/dxpl_")) < 0) assert(ret > 0); diff --git a/test/select_io_dset.c b/test/select_io_dset.c new file mode 100644 index 0000000..9a1de06 --- /dev/null +++ b/test/select_io_dset.c @@ -0,0 +1,3269 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Programmer: + * + * Purpose: Tests selection IO for the dataset interface (H5D) + */ + +#include "testhdf5.h" +#include "H5srcdir.h" + +const char *FILENAME[] = {"select_io", /* 0 */ + NULL}; + +#define FILENAME_BUF_SIZE 1024 + +/* + * Test configurations + */ +typedef enum { + TEST_NO_TYPE_CONV, /* no type conversion (null case) */ + TEST_NO_SIZE_CHANGE_NO_BKG, /* no size change, no bkg buffer */ + TEST_LARGER_MEM_NO_BKG, /* larger memory type, no bkg buffer */ + TEST_SMALLER_MEM_NO_BKG, /* smaller memory type, no bkg buffer */ + TEST_CMPD_WITH_BKG, /* compound types with bkg buffer */ + TEST_MULTI_CONV_NO_BKG, /* multi dataset test 1 */ + TEST_MULTI_CONV_BKG, /* multi dataset test 2 */ + TEST_MULTI_CONV_SIZE_CHANGE, /* multi dataset test 3 */ + TEST_MULTI_ALL, /* multi dataset test 4 */ + TEST_SELECT_NTESTS +} test_select_config_t; + +#define DSET_SELECT_DIM 100 +#define DSET_SELECT_CHUNK_DIM 10 + +#define MULTI_NUM_DSETS 3 +#define MULTI_MIN_DSETS 3 +#define DSET_NAME_LEN 64 + +/* Compound type */ +typedef struct s1_t { + int a; + int b; + int c; + int d; +} s1_t; + +/* + * Variation of s1 with: + * --no conversion for 2 member types + * --1 larger mem type, + * --1 smaller mem type + */ +typedef struct s2_t { + int a; + long b; + int c; + short d; +} s2_t; + +/* Variation of s1: reverse of s1_t */ +typedef struct s3_t { + int d; + int c; + int b; + int a; +} s3_t; + +/* Variations of s1: only 2 members in s1_t */ +typedef struct s4_t { + unsigned int b; + unsigned int d; +} s4_t; + +/* Defines for test_multi_dsets_all() */ +typedef enum { + DSET_WITH_NO_CONV, /* Dataset with no type conversion */ + DSET_WITH_CONV_AND_NO_BKG, /* Dataset with type conversion but no background buffer */ + DSET_WITH_CONV_AND_BKG, /* Dataset with type conversion and background buffer */ + DSET_NTTYPES +} multi_dset_type_t; + +/* Test setting A and B */ +#define SETTING_A 1 +#define SETTING_B 2 + +/* Definitions of the test modes for test_get_no_selection_io_cause() */ +#define TEST_DISABLE_BY_API 0x001 +#define TEST_DATATYPE_CONVERSION 0x002 +#define TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET 0x004 +#define TEST_CONTIGUOUS_SIEVE_BUFFER 0x008 +#define TEST_NO_VECTOR_OR_SELECTION_IO_CB 0x010 +#define TEST_PAGE_BUFFER 0x020 +#define TEST_DATASET_FILTER 0x040 +#define TEST_CHUNK_CACHE 0x080 +#define TEST_TCONV_BUF_TOO_SMALL 0x100 +#define TEST_IN_PLACE_TCONV 0x200 + +/* + * Case 1: single dataset read/write, no type conversion (null case) + * --create dataset with H5T_NATIVE_INT + * --write/read dataset with H5T_NATIVE_INT + */ +static herr_t +test_no_type_conv(hid_t fid, unsigned chunked, unsigned dtrans, unsigned mwbuf) +{ + int i; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t ntrans_dxpl = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + int wbuf[DSET_SELECT_DIM]; + int wbuf_bak[DSET_SELECT_DIM]; + int trans_wbuf[DSET_SELECT_DIM]; + int rbuf[DSET_SELECT_DIM]; + char dset_name[DSET_NAME_LEN]; + const char *expr = "2*x"; + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + FAIL_STACK_ERROR; + } + + /* Generate dataset name */ + HDsnprintf(dset_name, sizeof(dset_name), "no_tconv_%s_%s_%s", chunked ? "chunked" : "contig", + dtrans ? "xform" : "noxform", mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create dataset */ + if ((did = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* Initialize data */ + for (i = 0; i < DSET_SELECT_DIM; i++) { + wbuf[i] = i; + trans_wbuf[i] = 2 * wbuf[i]; + } + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + FAIL_STACK_ERROR; + + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_ON) < 0) + FAIL_STACK_ERROR; + + /* Set modify write buffer if requested */ + if (mwbuf) + if (H5Pset_modify_write_buf(dxpl, TRUE) < 0) + FAIL_STACK_ERROR; + + if ((ntrans_dxpl = H5Pcopy(dxpl)) < 0) + FAIL_STACK_ERROR; + + /* Set data transform */ + if (dtrans) + if (H5Pset_data_transform(dxpl, expr) < 0) + FAIL_STACK_ERROR; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(wbuf_bak, wbuf, sizeof(wbuf)); + + /* Write data to the dataset with/without data transform */ + if (H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, dxpl, wbuf) < 0) + FAIL_STACK_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(wbuf, wbuf_bak, sizeof(wbuf)); + + /* Read data from the dataset without data transform set in dxpl */ + if (H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, ntrans_dxpl, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify data or transformed data read */ + for (i = 0; i < DSET_SELECT_DIM; i++) + if (rbuf[i] != (dtrans ? trans_wbuf[i] : wbuf[i])) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" At index %d\n", i); + TEST_ERROR; + } + + if (dtrans) { + + /* Read the data from the dataset with data transform set in dxpl */ + if (H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, dxpl, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify data read is transformed a second time */ + for (i = 0; i < DSET_SELECT_DIM; i++) + if (rbuf[i] != (2 * trans_wbuf[i])) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" At index %d\n", i); + TEST_ERROR; + } + } + + if (H5Sclose(sid) < 0) + FAIL_STACK_ERROR; + if (H5Dclose(did) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dxpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(ntrans_dxpl) < 0) + FAIL_STACK_ERROR; + + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Sclose(sid); + H5Dclose(did); + H5Pclose(dcpl); + H5Pclose(dxpl); + H5Pclose(ntrans_dxpl); + } + H5E_END_TRY; + + return FAIL; + +} /* test_no_type_conv() */ + +/* + * Case 2: single dataset read/write, no size change, no background buffer + * --create dataset with H5T_STD_I32BE + * --write/read dataset with H5T_STD_I32LE + * --read again with H5T_STD_I32BE + */ +static herr_t +test_no_size_change_no_bkg(hid_t fid, unsigned chunked, unsigned mwbuf) +{ + int i; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + char *wbuf = NULL; + char *wbuf_bak = NULL; + char *rbuf = NULL; + char dset_name[DSET_NAME_LEN]; + + if ((wbuf = (char *)HDmalloc((size_t)(4 * DSET_SELECT_DIM))) == NULL) + FAIL_STACK_ERROR; + if (mwbuf && (wbuf_bak = (char *)HDmalloc((size_t)(4 * DSET_SELECT_DIM))) == NULL) + FAIL_STACK_ERROR; + if ((rbuf = (char *)HDmalloc((size_t)(4 * DSET_SELECT_DIM))) == NULL) + FAIL_STACK_ERROR; + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + FAIL_STACK_ERROR; + + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_ON) < 0) + FAIL_STACK_ERROR; + + /* Set modify write buffer if requested */ + if (mwbuf) + if (H5Pset_modify_write_buf(dxpl, TRUE) < 0) + FAIL_STACK_ERROR; + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + FAIL_STACK_ERROR; + } + + /* Generate dataset name */ + HDsnprintf(dset_name, sizeof(dset_name), "no_size_change_%s_%s", chunked ? "chunked" : "contig", + mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create 1d dataset */ + if ((did = H5Dcreate2(fid, dset_name, H5T_STD_I32BE, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* Initialize data */ + for (i = 0; i < DSET_SELECT_DIM; i++) { + wbuf[i * 4 + 3] = 0x1; + wbuf[i * 4 + 2] = 0x2; + wbuf[i * 4 + 1] = 0x3; + wbuf[i * 4 + 0] = 0x4; + } + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(wbuf_bak, wbuf, (size_t)(4 * DSET_SELECT_DIM)); + + /* Write the data to the dataset with little endian */ + if (H5Dwrite(did, H5T_STD_I32LE, H5S_ALL, H5S_ALL, dxpl, wbuf) < 0) + FAIL_STACK_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(wbuf, wbuf_bak, (size_t)(4 * DSET_SELECT_DIM)); + + /* Read the data from the dataset with little endian */ + if (H5Dread(did, H5T_STD_I32LE, H5S_ALL, H5S_ALL, dxpl, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify data read little endian */ + for (i = 0; i < DSET_SELECT_DIM; i++) + if (rbuf[4 * i + 0] != wbuf[4 * i + 0] || rbuf[4 * i + 1] != wbuf[4 * i + 1] || + rbuf[4 * i + 2] != wbuf[4 * i + 2] || rbuf[4 * i + 3] != wbuf[4 * i + 3]) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" At index %d\n", i); + TEST_ERROR; + } + + /* Read the data from the dataset with big endian */ + if (H5Dread(did, H5T_STD_I32BE, H5S_ALL, H5S_ALL, dxpl, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify data read in big endian */ + for (i = 0; i < DSET_SELECT_DIM; i++) + if (rbuf[4 * i + 0] != wbuf[4 * i + 3] || rbuf[4 * i + 1] != wbuf[4 * i + 2] || + rbuf[4 * i + 2] != wbuf[4 * i + 1] || rbuf[4 * i + 3] != wbuf[4 * i + 0]) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" At index %d\n", i); + TEST_ERROR; + } + + if (H5Sclose(sid) < 0) + FAIL_STACK_ERROR; + if (H5Dclose(did) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dxpl) < 0) + FAIL_STACK_ERROR; + + HDfree(wbuf); + HDfree(wbuf_bak); + HDfree(rbuf); + + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Sclose(sid); + H5Dclose(did); + H5Pclose(dcpl); + H5Pclose(dxpl); + } + H5E_END_TRY; + + if (wbuf) + HDfree(wbuf); + if (wbuf_bak) + HDfree(wbuf_bak); + if (wbuf) + HDfree(rbuf); + + return FAIL; + +} /* test_no_size_change_no_bkg() */ + +/* + * Case 3: single dataset read/write, larger mem type, no background buffer + * --create dataset with H5T_NATIVE_INT + * --write dataset with H5T_NATIVE_LONG + * --read dataset with H5T_NATIVE_LLONG + * + */ +static herr_t +test_larger_mem_type_no_bkg(hid_t fid, unsigned chunked, unsigned dtrans, unsigned mwbuf) +{ + int i; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t ntrans_dxpl = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + long wbuf[DSET_SELECT_DIM]; + long wbuf_bak[DSET_SELECT_DIM]; + long trans_wbuf[DSET_SELECT_DIM]; + long long rbuf[DSET_SELECT_DIM]; + char dset_name[DSET_NAME_LEN]; + const char *expr = "5 * (10 - x)"; + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + FAIL_STACK_ERROR; + } + + /* Generate dataset name */ + HDsnprintf(dset_name, sizeof(dset_name), "larger_no_bkg_%s_%s_%s", chunked ? "chunked" : "contig", + dtrans ? "xform" : "noxform", mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create dataset */ + if ((did = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* Initialize data */ + for (i = 0; i < DSET_SELECT_DIM; i++) { + wbuf[i] = i; + trans_wbuf[i] = 5 * (10 - wbuf[i]); + } + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + FAIL_STACK_ERROR; + + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_ON) < 0) + FAIL_STACK_ERROR; + + /* Set modify write buffer if requested */ + if (mwbuf) + if (H5Pset_modify_write_buf(dxpl, TRUE) < 0) + FAIL_STACK_ERROR; + + if ((ntrans_dxpl = H5Pcopy(dxpl)) < 0) + FAIL_STACK_ERROR; + + /* Set data transform */ + if (dtrans) + if (H5Pset_data_transform(dxpl, expr) < 0) + FAIL_STACK_ERROR; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(wbuf_bak, wbuf, sizeof(wbuf)); + + /* Write data to the dataset with/without data transform set in dxpl */ + if (H5Dwrite(did, H5T_NATIVE_LONG, H5S_ALL, H5S_ALL, dxpl, wbuf) < 0) + FAIL_STACK_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(wbuf, wbuf_bak, sizeof(wbuf)); + + /* Read the data from the dataset without data transform in dxpl */ + if (H5Dread(did, H5T_NATIVE_LLONG, H5S_ALL, H5S_ALL, ntrans_dxpl, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify data or transformed data read */ + for (i = 0; i < DSET_SELECT_DIM; i++) + if (rbuf[i] != (long long)(dtrans ? trans_wbuf[i] : wbuf[i])) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" At index %d\n", i); + TEST_ERROR; + } + + if (dtrans) { + + /* Read data from the dataset with data transform set in dxpl */ + if (H5Dread(did, H5T_NATIVE_LLONG, H5S_ALL, H5S_ALL, dxpl, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify data read is transformed a second time */ + for (i = 0; i < DSET_SELECT_DIM; i++) + if (rbuf[i] != (long long)(5 * (10 - trans_wbuf[i]))) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" At index %d\n", i); + TEST_ERROR; + } + } + + if (H5Sclose(sid) < 0) + FAIL_STACK_ERROR; + if (H5Dclose(did) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dxpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(ntrans_dxpl) < 0) + FAIL_STACK_ERROR; + + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Sclose(sid); + H5Dclose(did); + H5Pclose(dcpl); + H5Pclose(dxpl); + H5Pclose(ntrans_dxpl); + } + H5E_END_TRY; + + return FAIL; + +} /* test_larger_mem_type_no_bkg() */ + +/* + * Case 4: single dataset reader/write, smaller mem type, no background buffer + * --create dataset with H5T_NATIVE_INT + * --write dataset with H5T_NATIVE_SHORT + * --read dataset with H5T_NATIVE_SHORT + */ +static herr_t +test_smaller_mem_type_no_bkg(hid_t fid, unsigned chunked, unsigned dtrans, unsigned mwbuf) +{ + int i; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t ntrans_dxpl = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + short wbuf[DSET_SELECT_DIM]; + int wbuf_bak[DSET_SELECT_DIM]; + short trans_wbuf[DSET_SELECT_DIM]; + short rbuf[DSET_SELECT_DIM]; + char dset_name[DSET_NAME_LEN]; + const char *expr = "2 * (10 + x)"; + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + FAIL_STACK_ERROR; + } + + /* Generate dataset name */ + HDsnprintf(dset_name, sizeof(dset_name), "smaller_no_bkg_%s_%s_%s", chunked ? "chunked" : "contig", + dtrans ? "xform" : "noxform", mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create 1d chunked dataset with/without data transform */ + if ((did = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* Initialize data */ + for (i = 0; i < DSET_SELECT_DIM; i++) { + wbuf[i] = (short)i; + trans_wbuf[i] = (short)(2 * (10 + wbuf[i])); + } + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + FAIL_STACK_ERROR; + + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_ON) < 0) + FAIL_STACK_ERROR; + + /* Set modify write buffer if requested */ + if (mwbuf) + if (H5Pset_modify_write_buf(dxpl, TRUE) < 0) + FAIL_STACK_ERROR; + + if ((ntrans_dxpl = H5Pcopy(dxpl)) < 0) + FAIL_STACK_ERROR; + + /* Set data transform */ + if (dtrans) { + if (H5Pset_data_transform(dxpl, expr) < 0) + FAIL_STACK_ERROR; + } + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(wbuf_bak, wbuf, sizeof(wbuf)); + + /* Write data to the dataset with/without data transform in dxpl */ + if (H5Dwrite(did, H5T_NATIVE_SHORT, H5S_ALL, H5S_ALL, dxpl, wbuf) < 0) + FAIL_STACK_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(wbuf, wbuf_bak, sizeof(wbuf)); + + /* Read data from the dataset without data transform in dxpl */ + if (H5Dread(did, H5T_NATIVE_SHORT, H5S_ALL, H5S_ALL, ntrans_dxpl, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify data or transformed data read */ + for (i = 0; i < DSET_SELECT_DIM; i++) + if (rbuf[i] != (dtrans ? trans_wbuf[i] : wbuf[i])) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" At index %d\n", i); + TEST_ERROR; + } + + if (dtrans) { + + /* Read data from the dataset with data transform set in dxpl */ + if (H5Dread(did, H5T_NATIVE_SHORT, H5S_ALL, H5S_ALL, dxpl, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify data read is transformed a second time */ + for (i = 0; i < DSET_SELECT_DIM; i++) + if (rbuf[i] != (2 * (10 + trans_wbuf[i]))) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" At index %d\n", i); + TEST_ERROR; + } + } + + if (H5Sclose(sid) < 0) + FAIL_STACK_ERROR; + if (H5Dclose(did) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dxpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(ntrans_dxpl) < 0) + FAIL_STACK_ERROR; + + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Sclose(sid); + H5Dclose(did); + H5Pclose(dcpl); + H5Pclose(dxpl); + H5Pclose(ntrans_dxpl); + } + H5E_END_TRY; + + return FAIL; + +} /* test_smaller_mem_type_no_bkg() */ + +/* + * Case 5: single dataset reade/write, compound types with background buffer + * + * (a) Initialize compound buffer in memory with unique values + * Write all compound fields to disk + * Verify values read + * (b) Update all fields of the compound type in memory write buffer with new unique values + * Write some but not all all compound fields to disk + * Read the entire compound type + * Verify the fields have the correct (old or new) values + * (c) Update all fields of the compound type in memory read buffer with new unique values + * Read some but not all the compound fields to memory + * Verify the fields have the correct (old, middle or new) values + * (d) Set up a different compound type which has: + * --no conversion for member types + * --a field with larger mem type + * --a field with smaller mem type + * Write this compound type to disk + * Read the entire compound type + * Verify the values read + * + */ +static herr_t +test_cmpd_with_bkg(hid_t fid, unsigned chunked, unsigned mwbuf) +{ + int i; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t s1_tid = H5I_INVALID_HID; + hid_t s2_tid = H5I_INVALID_HID; + hid_t ss_ac_tid = H5I_INVALID_HID; + hid_t ss_bc_tid = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + s1_t *s1_wbuf = NULL; + s1_t *s1_wbuf_bak = NULL; + s1_t *s1_rbuf = NULL; + s2_t *s2_wbuf = NULL; + s2_t *s2_wbuf_bak = NULL; + s2_t *s2_rbuf = NULL; + char dset_name[DSET_NAME_LEN]; + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + FAIL_STACK_ERROR; + + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_ON) < 0) + FAIL_STACK_ERROR; + + /* Set modify write buffer if requested */ + if (mwbuf) + if (H5Pset_modify_write_buf(dxpl, TRUE) < 0) + FAIL_STACK_ERROR; + + /* Allocate buffers for datasets */ + if (NULL == (s1_wbuf = (s1_t *)HDmalloc(sizeof(s1_t) * DSET_SELECT_DIM))) + FAIL_STACK_ERROR; + if (mwbuf && NULL == (s1_wbuf_bak = (s1_t *)HDmalloc(sizeof(s1_t) * DSET_SELECT_DIM))) + FAIL_STACK_ERROR; + if (NULL == (s1_rbuf = (s1_t *)HDmalloc(sizeof(s1_t) * DSET_SELECT_DIM))) + FAIL_STACK_ERROR; + if (NULL == (s2_wbuf = (s2_t *)HDmalloc(sizeof(s2_t) * DSET_SELECT_DIM))) + FAIL_STACK_ERROR; + if (mwbuf && NULL == (s2_wbuf_bak = (s2_t *)HDmalloc(sizeof(s2_t) * DSET_SELECT_DIM))) + FAIL_STACK_ERROR; + if (NULL == (s2_rbuf = (s2_t *)HDmalloc(sizeof(s2_t) * DSET_SELECT_DIM))) + FAIL_STACK_ERROR; + + /* Create the memory data type */ + if ((s1_tid = H5Tcreate(H5T_COMPOUND, sizeof(s1_t))) < 0) + FAIL_STACK_ERROR; + + if (H5Tinsert(s1_tid, "a", HOFFSET(s1_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "b", HOFFSET(s1_t, b), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "c", HOFFSET(s1_t, c), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "d", HOFFSET(s1_t, d), H5T_NATIVE_INT) < 0) + FAIL_STACK_ERROR; + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + FAIL_STACK_ERROR; + } + + /* Case 5(a) */ + + /* Generate dataset name */ + HDsnprintf(dset_name, sizeof(dset_name), "cmpd_with_bkg_%s_%s", chunked ? "chunked" : "contig", + mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create 1d dataset */ + if ((did = H5Dcreate2(fid, dset_name, s1_tid, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* Initialize data */ + for (i = 0; i < DSET_SELECT_DIM; i++) { + s1_wbuf[i].a = 4 * i; + s1_wbuf[i].b = (4 * i) + 1; + s1_wbuf[i].c = (4 * i) + 2; + s1_wbuf[i].d = (4 * i) + 3; + } + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(s1_wbuf_bak, s1_wbuf, sizeof(s1_t) * DSET_SELECT_DIM); + + /* Write all the data to the dataset */ + if (H5Dwrite(did, s1_tid, H5S_ALL, H5S_ALL, dxpl, s1_wbuf) < 0) + FAIL_STACK_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(s1_wbuf, s1_wbuf_bak, sizeof(s1_t) * DSET_SELECT_DIM); + + /* Read all the data from the dataset */ + if (H5Dread(did, s1_tid, H5S_ALL, H5S_ALL, dxpl, s1_rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify data read */ + for (i = 0; i < DSET_SELECT_DIM; i++) { + if (s1_wbuf[i].a != s1_rbuf[i].a || s1_wbuf[i].b != s1_rbuf[i].b || s1_wbuf[i].c != s1_rbuf[i].c || + s1_wbuf[i].d != s1_rbuf[i].d) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" At index %d\n", i); + TEST_ERROR; + } + } + + /* Case 5(b) */ + + /* Update s1_wbuf with unique values */ + for (i = 0; i < DSET_SELECT_DIM; i++) { + s1_wbuf[i].a = (4 * i) + DSET_SELECT_DIM; + s1_wbuf[i].b = (4 * i) + DSET_SELECT_DIM + 1; + s1_wbuf[i].c = (4 * i) + DSET_SELECT_DIM + 2; + s1_wbuf[i].d = (4 * i) + DSET_SELECT_DIM + 3; + } + + /* Create a compound type same size as s1_t */ + if ((ss_ac_tid = H5Tcreate(H5T_COMPOUND, sizeof(s1_t))) < 0) + FAIL_STACK_ERROR; + + /* but contains only subset members of s1_t */ + if (H5Tinsert(ss_ac_tid, "a", HOFFSET(s1_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(ss_ac_tid, "c", HOFFSET(s1_t, c), H5T_NATIVE_INT) < 0) + FAIL_STACK_ERROR; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(s1_wbuf_bak, s1_wbuf, sizeof(s1_t) * DSET_SELECT_DIM); + + /* Write s1_wbuf to the dataset with only subset members in ss_tid */ + if (H5Dwrite(did, ss_ac_tid, H5S_ALL, H5S_ALL, dxpl, s1_wbuf) < 0) + FAIL_STACK_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(s1_wbuf, s1_wbuf_bak, sizeof(s1_t) * DSET_SELECT_DIM); + + /* Read the whole compound back */ + if (H5Dread(did, ss_ac_tid, H5S_ALL, H5S_ALL, dxpl, s1_rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify the compound fields have the correct (old or new) values */ + for (i = 0; i < DSET_SELECT_DIM; i++) { + if (s1_rbuf[i].a != s1_wbuf[i].a || s1_rbuf[i].b != ((4 * i) + 1) || s1_rbuf[i].c != s1_wbuf[i].c || + s1_rbuf[i].d != ((4 * i) + 3)) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" At index %d\n", i); + TEST_ERROR; + } + } + + /* Case 5(c) */ + + /* Update s1_rbuf with new unique values */ + for (i = 0; i < DSET_SELECT_DIM; i++) { + s1_rbuf[i].a = (4 * i) + (2 * DSET_SELECT_DIM); + s1_rbuf[i].b = (4 * i) + (2 * DSET_SELECT_DIM) + 1; + s1_rbuf[i].c = (4 * i) + (2 * DSET_SELECT_DIM) + 2; + s1_rbuf[i].d = (4 * i) + (2 * DSET_SELECT_DIM) + 3; + } + + /* Create a compound type same size as s1_t */ + if ((ss_bc_tid = H5Tcreate(H5T_COMPOUND, sizeof(s1_t))) < 0) + FAIL_STACK_ERROR; + + /* but contains only subset members of s1_t */ + if (H5Tinsert(ss_bc_tid, "b", HOFFSET(s1_t, b), H5T_NATIVE_INT) < 0 || + H5Tinsert(ss_bc_tid, "c", HOFFSET(s1_t, c), H5T_NATIVE_INT) < 0) + FAIL_STACK_ERROR; + + /* Read the dataset: will read only what is set in */ + if (H5Dread(did, ss_bc_tid, H5S_ALL, H5S_ALL, dxpl, s1_rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify data read */ + for (i = 0; i < DSET_SELECT_DIM; i++) { + if (s1_rbuf[i].a != ((4 * i) + (2 * DSET_SELECT_DIM)) || s1_rbuf[i].b != ((4 * i) + 1) || + s1_rbuf[i].c != ((4 * i) + DSET_SELECT_DIM + 2) || + s1_rbuf[i].d != ((4 * i) + (2 * DSET_SELECT_DIM) + 3)) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" At index %d\n", i); + TEST_ERROR; + } + } + + /* Case 5(d) */ + + /* Create s2_t compound type with: + * --no conversion for 2 member types, + * --1 larger mem type + * --1 smaller mem type + */ + if ((s2_tid = H5Tcreate(H5T_COMPOUND, sizeof(s2_t))) < 0) + FAIL_STACK_ERROR; + + if (H5Tinsert(s2_tid, "a", HOFFSET(s2_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(s2_tid, "b", HOFFSET(s2_t, b), H5T_NATIVE_LONG) < 0 || + H5Tinsert(s2_tid, "c", HOFFSET(s2_t, c), H5T_NATIVE_INT) < 0 || + H5Tinsert(s2_tid, "d", HOFFSET(s2_t, d), H5T_NATIVE_SHORT) < 0) + FAIL_STACK_ERROR; + + /* Update s2_wbuf with unique values */ + for (i = 0; i < DSET_SELECT_DIM; i++) { + s2_wbuf[i].a = 8 * i; + s2_wbuf[i].b = (long)((8 * i) + 1); + s2_wbuf[i].c = (8 * i) + 2; + s2_wbuf[i].d = (short)((8 * i) + 3); + } + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(s2_wbuf_bak, s2_wbuf, sizeof(s2_t) * DSET_SELECT_DIM); + + if (H5Dwrite(did, s2_tid, H5S_ALL, H5S_ALL, dxpl, s2_wbuf) < 0) + FAIL_STACK_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(s2_wbuf, s2_wbuf_bak, sizeof(s2_t) * DSET_SELECT_DIM); + + /* Read it back */ + if (H5Dread(did, s2_tid, H5S_ALL, H5S_ALL, dxpl, s2_rbuf) < 0) { + goto error; + } + + /* Verify data read */ + for (i = 0; i < DSET_SELECT_DIM; i++) { + if (s2_wbuf[i].a != s2_rbuf[i].a || s2_wbuf[i].b != s2_rbuf[i].b || s2_wbuf[i].c != s2_rbuf[i].c || + s2_wbuf[i].d != s2_rbuf[i].d) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" At index %d\n", i); + TEST_ERROR; + } + } + + if (H5Sclose(sid) < 0) + FAIL_STACK_ERROR; + if (H5Tclose(s1_tid) < 0) + FAIL_STACK_ERROR; + if (H5Tclose(s2_tid) < 0) + FAIL_STACK_ERROR; + if (H5Tclose(ss_ac_tid) < 0) + FAIL_STACK_ERROR; + if (H5Tclose(ss_bc_tid) < 0) + FAIL_STACK_ERROR; + if (H5Dclose(did) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dxpl) < 0) + FAIL_STACK_ERROR; + + /* Release buffers */ + HDfree(s1_wbuf); + HDfree(s1_wbuf_bak); + HDfree(s1_rbuf); + HDfree(s2_wbuf); + HDfree(s2_wbuf_bak); + HDfree(s2_rbuf); + + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Sclose(sid); + H5Tclose(s1_tid); + H5Tclose(s2_tid); + H5Tclose(ss_ac_tid); + H5Tclose(ss_bc_tid); + H5Dclose(did); + H5Pclose(dcpl); + H5Pclose(dxpl); + } + H5E_END_TRY; + + if (s1_wbuf) + HDfree(s1_wbuf); + if (s1_wbuf_bak) + HDfree(s1_wbuf_bak); + if (s1_rbuf) + HDfree(s1_rbuf); + if (s2_wbuf) + HDfree(s2_wbuf); + if (s2_wbuf_bak) + HDfree(s2_wbuf_bak); + if (s2_rbuf) + HDfree(s2_rbuf); + return FAIL; + +} /* test_cmpd_with_bkg() */ + +/* + * Test 1 for multi-dataset: + * --Datasets with/without type conversion+smaller/larger mem type+no background buffer + * + * Create datasets: randomized H5T_NATIVE_INT or H5T_NATIVE_LONG + * + * Case a--setting for multi write/read to ndsets: + * Datatype for all datasets: H5T_NATIVE_INT + * + * Case b--setting for multi write/read to ndsets: + * Datatype for all datasets: H5T_NATIVE_LONG + */ +static herr_t +test_multi_dsets_no_bkg(hid_t fid, unsigned chunked, unsigned dtrans, unsigned mwbuf) +{ + size_t ndsets; + int i, j; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t ntrans_dxpl = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + + hid_t file_sids[MULTI_NUM_DSETS]; + hid_t mem_sids[MULTI_NUM_DSETS]; + hid_t mem_tids[MULTI_NUM_DSETS]; + + char dset_names[MULTI_NUM_DSETS][DSET_NAME_LEN]; + hid_t dset_dids[MULTI_NUM_DSETS]; + + size_t buf_size; + int *total_wbuf = NULL; + int *total_wbuf_bak = NULL; + int *total_trans_wbuf = NULL; + int *total_rbuf = NULL; + long *total_lwbuf = NULL; + long *total_lwbuf_bak = NULL; + long *total_trans_lwbuf = NULL; + long *total_lrbuf = NULL; + + int *wbufi[MULTI_NUM_DSETS]; + int *trans_wbufi[MULTI_NUM_DSETS]; + int *rbufi[MULTI_NUM_DSETS]; + + long *lwbufi[MULTI_NUM_DSETS]; + long *trans_lwbufi[MULTI_NUM_DSETS]; + long *lrbufi[MULTI_NUM_DSETS]; + + const void *wbufs[MULTI_NUM_DSETS]; + void *rbufs[MULTI_NUM_DSETS]; + const char *expr = "2*x"; + + ndsets = MAX(MULTI_MIN_DSETS, MULTI_NUM_DSETS); + + dims[0] = DSET_SELECT_DIM; + + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + FAIL_STACK_ERROR; + } + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + FAIL_STACK_ERROR; + + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_ON) < 0) + FAIL_STACK_ERROR; + + /* Set modify write buffer if requested */ + if (mwbuf) + if (H5Pset_modify_write_buf(dxpl, TRUE) < 0) + FAIL_STACK_ERROR; + + if ((ntrans_dxpl = H5Pcopy(dxpl)) < 0) + FAIL_STACK_ERROR; + + /* Set data transform */ + if (dtrans) + if (H5Pset_data_transform(dxpl, expr) < 0) + FAIL_STACK_ERROR; + + /* Set up file space ids, mem space ids, and dataset ids */ + for (i = 0; i < (int)ndsets; i++) { + if ((file_sids[i] = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + + if ((mem_sids[i] = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + + /* Generate dataset name */ + HDsnprintf(dset_names[i], sizeof(dset_names[i]), "multi_dset%d_%s_%s_%s", i, + chunked ? "chunked" : "contig", dtrans ? "xform" : "noxform", mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create ith dataset */ + if ((dset_dids[i] = + H5Dcreate2(fid, dset_names[i], ((HDrandom() % 2) ? H5T_NATIVE_LONG : H5T_NATIVE_INT), + file_sids[i], H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + } + + buf_size = ndsets * DSET_SELECT_DIM * sizeof(int); + + /* Allocate buffers for all datasets */ + if (NULL == (total_wbuf = (int *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (mwbuf && NULL == (total_wbuf_bak = (int *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (NULL == (total_trans_wbuf = (int *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (NULL == (total_rbuf = (int *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + + buf_size = ndsets * DSET_SELECT_DIM * sizeof(long); + + if (NULL == (total_lwbuf = (long *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (mwbuf && NULL == (total_lwbuf_bak = (long *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (NULL == (total_trans_lwbuf = (long *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (NULL == (total_lrbuf = (long *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + + /* Initialize buffer indices */ + for (i = 0; i < (int)ndsets; i++) { + wbufi[i] = total_wbuf + (i * DSET_SELECT_DIM); + trans_wbufi[i] = total_trans_wbuf + (i * DSET_SELECT_DIM); + rbufi[i] = total_rbuf + (i * DSET_SELECT_DIM); + + wbufs[i] = wbufi[i]; + rbufs[i] = rbufi[i]; + } + + /* Initialize the buffer data */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < DSET_SELECT_DIM; j++) { + wbufi[i][j] = (int)j; + trans_wbufi[i][j] = 2 * wbufi[i][j]; + } + + /* Case a */ + + /* Datatype setting for multi write/read */ + for (i = 0; i < (int)ndsets; i++) + mem_tids[i] = H5T_NATIVE_INT; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_wbuf_bak, total_wbuf, ndsets * DSET_SELECT_DIM * sizeof(int)); + + /* Write data to the dataset with/without data transform */ + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_wbuf, total_wbuf_bak, ndsets * DSET_SELECT_DIM * sizeof(int)); + + /* Read data from the dataset (if dtrans, without data transform set in dxpl) */ + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, ntrans_dxpl, rbufs) < 0) + TEST_ERROR; + + /* Verify */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < DSET_SELECT_DIM; j++) + if (rbufi[i][j] != (dtrans ? trans_wbufi[i][j] : wbufi[i][j])) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + TEST_ERROR; + } + + if (dtrans) { + + /* Read the data from the dataset with data transform set in dxpl */ + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + TEST_ERROR; + + /* Verify */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < DSET_SELECT_DIM; j++) + if (rbufi[i][j] != (2 * trans_wbufi[i][j])) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + TEST_ERROR; + } + } + + /* Case b */ + + /* Datatype setting for multi write/read */ + for (i = 0; i < (int)ndsets; i++) + mem_tids[i] = H5T_NATIVE_LONG; + + /* Initialize buffer indices */ + for (i = 0; i < (int)ndsets; i++) { + lwbufi[i] = total_lwbuf + (i * DSET_SELECT_DIM); + trans_lwbufi[i] = total_trans_lwbuf + (i * DSET_SELECT_DIM); + lrbufi[i] = total_lrbuf + (i * DSET_SELECT_DIM); + wbufs[i] = lwbufi[i]; + rbufs[i] = lrbufi[i]; + } + + /* Initialize the buffer data */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < DSET_SELECT_DIM; j++) { + lwbufi[i][j] = (long)j + DSET_SELECT_DIM; + trans_lwbufi[i][j] = 2 * lwbufi[i][j]; + } + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_lwbuf_bak, total_lwbuf, ndsets * DSET_SELECT_DIM * sizeof(long)); + + /* Write data to the dataset with/without data transform */ + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_lwbuf, total_lwbuf_bak, ndsets * DSET_SELECT_DIM * sizeof(long)); + + /* Read data from the dataset (if dtrans, with data transform again in dxpl) */ + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + TEST_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + for (j = 0; j < DSET_SELECT_DIM; j++) { + if (lrbufi[i][j] != (dtrans ? (2 * trans_lwbufi[i][j]) : lwbufi[i][j])) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + TEST_ERROR; + } + } + } + + if (H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dxpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(ntrans_dxpl) < 0) + FAIL_STACK_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + if (H5Sclose(file_sids[i]) < 0) + FAIL_STACK_ERROR; + if (H5Sclose(mem_sids[i]) < 0) + FAIL_STACK_ERROR; + if (H5Dclose(dset_dids[i]) < 0) + FAIL_STACK_ERROR; + } + + HDfree(total_wbuf); + HDfree(total_wbuf_bak); + HDfree(total_rbuf); + HDfree(total_trans_wbuf); + HDfree(total_lwbuf); + HDfree(total_lwbuf_bak); + HDfree(total_lrbuf); + HDfree(total_trans_lwbuf); + + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + H5Pclose(dcpl); + H5Pclose(dxpl); + H5Pclose(ntrans_dxpl); + for (i = 0; i < (int)ndsets; i++) { + H5Sclose(file_sids[i]); + H5Sclose(mem_sids[i]); + H5Dclose(dset_dids[i]); + } + H5E_END_TRY; + + if (total_wbuf) + HDfree(total_wbuf); + if (total_wbuf_bak) + HDfree(total_wbuf_bak); + if (total_trans_wbuf) + HDfree(total_trans_wbuf); + if (total_rbuf) + HDfree(total_rbuf); + if (total_lwbuf) + HDfree(total_lwbuf); + if (total_lwbuf_bak) + HDfree(total_lwbuf_bak); + if (total_lrbuf) + HDfree(total_lrbuf); + if (total_trans_lwbuf) + HDfree(total_lrbuf); + + return FAIL; + +} /* test_multi_dsets_no_bkg() */ + +/* + * Test 2 for multi-dataset: + * + * Datasets with compound types+background buffer + * + * Create datasets with the same compound type + * (a) Initialize compound buffer in memory with unique values + * All datasets: + * --Write all compound fields to disk + * --Read the entire compound type for all datasets + * --Verify values read + * (b) Update all fields of the compound type in memory write buffer with new unique values + * dset0: + * --Write some but not all all compound fields to disk + * --Read and verify the fields have the correct (old(a) or new) values + * Remaining datasets: + * --Untouched + * --Read and verify the fields have the correct old(a) values + * (c) Update all fields of the compound type in memory read buffer with new unique values + * Randomized <mm> dataset: + * --Read some but not all the compound fields to memory + * --Verify the fields have the correct (old(a) or new) values + * dset0: + * --Untouched + * --Read and verify the fields have the correct (old(a) or middle(b)) values + * Remaining datasets: + * --Untouched + * --Read and verify the fields have the correct old(a) values + * (d) Set up a different compound type which has: + * --no type conversion for 2 member types + * --a field with larger mem type + * --a field with smaller mem type + * All datasets: + * --Write the compound fields to disk + * --Read the entire compound type + * --Verify values read + */ +static herr_t +test_multi_dsets_cmpd_with_bkg(hid_t fid, unsigned chunked, unsigned mwbuf) +{ + size_t ndsets; + int i, j, mm; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + + hid_t file_sids[MULTI_NUM_DSETS]; + hid_t mem_sids[MULTI_NUM_DSETS]; + hid_t mem_tids[MULTI_NUM_DSETS]; + + hid_t s1_tid = H5I_INVALID_HID; + hid_t ss_ac_tid = H5I_INVALID_HID; + hid_t ss_bc_tid = H5I_INVALID_HID; + hid_t s2_tid = H5I_INVALID_HID; + + char dset_names[MULTI_NUM_DSETS][DSET_NAME_LEN]; + hid_t dset_dids[MULTI_NUM_DSETS]; + + size_t buf_size; + size_t s2_buf_size; + + s1_t *total_wbuf = NULL; + s1_t *total_wbuf_bak = NULL; + s1_t *total_rbuf = NULL; + + s2_t *s2_total_wbuf = NULL; + s2_t *s2_total_wbuf_bak = NULL; + s2_t *s2_total_rbuf = NULL; + + s1_t *wbufi[MULTI_NUM_DSETS]; + s1_t *rbufi[MULTI_NUM_DSETS]; + + s2_t *s2_wbufi[MULTI_NUM_DSETS]; + s2_t *s2_rbufi[MULTI_NUM_DSETS]; + + const void *wbufs[MULTI_NUM_DSETS]; + void *rbufs[MULTI_NUM_DSETS]; + + ndsets = MAX(MULTI_MIN_DSETS, MULTI_NUM_DSETS); + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + FAIL_STACK_ERROR; + + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_ON) < 0) + FAIL_STACK_ERROR; + + /* Set modify write buffer if requested */ + if (mwbuf) + if (H5Pset_modify_write_buf(dxpl, TRUE) < 0) + FAIL_STACK_ERROR; + + dims[0] = DSET_SELECT_DIM; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + FAIL_STACK_ERROR; + } + + /* Create the memory data type */ + if ((s1_tid = H5Tcreate(H5T_COMPOUND, sizeof(s1_t))) < 0) + FAIL_STACK_ERROR; + + if (H5Tinsert(s1_tid, "a", HOFFSET(s1_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "b", HOFFSET(s1_t, b), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "c", HOFFSET(s1_t, c), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "d", HOFFSET(s1_t, d), H5T_NATIVE_INT) < 0) + FAIL_STACK_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + if ((file_sids[i] = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + if ((mem_sids[i] = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + + /* Generate dataset name */ + HDsnprintf(dset_names[i], sizeof(dset_names[i]), "multi_cmpd_dset%d_%s_%s", i, + chunked ? "chunked" : "contig", mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create ith dataset */ + if ((dset_dids[i] = + H5Dcreate2(fid, dset_names[i], s1_tid, file_sids[i], H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + } + + buf_size = ndsets * DSET_SELECT_DIM * sizeof(s1_t); + s2_buf_size = ndsets * DSET_SELECT_DIM * sizeof(s2_t); + + /* Allocate buffers */ + if (NULL == (total_wbuf = (s1_t *)HDmalloc(buf_size))) + TEST_ERROR; + if (mwbuf && NULL == (total_wbuf_bak = (s1_t *)HDmalloc(buf_size))) + TEST_ERROR; + if (NULL == (total_rbuf = (s1_t *)HDmalloc(buf_size))) + TEST_ERROR; + + if (NULL == (s2_total_wbuf = (s2_t *)HDmalloc(s2_buf_size))) + TEST_ERROR; + if (mwbuf && NULL == (s2_total_wbuf_bak = (s2_t *)HDmalloc(s2_buf_size))) + TEST_ERROR; + if (NULL == (s2_total_rbuf = (s2_t *)HDmalloc(s2_buf_size))) + TEST_ERROR; + + /* Initialize buffer indices */ + for (i = 0; i < (int)ndsets; i++) { + wbufi[i] = total_wbuf + (i * DSET_SELECT_DIM); + rbufi[i] = total_rbuf + (i * DSET_SELECT_DIM); + + wbufs[i] = wbufi[i]; + rbufs[i] = rbufi[i]; + } + + /* Case a */ + + /* Initialize the buffer data for all the datasets */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < DSET_SELECT_DIM; j++) { + wbufi[i][j].a = (4 * j); + wbufi[i][j].b = (4 * j) + 1; + wbufi[i][j].c = (4 * j) + 2; + wbufi[i][j].d = (4 * j) + 3; + } + + /* Datatype setting for multi write */ + for (i = 0; i < (int)ndsets; i++) + mem_tids[i] = s1_tid; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_wbuf_bak, total_wbuf, buf_size); + + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_wbuf, total_wbuf_bak, buf_size); + + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + TEST_ERROR; + + /* Verify data read */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < DSET_SELECT_DIM; j++) { + if (wbufi[i][j].a != rbufi[i][j].a || wbufi[i][j].b != rbufi[i][j].b || + wbufi[i][j].c != rbufi[i][j].c || wbufi[i][j].d != rbufi[i][j].d) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + TEST_ERROR; + } + } + + /* Case b */ + + /* Update data in wbufi for dset0 with unique values */ + for (j = 0; j < DSET_SELECT_DIM; j++) { + wbufi[0][j].a = (4 * j) + DSET_SELECT_DIM; + wbufi[0][j].b = (4 * j) + DSET_SELECT_DIM + 1; + wbufi[0][j].c = (4 * j) + DSET_SELECT_DIM + 2; + wbufi[0][j].d = (4 * j) + DSET_SELECT_DIM + 3; + } + + /* Create a compound type same size as s1_t */ + if ((ss_ac_tid = H5Tcreate(H5T_COMPOUND, sizeof(s1_t))) < 0) + FAIL_STACK_ERROR; + + /* but contains only subset members of s1_t */ + if (H5Tinsert(ss_ac_tid, "a", HOFFSET(s1_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(ss_ac_tid, "c", HOFFSET(s1_t, c), H5T_NATIVE_INT) < 0) + FAIL_STACK_ERROR; + + /* Untouched memory and file spaces for other datasets */ + for (i = 0; i < (int)ndsets; i++) { + if (i == 0) + continue; + + if (H5Sselect_none(mem_sids[i]) < 0) + TEST_ERROR; + if (H5Sselect_none(file_sids[i]) < 0) + TEST_ERROR; + } + + /* Datatype setting for write to dset0 */ + mem_tids[0] = ss_ac_tid; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_wbuf_bak, total_wbuf, buf_size); + + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_wbuf, total_wbuf_bak, buf_size); + + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + TEST_ERROR; + + /* Verify data read */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < DSET_SELECT_DIM; j++) + if (i == 0) { /* dset0 */ + if (wbufi[i][j].a != rbufi[i][j].a || ((4 * (int)j) + 1) != rbufi[i][j].b || + wbufi[i][j].c != rbufi[i][j].c || ((4 * (int)j) + 3) != rbufi[i][j].d) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + TEST_ERROR; + } + } + else { /* other datasets */ + for (j = 0; j < DSET_SELECT_DIM; j++) + if ((4 * (int)j) != rbufi[i][j].a || ((4 * (int)j) + 1) != rbufi[i][j].b || + ((4 * (int)j) + 2) != rbufi[i][j].c || ((4 * (int)j) + 3) != rbufi[i][j].d) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + TEST_ERROR; + } + } + + /* Case c */ + mm = HDrandom() % (int)ndsets; + if (!mm) + mm++; + + /* Update data in rbufi for dset1 with new unique values */ + for (j = 0; j < DSET_SELECT_DIM; j++) { + rbufi[mm][j].a = (4 * j) + (2 * DSET_SELECT_DIM); + rbufi[mm][j].b = (4 * j) + (2 * DSET_SELECT_DIM) + 1; + rbufi[mm][j].c = (4 * j) + (2 * DSET_SELECT_DIM) + 2; + rbufi[mm][j].d = (4 * j) + (2 * DSET_SELECT_DIM) + 3; + } + + /* Create a compound type same size as s1_t */ + if ((ss_bc_tid = H5Tcreate(H5T_COMPOUND, sizeof(s1_t))) < 0) + FAIL_STACK_ERROR; + + /* but contains only subset members of s1_t */ + if (H5Tinsert(ss_bc_tid, "b", HOFFSET(s1_t, b), H5T_NATIVE_INT) < 0 || + H5Tinsert(ss_bc_tid, "c", HOFFSET(s1_t, c), H5T_NATIVE_INT) < 0) + FAIL_STACK_ERROR; + + /* Reset memory and file space for <mm> dataset */ + if (H5Sselect_all(mem_sids[mm]) < 0) + FAIL_STACK_ERROR; + if (H5Sselect_all(file_sids[mm]) < 0) + FAIL_STACK_ERROR; + + /* Untouched memory and file space for other datasets */ + for (i = 0; i < (int)ndsets; i++) { + if (i == 0 || i == mm) + continue; + if (H5Sselect_none(mem_sids[i]) < 0) + TEST_ERROR; + if (H5Sselect_none(file_sids[i]) < 0) + TEST_ERROR; + } + + /* Datatype setting for read from <mm> dataset */ + mem_tids[mm] = ss_bc_tid; + + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + TEST_ERROR; + + /* Verify data read */ + /* dset0 */ + for (j = 0; j < DSET_SELECT_DIM; j++) + if (wbufi[0][j].a != rbufi[0][j].a || ((4 * (int)j) + 1) != rbufi[0][j].b || + wbufi[0][j].c != rbufi[0][j].c || ((4 * (int)j) + 3) != rbufi[0][j].d) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset0 at index %d\n", j); + TEST_ERROR; + } + + /* <mm> dset */ + for (j = 0; j < DSET_SELECT_DIM; j++) + if (rbufi[mm][j].a != ((4 * (int)j) + (2 * DSET_SELECT_DIM)) || + rbufi[mm][j].b != ((4 * (int)j) + 1) || rbufi[mm][j].c != ((4 * (int)j) + 2) || + rbufi[mm][j].d != ((4 * (int)j) + (2 * DSET_SELECT_DIM) + 3)) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset1 at index %d\n", j); + TEST_ERROR; + } + + /* other datasets */ + for (i = 0; i < (int)ndsets; i++) { + if (i == 0 || i == mm) + continue; + + for (j = 0; j < DSET_SELECT_DIM; j++) + if (rbufi[i][j].a != (4 * (int)j) || rbufi[i][j].b != ((4 * (int)j) + 1) || + rbufi[i][j].c != ((4 * (int)j) + 2) || rbufi[i][j].d != ((4 * (int)j) + 3)) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + TEST_ERROR; + } + } + + /* Case d */ + + /* Create s2_t compound type with: + * --no conversion for 2 member types, + * --1 larger mem type + * --1 smaller mem type + */ + if ((s2_tid = H5Tcreate(H5T_COMPOUND, sizeof(s2_t))) < 0) + FAIL_STACK_ERROR; + + if (H5Tinsert(s2_tid, "a", HOFFSET(s2_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(s2_tid, "b", HOFFSET(s2_t, b), H5T_NATIVE_LONG) < 0 || + H5Tinsert(s2_tid, "c", HOFFSET(s2_t, c), H5T_NATIVE_INT) < 0 || + H5Tinsert(s2_tid, "d", HOFFSET(s2_t, d), H5T_NATIVE_SHORT) < 0) + FAIL_STACK_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + s2_wbufi[i] = s2_total_wbuf + (i * DSET_SELECT_DIM); + s2_rbufi[i] = s2_total_rbuf + (i * DSET_SELECT_DIM); + + wbufs[i] = s2_wbufi[i]; + rbufs[i] = s2_rbufi[i]; + + mem_tids[i] = s2_tid; + + if (H5Sselect_all(mem_sids[i]) < 0) + TEST_ERROR; + if (H5Sselect_all(file_sids[i]) < 0) + TEST_ERROR; + } + + /* Initialize the buffer data for all the datasets */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < DSET_SELECT_DIM; j++) { + s2_wbufi[i][j].a = 8 * j; + s2_wbufi[i][j].b = (long)((8 * j) + 1); + s2_wbufi[i][j].c = (8 * j) + 2; + s2_wbufi[i][j].d = (short)((8 * j) + 3); + } + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(s2_total_wbuf_bak, s2_total_wbuf, s2_buf_size); + + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(s2_total_wbuf, s2_total_wbuf_bak, s2_buf_size); + + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + TEST_ERROR; + + /* Verify data read */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < DSET_SELECT_DIM; j++) + if (s2_wbufi[i][j].a != s2_rbufi[i][j].a || s2_wbufi[i][j].b != s2_rbufi[i][j].b || + s2_wbufi[i][j].c != s2_rbufi[i][j].c || s2_wbufi[i][j].d != s2_rbufi[i][j].d) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + TEST_ERROR; + } + + if (H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dxpl) < 0) + FAIL_STACK_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + if (H5Sclose(file_sids[i]) < 0) + FAIL_STACK_ERROR; + if (H5Sclose(mem_sids[i]) < 0) + FAIL_STACK_ERROR; + if (H5Dclose(dset_dids[i]) < 0) + FAIL_STACK_ERROR; + } + + HDfree(total_wbuf); + HDfree(total_wbuf_bak); + HDfree(total_rbuf); + HDfree(s2_total_wbuf); + HDfree(s2_total_wbuf_bak); + HDfree(s2_total_rbuf); + + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + H5Pclose(dcpl); + H5Pclose(dxpl); + for (i = 0; i < (int)ndsets; i++) { + H5Sclose(file_sids[i]); + H5Sclose(mem_sids[i]); + H5Dclose(dset_dids[i]); + } + H5E_END_TRY; + + if (total_wbuf) + HDfree(total_wbuf); + if (total_wbuf_bak) + HDfree(total_wbuf_bak); + if (total_rbuf) + HDfree(total_rbuf); + if (s2_total_wbuf) + HDfree(s2_total_wbuf); + if (s2_total_wbuf_bak) + HDfree(s2_total_wbuf_bak); + if (s2_total_rbuf) + HDfree(s2_total_rbuf); + + return FAIL; + +} /* test_multi_dsets_cmpd_with_bkg() */ + +/* + * Test 3 for multi-dataset: + * --Datasets with/without type conv+size change+no background buffer + * + * Create dset0: H5T_STD_I32BE + * Create other dateasets: randomized H5T_STD_I64LE or H5T_STD_I16LE + * + * Case a--setting for write/read to ndsets: + * Datatype for all datasets: H5T_STD_I32BE + * + * Case b--setting for write/read to ndsets + * Datatype for all datasets: H5T_STD_I64BE + * + * Case c--setting for write/read to ndsets + * Datatype for all datasets: H5T_STD_I16BE + */ +static herr_t +test_multi_dsets_size_change_no_bkg(hid_t fid, unsigned chunked, unsigned mwbuf) +{ + size_t ndsets; + int i, j; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + + hid_t file_sids[MULTI_NUM_DSETS]; + hid_t mem_sids[MULTI_NUM_DSETS]; + hid_t mem_tids[MULTI_NUM_DSETS]; + + char dset_names[MULTI_NUM_DSETS][DSET_NAME_LEN]; + hid_t dset_dids[MULTI_NUM_DSETS]; + + size_t buf_size, ss; + uint8_t *total_wbuf = NULL; + uint8_t *total_wbuf_bak = NULL; + uint8_t *total_rbuf = NULL; + uint8_t *total_lwbuf = NULL; + uint8_t *total_lwbuf_bak = NULL; + uint8_t *total_lrbuf = NULL; + uint8_t *total_swbuf = NULL; + uint8_t *total_swbuf_bak = NULL; + uint8_t *total_srbuf = NULL; + + uint8_t *wbufi[MULTI_NUM_DSETS]; + uint8_t *rbufi[MULTI_NUM_DSETS]; + uint8_t *lwbufi[MULTI_NUM_DSETS]; + uint8_t *lrbufi[MULTI_NUM_DSETS]; + uint8_t *swbufi[MULTI_NUM_DSETS]; + uint8_t *srbufi[MULTI_NUM_DSETS]; + + const void *wbufs[MULTI_NUM_DSETS]; + void *rbufs[MULTI_NUM_DSETS]; + + ndsets = MAX(MULTI_MIN_DSETS, MULTI_NUM_DSETS); + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + FAIL_STACK_ERROR; + + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_ON) < 0) + FAIL_STACK_ERROR; + + /* Set modify write buffer if requested */ + if (mwbuf) + if (H5Pset_modify_write_buf(dxpl, TRUE) < 0) + FAIL_STACK_ERROR; + + dims[0] = DSET_SELECT_DIM; + + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + FAIL_STACK_ERROR; + } + + /* Set up file space ids, mem space ids, and dataset ids */ + for (i = 0; i < (int)ndsets; i++) { + if ((file_sids[i] = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + + if ((mem_sids[i] = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + + /* Generate dataset name */ + HDsnprintf(dset_names[i], sizeof(dset_names[i]), "multi_size_dset%d_%s_%s", i, + chunked ? "chunked" : "contig", mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create ith dataset */ + if ((dset_dids[i] = H5Dcreate2(fid, dset_names[i], H5T_STD_I32BE, file_sids[i], H5P_DEFAULT, dcpl, + H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + } + + /* Case a */ + + ss = H5Tget_size(H5T_STD_I32BE); + buf_size = ndsets * ss * DSET_SELECT_DIM; + + /* Allocate buffers for all datasets */ + if (NULL == (total_wbuf = (uint8_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (NULL == (total_wbuf_bak = (uint8_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (NULL == (total_rbuf = (uint8_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + + /* Initialize buffer indices */ + for (i = 0; i < (int)ndsets; i++) { + wbufi[i] = total_wbuf + (i * (int)ss * DSET_SELECT_DIM); + rbufi[i] = total_rbuf + (i * (int)ss * DSET_SELECT_DIM); + + wbufs[i] = wbufi[i]; + rbufs[i] = rbufi[i]; + } + + /* Initialize the buffer data: big endian */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < DSET_SELECT_DIM; j++) { + wbufi[i][j * (int)ss + 0] = 0x1; + wbufi[i][j * (int)ss + 1] = 0x2; + wbufi[i][j * (int)ss + 2] = 0x3; + wbufi[i][j * (int)ss + 3] = (uint8_t)(0x4 + j); + } + + /* Datatype setting for multi write/read */ + for (i = 0; i < (int)ndsets; i++) + mem_tids[i] = H5T_STD_I32BE; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_wbuf_bak, total_wbuf, buf_size); + + /* Write data to the dataset */ + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_wbuf, total_wbuf_bak, buf_size); + + /* Read data from the dataset */ + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + TEST_ERROR; + + /* Verify */ + for (i = 0; i < (int)ndsets; i++) + /* Only compare when it's at least the size of H5T_STD_I32BE */ + if (H5Tget_size(H5Dget_type(dset_dids[i])) >= ss) { + for (j = 0; j < DSET_SELECT_DIM; j++) + if (rbufi[i][(int)ss * j + 0] != wbufi[i][(int)ss * j + 0] || + rbufi[i][(int)ss * j + 1] != wbufi[i][(int)ss * j + 1] || + rbufi[i][(int)ss * j + 2] != wbufi[i][(int)ss * j + 2] || + rbufi[i][(int)ss * j + 3] != wbufi[i][(int)ss * j + 3]) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + TEST_ERROR; + } + } + + /* Case b */ + + ss = H5Tget_size(H5T_STD_I64BE); + buf_size = ndsets * (ss * DSET_SELECT_DIM); + + /* Allocate buffers for all datasets */ + if (NULL == (total_lwbuf = (uint8_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (NULL == (total_lwbuf_bak = (uint8_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (NULL == (total_lrbuf = (uint8_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + + /* Initialize buffer indices */ + for (i = 0; i < (int)ndsets; i++) { + lwbufi[i] = total_lwbuf + (i * (int)ss * DSET_SELECT_DIM); + lrbufi[i] = total_lrbuf + (i * (int)ss * DSET_SELECT_DIM); + + wbufs[i] = lwbufi[i]; + rbufs[i] = lrbufi[i]; + } + + /* Initialize the buffer data: big endian */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < DSET_SELECT_DIM; j++) { + lwbufi[i][j * (int)ss + 0] = 0x1; + lwbufi[i][j * (int)ss + 1] = 0x2; + lwbufi[i][j * (int)ss + 2] = 0x3; + lwbufi[i][j * (int)ss + 3] = 0x4; + lwbufi[i][j * (int)ss + 4] = 0x5; + lwbufi[i][j * (int)ss + 5] = 0x6; + lwbufi[i][j * (int)ss + 6] = 0x7; + lwbufi[i][j * (int)ss + 7] = (uint8_t)(0x8 + j); + } + + /* Datatype setting for multi write/read */ + for (i = 0; i < (int)ndsets; i++) + mem_tids[i] = H5T_STD_I64BE; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_lwbuf_bak, total_lwbuf, buf_size); + + /* Write data to the dataset */ + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_lwbuf, total_lwbuf_bak, buf_size); + + /* Read data from the dataset */ + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + TEST_ERROR; + + /* Verify */ + for (i = 0; i < (int)ndsets; i++) + /* Only compare when it's the size of H5T_STD_I64BE */ + if (H5Tget_size(H5Dget_type(dset_dids[i])) >= ss) { + for (j = 0; j < DSET_SELECT_DIM; j++) + if (lrbufi[i][(int)ss * j + 0] != lwbufi[i][(int)ss * j + 0] || + lrbufi[i][(int)ss * j + 1] != lwbufi[i][(int)ss * j + 1] || + lrbufi[i][(int)ss * j + 2] != lwbufi[i][(int)ss * j + 2] || + lrbufi[i][(int)ss * j + 3] != lwbufi[i][(int)ss * j + 3] || + lrbufi[i][(int)ss * j + 4] != lwbufi[i][(int)ss * j + 4] || + lrbufi[i][(int)ss * j + 5] != lwbufi[i][(int)ss * j + 5] || + lrbufi[i][(int)ss * j + 6] != lwbufi[i][(int)ss * j + 6] || + lrbufi[i][(int)ss * j + 7] != lwbufi[i][(int)ss * j + 7]) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + TEST_ERROR; + } + } + + /* Case c */ + + ss = H5Tget_size(H5T_STD_I16BE); + buf_size = ndsets * (ss * DSET_SELECT_DIM); + + /* Allocate buffers for all datasets */ + if (NULL == (total_swbuf = (uint8_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (NULL == (total_swbuf_bak = (uint8_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (NULL == (total_srbuf = (uint8_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + + /* Initialize buffer indices */ + for (i = 0; i < (int)ndsets; i++) { + swbufi[i] = total_swbuf + (i * (int)ss * DSET_SELECT_DIM); + srbufi[i] = total_srbuf + (i * (int)ss * DSET_SELECT_DIM); + + wbufs[i] = swbufi[i]; + rbufs[i] = srbufi[i]; + } + + /* Initialize the buffer data: big endian */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < DSET_SELECT_DIM; j++) { + swbufi[i][j * (int)ss + 0] = 0x1; + swbufi[i][j * (int)ss + 1] = (uint8_t)(0x2 + j); + } + + /* Datatype setting for multi write/read */ + for (i = 0; i < (int)ndsets; i++) + mem_tids[i] = H5T_STD_I16BE; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_swbuf_bak, total_swbuf, buf_size); + + /* Write data to the dataset */ + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_swbuf, total_swbuf_bak, buf_size); + + /* Read data from the dataset */ + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + TEST_ERROR; + + /* Verify */ + for (i = 0; i < (int)ndsets; i++) + /* Can compare for all cases */ + for (j = 0; j < DSET_SELECT_DIM; j++) + if (srbufi[i][(int)ss * j + 0] != swbufi[i][(int)ss * j + 0] || + srbufi[i][(int)ss * j + 1] != swbufi[i][(int)ss * j + 1]) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + TEST_ERROR; + } + + if (H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dxpl) < 0) + FAIL_STACK_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + if (H5Sclose(file_sids[i]) < 0) + FAIL_STACK_ERROR; + if (H5Sclose(mem_sids[i]) < 0) + FAIL_STACK_ERROR; + if (H5Dclose(dset_dids[i]) < 0) + FAIL_STACK_ERROR; + } + + HDfree(total_wbuf); + HDfree(total_wbuf_bak); + HDfree(total_rbuf); + HDfree(total_lwbuf); + HDfree(total_lwbuf_bak); + HDfree(total_lrbuf); + HDfree(total_swbuf); + HDfree(total_swbuf_bak); + HDfree(total_srbuf); + + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + H5Pclose(dcpl); + H5Pclose(dxpl); + for (i = 0; i < (int)ndsets; i++) { + H5Sclose(file_sids[i]); + H5Sclose(mem_sids[i]); + H5Dclose(dset_dids[i]); + } + H5E_END_TRY; + + if (total_wbuf) + HDfree(total_wbuf); + if (total_wbuf_bak) + HDfree(total_wbuf_bak); + if (total_rbuf) + HDfree(total_rbuf); + if (total_lwbuf) + HDfree(total_lwbuf); + if (total_lwbuf_bak) + HDfree(total_lwbuf_bak); + if (total_lrbuf) + HDfree(total_lrbuf); + if (total_swbuf) + HDfree(total_swbuf); + if (total_swbuf_bak) + HDfree(total_swbuf_bak); + if (total_srbuf) + HDfree(total_srbuf); + + return FAIL; + +} /* test_multi_dsets_size_change_no_bkg() */ + +/* + * Test 4 for multi-dataset: + * + * Repeat the following test for niter times to ensure the + * random combinations of all dataset types are hit. + * + * Create randomized contiguous or chunked datasets with: + * --DSET_WITH_NO_CONV: + * --with no type conversion + * --dataset with H5T_NATIVE_INT + * --DSET_WITH_CONV_AND_NO_BKG: + * --type conversion without background buffer + * --dataset with H5T_NATIVE_LONG + * --DSET_WITH_CONV_AND_BKG: + * --type conversion with background buffer + * --dataset with compound type s1_t + * + * Do H5Dwrite_multi() and H5Dread_multi() for the above datasets: + * Setting A: + * --DSET_WITH_NO_CONV: + * --write: mem_tids[] = H5T_NATIVE_INT + * --read: r_mem_tids[] = H5T_NATIVE_INT + * --DSET_WITH_CONV_AND_NO_BKG: + * --write: mem_tids[] = H5T_NATIVE_ULONG + * --read: r_mem_tids[] = H5T_NATIVE_LONG + * --DSET_WITH_CONV_AND_BKG: + * --write: mem_tids[] = s1_tid; + * --read: r_mem_tids[i] = s3_tid; + * + * Setting B: + * --DSET_WITH_NO_CONV: + * --write: mem_tids[] = H5T_NATIVE_INT + * --read: r_mem_tids[] = H5T_NATIVE_INT + * --DSET_WITH_CONV_AND_NO_BKG: + * --write: mem_tids[] = H5T_NATIVE_LONG; + * --read: r_mem_tids[] = H5T_NATIVE_SHORT; + * --DSET_WITH_CONV_AND_BKG: + * --write: mem_tids[] = s4_tid; + * --read: r_mem_tids[i] = s1_tid; + * + * Verify the result read as below: + * Setting A: + * --DSET_WITH_NO_CONV: + * --verify data read in rbufi1[i][j] is same as wbufi1[i][j] + * --DSET_WITH_CONV_AND_NO_BKG: + * --verify data read in l_rbufi2[i][j] is all LONG_MAX + * --DSET_WITH_CONV_AND_BKG: + * --verify all fields read in s3_rbufi3[i][j] is the + * reverse of s1_wbufi3[i][j] + * Setting B: + * --DSET_WITH_NO_CONV: + * --verify data read in rbufi1[i][j] is same as wbufi1[i][j] + * --DSET_WITH_CONV_AND_NO_BKG: + * --verify data read in s_rbufi2[i][j] is all SHRT_MAX + * --DSET_WITH_CONV_AND_BKG: + * --verify fields read in s1_rbufi3[i][j] is as follows: + * --fields 'a' and 'c' are as s1_wbufi3[i][j].a and s1_wbufi3[i][j].c + * --fields 'b' and 'd' are (DSET_SELECT_DIM + j + start[0]) + * + */ +static herr_t +test_multi_dsets_all(int niter, hid_t fid, unsigned chunked, unsigned mwbuf) +{ + size_t ndsets; + int i, j, mm; + int s, n; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + + hid_t file_sids[MULTI_NUM_DSETS]; + hid_t mem_sids[MULTI_NUM_DSETS]; + hid_t mem_tids[MULTI_NUM_DSETS]; + hid_t r_mem_tids[MULTI_NUM_DSETS]; + + multi_dset_type_t dset_types[MULTI_NUM_DSETS]; + + hid_t s1_tid = H5I_INVALID_HID; + hid_t s3_tid = H5I_INVALID_HID; + hid_t s4_tid = H5I_INVALID_HID; + + char dset_names[MULTI_NUM_DSETS][DSET_NAME_LEN]; + hid_t dset_dids[MULTI_NUM_DSETS]; + + size_t buf_size; + + int *total_wbuf1 = NULL; + int *total_wbuf1_bak = NULL; + int *total_rbuf1 = NULL; + + int *wbufi1[MULTI_NUM_DSETS]; + int *rbufi1[MULTI_NUM_DSETS]; + + unsigned long *ul_total_wbuf2 = NULL; + unsigned long *ul_total_wbuf2_bak = NULL; + long *l_total_rbuf2 = NULL; + unsigned long *ul_wbufi2[MULTI_NUM_DSETS]; + long *l_rbufi2[MULTI_NUM_DSETS]; + + long *l_total_wbuf2 = NULL; + long *l_total_wbuf2_bak = NULL; + short *s_total_rbuf2 = NULL; + long *l_wbufi2[MULTI_NUM_DSETS]; + short *s_rbufi2[MULTI_NUM_DSETS]; + + s1_t *s1_total_wbuf3 = NULL; + s1_t *s1_total_wbuf3_bak = NULL; + s3_t *s3_total_rbuf3 = NULL; + s1_t *s1_wbufi3[MULTI_NUM_DSETS]; + s3_t *s3_rbufi3[MULTI_NUM_DSETS]; + + s4_t *s4_total_wbuf3 = NULL; + s4_t *s4_total_wbuf3_bak = NULL; + s1_t *s1_total_rbuf3 = NULL; + s4_t *s4_wbufi3[MULTI_NUM_DSETS]; + s1_t *s1_rbufi3[MULTI_NUM_DSETS]; + + const void *wbufs[MULTI_NUM_DSETS]; + void *rbufs[MULTI_NUM_DSETS]; + + /* for n niter to ensure that all randomized dset_types with multi_dset_type_t will be covered */ + for (n = 0; n < niter; n++) { + + /* Set up the number of datasets for testing */ + ndsets = MAX(MULTI_MIN_DSETS, MULTI_NUM_DSETS); + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + FAIL_STACK_ERROR; + + /* Enable selection I/O */ + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_ON) < 0) + FAIL_STACK_ERROR; + + /* Set modify write buffer if requested */ + if (mwbuf) + if (H5Pset_modify_write_buf(dxpl, TRUE) < 0) + FAIL_STACK_ERROR; + + /* Set dataset layout: contiguous or chunked */ + dims[0] = DSET_SELECT_DIM; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + FAIL_STACK_ERROR; + } + + /* Create compound data type: s1_t */ + if ((s1_tid = H5Tcreate(H5T_COMPOUND, sizeof(s1_t))) < 0) + FAIL_STACK_ERROR; + + if (H5Tinsert(s1_tid, "a", HOFFSET(s1_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "b", HOFFSET(s1_t, b), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "c", HOFFSET(s1_t, c), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "d", HOFFSET(s1_t, d), H5T_NATIVE_INT) < 0) + FAIL_STACK_ERROR; + + /* Create compound data type: s3_t */ + if ((s3_tid = H5Tcreate(H5T_COMPOUND, sizeof(s3_t))) < 0) + FAIL_STACK_ERROR; + + if (H5Tinsert(s3_tid, "a", HOFFSET(s3_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(s3_tid, "b", HOFFSET(s3_t, b), H5T_NATIVE_INT) < 0 || + H5Tinsert(s3_tid, "c", HOFFSET(s3_t, c), H5T_NATIVE_INT) < 0 || + H5Tinsert(s3_tid, "d", HOFFSET(s3_t, d), H5T_NATIVE_INT) < 0) + FAIL_STACK_ERROR; + + /* Create compound data type: s4_t */ + if ((s4_tid = H5Tcreate(H5T_COMPOUND, sizeof(s4_t))) < 0) + FAIL_STACK_ERROR; + + if (H5Tinsert(s4_tid, "b", HOFFSET(s4_t, b), H5T_NATIVE_UINT) < 0 || + H5Tinsert(s4_tid, "d", HOFFSET(s4_t, d), H5T_NATIVE_UINT) < 0) + FAIL_STACK_ERROR; + + /* Create dataset for i ndsets */ + for (i = 0; i < (int)ndsets; i++) { + + /* File space ids */ + if ((file_sids[i] = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + + /* Memory space ids */ + if ((mem_sids[i] = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + + mm = HDrandom() % (int)ndsets; + if (mm == 0) { + dset_types[i] = DSET_WITH_NO_CONV; + HDsnprintf(dset_names[i], sizeof(dset_names[i]), "multi_all_nconv_dset%d_%s_%s", i, + chunked ? "chunked" : "contig", mwbuf ? "mwbuf" : "nomwbuf"); + if ((dset_dids[i] = H5Dcreate2(fid, dset_names[i], H5T_NATIVE_INT, file_sids[i], H5P_DEFAULT, + dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + } + else if (mm == 1) { + dset_types[i] = DSET_WITH_CONV_AND_NO_BKG; + HDsnprintf(dset_names[i], sizeof(dset_names[i]), "multi_all_conv_nbkg_dset%d_%s_%s", i, + chunked ? "chunked" : "contig", mwbuf ? "mwbuf" : "nomwbuf"); + if ((dset_dids[i] = H5Dcreate2(fid, dset_names[i], H5T_NATIVE_LONG, file_sids[i], H5P_DEFAULT, + dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + } + else { + dset_types[i] = DSET_WITH_CONV_AND_BKG; + HDsnprintf(dset_names[i], sizeof(dset_names[i]), "multi_all_conv_bkg_dset%d_%s_%s", i, + chunked ? "chunked" : "contig", mwbuf ? "mwbuf" : "nomwbuf"); + if ((dset_dids[i] = H5Dcreate2(fid, dset_names[i], s1_tid, file_sids[i], H5P_DEFAULT, dcpl, + H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + } + + } /* end for i ndsets */ + + /* Allocate buffers for all datasets */ + + /* DSET_WITH_NO_CONV */ + buf_size = ndsets * DSET_SELECT_DIM * sizeof(int); + if (NULL == (total_wbuf1 = (int *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (mwbuf && NULL == (total_wbuf1_bak = (int *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (NULL == (total_rbuf1 = (int *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + + /* DSET_WITH_CONV_AND_NO_BKG */ + buf_size = ndsets * DSET_SELECT_DIM * sizeof(unsigned long); + if (NULL == (ul_total_wbuf2 = (unsigned long *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (mwbuf && NULL == (ul_total_wbuf2_bak = (unsigned long *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + buf_size = ndsets * DSET_SELECT_DIM * sizeof(long); + if (NULL == (l_total_rbuf2 = (long *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + + buf_size = ndsets * DSET_SELECT_DIM * sizeof(long); + if (NULL == (l_total_wbuf2 = (long *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (mwbuf && NULL == (l_total_wbuf2_bak = (long *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + buf_size = ndsets * DSET_SELECT_DIM * sizeof(short); + if (NULL == (s_total_rbuf2 = (short *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + + /* DSET_WITH_CONV_AND_BKG */ + buf_size = ndsets * DSET_SELECT_DIM * sizeof(s1_t); + if (NULL == (s1_total_wbuf3 = (s1_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (mwbuf && NULL == (s1_total_wbuf3_bak = (s1_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + buf_size = ndsets * DSET_SELECT_DIM * sizeof(s3_t); + if (NULL == (s3_total_rbuf3 = (s3_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + + buf_size = ndsets * DSET_SELECT_DIM * sizeof(s4_t); + if (NULL == (s4_total_wbuf3 = (s4_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + if (mwbuf && NULL == (s4_total_wbuf3_bak = (s4_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + buf_size = ndsets * DSET_SELECT_DIM * sizeof(s1_t); + if (NULL == (s1_total_rbuf3 = (s1_t *)HDmalloc(buf_size))) + FAIL_STACK_ERROR; + + /* Test with s settings for ndsets */ + for (s = SETTING_A; s <= SETTING_B; s++) { + + /* for i ndsets */ + for (i = 0; i < (int)ndsets; i++) { + + switch (dset_types[i]) { + + case DSET_WITH_NO_CONV: + /* Initialize buffer indices */ + wbufi1[i] = total_wbuf1 + (i * DSET_SELECT_DIM); + rbufi1[i] = total_rbuf1 + (i * DSET_SELECT_DIM); + + wbufs[i] = wbufi1[i]; + rbufs[i] = rbufi1[i]; + + /* Initialize the buffer data */ + for (j = 0; j < DSET_SELECT_DIM; j++) + wbufi1[i][j] = (int)j; + + /* Same for all cases */ + mem_tids[i] = H5T_NATIVE_INT; + r_mem_tids[i] = H5T_NATIVE_INT; + + break; + + case DSET_WITH_CONV_AND_NO_BKG: + + if (s == SETTING_A) { + /* Initialize buffer indices */ + ul_wbufi2[i] = ul_total_wbuf2 + (i * DSET_SELECT_DIM); + l_rbufi2[i] = l_total_rbuf2 + (i * DSET_SELECT_DIM); + + wbufs[i] = ul_wbufi2[i]; + rbufs[i] = l_rbufi2[i]; + + for (j = 0; j < DSET_SELECT_DIM; j++) + ul_wbufi2[i][j] = ULONG_MAX - (unsigned long)j; + + mem_tids[i] = H5T_NATIVE_ULONG; + r_mem_tids[i] = H5T_NATIVE_LONG; + } + else if (s == SETTING_B) { + /* Initialize buffer indices */ + l_wbufi2[i] = l_total_wbuf2 + (i * DSET_SELECT_DIM); + s_rbufi2[i] = s_total_rbuf2 + (i * DSET_SELECT_DIM); + + wbufs[i] = l_wbufi2[i]; + rbufs[i] = s_rbufi2[i]; + + /* Initialize the buffer data */ + for (j = 0; j < DSET_SELECT_DIM; j++) + l_wbufi2[i][j] = LONG_MAX - (long)j; + + mem_tids[i] = H5T_NATIVE_LONG; + r_mem_tids[i] = H5T_NATIVE_SHORT; + } + + break; + + case DSET_WITH_CONV_AND_BKG: + + if (s == SETTING_A) { + /* Initialize buffer indices */ + s1_wbufi3[i] = s1_total_wbuf3 + (i * DSET_SELECT_DIM); + s3_rbufi3[i] = s3_total_rbuf3 + (i * DSET_SELECT_DIM); + + wbufs[i] = s1_wbufi3[i]; + rbufs[i] = s3_rbufi3[i]; + + /* Initialize buffer data for s1_t */ + for (j = 0; j < DSET_SELECT_DIM; j++) { + s1_wbufi3[i][j].a = (4 * j); + s1_wbufi3[i][j].b = (4 * j) + 1; + s1_wbufi3[i][j].c = (4 * j) + 2; + s1_wbufi3[i][j].d = (4 * j) + 3; + } + mem_tids[i] = s1_tid; + r_mem_tids[i] = s3_tid; + } + else if (s == SETTING_B) { + /* Initialize buffer indices */ + s4_wbufi3[i] = s4_total_wbuf3 + (i * DSET_SELECT_DIM); + s1_rbufi3[i] = s1_total_rbuf3 + (i * DSET_SELECT_DIM); + + wbufs[i] = s4_wbufi3[i]; + rbufs[i] = s1_rbufi3[i]; + + /* Initialize buffer data for s4_t */ + for (j = 0; j < DSET_SELECT_DIM; j++) { + s4_wbufi3[i][j].b = DSET_SELECT_DIM + (unsigned int)j; + s4_wbufi3[i][j].d = DSET_SELECT_DIM + (unsigned int)j; + } + mem_tids[i] = s4_tid; + r_mem_tids[i] = s1_tid; + } + + break; + + case DSET_NTTYPES: + default: + TEST_ERROR; + + } /* end switch dset_types */ + + } /* end for i ndsets */ + + /* Copy wbufs if the library will be modifying them */ + if (mwbuf) { + HDmemcpy(total_wbuf1_bak, total_wbuf1, ndsets * DSET_SELECT_DIM * sizeof(int)); + HDmemcpy(ul_total_wbuf2_bak, ul_total_wbuf2, + ndsets * DSET_SELECT_DIM * sizeof(unsigned long)); + HDmemcpy(l_total_wbuf2_bak, l_total_wbuf2, ndsets * DSET_SELECT_DIM * sizeof(long)); + HDmemcpy(s1_total_wbuf3_bak, s1_total_wbuf3, ndsets * DSET_SELECT_DIM * sizeof(s1_t)); + HDmemcpy(s4_total_wbuf3_bak, s4_total_wbuf3, ndsets * DSET_SELECT_DIM * sizeof(s4_t)); + } + + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + TEST_ERROR; + + /* Restore wbufs from backup if the library modified them */ + if (mwbuf) { + HDmemcpy(total_wbuf1, total_wbuf1_bak, ndsets * DSET_SELECT_DIM * sizeof(int)); + HDmemcpy(ul_total_wbuf2, ul_total_wbuf2_bak, + ndsets * DSET_SELECT_DIM * sizeof(unsigned long)); + HDmemcpy(l_total_wbuf2, l_total_wbuf2_bak, ndsets * DSET_SELECT_DIM * sizeof(long)); + HDmemcpy(s1_total_wbuf3, s1_total_wbuf3_bak, ndsets * DSET_SELECT_DIM * sizeof(s1_t)); + HDmemcpy(s4_total_wbuf3, s4_total_wbuf3_bak, ndsets * DSET_SELECT_DIM * sizeof(s4_t)); + } + + if (H5Dread_multi(ndsets, dset_dids, r_mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + TEST_ERROR; + + /* Verify result read */ + /* for i ndsets */ + for (i = 0; i < (int)ndsets; i++) { + switch (dset_types[i]) { + + case DSET_WITH_NO_CONV: + for (j = 0; j < DSET_SELECT_DIM; j++) + if (rbufi1[i][j] != wbufi1[i][j]) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + TEST_ERROR; + } + + break; + + case DSET_WITH_CONV_AND_NO_BKG: + if (s == SETTING_A) { + for (j = 0; j < DSET_SELECT_DIM; j++) + if (l_rbufi2[i][j] != LONG_MAX) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + TEST_ERROR; + } + } + else if (s == SETTING_B) { + for (j = 0; j < DSET_SELECT_DIM; j++) + if (s_rbufi2[i][j] != SHRT_MAX) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + } + } + + break; + + case DSET_WITH_CONV_AND_BKG: + if (s == SETTING_A) { + for (j = 0; j < DSET_SELECT_DIM; j++) + if (s3_rbufi3[i][j].a != s1_wbufi3[i][j].a || + s3_rbufi3[i][j].b != s1_wbufi3[i][j].b || + s3_rbufi3[i][j].c != s1_wbufi3[i][j].c || + s3_rbufi3[i][j].d != s1_wbufi3[i][j].d) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + } + } + else if (s == SETTING_B) { + for (j = 0; j < DSET_SELECT_DIM; j++) + if (s1_rbufi3[i][j].a != s1_wbufi3[i][j].a || + s1_rbufi3[i][j].b != (DSET_SELECT_DIM + j) || + s1_rbufi3[i][j].c != s1_wbufi3[i][j].c || + s1_rbufi3[i][j].d != (DSET_SELECT_DIM + j)) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j); + } + } + + break; + + case DSET_NTTYPES: + default: + TEST_ERROR; + + } /* end switch dset_types */ + + } /* end for i ndsets */ + + } /* end for s settings */ + + /* Closing */ + if (H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dxpl) < 0) + FAIL_STACK_ERROR; + + if (H5Tclose(s1_tid) < 0) + FAIL_STACK_ERROR; + if (H5Tclose(s3_tid) < 0) + FAIL_STACK_ERROR; + if (H5Tclose(s4_tid) < 0) + FAIL_STACK_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + if (H5Sclose(file_sids[i]) < 0) + FAIL_STACK_ERROR; + if (H5Dclose(dset_dids[i]) < 0) + FAIL_STACK_ERROR; + /* Don't delete the last set of datasets */ + if ((n + 1) != niter) + if (H5Ldelete(fid, dset_names[i], H5P_DEFAULT) < 0) + FAIL_STACK_ERROR; + } + + /* Freeing */ + HDfree(total_wbuf1); + total_wbuf1 = NULL; + HDfree(total_wbuf1_bak); + total_wbuf1_bak = NULL; + HDfree(total_rbuf1); + total_rbuf1 = NULL; + + HDfree(ul_total_wbuf2); + ul_total_wbuf2 = NULL; + HDfree(ul_total_wbuf2_bak); + ul_total_wbuf2_bak = NULL; + HDfree(l_total_rbuf2); + l_total_rbuf2 = NULL; + HDfree(l_total_wbuf2); + l_total_wbuf2 = NULL; + HDfree(l_total_wbuf2_bak); + l_total_wbuf2_bak = NULL; + HDfree(s_total_rbuf2); + s_total_rbuf2 = NULL; + + HDfree(s1_total_wbuf3); + s1_total_wbuf3 = NULL; + HDfree(s1_total_wbuf3_bak); + s1_total_wbuf3_bak = NULL; + HDfree(s3_total_rbuf3); + s3_total_rbuf3 = NULL; + HDfree(s4_total_wbuf3); + s4_total_wbuf3 = NULL; + HDfree(s4_total_wbuf3_bak); + s4_total_wbuf3_bak = NULL; + HDfree(s1_total_rbuf3); + s1_total_rbuf3 = NULL; + + } /* end for n niter */ + + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + H5Pclose(dcpl); + H5Pclose(dxpl); + H5Tclose(s1_tid); + H5Tclose(s3_tid); + H5Tclose(s4_tid); + for (i = 0; i < (int)ndsets; i++) { + H5Sclose(file_sids[i]); + H5Sclose(mem_sids[i]); + H5Dclose(dset_dids[i]); + } + H5E_END_TRY; + + if (total_wbuf1) + HDfree(total_wbuf1); + if (total_wbuf1_bak) + HDfree(total_wbuf1_bak); + if (total_rbuf1) + HDfree(total_rbuf1); + + if (ul_total_wbuf2) + HDfree(ul_total_wbuf2); + if (ul_total_wbuf2_bak) + HDfree(ul_total_wbuf2_bak); + if (l_total_rbuf2) + HDfree(l_total_rbuf2); + if (l_total_wbuf2) + HDfree(l_total_wbuf2); + if (l_total_wbuf2_bak) + HDfree(l_total_wbuf2_bak); + if (s_total_rbuf2) + HDfree(s_total_rbuf2); + + if (s1_total_wbuf3) + HDfree(s1_total_wbuf3); + if (s1_total_wbuf3_bak) + HDfree(s1_total_wbuf3_bak); + if (s3_total_rbuf3) + HDfree(s3_total_rbuf3); + if (s4_total_wbuf3) + HDfree(s4_total_wbuf3); + if (s4_total_wbuf3_bak) + HDfree(s4_total_wbuf3_bak); + if (s1_total_rbuf3) + HDfree(s1_total_rbuf3); + + return FAIL; + +} /* test_multi_dsets_all() */ + +/* + * Verify H5Pset/get_selection_io API works as expected + */ +static herr_t +test_set_get_select_io_mode(hid_t fid) +{ + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + int i; + long wbuf[DSET_SELECT_DIM]; + H5D_selection_io_mode_t selection_io_mode; + + HDprintf("\n"); + TESTING("H5Pget/set_selection_io_mode()"); + + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + TEST_ERROR; + + /* default case */ + if (H5Pget_selection_io(dxpl, &selection_io_mode) < 0) + TEST_ERROR; + + if (selection_io_mode != H5D_SELECTION_IO_MODE_DEFAULT) + TEST_ERROR; + + /* Disable case */ + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_OFF) < 0) + TEST_ERROR; + + if (H5Pget_selection_io(dxpl, &selection_io_mode) < 0) + TEST_ERROR; + + if (selection_io_mode != H5D_SELECTION_IO_MODE_OFF) + TEST_ERROR; + + /* Enable case */ + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_ON) < 0) + TEST_ERROR; + + if (H5Pget_selection_io(dxpl, &selection_io_mode) < 0) + TEST_ERROR; + + if (selection_io_mode != H5D_SELECTION_IO_MODE_ON) + TEST_ERROR; + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + FAIL_STACK_ERROR; + + if ((did = H5Dcreate2(fid, "test_chk_dset", H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* Initialize data */ + for (i = 0; i < DSET_SELECT_DIM; i++) + wbuf[i] = i; + + /* May change the selection io actually performed */ + if (H5Dwrite(did, H5T_NATIVE_LONG, H5S_ALL, H5S_ALL, dxpl, wbuf) < 0) + FAIL_STACK_ERROR; + + if (H5Pget_selection_io(dxpl, &selection_io_mode) < 0) + TEST_ERROR; + + /* Should still be enabled */ + if (selection_io_mode != H5D_SELECTION_IO_MODE_ON) + TEST_ERROR; + + if (H5Dclose(did) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dxpl) < 0) + FAIL_STACK_ERROR; + if (H5Sclose(sid) < 0) + FAIL_STACK_ERROR; + + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Sclose(sid); + H5Dclose(did); + H5Pclose(dcpl); + H5Pclose(dxpl); + } + H5E_END_TRY; + + return FAIL; +} /* test_set_get_select_io_mode() */ + +/* + * To test with various test_mode that no selelction I/O is performed + * + * Note: It's the responsibility of the tester to feed proper combination + * of test_mode as needed. + */ +static herr_t +test_no_selection_io_cause_mode(const char *filename, hid_t fapl, uint32_t test_mode) +{ + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t fid = H5I_INVALID_HID; + hid_t fcpl = H5I_INVALID_HID; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + hbool_t is_chunked = FALSE; + hid_t tid = H5T_NATIVE_INT; + uint32_t no_selection_io_cause_write = 0; + uint32_t no_selection_io_cause_read = 0; + uint32_t no_selection_io_cause_write_expected = 0; + uint32_t no_selection_io_cause_read_expected = 0; + int wbuf[DSET_SELECT_DIM]; + int rbuf[DSET_SELECT_DIM]; + int i; + + if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR; + + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + FAIL_STACK_ERROR; + + /* Enable page buffering to trigger H5D_PAGE_BUFFER */ + if (test_mode & TEST_PAGE_BUFFER) { + if (H5Pset_page_buffer_size(fapl, 4096, 0, 0) < 0) + FAIL_STACK_ERROR; + if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + FAIL_STACK_ERROR; + } + else { + /* Not page buffer test, reset to default */ + if (H5Pset_page_buffer_size(fapl, 0, 0, 0) < 0) + FAIL_STACK_ERROR; + if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, 0, (hsize_t)1) < 0) + FAIL_STACK_ERROR; + } + + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR; + + /* If default mode, 1st write will trigger cb, 2nd write will trigger sieve */ + /* If on mode, will trigger nothing because the on mode path is different */ + /* Need 2 writes */ + if (test_mode & TEST_CONTIGUOUS_SIEVE_BUFFER) { + no_selection_io_cause_write_expected |= H5D_SEL_IO_CONTIGUOUS_SIEVE_BUFFER; + no_selection_io_cause_read_expected |= H5D_SEL_IO_CONTIGUOUS_SIEVE_BUFFER; + } + + if (test_mode & TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET) { + if (H5Pset_layout(dcpl, H5D_COMPACT) < 0) + FAIL_STACK_ERROR; + no_selection_io_cause_write_expected |= H5D_SEL_IO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; + no_selection_io_cause_read_expected |= H5D_SEL_IO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; + } + + if (test_mode == TEST_DATASET_FILTER) { + if (H5Pset_deflate(dcpl, 9) < 0) + FAIL_STACK_ERROR; + is_chunked = TRUE; + no_selection_io_cause_write_expected |= H5D_SEL_IO_DATASET_FILTER; + no_selection_io_cause_read_expected |= H5D_SEL_IO_DATASET_FILTER; + } + + if (test_mode == TEST_CHUNK_CACHE) { + is_chunked = TRUE; + no_selection_io_cause_write_expected |= H5D_SEL_IO_CHUNK_CACHE; + no_selection_io_cause_read_expected |= H5D_SEL_IO_CHUNK_CACHE; + } + + if (test_mode == TEST_DISABLE_BY_API) { + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_OFF) < 0) + FAIL_STACK_ERROR; + no_selection_io_cause_write_expected |= H5D_SEL_IO_DISABLE_BY_API; + no_selection_io_cause_read_expected |= H5D_SEL_IO_DISABLE_BY_API; + } + + if (test_mode & TEST_NO_VECTOR_OR_SELECTION_IO_CB) { + no_selection_io_cause_write_expected |= H5D_SEL_IO_DEFAULT_OFF; + no_selection_io_cause_read_expected |= H5D_SEL_IO_DEFAULT_OFF; + } + + /* Datatype conversion */ + if (test_mode & TEST_DATATYPE_CONVERSION) { + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_ON) < 0) + FAIL_STACK_ERROR; + tid = H5T_NATIVE_UINT; + + /* If we're testing a too small tconv buffer, set the buffer to be too small */ + if (test_mode & TEST_TCONV_BUF_TOO_SMALL) { + if (H5Pset_buffer(dxpl, sizeof(int), NULL, NULL) < 0) + FAIL_STACK_ERROR; + + /* If we're using in-place type conversion sel io will succeed and only switch to scalar at the + * VFL */ + if (test_mode & TEST_IN_PLACE_TCONV) { + if (H5Pset_modify_write_buf(dxpl, TRUE) < 0) + FAIL_STACK_ERROR; + no_selection_io_cause_write_expected |= H5D_SEL_IO_NO_VECTOR_OR_SELECTION_IO_CB; + } + else + no_selection_io_cause_write_expected |= H5D_SEL_IO_TCONV_BUF_TOO_SMALL; + + /* In-place type conversion for read doesn't require modify_write_buf */ + no_selection_io_cause_read_expected |= H5D_SEL_IO_NO_VECTOR_OR_SELECTION_IO_CB; + } + else { + /* sel io will succeed and only switch to scalar at the VFL */ + no_selection_io_cause_write_expected |= H5D_SEL_IO_NO_VECTOR_OR_SELECTION_IO_CB; + no_selection_io_cause_read_expected |= H5D_SEL_IO_NO_VECTOR_OR_SELECTION_IO_CB; + } + } + + if (test_mode & TEST_PAGE_BUFFER) { + no_selection_io_cause_write_expected |= H5D_SEL_IO_PAGE_BUFFER; + no_selection_io_cause_read_expected |= H5D_SEL_IO_PAGE_BUFFER; + } + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + FAIL_STACK_ERROR; + + if (is_chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + FAIL_STACK_ERROR; + } + + if ((did = H5Dcreate2(fid, "no_selection_io_cause", H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, + H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* Initialize data */ + for (i = 0; i < DSET_SELECT_DIM; i++) + wbuf[i] = i; + + if (H5Dwrite(did, tid, H5S_ALL, H5S_ALL, dxpl, wbuf) < 0) + FAIL_STACK_ERROR; + + if (test_mode & TEST_CONTIGUOUS_SIEVE_BUFFER) { + if (H5Dwrite(did, tid, H5S_ALL, H5S_ALL, dxpl, wbuf) < 0) + FAIL_STACK_ERROR; + } + + if (H5Pget_no_selection_io_cause(dxpl, &no_selection_io_cause_write) < 0) + TEST_ERROR; + + /* Verify causes of no selection I/O for write are as expected */ + if (no_selection_io_cause_write != no_selection_io_cause_write_expected) + TEST_ERROR; + + /* Flush to clear the sieve buf */ + if (test_mode & TEST_NO_VECTOR_OR_SELECTION_IO_CB || test_mode & TEST_DATATYPE_CONVERSION || + test_mode & TEST_PAGE_BUFFER) { + + if (H5Dflush(did) < 0) + FAIL_STACK_ERROR; + } + + if (H5Dread(did, tid, H5S_ALL, H5S_ALL, dxpl, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify causes of no selection I/O for write is as expected */ + if (H5Pget_no_selection_io_cause(dxpl, &no_selection_io_cause_read) < 0) + TEST_ERROR; + + /* Verify causes of no selection I/O for read are as expected */ + if (no_selection_io_cause_read != no_selection_io_cause_read_expected) + TEST_ERROR; + + if (H5Dclose(did) < 0) + FAIL_STACK_ERROR; + if (H5Sclose(sid) < 0) + FAIL_STACK_ERROR; + + if (H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dxpl) < 0) + FAIL_STACK_ERROR; + + if (H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + + if (H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Pclose(fcpl); + H5Pclose(dcpl); + H5Pclose(dxpl); + H5Dclose(did); + H5Dclose(sid); + H5Fclose(fid); + } + H5E_END_TRY; + + return FAIL; +} /* test_no_selection_io_cause_mode() */ + +/* + * Test for causes of not performing selection I/O + */ +static herr_t +test_get_no_selection_io_cause(const char *filename, hid_t fapl) +{ + + int errs = 0; + + HDprintf("\n"); + TESTING("H5Pget_no_selection_io_cause()"); + + errs += test_no_selection_io_cause_mode(filename, fapl, TEST_DISABLE_BY_API); + errs += test_no_selection_io_cause_mode(filename, fapl, TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET); + errs += test_no_selection_io_cause_mode(filename, fapl, TEST_CONTIGUOUS_SIEVE_BUFFER); + errs += test_no_selection_io_cause_mode(filename, fapl, TEST_DATASET_FILTER); + errs += test_no_selection_io_cause_mode(filename, fapl, TEST_CHUNK_CACHE); + errs += test_no_selection_io_cause_mode(filename, fapl, TEST_NO_VECTOR_OR_SELECTION_IO_CB); + errs += test_no_selection_io_cause_mode(filename, fapl, TEST_DATATYPE_CONVERSION); + errs += + test_no_selection_io_cause_mode(filename, fapl, TEST_DATATYPE_CONVERSION | TEST_TCONV_BUF_TOO_SMALL); + errs += test_no_selection_io_cause_mode( + filename, fapl, TEST_DATATYPE_CONVERSION | TEST_TCONV_BUF_TOO_SMALL | TEST_IN_PLACE_TCONV); +#ifndef H5_HAVE_PARALLEL + errs += test_no_selection_io_cause_mode(filename, fapl, TEST_PAGE_BUFFER); +#endif + + if (errs) { + HDprintf(" FAILED\n"); + return FAIL; + } + else { + PASSED(); + return SUCCEED; + } +} + +/*------------------------------------------------------------------------- + * Function: main + * + * Purpose: Test cases for selection I/O + * + * Return: EXIT_SUCCESS/EXIT_FAILURE + * + * Programmer: + * + *------------------------------------------------------------------------- + */ +int +main(void) +{ + int nerrors = 0; + char filename[FILENAME_BUF_SIZE]; + hid_t fapl = H5I_INVALID_HID; + hid_t fid = H5I_INVALID_HID; + int test_select_config; + unsigned chunked; + unsigned dtrans; + unsigned mwbuf; + + /* Testing setup */ + h5_reset(); + fapl = h5_fileaccess(); + + h5_fixname(FILENAME[0], fapl, filename, sizeof filename); + + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + TEST_ERROR; + + /* Test with contiguous or chunked dataset */ + for (chunked = FALSE; chunked <= TRUE; chunked++) { + + /* Data transforms only apply to integer or floating-point datasets */ + /* therefore, not all tests are run with data transform */ + for (dtrans = FALSE; dtrans <= TRUE; dtrans++) { + + /* Test with and without modify_write_buf turned on */ + for (mwbuf = FALSE; mwbuf <= TRUE; mwbuf++) { + /* Print configuration message */ + printf("Testing for selection I/O "); + if (chunked) + printf("with chunked dataset, "); + else + printf("with contiguous dataset, "); + if (dtrans) + printf("data transform, "); + else + printf("without data transform, "); + if (mwbuf) + printf("and with modifying write buffers\n"); + else + printf("and without modifying write buffers\n"); + + for (test_select_config = (int)TEST_NO_TYPE_CONV; + test_select_config < (int)TEST_SELECT_NTESTS; test_select_config++) { + + switch (test_select_config) { + case TEST_NO_TYPE_CONV: /* case 1 */ + TESTING_2("No type conversion (null case)"); + + nerrors += (test_no_type_conv(fid, chunked, dtrans, mwbuf) < 0 ? 1 : 0); + + break; + + case TEST_NO_SIZE_CHANGE_NO_BKG: /* case 2 */ + TESTING_2("No size change, no background buffer"); + + /* Data transforms does not apply to the dataset datatype for this test */ + if (dtrans) + SKIPPED(); + else + nerrors += (test_no_size_change_no_bkg(fid, chunked, mwbuf) < 0 ? 1 : 0); + + break; + + case TEST_LARGER_MEM_NO_BKG: /* case 3 */ + TESTING_2("Larger memory type, no background buffer"); + + nerrors += (test_larger_mem_type_no_bkg(fid, chunked, dtrans, mwbuf) < 0 ? 1 : 0); + + break; + + case TEST_SMALLER_MEM_NO_BKG: /* case 4 */ + TESTING_2("Smaller memory type, no background buffer"); + + nerrors += + (test_smaller_mem_type_no_bkg(fid, chunked, dtrans, mwbuf) < 0 ? 1 : 0); + + break; + + case TEST_CMPD_WITH_BKG: /* case 5 */ + TESTING_2("Compound types with background buffer"); + + /* Data transforms does not apply to the dataset datatype for this test */ + if (dtrans) + SKIPPED(); + else + nerrors += (test_cmpd_with_bkg(fid, chunked, mwbuf) < 0 ? 1 : 0); + + break; + + case TEST_MULTI_CONV_NO_BKG: /* case 6 */ + TESTING_2("multi-datasets: type conv + no bkg buffer"); + + nerrors += test_multi_dsets_no_bkg(fid, chunked, dtrans, mwbuf); + break; + + case TEST_MULTI_CONV_BKG: /* case 7 */ + TESTING_2("multi-datasets: type conv + bkg buffer"); + + /* Data transforms does not apply to the dataset datatype for this test */ + if (dtrans) + SKIPPED(); + else + nerrors += test_multi_dsets_cmpd_with_bkg(fid, chunked, mwbuf); + + break; + + case TEST_MULTI_CONV_SIZE_CHANGE: /* case 8 */ + TESTING_2("multi-datasets: type conv + size change + no bkg buffer"); + + /* Data transforms does not apply to the dataset datatype for this test */ + if (dtrans) + SKIPPED(); + else + nerrors += test_multi_dsets_size_change_no_bkg(fid, chunked, mwbuf); + + break; + + case TEST_MULTI_ALL: /* case 9 */ + TESTING_2("multi-datasets: no conv + conv without bkg + conv with bkg"); + + /* Data transforms does not apply to the dataset datatype for this test */ + if (dtrans) + SKIPPED(); + else + nerrors += test_multi_dsets_all(10, fid, chunked, mwbuf); + + break; + + case TEST_SELECT_NTESTS: + default: + TEST_ERROR; + + } /* end switch */ + } /* end for test_select_config */ + + } /* end mwbuf */ + + } /* end dtrans */ + + } /* end chunked */ + + nerrors += test_set_get_select_io_mode(fid); + + if (H5Fclose(fid) < 0) + TEST_ERROR; + + /* Use own file */ + nerrors += test_get_no_selection_io_cause(filename, fapl); + + if (nerrors) + goto error; + + printf("\n===================================\n"); + HDprintf("All selection I/O dataset tests passed.\n"); + printf("===================================\n"); + + h5_cleanup(FILENAME, fapl); + + HDexit(EXIT_SUCCESS); + +error: + nerrors = MAX(1, nerrors); + HDprintf("***** %d SELECTION I/O DATASET TEST%s FAILED! *****\n", nerrors, 1 == nerrors ? "" : "S"); + HDexit(EXIT_FAILURE); + +} /* end main() */ diff --git a/test/testfiles/plist_files/def_dxpl_32be b/test/testfiles/plist_files/def_dxpl_32be Binary files differindex b13f456..77ed5d1 100644 --- a/test/testfiles/plist_files/def_dxpl_32be +++ b/test/testfiles/plist_files/def_dxpl_32be diff --git a/test/testfiles/plist_files/def_dxpl_32le b/test/testfiles/plist_files/def_dxpl_32le Binary files differindex b13f456..77ed5d1 100644 --- a/test/testfiles/plist_files/def_dxpl_32le +++ b/test/testfiles/plist_files/def_dxpl_32le diff --git a/test/testfiles/plist_files/def_dxpl_64be b/test/testfiles/plist_files/def_dxpl_64be Binary files differindex b13f456..77ed5d1 100644 --- a/test/testfiles/plist_files/def_dxpl_64be +++ b/test/testfiles/plist_files/def_dxpl_64be diff --git a/test/testfiles/plist_files/def_dxpl_64le b/test/testfiles/plist_files/def_dxpl_64le Binary files differindex b13f456..77ed5d1 100644 --- a/test/testfiles/plist_files/def_dxpl_64le +++ b/test/testfiles/plist_files/def_dxpl_64le diff --git a/test/testfiles/plist_files/dxpl_32be b/test/testfiles/plist_files/dxpl_32be Binary files differindex 5ff2ea0..b6ff37d 100644 --- a/test/testfiles/plist_files/dxpl_32be +++ b/test/testfiles/plist_files/dxpl_32be diff --git a/test/testfiles/plist_files/dxpl_32le b/test/testfiles/plist_files/dxpl_32le Binary files differindex 5ff2ea0..b6ff37d 100644 --- a/test/testfiles/plist_files/dxpl_32le +++ b/test/testfiles/plist_files/dxpl_32le diff --git a/test/testfiles/plist_files/dxpl_64be b/test/testfiles/plist_files/dxpl_64be Binary files differindex 5ff2ea0..b6ff37d 100644 --- a/test/testfiles/plist_files/dxpl_64be +++ b/test/testfiles/plist_files/dxpl_64be diff --git a/test/testfiles/plist_files/dxpl_64le b/test/testfiles/plist_files/dxpl_64le Binary files differindex 5ff2ea0..b6ff37d 100644 --- a/test/testfiles/plist_files/dxpl_64le +++ b/test/testfiles/plist_files/dxpl_64le diff --git a/testpar/CMakeLists.txt b/testpar/CMakeLists.txt index d876a21..fb66e76 100644 --- a/testpar/CMakeLists.txt +++ b/testpar/CMakeLists.txt @@ -8,6 +8,7 @@ project (HDF5_TEST_PAR C) set (testphdf5_SOURCES ${HDF5_TEST_PAR_SOURCE_DIR}/testphdf5.c ${HDF5_TEST_PAR_SOURCE_DIR}/t_dset.c + ${HDF5_TEST_PAR_SOURCE_DIR}/t_select_io_dset.c ${HDF5_TEST_PAR_SOURCE_DIR}/t_file.c ${HDF5_TEST_PAR_SOURCE_DIR}/t_file_image.c ${HDF5_TEST_PAR_SOURCE_DIR}/t_mdset.c @@ -96,6 +97,7 @@ set (H5P_TESTS t_prestart t_init_term t_pmulti_dset + t_select_io_dset t_shapesame t_filters_parallel t_subfiling_vfd diff --git a/testpar/Makefile.am b/testpar/Makefile.am index 539750a..59d47e1 100644 --- a/testpar/Makefile.am +++ b/testpar/Makefile.am @@ -33,7 +33,7 @@ check_SCRIPTS = $(TEST_SCRIPT_PARA) # Test programs. These are our main targets. # -TEST_PROG_PARA=t_mpi t_bigio testphdf5 t_cache t_cache_image t_pread t_pshutdown t_prestart t_init_term t_pmulti_dset t_shapesame t_filters_parallel t_2Gio t_vfd +TEST_PROG_PARA=t_mpi t_bigio testphdf5 t_cache t_cache_image t_pread t_pshutdown t_prestart t_init_term t_pmulti_dset t_select_io_dset t_shapesame t_filters_parallel t_2Gio t_vfd if SUBFILING_VFD_CONDITIONAL TEST_PROG_PARA += t_subfiling_vfd diff --git a/testpar/t_2Gio.c b/testpar/t_2Gio.c index d62fb55..8b67dd9 100644 --- a/testpar/t_2Gio.c +++ b/testpar/t_2Gio.c @@ -3868,6 +3868,10 @@ test_no_collective_cause_mode(int selection_mode) uint32_t no_collective_cause_global_expected = 0; // hsize_t coord[NELM][MAX_RANK]; + uint32_t no_selection_io_cause_write = 0; + uint32_t no_selection_io_cause_read = 0; + uint32_t no_selection_io_cause_expected = 0; + const char *filename; const char *test_name; hbool_t is_chunked = 1; @@ -3968,27 +3972,50 @@ test_no_collective_cause_mode(int selection_mode) dataset = H5Dcreate2(fid, "nocolcause", data_type, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); VRFY((dataset >= 0), "H5Dcreate2() dataset succeeded"); + /* Set up the dxpl for the write */ + dxpl_write = H5Pcreate(H5P_DATASET_XFER); + VRFY((dxpl_write >= 0), "H5Pcreate(H5P_DATASET_XFER) succeeded"); + /* * Set expected causes and some tweaks based on the type of test */ if (selection_mode & TEST_DATATYPE_CONVERSION) { test_name = "Broken Collective I/O - Datatype Conversion"; - no_collective_cause_local_expected |= H5D_MPIO_DATATYPE_CONVERSION; - no_collective_cause_global_expected |= H5D_MPIO_DATATYPE_CONVERSION; + /* set different sign to trigger type conversion */ data_type = H5T_NATIVE_UINT; + + /* Disable selection I/O since datatype conversion is supported in collective with selection I/O */ + ret = H5Pset_selection_io(dxpl_write, H5D_SELECTION_IO_MODE_OFF); + VRFY((ret >= 0), "H5Pset_selection_io succeeded"); + + no_collective_cause_local_expected |= H5D_MPIO_DATATYPE_CONVERSION | H5D_MPIO_NO_SELECTION_IO; + no_collective_cause_global_expected |= H5D_MPIO_DATATYPE_CONVERSION | H5D_MPIO_NO_SELECTION_IO; + no_selection_io_cause_expected |= H5D_SEL_IO_DISABLE_BY_API; } if (selection_mode & TEST_DATA_TRANSFORMS) { test_name = "Broken Collective I/O - DATA Transforms"; - no_collective_cause_local_expected |= H5D_MPIO_DATA_TRANSFORMS; - no_collective_cause_global_expected |= H5D_MPIO_DATA_TRANSFORMS; + + /* Set transform */ + ret = H5Pset_data_transform(dxpl_write, "x+1"); + VRFY((ret >= 0), "H5Pset_data_transform succeeded"); + + /* Disable selection I/O since data transforms are supported in collective with selection I/O */ + ret = H5Pset_selection_io(dxpl_write, H5D_SELECTION_IO_MODE_OFF); + VRFY((ret >= 0), "H5Pset_selection_io succeeded"); + + no_collective_cause_local_expected |= H5D_MPIO_DATA_TRANSFORMS | H5D_MPIO_NO_SELECTION_IO; + no_collective_cause_global_expected |= H5D_MPIO_DATA_TRANSFORMS | H5D_MPIO_NO_SELECTION_IO; + no_selection_io_cause_expected |= H5D_SEL_IO_DISABLE_BY_API; } if (selection_mode & TEST_NOT_SIMPLE_OR_SCALAR_DATASPACES) { test_name = "Broken Collective I/O - No Simple or Scalar DataSpace"; no_collective_cause_local_expected |= H5D_MPIO_NOT_SIMPLE_OR_SCALAR_DATASPACES; no_collective_cause_global_expected |= H5D_MPIO_NOT_SIMPLE_OR_SCALAR_DATASPACES; + no_collective_cause_local_expected &= ~(unsigned)H5D_MPIO_NO_SELECTION_IO; + no_collective_cause_global_expected &= ~(unsigned)H5D_MPIO_NO_SELECTION_IO; } if (selection_mode & TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET_COMPACT || @@ -3996,6 +4023,8 @@ test_no_collective_cause_mode(int selection_mode) test_name = "Broken Collective I/O - No CONTI or CHUNKED Dataset"; no_collective_cause_local_expected |= H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; no_collective_cause_global_expected |= H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; + no_collective_cause_local_expected &= ~(unsigned)H5D_MPIO_NO_SELECTION_IO; + no_collective_cause_global_expected &= ~(unsigned)H5D_MPIO_NO_SELECTION_IO; } if (selection_mode & TEST_COLLECTIVE) { @@ -4008,6 +4037,8 @@ test_no_collective_cause_mode(int selection_mode) test_name = "Broken Collective I/O - Independent"; no_collective_cause_local_expected = H5D_MPIO_SET_INDEPENDENT; no_collective_cause_global_expected = H5D_MPIO_SET_INDEPENDENT; + no_collective_cause_local_expected &= ~(unsigned)H5D_MPIO_NO_SELECTION_IO; + no_collective_cause_global_expected &= ~(unsigned)H5D_MPIO_NO_SELECTION_IO; /* switch to independent io */ is_independent = 1; } @@ -4037,10 +4068,6 @@ test_no_collective_cause_mode(int selection_mode) for (i = 0; i < length; i++) buffer[i] = i; - /* Set up the dxpl for the write */ - dxpl_write = H5Pcreate(H5P_DATASET_XFER); - VRFY((dxpl_write >= 0), "H5Pcreate(H5P_DATASET_XFER) succeeded"); - if (is_independent) { /* Set Independent I/O */ ret = H5Pset_dxpl_mpio(dxpl_write, H5FD_MPIO_INDEPENDENT); @@ -4052,11 +4079,6 @@ test_no_collective_cause_mode(int selection_mode) VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); } - if (selection_mode & TEST_DATA_TRANSFORMS) { - ret = H5Pset_data_transform(dxpl_write, "x+1"); - VRFY((ret >= 0), "H5Pset_data_transform succeeded"); - } - /*--------------------- * Test Write access *---------------------*/ @@ -4072,6 +4094,20 @@ test_no_collective_cause_mode(int selection_mode) &no_collective_cause_global_write); VRFY((ret >= 0), "retrieving no collective cause succeeded"); + ret = H5Pget_no_selection_io_cause(dxpl_write, &no_selection_io_cause_write); + VRFY((ret >= 0), "retrieving no selection io cause succeeded"); + + if (no_collective_cause_local_write & H5D_MPIO_NO_SELECTION_IO) { + VRFY((no_selection_io_cause_write == no_selection_io_cause_expected), + "H5D_MPIO_NO_SELECTION_IO for write is as expected"); + } + + if (no_collective_cause_global_write & H5D_MPIO_NO_SELECTION_IO) { + + VRFY((no_selection_io_cause_write == no_selection_io_cause_expected), + "H5D_MPIO_NO_SELECTION_IO for write is as expected"); + } + /*--------------------- * Test Read access *---------------------*/ @@ -4092,6 +4128,21 @@ test_no_collective_cause_mode(int selection_mode) &no_collective_cause_global_read); VRFY((ret >= 0), "retrieving no collective cause succeeded"); + ret = H5Pget_no_selection_io_cause(dxpl_read, &no_selection_io_cause_read); + VRFY((ret >= 0), "retrieving no selection io cause succeeded"); + + if (no_collective_cause_local_read & H5D_MPIO_NO_SELECTION_IO) { + + VRFY((no_selection_io_cause_read == no_selection_io_cause_expected), + "H5D_MPIO_NO_SELECTION_IO for read is as expected"); + } + + if (no_collective_cause_global_read & H5D_MPIO_NO_SELECTION_IO) { + + VRFY((no_selection_io_cause_read == no_selection_io_cause_expected), + "H5D_MPIO_NO_SELECTION_IO for read is as expected"); + } + /* Check write vs read */ VRFY((no_collective_cause_local_read == no_collective_cause_local_write), "reading and writing are the same for local cause of Broken Collective I/O"); diff --git a/testpar/t_coll_chunk.c b/testpar/t_coll_chunk.c index 5f853e3..6636ffa 100644 --- a/testpar/t_coll_chunk.c +++ b/testpar/t_coll_chunk.c @@ -566,7 +566,8 @@ coll_chunktest(const char *filename, int chunk_factor, int select_factor, int ap hsize_t start[RANK], count[RANK], stride[RANK], block[RANK]; #ifdef H5_HAVE_INSTRUMENTED_LIBRARY - unsigned prop_value; + unsigned prop_value; + H5D_selection_io_mode_t selection_io_mode; #endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ int mpi_size, mpi_rank; @@ -804,7 +805,11 @@ coll_chunktest(const char *filename, int chunk_factor, int select_factor, int ap /* Only check chunk optimization mode if selection I/O is not being used - * selection I/O bypasses this IO mode decision - it's effectively always * multi chunk currently */ - if (facc_type == FACC_MPIO && !H5_use_selection_io_g) { + + status = H5Pget_selection_io(xfer_plist, &selection_io_mode); + VRFY((status >= 0), "testing property list get succeeded"); + + if (facc_type == FACC_MPIO && (selection_io_mode != H5D_SELECTION_IO_MODE_ON)) { switch (api_option) { case API_LINK_HARD: status = H5Pget(xfer_plist, H5D_XFER_COLL_CHUNK_LINK_HARD_NAME, &prop_value); diff --git a/testpar/t_dset.c b/testpar/t_dset.c index 34c4d97..d144235 100644 --- a/testpar/t_dset.c +++ b/testpar/t_dset.c @@ -3348,13 +3348,27 @@ test_actual_io_mode(int selection_mode) void actual_io_mode_tests(void) { - int mpi_size = -1; + H5D_selection_io_mode_t selection_io_mode; + hid_t dxpl_id = H5I_INVALID_HID; + herr_t ret; + int mpi_size = -1; + int mpi_rank = -1; + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); /* Only run these tests if selection I/O is not being used - selection I/O * bypasses this IO mode decision - it's effectively always multi chunk * currently */ - if (!H5_use_selection_io_g) { + + dxpl_id = H5Pcreate(H5P_DATASET_XFER); + VRFY((dxpl_id >= 0), "H5Pcreate(H5P_DATASET_XFER) succeeded"); + ret = H5Pget_selection_io(dxpl_id, &selection_io_mode); + VRFY((ret >= 0), "retrieving selection io mode succeeded"); + ret = H5Pclose(dxpl_id); + VRFY((ret >= 0), "H5Pclose succeeded"); + + if (selection_io_mode != H5D_SELECTION_IO_MODE_ON) { test_actual_io_mode(TEST_ACTUAL_IO_NO_COLLECTIVE); /* @@ -3438,6 +3452,10 @@ test_no_collective_cause_mode(int selection_mode) uint32_t no_collective_cause_global_read = 0; uint32_t no_collective_cause_global_expected = 0; + uint32_t no_selection_io_cause_write = 0; + uint32_t no_selection_io_cause_read = 0; + uint32_t no_selection_io_cause_expected = 0; + const char *filename; const char *test_name; hbool_t is_chunked = 1; @@ -3538,27 +3556,50 @@ test_no_collective_cause_mode(int selection_mode) dataset = H5Dcreate2(fid, "nocolcause", data_type, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); VRFY((dataset >= 0), "H5Dcreate2() dataset succeeded"); + /* Set up the dxpl for the write */ + dxpl_write = H5Pcreate(H5P_DATASET_XFER); + VRFY((dxpl_write >= 0), "H5Pcreate(H5P_DATASET_XFER) succeeded"); + /* * Set expected causes and some tweaks based on the type of test */ if (selection_mode & TEST_DATATYPE_CONVERSION) { test_name = "Broken Collective I/O - Datatype Conversion"; - no_collective_cause_local_expected |= H5D_MPIO_DATATYPE_CONVERSION; - no_collective_cause_global_expected |= H5D_MPIO_DATATYPE_CONVERSION; + /* set different sign to trigger type conversion */ data_type = H5T_NATIVE_UINT; + + /* Disable selection I/O since datatype conversion is supported in collective with selection I/O */ + ret = H5Pset_selection_io(dxpl_write, H5D_SELECTION_IO_MODE_OFF); + VRFY((ret >= 0), "H5Pset_selection_io succeeded"); + + no_collective_cause_local_expected |= H5D_MPIO_DATATYPE_CONVERSION | H5D_MPIO_NO_SELECTION_IO; + no_collective_cause_global_expected |= H5D_MPIO_DATATYPE_CONVERSION | H5D_MPIO_NO_SELECTION_IO; + no_selection_io_cause_expected |= H5D_SEL_IO_DISABLE_BY_API; } if (selection_mode & TEST_DATA_TRANSFORMS) { test_name = "Broken Collective I/O - DATA Transforms"; - no_collective_cause_local_expected |= H5D_MPIO_DATA_TRANSFORMS; - no_collective_cause_global_expected |= H5D_MPIO_DATA_TRANSFORMS; + + /* Set transform */ + ret = H5Pset_data_transform(dxpl_write, "x+1"); + VRFY((ret >= 0), "H5Pset_data_transform succeeded"); + + /* Disable selection I/O since data transforms are supported in collective with selection I/O */ + ret = H5Pset_selection_io(dxpl_write, H5D_SELECTION_IO_MODE_OFF); + VRFY((ret >= 0), "H5Pset_selection_io succeeded"); + + no_collective_cause_local_expected |= H5D_MPIO_DATA_TRANSFORMS | H5D_MPIO_NO_SELECTION_IO; + no_collective_cause_global_expected |= H5D_MPIO_DATA_TRANSFORMS | H5D_MPIO_NO_SELECTION_IO; + no_selection_io_cause_expected |= H5D_SEL_IO_DISABLE_BY_API; } if (selection_mode & TEST_NOT_SIMPLE_OR_SCALAR_DATASPACES) { test_name = "Broken Collective I/O - No Simple or Scalar DataSpace"; no_collective_cause_local_expected |= H5D_MPIO_NOT_SIMPLE_OR_SCALAR_DATASPACES; no_collective_cause_global_expected |= H5D_MPIO_NOT_SIMPLE_OR_SCALAR_DATASPACES; + no_collective_cause_local_expected &= ~(unsigned)H5D_MPIO_NO_SELECTION_IO; + no_collective_cause_global_expected &= ~(unsigned)H5D_MPIO_NO_SELECTION_IO; } if (selection_mode & TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET_COMPACT || @@ -3566,6 +3607,8 @@ test_no_collective_cause_mode(int selection_mode) test_name = "Broken Collective I/O - No CONTI or CHUNKED Dataset"; no_collective_cause_local_expected |= H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; no_collective_cause_global_expected |= H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; + no_collective_cause_local_expected &= ~(unsigned)H5D_MPIO_NO_SELECTION_IO; + no_collective_cause_global_expected &= ~(unsigned)H5D_MPIO_NO_SELECTION_IO; } if (selection_mode & TEST_COLLECTIVE) { @@ -3578,6 +3621,8 @@ test_no_collective_cause_mode(int selection_mode) test_name = "Broken Collective I/O - Independent"; no_collective_cause_local_expected = H5D_MPIO_SET_INDEPENDENT; no_collective_cause_global_expected = H5D_MPIO_SET_INDEPENDENT; + no_collective_cause_local_expected &= ~(unsigned)H5D_MPIO_NO_SELECTION_IO; + no_collective_cause_global_expected &= ~(unsigned)H5D_MPIO_NO_SELECTION_IO; /* switch to independent io */ is_independent = 1; } @@ -3607,10 +3652,6 @@ test_no_collective_cause_mode(int selection_mode) for (i = 0; i < length; i++) buffer[i] = i; - /* Set up the dxpl for the write */ - dxpl_write = H5Pcreate(H5P_DATASET_XFER); - VRFY((dxpl_write >= 0), "H5Pcreate(H5P_DATASET_XFER) succeeded"); - if (is_independent) { /* Set Independent I/O */ ret = H5Pset_dxpl_mpio(dxpl_write, H5FD_MPIO_INDEPENDENT); @@ -3642,6 +3683,20 @@ test_no_collective_cause_mode(int selection_mode) &no_collective_cause_global_write); VRFY((ret >= 0), "retrieving no collective cause succeeded"); + ret = H5Pget_no_selection_io_cause(dxpl_write, &no_selection_io_cause_write); + VRFY((ret >= 0), "retrieving no selection io cause succeeded"); + + if (no_collective_cause_local_write & H5D_MPIO_NO_SELECTION_IO) { + VRFY((no_selection_io_cause_write == no_selection_io_cause_expected), + "H5D_MPIO_NO_SELECTION_IO for write is as expected"); + } + + if (no_collective_cause_global_write & H5D_MPIO_NO_SELECTION_IO) { + + VRFY((no_selection_io_cause_write == no_selection_io_cause_expected), + "H5D_MPIO_NO_SELECTION_IO for write is as expected"); + } + /*--------------------- * Test Read access *---------------------*/ @@ -3662,6 +3717,21 @@ test_no_collective_cause_mode(int selection_mode) &no_collective_cause_global_read); VRFY((ret >= 0), "retrieving no collective cause succeeded"); + ret = H5Pget_no_selection_io_cause(dxpl_read, &no_selection_io_cause_read); + VRFY((ret >= 0), "retrieving no selection io cause succeeded"); + + if (no_collective_cause_local_read & H5D_MPIO_NO_SELECTION_IO) { + + VRFY((no_selection_io_cause_read == no_selection_io_cause_expected), + "H5D_MPIO_NO_SELECTION_IO for read is as expected"); + } + + if (no_collective_cause_global_read & H5D_MPIO_NO_SELECTION_IO) { + + VRFY((no_selection_io_cause_read == no_selection_io_cause_expected), + "H5D_MPIO_NO_SELECTION_IO for read is as expected"); + } + /* Check write vs read */ VRFY((no_collective_cause_local_read == no_collective_cause_local_write), "reading and writing are the same for local cause of Broken Collective I/O"); diff --git a/testpar/t_select_io_dset.c b/testpar/t_select_io_dset.c new file mode 100644 index 0000000..10b29e4 --- /dev/null +++ b/testpar/t_select_io_dset.c @@ -0,0 +1,3786 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * + * Purpose: Test selection I/O + */ + +#include "h5test.h" +#include "testpar.h" + +#define FILENAME "pselect_io.h5" + +/* MPI variables */ +int mpi_size; +int mpi_rank; + +/* Number of errors */ +int nerrors = 0; +int curr_nerrors = 0; + +#define P_TEST_ERROR \ + do { \ + nerrors++; \ + H5_FAILED(); \ + AT(); \ + } while (0) + +#define CHECK_PASSED() \ + do { \ + int err_result = (nerrors > curr_nerrors); \ + \ + MPI_Allreduce(MPI_IN_PLACE, &err_result, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); \ + \ + if (MAINPROCESS) { \ + if (err_result == 0) \ + PASSED(); \ + else \ + HDputs(" ***TEST FAILED***"); \ + } \ + } while (0) + +/* + * Test configurations + */ +typedef enum { + TEST_NO_TYPE_CONV, /* no type conversion (null case) */ + TEST_NO_SIZE_CHANGE_NO_BKG, /* no size change, no bkg buffer */ + TEST_LARGER_MEM_NO_BKG, /* larger memory type, no bkg buffer */ + TEST_SMALLER_MEM_NO_BKG, /* smaller memory type, no bkg buffer */ + TEST_CMPD_WITH_BKG, /* compound types with bkg buffer */ + TEST_TYPE_CONV_SEL_EMPTY, /* some processes have null/empty selections and with type conversion */ + TEST_MULTI_CONV_NO_BKG, /* multi dataset test 1 */ + TEST_MULTI_CONV_BKG, /* multi dataset test 2 */ + TEST_MULTI_CONV_SIZE_CHANGE, /* multi dataset test 3 */ + TEST_MULTI_CONV_SEL_EMPTY, /* multi dataset test 4 */ + TEST_MULTI_ALL, /* multi dataset test 5 */ + TEST_SELECT_NTESTS +} test_select_config_t; + +#define DSET_SELECT_DIM 100 +#define DSET_SELECT_CHUNK_DIM 10 + +#define MULTI_NUM_DSETS 3 +#define MULTI_MIN_DSETS 3 +#define DSET_NAME_LEN 64 + +/* Compound type */ +typedef struct s1_t { + int a; + int b; + int c; + int d; +} s1_t; + +/* + * Variation of s1 with: + * --no conversion for 2 member types + * --1 larger mem type, + * --1 smaller mem type + */ +typedef struct s2_t { + int a; + long long b; + int c; + short d; +} s2_t; + +/* Variation of s1: reverse of s1_t */ +typedef struct s3_t { + int d; + int c; + int b; + int a; +} s3_t; + +/* Variations of s1: only 2 members in s1_t */ +typedef struct s4_t { + unsigned int b; + unsigned int d; +} s4_t; + +/* Defines for test_multi_dsets_all() */ +typedef enum { + DSET_WITH_NO_CONV, /* Dataset with no type conversion */ + DSET_WITH_CONV_AND_NO_BKG, /* Dataset with type conversion but no background buffer */ + DSET_WITH_CONV_AND_BKG, /* Dataset with type conversion and background buffer */ + DSET_NTTYPES +} multi_dset_type_t; + +/* Test setting A and B */ +#define SETTING_A 1 +#define SETTING_B 2 + +/* Definitions of the test modes for test_get_no_selection_io_cause() */ +#define TEST_DISABLE_BY_API 0x001 +#define TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET 0x002 +#define TEST_DATATYPE_CONVERSION 0x004 +#define TEST_TCONV_BUF_TOO_SMALL 0x008 +#define TEST_IN_PLACE_TCONV 0x010 + +/* + * Helper routine to set dxpl + * --selection I/O mode + * --type of I/O + * --type of collective I/O + */ +static void +set_dxpl(hid_t dxpl, H5D_selection_io_mode_t select_io_mode, H5FD_mpio_xfer_t mpio_type, + H5FD_mpio_collective_opt_t mpio_coll_opt, unsigned mwbuf) +{ + if (H5Pset_selection_io(dxpl, select_io_mode) < 0) + P_TEST_ERROR; + + if (H5Pset_dxpl_mpio(dxpl, mpio_type) < 0) + P_TEST_ERROR; + + if (H5Pset_dxpl_mpio_collective_opt(dxpl, mpio_coll_opt) < 0) + P_TEST_ERROR; + + if (mwbuf) + if (H5Pset_modify_write_buf(dxpl, TRUE) < 0) + P_TEST_ERROR; + +} /* set_dxpl() */ + +/* + * Helper routine to check actual I/O mode on a dxpl + */ +static void +check_io_mode(hid_t dxpl, unsigned chunked) +{ + H5D_mpio_actual_io_mode_t actual_io_mode = H5D_MPIO_NO_COLLECTIVE; + + if (H5Pget_mpio_actual_io_mode(dxpl, &actual_io_mode) < 0) + P_TEST_ERROR; + + if (chunked) { + if (actual_io_mode != H5D_MPIO_CHUNK_COLLECTIVE) { + nerrors++; + if (MAINPROCESS) + HDprintf("\n Failed: Incorrect I/O mode (expected chunked, returned %u)", + (unsigned)actual_io_mode); + } + } + else if (actual_io_mode != H5D_MPIO_CONTIGUOUS_COLLECTIVE) { + nerrors++; + if (MAINPROCESS) + HDprintf("\n Failed: Incorrect I/O mode (expected contiguous, returned %u)", + (unsigned)actual_io_mode); + } + +} /* check_io_mode() */ + +/* + * Case 1: single dataset read/write, no type conversion (null case) + */ +static void +test_no_type_conv(hid_t fid, unsigned chunked, unsigned dtrans, unsigned mwbuf) +{ + int i; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t ntrans_dxpl = H5I_INVALID_HID; + hid_t fspace_id = H5I_INVALID_HID; + hid_t mspace_id = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + hsize_t start[1], stride[1], count[1], block[1]; + int wbuf[DSET_SELECT_DIM]; + int wbuf_bak[DSET_SELECT_DIM]; + int trans_wbuf[DSET_SELECT_DIM]; + int rbuf[DSET_SELECT_DIM]; + char dset_name[DSET_NAME_LEN]; + const char *expr = "2*x"; + + curr_nerrors = nerrors; + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + P_TEST_ERROR; + + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + P_TEST_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + P_TEST_ERROR; + } + + /* Generate dataset name */ + HDsnprintf(dset_name, sizeof(dset_name), "no_tconv_%s_%s_%s", chunked ? "chunked" : "contig", + dtrans ? "xform" : "noxform", mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create dataset */ + if ((did = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + P_TEST_ERROR; + + /* Each process takes x number of elements */ + block[0] = dims[0] / (hsize_t)mpi_size; + stride[0] = block[0]; + count[0] = 1; + start[0] = (hsize_t)mpi_rank * block[0]; + + /* Initialize data */ + for (i = 0; i < (int)block[0]; i++) { + wbuf[i] = i + (int)start[0]; + trans_wbuf[i] = 2 * wbuf[i]; + } + + /* Create a memory dataspace */ + if ((mspace_id = H5Screate_simple(1, block, NULL)) < 0) + P_TEST_ERROR; + + /* Create a file dataspace */ + if ((fspace_id = H5Dget_space(did)) < 0) + P_TEST_ERROR; + + if (H5Sselect_hyperslab(fspace_id, H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + P_TEST_ERROR; + + /* Set selection I/O mode, type of I/O and type of collective I/O */ + set_dxpl(dxpl, H5D_SELECTION_IO_MODE_ON, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, mwbuf); + + if ((ntrans_dxpl = H5Pcopy(dxpl)) < 0) + P_TEST_ERROR; + + /* Set data transform */ + if (dtrans) + if (H5Pset_data_transform(dxpl, expr) < 0) + P_TEST_ERROR; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(wbuf_bak, wbuf, sizeof(wbuf)); + + /* Write data to the dataset with/without data transform */ + if (H5Dwrite(did, H5T_NATIVE_INT, mspace_id, fspace_id, dxpl, wbuf) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(wbuf, wbuf_bak, sizeof(wbuf)); + + check_io_mode(dxpl, chunked); + + /* Read data from the dataset (if dtrans, without data transform set in dxpl) */ + if (H5Dread(did, H5T_NATIVE_INT, mspace_id, fspace_id, ntrans_dxpl, rbuf) < 0) + P_TEST_ERROR; + + /* Verify data read (if dtrans, verify data is transformed) */ + for (i = 0; i < (int)block[0]; i++) + if (rbuf[i] != (dtrans ? trans_wbuf[i] : wbuf[i])) { + nerrors++; + HDprintf("\n Error in first data verification:\n"); + HDprintf(" At index %d: %d, %d\n", i + (int)start[0], dtrans ? trans_wbuf[i] : wbuf[i], + rbuf[i]); + break; + } + + if (dtrans) { + + /* Read the data from the dataset with data transform set in dxpl */ + if (H5Dread(did, H5T_NATIVE_INT, mspace_id, fspace_id, dxpl, rbuf) < 0) + P_TEST_ERROR; + + /* Verify data read is transformed a second time */ + for (i = 0; i < (int)block[0]; i++) + if (rbuf[i] != (2 * trans_wbuf[i])) { + nerrors++; + HDprintf("\n Error in second data verification:.\n"); + HDprintf(" At index %d: %d, %d\n", i + (int)start[0], 2 * trans_wbuf[i], rbuf[i]); + break; + } + } + + if (H5Sclose(mspace_id) < 0) + P_TEST_ERROR; + if (H5Sclose(fspace_id) < 0) + P_TEST_ERROR; + + if (H5Sclose(sid) < 0) + P_TEST_ERROR; + if (H5Dclose(did) < 0) + P_TEST_ERROR; + if (H5Pclose(dxpl) < 0) + P_TEST_ERROR; + if (H5Pclose(ntrans_dxpl) < 0) + P_TEST_ERROR; + + CHECK_PASSED(); + + return; +} /* test_no_type_conv() */ + +/* + * Case 2: single dataset read/write, no size change, no background buffer + */ +static void +test_no_size_change_no_bkg(hid_t fid, unsigned chunked, unsigned mwbuf) +{ + int i; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t fspace_id = H5I_INVALID_HID; + hid_t mspace_id = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + hsize_t start[1], stride[1], count[1], block[1]; + char *wbuf = NULL; + char *wbuf_bak = NULL; + char *rbuf = NULL; + char dset_name[DSET_NAME_LEN]; + + curr_nerrors = nerrors; + + if ((wbuf = (char *)HDmalloc((size_t)(4 * DSET_SELECT_DIM))) == NULL) + P_TEST_ERROR; + if (mwbuf && (wbuf_bak = (char *)HDmalloc((size_t)(4 * DSET_SELECT_DIM))) == NULL) + P_TEST_ERROR; + if ((rbuf = (char *)HDmalloc((size_t)(4 * DSET_SELECT_DIM))) == NULL) + P_TEST_ERROR; + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + P_TEST_ERROR; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + P_TEST_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + P_TEST_ERROR; + } + + /* Generate dataset name */ + HDsnprintf(dset_name, sizeof(dset_name), "no_size_change_%s_%s", chunked ? "chunked" : "contig", + mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create 1d dataset */ + if ((did = H5Dcreate2(fid, dset_name, H5T_STD_I32BE, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + P_TEST_ERROR; + + /* Each process takes x number of elements */ + block[0] = dims[0] / (hsize_t)mpi_size; + stride[0] = block[0]; + count[0] = 1; + start[0] = (hsize_t)mpi_rank * block[0]; + + /* Initialize data */ + for (i = 0; i < (int)block[0]; i++) { + wbuf[i * 4 + 3] = 0x1; + wbuf[i * 4 + 2] = 0x2; + wbuf[i * 4 + 1] = 0x3; + wbuf[i * 4 + 0] = 0x4; + } + + /* Create a memory dataspace independently */ + if ((mspace_id = H5Screate_simple(1, block, NULL)) < 0) + P_TEST_ERROR; + + /* Create a file dataspace independently */ + if ((fspace_id = H5Dget_space(did)) < 0) + P_TEST_ERROR; + + if (H5Sselect_hyperslab(fspace_id, H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + P_TEST_ERROR; + + /* Set selection I/O mode, type of I/O and type of collective I/O */ + set_dxpl(dxpl, H5D_SELECTION_IO_MODE_ON, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, mwbuf); + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(wbuf_bak, wbuf, (size_t)(4 * DSET_SELECT_DIM)); + + /* Write the data to the dataset with little endian */ + if (H5Dwrite(did, H5T_STD_I32LE, mspace_id, fspace_id, dxpl, wbuf) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(wbuf, wbuf_bak, (size_t)(4 * DSET_SELECT_DIM)); + + check_io_mode(dxpl, chunked); + + /* Read the data from the dataset with little endian */ + if (H5Dread(did, H5T_STD_I32LE, mspace_id, fspace_id, dxpl, rbuf) < 0) + P_TEST_ERROR; + + /* Verify data read */ + for (i = 0; i < (int)block[0]; i++) { + if (rbuf[4 * i + 0] != wbuf[4 * i + 0] || rbuf[4 * i + 1] != wbuf[4 * i + 1] || + rbuf[4 * i + 2] != wbuf[4 * i + 2] || rbuf[4 * i + 3] != wbuf[4 * i + 3]) { + nerrors++; + HDprintf("\n Error in data verification:\n"); + HDprintf("\n Error in data verification at index %d\n", i + (int)start[0]); + break; + } + } + + /* Read the data from the dataset with big endian */ + if (H5Dread(did, H5T_STD_I32BE, mspace_id, fspace_id, dxpl, rbuf) < 0) + P_TEST_ERROR; + + /* Verify data read */ + for (i = 0; i < (int)block[0]; i++) { + if (rbuf[4 * i + 0] != wbuf[4 * i + 3] || rbuf[4 * i + 1] != wbuf[4 * i + 2] || + rbuf[4 * i + 2] != wbuf[4 * i + 1] || rbuf[4 * i + 3] != wbuf[4 * i + 0]) { + nerrors++; + HDprintf("\n Error in data verification at index %d\n", i + (int)start[0]); + break; + } + } + + if (H5Sclose(mspace_id) < 0) + P_TEST_ERROR; + if (H5Sclose(fspace_id) < 0) + P_TEST_ERROR; + + if (H5Sclose(sid) < 0) + P_TEST_ERROR; + if (H5Dclose(did) < 0) + P_TEST_ERROR; + + if (wbuf) + HDfree(wbuf); + + if (wbuf_bak) + HDfree(wbuf_bak); + + if (rbuf) + HDfree(rbuf); + + CHECK_PASSED(); + + return; +} /* test_no_size_change_no_bkg() */ + +/* + * Case 3: single dataset read/write, larger mem type, no background buffer + */ +static void +test_larger_mem_type_no_bkg(hid_t fid, unsigned chunked, unsigned dtrans, unsigned mwbuf) +{ + int i; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t ntrans_dxpl = H5I_INVALID_HID; + hid_t fspace_id = H5I_INVALID_HID; + hid_t mspace_id = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + hsize_t start[1], stride[1], count[1], block[1]; + long wbuf[DSET_SELECT_DIM]; + long wbuf_bak[DSET_SELECT_DIM]; + long trans_wbuf[DSET_SELECT_DIM]; + long long rbuf[DSET_SELECT_DIM]; + char dset_name[DSET_NAME_LEN]; + const char *expr = "100 - x"; + + curr_nerrors = nerrors; + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + P_TEST_ERROR; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + P_TEST_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + P_TEST_ERROR; + } + + /* Generate dataset name */ + HDsnprintf(dset_name, sizeof(dset_name), "larger_no_bkg_%s_%s_%s", chunked ? "chunked" : "contig", + dtrans ? "xform" : "noxform", mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create 1d chunked dataset with/without data transform */ + if ((did = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + P_TEST_ERROR; + + /* Each process takes x number of elements */ + block[0] = dims[0] / (hsize_t)mpi_size; + stride[0] = block[0]; + count[0] = 1; + start[0] = (hsize_t)mpi_rank * block[0]; + + /* Initialize data */ + for (i = 0; i < (int)block[0]; i++) { + wbuf[i] = i + (int)start[0]; + trans_wbuf[i] = 100 - wbuf[i]; + } + + /* Create a memory dataspace */ + if ((mspace_id = H5Screate_simple(1, block, NULL)) < 0) + P_TEST_ERROR; + + /* Create a file dataspace */ + if ((fspace_id = H5Dget_space(did)) < 0) + P_TEST_ERROR; + + if (H5Sselect_hyperslab(fspace_id, H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + P_TEST_ERROR; + + /* Set selection I/O mode, type of I/O and type of collective I/O */ + set_dxpl(dxpl, H5D_SELECTION_IO_MODE_ON, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, mwbuf); + + if ((ntrans_dxpl = H5Pcopy(dxpl)) < 0) + P_TEST_ERROR; + + /* Set data transform */ + if (dtrans) + if (H5Pset_data_transform(dxpl, expr) < 0) + P_TEST_ERROR; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(wbuf_bak, wbuf, sizeof(wbuf)); + + /* Write data to the dataset with/without data transform set in dxpl */ + if (H5Dwrite(did, H5T_NATIVE_LONG, mspace_id, fspace_id, dxpl, wbuf) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(wbuf, wbuf_bak, sizeof(wbuf)); + + check_io_mode(dxpl, chunked); + + /* Read data from the dataset (if dtrans, without data transform set in dxpl) */ + if (H5Dread(did, H5T_NATIVE_LLONG, mspace_id, fspace_id, ntrans_dxpl, rbuf) < 0) + P_TEST_ERROR; + + /* Verify data read (if dtrans, verify data is transformed) */ + for (i = 0; i < (int)block[0]; i++) + if (rbuf[i] != (long long)(dtrans ? trans_wbuf[i] : wbuf[i])) { + nerrors++; + HDprintf("\n Error in first data verification:\n"); + HDprintf(" At index %d: %lld, %lld\n", i + (int)start[0], + (long long)(dtrans ? trans_wbuf[i] : wbuf[i]), rbuf[i]); + break; + } + + if (dtrans) { + + /* Read data from the dataset with data transform set in dxpl */ + if (H5Dread(did, H5T_NATIVE_LLONG, mspace_id, fspace_id, dxpl, rbuf) < 0) + P_TEST_ERROR; + + /* Verify data read is transformed a second time */ + for (i = 0; i < (int)block[0]; i++) + if (rbuf[i] != (long long)(100 - trans_wbuf[i])) { + nerrors++; + HDprintf("\n Error in second data verification:.\n"); + HDprintf(" At index %d: %lld, %lld\n", i + (int)start[0], + (long long)(100 - trans_wbuf[i]), rbuf[i]); + break; + } + } + if (H5Sclose(mspace_id) < 0) + P_TEST_ERROR; + if (H5Sclose(fspace_id) < 0) + P_TEST_ERROR; + if (H5Sclose(sid) < 0) + P_TEST_ERROR; + if (H5Dclose(did) < 0) + P_TEST_ERROR; + if (H5Pclose(dxpl) < 0) + P_TEST_ERROR; + if (H5Pclose(ntrans_dxpl) < 0) + P_TEST_ERROR; + + CHECK_PASSED(); + + return; + +} /* test_larger_mem_type_no_bkg() */ + +/* + * Case 4: single dataset reader/write, smaller mem type, no background buffer + */ +static void +test_smaller_mem_type_no_bkg(hid_t fid, unsigned chunked, unsigned dtrans, unsigned mwbuf) +{ + int i; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t ntrans_dxpl = H5I_INVALID_HID; + hid_t fspace_id = H5I_INVALID_HID; + hid_t mspace_id = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + hsize_t start[1], stride[1], count[1], block[1]; + short wbuf[DSET_SELECT_DIM]; + int wbuf_bak[DSET_SELECT_DIM]; + short trans_wbuf[DSET_SELECT_DIM]; + short rbuf[DSET_SELECT_DIM]; + char dset_name[DSET_NAME_LEN]; + const char *expr = "2 * (10 + x)"; + + curr_nerrors = nerrors; + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + P_TEST_ERROR; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + P_TEST_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + P_TEST_ERROR; + } + + /* Generate dataset name */ + HDsnprintf(dset_name, sizeof(dset_name), "smaller_no_bkg_%s_%s_%s", chunked ? "chunked" : "contig", + dtrans ? "xform" : "noxform", mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create 1d chunked dataset with/without data transform */ + if ((did = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + P_TEST_ERROR; + + /* Each process takes x number of elements */ + block[0] = dims[0] / (hsize_t)mpi_size; + stride[0] = block[0]; + count[0] = 1; + start[0] = (hsize_t)mpi_rank * block[0]; + + /* Initialize data */ + for (i = 0; i < (int)block[0]; i++) { + wbuf[i] = (short)(i + (int)start[0]); + trans_wbuf[i] = (short)(2 * (10 + wbuf[i])); + } + + /* Create a memory dataspace */ + if ((mspace_id = H5Screate_simple(1, block, NULL)) < 0) + P_TEST_ERROR; + + /* Create a file dataspace */ + if ((fspace_id = H5Dget_space(did)) < 0) + P_TEST_ERROR; + + if (H5Sselect_hyperslab(fspace_id, H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + P_TEST_ERROR; + + /* Set selection I/O mode, type of I/O and type of collective I/O */ + set_dxpl(dxpl, H5D_SELECTION_IO_MODE_ON, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, mwbuf); + + if ((ntrans_dxpl = H5Pcopy(dxpl)) < 0) + P_TEST_ERROR; + + /* Set data transform */ + if (dtrans) { + if (H5Pset_data_transform(dxpl, expr) < 0) + P_TEST_ERROR; + } + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(wbuf_bak, wbuf, sizeof(wbuf)); + + /* Write data to the dataset with/without data transform in dxpl */ + if (H5Dwrite(did, H5T_NATIVE_SHORT, mspace_id, fspace_id, dxpl, wbuf) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(wbuf, wbuf_bak, sizeof(wbuf)); + + check_io_mode(dxpl, chunked); + + /* Read data from the dataset (if dtrans, without data transform set in dxpl) */ + if (H5Dread(did, H5T_NATIVE_SHORT, mspace_id, fspace_id, ntrans_dxpl, rbuf) < 0) + P_TEST_ERROR; + + /* Verify data read (if dtrans, verify data is transformed) */ + for (i = 0; i < (int)block[0]; i++) + if (rbuf[i] != (dtrans ? trans_wbuf[i] : wbuf[i])) { + nerrors++; + HDprintf("\n Error in first data verification:\n"); + HDprintf(" At index %d: %d, %d\n", i + (int)start[0], wbuf[i], rbuf[i]); + break; + } + + if (dtrans) { + + /* Read data from the dataset with data transform set in dxpl */ + if (H5Dread(did, H5T_NATIVE_SHORT, mspace_id, fspace_id, dxpl, rbuf) < 0) + P_TEST_ERROR; + + /* Verify data read is transformed a second time */ + for (i = 0; i < (int)block[0]; i++) + if (rbuf[i] != (2 * (10 + trans_wbuf[i]))) { + nerrors++; + HDprintf("\n Error in second data verification:.\n"); + HDprintf(" At index %d: %d, %d\n", i + (int)start[0], (2 * (10 - trans_wbuf[i])), + rbuf[i]); + break; + } + } + + if (H5Sclose(mspace_id) < 0) + P_TEST_ERROR; + if (H5Sclose(fspace_id) < 0) + P_TEST_ERROR; + + if (H5Sclose(sid) < 0) + P_TEST_ERROR; + if (H5Dclose(did) < 0) + P_TEST_ERROR; + if (H5Pclose(dxpl) < 0) + P_TEST_ERROR; + if (H5Pclose(ntrans_dxpl) < 0) + P_TEST_ERROR; + + CHECK_PASSED(); + + return; + +} /* test_smaller_mem_type_no_bkg() */ + +/* + * Case 5: single dataset reade/write, compound types with background buffer + * + * (a) Initialize compound buffer in memory with unique values + * Write all compound fields to disk + * Verify values read + * (b) Update all fields of the compound type in memory write buffer with new unique values + * Write some but not all all compound fields to disk + * Read the entire compound type + * Verify the fields have the correct (old or new) values + * (c) Update all fields of the compound type in memory read buffer with new unique values + * Read some but not all the compound fields to memory + * Verify the fields have the correct (old, middle or new) values + * (d) Set up a different compound type which has: + * --no conversion for member types + * --a field with larger mem type + * --a field with smaller mem type + * Write this compound type to disk + * Read the entire compound type + * Verify the values read + */ +static void +test_cmpd_with_bkg(hid_t fid, unsigned chunked, unsigned mwbuf) +{ + int i; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t s1_tid = H5I_INVALID_HID; + hid_t s2_tid = H5I_INVALID_HID; + hid_t ss_ac_tid = H5I_INVALID_HID; + hid_t ss_bc_tid = H5I_INVALID_HID; + hid_t fspace_id = H5I_INVALID_HID; + hid_t mspace_id = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + hsize_t start[1], stride[1], count[1], block[1]; + s1_t *s1_wbuf = NULL; + s1_t *s1_wbuf_bak = NULL; + s1_t *s1_rbuf = NULL; + s2_t *s2_wbuf = NULL; + s2_t *s2_wbuf_bak = NULL; + s2_t *s2_rbuf = NULL; + char dset_name[DSET_NAME_LEN]; + + curr_nerrors = nerrors; + + /* Allocate buffers for datasets */ + if (NULL == (s1_wbuf = (s1_t *)HDmalloc(sizeof(s1_t) * DSET_SELECT_DIM))) + P_TEST_ERROR; + if (mwbuf && NULL == (s1_wbuf_bak = (s1_t *)HDmalloc(sizeof(s1_t) * DSET_SELECT_DIM))) + P_TEST_ERROR; + if (NULL == (s1_rbuf = (s1_t *)HDmalloc(sizeof(s1_t) * DSET_SELECT_DIM))) + P_TEST_ERROR; + if (NULL == (s2_wbuf = (s2_t *)HDmalloc(sizeof(s2_t) * DSET_SELECT_DIM))) + P_TEST_ERROR; + if (mwbuf && NULL == (s2_wbuf_bak = (s2_t *)HDmalloc(sizeof(s2_t) * DSET_SELECT_DIM))) + P_TEST_ERROR; + if (NULL == (s2_rbuf = (s2_t *)HDmalloc(sizeof(s2_t) * DSET_SELECT_DIM))) + P_TEST_ERROR; + + /* Create the memory data type */ + if ((s1_tid = H5Tcreate(H5T_COMPOUND, sizeof(s1_t))) < 0) + P_TEST_ERROR; + + if (H5Tinsert(s1_tid, "a", HOFFSET(s1_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "b", HOFFSET(s1_t, b), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "c", HOFFSET(s1_t, c), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "d", HOFFSET(s1_t, d), H5T_NATIVE_INT) < 0) + P_TEST_ERROR; + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + P_TEST_ERROR; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + P_TEST_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + P_TEST_ERROR; + } + + /* Case 5(a) */ + + /* Generate dataset name */ + HDsnprintf(dset_name, sizeof(dset_name), "cmpd_with_bkg_%s_%s", chunked ? "chunked" : "contig", + mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create 1d dataset */ + if ((did = H5Dcreate2(fid, dset_name, s1_tid, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + P_TEST_ERROR; + + /* Each process takes x number of elements */ + block[0] = dims[0] / (hsize_t)mpi_size; + stride[0] = block[0]; + count[0] = 1; + start[0] = (hsize_t)mpi_rank * block[0]; + + /* Initialize data */ + for (i = 0; i < (int)block[0]; i++) { + s1_wbuf[i].a = 4 * (i + (int)start[0]); + s1_wbuf[i].b = 4 * (i + (int)start[0]) + 1; + s1_wbuf[i].c = 4 * (i + (int)start[0]) + 2; + s1_wbuf[i].d = 4 * (i + (int)start[0]) + 3; + } + + /* Create a memory dataspace */ + if ((mspace_id = H5Screate_simple(1, block, NULL)) < 0) + P_TEST_ERROR; + + /* Create a file dataspace */ + if ((fspace_id = H5Dget_space(did)) < 0) + P_TEST_ERROR; + + if (H5Sselect_hyperslab(fspace_id, H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + P_TEST_ERROR; + + /* Set selection I/O mode, type of I/O and type of collective I/O */ + set_dxpl(dxpl, H5D_SELECTION_IO_MODE_ON, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, mwbuf); + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(s1_wbuf_bak, s1_wbuf, sizeof(s1_t) * DSET_SELECT_DIM); + + /* Write all the data to the dataset */ + if (H5Dwrite(did, s1_tid, mspace_id, fspace_id, dxpl, s1_wbuf) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(s1_wbuf, s1_wbuf_bak, sizeof(s1_t) * DSET_SELECT_DIM); + + check_io_mode(dxpl, chunked); + + /* Read all the data from the dataset */ + HDmemset(s1_rbuf, 0, sizeof(s1_t) * DSET_SELECT_DIM); + if (H5Dread(did, s1_tid, mspace_id, fspace_id, dxpl, s1_rbuf) < 0) + P_TEST_ERROR; + + /* Verify data read */ + for (i = 0; i < (int)block[0]; i++) + if (s1_wbuf[i].a != s1_rbuf[i].a || s1_wbuf[i].b != s1_rbuf[i].b || s1_wbuf[i].c != s1_rbuf[i].c || + s1_wbuf[i].d != s1_rbuf[i].d) { + nerrors++; + HDprintf("\n Error in 1st data verification:\n"); + HDprintf(" At index %d: %d/%d, %d/%d, %d/%d, %d/%d\n", i + (int)start[0], s1_wbuf[i].a, + s1_rbuf[i].a, s1_wbuf[i].b, s1_rbuf[i].b, s1_wbuf[i].c, s1_rbuf[i].c, s1_wbuf[i].d, + s1_rbuf[i].d); + break; + } + + /* Case 5(b) */ + + /* Update s1_wbuf with unique values */ + for (i = 0; i < (int)block[0]; i++) { + s1_wbuf[i].a = 4 * (i + (int)start[0]) + DSET_SELECT_DIM; + s1_wbuf[i].b = 4 * (i + (int)start[0]) + DSET_SELECT_DIM + 1; + s1_wbuf[i].c = 4 * (i + (int)start[0]) + DSET_SELECT_DIM + 2; + s1_wbuf[i].d = 4 * (i + (int)start[0]) + DSET_SELECT_DIM + 3; + } + + /* Create a compound type same size as s1_t */ + if ((ss_ac_tid = H5Tcreate(H5T_COMPOUND, sizeof(s1_t))) < 0) + P_TEST_ERROR; + + /* but contains only subset members of s1_t */ + if (H5Tinsert(ss_ac_tid, "a", HOFFSET(s1_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(ss_ac_tid, "c", HOFFSET(s1_t, c), H5T_NATIVE_INT) < 0) + P_TEST_ERROR; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(s1_wbuf_bak, s1_wbuf, sizeof(s1_t) * DSET_SELECT_DIM); + + /* Write s1_wbuf to the dataset but with only subset members in ss_tid */ + if (H5Dwrite(did, ss_ac_tid, mspace_id, fspace_id, dxpl, s1_wbuf) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(s1_wbuf, s1_wbuf_bak, sizeof(s1_t) * DSET_SELECT_DIM); + + /* Read the whole compound back */ + HDmemset(s1_rbuf, 0, sizeof(s1_t) * DSET_SELECT_DIM); + if (H5Dread(did, s1_tid, mspace_id, fspace_id, dxpl, s1_rbuf) < 0) + P_TEST_ERROR; + + /* Verify the compound fields have the correct (old or new) values */ + for (i = 0; i < (int)block[0]; i++) + if (s1_rbuf[i].a != s1_wbuf[i].a || s1_rbuf[i].b != (4 * (i + (int)start[0]) + 1) || + s1_rbuf[i].c != s1_wbuf[i].c || s1_rbuf[i].d != (4 * (i + (int)start[0]) + 3)) { + nerrors++; + HDprintf("\n Error in 2nd data verification:\n"); + HDprintf(" At index %d: %d/%d, %d/%d, %d/%d, %d/%d\n", i + (int)start[0], s1_wbuf[i].a, + s1_rbuf[i].a, (4 * (i + (int)start[0]) + 1), s1_rbuf[i].b, s1_wbuf[i].c, s1_rbuf[i].c, + (4 * (i + (int)start[0]) + 3), s1_rbuf[i].d); + break; + } + + /* Case 5(c) */ + + /* Update s1_rbuf with new unique values */ + for (i = 0; i < (int)block[0]; i++) { + s1_rbuf[i].a = (4 * (i + (int)start[0])) + (2 * DSET_SELECT_DIM); + s1_rbuf[i].b = (4 * (i + (int)start[0])) + (2 * DSET_SELECT_DIM) + 1; + s1_rbuf[i].c = (4 * (i + (int)start[0])) + (2 * DSET_SELECT_DIM) + 2; + s1_rbuf[i].d = (4 * (i + (int)start[0])) + (2 * DSET_SELECT_DIM) + 3; + } + + /* Create a compound type same size as s1_t */ + if ((ss_bc_tid = H5Tcreate(H5T_COMPOUND, sizeof(s1_t))) < 0) + P_TEST_ERROR; + + /* but contains only subset members of s1_t */ + if (H5Tinsert(ss_bc_tid, "b", HOFFSET(s1_t, b), H5T_NATIVE_INT) < 0 || + H5Tinsert(ss_bc_tid, "c", HOFFSET(s1_t, c), H5T_NATIVE_INT) < 0) + P_TEST_ERROR; + + /* Read the dataset: will read only what is set in ss_bc_tid */ + if (H5Dread(did, ss_bc_tid, mspace_id, fspace_id, dxpl, s1_rbuf) < 0) + P_TEST_ERROR; + + /* Verify data read */ + for (i = 0; i < (int)block[0]; i++) + if (s1_rbuf[i].a != ((4 * (i + (int)start[0])) + (2 * DSET_SELECT_DIM)) || + s1_rbuf[i].b != (4 * (i + (int)start[0]) + 1) || + s1_rbuf[i].c != (4 * (i + (int)start[0]) + DSET_SELECT_DIM + 2) || + s1_rbuf[i].d != ((4 * (i + (int)start[0])) + (2 * DSET_SELECT_DIM) + 3)) { + nerrors++; + HDprintf("\n Error in 3rd data verification:\n"); + HDprintf(" At index %d: %d/%d, %d/%d, %d/%d, %d/%d\n", i + (int)start[0], + ((4 * (i + (int)start[0])) + (2 * DSET_SELECT_DIM)), s1_rbuf[i].a, + (4 * (i + (int)start[0]) + 1), s1_rbuf[i].b, + (4 * (i + (int)start[0]) + DSET_SELECT_DIM + 2), s1_rbuf[i].c, + ((4 * (i + (int)start[0])) + (2 * DSET_SELECT_DIM) + 3), s1_rbuf[i].d); + break; + } + + /* Case 5(d) */ + + /* Create s2_t compound type with: + * --no conversion for 2 member types, + * --1 larger mem type + * --1 smaller mem type + */ + if ((s2_tid = H5Tcreate(H5T_COMPOUND, sizeof(s2_t))) < 0) + P_TEST_ERROR; + + if (H5Tinsert(s2_tid, "a", HOFFSET(s2_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(s2_tid, "b", HOFFSET(s2_t, b), H5T_NATIVE_LLONG) < 0 || + H5Tinsert(s2_tid, "c", HOFFSET(s2_t, c), H5T_NATIVE_INT) < 0 || + H5Tinsert(s2_tid, "d", HOFFSET(s2_t, d), H5T_NATIVE_SHORT) < 0) + P_TEST_ERROR; + + /* Update s2_wbuf with unique values */ + for (i = 0; i < (int)block[0]; i++) { + s2_wbuf[i].a = (8 * (i + (int)start[0])); + s2_wbuf[i].b = (long long)(8 * (i + (int)start[0]) + 1); + s2_wbuf[i].c = (8 * (i + (int)start[0]) + 2); + s2_wbuf[i].d = (short)(8 * (i + (int)start[0]) + 3); + } + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(s2_wbuf_bak, s2_wbuf, sizeof(s2_t) * DSET_SELECT_DIM); + + if (H5Dwrite(did, s2_tid, mspace_id, fspace_id, dxpl, s2_wbuf) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(s2_wbuf, s2_wbuf_bak, sizeof(s2_t) * DSET_SELECT_DIM); + + /* Read it back */ + HDmemset(s2_rbuf, 0, sizeof(s2_t) * DSET_SELECT_DIM); + if (H5Dread(did, s2_tid, mspace_id, fspace_id, dxpl, s2_rbuf) < 0) + P_TEST_ERROR; + + /* Verify data read */ + for (i = 0; i < (int)block[0]; i++) + if (s2_wbuf[i].a != s2_rbuf[i].a || s2_wbuf[i].b != s2_rbuf[i].b || s2_wbuf[i].c != s2_rbuf[i].c || + s2_wbuf[i].d != s2_rbuf[i].d) { + nerrors++; + HDprintf("\n Error in 4th data verification:\n"); + HDprintf(" At index %d: %d/%d, %lld/%lld, %d/%d, %d/%d\n", i + (int)start[0], s2_wbuf[i].a, + s2_rbuf[i].a, s2_wbuf[i].b, s2_rbuf[i].b, s2_wbuf[i].c, s2_rbuf[i].c, s2_wbuf[i].d, + s2_rbuf[i].d); + break; + } + + if (H5Sclose(mspace_id) < 0) + P_TEST_ERROR; + if (H5Sclose(fspace_id) < 0) + P_TEST_ERROR; + + if (H5Sclose(sid) < 0) + P_TEST_ERROR; + if (H5Tclose(s1_tid) < 0) + P_TEST_ERROR; + if (H5Tclose(s2_tid) < 0) + P_TEST_ERROR; + if (H5Tclose(ss_ac_tid) < 0) + P_TEST_ERROR; + if (H5Tclose(ss_bc_tid) < 0) + P_TEST_ERROR; + if (H5Dclose(did) < 0) + P_TEST_ERROR; + + /* Release buffers */ + HDfree(s1_wbuf); + HDfree(s1_wbuf_bak); + HDfree(s1_rbuf); + HDfree(s2_wbuf); + HDfree(s2_wbuf_bak); + HDfree(s2_rbuf); + + CHECK_PASSED(); + + return; + +} /* test_cmpd_with_bkg() */ + +/* + * Case 6: Type conversions + some processes have null/empty selections in datasets + */ +static void +test_type_conv_sel_empty(hid_t fid, unsigned chunked, unsigned dtrans, unsigned mwbuf) +{ + int i; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t ntrans_dxpl = H5I_INVALID_HID; + hid_t fspace_id = H5I_INVALID_HID; + hid_t mspace_id = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + hsize_t start[1], stride[1], count[1], block[1]; + + long lwbuf[DSET_SELECT_DIM]; + long lwbuf_bak[DSET_SELECT_DIM]; + long trans_lwbuf[DSET_SELECT_DIM]; + long lrbuf[DSET_SELECT_DIM]; + short srbuf[DSET_SELECT_DIM]; + short swbuf[DSET_SELECT_DIM]; + short swbuf_bak[DSET_SELECT_DIM]; + short trans_swbuf[DSET_SELECT_DIM]; + long long llrbuf[DSET_SELECT_DIM]; + char dset_name[DSET_NAME_LEN]; + + const char *expr = "2*x"; + + curr_nerrors = nerrors; + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + P_TEST_ERROR; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + P_TEST_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + P_TEST_ERROR; + } + + /* Generate dataset name */ + HDsnprintf(dset_name, sizeof(dset_name), "tconv_sel_empty_%s_%s_%s", chunked ? "chunked" : "contig", + dtrans ? "xform" : "noxform", mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create dataset */ + if ((did = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + P_TEST_ERROR; + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + P_TEST_ERROR; + + /* Set selection I/O mode, type of I/O and type of collective I/O */ + set_dxpl(dxpl, H5D_SELECTION_IO_MODE_ON, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, mwbuf); + + if ((ntrans_dxpl = H5Pcopy(dxpl)) < 0) + P_TEST_ERROR; + + /* Set data transform */ + if (dtrans) { + if (H5Pset_data_transform(dxpl, expr) < 0) + P_TEST_ERROR; + } + + /* Each process takes x number of elements */ + block[0] = dims[0] / (hsize_t)mpi_size; + stride[0] = block[0]; + count[0] = 1; + start[0] = (hsize_t)mpi_rank * block[0]; + + /* Initialize data */ + for (i = 0; i < (int)block[0]; i++) { + lwbuf[i] = i + (int)start[0]; + trans_lwbuf[i] = 2 * lwbuf[i]; + } + + /* Case 6(a) process 0: hyperslab; other processes: select none */ + + /* Create a file dataspace */ + if ((fspace_id = H5Dget_space(did)) < 0) + P_TEST_ERROR; + if (MAINPROCESS) { + if (H5Sselect_hyperslab(fspace_id, H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + } + else { + if (H5Sselect_none(fspace_id) < 0) + P_TEST_ERROR; + } + + /* Create a memory dataspace */ + if ((mspace_id = H5Screate_simple(1, block, NULL)) < 0) + P_TEST_ERROR; + if (mpi_rank) { + if (H5Sselect_none(mspace_id) < 0) + P_TEST_ERROR; + } + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(lwbuf_bak, lwbuf, sizeof(lwbuf)); + + /* Write data to the dataset with/without data transform in dxpl */ + if (H5Dwrite(did, H5T_NATIVE_LONG, mspace_id, fspace_id, dxpl, lwbuf) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(lwbuf, lwbuf_bak, sizeof(lwbuf)); + + check_io_mode(dxpl, chunked); + + /* Read the data from the dataset: type conversion int-->long */ + /* If dtrans, without data transform set in dxpl */ + if (H5Dread(did, H5T_NATIVE_LONG, mspace_id, fspace_id, ntrans_dxpl, lrbuf) < 0) + P_TEST_ERROR; + + if (MAINPROCESS) { + for (i = 0; i < (int)block[0]; i++) + if (lrbuf[i] != (dtrans ? trans_lwbuf[i] : lwbuf[i])) { + nerrors++; + HDprintf("\n Error in first data verification:\n"); + HDprintf(" At index %d: %ld, %ld\n", i + (int)start[0], + dtrans ? trans_lwbuf[i] : lwbuf[i], lrbuf[i]); + break; + } + } + + if (H5Sclose(mspace_id) < 0) + P_TEST_ERROR; + if (H5Sclose(fspace_id) < 0) + P_TEST_ERROR; + + /* Case 6(b) process 0: get 0 row; other processes: hyperslab */ + + block[0] = mpi_rank ? (dims[0] / (hsize_t)mpi_size) : 0; + stride[0] = mpi_rank ? block[0] : 1; + count[0] = 1; + start[0] = mpi_rank ? ((hsize_t)mpi_rank * block[0]) : 0; + + /* Create a file dataspace */ + if ((fspace_id = H5Dget_space(did)) < 0) + P_TEST_ERROR; + + if (H5Sselect_hyperslab(fspace_id, H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + + /* Create a memory dataspace */ + if ((mspace_id = H5Screate_simple(1, block, NULL)) < 0) + P_TEST_ERROR; + + /* need to make memory space to match for process 0 */ + if (MAINPROCESS) { + if (H5Sselect_hyperslab(mspace_id, H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + } + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(lwbuf_bak, lwbuf, sizeof(lwbuf)); + + /* Write data to the dataset with/without data transform */ + if (H5Dwrite(did, H5T_NATIVE_LONG, mspace_id, fspace_id, dxpl, lwbuf) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(lwbuf, lwbuf_bak, sizeof(lwbuf)); + + /* Read the data from the dataset: type conversion int-->short */ + /* If dtrans, without data transform set in dxpl */ + if (H5Dread(did, H5T_NATIVE_SHORT, mspace_id, fspace_id, ntrans_dxpl, srbuf) < 0) + P_TEST_ERROR; + + if (mpi_rank) { + for (i = 0; i < (int)block[0]; i++) + if (srbuf[i] != (short)(dtrans ? trans_lwbuf[i] : lwbuf[i])) { + HDprintf("\n Error in second data verification:\n"); + HDprintf(" At index %d: %d, %d\n", i + (int)start[0], + (short)(dtrans ? trans_lwbuf[i] : lwbuf[i]), srbuf[i]); + break; + } + } + + if (H5Sclose(mspace_id) < 0) + P_TEST_ERROR; + if (H5Sclose(fspace_id) < 0) + P_TEST_ERROR; + + /* Case 6(c) process 0: select none; other processes: select all */ + + /* Initialize data */ + block[0] = DSET_SELECT_DIM; + for (i = 0; i < (int)block[0]; i++) { + swbuf[i] = (short)(i + DSET_SELECT_DIM); + trans_swbuf[i] = (short)(2 * swbuf[i]); + } + + /* Create a file dataspace */ + if ((fspace_id = H5Dget_space(did)) < 0) + P_TEST_ERROR; + if (MAINPROCESS) { + if (H5Sselect_none(fspace_id) < 0) + P_TEST_ERROR; + } + else { + if (H5Sselect_all(fspace_id) < 0) + P_TEST_ERROR; + } + + /* Create a memory dataspace */ + if ((mspace_id = H5Screate_simple(1, block, NULL)) < 0) + P_TEST_ERROR; + if (MAINPROCESS) { + if (H5Sselect_none(mspace_id) < 0) + P_TEST_ERROR; + } + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(swbuf_bak, swbuf, sizeof(swbuf)); + + /* Write data to the dataset with/without data transform */ + if (H5Dwrite(did, H5T_NATIVE_SHORT, mspace_id, fspace_id, dxpl, swbuf) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(swbuf, swbuf_bak, sizeof(swbuf)); + + /* Read the data from the dataset: type conversion int-->llong */ + /* If dtrans, without data transform set in dxpl */ + if (H5Dread(did, H5T_NATIVE_LLONG, mspace_id, fspace_id, ntrans_dxpl, llrbuf) < 0) + P_TEST_ERROR; + + if (mpi_rank) { + for (i = 0; i < (int)block[0]; i++) + if (llrbuf[i] != (long long)(dtrans ? trans_swbuf[i] : swbuf[i])) { + HDprintf("\n Error in third data verification:\n"); + HDprintf(" At index %d: %lld, %lld\n", i + (int)start[0], + (long long)(dtrans ? trans_swbuf[i] : swbuf[i]), llrbuf[i]); + break; + } + } + + if (H5Sclose(mspace_id) < 0) + P_TEST_ERROR; + if (H5Sclose(fspace_id) < 0) + P_TEST_ERROR; + + if (H5Sclose(sid) < 0) + P_TEST_ERROR; + if (H5Dclose(did) < 0) + P_TEST_ERROR; + if (H5Pclose(dxpl) < 0) + P_TEST_ERROR; + if (H5Pclose(ntrans_dxpl) < 0) + P_TEST_ERROR; + + CHECK_PASSED(); + + return; + +} /* test_type_conv_sel_empty() */ + +/* + * Test 1 for multi-dataset: + * --Datasets with/without type conversion+smaller/larger mem type+no background buffer + * + * Create datasets: randomized H5T_NATIVE_INT or H5T_NATIVE_LONG + * + * Case a--setting for multi write/read to ndsets: + * Datatype for all datasets: H5T_NATIVE_INT + * + * Case b--setting for multi write/read to ndsets: + * Datatype for all datasets: H5T_NATIVE_LONG + */ +static void +test_multi_dsets_no_bkg(hid_t fid, unsigned chunked, unsigned dtrans, unsigned mwbuf) +{ + size_t ndsets; + int i, j; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t ntrans_dxpl = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + hsize_t start[1], stride[1], count[1], block[1]; + + hid_t file_sids[MULTI_NUM_DSETS]; + hid_t mem_sids[MULTI_NUM_DSETS]; + hid_t mem_tids[MULTI_NUM_DSETS]; + + char dset_names[MULTI_NUM_DSETS][DSET_NAME_LEN]; + hid_t dset_dids[MULTI_NUM_DSETS]; + + size_t buf_size; + + int *total_wbuf = NULL; + int *total_wbuf_bak = NULL; + int *total_trans_wbuf = NULL; + int *total_rbuf = NULL; + + long *total_lwbuf = NULL; + long *total_lwbuf_bak = NULL; + long *total_trans_lwbuf = NULL; + long *total_lrbuf = NULL; + + int *wbufi[MULTI_NUM_DSETS]; + int *trans_wbufi[MULTI_NUM_DSETS]; + int *rbufi[MULTI_NUM_DSETS]; + + long *lwbufi[MULTI_NUM_DSETS]; + long *trans_lwbufi[MULTI_NUM_DSETS]; + long *lrbufi[MULTI_NUM_DSETS]; + + const void *wbufs[MULTI_NUM_DSETS]; + void *rbufs[MULTI_NUM_DSETS]; + const char *expr = "2*x"; + + curr_nerrors = nerrors; + + ndsets = MAX(MULTI_MIN_DSETS, MULTI_NUM_DSETS); + + dims[0] = DSET_SELECT_DIM; + + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + P_TEST_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + P_TEST_ERROR; + } + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + P_TEST_ERROR; + + /* Set selection I/O mode, type of I/O and type of collective I/O */ + set_dxpl(dxpl, H5D_SELECTION_IO_MODE_ON, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, mwbuf); + + if ((ntrans_dxpl = H5Pcopy(dxpl)) < 0) + P_TEST_ERROR; + + /* Set data transform */ + if (dtrans) + if (H5Pset_data_transform(dxpl, expr) < 0) + P_TEST_ERROR; + + /* Set up file space ids and dataset ids */ + for (i = 0; i < (int)ndsets; i++) { + if ((file_sids[i] = H5Screate_simple(1, dims, NULL)) < 0) + P_TEST_ERROR; + + /* Generate dataset name */ + HDsnprintf(dset_names[i], sizeof(dset_names[i]), "multi_dset%d_%s_%s_%s", i, + chunked ? "chunked" : "contig", dtrans ? "xform" : "noxform", mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create ith dataset */ + if ((dset_dids[i] = + H5Dcreate2(fid, dset_names[i], ((HDrandom() % 2) ? H5T_NATIVE_LONG : H5T_NATIVE_INT), + file_sids[i], H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + P_TEST_ERROR; + } + + /* Each process takes x number of elements */ + block[0] = dims[0] / (hsize_t)mpi_size; + stride[0] = block[0]; + count[0] = 1; + start[0] = (hsize_t)mpi_rank * block[0]; + + for (i = 0; i < (int)ndsets; i++) { + if ((mem_sids[i] = H5Screate_simple(1, block, NULL)) < 0) + P_TEST_ERROR; + + if (H5Sselect_hyperslab(file_sids[i], H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + } + + buf_size = ndsets * DSET_SELECT_DIM * sizeof(int); + + /* Allocate buffers for all datasets */ + if (NULL == (total_wbuf = (int *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (mwbuf && NULL == (total_wbuf_bak = (int *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_trans_wbuf = (int *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_rbuf = (int *)HDmalloc(buf_size))) + P_TEST_ERROR; + + buf_size = ndsets * DSET_SELECT_DIM * sizeof(long); + + if (NULL == (total_lwbuf = (long *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (mwbuf && NULL == (total_lwbuf_bak = (long *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_trans_lwbuf = (long *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_lrbuf = (long *)HDmalloc(buf_size))) + P_TEST_ERROR; + + /* Initialize buffer indices */ + for (i = 0; i < (int)ndsets; i++) { + wbufi[i] = total_wbuf + (i * DSET_SELECT_DIM); + rbufi[i] = total_rbuf + (i * DSET_SELECT_DIM); + trans_wbufi[i] = total_trans_wbuf + (i * DSET_SELECT_DIM); + + wbufs[i] = wbufi[i]; + rbufs[i] = rbufi[i]; + } + + /* Case a */ + + /* Initialize the buffer data */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < (int)block[0]; j++) { + wbufi[i][j] = j + (int)start[0]; + trans_wbufi[i][j] = 2 * wbufi[i][j]; + } + + /* Datatype setting for multi write/read */ + for (i = 0; i < (int)ndsets; i++) + mem_tids[i] = H5T_NATIVE_INT; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_wbuf_bak, total_wbuf, ndsets * DSET_SELECT_DIM * sizeof(int)); + + /* Write data to the dataset with/without data transform */ + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_wbuf, total_wbuf_bak, ndsets * DSET_SELECT_DIM * sizeof(int)); + + check_io_mode(dxpl, chunked); + + /* Read data from the dataset (if dtrans, without data transform set in dxpl) */ + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, ntrans_dxpl, rbufs) < 0) + P_TEST_ERROR; + + /* Verify */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < (int)block[0]; j++) + if (rbufi[i][j] != (dtrans ? trans_wbufi[i][j] : wbufi[i][j])) { + nerrors++; + HDprintf("\n Error in 1st data verification for dset %d:\n", i); + HDprintf(" At index %d: %d, %d\n", j + (int)start[0], + dtrans ? trans_wbufi[i][j] : wbufi[i][j], rbufi[i][j]); + break; + } + + if (dtrans) { + + /* Read the data from the dataset with data transform set in dxpl */ + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + P_TEST_ERROR; + + /* Verify */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < (int)block[0]; j++) + if (rbufi[i][j] != (2 * trans_wbufi[i][j])) { + nerrors++; + HDprintf("\n Error in 1st (with dtrans) data verification for dset %d:\n", i); + HDprintf(" At index %d: %d, %d\n", j + (int)start[0], 2 * trans_wbufi[i][j], + rbufi[i][j]); + break; + } + } + + /* Case b */ + + /* Initialize buffer indices */ + for (i = 0; i < (int)ndsets; i++) { + lwbufi[i] = total_lwbuf + (i * DSET_SELECT_DIM); + trans_lwbufi[i] = total_trans_lwbuf + (i * DSET_SELECT_DIM); + lrbufi[i] = total_lrbuf + (i * DSET_SELECT_DIM); + wbufs[i] = lwbufi[i]; + rbufs[i] = lrbufi[i]; + } + + /* Initialize the buffer data */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < (int)block[0]; j++) { + lwbufi[i][j] = j + (int)start[0] + DSET_SELECT_DIM; + trans_lwbufi[i][j] = 2 * lwbufi[i][j]; + } + + /* Datatype setting for multi write/read */ + for (i = 0; i < (int)ndsets; i++) + mem_tids[i] = H5T_NATIVE_LONG; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_lwbuf_bak, total_lwbuf, ndsets * DSET_SELECT_DIM * sizeof(long)); + + /* Write data to the dataset with/without data transform */ + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_lwbuf, total_lwbuf_bak, ndsets * DSET_SELECT_DIM * sizeof(long)); + + /* Read data from the dataset (if dtrans, with data transform again in dxpl */ + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + P_TEST_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + for (j = 0; j < (int)block[0]; j++) { + if (lrbufi[i][j] != (dtrans ? (2 * trans_lwbufi[i][j]) : lwbufi[i][j])) { + nerrors++; + HDprintf("\n Error in 2nd data verification for dset %d:\n", i); + HDprintf(" At index %d: %ld/%ld\n", j + (int)start[0], + (dtrans ? (2 * trans_lwbufi[i][j]) : lwbufi[i][j]), lrbufi[i][j]); + break; + } + } + } + + if (H5Pclose(dcpl) < 0) + P_TEST_ERROR; + if (H5Pclose(dxpl) < 0) + P_TEST_ERROR; + if (H5Pclose(ntrans_dxpl) < 0) + P_TEST_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + if (H5Sclose(file_sids[i]) < 0) + P_TEST_ERROR; + if (H5Sclose(mem_sids[i]) < 0) + P_TEST_ERROR; + if (H5Dclose(dset_dids[i]) < 0) + P_TEST_ERROR; + } + + HDfree(total_wbuf); + HDfree(total_wbuf_bak); + HDfree(total_rbuf); + HDfree(total_trans_wbuf); + HDfree(total_lwbuf); + HDfree(total_lwbuf_bak); + HDfree(total_trans_lwbuf); + HDfree(total_lrbuf); + + CHECK_PASSED(); + + return; + +} /* test_multi_dsets_no_bkg() */ + +/* + * Test 2 for multi-dataset: + * Datasets with compound types+background buffer + * + * Create datasets with the same compound type + * Case (a) Initialize compound buffer in memory with unique values + * All datasets: + * --Write all compound fields to disk + * --Read the entire compound type for all datasets + * --Verify values read + * Case (b) Update all fields of the compound type in memory write buffer with new unique values + * dset0: + * --Write some but not all all compound fields to disk + * --Read and verify the fields have the correct (old(a) or new) values + * Remaining datasets: + * --Untouched + * --Read and verify the fields have the correct old(a) values + * Case (c) Update all fields of the compound type in memory read buffer with new unique values + * Randomized <mm> dataset: + * --Read some but not all the compound fields to memory + * --Verify the fields have the correct (old(a) or new) values + * dset0: + * --Untouched + * --Read and verify the fields have the correct (old(a) or middle(b)) values + * Remaining datasets: + * --Untouched + * --Read and verify the fields have the correct old(a) values + * Case (d) Set up a different compound type which has: + * --no type conversion for 2 member types + * --a field with larger mem type + * --a field with smaller mem type + * All datasets: + * --Write the compound fields to disk + * --Read the entire compound type + * --Verify values read + */ +static void +test_multi_dsets_cmpd_with_bkg(hid_t fid, unsigned chunked, unsigned mwbuf) +{ + size_t ndsets; + int i, j, mm; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + hsize_t start[1], stride[1], count[1], block[1]; + + hid_t file_sids[MULTI_NUM_DSETS]; + hid_t mem_sids[MULTI_NUM_DSETS]; + hid_t mem_tids[MULTI_NUM_DSETS]; + + hid_t s1_tid = H5I_INVALID_HID; + hid_t ss_ac_tid = H5I_INVALID_HID; + hid_t ss_bc_tid = H5I_INVALID_HID; + hid_t s2_tid = H5I_INVALID_HID; + + char dset_names[MULTI_NUM_DSETS][DSET_NAME_LEN]; + hid_t dset_dids[MULTI_NUM_DSETS]; + + size_t buf_size; + size_t s2_buf_size; + + s1_t *total_wbuf = NULL; + s1_t *total_wbuf_bak = NULL; + s1_t *total_rbuf = NULL; + + s2_t *s2_total_wbuf = NULL; + s2_t *s2_total_wbuf_bak = NULL; + s2_t *s2_total_rbuf = NULL; + + s1_t *wbufi[MULTI_NUM_DSETS]; + s1_t *rbufi[MULTI_NUM_DSETS]; + + s2_t *s2_wbufi[MULTI_NUM_DSETS]; + s2_t *s2_rbufi[MULTI_NUM_DSETS]; + + const void *wbufs[MULTI_NUM_DSETS]; + void *rbufs[MULTI_NUM_DSETS]; + + curr_nerrors = nerrors; + + ndsets = MAX(MULTI_MIN_DSETS, MULTI_NUM_DSETS); + + dims[0] = DSET_SELECT_DIM; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + P_TEST_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + P_TEST_ERROR; + } + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + P_TEST_ERROR; + + /* Set selection I/O mode, type of I/O and type of collective I/O */ + set_dxpl(dxpl, H5D_SELECTION_IO_MODE_ON, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, mwbuf); + + /* Each process takes x number of elements */ + block[0] = dims[0] / (hsize_t)mpi_size; + stride[0] = block[0]; + count[0] = 1; + start[0] = (hsize_t)mpi_rank * block[0]; + + /* Create the memory data type */ + if ((s1_tid = H5Tcreate(H5T_COMPOUND, sizeof(s1_t))) < 0) + P_TEST_ERROR; + + if (H5Tinsert(s1_tid, "a", HOFFSET(s1_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "b", HOFFSET(s1_t, b), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "c", HOFFSET(s1_t, c), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "d", HOFFSET(s1_t, d), H5T_NATIVE_INT) < 0) + P_TEST_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + if ((file_sids[i] = H5Screate_simple(1, dims, NULL)) < 0) + P_TEST_ERROR; + if ((mem_sids[i] = H5Screate_simple(1, block, NULL)) < 0) + P_TEST_ERROR; + + /* Generate dataset name */ + HDsnprintf(dset_names[i], sizeof(dset_names[i]), "multi_cmpd_dset%d_%s_%s", i, + chunked ? "chunked" : "contig", mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create ith dataset */ + if ((dset_dids[i] = + H5Dcreate2(fid, dset_names[i], s1_tid, file_sids[i], H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + P_TEST_ERROR; + + if (H5Sselect_hyperslab(file_sids[i], H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + } + + buf_size = ndsets * DSET_SELECT_DIM * sizeof(s1_t); + s2_buf_size = ndsets * DSET_SELECT_DIM * sizeof(s2_t); + + /* Allocate buffers for all datasets */ + if (NULL == (total_wbuf = (s1_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (mwbuf && NULL == (total_wbuf_bak = (s1_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_rbuf = (s1_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + + if (NULL == (s2_total_wbuf = (s2_t *)HDmalloc(s2_buf_size))) + P_TEST_ERROR; + if (mwbuf && NULL == (s2_total_wbuf_bak = (s2_t *)HDmalloc(s2_buf_size))) + P_TEST_ERROR; + if (NULL == (s2_total_rbuf = (s2_t *)HDmalloc(s2_buf_size))) + P_TEST_ERROR; + + /* Initialize buffer indices */ + for (i = 0; i < (int)ndsets; i++) { + wbufi[i] = total_wbuf + (i * DSET_SELECT_DIM); + rbufi[i] = total_rbuf + (i * DSET_SELECT_DIM); + + wbufs[i] = wbufi[i]; + rbufs[i] = rbufi[i]; + } + + /* Case a */ + + /* Initialize the buffer data for all the datasets */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < (int)block[0]; j++) { + wbufi[i][j].a = 4 * (j + (int)start[0]); + wbufi[i][j].b = 4 * (j + (int)start[0]) + 1; + wbufi[i][j].c = 4 * (j + (int)start[0]) + 2; + wbufi[i][j].d = 4 * (j + (int)start[0]) + 3; + } + + /* Datatype setting for multi write */ + for (i = 0; i < (int)ndsets; i++) + mem_tids[i] = s1_tid; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_wbuf_bak, total_wbuf, buf_size); + + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_wbuf, total_wbuf_bak, buf_size); + + check_io_mode(dxpl, chunked); + + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + P_TEST_ERROR; + + /* Verify data read */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < (int)block[0]; j++) { + if (wbufi[i][j].a != rbufi[i][j].a || wbufi[i][j].b != rbufi[i][j].b || + wbufi[i][j].c != rbufi[i][j].c || wbufi[i][j].d != rbufi[i][j].d) { + nerrors++; + HDprintf("\n Error in 1st data verification for dset %d:\n", i); + HDprintf(" At index %d: %d/%d, %d/%d, %d/%d, %d/%d\n", j + (int)start[0], wbufi[i][j].a, + rbufi[i][j].a, wbufi[i][j].b, rbufi[i][j].b, wbufi[i][j].c, rbufi[i][j].c, + wbufi[i][j].d, rbufi[i][j].d); + + break; + } + } + + /* Case b */ + + /* Update data in wbufi for dset0 with unique values */ + for (j = 0; j < (int)block[0]; j++) { + wbufi[0][j].a = (4 * (j + (int)start[0])) + DSET_SELECT_DIM; + wbufi[0][j].b = (4 * (j + (int)start[0])) + DSET_SELECT_DIM + 1; + wbufi[0][j].c = (4 * (j + (int)start[0])) + DSET_SELECT_DIM + 2; + wbufi[0][j].d = (4 * (j + (int)start[0])) + DSET_SELECT_DIM + 3; + } + + /* Create a compound type same size as s1_t */ + if ((ss_ac_tid = H5Tcreate(H5T_COMPOUND, sizeof(s1_t))) < 0) + P_TEST_ERROR; + + /* but contains only subset members of s1_t */ + if (H5Tinsert(ss_ac_tid, "a", HOFFSET(s1_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(ss_ac_tid, "c", HOFFSET(s1_t, c), H5T_NATIVE_INT) < 0) + P_TEST_ERROR; + + /* Datatype setting for write to dset0 */ + mem_tids[0] = ss_ac_tid; + + /* Untouched memory and file spaces for other datasets */ + for (i = 0; i < (int)ndsets; i++) { + if (i == 0) + continue; + + if (H5Sselect_none(mem_sids[i]) < 0) + P_TEST_ERROR; + if (H5Sselect_none(file_sids[i]) < 0) + P_TEST_ERROR; + } + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_wbuf_bak, total_wbuf, buf_size); + + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_wbuf, total_wbuf_bak, buf_size); + + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + P_TEST_ERROR; + + /* Verify data read */ + for (i = 0; i < (int)ndsets; i++) + if (i == 0) { /* dset0 */ + for (j = 0; j < (int)block[0]; j++) + if (wbufi[i][j].a != rbufi[i][j].a || (4 * (j + (int)start[0]) + 1) != rbufi[i][j].b || + wbufi[i][j].c != rbufi[i][j].c || (4 * (j + (int)start[0]) + 3) != rbufi[i][j].d) { + nerrors++; + HDprintf("\n Error in 2nd data verification for dset %d:\n", i); + HDprintf(" At index %d: %d/%d, %d/%d, %d/%d, %d/%d\n", j + (int)start[0], + wbufi[i][j].a, rbufi[i][j].a, (4 * (j + (int)start[0]) + 1), rbufi[i][j].b, + wbufi[i][j].c, rbufi[i][j].c, (4 * (j + (int)start[0]) + 3), rbufi[i][j].d); + break; + } + } + else { /* other datasets */ + for (j = 0; j < (int)block[0]; j++) + if ((4 * (j + (int)start[0])) != rbufi[i][j].a || + (4 * (j + (int)start[0]) + 1) != rbufi[i][j].b || + (4 * (j + (int)start[0]) + 2) != rbufi[i][j].c || + (4 * (j + (int)start[0]) + 3) != rbufi[i][j].d) { + nerrors++; + HDprintf("\n Error in 2nd data verification for dset %d:\n", i); + HDprintf(" At index %d: %d/%d, %d/%d, %d/%d, %d/%d\n", j + (int)start[0], + (4 * (j + (int)start[0])), rbufi[i][j].a, (4 * (j + (int)start[0]) + 1), + rbufi[i][j].b, (4 * (j + (int)start[0]) + 2), rbufi[i][j].c, + (4 * (j + (int)start[0]) + 3), rbufi[i][j].d); + break; + } + } + + /* Case c */ + mm = HDrandom() % (int)ndsets; + if (!mm) + mm++; + + /* Update data in rbufi for <mm> dset with new unique values */ + for (j = 0; j < (int)block[0]; j++) { + rbufi[mm][j].a = (4 * (j + (int)start[0])) + (2 * DSET_SELECT_DIM); + rbufi[mm][j].b = (4 * (j + (int)start[0])) + (2 * DSET_SELECT_DIM) + 1; + rbufi[mm][j].c = (4 * (j + (int)start[0])) + (2 * DSET_SELECT_DIM) + 2; + rbufi[mm][j].d = (4 * (j + (int)start[0])) + (2 * DSET_SELECT_DIM) + 3; + } + + /* Create a compound type same size as s1_t */ + if ((ss_bc_tid = H5Tcreate(H5T_COMPOUND, sizeof(s1_t))) < 0) + P_TEST_ERROR; + + /* but contains only subset members of s1_t */ + if (H5Tinsert(ss_bc_tid, "b", HOFFSET(s1_t, b), H5T_NATIVE_INT) < 0 || + H5Tinsert(ss_bc_tid, "c", HOFFSET(s1_t, c), H5T_NATIVE_INT) < 0) + P_TEST_ERROR; + + /* Reset memory and file spaces for <mm> dset */ + if (H5Sselect_all(mem_sids[mm]) < 0) + P_TEST_ERROR; + if (H5Sselect_all(file_sids[mm]) < 0) + P_TEST_ERROR; + if (H5Sselect_hyperslab(file_sids[mm], H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + + /* Untouched memory and file space for other datasets */ + for (i = 0; i < (int)ndsets; i++) { + if (i == 0 || i == mm) + continue; + if (H5Sselect_none(mem_sids[i]) < 0) + P_TEST_ERROR; + if (H5Sselect_none(file_sids[i]) < 0) + P_TEST_ERROR; + } + + /* Datatype setting for read from <mm> dataset */ + mem_tids[mm] = ss_bc_tid; + + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + P_TEST_ERROR; + + /* Verify data read */ + /* dset0 */ + for (j = 0; j < (int)block[0]; j++) + if (wbufi[0][j].a != rbufi[0][j].a || ((4 * (j + (int)start[0])) + 1) != rbufi[0][j].b || + wbufi[0][j].c != rbufi[0][j].c || ((4 * (j + (int)start[0])) + 3) != rbufi[0][j].d) { + nerrors++; + HDprintf("\n Error in 3rd data verification for dset0:\n"); + HDprintf(" At index %d: %d/%d, %d/%d, %d/%d, %d/%d\n", j + (int)start[0], wbufi[0][j].a, + rbufi[0][j].a, (4 * (j + (int)start[0]) + 1), rbufi[0][j].b, wbufi[0][j].c, + rbufi[0][j].c, (4 * (j + (int)start[0]) + 3), rbufi[0][j].d); + break; + } + + /* <mm> dset */ + for (j = 0; j < (int)block[0]; j++) + if (rbufi[mm][j].a != ((4 * (j + (int)start[0])) + (2 * DSET_SELECT_DIM)) || + rbufi[mm][j].b != ((4 * (j + (int)start[0])) + 1) || + rbufi[mm][j].c != ((4 * (j + (int)start[0])) + 2) || + rbufi[mm][j].d != ((4 * (j + (int)start[0])) + (2 * DSET_SELECT_DIM) + 3)) { + nerrors++; + HDprintf("\n Error in 3rd data verification for dset %d:\n", mm); + HDprintf(" At index %d: %d/%d, %d/%d, %d/%d, %d/%d\n", j + (int)start[0], + ((4 * (j + (int)start[0])) + (2 * DSET_SELECT_DIM)), rbufi[mm][j].a, + ((4 * (j + (int)start[0])) + 1), rbufi[mm][j].b, ((4 * (j + (int)start[0])) + 2), + rbufi[mm][j].c, ((4 * (j + (int)start[0])) + (2 * DSET_SELECT_DIM) + 3), rbufi[mm][j].d); + break; + } + + /* other datasets */ + for (i = 0; i < (int)ndsets; i++) { + if (i == 0 || i == mm) + continue; + + for (j = 0; j < (int)block[0]; j++) + if (rbufi[i][j].a != ((4 * (j + (int)start[0]))) || + rbufi[i][j].b != ((4 * (j + (int)start[0])) + 1) || + rbufi[i][j].c != ((4 * (j + (int)start[0])) + 2) || + rbufi[i][j].d != ((4 * (j + (int)start[0])) + 3)) { + nerrors++; + HDprintf("\n Error in 3rd data verification for dset %d:\n", i); + HDprintf(" At index %d: %d/%d, %d/%d, %d/%d, %d/%d\n", j + (int)start[0], + ((4 * (j + (int)start[0]))), rbufi[i][j].a, ((4 * (j + (int)start[0])) + 1), + rbufi[i][j].b, ((4 * (j + (int)start[0])) + 2), rbufi[i][j].c, + ((4 * (j + (int)start[0])) + 3), rbufi[i][j].d); + break; + } + } + + /* Case d */ + + /* Create s2_t compound type with: + * --no conversion for 2 member types, + * --1 larger mem type + * --1 smaller mem type + */ + if ((s2_tid = H5Tcreate(H5T_COMPOUND, sizeof(s2_t))) < 0) + P_TEST_ERROR; + + if (H5Tinsert(s2_tid, "a", HOFFSET(s2_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(s2_tid, "b", HOFFSET(s2_t, b), H5T_NATIVE_LLONG) < 0 || + H5Tinsert(s2_tid, "c", HOFFSET(s2_t, c), H5T_NATIVE_INT) < 0 || + H5Tinsert(s2_tid, "d", HOFFSET(s2_t, d), H5T_NATIVE_SHORT) < 0) + P_TEST_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + s2_wbufi[i] = s2_total_wbuf + (i * DSET_SELECT_DIM); + s2_rbufi[i] = s2_total_rbuf + (i * DSET_SELECT_DIM); + + wbufs[i] = s2_wbufi[i]; + rbufs[i] = s2_rbufi[i]; + + mem_tids[i] = s2_tid; + + if (H5Sselect_all(mem_sids[i]) < 0) + P_TEST_ERROR; + if (H5Sselect_all(file_sids[i]) < 0) + P_TEST_ERROR; + if (H5Sselect_hyperslab(file_sids[i], H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + } + + /* Initialize the buffer data for all the datasets */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < (int)block[0]; j++) { + s2_wbufi[i][j].a = 8 * (j + (int)start[0]); + s2_wbufi[i][j].b = (long long)((8 * (j + (int)start[0])) + 1); + s2_wbufi[i][j].c = (8 * (j + (int)start[0])) + 2; + s2_wbufi[i][j].d = (short)((8 * (j + (int)start[0])) + 3); + } + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(s2_total_wbuf_bak, s2_total_wbuf, s2_buf_size); + + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(s2_total_wbuf, s2_total_wbuf_bak, s2_buf_size); + + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + P_TEST_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + for (j = 0; j < (int)block[0]; j++) + if (s2_rbufi[i][j].a != s2_wbufi[i][j].a || s2_rbufi[i][j].b != s2_wbufi[i][j].b || + s2_rbufi[i][j].c != s2_wbufi[i][j].c || s2_rbufi[i][j].d != s2_wbufi[i][j].d) { + nerrors++; + HDprintf("\n Error in 3rd data verification for dset %d:\n", i); + HDprintf(" At index %d: %d/%d, %lld/%lld, %d/%d, %d/%d\n", j + (int)start[0], + s2_wbufi[i][j].a, s2_rbufi[i][j].a, s2_wbufi[i][j].b, s2_rbufi[i][j].b, + s2_wbufi[i][j].c, s2_rbufi[i][j].c, s2_wbufi[i][j].d, s2_rbufi[i][j].d); + break; + } + } + + if (H5Pclose(dcpl) < 0) + P_TEST_ERROR; + + if (H5Pclose(dxpl) < 0) + P_TEST_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + if (H5Sclose(file_sids[i]) < 0) + P_TEST_ERROR; + if (H5Sclose(mem_sids[i]) < 0) + P_TEST_ERROR; + if (H5Dclose(dset_dids[i]) < 0) + P_TEST_ERROR; + } + + HDfree(total_wbuf); + HDfree(total_wbuf_bak); + HDfree(total_rbuf); + HDfree(s2_total_wbuf); + HDfree(s2_total_wbuf_bak); + HDfree(s2_total_rbuf); + + CHECK_PASSED(); + + return; + +} /* test_multi_dsets_cmpd_with_bkg() */ + +/* + * Test 3 for multi-dataset: + * --Datasets with/without type conv+size change+no background buffer + * + * Create dset0: H5T_STD_I32BE + * Create other dateasets: randomized H5T_STD_I64LE or H5T_STD_I16LE + * + * Case a--setting for multi write/read to ndsets: + * Datatype for all datasets: H5T_STD_I32BE + * + * Case b--setting for multi write/read to ndsets + * Datatype for all datasets: H5T_STD_I64BE + * + * Case c--setting for multi write/read to ndsets + * Datatype for all datasets: H5T_STD_I16BE + */ +static void +test_multi_dsets_size_change_no_bkg(hid_t fid, unsigned chunked, unsigned mwbuf) +{ + size_t ndsets; + int i, j; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + hsize_t start[1], stride[1], count[1], block[1]; + + hid_t file_sids[MULTI_NUM_DSETS]; + hid_t mem_sids[MULTI_NUM_DSETS]; + hid_t mem_tids[MULTI_NUM_DSETS]; + + char dset_names[MULTI_NUM_DSETS][DSET_NAME_LEN]; + hid_t dset_dids[MULTI_NUM_DSETS]; + + size_t buf_size, ss; + uint8_t *total_wbuf = NULL; + uint8_t *total_wbuf_bak = NULL; + uint8_t *total_rbuf = NULL; + uint8_t *total_lwbuf = NULL; + uint8_t *total_lwbuf_bak = NULL; + uint8_t *total_lrbuf = NULL; + uint8_t *total_swbuf = NULL; + uint8_t *total_swbuf_bak = NULL; + uint8_t *total_srbuf = NULL; + + uint8_t *wbufi[MULTI_NUM_DSETS]; + uint8_t *rbufi[MULTI_NUM_DSETS]; + uint8_t *lwbufi[MULTI_NUM_DSETS]; + uint8_t *lrbufi[MULTI_NUM_DSETS]; + uint8_t *swbufi[MULTI_NUM_DSETS]; + uint8_t *srbufi[MULTI_NUM_DSETS]; + + const void *wbufs[MULTI_NUM_DSETS]; + void *rbufs[MULTI_NUM_DSETS]; + + curr_nerrors = nerrors; + + ndsets = MAX(MULTI_MIN_DSETS, MULTI_NUM_DSETS); + + dims[0] = DSET_SELECT_DIM; + + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + P_TEST_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + P_TEST_ERROR; + } + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + P_TEST_ERROR; + + /* Set selection I/O mode, type of I/O and type of collective I/O */ + set_dxpl(dxpl, H5D_SELECTION_IO_MODE_ON, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, mwbuf); + + /* Set up file space ids, mem space ids, and dataset ids */ + for (i = 0; i < (int)ndsets; i++) { + if ((file_sids[i] = H5Screate_simple(1, dims, NULL)) < 0) + P_TEST_ERROR; + + /* Generate dataset name */ + HDsnprintf(dset_names[i], sizeof(dset_names[i]), "multi_size_dset%d_%s_%s", i, + chunked ? "chunked" : "contig", mwbuf ? "mwbuf" : "nomwbuf"); + + /* Create ith dataset */ + if ((dset_dids[i] = H5Dcreate2(fid, dset_names[i], H5T_STD_I32BE, file_sids[i], H5P_DEFAULT, dcpl, + H5P_DEFAULT)) < 0) + P_TEST_ERROR; + } + + /* Each process takes x number of elements */ + block[0] = dims[0] / (hsize_t)mpi_size; + stride[0] = block[0]; + count[0] = 1; + start[0] = (hsize_t)mpi_rank * block[0]; + + for (i = 0; i < (int)ndsets; i++) { + if ((mem_sids[i] = H5Screate_simple(1, block, NULL)) < 0) + P_TEST_ERROR; + + if (H5Sselect_hyperslab(file_sids[i], H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + } + + /* Case a */ + + ss = H5Tget_size(H5T_STD_I32BE); + buf_size = ndsets * ss * DSET_SELECT_DIM; + + /* Allocate buffers for all datasets */ + if (NULL == (total_wbuf = (uint8_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_wbuf_bak = (uint8_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_rbuf = (uint8_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + + /* Initialize buffer indices */ + for (i = 0; i < (int)ndsets; i++) { + wbufi[i] = total_wbuf + (i * (int)ss * DSET_SELECT_DIM); + rbufi[i] = total_rbuf + (i * (int)ss * DSET_SELECT_DIM); + + wbufs[i] = wbufi[i]; + rbufs[i] = rbufi[i]; + } + + /* Initialize the buffer data: big endian */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < (int)block[0]; j++) { + wbufi[i][j * (int)ss + 0] = 0x1; + wbufi[i][j * (int)ss + 1] = 0x2; + wbufi[i][j * (int)ss + 2] = 0x3; + wbufi[i][j * (int)ss + 3] = (uint8_t)(0x4 + j + (int)start[0]); + } + + /* Datatype setting for multi write/read */ + for (i = 0; i < (int)ndsets; i++) + mem_tids[i] = H5T_STD_I32BE; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_wbuf_bak, total_wbuf, buf_size); + + /* Write data to the dataset */ + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_wbuf, total_wbuf_bak, buf_size); + + check_io_mode(dxpl, chunked); + + /* Read data from the dataset */ + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + P_TEST_ERROR; + + /* Verify */ + for (i = 0; i < (int)ndsets; i++) + /* Only compare when it's at least the size of H5T_STD_I32BE */ + if (H5Tget_size(H5Dget_type(dset_dids[i])) >= ss) { + for (j = 0; j < (int)block[0]; j++) + if (rbufi[i][(int)ss * j + 0] != wbufi[i][(int)ss * j + 0] || + rbufi[i][(int)ss * j + 1] != wbufi[i][(int)ss * j + 1] || + rbufi[i][(int)ss * j + 2] != wbufi[i][(int)ss * j + 2] || + rbufi[i][(int)ss * j + 3] != wbufi[i][(int)ss * j + 3]) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j + (int)start[0]); + P_TEST_ERROR; + } + } + + /* Case b */ + + ss = H5Tget_size(H5T_STD_I64BE); + buf_size = ndsets * (ss * DSET_SELECT_DIM); + + /* Allocate buffers for all datasets */ + if (NULL == (total_lwbuf = (uint8_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_lwbuf_bak = (uint8_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_lrbuf = (uint8_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + + /* Initialize buffer indices */ + for (i = 0; i < (int)ndsets; i++) { + lwbufi[i] = total_lwbuf + (i * (int)ss * DSET_SELECT_DIM); + lrbufi[i] = total_lrbuf + (i * (int)ss * DSET_SELECT_DIM); + + wbufs[i] = lwbufi[i]; + rbufs[i] = lrbufi[i]; + } + + /* Initialize the buffer data: big endian */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < (int)block[0]; j++) { + lwbufi[i][j * (int)ss + 0] = 0x1; + lwbufi[i][j * (int)ss + 1] = 0x2; + lwbufi[i][j * (int)ss + 2] = 0x3; + lwbufi[i][j * (int)ss + 3] = 0x4; + lwbufi[i][j * (int)ss + 4] = 0x5; + lwbufi[i][j * (int)ss + 5] = 0x6; + lwbufi[i][j * (int)ss + 6] = 0x7; + lwbufi[i][j * (int)ss + 7] = (uint8_t)(0x8 + j + (int)start[0]); + } + + /* Datatype setting for multi write/read */ + for (i = 0; i < (int)ndsets; i++) + mem_tids[i] = H5T_STD_I64BE; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_lwbuf_bak, total_lwbuf, buf_size); + + /* Write data to the dataset */ + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_lwbuf, total_lwbuf_bak, buf_size); + + /* Read data from the dataset */ + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + P_TEST_ERROR; + + /* Verify */ + for (i = 0; i < (int)ndsets; i++) + /* Only compare when it's the size of H5T_STD_I64BE */ + if (H5Tget_size(H5Dget_type(dset_dids[i])) >= ss) { + for (j = 0; j < (int)block[0]; j++) + if (lrbufi[i][(int)ss * j + 0] != lwbufi[i][(int)ss * j + 0] || + lrbufi[i][(int)ss * j + 1] != lwbufi[i][(int)ss * j + 1] || + lrbufi[i][(int)ss * j + 2] != lwbufi[i][(int)ss * j + 2] || + lrbufi[i][(int)ss * j + 3] != lwbufi[i][(int)ss * j + 3] || + lrbufi[i][(int)ss * j + 4] != lwbufi[i][(int)ss * j + 4] || + lrbufi[i][(int)ss * j + 5] != lwbufi[i][(int)ss * j + 5] || + lrbufi[i][(int)ss * j + 6] != lwbufi[i][(int)ss * j + 6] || + lrbufi[i][(int)ss * j + 7] != lwbufi[i][(int)ss * j + 7]) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j + (int)start[0]); + P_TEST_ERROR; + } + } + + /* Case c */ + + ss = H5Tget_size(H5T_STD_I16BE); + buf_size = ndsets * (ss * DSET_SELECT_DIM); + + /* Allocate buffers for all datasets */ + if (NULL == (total_swbuf = (uint8_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_swbuf_bak = (uint8_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_srbuf = (uint8_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + + /* Initialize buffer indices */ + for (i = 0; i < (int)ndsets; i++) { + swbufi[i] = total_swbuf + (i * (int)ss * DSET_SELECT_DIM); + srbufi[i] = total_srbuf + (i * (int)ss * DSET_SELECT_DIM); + + wbufs[i] = swbufi[i]; + rbufs[i] = srbufi[i]; + } + + /* Initialize the buffer data: big endian */ + for (i = 0; i < (int)ndsets; i++) + for (j = 0; j < (int)block[0]; j++) { + swbufi[i][j * (int)ss + 0] = 0x1; + swbufi[i][j * (int)ss + 1] = (uint8_t)(0x2 + j + (int)start[0]); + } + + /* Datatype setting for multi write/read */ + for (i = 0; i < (int)ndsets; i++) + mem_tids[i] = H5T_STD_I16BE; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_swbuf_bak, total_swbuf, buf_size); + + /* Write data to the dataset */ + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_swbuf, total_swbuf_bak, buf_size); + + /* Read data from the dataset */ + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + P_TEST_ERROR; + + /* Verify */ + for (i = 0; i < (int)ndsets; i++) + /* Can compare for all cases */ + for (j = 0; j < (int)block[0]; j++) + if (srbufi[i][(int)ss * j + 0] != swbufi[i][(int)ss * j + 0] || + srbufi[i][(int)ss * j + 1] != swbufi[i][(int)ss * j + 1]) { + H5_FAILED(); + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j + (int)start[0]); + P_TEST_ERROR; + } + + if (H5Pclose(dcpl) < 0) + P_TEST_ERROR; + if (H5Pclose(dxpl) < 0) + P_TEST_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + if (H5Sclose(file_sids[i]) < 0) + P_TEST_ERROR; + if (H5Sclose(mem_sids[i]) < 0) + P_TEST_ERROR; + if (H5Dclose(dset_dids[i]) < 0) + P_TEST_ERROR; + } + + HDfree(total_wbuf); + HDfree(total_wbuf_bak); + HDfree(total_rbuf); + HDfree(total_lwbuf); + HDfree(total_lwbuf_bak); + HDfree(total_lrbuf); + HDfree(total_swbuf); + HDfree(total_swbuf_bak); + HDfree(total_srbuf); + + CHECK_PASSED(); + + return; + +} /* test_multi_dsets_size_change_no_bkg() */ + +/* + * Test 4 for multi-dataset: + * Datasets with type conversions+some processes have null/empty selections + * + * Create dset0: H5T_NATIVE_INT + * Create other datasets: randomized H5T_NATIVE_LLONG or H5T_NATIVE_SHORT + * Type conversions + some processes have null/empty selections in datasets + * + * Case (a): dset0 + * process 0: hyperslab; other processes: select none + * Case (b): randomized dset <mm> + * process 0: get 0 row; other processes: hyperslab + * Case (c): randomized dset <ll> + * process 0: select none; other processes: select all + * + * Memory datatype for multi write to all datasets: H5T_NATIVE_INT + * --this will not trigger type conversion for case (a) but + * type conversion for cases (b) & (c) + * Memory datatype for multi read to all datasets: H5T_NATIVE_LONG + * --this will trigger type conversion for (a), (b) & (c) + */ +static void +test_multi_dsets_conv_sel_empty(hid_t fid, unsigned chunked, unsigned dtrans, unsigned mwbuf) +{ + size_t ndsets; + int i, j; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t ntrans_dxpl = H5I_INVALID_HID; + + hsize_t dims[1]; + hsize_t cdims[1]; + hsize_t start[1], stride[1], count[1], block[1]; + + hid_t file_sids[MULTI_NUM_DSETS]; + hid_t mem_sids[MULTI_NUM_DSETS]; + hid_t mem_tids[MULTI_NUM_DSETS]; + + char dset_names[MULTI_NUM_DSETS][DSET_NAME_LEN]; + hid_t dset_dids[MULTI_NUM_DSETS]; + + size_t buf_size; + int *total_wbuf = NULL; + int *total_wbuf_bak = NULL; + int *total_trans_wbuf = NULL; + long *total_lrbuf = NULL; + + int *wbufi[MULTI_NUM_DSETS]; + int *trans_wbufi[MULTI_NUM_DSETS]; + long *l_rbufi[MULTI_NUM_DSETS]; + + const void *wbufs[MULTI_NUM_DSETS]; + void *rbufs[MULTI_NUM_DSETS]; + + int save_block0; + int mm, ll; + + const char *expr = "2*x"; + + curr_nerrors = nerrors; + + ndsets = MAX(MULTI_MIN_DSETS, MULTI_NUM_DSETS); + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + P_TEST_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + P_TEST_ERROR; + } + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + P_TEST_ERROR; + + /* Set selection I/O mode, type of I/O and type of collective I/O */ + set_dxpl(dxpl, H5D_SELECTION_IO_MODE_ON, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, mwbuf); + + if ((ntrans_dxpl = H5Pcopy(dxpl)) < 0) + P_TEST_ERROR; + + /* Set data transform */ + if (dtrans) + if (H5Pset_data_transform(dxpl, expr) < 0) + P_TEST_ERROR; + + /* Set up file space ids and dataset ids */ + for (i = 0; i < (int)ndsets; i++) { + if ((file_sids[i] = H5Screate_simple(1, dims, NULL)) < 0) + P_TEST_ERROR; + + /* Generate dataset name */ + HDsnprintf(dset_names[i], sizeof(dset_names[i]), "multi_sel_dset%d_%s_%s_%s", i, + chunked ? "chunked" : "contig", dtrans ? "xform" : "noxform", mwbuf ? "mwbuf" : "nomwbuf"); + + if (i == 0) { + if ((dset_dids[i] = H5Dcreate2(fid, dset_names[i], H5T_NATIVE_INT, file_sids[i], H5P_DEFAULT, + dcpl, H5P_DEFAULT)) < 0) + P_TEST_ERROR; + } + else { + if ((dset_dids[i] = + H5Dcreate2(fid, dset_names[i], ((HDrandom() % 2) ? H5T_NATIVE_LLONG : H5T_NATIVE_SHORT), + file_sids[i], H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + P_TEST_ERROR; + } + } + + buf_size = ndsets * DSET_SELECT_DIM * sizeof(int); + + /* Allocate buffers for all datasets */ + if (NULL == (total_wbuf = (int *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_wbuf_bak = (int *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_trans_wbuf = (int *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_lrbuf = (long *)HDmalloc(ndsets * DSET_SELECT_DIM * sizeof(long)))) + P_TEST_ERROR; + + /* Initialize buffer indices */ + for (i = 0; i < (int)ndsets; i++) { + wbufi[i] = total_wbuf + (i * DSET_SELECT_DIM); + trans_wbufi[i] = total_trans_wbuf + (i * DSET_SELECT_DIM); + + wbufs[i] = wbufi[i]; + } + + /* + * Case (a): dset0 + * process 0: hyperslab; other processes: select none + */ + + /* Each process takes x number of elements */ + block[0] = dims[0] / (hsize_t)mpi_size; + save_block0 = (int)block[0]; + stride[0] = block[0]; + count[0] = 1; + start[0] = (hsize_t)mpi_rank * block[0]; + + /* Get file dataspace */ + if ((file_sids[0] = H5Dget_space(dset_dids[0])) < 0) + P_TEST_ERROR; + + if (MAINPROCESS) { + if (H5Sselect_hyperslab(file_sids[0], H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + } + else { + if (H5Sselect_none(file_sids[0]) < 0) + P_TEST_ERROR; + } + + /* Create memory dataspace */ + if ((mem_sids[0] = H5Screate_simple(1, block, NULL)) < 0) + P_TEST_ERROR; + if (mpi_rank) { + if (H5Sselect_none(mem_sids[0]) < 0) + P_TEST_ERROR; + } + + /* Initialize data for wbufi[0] */ + for (j = 0; j < (int)block[0]; j++) { + wbufi[0][j] = j + (int)start[0]; + trans_wbufi[0][j] = 2 * wbufi[0][j]; + } + + /* + * Case (b): choose a dataset -- dset <mm> + * process 0: get 0 row; other processes: hyperslab + */ + + mm = HDrandom() % (int)ndsets; + if (mm == 0) + mm++; + + block[0] = mpi_rank ? (dims[0] / (hsize_t)mpi_size) : 0; + stride[0] = mpi_rank ? block[0] : 1; + count[0] = 1; + start[0] = mpi_rank ? ((hsize_t)mpi_rank * block[0]) : 0; + + /* Get file dataspace */ + if ((file_sids[mm] = H5Dget_space(dset_dids[mm])) < 0) + P_TEST_ERROR; + + if (H5Sselect_hyperslab(file_sids[mm], H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + + /* Create a memory dataspace */ + if ((mem_sids[mm] = H5Screate_simple(1, block, NULL)) < 0) + P_TEST_ERROR; + + /* need to make memory space to match for process 0 */ + if (MAINPROCESS) { + if (H5Sselect_hyperslab(mem_sids[mm], H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + } + else { + if (H5Sselect_all(mem_sids[mm]) < 0) + P_TEST_ERROR; + } + + /* Initialize data for wbufi[1] */ + for (j = 0; j < (int)block[0]; j++) { + wbufi[mm][j] = j + (int)start[0]; + trans_wbufi[mm][j] = 2 * wbufi[mm][j]; + } + + /* + * Case (c): choose a dataset -- dset <ll> + * process 0: select none; other processes: select all + */ + + ll = mm + 1; + if ((ll % (int)ndsets) == 0) + ll = 1; + + /* Get file dataspace */ + if ((file_sids[ll] = H5Dget_space(dset_dids[ll])) < 0) + P_TEST_ERROR; + if (MAINPROCESS) { + if (H5Sselect_none(file_sids[ll]) < 0) + P_TEST_ERROR; + } + else { + if (H5Sselect_all(file_sids[ll]) < 0) + P_TEST_ERROR; + } + + /* Create a memory dataspace */ + if ((mem_sids[ll] = H5Screate_simple(1, dims, NULL)) < 0) + P_TEST_ERROR; + if (MAINPROCESS) { + if (H5Sselect_none(mem_sids[ll]) < 0) + P_TEST_ERROR; + } + else if (H5Sselect_all(mem_sids[ll]) < 0) + P_TEST_ERROR; + + /* Initialize data for wbufi[ll] */ + for (j = 0; j < (int)dims[0]; j++) { + wbufi[ll][j] = (int)j + DSET_SELECT_DIM; + trans_wbufi[ll][j] = 2 * wbufi[ll][j]; + } + + /* Set up remaining dsets */ + for (i = 0; i < (int)ndsets; i++) { + if (i == 0 || i == mm || i == ll) + continue; + /* Get file dataspace */ + if ((file_sids[i] = H5Dget_space(dset_dids[i])) < 0) + P_TEST_ERROR; + if (H5Sselect_none(file_sids[i]) < 0) + P_TEST_ERROR; + + if ((mem_sids[i] = H5Screate_simple(1, dims, NULL)) < 0) + P_TEST_ERROR; + if (H5Sselect_none(mem_sids[i]) < 0) + P_TEST_ERROR; + } + + /* Set up mem_tids[] for multi write */ + for (i = 0; i < (int)ndsets; i++) + mem_tids[i] = H5T_NATIVE_INT; + + /* Copy wbuf if the library will be modifying it */ + if (mwbuf) + HDmemcpy(total_wbuf_bak, total_wbuf, buf_size); + + /* Write data to the dataset with/without data transform */ + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + P_TEST_ERROR; + + /* Restore wbuf from backup if the library modified it */ + if (mwbuf) + HDmemcpy(total_wbuf, total_wbuf_bak, buf_size); + + check_io_mode(dxpl, chunked); + + /* Initialize buffer indices */ + for (i = 0; i < (int)ndsets; i++) { + l_rbufi[i] = total_lrbuf + (i * DSET_SELECT_DIM); + rbufs[i] = l_rbufi[i]; + } + + /* Set up mem_tids[] for multi read */ + for (i = 0; i < (int)ndsets; i++) + mem_tids[i] = H5T_NATIVE_LONG; + + if (H5Dread_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, ntrans_dxpl, rbufs) < 0) + P_TEST_ERROR; + + if (MAINPROCESS) { + /* Case a: verify dset0 */ + for (j = 0; j < save_block0; j++) + if (l_rbufi[0][j] != (dtrans ? (long)trans_wbufi[0][j] : (long)wbufi[0][j])) { + nerrors++; + HDprintf(" Verify dset0 at index %d: %ld, %ld\n", j + (int)start[0], + dtrans ? (long)trans_wbufi[0][j] : (long)wbufi[0][j], l_rbufi[0][j]); + break; + } + } + + if (mpi_rank) { + /* Case b: verify dset <mm> */ + for (j = 0; j < (int)block[0]; j++) + if (l_rbufi[mm][j] != (long)(dtrans ? trans_wbufi[mm][j] : wbufi[mm][j])) { + nerrors++; + HDprintf(" Verify dset %d at index %d: %ld, %ld\n", mm, j + (int)start[0], + (long)(dtrans ? trans_wbufi[mm][j] : wbufi[mm][j]), l_rbufi[mm][j]); + break; + } + + /* Case c: verify dset <ll> */ + for (j = 0; j < (int)dims[0]; j++) + if (l_rbufi[ll][j] != (long)(dtrans ? trans_wbufi[ll][j] : wbufi[ll][j])) { + nerrors++; + HDprintf(" Verify dset %d at index %d: %ld, %ld\n", ll, j, + (long)(dtrans ? trans_wbufi[ll][j] : wbufi[ll][j]), l_rbufi[ll][j]); + break; + } + } + + if (H5Pclose(dcpl) < 0) + P_TEST_ERROR; + if (H5Pclose(dxpl) < 0) + P_TEST_ERROR; + if (H5Pclose(ntrans_dxpl) < 0) + P_TEST_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + if (H5Sclose(file_sids[i]) < 0) + P_TEST_ERROR; + if (H5Sclose(mem_sids[i]) < 0) + P_TEST_ERROR; + if (H5Dclose(dset_dids[i]) < 0) + P_TEST_ERROR; + } + + HDfree(total_wbuf); + HDfree(total_wbuf_bak); + HDfree(total_trans_wbuf); + HDfree(total_lrbuf); + + CHECK_PASSED(); + + return; + +} /* test_multi_dsets_conv_sel_empty() */ + +/* + * Test 5 for multi-dataset: + * + * Repeat the following test for niter times to ensure the + * random combinations of all dataset types are hit. + * + * Create randomized contiguous or chunked datasets with: + * --DSET_WITH_NO_CONV: + * --with no type conversion + * --dataset with H5T_NATIVE_INT + * --DSET_WITH_CONV_AND_NO_BKG: + * --type conversion without background buffer + * --dataset with H5T_NATIVE_LONG + * --DSET_WITH_CONV_AND_BKG: + * --type conversion with background buffer + * --dataset with compound type s1_t + * + * Do H5Dwrite_multi() and H5Dread_multi() for the above randomized + * datasets with the settings below: + * Setting A: + * --DSET_WITH_NO_CONV: + * --write: mem_tids[] = H5T_NATIVE_INT + * --read: r_mem_tids[] = H5T_NATIVE_INT + * --DSET_WITH_CONV_AND_NO_BKG: + * --write: mem_tids[] = H5T_NATIVE_ULONG + * --read: r_mem_tids[] = H5T_NATIVE_LONG + * --DSET_WITH_CONV_AND_BKG: + * --write: mem_tids[] = s1_tid; + * --read: r_mem_tids[i] = s3_tid; + * + * Setting B: + * --DSET_WITH_NO_CONV: + * --write: mem_tids[] = H5T_NATIVE_INT + * --read: r_mem_tids[] = H5T_NATIVE_INT + * --DSET_WITH_CONV_AND_NO_BKG: + * --write: mem_tids[] = H5T_NATIVE_LONG; + * --read: r_mem_tids[] = H5T_NATIVE_SHORT; + * --DSET_WITH_CONV_AND_BKG: + * --write: mem_tids[] = s4_tid; + * --read: r_mem_tids[i] = s1_tid; + * + * Verify the result read as below: + * Setting A: + * --DSET_WITH_NO_CONV: + * --verify data read in rbufi1[i][j] is same as wbufi1[i][j] + * --DSET_WITH_CONV_AND_NO_BKG: + * --verify data read in l_rbufi2[i][j] is all LONG_MAX + * --DSET_WITH_CONV_AND_BKG: + * --verify all fields read in s3_rbufi3[i][j] is the + * reverse of s1_wbufi3[i][j] + * Setting B: + * --DSET_WITH_NO_CONV: + * --verify data read in rbufi1[i][j] is same as wbufi1[i][j] + * --DSET_WITH_CONV_AND_NO_BKG: + * --verify data read in s_rbufi2[i][j] is all SHRT_MAX + * --DSET_WITH_CONV_AND_BKG: + * --verify fields read in s1_rbufi3[i][j] is as follows: + * --fields 'a' and 'c' are as s1_wbufi3[i][j].a and s1_wbufi3[i][j].c + * --fields 'b' and 'd' are (DSET_SELECT_DIM + j + start[0]) + */ +static void +test_multi_dsets_all(int niter, hid_t fid, unsigned chunked, unsigned mwbuf) +{ + size_t ndsets; + int i, j, mm; + int s, n; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + + hsize_t dims[1]; + hsize_t cdims[1]; + hsize_t start[1], stride[1], count[1], block[1]; + + hid_t file_sids[MULTI_NUM_DSETS]; + hid_t mem_sids[MULTI_NUM_DSETS]; + hid_t mem_tids[MULTI_NUM_DSETS]; + hid_t r_mem_tids[MULTI_NUM_DSETS]; + + multi_dset_type_t dset_types[MULTI_NUM_DSETS]; + + hid_t s1_tid = H5I_INVALID_HID; + hid_t s3_tid = H5I_INVALID_HID; + hid_t s4_tid = H5I_INVALID_HID; + + char dset_names[MULTI_NUM_DSETS][DSET_NAME_LEN]; + hid_t dset_dids[MULTI_NUM_DSETS]; + + size_t buf_size; + + int *total_wbuf1 = NULL; + int *total_wbuf1_bak = NULL; + int *total_rbuf1 = NULL; + + int *wbufi1[MULTI_NUM_DSETS]; + int *rbufi1[MULTI_NUM_DSETS]; + + unsigned long *ul_total_wbuf2 = NULL; + unsigned long *ul_total_wbuf2_bak = NULL; + long *l_total_rbuf2 = NULL; + unsigned long *ul_wbufi2[MULTI_NUM_DSETS]; + long *l_rbufi2[MULTI_NUM_DSETS]; + + long *l_total_wbuf2 = NULL; + long *l_total_wbuf2_bak = NULL; + short *s_total_rbuf2 = NULL; + long *l_wbufi2[MULTI_NUM_DSETS]; + short *s_rbufi2[MULTI_NUM_DSETS]; + + s1_t *s1_total_wbuf3 = NULL; + s1_t *s1_total_wbuf3_bak = NULL; + s3_t *s3_total_rbuf3 = NULL; + s1_t *s1_wbufi3[MULTI_NUM_DSETS]; + s3_t *s3_rbufi3[MULTI_NUM_DSETS]; + + s4_t *s4_total_wbuf3 = NULL; + s4_t *s4_total_wbuf3_bak = NULL; + s1_t *s1_total_rbuf3 = NULL; + s4_t *s4_wbufi3[MULTI_NUM_DSETS]; + s1_t *s1_rbufi3[MULTI_NUM_DSETS]; + + const void *wbufs[MULTI_NUM_DSETS]; + void *rbufs[MULTI_NUM_DSETS]; + + /* for n niter to ensure that all randomized dset_types with multi_dset_type_t will be covered */ + for (n = 0; n < niter; n++) { + + /* Set up the number of datasets for testing */ + ndsets = MAX(MULTI_MIN_DSETS, MULTI_NUM_DSETS); + + /* Create dataset transfer property list */ + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + P_TEST_ERROR; + + /* Set selection I/O mode, type of I/O and type of collective I/O */ + set_dxpl(dxpl, H5D_SELECTION_IO_MODE_ON, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, mwbuf); + + /* Set dataset layout: contiguous or chunked */ + dims[0] = DSET_SELECT_DIM; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + P_TEST_ERROR; + + if (chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + P_TEST_ERROR; + } + + /* Each process takes x number of elements */ + block[0] = dims[0] / (hsize_t)mpi_size; + stride[0] = block[0]; + count[0] = 1; + start[0] = (hsize_t)mpi_rank * block[0]; + + /* Create compound data type: s1_t */ + if ((s1_tid = H5Tcreate(H5T_COMPOUND, sizeof(s1_t))) < 0) + P_TEST_ERROR; + + if (H5Tinsert(s1_tid, "a", HOFFSET(s1_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "b", HOFFSET(s1_t, b), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "c", HOFFSET(s1_t, c), H5T_NATIVE_INT) < 0 || + H5Tinsert(s1_tid, "d", HOFFSET(s1_t, d), H5T_NATIVE_INT) < 0) + P_TEST_ERROR; + + /* Create compound data type: s3_t */ + if ((s3_tid = H5Tcreate(H5T_COMPOUND, sizeof(s3_t))) < 0) + P_TEST_ERROR; + + if (H5Tinsert(s3_tid, "a", HOFFSET(s3_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(s3_tid, "b", HOFFSET(s3_t, b), H5T_NATIVE_INT) < 0 || + H5Tinsert(s3_tid, "c", HOFFSET(s3_t, c), H5T_NATIVE_INT) < 0 || + H5Tinsert(s3_tid, "d", HOFFSET(s3_t, d), H5T_NATIVE_INT) < 0) + P_TEST_ERROR; + + /* Create compound data type: s4_t */ + if ((s4_tid = H5Tcreate(H5T_COMPOUND, sizeof(s4_t))) < 0) + P_TEST_ERROR; + + if (H5Tinsert(s4_tid, "b", HOFFSET(s4_t, b), H5T_NATIVE_UINT) < 0 || + H5Tinsert(s4_tid, "d", HOFFSET(s4_t, d), H5T_NATIVE_UINT) < 0) + P_TEST_ERROR; + + /* Create dataset for i ndsets */ + for (i = 0; i < (int)ndsets; i++) { + + /* File space ids */ + if ((file_sids[i] = H5Screate_simple(1, dims, NULL)) < 0) + P_TEST_ERROR; + + /* Memory space ids */ + if ((mem_sids[i] = H5Screate_simple(1, block, NULL)) < 0) + P_TEST_ERROR; + + mm = HDrandom() % (int)ndsets; + if (mm == 0) { + dset_types[i] = DSET_WITH_NO_CONV; + HDsnprintf(dset_names[i], sizeof(dset_names[i]), "multi_all_nconv_dset%d_%s_%s", i, + chunked ? "chunked" : "contig", mwbuf ? "mwbuf" : "nomwbuf"); + if ((dset_dids[i] = H5Dcreate2(fid, dset_names[i], H5T_NATIVE_INT, file_sids[i], H5P_DEFAULT, + dcpl, H5P_DEFAULT)) < 0) + P_TEST_ERROR; + } + else if (mm == 1) { + dset_types[i] = DSET_WITH_CONV_AND_NO_BKG; + HDsnprintf(dset_names[i], sizeof(dset_names[i]), "multi_all_conv_nbkg_dset%d_%s_%s", i, + chunked ? "chunked" : "contig", mwbuf ? "mwbuf" : "nomwbuf"); + if ((dset_dids[i] = H5Dcreate2(fid, dset_names[i], H5T_NATIVE_LONG, file_sids[i], H5P_DEFAULT, + dcpl, H5P_DEFAULT)) < 0) + P_TEST_ERROR; + } + else { + dset_types[i] = DSET_WITH_CONV_AND_BKG; + HDsnprintf(dset_names[i], sizeof(dset_names[i]), "multi_all_conv_bkg_dset%d_%s_%s", i, + chunked ? "chunked" : "contig", mwbuf ? "mwbuf" : "nomwbuf"); + if ((dset_dids[i] = H5Dcreate2(fid, dset_names[i], s1_tid, file_sids[i], H5P_DEFAULT, dcpl, + H5P_DEFAULT)) < 0) + P_TEST_ERROR; + } + + if (H5Sselect_hyperslab(file_sids[i], H5S_SELECT_SET, start, stride, count, block) < 0) + P_TEST_ERROR; + + } /* end for i ndsets */ + + /* Allocate buffers for all datasets */ + + /* DSET_WITH_NO_CONV */ + buf_size = ndsets * DSET_SELECT_DIM * sizeof(int); + if (NULL == (total_wbuf1 = (int *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (mwbuf && NULL == (total_wbuf1_bak = (int *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (NULL == (total_rbuf1 = (int *)HDmalloc(buf_size))) + P_TEST_ERROR; + + /* DSET_WITH_CONV_AND_NO_BKG */ + buf_size = ndsets * DSET_SELECT_DIM * sizeof(unsigned long); + if (NULL == (ul_total_wbuf2 = (unsigned long *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (mwbuf && NULL == (ul_total_wbuf2_bak = (unsigned long *)HDmalloc(buf_size))) + P_TEST_ERROR; + buf_size = ndsets * DSET_SELECT_DIM * sizeof(long); + if (NULL == (l_total_rbuf2 = (long *)HDmalloc(buf_size))) + P_TEST_ERROR; + + buf_size = ndsets * DSET_SELECT_DIM * sizeof(long); + if (NULL == (l_total_wbuf2 = (long *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (mwbuf && NULL == (l_total_wbuf2_bak = (long *)HDmalloc(buf_size))) + P_TEST_ERROR; + buf_size = ndsets * DSET_SELECT_DIM * sizeof(short); + if (NULL == (s_total_rbuf2 = (short *)HDmalloc(buf_size))) + P_TEST_ERROR; + + /* DSET_WITH_CONV_AND_BKG */ + buf_size = ndsets * DSET_SELECT_DIM * sizeof(s1_t); + if (NULL == (s1_total_wbuf3 = (s1_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (mwbuf && NULL == (s1_total_wbuf3_bak = (s1_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + buf_size = ndsets * DSET_SELECT_DIM * sizeof(s3_t); + if (NULL == (s3_total_rbuf3 = (s3_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + + buf_size = ndsets * DSET_SELECT_DIM * sizeof(s4_t); + if (NULL == (s4_total_wbuf3 = (s4_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + if (mwbuf && NULL == (s4_total_wbuf3_bak = (s4_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + buf_size = ndsets * DSET_SELECT_DIM * sizeof(s1_t); + if (NULL == (s1_total_rbuf3 = (s1_t *)HDmalloc(buf_size))) + P_TEST_ERROR; + + /* Test with s settings for ndsets */ + for (s = SETTING_A; s <= SETTING_B; s++) { + + /* for i ndsets */ + for (i = 0; i < (int)ndsets; i++) { + + switch (dset_types[i]) { + + case DSET_WITH_NO_CONV: + /* Initialize buffer indices */ + wbufi1[i] = total_wbuf1 + (i * DSET_SELECT_DIM); + rbufi1[i] = total_rbuf1 + (i * DSET_SELECT_DIM); + + wbufs[i] = wbufi1[i]; + rbufs[i] = rbufi1[i]; + + /* Initialize the buffer data */ + for (j = 0; j < (int)block[0]; j++) + wbufi1[i][j] = j + (int)start[0]; + + /* Same for all cases */ + mem_tids[i] = H5T_NATIVE_INT; + r_mem_tids[i] = H5T_NATIVE_INT; + + break; + + case DSET_WITH_CONV_AND_NO_BKG: + if (s == SETTING_A) { + /* Initialize buffer indices */ + ul_wbufi2[i] = ul_total_wbuf2 + (i * DSET_SELECT_DIM); + l_rbufi2[i] = l_total_rbuf2 + (i * DSET_SELECT_DIM); + + wbufs[i] = ul_wbufi2[i]; + rbufs[i] = l_rbufi2[i]; + + for (j = 0; j < (int)block[0]; j++) + ul_wbufi2[i][j] = ULONG_MAX - (unsigned long)(j + (int)start[0]); + + mem_tids[i] = H5T_NATIVE_ULONG; + r_mem_tids[i] = H5T_NATIVE_LONG; + } + else if (s == SETTING_B) { + /* Initialize buffer indices */ + l_wbufi2[i] = l_total_wbuf2 + (i * DSET_SELECT_DIM); + s_rbufi2[i] = s_total_rbuf2 + (i * DSET_SELECT_DIM); + + wbufs[i] = l_wbufi2[i]; + rbufs[i] = s_rbufi2[i]; + + /* Initialize the buffer data */ + for (j = 0; j < (int)block[0]; j++) + l_wbufi2[i][j] = LONG_MAX - (long)(j + (int)start[0]); + + mem_tids[i] = H5T_NATIVE_LONG; + r_mem_tids[i] = H5T_NATIVE_SHORT; + } + + break; + + case DSET_WITH_CONV_AND_BKG: + + if (s == SETTING_A) { + /* Initialize buffer indices */ + s1_wbufi3[i] = s1_total_wbuf3 + (i * DSET_SELECT_DIM); + s3_rbufi3[i] = s3_total_rbuf3 + (i * DSET_SELECT_DIM); + + wbufs[i] = s1_wbufi3[i]; + rbufs[i] = s3_rbufi3[i]; + + /* Initialize buffer data for s1_t */ + for (j = 0; j < (int)block[0]; j++) { + s1_wbufi3[i][j].a = (4 * j + (int)start[0]); + s1_wbufi3[i][j].b = (4 * j + (int)start[0]) + 1; + s1_wbufi3[i][j].c = (4 * j + (int)start[0]) + 2; + s1_wbufi3[i][j].d = (4 * j + (int)start[0]) + 3; + } + mem_tids[i] = s1_tid; + r_mem_tids[i] = s3_tid; + } + else if (s == SETTING_B) { + /* Initialize buffer indices */ + s4_wbufi3[i] = s4_total_wbuf3 + (i * DSET_SELECT_DIM); + s1_rbufi3[i] = s1_total_rbuf3 + (i * DSET_SELECT_DIM); + + wbufs[i] = s4_wbufi3[i]; + rbufs[i] = s1_rbufi3[i]; + + /* Initialize buffer data for s4_t */ + for (j = 0; j < (int)block[0]; j++) { + s4_wbufi3[i][j].b = DSET_SELECT_DIM + (unsigned int)(j + (int)start[0]); + s4_wbufi3[i][j].d = DSET_SELECT_DIM + (unsigned int)(j + (int)start[0]); + } + mem_tids[i] = s4_tid; + r_mem_tids[i] = s1_tid; + } + + break; + + case DSET_NTTYPES: + default: + P_TEST_ERROR; + + } /* end switch dset_types */ + + } /* end for i ndsets */ + + /* Copy wbufs if the library will be modifying them */ + if (mwbuf) { + HDmemcpy(total_wbuf1_bak, total_wbuf1, ndsets * DSET_SELECT_DIM * sizeof(int)); + HDmemcpy(ul_total_wbuf2_bak, ul_total_wbuf2, + ndsets * DSET_SELECT_DIM * sizeof(unsigned long)); + HDmemcpy(l_total_wbuf2_bak, l_total_wbuf2, ndsets * DSET_SELECT_DIM * sizeof(long)); + HDmemcpy(s1_total_wbuf3_bak, s1_total_wbuf3, ndsets * DSET_SELECT_DIM * sizeof(s1_t)); + HDmemcpy(s4_total_wbuf3_bak, s4_total_wbuf3, ndsets * DSET_SELECT_DIM * sizeof(s4_t)); + } + + if (H5Dwrite_multi(ndsets, dset_dids, mem_tids, mem_sids, file_sids, dxpl, wbufs) < 0) + P_TEST_ERROR; + + /* Restore wbufs from backup if the library modified them */ + if (mwbuf) { + HDmemcpy(total_wbuf1, total_wbuf1_bak, ndsets * DSET_SELECT_DIM * sizeof(int)); + HDmemcpy(ul_total_wbuf2, ul_total_wbuf2_bak, + ndsets * DSET_SELECT_DIM * sizeof(unsigned long)); + HDmemcpy(l_total_wbuf2, l_total_wbuf2_bak, ndsets * DSET_SELECT_DIM * sizeof(long)); + HDmemcpy(s1_total_wbuf3, s1_total_wbuf3_bak, ndsets * DSET_SELECT_DIM * sizeof(s1_t)); + HDmemcpy(s4_total_wbuf3, s4_total_wbuf3_bak, ndsets * DSET_SELECT_DIM * sizeof(s4_t)); + } + + if (H5Dread_multi(ndsets, dset_dids, r_mem_tids, mem_sids, file_sids, dxpl, rbufs) < 0) + P_TEST_ERROR; + + check_io_mode(dxpl, chunked); + + /* Verify result read */ + /* for i ndsets */ + for (i = 0; i < (int)ndsets; i++) { + switch (dset_types[i]) { + + case DSET_WITH_NO_CONV: + for (j = 0; j < (int)block[0]; j++) + if (rbufi1[i][j] != wbufi1[i][j]) { + nerrors++; + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j + (int)start[0]); + break; + } + + break; + + case DSET_WITH_CONV_AND_NO_BKG: + if (s == SETTING_A) { + for (j = 0; j < (int)block[0]; j++) + if (l_rbufi2[i][j] != LONG_MAX) { + nerrors++; + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j + (int)start[0]); + break; + } + } + else if (s == SETTING_B) { + for (j = 0; j < (int)block[0]; j++) + if (s_rbufi2[i][j] != SHRT_MAX) { + nerrors++; + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j + (int)start[0]); + break; + } + } + + break; + + case DSET_WITH_CONV_AND_BKG: + if (s == SETTING_A) { + for (j = 0; j < (int)block[0]; j++) + if (s3_rbufi3[i][j].a != s1_wbufi3[i][j].a || + s3_rbufi3[i][j].b != s1_wbufi3[i][j].b || + s3_rbufi3[i][j].c != s1_wbufi3[i][j].c || + s3_rbufi3[i][j].d != s1_wbufi3[i][j].d) { + nerrors++; + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j + (int)start[0]); + break; + } + } + else if (s == SETTING_B) { + for (j = 0; j < (int)block[0]; j++) + if (s1_rbufi3[i][j].a != s1_wbufi3[i][j].a || + s1_rbufi3[i][j].b != (DSET_SELECT_DIM + j + (int)start[0]) || + s1_rbufi3[i][j].c != s1_wbufi3[i][j].c || + s1_rbufi3[i][j].d != (DSET_SELECT_DIM + j + (int)start[0])) { + nerrors++; + HDprintf(" Read different values than written.\n"); + HDprintf(" For dset %d at index %d\n", i, j + (int)start[0]); + break; + } + } + + break; + + case DSET_NTTYPES: + default: + P_TEST_ERROR; + + } /* end switch dset_types */ + + } /* end for i ndsets */ + + } /* end for s settings */ + + /* Closing */ + if (H5Pclose(dcpl) < 0) + P_TEST_ERROR; + if (H5Pclose(dxpl) < 0) + P_TEST_ERROR; + + if (H5Tclose(s1_tid) < 0) + P_TEST_ERROR; + if (H5Tclose(s3_tid) < 0) + P_TEST_ERROR; + if (H5Tclose(s4_tid) < 0) + P_TEST_ERROR; + + for (i = 0; i < (int)ndsets; i++) { + if (H5Sclose(file_sids[i]) < 0) + P_TEST_ERROR; + if (H5Dclose(dset_dids[i]) < 0) + P_TEST_ERROR; + /* Don't delete the last set of datasets */ + if ((n + 1) != niter) + if (H5Ldelete(fid, dset_names[i], H5P_DEFAULT) < 0) + P_TEST_ERROR; + } + + /* Freeing */ + HDfree(total_wbuf1); + total_wbuf1 = NULL; + HDfree(total_wbuf1_bak); + total_wbuf1_bak = NULL; + HDfree(total_rbuf1); + total_rbuf1 = NULL; + + HDfree(ul_total_wbuf2); + ul_total_wbuf2 = NULL; + HDfree(ul_total_wbuf2_bak); + ul_total_wbuf2_bak = NULL; + HDfree(l_total_rbuf2); + l_total_rbuf2 = NULL; + HDfree(l_total_wbuf2); + l_total_wbuf2 = NULL; + HDfree(l_total_wbuf2_bak); + l_total_wbuf2_bak = NULL; + HDfree(s_total_rbuf2); + s_total_rbuf2 = NULL; + + HDfree(s1_total_wbuf3); + s1_total_wbuf3 = NULL; + HDfree(s1_total_wbuf3_bak); + s1_total_wbuf3_bak = NULL; + HDfree(s3_total_rbuf3); + s3_total_rbuf3 = NULL; + HDfree(s4_total_wbuf3); + s4_total_wbuf3 = NULL; + HDfree(s4_total_wbuf3_bak); + s4_total_wbuf3_bak = NULL; + HDfree(s1_total_rbuf3); + s1_total_rbuf3 = NULL; + + } /* end for n niter */ + + CHECK_PASSED(); + + return; + +} /* test_multi_dsets_all() */ + +/* + * Test with various test_mode that no selection I/O is performed + * + * Note: It is the responsibility of the tester to + * understand and feed proper combination of test_mode + * as needed. + */ +static void +test_no_selection_io_cause_mode(const char *filename, hid_t fapl, uint32_t test_mode) +{ + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + hid_t fid = H5I_INVALID_HID; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hsize_t dims[1]; + hsize_t cdims[1]; + hbool_t is_chunked = FALSE; + hid_t tid = H5T_NATIVE_INT; + uint32_t no_selection_io_cause_write = 0; + uint32_t no_selection_io_cause_read = 0; + uint32_t no_selection_io_cause_write_expected = 0; + uint32_t no_selection_io_cause_read_expected = 0; + int wbuf[DSET_SELECT_DIM]; + int rbuf[DSET_SELECT_DIM]; + int i; + + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + P_TEST_ERROR; + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + P_TEST_ERROR; + + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + P_TEST_ERROR; + + if (test_mode & TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET) { + if (H5Pset_layout(dcpl, H5D_COMPACT) < 0) + P_TEST_ERROR; + no_selection_io_cause_write_expected |= H5D_SEL_IO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; + no_selection_io_cause_read_expected |= H5D_SEL_IO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; + } + + if (test_mode == TEST_DISABLE_BY_API) { + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_OFF) < 0) + P_TEST_ERROR; + no_selection_io_cause_write_expected |= H5D_SEL_IO_DISABLE_BY_API; + no_selection_io_cause_read_expected |= H5D_SEL_IO_DISABLE_BY_API; + } + + /* Datatype conversion */ + if (test_mode & TEST_DATATYPE_CONVERSION) { + if (H5Pset_selection_io(dxpl, H5D_SELECTION_IO_MODE_ON) < 0) + P_TEST_ERROR; + tid = H5T_NATIVE_UINT; + + /* If we're testing a too small tconv buffer, set the buffer to be too small */ + if (test_mode & TEST_TCONV_BUF_TOO_SMALL) { + if (H5Pset_buffer(dxpl, sizeof(int), NULL, NULL) < 0) + P_TEST_ERROR; + + /* If we're using in-place type conversion sel io will succeed */ + if (test_mode & TEST_IN_PLACE_TCONV) { + if (H5Pset_modify_write_buf(dxpl, TRUE) < 0) + P_TEST_ERROR; + } + else + no_selection_io_cause_write_expected |= H5D_SEL_IO_TCONV_BUF_TOO_SMALL; + + /* In-place type conversion for read doesn't require modify_write_buf */ + } + + /* If the tconv buf is largge enough sel io will succeed */ + } + + /* Create 1d data space */ + dims[0] = DSET_SELECT_DIM; + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + P_TEST_ERROR; + + if (is_chunked) { + cdims[0] = DSET_SELECT_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, cdims) < 0) + P_TEST_ERROR; + } + + if ((did = H5Dcreate2(fid, "no_selection_io_cause", H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, + H5P_DEFAULT)) < 0) + P_TEST_ERROR; + + /* Initialize data */ + for (i = 0; i < DSET_SELECT_DIM; i++) + wbuf[i] = i; + + if (H5Dwrite(did, tid, H5S_ALL, H5S_ALL, dxpl, wbuf) < 0) + P_TEST_ERROR; + + if (H5Pget_no_selection_io_cause(dxpl, &no_selection_io_cause_write) < 0) + P_TEST_ERROR; + + /* Verify causes of no selection I/O for write are as expected */ + if (no_selection_io_cause_write != no_selection_io_cause_write_expected) + P_TEST_ERROR; + + if (H5Dread(did, tid, H5S_ALL, H5S_ALL, dxpl, rbuf) < 0) + P_TEST_ERROR; + + if (H5Pget_no_selection_io_cause(dxpl, &no_selection_io_cause_read) < 0) + P_TEST_ERROR; + + /* Verify causes of no selection I/O for read are as expected */ + if (no_selection_io_cause_read != no_selection_io_cause_read_expected) + P_TEST_ERROR; + + if (H5Dclose(did) < 0) + P_TEST_ERROR; + + if (H5Pclose(dcpl) < 0) + P_TEST_ERROR; + + if (H5Pclose(dxpl) < 0) + P_TEST_ERROR; + + if (H5Sclose(sid) < 0) + P_TEST_ERROR; + + if (H5Fclose(fid) < 0) + P_TEST_ERROR; + + return; + +} /* test_no_selection_io_cause_mode() */ + +/* + * Test for causes of not performing selection I/O + */ +static void +test_get_no_selection_io_cause(const char *filename, hid_t fapl) +{ + test_no_selection_io_cause_mode(filename, fapl, TEST_DISABLE_BY_API); + test_no_selection_io_cause_mode(filename, fapl, TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET); + test_no_selection_io_cause_mode(filename, fapl, TEST_DATATYPE_CONVERSION); + test_no_selection_io_cause_mode(filename, fapl, TEST_DATATYPE_CONVERSION | TEST_TCONV_BUF_TOO_SMALL); + test_no_selection_io_cause_mode( + filename, fapl, TEST_DATATYPE_CONVERSION | TEST_TCONV_BUF_TOO_SMALL | TEST_IN_PLACE_TCONV); + + CHECK_PASSED(); + + return; +} /* test_get_no_selection_io_cause() */ + +/*------------------------------------------------------------------------- + * Function: main + * + * Purpose: Runs tests with all combinations of configuration + * flags. + * + * Return: Success: 0 + * Failure: 1 + * + *------------------------------------------------------------------------- + */ +int +main(int argc, char *argv[]) +{ + int ret; + hid_t fapl = H5I_INVALID_HID; + hid_t fid = H5I_INVALID_HID; + int test_select_config; + unsigned chunked; + unsigned dtrans; + unsigned mwbuf; + + h5_reset(); + + /* Initialize MPI */ + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + + if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + P_TEST_ERROR; + + /* Set MPIO file driver */ + if (H5Pset_fapl_mpio(fapl, MPI_COMM_WORLD, MPI_INFO_NULL) < 0) + P_TEST_ERROR; + + if ((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + P_TEST_ERROR; + + /* Test with contiguous or chunked dataset */ + for (chunked = FALSE; chunked <= TRUE; chunked++) { + + /* Data transforms only apply to integer or floating-point datasets */ + /* therefore, not all tests are run with data transform */ + for (dtrans = FALSE; dtrans <= TRUE; dtrans++) { + + /* Test with and without modify_write_buf turned on */ + for (mwbuf = FALSE; mwbuf <= TRUE; mwbuf++) { + + if (MAINPROCESS) { + /* Print configuration message */ + printf("Testing for selection I/O "); + if (chunked) + printf("with chunked dataset, "); + else + printf("with contiguous dataset, "); + if (dtrans) + printf("data transform, "); + else + printf("without data transform, "); + if (mwbuf) + printf("and with modifying write buffers\n"); + else + printf("and without modifying write buffers\n"); + } + + for (test_select_config = (int)TEST_NO_TYPE_CONV; + test_select_config < (int)TEST_SELECT_NTESTS; test_select_config++) { + + switch (test_select_config) { + + case TEST_NO_TYPE_CONV: /* case 1 */ + if (MAINPROCESS) + TESTING_2("No type conversion (null case)"); + + test_no_type_conv(fid, chunked, dtrans, mwbuf); + + break; + + case TEST_NO_SIZE_CHANGE_NO_BKG: /* case 2 */ + if (MAINPROCESS) + TESTING_2("No size change, no background buffer"); + + /* Data transforms does not apply to the dataset datatype for this test */ + if (dtrans) { + if (MAINPROCESS) + SKIPPED(); + continue; + } + + test_no_size_change_no_bkg(fid, chunked, mwbuf); + + break; + + case TEST_LARGER_MEM_NO_BKG: /* case 3 */ + if (MAINPROCESS) + TESTING_2("Larger memory type, no background buffer"); + + test_larger_mem_type_no_bkg(fid, chunked, dtrans, mwbuf); + + break; + + case TEST_SMALLER_MEM_NO_BKG: /* case 4 */ + if (MAINPROCESS) + TESTING_2("Smaller memory type, no background buffer"); + + test_smaller_mem_type_no_bkg(fid, chunked, dtrans, mwbuf); + + break; + + case TEST_CMPD_WITH_BKG: /* case 5 */ + if (MAINPROCESS) + TESTING_2("Compound types with background buffer"); + /* Data transforms does not apply to the dataset datatype for this test */ + if (dtrans) { + if (MAINPROCESS) + SKIPPED(); + continue; + } + + test_cmpd_with_bkg(fid, chunked, mwbuf); + + break; + + case TEST_TYPE_CONV_SEL_EMPTY: /* case 6 */ + if (MAINPROCESS) + TESTING_2("Empty selections + Type conversion"); + + test_type_conv_sel_empty(fid, chunked, dtrans, mwbuf); + + break; + + case TEST_MULTI_CONV_NO_BKG: /* case 7 */ + if (MAINPROCESS) + TESTING_2("multi-datasets: type conv + no bkg buffer"); + + test_multi_dsets_no_bkg(fid, chunked, dtrans, mwbuf); + + break; + + case TEST_MULTI_CONV_BKG: /* case 8 */ + if (MAINPROCESS) + TESTING_2("multi-datasets: type conv + bkg buffer"); + + /* Data transforms does not apply to the dataset datatype for this test */ + if (dtrans) { + if (MAINPROCESS) + SKIPPED(); + } + else + test_multi_dsets_cmpd_with_bkg(fid, chunked, mwbuf); + + break; + + case TEST_MULTI_CONV_SIZE_CHANGE: /* case 9 */ + if (MAINPROCESS) + TESTING_2("multi-datasets: type conv + size change + no bkg buffer"); + + /* Data transforms does not apply to the dataset datatype for this test */ + if (dtrans) { + if (MAINPROCESS) + SKIPPED(); + } + else + test_multi_dsets_size_change_no_bkg(fid, chunked, mwbuf); + + break; + + case TEST_MULTI_CONV_SEL_EMPTY: /* case 10 */ + if (MAINPROCESS) + TESTING_2("multi-datasets: type conv + empty selections"); + + test_multi_dsets_conv_sel_empty(fid, chunked, dtrans, mwbuf); + + break; + + case TEST_MULTI_ALL: /* case 11 */ + if (MAINPROCESS) + TESTING_2("multi-datasets: no conv + conv without bkg + conv with bkg"); + + /* Data transforms does not apply to the dataset datatype for this test */ + if (dtrans) { + if (MAINPROCESS) + SKIPPED(); + } + else + test_multi_dsets_all(2, fid, chunked, mwbuf); + + break; + + case TEST_SELECT_NTESTS: + default: + P_TEST_ERROR; + break; + + } /* end switch */ + + } /* end for test_select_config */ + + } /* end mwbuf */ + + } /* end dtrans */ + } /* end chunked */ + + if (H5Fclose(fid) < 0) + P_TEST_ERROR; + + if (MAINPROCESS) { + printf("\n"); + TESTING("Testing for H5Pget_no_selection_io_cause()"); + } + test_get_no_selection_io_cause(FILENAME, fapl); + + /* Barrier to make sure all ranks are done before deleting the file, and + * also to clean up output (make sure PASSED is printed before any of the + * following messages) */ + if (MPI_Barrier(MPI_COMM_WORLD) != MPI_SUCCESS) + P_TEST_ERROR; + + /* Delete file */ + if (H5Fdelete(FILENAME, fapl) < 0) + P_TEST_ERROR; + + if (H5Pclose(fapl) < 0) + P_TEST_ERROR; + + /* Gather errors from all processes */ + MPI_Allreduce(&nerrors, &ret, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); + nerrors = ret; + + if (MAINPROCESS) { + printf("\n===================================\n"); + if (nerrors) + HDprintf("***Parallel selection I/O dataset tests detected %d errors***\n", nerrors); + else + HDprintf("Parallel selection I/O dataset tests finished with no errors\n"); + printf("===================================\n"); + } + + /* close HDF5 library */ + H5close(); + + /* MPI_Finalize must be called AFTER H5close which may use MPI calls */ + MPI_Finalize(); + + /* cannot just return (nerrors) because exit code is limited to 1 byte */ + return (nerrors != 0); +} /* end main() */ diff --git a/testpar/t_subfiling_vfd.c b/testpar/t_subfiling_vfd.c index 85df3bd..f827aa5 100644 --- a/testpar/t_subfiling_vfd.c +++ b/testpar/t_subfiling_vfd.c @@ -425,7 +425,10 @@ test_stripe_sizes(void) VRFY(tmp_filename, "HDmalloc succeeded"); dxpl_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((dxpl_id >= 0), "DCPL creation succeeded"); + VRFY((dxpl_id >= 0), "DXPL creation succeeded"); + + /* Set selection I/O mode on DXPL */ + VRFY((H5Pset_selection_io(dxpl_id, H5D_SELECTION_IO_MODE_ON) >= 0), "H5Pset_selection_io succeeded"); for (size_t i = 0; i < SUBF_NITER; i++) { H5FD_subfiling_params_t cfg; @@ -1011,12 +1014,19 @@ test_read_different_stripe_size(void) hid_t file_id = H5I_INVALID_HID; hid_t fapl_id = H5I_INVALID_HID; hid_t dset_id = H5I_INVALID_HID; + hid_t dxpl_id = H5I_INVALID_HID; hid_t fspace_id = H5I_INVALID_HID; char *tmp_filename = NULL; void *buf = NULL; curr_nerrors = nerrors; + dxpl_id = H5Pcreate(H5P_DATASET_XFER); + VRFY((dxpl_id >= 0), "DXPL creation succeeded"); + + /* Set selection I/O mode on DXPL */ + VRFY((H5Pset_selection_io(dxpl_id, H5D_SELECTION_IO_MODE_ON) >= 0), "H5Pset_selection_io succeeded"); + if (MAINPROCESS) TESTING_2("file re-opening with different stripe size"); @@ -1066,7 +1076,7 @@ test_read_different_stripe_size(void) for (size_t i = 0; i < count[0]; i++) ((SUBF_C_TYPE *)buf)[i] = (SUBF_C_TYPE)((size_t)mpi_rank + i); - VRFY((H5Dwrite(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, fspace_id, H5P_DEFAULT, buf) >= 0), + VRFY((H5Dwrite(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, fspace_id, dxpl_id, buf) >= 0), "Dataset write succeeded"); HDfree(buf); @@ -1133,7 +1143,7 @@ test_read_different_stripe_size(void) buf = HDcalloc(1, count[0] * sizeof(SUBF_C_TYPE)); VRFY(buf, "HDcalloc succeeded"); - VRFY((H5Dread(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, fspace_id, H5P_DEFAULT, buf) >= 0), + VRFY((H5Dread(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, fspace_id, dxpl_id, buf) >= 0), "Dataset read succeeded"); for (size_t i = 0; i < count[0]; i++) { @@ -1185,6 +1195,7 @@ test_read_different_stripe_size(void) } H5E_END_TRY; + VRFY((H5Pclose(dxpl_id) >= 0), "DXPL close succeeded"); VRFY((H5Pclose(fapl_id) >= 0), "FAPL close succeeded"); HDfree(tmp_filename); @@ -1214,11 +1225,18 @@ test_subfiling_precreate_rank_0(void) hid_t file_id = H5I_INVALID_HID; hid_t fapl_id = H5I_INVALID_HID; hid_t dset_id = H5I_INVALID_HID; + hid_t dxpl_id = H5I_INVALID_HID; hid_t fspace_id = H5I_INVALID_HID; void *buf = NULL; curr_nerrors = nerrors; + dxpl_id = H5Pcreate(H5P_DATASET_XFER); + VRFY((dxpl_id >= 0), "DXPL creation succeeded"); + + /* Set selection I/O mode on DXPL */ + VRFY((H5Pset_selection_io(dxpl_id, H5D_SELECTION_IO_MODE_ON) >= 0), "H5Pset_selection_io succeeded"); + if (MAINPROCESS) TESTING_2("file pre-creation on rank 0"); @@ -1278,7 +1296,7 @@ test_subfiling_precreate_rank_0(void) for (size_t i = 0; i < dset_dims[0]; i++) ((SUBF_C_TYPE *)buf)[i] = (SUBF_C_TYPE)((i / n_elements_per_rank) + (i % n_elements_per_rank)); - VRFY((H5Dwrite(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, fspace_id, H5P_DEFAULT, buf) >= 0), + VRFY((H5Dwrite(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, fspace_id, dxpl_id, buf) >= 0), "Dataset write succeeded"); HDfree(buf); @@ -1357,7 +1375,7 @@ test_subfiling_precreate_rank_0(void) buf = HDcalloc(1, count[0] * sizeof(SUBF_C_TYPE)); VRFY(buf, "HDcalloc succeeded"); - VRFY((H5Dread(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, fspace_id, H5P_DEFAULT, buf) >= 0), + VRFY((H5Dread(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, fspace_id, dxpl_id, buf) >= 0), "Dataset read succeeded"); for (size_t i = 0; i < n_elements_per_rank; i++) { @@ -1380,6 +1398,7 @@ test_subfiling_precreate_rank_0(void) H5E_END_TRY; VRFY((H5Pclose(fapl_id) >= 0), "FAPL close succeeded"); + VRFY((H5Pclose(dxpl_id) >= 0), "DXPL close succeeded"); CHECK_PASSED(); } @@ -1405,11 +1424,18 @@ test_subfiling_write_many_read_one(void) hid_t file_id = H5I_INVALID_HID; hid_t fapl_id = H5I_INVALID_HID; hid_t dset_id = H5I_INVALID_HID; + hid_t dxpl_id = H5I_INVALID_HID; hid_t fspace_id = H5I_INVALID_HID; void *buf = NULL; curr_nerrors = nerrors; + dxpl_id = H5Pcreate(H5P_DATASET_XFER); + VRFY((dxpl_id >= 0), "DXPL creation succeeded"); + + /* Set selection I/O mode on DXPL */ + VRFY((H5Pset_selection_io(dxpl_id, H5D_SELECTION_IO_MODE_ON) >= 0), "H5Pset_selection_io succeeded"); + if (MAINPROCESS) TESTING_2("reading back file with single MPI rank"); @@ -1461,7 +1487,7 @@ test_subfiling_write_many_read_one(void) for (size_t i = 0; i < count[0]; i++) ((SUBF_C_TYPE *)buf)[i] = (SUBF_C_TYPE)((size_t)mpi_rank + i); - VRFY((H5Dwrite(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, fspace_id, H5P_DEFAULT, buf) >= 0), + VRFY((H5Dwrite(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, fspace_id, dxpl_id, buf) >= 0), "Dataset write succeeded"); HDfree(buf); @@ -1486,7 +1512,7 @@ test_subfiling_write_many_read_one(void) buf = HDcalloc(1, target_size); VRFY(buf, "HDcalloc succeeded"); - VRFY((H5Dread(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, H5S_ALL, H5P_DEFAULT, buf) >= 0), + VRFY((H5Dread(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, H5S_ALL, dxpl_id, buf) >= 0), "Dataset read succeeded"); for (size_t i = 0; i < (size_t)mpi_size; i++) { @@ -1516,6 +1542,7 @@ test_subfiling_write_many_read_one(void) VRFY((mpi_code_g == MPI_SUCCESS), "MPI_Barrier succeeded"); VRFY((H5Sclose(fspace_id) >= 0), "File dataspace close succeeded"); + VRFY((H5Pclose(dxpl_id) >= 0), "DXPL close succeeded"); CHECK_PASSED(); } @@ -1543,11 +1570,18 @@ test_subfiling_write_many_read_few(void) hid_t file_id = H5I_INVALID_HID; hid_t fapl_id = H5I_INVALID_HID; hid_t dset_id = H5I_INVALID_HID; + hid_t dxpl_id = H5I_INVALID_HID; hid_t fspace_id = H5I_INVALID_HID; void *buf = NULL; curr_nerrors = nerrors; + dxpl_id = H5Pcreate(H5P_DATASET_XFER); + VRFY((dxpl_id >= 0), "DXPL creation succeeded"); + + /* Set selection I/O mode on DXPL */ + VRFY((H5Pset_selection_io(dxpl_id, H5D_SELECTION_IO_MODE_ON) >= 0), "H5Pset_selection_io succeeded"); + if (MAINPROCESS) TESTING_2("reading back file with fewer MPI ranks than written with"); @@ -1609,7 +1643,7 @@ test_subfiling_write_many_read_few(void) for (size_t i = 0; i < count[0]; i++) ((SUBF_C_TYPE *)buf)[i] = (SUBF_C_TYPE)((size_t)mpi_rank + i); - VRFY((H5Dwrite(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, fspace_id, H5P_DEFAULT, buf) >= 0), + VRFY((H5Dwrite(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, fspace_id, dxpl_id, buf) >= 0), "Dataset write succeeded"); HDfree(buf); @@ -1664,7 +1698,7 @@ test_subfiling_write_many_read_few(void) buf = HDcalloc(1, target_size); VRFY(buf, "HDcalloc succeeded"); - VRFY((H5Dread(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, H5S_ALL, H5P_DEFAULT, buf) >= 0), + VRFY((H5Dread(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, H5S_ALL, dxpl_id, buf) >= 0), "Dataset read succeeded"); for (size_t i = 0; i < (size_t)mpi_size; i++) { @@ -1699,6 +1733,7 @@ test_subfiling_write_many_read_few(void) VRFY((mpi_code_g == MPI_SUCCESS), "MPI_Barrier succeeded"); VRFY((H5Sclose(fspace_id) >= 0), "File dataspace close succeeded"); + VRFY((H5Pclose(dxpl_id) >= 0), "DXPL close succeeded"); CHECK_PASSED(); } @@ -1727,6 +1762,7 @@ test_subfiling_h5fuse(void) hid_t file_id = H5I_INVALID_HID; hid_t fapl_id = H5I_INVALID_HID; hid_t dset_id = H5I_INVALID_HID; + hid_t dxpl_id = H5I_INVALID_HID; hid_t fspace_id = H5I_INVALID_HID; void *buf = NULL; int skip_test = 0; @@ -1734,6 +1770,12 @@ test_subfiling_h5fuse(void) curr_nerrors = nerrors; + dxpl_id = H5Pcreate(H5P_DATASET_XFER); + VRFY((dxpl_id >= 0), "DXPL creation succeeded"); + + /* Set selection I/O mode on DXPL */ + VRFY((H5Pset_selection_io(dxpl_id, H5D_SELECTION_IO_MODE_ON) >= 0), "H5Pset_selection_io succeeded"); + if (MAINPROCESS) TESTING_2("h5fuse utility"); @@ -1826,7 +1868,7 @@ test_subfiling_h5fuse(void) for (size_t i = 0; i < count[0]; i++) ((SUBF_C_TYPE *)buf)[i] = (SUBF_C_TYPE)((size_t)mpi_rank + i); - VRFY((H5Dwrite(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, fspace_id, H5P_DEFAULT, buf) >= 0), + VRFY((H5Dwrite(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, fspace_id, dxpl_id, buf) >= 0), "Dataset write succeeded"); HDfree(buf); @@ -1899,7 +1941,7 @@ test_subfiling_h5fuse(void) buf = HDcalloc(1, target_size); VRFY(buf, "HDcalloc succeeded"); - VRFY((H5Dread(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, H5S_ALL, H5P_DEFAULT, buf) >= 0), + VRFY((H5Dread(dset_id, SUBF_HDF5_TYPE, H5S_BLOCK, H5S_ALL, dxpl_id, buf) >= 0), "Dataset read succeeded"); for (size_t i = 0; i < (size_t)mpi_size; i++) { @@ -1969,6 +2011,7 @@ test_subfiling_h5fuse(void) } VRFY((H5Pclose(fapl_id) >= 0), "FAPL close succeeded"); + VRFY((H5Pclose(dxpl_id) >= 0), "DXPL close succeeded"); mpi_code_g = MPI_Barrier(comm_g); VRFY((mpi_code_g == MPI_SUCCESS), "MPI_Barrier succeeded"); @@ -2132,9 +2175,6 @@ main(int argc, char **argv) H5open(); - /* Enable selection I/O using internal temporary workaround */ - H5_use_selection_io_g = TRUE; - if (MAINPROCESS) { HDprintf("Testing Subfiling VFD functionality\n"); } |