From 6e489435e9ca484a4927d3bb83f2c0058147d7ba Mon Sep 17 00:00:00 2001 From: "M. Scot Breitenfeld" Date: Wed, 12 Dec 2018 15:39:18 -0600 Subject: merged: HDFFV-10652 HDFFV-10443 --- config/cmake/H5pubconf.h.in | 3 + config/cmake_ext_mod/ConfigureChecks.cmake | 1 + configure.ac | 9 + fortran/src/H5Of.c | 48 ++-- fortran/src/H5Off.F90 | 107 ++++++-- fortran/src/H5_f.c | 11 + fortran/src/H5_ff.F90 | 74 ++++- fortran/src/H5f90global.F90 | 13 + fortran/src/H5f90proto.h | 10 +- fortran/src/H5match_types.c | 13 + fortran/src/hdf5_fortrandll.def.in | 1 + fortran/test/tH5O_F03.F90 | 390 ++++++++++++++++++++++++-- release_docs/RELEASE.txt | 302 +++++++++++++++++++++ src/H5CX.c | 92 +++++++ src/H5CXprivate.h | 4 + src/H5Dio.c | 18 +- src/H5Dmpio.c | 121 +++++++-- src/H5Dprivate.h | 8 +- src/H5FDmpio.c | 89 ++++-- src/H5Pdxpl.c | 3 +- src/H5Ppublic.h | 3 +- src/H5T.c | 109 +++++++- src/H5Tprivate.h | 1 + testpar/t_bigio.c | 4 +- testpar/t_dset.c | 2 +- testpar/t_mdset.c | 173 ++++++++---- testpar/t_pread.c | 422 +++++++++++++++++++++++++++-- 27 files changed, 1813 insertions(+), 218 deletions(-) diff --git a/config/cmake/H5pubconf.h.in b/config/cmake/H5pubconf.h.in index b079297..4edaff0 100644 --- a/config/cmake/H5pubconf.h.in +++ b/config/cmake/H5pubconf.h.in @@ -599,6 +599,9 @@ /* The size of `short', as computed by sizeof. */ #cmakedefine H5_SIZEOF_SHORT @H5_SIZEOF_SHORT@ +/* The size of `time_t', as computed by sizeof. */ +#cmakedefine H5_SIZEOF_TIME_T @H5_SIZEOF_TIME_T@ + /* The size of `uint16_t', as computed by sizeof. */ #cmakedefine H5_SIZEOF_UINT16_T @H5_SIZEOF_UINT16_T@ diff --git a/config/cmake_ext_mod/ConfigureChecks.cmake b/config/cmake_ext_mod/ConfigureChecks.cmake index 0424ed6..c24c1f8 100644 --- a/config/cmake_ext_mod/ConfigureChecks.cmake +++ b/config/cmake_ext_mod/ConfigureChecks.cmake @@ -413,6 +413,7 @@ HDF_CHECK_TYPE_SIZE (off64_t ${HDF_PREFIX}_SIZEOF_OFF64_T) if (NOT ${HDF_PREFIX}_SIZEOF_OFF64_T) set (${HDF_PREFIX}_SIZEOF_OFF64_T 0) endif () +HDF_CHECK_TYPE_SIZE (time_t ${HDF_PREFIX}_SIZEOF_TIME_T) #----------------------------------------------------------------------------- # Extra C99 types diff --git a/configure.ac b/configure.ac index e724f06..6b591ae 100644 --- a/configure.ac +++ b/configure.ac @@ -1211,6 +1211,15 @@ EOF AC_CHECK_SIZEOF([bool]) fi +AC_CHECK_SIZEOF(time_t, [], [ +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_TIME_H +#include +#endif +]) + ## Checkpoint the cache AC_CACHE_SAVE diff --git a/fortran/src/H5Of.c b/fortran/src/H5Of.c index 45b7f09..08305ea 100644 --- a/fortran/src/H5Of.c +++ b/fortran/src/H5Of.c @@ -27,11 +27,15 @@ fill_h5o_info_t_f(H5O_info_t Oinfo, H5O_info_t_f *object_info); int_f fill_h5o_info_t_f(H5O_info_t Oinfo, H5O_info_t_f *object_info) { + /* This function does not used the field parameter because we want + * this function to fill the unfilled fields with C's default values. + */ + struct tm *ts; object_info->fileno = Oinfo.fileno; object_info->addr = (haddr_t_f)Oinfo.addr; - + object_info->type = (int_f)Oinfo.type; object_info->rc = (int_f)Oinfo.rc; @@ -96,6 +100,8 @@ fill_h5o_info_t_f(H5O_info_t Oinfo, H5O_info_t_f *object_info) { object_info->meta_size.obj.index_size = (hsize_t_f)Oinfo.meta_size.obj.index_size; object_info->meta_size.obj.heap_size = (hsize_t_f)Oinfo.meta_size.obj.heap_size; + object_info->meta_size.attr.index_size = (hsize_t_f)Oinfo.meta_size.attr.index_size; + object_info->meta_size.attr.heap_size = (hsize_t_f)Oinfo.meta_size.attr.heap_size; return 0; @@ -138,7 +144,7 @@ h5olink_c (hid_t_f *object_id, hid_t_f *new_loc_id, _fcd name, size_t_f *namelen * Call H5Olink function. */ if((hid_t_f)H5Olink((hid_t)*object_id, (hid_t)*new_loc_id, c_name, - (hid_t)*lcpl_id, (hid_t)*lapl_id) < 0) + (hid_t)*lcpl_id, (hid_t)*lapl_id) < 0) HGOTO_DONE(FAIL); done: @@ -229,6 +235,7 @@ h5oclose_c ( hid_t_f *object_id ) * idx - Iteration position at which to start * op - Callback function passing data regarding the link to the calling application * op_data - User-defined pointer to data required by the application for its processing of the link + * fields - Flags specifying the fields to include in object_info. * * OUTPUTS * idx - Position at which an interrupted iteration may be restarted @@ -241,7 +248,7 @@ h5oclose_c ( hid_t_f *object_id ) * SOURCE */ int_f -h5ovisit_c(hid_t_f *group_id, int_f *index_type, int_f *order, H5O_iterate_t op, void *op_data ) +h5ovisit_c(hid_t_f *group_id, int_f *index_type, int_f *order, H5O_iterate_t op, void *op_data, int_f *fields ) /******/ { int_f ret_value = -1; /* Return value */ @@ -250,7 +257,8 @@ h5ovisit_c(hid_t_f *group_id, int_f *index_type, int_f *order, H5O_iterate_t op, /* * Call H5Ovisit2 */ - func_ret_value = H5Ovisit2( (hid_t)*group_id, (H5_index_t)*index_type, (H5_iter_order_t)*order, op, op_data, H5O_INFO_ALL); + + func_ret_value = H5Ovisit2( (hid_t)*group_id, (H5_index_t)*index_type, (H5_iter_order_t)*order, op, op_data, (unsigned)*fields); ret_value = (int_f)func_ret_value; @@ -302,6 +310,7 @@ h5oopen_by_addr_c (hid_t_f *loc_id, haddr_t_f *addr, hid_t_f *obj_id) * name - Name of group, relative to loc_id. * namelen - Name length. * lapl_id - Link access property list. + * fields - Flags specifying the fields to include in object_info. * OUTPUTS * object_info - Buffer in which to return object information. * @@ -314,7 +323,7 @@ h5oopen_by_addr_c (hid_t_f *loc_id, haddr_t_f *addr, hid_t_f *obj_id) */ int_f h5oget_info_by_name_c (hid_t_f *loc_id, _fcd name, size_t_f *namelen, hid_t_f *lapl_id, - H5O_info_t_f *object_info) + H5O_info_t_f *object_info, int_f *fields) /******/ { char *c_name = NULL; /* Buffer to hold C string */ @@ -331,10 +340,10 @@ h5oget_info_by_name_c (hid_t_f *loc_id, _fcd name, size_t_f *namelen, hid_t_f *l * Call H5Oinfo_by_name function. */ if(H5Oget_info_by_name2((hid_t)*loc_id, c_name, - &Oinfo, H5O_INFO_ALL, (hid_t)*lapl_id) < 0) + &Oinfo, (unsigned)*fields, (hid_t)*lapl_id) < 0) HGOTO_DONE(FAIL); - ret_value = fill_h5o_info_t_f(Oinfo,object_info); + ret_value = fill_h5o_info_t_f(Oinfo, object_info); done: if(c_name) @@ -354,6 +363,7 @@ h5oget_info_by_name_c (hid_t_f *loc_id, _fcd name, size_t_f *namelen, hid_t_f *l * lapl_id - Link access property list. * OUTPUTS * object_info - Buffer in which to return object information. + * fields - Flags specifying the fields to include in object_info. * * RETURNS * 0 on success, -1 on failure @@ -364,7 +374,7 @@ h5oget_info_by_name_c (hid_t_f *loc_id, _fcd name, size_t_f *namelen, hid_t_f *l */ int_f h5oget_info_by_idx_c (hid_t_f *loc_id, _fcd group_name, size_t_f *namelen, - int_f *index_field, int_f *order, hsize_t_f *n, hid_t_f *lapl_id, H5O_info_t_f *object_info) + int_f *index_field, int_f *order, hsize_t_f *n, hid_t_f *lapl_id, H5O_info_t_f *object_info, int_f *fields) /******/ { char *c_group_name = NULL; /* Buffer to hold C string */ @@ -386,7 +396,7 @@ h5oget_info_by_idx_c (hid_t_f *loc_id, _fcd group_name, size_t_f *namelen, * Call H5Oinfo_by_idx function. */ if(H5Oget_info_by_idx2((hid_t)*loc_id, c_group_name, c_index_field, c_order, (hsize_t)*n, - &Oinfo, H5O_INFO_ALL, (hid_t)*lapl_id) < 0) + &Oinfo, (unsigned)*fields, (hid_t)*lapl_id) < 0) HGOTO_DONE(FAIL); ret_value = fill_h5o_info_t_f(Oinfo,object_info); @@ -404,6 +414,7 @@ h5oget_info_by_idx_c (hid_t_f *loc_id, _fcd group_name, size_t_f *namelen, * Calls H5Oget_info * INPUTS * object_id - Identifier for target object. + * fields - Flags specifying the fields to include in object_info. * OUTPUTS * object_info - Buffer in which to return object information. * @@ -415,7 +426,7 @@ h5oget_info_by_idx_c (hid_t_f *loc_id, _fcd group_name, size_t_f *namelen, * SOURCE */ int_f -h5oget_info_c (hid_t_f *object_id, H5O_info_t_f *object_info) +h5oget_info_c (hid_t_f *object_id, H5O_info_t_f *object_info, int_f *fields) /******/ { int_f ret_value = 0; /* Return value */ @@ -424,7 +435,7 @@ h5oget_info_c (hid_t_f *object_id, H5O_info_t_f *object_info) /* * Call H5Oinfo_by_name function. */ - if(H5Oget_info2((hid_t)*object_id, &Oinfo, H5O_INFO_ALL) < 0) + if(H5Oget_info2((hid_t)*object_id, &Oinfo, (unsigned)*fields) < 0) HGOTO_DONE(FAIL); ret_value = fill_h5o_info_t_f(Oinfo,object_info); @@ -457,8 +468,8 @@ h5oget_info_c (hid_t_f *object_id, H5O_info_t_f *object_info) */ int_f h5ocopy_c (hid_t_f *src_loc_id, _fcd src_name, size_t_f *src_name_len, - hid_t_f *dst_loc_id, _fcd dst_name, size_t_f *dst_name_len, - hid_t_f *ocpypl_id, hid_t_f *lcpl_id ) + hid_t_f *dst_loc_id, _fcd dst_name, size_t_f *dst_name_len, + hid_t_f *ocpypl_id, hid_t_f *lcpl_id ) /******/ { char *c_src_name = NULL; /* Buffer to hold C string */ @@ -478,7 +489,7 @@ h5ocopy_c (hid_t_f *src_loc_id, _fcd src_name, size_t_f *src_name_len, * Call H5Ocopy function. */ if(H5Ocopy( (hid_t)*src_loc_id, c_src_name, (hid_t)*dst_loc_id, c_dst_name, - (hid_t)*ocpypl_id, (hid_t)*lcpl_id) < 0) + (hid_t)*ocpypl_id, (hid_t)*lcpl_id) < 0) HGOTO_DONE(FAIL); done: @@ -503,6 +514,7 @@ h5ocopy_c (hid_t_f *src_loc_id, _fcd src_name, size_t_f *src_name_len, * idx - Iteration position at which to start * op - Callback function passing data regarding the link to the calling application * op_data - User-defined pointer to data required by the application for its processing of the link + * fields - Flags specifying the fields to include in object_info. * * OUTPUTS * idx - Position at which an interrupted iteration may be restarted @@ -516,7 +528,7 @@ h5ocopy_c (hid_t_f *src_loc_id, _fcd src_name, size_t_f *src_name_len, */ int_f h5ovisit_by_name_c(hid_t_f *loc_id, _fcd object_name, size_t_f *namelen, int_f *index_type, int_f *order, - H5O_iterate_t op, void *op_data, hid_t_f *lapl_id ) + H5O_iterate_t op, void *op_data, hid_t_f *lapl_id, int_f *fields ) /******/ { int_f ret_value = -1; /* Return value */ @@ -533,7 +545,7 @@ h5ovisit_by_name_c(hid_t_f *loc_id, _fcd object_name, size_t_f *namelen, int_f * Call H5Ovisit */ func_ret_value = H5Ovisit_by_name2( (hid_t)*loc_id, c_object_name, (H5_index_t)*index_type, (H5_iter_order_t)*order, - op, op_data, H5O_INFO_ALL, (hid_t)*lapl_id); + op, op_data, (unsigned)*fields, (hid_t)*lapl_id); ret_value = (int_f)func_ret_value; done: @@ -763,7 +775,7 @@ h5oset_comment_by_name_c (hid_t_f *object_id, _fcd name, size_t_f *namelen, _fc */ int_f h5oopen_by_idx_c (hid_t_f *loc_id, _fcd group_name, size_t_f *group_namelen, - int_f *index_type, int_f *order, hsize_t_f *n, hid_t_f *obj_id, hid_t_f *lapl_id) + int_f *index_type, int_f *order, hsize_t_f *n, hid_t_f *obj_id, hid_t_f *lapl_id) /******/ { char *c_group_name = NULL; /* Buffer to hold C string */ @@ -868,7 +880,7 @@ h5oget_comment_c (hid_t_f *object_id, _fcd comment, size_t_f *commentsize, hssi */ int_f h5oget_comment_by_name_c (hid_t_f *loc_id, _fcd name, size_t_f *name_size, - _fcd comment, size_t_f *commentsize, size_t_f *bufsize, hid_t_f *lapl_id) + _fcd comment, size_t_f *commentsize, size_t_f *bufsize, hid_t_f *lapl_id) /******/ { char *c_comment = NULL; /* Buffer to hold C string */ diff --git a/fortran/src/H5Off.F90 b/fortran/src/H5Off.F90 index 243ec29..8c77230 100644 --- a/fortran/src/H5Off.F90 +++ b/fortran/src/H5Off.F90 @@ -69,6 +69,15 @@ MODULE H5O TYPE(mesg_t) :: mesg END TYPE hdr_t + TYPE, BIND(C) :: c_hdr_t + INTEGER(C_INT) :: version ! Version number of header format in file + INTEGER(C_INT) :: nmesgs ! Number of object header messages + INTEGER(C_INT) :: nchunks ! Number of object header chunks + INTEGER(C_INT) :: flags ! Object header status flags + TYPE(space_t) :: space + TYPE(mesg_t) :: mesg + END TYPE c_hdr_t + ! Extra metadata storage for obj & attributes TYPE, BIND(C) :: H5_ih_info_t INTEGER(hsize_t) :: index_size ! btree and/or list @@ -83,7 +92,7 @@ MODULE H5O TYPE, BIND(C) :: h5o_info_t INTEGER(C_LONG) :: fileno ! File number that object is located in INTEGER(haddr_t) :: addr ! Object address in file - INTEGER(C_INT) :: type ! Basic object type (group, dataset, etc.) + INTEGER(C_INT) :: type ! Basic object type (group, dataset, etc.) INTEGER :: rc ! Reference count of object INTEGER, DIMENSION(8) :: atime ! Access time ! -- NOTE -- @@ -98,6 +107,28 @@ MODULE H5O TYPE(meta_size_t) :: meta_size END TYPE h5o_info_t +! C interoperable structure for h5o_info_t. The Fortran derived type returns the time +! values as an integer array as specified in the Fortran intrinsic DATE_AND_TIME(VALUES). +! Whereas, this derived type does not. + + TYPE, BIND(C) :: c_h5o_info_t + INTEGER(C_LONG) :: fileno ! File number that object is located in + INTEGER(haddr_t) :: addr ! Object address in file + INTEGER(C_INT) :: type ! Basic object type (group, dataset, etc.) + INTEGER(C_INT) :: rc ! Reference count of object + + INTEGER(KIND=TIME_T) :: atime ! Access time + INTEGER(KIND=TIME_T) :: mtime ! modify time + INTEGER(KIND=TIME_T) :: ctime ! create time + INTEGER(KIND=TIME_T) :: btime ! Access time + + INTEGER(hsize_t) :: num_attrs ! # of attributes attached to object + + TYPE(c_hdr_t) :: hdr + + TYPE(meta_size_t) :: meta_size + END TYPE c_h5o_info_t + !***** CONTAINS @@ -834,12 +865,16 @@ CONTAINS ! return_value - returns the return value of the first operator that returns a positive value, or ! zero if all members were processed with no operator returning non-zero. ! hdferr - Returns 0 if successful and -1 if fails +! +! Optional parameters: +! fields - Flags specifying the fields to include in object_info. +! ! AUTHOR ! M. Scot Breitenfeld ! November 19, 2008 ! ! Fortran2003 Interface: - SUBROUTINE h5ovisit_f(object_id, index_type, order, op, op_data, return_value, hdferr) + SUBROUTINE h5ovisit_f(object_id, index_type, order, op, op_data, return_value, hdferr, fields) IMPLICIT NONE INTEGER(HID_T), INTENT(IN) :: object_id INTEGER, INTENT(IN) :: index_type @@ -849,10 +884,12 @@ CONTAINS TYPE(C_PTR) :: op_data INTEGER, INTENT(OUT) :: return_value INTEGER, INTENT(OUT) :: hdferr + INTEGER, INTENT(IN), OPTIONAL :: fields !***** + INTEGER :: fields_c INTERFACE - INTEGER FUNCTION h5ovisit_c(object_id, index_type, order, op, op_data) & + INTEGER FUNCTION h5ovisit_c(object_id, index_type, order, op, op_data, fields) & BIND(C, NAME='h5ovisit_c') IMPORT :: C_FUNPTR, C_PTR IMPORT :: HID_T @@ -862,10 +899,14 @@ CONTAINS INTEGER, INTENT(IN) :: order TYPE(C_FUNPTR), VALUE :: op TYPE(C_PTR), VALUE :: op_data + INTEGER, INTENT(IN) :: fields END FUNCTION h5ovisit_c END INTERFACE - return_value = h5ovisit_c(object_id, index_type, order, op, op_data) + fields_c = H5O_INFO_ALL_F + IF(PRESENT(fields)) fields_c = fields + + return_value = h5ovisit_c(object_id, index_type, order, op, op_data, fields_c) IF(return_value.GE.0)THEN hdferr = 0 @@ -894,26 +935,29 @@ CONTAINS ! ! Optional parameters: ! lapl_id - Link access property list. +! fields - Flags specifying the fields to include in object_info. ! ! AUTHOR ! M. Scot Breitenfeld ! December 1, 2008 ! ! Fortran2003 Interface: - SUBROUTINE h5oget_info_by_name_f(loc_id, name, object_info, hdferr, lapl_id) + SUBROUTINE h5oget_info_by_name_f(loc_id, name, object_info, hdferr, lapl_id, fields) IMPLICIT NONE INTEGER(HID_T) , INTENT(IN) :: loc_id CHARACTER(LEN=*), INTENT(IN) :: name TYPE(h5o_info_t), INTENT(OUT), TARGET :: object_info INTEGER , INTENT(OUT) :: hdferr INTEGER(HID_T) , INTENT(IN) , OPTIONAL :: lapl_id + INTEGER , INTENT(IN) , OPTIONAL :: fields !***** INTEGER(SIZE_T) :: namelen INTEGER(HID_T) :: lapl_id_default TYPE(C_PTR) :: ptr + INTEGER :: fields_c INTERFACE - INTEGER FUNCTION h5oget_info_by_name_c(loc_id, name, namelen, lapl_id_default, object_info) & + INTEGER FUNCTION h5oget_info_by_name_c(loc_id, name, namelen, lapl_id_default, object_info, fields) & BIND(C, NAME='h5oget_info_by_name_c') IMPORT :: c_char, c_ptr IMPORT :: HID_T, SIZE_T @@ -923,10 +967,13 @@ CONTAINS INTEGER(SIZE_T) , INTENT(IN) :: namelen INTEGER(HID_T) , INTENT(IN) :: lapl_id_default TYPE(C_PTR),VALUE :: object_info - + INTEGER , INTENT(IN) :: fields END FUNCTION h5oget_info_by_name_c END INTERFACE + fields_c = H5O_INFO_ALL_F + IF(PRESENT(fields)) fields_c = fields + namelen = LEN(name) lapl_id_default = H5P_DEFAULT_F @@ -934,7 +981,7 @@ CONTAINS ptr = C_LOC(object_info) - hdferr = H5Oget_info_by_name_c(loc_id, name, namelen, lapl_id_default, ptr) + hdferr = H5Oget_info_by_name_c(loc_id, name, namelen, lapl_id_default, ptr, fields_c) END SUBROUTINE H5Oget_info_by_name_f @@ -953,34 +1000,43 @@ CONTAINS ! object_info - Buffer in which to return object information. ! hdferr - Returns 0 if successful and -1 if fails. ! +! Optional parameters: +! fields - Flags specifying the fields to include in object_info. +! ! AUTHOR ! M. Scot Breitenfeld ! May 11, 2012 ! ! Fortran2003 Interface: - SUBROUTINE h5oget_info_f(object_id, object_info, hdferr) + SUBROUTINE h5oget_info_f(object_id, object_info, hdferr, fields) USE, INTRINSIC :: ISO_C_BINDING IMPLICIT NONE INTEGER(HID_T) , INTENT(IN) :: object_id TYPE(h5o_info_t), INTENT(OUT), TARGET :: object_info INTEGER , INTENT(OUT) :: hdferr + INTEGER , INTENT(IN), OPTIONAL :: fields !***** TYPE(C_PTR) :: ptr - + INTEGER :: fields_c + INTERFACE - INTEGER FUNCTION h5oget_info_c(object_id, object_info) & + INTEGER FUNCTION h5oget_info_c(object_id, object_info, fields) & BIND(C, NAME='h5oget_info_c') IMPORT :: C_PTR IMPORT :: HID_T IMPLICIT NONE INTEGER(HID_T), INTENT(IN) :: object_id TYPE(C_PTR), VALUE :: object_info + INTEGER, INTENT(IN) :: fields END FUNCTION h5oget_info_c END INTERFACE + fields_c = H5O_INFO_ALL_F + IF(PRESENT(fields)) fields_c = fields + ptr = C_LOC(object_info) - hdferr = H5Oget_info_c(object_id, ptr) + hdferr = H5Oget_info_c(object_id, ptr, fields_c) END SUBROUTINE H5Oget_info_f @@ -1006,6 +1062,7 @@ CONTAINS ! ! Optional parameters: ! lapl_id - Link access property list. (Not currently used.) +! fields - Flags specifying the fields to include in object_info. ! ! AUTHOR ! M. Scot Breitenfeld @@ -1013,7 +1070,7 @@ CONTAINS ! ! Fortran2003 Interface: SUBROUTINE h5oget_info_by_idx_f(loc_id, group_name, index_field, order, n, & - object_info, hdferr, lapl_id) + object_info, hdferr, lapl_id, fields) USE, INTRINSIC :: ISO_C_BINDING IMPLICIT NONE @@ -1025,14 +1082,16 @@ CONTAINS TYPE(h5o_info_t), INTENT(OUT), TARGET :: object_info INTEGER , INTENT(OUT) :: hdferr INTEGER(HID_T) , INTENT(IN) , OPTIONAL :: lapl_id + INTEGER , INTENT(IN) , OPTIONAL :: fields !***** INTEGER(SIZE_T) :: namelen INTEGER(HID_T) :: lapl_id_default TYPE(C_PTR) :: ptr + INTEGER :: fields_c INTERFACE INTEGER FUNCTION h5oget_info_by_idx_c(loc_id, group_name, namelen, & - index_field, order, n, lapl_id_default, object_info) BIND(C, NAME='h5oget_info_by_idx_c') + index_field, order, n, lapl_id_default, object_info, fields) BIND(C, NAME='h5oget_info_by_idx_c') IMPORT :: c_char, c_ptr, c_funptr IMPORT :: HID_T, SIZE_T, HSIZE_T INTEGER(HID_T) , INTENT(IN) :: loc_id @@ -1043,17 +1102,20 @@ CONTAINS INTEGER(HSIZE_T), INTENT(IN) :: n INTEGER(HID_T) , INTENT(IN) :: lapl_id_default TYPE(C_PTR), VALUE :: object_info - + INTEGER, INTENT(IN) :: fields END FUNCTION h5oget_info_by_idx_c END INTERFACE + fields_c = H5O_INFO_ALL_F + IF(PRESENT(fields)) fields_c = fields + namelen = LEN(group_name) lapl_id_default = H5P_DEFAULT_F IF(PRESENT(lapl_id)) lapl_id_default = lapl_id ptr = C_LOC(object_info) - hdferr = H5Oget_info_by_idx_c(loc_id, group_name, namelen, index_field, order, n, lapl_id_default, ptr) + hdferr = H5Oget_info_by_idx_c(loc_id, group_name, namelen, index_field, order, n, lapl_id_default, ptr, fields_c) END SUBROUTINE H5Oget_info_by_idx_f @@ -1086,6 +1148,7 @@ CONTAINS ! ! Optional parameters: ! lapl_id - Link access property list identifier. +! fields - Flags specifying the fields to include in object_info. ! ! AUTHOR ! M. Scot Breitenfeld @@ -1093,7 +1156,7 @@ CONTAINS ! ! Fortran2003 Interface: SUBROUTINE h5ovisit_by_name_f(loc_id, object_name, index_type, order, op, op_data, & - return_value, hdferr, lapl_id) + return_value, hdferr, lapl_id, fields) IMPLICIT NONE INTEGER(HID_T) , INTENT(IN) :: loc_id CHARACTER(LEN=*), INTENT(IN) :: object_name @@ -1105,14 +1168,16 @@ CONTAINS INTEGER , INTENT(OUT) :: return_value INTEGER , INTENT(OUT) :: hdferr INTEGER(HID_T) , INTENT(IN) , OPTIONAL :: lapl_id + INTEGER , INTENT(IN) , OPTIONAL :: fields !***** INTEGER(SIZE_T) :: namelen INTEGER(HID_T) :: lapl_id_default + INTEGER :: fields_c INTERFACE INTEGER FUNCTION h5ovisit_by_name_c(loc_id, object_name, namelen, index_type, order, & - op, op_data, lapl_id) BIND(C, NAME='h5ovisit_by_name_c') + op, op_data, lapl_id, fields) BIND(C, NAME='h5ovisit_by_name_c') IMPORT :: C_CHAR, C_PTR, C_FUNPTR IMPORT :: HID_T, SIZE_T IMPLICIT NONE @@ -1124,16 +1189,20 @@ CONTAINS TYPE(C_FUNPTR) , VALUE :: op TYPE(C_PTR) , VALUE :: op_data INTEGER(HID_T) , INTENT(IN) :: lapl_id + INTEGER , INTENT(IN) :: fields END FUNCTION h5ovisit_by_name_c END INTERFACE + fields_c = H5O_INFO_ALL_F + IF(PRESENT(fields)) fields_c = fields + namelen = LEN(object_name) lapl_id_default = H5P_DEFAULT_F IF(PRESENT(lapl_id)) lapl_id_default = lapl_id return_value = h5ovisit_by_name_c(loc_id, object_name, namelen, index_type, order, & - op, op_data, lapl_id_default) + op, op_data, lapl_id_default, fields_c) IF(return_value.GE.0)THEN hdferr = 0 diff --git a/fortran/src/H5_f.c b/fortran/src/H5_f.c index 352ffab..69ba8b3 100644 --- a/fortran/src/H5_f.c +++ b/fortran/src/H5_f.c @@ -556,6 +556,17 @@ h5init_flags_c( int_f *h5d_flags, size_t_f *h5d_size_flags, h5o_flags[24] = (int_f)H5O_TYPE_DATASET; /* Object is a dataset */ h5o_flags[25] = (int_f)H5O_TYPE_NAMED_DATATYPE; /* Object is a named data type */ h5o_flags[26] = (int_f)H5O_TYPE_NTYPES; /* Number of different object types */ + +/* Flags for H5Oget_info. + * These flags determine which fields will be filled in in the H5O_info_t + * struct. + */ + h5o_flags[27] = (int_f)H5O_INFO_ALL; /* (H5O_INFO_BASIC|H5O_INFO_TIME|H5O_INFO_NUM_ATTRS|H5O_INFO_HDR|H5O_INFO_META_SIZE) */ + h5o_flags[28] = (int_f)H5O_INFO_BASIC; /* Fill in the fileno, addr, type, and rc fields */ + h5o_flags[29] = (int_f)H5O_INFO_TIME; /* Fill in the atime, mtime, ctime, and btime fields */ + h5o_flags[30] = (int_f)H5O_INFO_NUM_ATTRS; /* Fill in the num_attrs field */ + h5o_flags[31] = (int_f)H5O_INFO_HDR; /* Fill in the hdr field */ + h5o_flags[32] = (int_f)H5O_INFO_META_SIZE; /* Fill in the meta_size field */ /* * H5P flags */ diff --git a/fortran/src/H5_ff.F90 b/fortran/src/H5_ff.F90 index f63e734..84529e4 100644 --- a/fortran/src/H5_ff.F90 +++ b/fortran/src/H5_ff.F90 @@ -98,8 +98,8 @@ MODULE H5LIB ! ! H5O flags declaration ! - INTEGER, PARAMETER :: H5O_FLAGS_LEN = 27 - INTEGER, DIMENSION(1:H5O_FLAGS_LEN) :: H5o_flags + INTEGER, PARAMETER :: H5O_FLAGS_LEN = 33 + INTEGER, DIMENSION(1:H5O_FLAGS_LEN) :: H5O_flags ! ! H5P flags declaration ! @@ -139,8 +139,8 @@ MODULE H5LIB ! INTEGER, PARAMETER :: H5LIB_FLAGS_LEN = 2 INTEGER, DIMENSION(1:H5LIB_FLAGS_LEN) :: H5LIB_flags - - PUBLIC :: h5open_f, h5close_f, h5get_libversion_f, h5dont_atexit_f, h5kind_to_type, h5offsetof + + PUBLIC :: h5open_f, h5close_f, h5get_libversion_f, h5dont_atexit_f, h5kind_to_type, h5offsetof, h5gmtime PUBLIC :: h5garbage_collect_f, h5check_version_f CONTAINS @@ -488,7 +488,13 @@ CONTAINS H5O_TYPE_GROUP_F = h5o_flags(24) H5O_TYPE_DATASET_F = h5o_flags(25) H5O_TYPE_NAMED_DATATYPE_F = h5o_flags(26) - H5O_TYPE_NTYPES_F = h5o_flags(27) + H5O_TYPE_NTYPES_F = h5o_flags(27) + H5O_INFO_ALL_F = h5o_flags(28) + H5O_INFO_BASIC_F = h5o_flags(29) + H5O_INFO_TIME_F = h5o_flags(30) + H5O_INFO_NUM_ATTRS_F = h5o_flags(31) + H5O_INFO_HDR_F = h5o_flags(32) + H5O_INFO_META_SIZE_F = h5o_flags(33) ! ! H5P flags ! @@ -898,4 +904,62 @@ CONTAINS END FUNCTION h5offsetof +!****f* H5LIB_PROVISIONAL/h5gmtime +! +! NAME +! h5gmtime +! +! PURPOSE +! Convert time_t structure (C) to Fortran DATE AND TIME storage format. +! +! Inputs: +! stdtime_t - Object of type time_t that contains a time value +! +! Outputs: +! datetime - A date/time array using Fortran conventions: +! datetime(1) = year +! datetime(2) = month +! datetime(3) = day +! datetime(4) = 0 ! time is expressed as UTC (or GMT timezone) */ +! datetime(5) = hour +! datetime(6) = minute +! datetime(7) = second +! datetime(8) = millisecond -- not available, assigned - HUGE(0) +! +! AUTHOR +! M. Scot Breitenfeld +! January, 2019 +! +! Fortran Interface: + FUNCTION h5gmtime(stdtime_t) + IMPLICIT NONE + INTEGER(KIND=TIME_T), INTENT(IN) :: stdtime_t + INTEGER, DIMENSION(1:8) :: h5gmtime +!***** + TYPE(C_PTR) :: cptr + INTEGER(C_INT), DIMENSION(:), POINTER :: c_time + + INTERFACE + FUNCTION gmtime(stdtime_t) BIND(C, NAME='gmtime') + IMPORT :: TIME_T, C_PTR + IMPLICIT NONE + INTEGER(KIND=TIME_T) :: stdtime_t + TYPE(C_PTR) :: gmtime + END FUNCTION gmtime + END INTERFACE + + cptr = gmtime(stdtime_t) + CALL C_F_POINTER(cptr, c_time, [9]) + + h5gmtime(1) = INT(c_time(6)+1900) ! year starts at 1900 + h5gmtime(2) = INT(c_time(5)+1) ! month starts at 0 in C + h5gmtime(3) = INT(c_time(4)) ! day + h5gmtime(4) = 0 ! time is expressed as UTC (or GMT timezone) + h5gmtime(5) = INT(c_time(3)) ! hour + h5gmtime(6) = INT(c_time(2)) ! minute + h5gmtime(7) = INT(c_time(1)) ! second + h5gmtime(8) = -32767 ! millisecond is not available, assign it -HUGE(0) + + END FUNCTION h5gmtime + END MODULE H5LIB diff --git a/fortran/src/H5f90global.F90 b/fortran/src/H5f90global.F90 index 078778a..b705cc1 100644 --- a/fortran/src/H5f90global.F90 +++ b/fortran/src/H5f90global.F90 @@ -493,6 +493,13 @@ MODULE H5GLOBAL !DEC$ATTRIBUTES DLLEXPORT :: H5O_TYPE_DATASET_F !DEC$ATTRIBUTES DLLEXPORT :: H5O_TYPE_NAMED_DATATYPE_F !DEC$ATTRIBUTES DLLEXPORT :: H5O_TYPE_NTYPES_F + !DEC$ATTRIBUTES DLLEXPORT :: H5O_INFO_ALL_F + !DEC$ATTRIBUTES DLLEXPORT :: H5O_INFO_BASIC_F + !DEC$ATTRIBUTES DLLEXPORT :: H5O_INFO_TIME_F + !DEC$ATTRIBUTES DLLEXPORT :: H5O_INFO_NUM_ATTRS_F + !DEC$ATTRIBUTES DLLEXPORT :: H5O_INFO_HDR_F + !DEC$ATTRIBUTES DLLEXPORT :: H5O_INFO_META_SIZE_F + ! !DEC$endif INTEGER :: H5O_COPY_SHALLOW_HIERARCHY_F ! *** THESE VARIABLES DO @@ -522,6 +529,12 @@ MODULE H5GLOBAL INTEGER :: H5O_TYPE_DATASET_F INTEGER :: H5O_TYPE_NAMED_DATATYPE_F INTEGER :: H5O_TYPE_NTYPES_F + INTEGER :: H5O_INFO_ALL_F + INTEGER :: H5O_INFO_BASIC_F + INTEGER :: H5O_INFO_TIME_F + INTEGER :: H5O_INFO_NUM_ATTRS_F + INTEGER :: H5O_INFO_HDR_F + INTEGER :: H5O_INFO_META_SIZE_F ! ! H5P flags declaration ! diff --git a/fortran/src/H5f90proto.h b/fortran/src/H5f90proto.h index 46ef8ef..fc6567c 100644 --- a/fortran/src/H5f90proto.h +++ b/fortran/src/H5f90proto.h @@ -315,14 +315,14 @@ H5_FCDLL int_f h5oclose_c(hid_t_f *object_id ); H5_FCDLL int_f h5oopen_by_addr_c(hid_t_f *loc_id, haddr_t_f *addr, hid_t_f *obj_id); H5_FCDLL int_f h5olink_c(hid_t_f *object_id, hid_t_f *new_loc_id, _fcd name, size_t_f *namelen, hid_t_f *lcpl_id, hid_t_f *lapl_id); -H5_FCDLL int_f h5ovisit_c(hid_t_f *group_id, int_f *index_type, int_f *order, H5O_iterate_t op, void *op_data); +H5_FCDLL int_f h5ovisit_c(hid_t_f *group_id, int_f *index_type, int_f *order, H5O_iterate_t op, void *op_data, int_f *fields); H5_FCDLL int_f h5ovisit_by_name_c(hid_t_f *loc_id, _fcd object_name, size_t_f *namelen, int_f *index_type, int_f *order, - H5O_iterate_t op, void *op_data, hid_t_f *lapl_id ); -H5_FCDLL int_f h5oget_info_c(hid_t_f *object_id, H5O_info_t_f *object_info); + H5O_iterate_t op, void *op_data, hid_t_f *lapl_id, int_f *fields ); +H5_FCDLL int_f h5oget_info_c(hid_t_f *object_id, H5O_info_t_f *object_info, int_f *fields); H5_FCDLL int_f h5oget_info_by_idx_c(hid_t_f *loc_id, _fcd group_name, size_t_f *namelen, - int_f *index_field, int_f *order, hsize_t_f *n, hid_t_f *lapl_id, H5O_info_t_f *object_info); + int_f *index_field, int_f *order, hsize_t_f *n, hid_t_f *lapl_id, H5O_info_t_f *object_info, int_f *fields); H5_FCDLL int_f h5oget_info_by_name_c(hid_t_f *loc_id, _fcd name, size_t_f *namelen, hid_t_f *lapl_id, - H5O_info_t_f *object_info); + H5O_info_t_f *object_info, int_f *fields); H5_FCDLL int_f h5ocopy_c(hid_t_f *src_loc_id, _fcd src_name, size_t_f *src_name_len, hid_t_f *dst_loc_id, _fcd dst_name, size_t_f *dst_name_len, hid_t_f *ocpypl_id, hid_t_f *lcpl_id ); diff --git a/fortran/src/H5match_types.c b/fortran/src/H5match_types.c index 7e0b7e8..57f7dda 100644 --- a/fortran/src/H5match_types.c +++ b/fortran/src/H5match_types.c @@ -191,6 +191,8 @@ int main(void) } if(sizeof(size_t) == IntKinds_SizeOf[i]) writeTypedef("size_t", "size_t", IntKinds[i]); + if(sizeof(time_t) == IntKinds_SizeOf[i]) + writeTypedef("time_t", "time_t", IntKinds[i]); if(sizeof(hsize_t) == IntKinds_SizeOf[i]) writeTypedef("hsize_t", "hsize_t", IntKinds[i]); } @@ -306,6 +308,17 @@ int main(void) return -1; } + /* time_t */ + for(i=0;i< FORTRAN_NUM_INTEGER_KINDS;i++) { + if(IntKinds_SizeOf[i] == H5_SIZEOF_TIME_T) { + writeToFiles("time_t","TIME_T", "time_t_f", IntKinds[i]); + break; + } + if(i == (FORTRAN_NUM_INTEGER_KINDS-1) ) + /* Error: couldn't find a size for time_t */ + return -1; + } + /* int */ writeToFiles("int","Fortran_INTEGER", "int_f", H5_FORTRAN_NATIVE_INTEGER_KIND); diff --git a/fortran/src/hdf5_fortrandll.def.in b/fortran/src/hdf5_fortrandll.def.in index f36a5c8..b9e2314 100644 --- a/fortran/src/hdf5_fortrandll.def.in +++ b/fortran/src/hdf5_fortrandll.def.in @@ -8,6 +8,7 @@ H5LIB_mp_H5GARBAGE_COLLECT_F H5LIB_mp_H5DONT_ATEXIT_F H5LIB_mp_H5KIND_TO_TYPE H5LIB_mp_H5OFFSETOF +H5LIB_mp_H5GMTIME ; H5A H5A_mp_H5AWRITE_CHAR_SCALAR H5A_mp_H5AREAD_CHAR_SCALAR diff --git a/fortran/test/tH5O_F03.F90 b/fortran/test/tH5O_F03.F90 index 44c4bff..d1a9ddb 100644 --- a/fortran/test/tH5O_F03.F90 +++ b/fortran/test/tH5O_F03.F90 @@ -58,21 +58,224 @@ MODULE visit_cb TYPE, bind(c) :: ovisit_ud_t INTEGER :: idx ! Index in object visit structure TYPE(obj_visit_t), DIMENSION(1:info_size) :: info ! Pointer to the object visit structure to use + INTEGER :: field END TYPE ovisit_ud_t CONTAINS - INTEGER FUNCTION visit_obj_cb( group_id, name, oinfo, op_data) bind(C) +! Compares the field values of a C h5O_info_t and a Fortran H5O_info_t. + + INTEGER FUNCTION compare_h5o_info_t( oinfo_f, oinfo_c, field, full_f_field ) RESULT(status) + + IMPLICIT NONE + TYPE(h5o_info_t) :: oinfo_f + TYPE(c_h5o_info_t) :: oinfo_c + INTEGER :: field + LOGICAL :: full_f_field ! All the fields of Fortran H5O_info_t where filled +! local + INTEGER(C_INT), DIMENSION(1:8) :: atime, btime, ctime, mtime + INTEGER :: i + + status = 0 + + IF( (field .EQ. H5O_INFO_BASIC_F).OR.(field .EQ. H5O_INFO_ALL_F) )THEN + IF( (oinfo_f%fileno.LE.0) .OR. (oinfo_c%fileno .NE. oinfo_f%fileno) )THEN + status = -1 + RETURN + ENDIF + IF( (oinfo_f%addr.LE.0) .OR. (oinfo_c%addr .NE. oinfo_f%addr) )THEN + status = -1 + RETURN + ENDIF + IF( (oinfo_f%type.LT.0) .OR. (oinfo_c%type .NE. oinfo_f%type) )THEN + status = -1 + RETURN + ENDIF + IF( (oinfo_f%rc.LT.0) .OR. (oinfo_c%rc .NE. oinfo_f%rc) )THEN + status = -1 + RETURN + ENDIF + + ENDIF + + IF((field .EQ. H5O_INFO_TIME_F).OR.(field .EQ. H5O_INFO_ALL_F))THEN + + atime(1:8) = h5gmtime(oinfo_c%atime) + btime(1:8) = h5gmtime(oinfo_c%btime) + ctime(1:8) = h5gmtime(oinfo_c%ctime) + mtime(1:8) = h5gmtime(oinfo_c%mtime) + + DO i = 1, 8 + IF( (atime(i) .NE. oinfo_f%atime(i)) )THEN + status = -1 + RETURN + ENDIF + + IF( (btime(i) .NE. oinfo_f%btime(i)) )THEN + status = -1 + RETURN + ENDIF + + IF( (ctime(i) .NE. oinfo_f%ctime(i)) )THEN + status = -1 + RETURN + ENDIF + + IF( (mtime(i) .NE. oinfo_f%mtime(i)) )THEN + status = -1 + RETURN + ENDIF + ENDDO + + ELSE IF(field .EQ. H5O_INFO_TIME_F.AND. full_f_field)THEN + ! check other field values are not filled (using only a small subset to check) + status = 0 + IF( oinfo_c%fileno .NE. oinfo_f%fileno) status = status + 1 + IF( oinfo_c%addr .NE. oinfo_f%addr) status = status + 1 + IF( oinfo_c%type .NE. oinfo_f%type) status = status + 1 + IF( oinfo_c%rc .NE. oinfo_f%rc) status = status + 1 + IF(status.EQ.0) THEN ! There was no difference found, which is only possible if the field was filled. + status = -1 + RETURN + ENDIF + status = 0 ! reset + ENDIF + + IF((field .EQ. H5O_INFO_NUM_ATTRS_F).OR.(field .EQ. H5O_INFO_ALL_F))THEN + IF( (oinfo_f%num_attrs.LT.0) .OR. (oinfo_c%num_attrs .NE. oinfo_f%num_attrs) )THEN + status = -1 + RETURN + ENDIF + ELSE IF( field .EQ. H5O_INFO_ALL_F.AND.full_f_field)THEN + ! check other field values are not filled (using only a small subset to check) + status = 0 + IF( oinfo_c%fileno .NE. oinfo_f%fileno) status = status + 1 + IF( oinfo_c%addr .NE. oinfo_f%addr) status = status + 1 + IF( oinfo_c%type .NE. oinfo_f%type) status = status + 1 + IF( oinfo_c%rc .NE. oinfo_f%rc) status = status + 1 + IF(status.EQ.0) THEN ! There was no difference found, which is only possible if the field was filled. + status = -1 + RETURN + ENDIF + status = 0 ! reset + + ENDIF + + IF((field).EQ.H5O_INFO_HDR_F.OR.(field .EQ. H5O_INFO_ALL_F))THEN + IF( (oinfo_f%hdr%version.LT.0) .OR. (oinfo_c%hdr%version .NE. oinfo_f%hdr%version) )THEN + status = -1 + RETURN + ENDIF + IF( (oinfo_f%hdr%nmesgs.LT.0) .OR. (oinfo_c%hdr%nmesgs .NE. oinfo_f%hdr%nmesgs) )THEN + status = -1 + RETURN + ENDIF + IF( (oinfo_f%hdr%nchunks.LT.0) .OR. (oinfo_c%hdr%nchunks .NE. oinfo_f%hdr%nchunks) )THEN + status = -1 + RETURN + ENDIF + IF( (oinfo_f%hdr%flags.LT.0) .OR. (oinfo_c%hdr%flags .NE. oinfo_f%hdr%flags) )THEN + status = -1 + RETURN + ENDIF + IF( (oinfo_f%hdr%space%total.LT.0) .OR. (oinfo_c%hdr%space%total .NE. oinfo_f%hdr%space%total) )THEN + status = -1 + RETURN + ENDIF + IF( (oinfo_f%hdr%space%meta.LT.0) .OR. (oinfo_c%hdr%space%meta .NE. oinfo_f%hdr%space%meta) )THEN + status = -1 + RETURN + ENDIF + IF( (oinfo_f%hdr%space%mesg.LT.0) .OR. (oinfo_c%hdr%space%mesg .NE. oinfo_f%hdr%space%mesg) )THEN + status = -1 + RETURN + ENDIF + IF( (oinfo_f%hdr%space%free.LT.0) .OR. (oinfo_c%hdr%space%free .NE. oinfo_f%hdr%space%free) )THEN + status = -1 + RETURN + ENDIF + IF( (oinfo_f%hdr%mesg%present.LT.0) .OR. (oinfo_c%hdr%mesg%present .NE. oinfo_f%hdr%mesg%present) )THEN + status = -1 + RETURN + ENDIF + IF( (oinfo_f%hdr%mesg%shared.LT.0) .OR. (oinfo_c%hdr%mesg%shared .NE. oinfo_f%hdr%mesg%shared) )THEN + status = -1 + RETURN + ENDIF + ELSE IF( field .EQ. H5O_INFO_HDR_F.AND.full_f_field)THEN + ! check other field values are not filled (using only a small subset to check) + status = 0 + IF( oinfo_c%fileno .NE. oinfo_f%fileno) status = status + 1 + IF( oinfo_c%addr .NE. oinfo_f%addr) status = status + 1 + IF( oinfo_c%type .NE. oinfo_f%type) status = status + 1 + IF( oinfo_c%rc .NE. oinfo_f%rc) status = status + 1 + IF(status.EQ.0) THEN ! There was no difference found, which is only possible if the field was filled. + status = -1 + RETURN + ENDIF + status = 0 ! reset + ENDIF + IF((field).EQ.H5O_INFO_META_SIZE_F.OR.(field .EQ. H5O_INFO_ALL_F))THEN + IF((oinfo_f%meta_size%obj%index_size.LT.0).OR.(oinfo_c%meta_size%obj%index_size.NE.oinfo_f%meta_size%obj%index_size))THEN + status = -1 + RETURN + ENDIF + IF((oinfo_f%meta_size%obj%heap_size.LT.0).OR.(oinfo_c%meta_size%obj%heap_size.NE.oinfo_f%meta_size%obj%heap_size))THEN + status = -1 + RETURN + ENDIF + IF((oinfo_f%meta_size%attr%index_size.LT.0).OR.(oinfo_c%meta_size%attr%index_size.NE.oinfo_f%meta_size%attr%index_size))THEN + status = -1 + RETURN + ENDIF + IF((oinfo_f%meta_size%attr%heap_size.LT.0).OR.(oinfo_c%meta_size%attr%heap_size.NE.oinfo_f%meta_size%attr%heap_size))THEN + status = -1 + RETURN + ENDIF + ELSE IF( field .EQ. H5O_INFO_META_SIZE_F.AND.full_f_field)THEN + ! check other field values are not filled (using only a small subset to check) + status = 0 + IF( oinfo_c%fileno .NE. oinfo_f%fileno) status = status + 1 + IF( oinfo_c%addr .NE. oinfo_f%addr) status = status + 1 + IF( oinfo_c%type .NE. oinfo_f%type) status = status + 1 + IF( oinfo_c%rc .NE. oinfo_f%rc) status = status + 1 + IF(status.EQ.0) THEN ! There was no difference found, which is only possible if the field was filled. + status = -1 + RETURN + ENDIF + status = 0 ! reset + ENDIF + + END FUNCTION compare_h5o_info_t + + INTEGER FUNCTION visit_obj_cb( group_id, name, oinfo_c, op_data) bind(C) IMPLICIT NONE INTEGER(HID_T), VALUE :: group_id CHARACTER(LEN=1), DIMENSION(1:180) :: name - TYPE(h5o_info_t) :: oinfo + CHARACTER(LEN=180) :: name2 + TYPE(c_h5o_info_t) :: oinfo_c TYPE(ovisit_ud_t) :: op_data - + TYPE(h5o_info_t) :: oinfo_f +! +! MEMBER | TYPE | MEANING | RANGE +! A(1) = tm_sec int seconds after the minute 0-61* +! A(2) = tm_min int minutes after the hour 0-59 +! A(3) = tm_hour int hours since midnight 0-23 +! A(4) = tm_mday int day of the month 1-31 +! A(5) = tm_mon int months since January 0-11 +! A(6) = tm_year int years since 1900 +! A(7) = tm_wday int days since Sunday 0-6 +! A(8) = tm_yday int days since January 1 0-365 +! A(9) = tm_isdst int Daylight Saving Time flag +! + INTEGER(C_INT), DIMENSION(:), POINTER :: c_atime, c_btime, c_ctime, c_mtime + INTEGER(C_INT), DIMENSION(1:8) :: atime, btime, ctime, mtime INTEGER :: len, i INTEGER :: idx + INTEGER :: ierr + TYPE(C_PTR) :: cptr visit_obj_cb = 0 @@ -87,21 +290,53 @@ CONTAINS len = len - 1 ! Check for correct object information + name2(1:180) = "" + DO i = 1, len + name2(i:i) = name(i)(1:1) + ENDDO - idx = op_data%idx + IF(op_data%field .EQ. H5O_INFO_ALL_F)THEN - DO i = 1, len - IF(op_data%info(idx)%path(i)(1:1) .NE. name(i)(1:1))THEN - visit_obj_cb = -1 - RETURN - ENDIF - - IF(op_data%info(idx)%type_obj .NE. oinfo%type)THEN - visit_obj_cb = -1 - RETURN - ENDIF + idx = op_data%idx + + DO i = 1, len + IF(op_data%info(idx)%path(i)(1:1) .NE. name(i)(1:1))THEN + visit_obj_cb = -1 + RETURN + ENDIF + + IF(op_data%info(idx)%type_obj .NE. oinfo_c%type)THEN + visit_obj_cb = -1 + RETURN + ENDIF + ENDDO - ENDDO + ENDIF + + ! Check H5Oget_info_by_name_f; if partial field values where filled correctly + CALL H5Oget_info_by_name_f(group_id, name2, oinfo_f, ierr); + visit_obj_cb = compare_h5o_info_t( oinfo_f, oinfo_c, op_data%field, .TRUE. ) + IF(visit_obj_cb.EQ.-1) RETURN + + ! Check H5Oget_info_by_name_f, only check field values + CALL H5Oget_info_by_name_f(group_id, name2, oinfo_f, ierr, fields = op_data%field); + visit_obj_cb = compare_h5o_info_t( oinfo_f, oinfo_c, op_data%field, .FALSE. ) + IF(visit_obj_cb.EQ.-1) RETURN + + + IF(op_data%idx.EQ.1)THEN + + ! Check H5Oget_info_f, only check field values + CALL H5Oget_info_f(group_id, oinfo_f, ierr, fields = op_data%field); + visit_obj_cb = compare_h5o_info_t( oinfo_f, oinfo_c, op_data%field, .FALSE. ) + IF(visit_obj_cb.EQ.-1) RETURN + + ! Check H5Oget_info_f; if partial field values where filled correctly + CALL H5Oget_info_f(group_id, oinfo_f, ierr); + visit_obj_cb = compare_h5o_info_t( oinfo_f, oinfo_c, op_data%field, .TRUE. ) + IF(visit_obj_cb.EQ.-1) RETURN + + ENDIF ! Advance to next location in expected output op_data%idx = op_data%idx + 1 @@ -110,7 +345,6 @@ CONTAINS END MODULE visit_cb - MODULE TH5O_F03 CONTAINS @@ -310,29 +544,110 @@ SUBROUTINE obj_visit(total_error) udata%info(9)%type_obj = H5O_TYPE_NAMED_DATATYPE_F ! Visit all the objects reachable from the root group (with file ID) - udata%idx = 1 fun_ptr = C_FUNLOC(visit_obj_cb) f_ptr = C_LOC(udata) ! Test h5ovisit_f + udata%field = H5O_INFO_ALL_F + udata%idx = 1 CALL h5ovisit_f(fid, H5_INDEX_NAME_F, H5_ITER_INC_F, fun_ptr, f_ptr, ret_val, error) CALL check("h5ovisit_f", error, total_error) IF(ret_val.LT.0)THEN CALL check("h5ovisit_f", -1, total_error) ENDIF - ! Test h5ovisit_by_name_f + ! Test fields option + udata%field = H5O_INFO_ALL_F + udata%idx = 1 + CALL h5ovisit_f(fid, H5_INDEX_NAME_F, H5_ITER_INC_F, fun_ptr, f_ptr, ret_val, error, fields=udata%field) + CALL check("h5ovisit_f", error, total_error) + IF(ret_val.LT.0)THEN + CALL check("h5ovisit_f", -1, total_error) + ENDIF + udata%field = H5O_INFO_BASIC_F + udata%idx = 1 + CALL h5ovisit_f(fid, H5_INDEX_NAME_F, H5_ITER_INC_F, fun_ptr, f_ptr, ret_val, error, fields=udata%field) + CALL check("h5ovisit_f", error, total_error) + IF(ret_val.LT.0)THEN + CALL check("h5ovisit_f", -1, total_error) + ENDIF + udata%field = H5O_INFO_TIME_F + udata%idx = 1 + CALL h5ovisit_f(fid, H5_INDEX_NAME_F, H5_ITER_INC_F, fun_ptr, f_ptr, ret_val, error, fields=udata%field) + CALL check("h5ovisit_f", error, total_error) + IF(ret_val.LT.0)THEN + CALL check("h5ovisit_f", -1, total_error) + ENDIF + udata%field = H5O_INFO_NUM_ATTRS_F + udata%idx = 1 + CALL h5ovisit_f(fid, H5_INDEX_NAME_F, H5_ITER_INC_F, fun_ptr, f_ptr, ret_val, error, fields=udata%field) + CALL check("h5ovisit_f", error, total_error) + IF(ret_val.LT.0)THEN + CALL check("h5ovisit_f", -1, total_error) + ENDIF + udata%field = H5O_INFO_HDR_F + udata%idx = 1 + CALL h5ovisit_f(fid, H5_INDEX_NAME_F, H5_ITER_INC_F, fun_ptr, f_ptr, ret_val, error, fields=udata%field) + CALL check("h5ovisit_f", error, total_error) + IF(ret_val.LT.0)THEN + CALL check("h5ovisit_f", -1, total_error) + ENDIF + udata%field = H5O_INFO_META_SIZE_F + udata%idx = 1 + CALL h5ovisit_f(fid, H5_INDEX_NAME_F, H5_ITER_INC_F, fun_ptr, f_ptr, ret_val, error, fields=udata%field) + CALL check("h5ovisit_f", error, total_error) + IF(ret_val.LT.0)THEN + CALL check("h5ovisit_f", -1, total_error) + ENDIF + ! Test h5ovisit_by_name_f object_name = "/" udata%idx = 1 - - CALL h5ovisit_by_name_f(fid, object_name, H5_INDEX_NAME_F, H5_ITER_INC_F, fun_ptr, f_ptr, ret_val, error) + udata%field = H5O_INFO_ALL_F + CALL h5ovisit_by_name_f(fid, object_name, H5_INDEX_NAME_F, H5_ITER_INC_F, fun_ptr, f_ptr, ret_val, error, fields=udata%field) CALL check("h5ovisit_by_name_f", error, total_error) IF(ret_val.LT.0)THEN CALL check("h5ovisit_by_name_f", -1, total_error) ENDIF + ! Test fields option + udata%idx = 1 + udata%field = H5O_INFO_BASIC_F + CALL h5ovisit_by_name_f(fid, object_name, H5_INDEX_NAME_F, H5_ITER_INC_F, fun_ptr, f_ptr, ret_val, error, fields=udata%field) + CALL check("h5ovisit_by_name_f", error, total_error) + IF(ret_val.LT.0)THEN + CALL check("h5ovisit_by_name_f", -1, total_error) + ENDIF + udata%idx = 1 + udata%field = H5O_INFO_TIME_F + CALL h5ovisit_by_name_f(fid, object_name, H5_INDEX_NAME_F, H5_ITER_INC_F, fun_ptr, f_ptr, ret_val, error, fields=udata%field) + CALL check("h5ovisit_by_name_f", error, total_error) + IF(ret_val.LT.0)THEN + CALL check("h5ovisit_by_name_f", -1, total_error) + ENDIF + udata%idx = 1 + udata%field = H5O_INFO_NUM_ATTRS_F + CALL h5ovisit_by_name_f(fid, object_name, H5_INDEX_NAME_F, H5_ITER_INC_F, fun_ptr, f_ptr, ret_val, error, fields=udata%field) + CALL check("h5ovisit_by_name_f", error, total_error) + IF(ret_val.LT.0)THEN + CALL check("h5ovisit_by_name_f", -1, total_error) + ENDIF + udata%idx = 1 + udata%field = H5O_INFO_HDR_F + CALL h5ovisit_by_name_f(fid, object_name, H5_INDEX_NAME_F, H5_ITER_INC_F, fun_ptr, f_ptr, ret_val, error, fields=udata%field) + CALL check("h5ovisit_by_name_f", error, total_error) + IF(ret_val.LT.0)THEN + CALL check("h5ovisit_by_name_f", -1, total_error) + ENDIF + udata%idx = 1 + udata%field = H5O_INFO_META_SIZE_F + CALL h5ovisit_by_name_f(fid, object_name, H5_INDEX_NAME_F, H5_ITER_INC_F, fun_ptr, f_ptr, ret_val, error, fields=udata%field) + CALL check("h5ovisit_by_name_f", error, total_error) + IF(ret_val.LT.0)THEN + CALL check("h5ovisit_by_name_f", -1, total_error) + ENDIF + CALL h5fclose_f(fid, error) CALL check("h5fclose_f",error, total_error) @@ -450,11 +765,32 @@ SUBROUTINE obj_info(total_error) IF(oinfo%rc.NE.1)THEN CALL check("h5oget_info_by_idx_f", -1, total_error) ENDIF + IF(oinfo%type.NE.H5O_TYPE_DATASET_F)THEN + CALL check("h5oget_info_by_idx_f", -1, total_error) + ENDIF + ! Check partial fields + CALL h5oget_info_by_idx_f(gid, ".", H5_INDEX_NAME_F, H5_ITER_INC_F, 0_hsize_t, oinfo, error, fields=H5O_INFO_BASIC_F ) + CALL check("h5oget_info_by_idx_f", error, total_error) + + IF(oinfo%rc.NE.1)THEN + CALL check("h5oget_info_by_idx_f", -1, total_error) + ENDIF IF(oinfo%type.NE.H5O_TYPE_DATASET_F)THEN CALL check("h5oget_info_by_idx_f", -1, total_error) ENDIF + CALL h5oget_info_by_idx_f(gid, ".", H5_INDEX_NAME_F, H5_ITER_INC_F, 0_hsize_t, oinfo, error, fields=H5O_INFO_TIME_F ) + CALL check("h5oget_info_by_idx_f", error, total_error) + ! These field values should not be filled + IF(oinfo%rc.EQ.1)THEN + CALL check("h5oget_info_by_idx_f", -1, total_error) + ENDIF + IF(oinfo%type.EQ.H5O_TYPE_DATASET_F)THEN + CALL check("h5oget_info_by_idx_f", -1, total_error) + ENDIF + + ! Close objects CALL h5dclose_f(did, error) CALL check("h5dclose_f", error, total_error) @@ -483,11 +819,12 @@ SUBROUTINE build_visit_file(fid) USE TH5_MISC IMPLICIT NONE - INTEGER(hid_t) :: fid ! File ID - INTEGER(hid_t) :: gid = -1, gid2 = -1 ! Group IDs + INTEGER(hid_t) :: fid ! File ID + INTEGER(hid_t) :: gid = -1, gid2 = -1 ! Group IDs INTEGER(hid_t) :: sid = -1 ! Dataspace ID INTEGER(hid_t) :: did = -1 ! Dataset ID INTEGER(hid_t) :: tid = -1 ! Datatype ID + INTEGER(hid_t) :: aid = -1, aid2 = -1, aid3 = -1 ! Attribute ID CHARACTER(LEN=20) :: filename = 'visit.h5' INTEGER :: error @@ -500,6 +837,15 @@ SUBROUTINE build_visit_file(fid) ! Create nested group CALL H5Gcreate_f(gid, "Group2", gid2, error) + CALL H5Screate_f(H5S_SCALAR_F, sid, error) + CALL H5Acreate_f(gid2, "Attr1", H5T_NATIVE_INTEGER, sid, aid, error) + CALL H5Acreate_f(gid2, "Attr2", H5T_NATIVE_INTEGER, sid, aid2, error) + CALL H5Acreate_f(gid2, "Attr3", H5T_NATIVE_INTEGER, sid, aid3, error) + CALL H5Aclose_f(aid,error) + CALL H5Aclose_f(aid2,error) + CALL H5Aclose_f(aid3,error) + CALL H5Sclose_f(sid,error) + ! Close groups CALL h5gclose_f(gid2, error) CALL h5gclose_f(gid, error) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index d33b7eb..06c7c6a 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -194,6 +194,14 @@ New Features (DER - 2018/12/30, HDFFV-10566) + - Changed the default behavior in parallel when reading the same dataset in its entirely + (i.e. H5S_ALL dataset selection) which is being read by all the processes collectively. + The dataset mush be contiguous, less than 2GB, and of an atomic datatype. + The new behavior is the HDF5 library will use an MPI_Bcast to pass the data read from + the disk by the root process to the remain processes in the MPI communicator associated + with the HDF5 file. + + (MSB - 2019/01/02, HDFFV-10652) Fortran Library: ---------------- @@ -209,6 +217,22 @@ New Features (DER - 2019/01/09, TRILAB-45) + - Added new Fortran derived type, c_h5o_info_t, which is interoperable with + C's h5o_info_t. This is needed for callback functions which + pass C's h5o_info_t data type definition. + + (MSB, 2019/01/08, HDFFV-10443) + + - Added new Fortran API, H5gmtime, which converts (C) 'time_t' structure + to Fortran DATE AND TIME storage format. + + (MSB, 2019/01/08, HDFFV-10443) + + - Added new Fortran 'fields' optional parameter to: h5ovisit_f, h5oget_info_by_name_f, + h5oget_info, h5oget_info_by_idx and h5ovisit_by_name_f. + + (MSB, 2019/01/08, HDFFV-10443) + C++ Library: ------------ - @@ -384,6 +408,284 @@ Bug Fixes since HDF5-1.10.3 release (MSB, 2018/12/04, HDFFV-10511) +<<<<<<< HEAD +======= + - Fixed issue with Fortran not returning h5o_info_t field values + meta_size%attr%index_size and meta_size%attr%heap_size. + + (MSB, 2018/1/8, HDFFV-10443) + + + Tools + ----- + - + + High-Level APIs: + ------ + - + + Fortran High-Level APIs: + ------ + - + + Documentation + ------------- + - + + F90 APIs + -------- + - + + C++ APIs + -------- + - + + Testing + ------- + +Bug Fixes since HDF5-1.10.2 release +================================== + + Library + ------- + - Java HDF5LibraryException class + + The error minor and major values would be lost after the + constructor executed. + + Created two local class variables to hold the values obtained during + execution of the constructor. Refactored the class functions to retrieve + the class values rather then calling the native functions. + The native functions were renamed and called only during execution + of the constructor. + Added error checking to calling class constructors in JNI classes. + + (ADB - 2018/08/06, HDFFV-10544) + + - Added checks of the defined MPI_VERSION to guard against usage of + MPI-3 functions in the Parallel Compression and "big Parallel I/O" + features when HDF5 is built with MPI-2. Previously, the configure + step would pass but the build itself would fail when it could not + locate the MPI-3 functions used. + + As a result of these new checks, HDF5 can again be built with MPI-2, + but the Parallel Compression feature will be disabled as it relies + on the MPI-3 functions used. + + (JTH - 2018/08/02, HDFFV-10512) + + - User's patches: CVEs + + The following patches have been applied: + + CVE-2018-11202 - NULL pointer dereference was discovered in + H5S_hyper_make_spans in H5Shyper.c (HDFFV-10476) + https://security-tracker.debian.org/tracker/CVE-2018-11202 + https://cve.mitre.org/cgi-bin/cvename.cgi?name=3DCVE-2018-11202 + + CVE-2018-11203 - A division by zero was discovered in + H5D__btree_decode_key in H5Dbtree.c (HDFFV-10477) + https://security-tracker.debian.org/tracker/CVE-2018-11203 + https://cve.mitre.org/cgi-bin/cvename.cgi?name=3DCVE-2018-11203 + + CVE-2018-11204 - A NULL pointer dereference was discovered in + H5O__chunk_deserialize in H5Ocache.c (HDFFV-10478) + https://security-tracker.debian.org/tracker/CVE-2018-11204 + https://cve.mitre.org/cgi-bin/cvename.cgi?name=3DCVE-2018-11204 + + CVE-2018-11206 - An out of bound read was discovered in + H5O_fill_new_decode and H5O_fill_old_decode in H5Ofill.c + (HDFFV-10480) + https://security-tracker.debian.org/tracker/CVE-2018-11206 + https://cve.mitre.org/cgi-bin/cvename.cgi?name=3DCVE-2018-11206 + + CVE-2018-11207 - A division by zero was discovered in + H5D__chunk_init in H5Dchunk.c (HDFFV-10481) + https://security-tracker.debian.org/tracker/CVE-2018-11207 + https://cve.mitre.org/cgi-bin/cvename.cgi?name=3DCVE-2018-11207 + + (BMR - 2018/7/22, PR#s: 1134 and 1139, + HDFFV-10476, HDFFV-10477, HDFFV-10478, HDFFV-10480, HDFFV-10481) + + - H5Adelete + + H5Adelete failed when deleting the last "large" attribute that + is stored densely via fractal heap/v2 b-tree. + + After removing the attribute, update the ainfo message. If the + number of attributes goes to zero, remove the message. + + (VC - 2018/07/20, HDFFV-9277) + + - A bug was discovered in the parallel library which caused partial + parallel reads of filtered datasets to return incorrect data. The + library used the incorrect dataspace for each chunk read, causing + the selection used in each chunk to be wrong. + + The bug was not caught during testing because all of the current + tests which do parallel reads of filtered data read all of the data + using an H5S_ALL selection. Several tests were added which exercise + partial parallel reads. + + (JTH - 2018/07/16, HDFFV-10467) + + - A bug was discovered in the parallel library which caused parallel + writes of filtered datasets to trigger an assertion failure in the + file free space manager. + + This occurred when the filter used caused chunks to repeatedly shrink + and grow over the course of several dataset writes. The previous chunk + information, such as the size of the chunk and the offset in the file, + was being cached and not updated after each write, causing the next write + to the chunk to retrieve the incorrect cached information and run into + issues when reallocating space in the file for the chunk. + + (JTH - 2018/07/16, HDFFV-10509) + + - A bug was discovered in the parallel library which caused the + H5D__mpio_array_gatherv() function to allocate too much memory. + + When the function is called with the 'allgather' parameter set + to a non-true value, the function will receive data from all MPI + ranks and gather it to the single rank specied by the 'root' + parameter. However, the bug in the function caused memory for + the received data to be allocated on all MPI ranks, not just the + singular rank specified as the receiver. In some circumstances, + this would cause an application to fail due to the large amounts + of memory being allocated. + + (JTH - 2018/07/16, HDFFV-10467) + + - Error checks in h5stat and when decoding messages + + h5stat exited with seg fault/core dumped when + errors are encountered in the internal library. + + Add error checks and --enable-error-stack option to h5stat. + Add range checks when decoding messages: old fill value, old + layout and refcount. + + (VC - 2018/07/11, HDFFV-10333) + + - If an HDF5 file contains a malformed compound datatype with a + suitably large offset, the type conversion code can run off + the end of the type conversion buffer, causing a segmentation + fault. + + This issue was reported to The HDF Group as issue #CVE-2017-17507. + + NOTE: The HDF5 C library cannot produce such a file. This condition + should only occur in a corrupt (or deliberately altered) file + or a file created by third-party software. + + THE HDF GROUP WILL NOT FIX THIS BUG AT THIS TIME + + Fixing this problem would involve updating the publicly visible + H5T_conv_t function pointer typedef and versioning the API calls + which use it. We normally only modify the public API during + major releases, so this bug will not be fixed at this time. + + (DER - 2018/02/26, HDFFV-10356) + + Configuration + ------------- + - Applied patches to address Cywin build issues + + There were three issues for Cygwin builds: + - Shared libs were not built. + - The -std=c99 flag caused a SIG_SETMASK undeclared error. + - Undefined errors when buildbing test shared libraries. + + Patches to address these issues were received and incorporated in this version. + + (LRK - 2018/07/18, HDFFV-10475) + + - Moved the location of gcc attribute. + + The gcc attribute(no_sanitize), named as the macro HDF_NO_UBSAN, + was located after the function name. Builds with GCC 7 did not + indicate any problem, but GCC 8 issued errors. Moved the + attribute before the function name, as required. + + (ADB - 2018/05/22, HDFFV-10473) + + - Reworked java test suite into individual JUnit tests. + + Testing the whole suite of java unit tests in a single JUnit run + made it difficult to determine actual failures when tests would fail. + Running each file set of tests individually, allows individual failures + to be diagnosed easier. A side benefit is that tests for optional components + of the library can be disabled if not configured. + + (ADB - 2018/05/16, HDFFV-9739) + + - Converted CMake global commands ADD_DEFINITIONS and INCLUDE_DIRECTORIES + to use target_* type commands. This change modernizes the CMake usage + in the HDF5 library. + + In addition, there is the intention to convert to generator expressions, + where possible. The exception is Fortran FLAGS on Windows Visual Studio. + The HDF macros TARGET_C_PROPERTIES and TARGET_FORTRAN_PROPERTIES have + been removed with this change in usage. + + The additional language (C++ and Fortran) checks have also been localized + to only be checked when that language is enabled. + + (ADB - 2018/05/08) + + Performance + ------------- + - + + Fortran + -------- + - + + Tools + ----- + - + + High-Level APIs: + ------ + - + + Fortran High-Level APIs: + ------ + - + + Documentation + ------------- + - + + F90 APIs + -------- + - + + C++ APIs + -------- + - Adding default arguments to existing functions + + Added the following items: + + Two more property list arguments are added to H5Location::createDataSet: + const DSetAccPropList& dapl = DSetAccPropList::DEFAULT + const LinkCreatPropList& lcpl = LinkCreatPropList::DEFAULT + + + One more property list argument is added to H5Location::openDataSet: + const DSetAccPropList& dapl = DSetAccPropList::DEFAULT + + (BMR - 2018/07/21, PR# 1146) + + - Improvement C++ documentation + + Replaced the table in main page of the C++ documentation from mht to htm format + for portability. + + (BMR - 2018/07/17, PR# 1141) + + Testing + ------- + +>>>>>>> fb5d83e146... Updated Fortran Bug and new features Supported Platforms =================== diff --git a/src/H5CX.c b/src/H5CX.c index f809d1c..9ef2886 100644 --- a/src/H5CX.c +++ b/src/H5CX.c @@ -198,6 +198,7 @@ typedef struct H5CX_t { MPI_Datatype btype; /* MPI datatype for buffer, when using collective I/O */ MPI_Datatype ftype; /* MPI datatype for file, when using collective I/O */ hbool_t mpi_file_flushing; /* Whether an MPI-opened file is being flushed */ + hbool_t rank0_bcast; /* Whether a dataset meets read-with-rank0-and-bcast requirements */ #endif /* H5_HAVE_PARALLEL */ /* Cached DXPL properties */ @@ -261,6 +262,8 @@ typedef struct H5CX_t { hbool_t mpio_coll_chunk_multi_ratio_coll_set; /* Whether instrumented "collective chunk multi ratio coll" value is set */ int mpio_coll_chunk_multi_ratio_ind; /* Instrumented "collective chunk multi ratio ind" value (H5D_XFER_COLL_CHUNK_MULTI_RATIO_IND_NAME) */ hbool_t mpio_coll_chunk_multi_ratio_ind_set; /* Whether instrumented "collective chunk multi ratio ind" value is set */ + hbool_t mpio_coll_rank0_bcast; /* Instrumented "collective chunk multi ratio ind" value (H5D_XFER_COLL_CHUNK_MULTI_RATIO_IND_NAME) */ + hbool_t mpio_coll_rank0_bcast_set; /* Whether instrumented "collective chunk multi ratio ind" value is set */ #endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ #endif /* H5_HAVE_PARALLEL */ @@ -1106,6 +1109,32 @@ H5CX_get_mpi_file_flushing(void) FUNC_LEAVE_NOAPI((*head)->ctx.mpi_file_flushing) } /* end H5CX_get_mpi_file_flushing() */ + + +/*------------------------------------------------------------------------- + * Function: H5CX_get_mpio_rank0_bcast + * + * Purpose: Retrieves if the dataset meets read-with-rank0-and-bcast requirements for the current API call context. + * + * Return: Non-negative on success / Negative on failure + * + * Programmer: M. Breitenfeld + * December 31, 2018 + * + *------------------------------------------------------------------------- + */ +hbool_t +H5CX_get_mpio_rank0_bcast(void) +{ + H5CX_node_t **head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Sanity check */ + HDassert(head && *head); + + FUNC_LEAVE_NOAPI((*head)->ctx.rank0_bcast) +} /* end H5CX_get_mpio_rank0_bcast() */ #endif /* H5_HAVE_PARALLEL */ @@ -2037,6 +2066,34 @@ H5CX_set_mpi_file_flushing(hbool_t flushing) FUNC_LEAVE_NOAPI_VOID } /* end H5CX_set_mpi_file_flushing() */ + + +/*------------------------------------------------------------------------- + * Function: H5CX_set_mpio_rank0_bcast + * + * Purpose: Sets the "dataset meets read-with-rank0-and-bcast requirements" flag for the current API call context. + * + * Return: + * + * Programmer: M. Breitenfeld + * December 31, 2018 + * + *------------------------------------------------------------------------- + */ +void +H5CX_set_mpio_rank0_bcast(hbool_t rank0_bcast) +{ + H5CX_node_t **head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Sanity checks */ + HDassert(head && *head); + + (*head)->ctx.rank0_bcast = rank0_bcast; + + FUNC_LEAVE_NOAPI_VOID +} /* end H5CX_set_mpio_rank0_bcast() */ #endif /* H5_HAVE_PARALLEL */ @@ -2448,6 +2505,40 @@ H5CX_test_set_mpio_coll_chunk_multi_ratio_ind(int mpio_coll_chunk_multi_ratio_in done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5CX_test_set_mpio_coll_chunk_multi_ratio_ind() */ + + +/*------------------------------------------------------------------------- + * Function: H5CX_test_set_mpio_coll_rank0_bcast + * + * Purpose: Sets the instrumented "read-with-rank0-bcast" flag for the current API call context. + * + * Note: Only sets value if property set in DXPL + * + * Return: Non-negative on success / Negative on failure + * + * Programmer: Quincey Koziol + * January 2, 2019 + * + *------------------------------------------------------------------------- + */ +herr_t +H5CX_test_set_mpio_coll_rank0_bcast(hbool_t mpio_coll_rank0_bcast) +{ + H5CX_node_t **head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Sanity checks */ + HDassert(head && *head); + HDassert(!((*head)->ctx.dxpl_id == H5P_DEFAULT || + (*head)->ctx.dxpl_id == H5P_DATASET_XFER_DEFAULT)); + + H5CX_TEST_SET_PROP(H5D_XFER_COLL_RANK0_BCAST_NAME, mpio_coll_rank0_bcast) + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5CX_test_set_mpio_coll_rank0_bcast() */ #endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ #endif /* H5_HAVE_PARALLEL */ @@ -2488,6 +2579,7 @@ H5CX__pop_common(void) H5CX_SET_PROP(H5D_XFER_COLL_CHUNK_LINK_NUM_FALSE_NAME, mpio_coll_chunk_link_num_false) H5CX_SET_PROP(H5D_XFER_COLL_CHUNK_MULTI_RATIO_COLL_NAME, mpio_coll_chunk_multi_ratio_coll) H5CX_SET_PROP(H5D_XFER_COLL_CHUNK_MULTI_RATIO_IND_NAME, mpio_coll_chunk_multi_ratio_ind) + H5CX_SET_PROP(H5D_XFER_COLL_RANK0_BCAST_NAME, mpio_coll_rank0_bcast) #endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5CXprivate.h b/src/H5CXprivate.h index 44a4067..6b8e3b2 100644 --- a/src/H5CXprivate.h +++ b/src/H5CXprivate.h @@ -73,6 +73,7 @@ H5_DLL H5AC_ring_t H5CX_get_ring(void); H5_DLL hbool_t H5CX_get_coll_metadata_read(void); H5_DLL herr_t H5CX_get_mpi_coll_datatypes(MPI_Datatype *btype, MPI_Datatype *ftype); H5_DLL hbool_t H5CX_get_mpi_file_flushing(void); +H5_DLL hbool_t H5CX_get_mpio_rank0_bcast(void); #endif /* H5_HAVE_PARALLEL */ /* "Getter" routines for DXPL properties cached in API context */ @@ -108,6 +109,7 @@ H5_DLL void H5CX_set_coll_metadata_read(hbool_t cmdr); H5_DLL herr_t H5CX_set_mpi_coll_datatypes(MPI_Datatype btype, MPI_Datatype ftype); H5_DLL herr_t H5CX_set_mpio_coll_opt(H5FD_mpio_collective_opt_t mpio_coll_opt); H5_DLL void H5CX_set_mpi_file_flushing(hbool_t flushing); +H5_DLL void H5CX_set_mpio_rank0_bcast(hbool_t rank0_bcast); #endif /* H5_HAVE_PARALLEL */ /* "Setter" routines for DXPL properties cached in API context */ @@ -126,6 +128,7 @@ H5_DLL void H5CX_set_mpio_actual_chunk_opt(H5D_mpio_actual_chunk_opt_mode_t chun H5_DLL void H5CX_set_mpio_actual_io_mode(H5D_mpio_actual_io_mode_t actual_io_mode); H5_DLL void H5CX_set_mpio_local_no_coll_cause(uint32_t mpio_local_no_coll_cause); H5_DLL void H5CX_set_mpio_global_no_coll_cause(uint32_t mpio_global_no_coll_cause); +H5_DLL void H5CX_set_mpio_Proc0_BCast(hbool_t mpio_Proc0_BCast); #ifdef H5_HAVE_INSTRUMENTED_LIBRARY H5_DLL herr_t H5CX_test_set_mpio_coll_chunk_link_hard(int mpio_coll_chunk_link_hard); H5_DLL herr_t H5CX_test_set_mpio_coll_chunk_multi_hard(int mpio_coll_chunk_multi_hard); @@ -133,6 +136,7 @@ H5_DLL herr_t H5CX_test_set_mpio_coll_chunk_link_num_true(int mpio_coll_chunk_li H5_DLL herr_t H5CX_test_set_mpio_coll_chunk_link_num_false(int mpio_coll_chunk_link_num_false); H5_DLL herr_t H5CX_test_set_mpio_coll_chunk_multi_ratio_coll(int mpio_coll_chunk_multi_ratio_coll); H5_DLL herr_t H5CX_test_set_mpio_coll_chunk_multi_ratio_ind(int mpio_coll_chunk_multi_ratio_ind); +H5_DLL herr_t H5CX_test_set_mpio_coll_rank0_bcast(hbool_t rank0_bcast); #endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5Dio.c b/src/H5Dio.c index ed384ca..c762389 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -685,22 +685,12 @@ H5D__write(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, /* Various MPI based checks */ #ifdef H5_HAVE_PARALLEL - if H5F_HAS_FEATURE(dataset->oloc.file, H5FD_FEAT_HAS_MPI) { - /* If MPI based VFD is used, no VL datatype support yet. */ + if(H5F_HAS_FEATURE(dataset->oloc.file, H5FD_FEAT_HAS_MPI)) { + /* If MPI based VFD is used, no VL or region reference datatype support yet. */ /* This is because they use the global heap in the file and we don't */ /* support parallel access of that yet */ - if(H5T_detect_class(type_info.mem_type, H5T_VLEN, FALSE) > 0) - HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "Parallel IO does not support writing VL datatypes yet") - - /* If MPI based VFD is used, no VL datatype support yet. */ - /* This is because they use the global heap in the file and we don't */ - /* support parallel access of that yet */ - /* We should really use H5T_detect_class() here, but it will be difficult - * to detect the type of the reference if it is nested... -QAK - */ - if(H5T_get_class(type_info.mem_type, TRUE) == H5T_REFERENCE && - H5T_get_ref_type(type_info.mem_type) == H5R_DATASET_REGION) - HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "Parallel IO does not support writing region reference datatypes yet") + if(H5T_is_vl_storage(type_info.mem_type) > 0) + HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "Parallel IO does not support writing VL or region reference datatypes yet") } /* end if */ else { H5FD_mpio_xfer_t io_xfer_mode; /* MPI I/O transfer mode */ diff --git a/src/H5Dmpio.c b/src/H5Dmpio.c index baec4c4..20b2066 100644 --- a/src/H5Dmpio.c +++ b/src/H5Dmpio.c @@ -37,7 +37,6 @@ #include "H5Eprivate.h" /* Error handling */ #include "H5Fprivate.h" /* File access */ #include "H5FDprivate.h" /* File drivers */ -#include "H5FDmpi.h" /* MPI-based file drivers */ #include "H5Iprivate.h" /* IDs */ #include "H5MMprivate.h" /* Memory management */ #include "H5Oprivate.h" /* Object headers */ @@ -83,16 +82,30 @@ /* Macros to represent the regularity of the selection for multiple chunk IO case. */ #define H5D_CHUNK_SELECT_REG 1 +/* Macros for reasons to not enable read-proc-and-bcast. */ +#define H5D_MPIO_PROC0_BCAST 0x00 +#define H5D_MPIO_NOT_H5S_ALL 0x01 +#define H5D_MPIO_GREATER_THAN_2GB 0x02 /******************/ /* Local Typedefs */ /******************/ /* Combine chunk address and chunk info into a struct for better performance. */ typedef struct H5D_chunk_addr_info_t { - haddr_t chunk_addr; - H5D_chunk_info_t chunk_info; + haddr_t chunk_addr; + H5D_chunk_info_t chunk_info; } H5D_chunk_addr_info_t; +/* Rank 0 Bcast values */ +typedef enum H5D_mpio_no_rank0_bcast_cause_t { + H5D_MPIO_RANK0_BCAST = 0x00, + H5D_MPIO_RANK0_NOT_H5S_ALL = 0x01, + H5D_MPIO_RANK0_NOT_CONTIGUOUS = 0x02, + H5D_MPIO_RANK0_NOT_FIXED_SIZE = 0x04, + H5D_MPIO_RANK0_GREATER_THAN_2GB = 0x08 +} H5D_mpio_no_rank0_bcast_cause_t; + + /* * Information about a single chunk when performing collective filtered I/O. All * of the fields of one of these structs are initialized at the start of collective @@ -281,9 +294,12 @@ H5D__mpio_opt_possible(const H5D_io_info_t *io_info, const H5S_t *file_space, const H5S_t *mem_space, const H5D_type_info_t *type_info) { H5FD_mpio_xfer_t io_xfer_mode; /* MPI I/O transfer mode */ - unsigned local_cause = 0; /* Local reason(s) for breaking collective mode */ - unsigned global_cause = 0; /* Global reason(s) for breaking collective mode */ + unsigned local_cause[2] = {0,0}; /* [0] Local reason(s) for breaking collective mode */ + /* [1] Flag if dataset is both: H5S_ALL and small */ + unsigned global_cause[2] = {0,0}; /* Global reason(s) for breaking collective mode */ + htri_t is_vl_storage; /* Whether the dataset's datatype is stored in a variable-length form */ htri_t ret_value = SUCCEED; /* Return value */ + hbool_t H5FD_MPIO_Proc0_BCast; /* Flag if dataset is both: H5S_ALL and < 2GB */ FUNC_ENTER_PACKAGE @@ -296,36 +312,37 @@ H5D__mpio_opt_possible(const H5D_io_info_t *io_info, const H5S_t *file_space, /* For independent I/O, get out quickly and don't try to form consensus */ if(H5CX_get_io_xfer_mode(&io_xfer_mode) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get MPI-I/O transfer mode") + /* Set error flag, but keep going */ + local_cause[0] |= H5D_MPIO_ERROR_WHILE_CHECKING_COLLECTIVE_POSSIBLE; if(io_xfer_mode == H5FD_MPIO_INDEPENDENT) - local_cause |= H5D_MPIO_SET_INDEPENDENT; + local_cause[0] |= H5D_MPIO_SET_INDEPENDENT; /* Optimized MPI types flag must be set */ /* (based on 'HDF5_MPI_OPT_TYPES' environment variable) */ if(!H5FD_mpi_opt_types_g) - local_cause |= H5D_MPIO_MPI_OPT_TYPES_ENV_VAR_DISABLED; + local_cause[0] |= H5D_MPIO_MPI_OPT_TYPES_ENV_VAR_DISABLED; /* Don't allow collective operations if datatype conversions need to happen */ if(!type_info->is_conv_noop) - local_cause |= H5D_MPIO_DATATYPE_CONVERSION; + local_cause[0] |= H5D_MPIO_DATATYPE_CONVERSION; /* Don't allow collective operations if data transform operations should occur */ if(!type_info->is_xform_noop) - local_cause |= H5D_MPIO_DATA_TRANSFORMS; + local_cause[0] |= H5D_MPIO_DATA_TRANSFORMS; /* Check whether these are both simple or scalar dataspaces */ if(!((H5S_SIMPLE == H5S_GET_EXTENT_TYPE(mem_space) || H5S_SCALAR == H5S_GET_EXTENT_TYPE(mem_space)) && (H5S_SIMPLE == H5S_GET_EXTENT_TYPE(file_space) || H5S_SCALAR == H5S_GET_EXTENT_TYPE(file_space)))) - local_cause |= H5D_MPIO_NOT_SIMPLE_OR_SCALAR_DATASPACES; + local_cause[0] |= H5D_MPIO_NOT_SIMPLE_OR_SCALAR_DATASPACES; /* Dataset storage must be contiguous or chunked */ if(!(io_info->dset->shared->layout.type == H5D_CONTIGUOUS || io_info->dset->shared->layout.type == H5D_CHUNKED)) - local_cause |= H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; + local_cause[0] |= H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; /* check if external-file storage is used */ if(io_info->dset->shared->dcpl_cache.efl.nused > 0) - local_cause |= H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; + local_cause[0] |= H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; /* The handling of memory space is different for chunking and contiguous * storage. For contiguous storage, mem_space and file_space won't change @@ -340,31 +357,85 @@ H5D__mpio_opt_possible(const H5D_io_info_t *io_info, const H5S_t *file_space, * is less than 3. The functions needed (MPI_Mprobe and MPI_Imrecv) will * not be available. */ - if (io_info->op_type == H5D_IO_OP_WRITE && - io_info->dset->shared->layout.type == H5D_CHUNKED && - io_info->dset->shared->dcpl_cache.pline.nused > 0) - local_cause |= H5D_MPIO_PARALLEL_FILTERED_WRITES_DISABLED; + if(io_info->op_type == H5D_IO_OP_WRITE && + io_info->dset->shared->layout.type == H5D_CHUNKED && + io_info->dset->shared->dcpl_cache.pline.nused > 0) + local_cause[0] |= H5D_MPIO_PARALLEL_FILTERED_WRITES_DISABLED; #endif + /* Check if we are able to do a MPI_Bcast of the data from one rank + * instead of having all the processes involved in the collective I/O call. + */ + + /* Check to see if the process is reading the entire dataset */ + if(H5S_GET_SELECT_TYPE(file_space) != H5S_SEL_ALL) + local_cause[1] |= H5D_MPIO_RANK0_NOT_H5S_ALL; + /* Only perform this optimization for contigous datasets, currently */ + else if(H5D_CONTIGUOUS != io_info->dset->shared->layout.type) + /* Flag to do a MPI_Bcast of the data from one proc instead of + * having all the processes involved in the collective I/O. + */ + local_cause[1] |= H5D_MPIO_RANK0_NOT_CONTIGUOUS; + else if((is_vl_storage = H5T_is_vl_storage(type_info->dset_type)) < 0) + local_cause[0] |= H5D_MPIO_ERROR_WHILE_CHECKING_COLLECTIVE_POSSIBLE; + else if(is_vl_storage) + local_cause[1] |= H5D_MPIO_RANK0_NOT_FIXED_SIZE; + else { + size_t type_size; /* Size of dataset's datatype */ + + /* Retrieve the size of the dataset's datatype */ + if(0 == (type_size = H5T_GET_SIZE(type_info->dset_type))) + local_cause[0] |= H5D_MPIO_ERROR_WHILE_CHECKING_COLLECTIVE_POSSIBLE; + else { + hssize_t snelmts; /* [Signed] # of elements in dataset's dataspace */ + + /* Retrieve the size of the dataset's datatype */ + if((snelmts = H5S_GET_EXTENT_NPOINTS(file_space)) < 0) + local_cause[0] |= H5D_MPIO_ERROR_WHILE_CHECKING_COLLECTIVE_POSSIBLE; + else { + hsize_t dset_size; + + /* Determine dataset size */ + dset_size = ((hsize_t)snelmts) * type_size; + + /* If the size of the dataset is less than 2GB then do an MPI_Bcast + * of the data from one process instead of having all the processes + * involved in the collective I/O. + */ + if(dset_size > ((hsize_t)(2.0F * H5_GB) - 1)) + local_cause[1] |= H5D_MPIO_RANK0_GREATER_THAN_2GB; + } /* end else */ + } /* end else */ + } /* end else */ + /* Check for independent I/O */ - if(local_cause & H5D_MPIO_SET_INDEPENDENT) - global_cause = local_cause; + if(local_cause[0] & H5D_MPIO_SET_INDEPENDENT) + global_cause[0] = local_cause[0]; else { int mpi_code; /* MPI error code */ /* Form consensus opinion among all processes about whether to perform * collective I/O */ - if(MPI_SUCCESS != (mpi_code = MPI_Allreduce(&local_cause, &global_cause, 1, MPI_UNSIGNED, MPI_BOR, io_info->comm))) + if(MPI_SUCCESS != (mpi_code = MPI_Allreduce(&local_cause, &global_cause, 2, MPI_UNSIGNED, MPI_BOR, io_info->comm))) HMPI_GOTO_ERROR(FAIL, "MPI_Allreduce failed", mpi_code) + } /* end else */ /* Set the local & global values of no-collective-cause in the API context */ - H5CX_set_mpio_local_no_coll_cause(local_cause); - H5CX_set_mpio_global_no_coll_cause(global_cause); + H5CX_set_mpio_local_no_coll_cause(local_cause[0]); + H5CX_set_mpio_global_no_coll_cause(global_cause[0]); + + /* Set read-with-rank0-and-bcast flag if possible */ + if(global_cause[0] == 0 && global_cause[1] == 0) { + H5CX_set_mpio_rank0_bcast(TRUE); +#ifdef H5_HAVE_INSTRUMENTED_LIBRARY + H5CX_test_set_mpio_coll_rank0_bcast(TRUE); +#endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ + } /* end if */ /* Set the return value, based on the global cause */ - ret_value = global_cause > 0 ? FALSE : TRUE; + ret_value = global_cause[0] > 0 ? FALSE : TRUE; done: FUNC_LEAVE_NOAPI(ret_value) @@ -3076,8 +3147,8 @@ H5D__filtered_collective_chunk_entry_io(H5D_filtered_collective_io_info_t *chunk chunk_entry->chunk_states.new_chunk.length = chunk_entry->chunk_states.chunk_current.length; /* Currently, these chunk reads are done independently and will likely - * cause issues with collective metadata reads enabled. In the future, - * this should be refactored to use collective chunk reads - JTH */ + * cause issues with collective metadata reads enabled. In the future, + * this should be refactored to use collective chunk reads - JTH */ /* Get the original state of parallel I/O transfer mode */ if(H5CX_get_io_xfer_mode(&xfer_mode) < 0) diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h index ccbae9d..00e1e28 100644 --- a/src/H5Dprivate.h +++ b/src/H5Dprivate.h @@ -95,7 +95,13 @@ /* Definitions for all collective chunk instrumentation properties */ #define H5D_XFER_COLL_CHUNK_SIZE sizeof(unsigned) #define H5D_XFER_COLL_CHUNK_DEF 1 -#define H5D_XFER_COLL_CHUNK_FIX 0 + +/* General collective I/O instrumentation properties */ +#define H5D_XFER_COLL_RANK0_BCAST_NAME "coll_rank0_bcast" + +/* Definitions for general collective I/O instrumentation properties */ +#define H5D_XFER_COLL_RANK0_BCAST_SIZE sizeof(hbool_t) +#define H5D_XFER_COLL_RANK0_BCAST_DEF FALSE #endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ /* Default temporary buffer size */ diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c index 87f8b6a..689f43c 100644 --- a/src/H5FDmpio.c +++ b/src/H5FDmpio.c @@ -1414,31 +1414,32 @@ static herr_t H5FD_mpio_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNUSED dxpl_id, haddr_t addr, size_t size, void *buf/*out*/) { - H5FD_mpio_t *file = (H5FD_mpio_t*)_file; - MPI_Offset mpi_off; - MPI_Status mpi_stat; /* Status from I/O operation */ - int mpi_code; /* mpi return code */ - MPI_Datatype buf_type = MPI_BYTE; /* MPI description of the selection in memory */ - int size_i; /* Integer copy of 'size' to read */ + H5FD_mpio_t *file = (H5FD_mpio_t*)_file; + MPI_Offset mpi_off; + MPI_Status mpi_stat; /* Status from I/O operation */ + int mpi_code; /* mpi return code */ + MPI_Datatype buf_type = MPI_BYTE; /* MPI description of the selection in memory */ + int size_i; /* Integer copy of 'size' to read */ #if MPI_VERSION >= 3 - MPI_Count bytes_read; /* Number of bytes read in */ + MPI_Count bytes_read = 0; /* Number of bytes read in */ MPI_Count type_size; /* MPI datatype used for I/O's size */ MPI_Count io_size; /* Actual number of bytes requested */ MPI_Count n; #else - int bytes_read; /* Number of bytes read in */ + int bytes_read = 0; /* Number of bytes read in */ int type_size; /* MPI datatype used for I/O's size */ int io_size; /* Actual number of bytes requested */ - int n; + int n; #endif - hbool_t use_view_this_time = FALSE; - herr_t ret_value = SUCCEED; + hbool_t use_view_this_time = FALSE; + hbool_t rank0_bcast = FALSE; /* If read-with-rank0-and-bcast flag was used */ + herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI_NOINIT #ifdef H5FDmpio_DEBUG if (H5FD_mpio_Debug[(int)'t']) - fprintf(stdout, "Entering H5FD_mpio_read\n" ); + fprintf(stdout, "Entering H5FD_mpio_read\n" ); #endif HDassert(file); HDassert(H5FD_MPIO==file->pub.driver_id); @@ -1457,12 +1458,12 @@ H5FD_mpio_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, #ifdef H5FDmpio_DEBUG if (H5FD_mpio_Debug[(int)'r']) fprintf(stdout, "in H5FD_mpio_read mpi_off=%ld size_i=%d\n", - (long)mpi_off, size_i ); + (long)mpi_off, size_i ); #endif /* Only look for MPI views for raw data transfers */ if(type == H5FD_MEM_DRAW) { - H5FD_mpio_xfer_t xfer_mode; /* I/O transfer mode */ + H5FD_mpio_xfer_t xfer_mode; /* I/O transfer mode */ /* Get the transfer mode from the API context */ if(H5CX_get_io_xfer_mode(&xfer_mode) < 0) @@ -1475,7 +1476,7 @@ H5FD_mpio_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, * could mean "use MPI_BYTE" by convention). */ if(xfer_mode==H5FD_MPIO_COLLECTIVE) { - MPI_Datatype file_type; + MPI_Datatype file_type; /* Remember that views are used */ use_view_this_time = TRUE; @@ -1502,8 +1503,8 @@ H5FD_mpio_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, H5FD_mpio_collective_opt_t coll_opt_mode; #ifdef H5FDmpio_DEBUG - if (H5FD_mpio_Debug[(int)'t']) - fprintf(stdout, "H5FD_mpio_read: using MPIO collective mode\n"); + if (H5FD_mpio_Debug[(int)'t']) + fprintf(stdout, "H5FD_mpio_read: using MPIO collective mode\n"); #endif /* Get the collective_opt property to check whether the application wants to do IO individually. */ if(H5CX_get_mpio_coll_opt(&coll_opt_mode) < 0) @@ -1514,8 +1515,25 @@ H5FD_mpio_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, if(H5FD_mpio_Debug[(int)'t']) fprintf(stdout, "H5FD_mpio_read: doing MPI collective IO\n"); #endif - if(MPI_SUCCESS != (mpi_code = MPI_File_read_at_all(file->f, mpi_off, buf, size_i, buf_type, &mpi_stat))) - HMPI_GOTO_ERROR(FAIL, "MPI_File_read_at_all failed", mpi_code) + /* Check whether we should read from rank 0 and broadcast to other ranks */ + if(H5CX_get_mpio_rank0_bcast()) { +#ifdef H5FDmpio_DEBUG + if(H5FD_mpio_Debug[(int)'r']) + HDfprintf(stdout, "%s: doing read-rank0-and-MPI_Bcast\n", FUNC); +#endif + /* Indicate path we've taken */ + rank0_bcast = TRUE; + + /* Read on rank 0 Bcast to other ranks */ + if(file->mpi_rank == 0) + if(MPI_SUCCESS != (mpi_code = MPI_File_read_at(file->f, mpi_off, buf, size_i, buf_type, &mpi_stat))) + HMPI_GOTO_ERROR(FAIL, "MPI_File_read_at failed", mpi_code) + if(MPI_SUCCESS != (mpi_code = MPI_Bcast(buf, size_i, buf_type, 0, file->comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_code) + } /* end if */ + else + if(MPI_SUCCESS != (mpi_code = MPI_File_read_at_all(file->f, mpi_off, buf, size_i, buf_type, &mpi_stat))) + HMPI_GOTO_ERROR(FAIL, "MPI_File_read_at_all failed", mpi_code) } /* end if */ else { #ifdef H5FDmpio_DEBUG @@ -1534,24 +1552,37 @@ H5FD_mpio_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, HMPI_GOTO_ERROR(FAIL, "MPI_File_set_view failed", mpi_code) } else { if(MPI_SUCCESS != (mpi_code = MPI_File_read_at(file->f, mpi_off, buf, size_i, buf_type, &mpi_stat))) - HMPI_GOTO_ERROR(FAIL, "MPI_File_read_at failed", mpi_code) + HMPI_GOTO_ERROR(FAIL, "MPI_File_read_at failed", mpi_code) } - /* How many bytes were actually read? */ + /* Only retrieve bytes read if this rank _actually_ participated in I/O */ + if(!rank0_bcast || (rank0_bcast && file->mpi_rank == 0) ) { + /* How many bytes were actually read? */ #if MPI_VERSION >= 3 - if (MPI_SUCCESS != (mpi_code = MPI_Get_elements_x(&mpi_stat, buf_type, &bytes_read))) + if(MPI_SUCCESS != (mpi_code = MPI_Get_elements_x(&mpi_stat, buf_type, &bytes_read))) #else - if (MPI_SUCCESS != (mpi_code = MPI_Get_elements(&mpi_stat, MPI_BYTE, &bytes_read))) + if(MPI_SUCCESS != (mpi_code = MPI_Get_elements(&mpi_stat, MPI_BYTE, &bytes_read))) #endif - HMPI_GOTO_ERROR(FAIL, "MPI_Get_elements failed", mpi_code) + HMPI_GOTO_ERROR(FAIL, "MPI_Get_elements failed", mpi_code) + } /* end if */ + + /* If the rank0-bcast feature was used, broadcast the # of bytes read to + * other ranks, which didn't perform any I/O. + */ + /* NOTE: This could be optimized further to be combined with the broadcast + * of the data. (QAK - 2019/1/2) + */ + if(rank0_bcast) + if(MPI_SUCCESS != MPI_Bcast(&bytes_read, 1, MPI_LONG_LONG, 0, file->comm)) + HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", 0) /* Get the type's size */ #if MPI_VERSION >= 3 - if (MPI_SUCCESS != (mpi_code = MPI_Type_size_x(buf_type, &type_size))) + if(MPI_SUCCESS != (mpi_code = MPI_Type_size_x(buf_type, &type_size))) #else - if (MPI_SUCCESS != (mpi_code = MPI_Type_size(buf_type, &type_size))) + if(MPI_SUCCESS != (mpi_code = MPI_Type_size(buf_type, &type_size))) #endif - HMPI_GOTO_ERROR(FAIL, "MPI_Type_size failed", mpi_code) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_size failed", mpi_code) /* Compute the actual number of bytes requested */ io_size=type_size*size_i; @@ -1564,12 +1595,12 @@ H5FD_mpio_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, * This gives us zeroes beyond end of physical MPI file. */ if ((n=(io_size-bytes_read)) > 0) - HDmemset((char*)buf+bytes_read, 0, (size_t)n); + HDmemset((char*)buf+bytes_read, 0, (size_t)n); done: #ifdef H5FDmpio_DEBUG if (H5FD_mpio_Debug[(int)'t']) - fprintf(stdout, "Leaving H5FD_mpio_read\n" ); + fprintf(stdout, "Leaving H5FD_mpio_read\n" ); #endif FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5Pdxpl.c b/src/H5Pdxpl.c index dcfe89f..1c97589 100644 --- a/src/H5Pdxpl.c +++ b/src/H5Pdxpl.c @@ -2017,6 +2017,7 @@ done: FUNC_LEAVE_API(ret_value) } /* end H5Pget_mpio_actual_io_mode() */ + /*------------------------------------------------------------------------- * Function: H5Pget_mpio_no_collective_cause * @@ -2053,8 +2054,6 @@ H5Pget_mpio_no_collective_cause(hid_t plist_id, uint32_t *local_no_collective_ca done: FUNC_LEAVE_API(ret_value) } /* end H5Pget_mpio_no_collective_cause() */ - - #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index b566085..5d6dea8 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -165,7 +165,8 @@ typedef enum H5D_mpio_no_collective_cause_t { H5D_MPIO_NOT_SIMPLE_OR_SCALAR_DATASPACES = 0x10, H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET = 0x20, H5D_MPIO_PARALLEL_FILTERED_WRITES_DISABLED = 0x40, - H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE = 0x80 + H5D_MPIO_ERROR_WHILE_CHECKING_COLLECTIVE_POSSIBLE = 0x80, + H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE = 0x100 } H5D_mpio_no_collective_cause_t; /********************/ diff --git a/src/H5T.c b/src/H5T.c index 1ab5b42..023a65f 100644 --- a/src/H5T.c +++ b/src/H5T.c @@ -301,8 +301,8 @@ static herr_t H5T__unregister(H5T_pers_t pers, const char *name, H5T_t *src, static htri_t H5T__compiler_conv(H5T_t *src, H5T_t *dst); static herr_t H5T__set_size(H5T_t *dt, size_t size); static herr_t H5T__close_cb(H5T_t *dt); -static H5T_path_t *H5T__path_find_real(const H5T_t *src, const H5T_t *dst, - const char *name, H5T_conv_func_t *conv); +static H5T_path_t *H5T__path_find_real(const H5T_t *src, const H5T_t *dst, const char *name, H5T_conv_func_t *conv); +static hbool_t H5T__detect_reg_ref(const H5T_t *dt); /*****************************/ @@ -5517,6 +5517,111 @@ done: /*------------------------------------------------------------------------- + * Function: H5T_detect_reg_ref + * + * Purpose: Check whether a datatype contains (or is) a region reference + * datatype. + * + * Return: TRUE (1) or FALSE (0) on success + * (Can't fail) + * + * Programmer: Quincey Koziol + * Saturday, January 5, 2019 + * + *------------------------------------------------------------------------- + */ +static hbool_t +H5T__detect_reg_ref(const H5T_t *dt) +{ + unsigned u; /* Local index variable */ + hbool_t ret_value = FALSE; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Sanity checks */ + HDassert(dt); + + /* Check if this datatype is a region reference */ + if(H5T_REFERENCE == dt->shared->type && H5R_DATASET_REGION == dt->shared->u.atomic.u.r.rtype) + HGOTO_DONE(TRUE); + + /* Check for types that might have the correct type as a component */ + switch(dt->shared->type) { + case H5T_COMPOUND: + /* Iterate over all the compound datatype's fields */ + for(u = 0; u < dt->shared->u.compnd.nmembs; u++) + /* Recurse on field's datatype */ + if(H5T__detect_reg_ref(dt->shared->u.compnd.memb[u].type)) + HGOTO_DONE(TRUE); + break; + + case H5T_ARRAY: + case H5T_VLEN: + case H5T_ENUM: + HGOTO_DONE(H5T__detect_reg_ref(dt->shared->parent)); + break; + + case H5T_NO_CLASS: + case H5T_INTEGER: + case H5T_FLOAT: + case H5T_TIME: + case H5T_STRING: + case H5T_BITFIELD: + case H5T_OPAQUE: + case H5T_REFERENCE: + case H5T_NCLASSES: + default: + break; + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5T__detect_reg_ref() */ + + +/*------------------------------------------------------------------------- + * Function: H5T_is_vl_storage + * + * Purpose: Check if a datatype will be stored in a variable-length form. + * + * Notes: Currently, only variable-length string & sequences and region + * references are stored in a variable-length form. + * + * Return: + * One of two values on success: + * TRUE - If the datatype will be stored in a variable-length form + * FALSE - If the datatype will NOT be stored in a variable-length form + * <0 is returned on failure + * + * Programmer: Quincey Koziol + * Saturday, January 5, 2019 + * + *------------------------------------------------------------------------- + */ +htri_t +H5T_is_vl_storage(const H5T_t *dt) +{ + htri_t ret_value = FALSE; + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity check */ + HDassert(dt); + + /* VL and region reference datatypes are stored in variable-length form */ + if(H5T_detect_class(dt, H5T_VLEN, FALSE)) + ret_value = TRUE; + else if(H5T_detect_class(dt, H5T_REFERENCE, FALSE)) + ret_value = H5T__detect_reg_ref(dt); + else + ret_value = FALSE; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5T_is_vl_storage() */ + + +/*------------------------------------------------------------------------- * Function: H5T_upgrade_version_cb * * Purpose: H5T__visit callback to Upgrade the version of a datatype diff --git a/src/H5Tprivate.h b/src/H5Tprivate.h index 89cdcfd..aff3f6d 100644 --- a/src/H5Tprivate.h +++ b/src/H5Tprivate.h @@ -138,6 +138,7 @@ H5_DLL herr_t H5T_set_version(H5F_t *f, H5T_t *dt); H5_DLL herr_t H5T_patch_file(H5T_t *dt, H5F_t *f); H5_DLL herr_t H5T_patch_vlen_file(H5T_t *dt, H5F_t *f); H5_DLL htri_t H5T_is_variable_str(const H5T_t *dt); +H5_DLL htri_t H5T_is_vl_storage(const H5T_t *dt); /* Reference specific functions */ H5_DLL H5R_type_t H5T_get_ref_type(const H5T_t *dt); diff --git a/testpar/t_bigio.c b/testpar/t_bigio.c index fdd3488..1d882b8 100644 --- a/testpar/t_bigio.c +++ b/testpar/t_bigio.c @@ -671,7 +671,7 @@ dataset_big_write(void) /* create a memory dataspace independently */ mem_dataspace = H5Screate_simple (RANK, dims, NULL); VRFY((mem_dataspace >= 0), ""); - if(!mpi_rank == 0) { + if(mpi_rank != 0) { ret = H5Sselect_none(mem_dataspace); VRFY((ret >= 0), "H5Sset_none succeeded"); } @@ -980,7 +980,7 @@ dataset_big_read(void) /* create a memory dataspace independently */ mem_dataspace = H5Screate_simple (RANK, dims, NULL); VRFY((mem_dataspace >= 0), ""); - if(!mpi_rank == 0) { + if(mpi_rank != 0) { ret = H5Sselect_none(mem_dataspace); VRFY((ret >= 0), "H5Sset_none succeeded"); } diff --git a/testpar/t_dset.c b/testpar/t_dset.c index 281d027..35501d8 100644 --- a/testpar/t_dset.c +++ b/testpar/t_dset.c @@ -2649,7 +2649,7 @@ compress_readAll(void) /* Try reading the data */ ret = H5Dread(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, xfer_plist, data_read); - VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); + VRFY((ret >= 0), "H5Dread succeeded"); /* Verify data read */ for(u=0; u= 0),"set independent IO collectively succeeded"); } - dataset = H5Dopen2(iof, dname, H5P_DEFAULT); VRFY((dataset >= 0), "H5Dopen2 succeeded"); +#ifdef H5_HAVE_INSTRUMENTED_LIBRARY + hbool_t prop_value; + prop_value = H5D_XFER_COLL_RANK0_BCAST_DEF; + ret = H5Pinsert2(dxpl, H5D_XFER_COLL_RANK0_BCAST_NAME, H5D_XFER_COLL_RANK0_BCAST_SIZE, &prop_value, + NULL, NULL, NULL, NULL, NULL, NULL); + VRFY((ret >= 0), "H5Pinsert2() succeeded"); +#endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ + ret = H5Dread(dataset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, dxpl, inme); VRFY((ret >= 0), "H5Dread succeeded"); +#ifdef H5_HAVE_INSTRUMENTED_LIBRARY + prop_value = FALSE; + ret = H5Pget(dxpl, H5D_XFER_COLL_RANK0_BCAST_NAME, &prop_value); + VRFY((ret >= 0), "H5Pget succeeded"); + VRFY((prop_value == FALSE && dxfer_coll_type == DXFER_COLLECTIVE_IO),"rank 0 Bcast optimization was performed for a compact dataset"); +#endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ + /* Verify data value */ for(i = 0; i < size; i++) for(j = 0; j < size; j++) @@ -603,8 +618,8 @@ void dataset_fillvalue(void) hsize_t req_count[4] = {1, 6, 7, 8}; hsize_t dset_size; /* Dataset size */ int *rdata, *wdata; /* Buffers for data to read and write */ - int *twdata, *trdata; /* Temporary pointer into buffer */ - int acc, i, j, k, l; /* Local index variables */ + int *twdata, *trdata; /* Temporary pointer into buffer */ + int acc, i, j, k, l, ii; /* Local index variables */ herr_t ret; /* Generic return value */ const char *filename; @@ -645,27 +660,60 @@ void dataset_fillvalue(void) /* * Read dataset before any data is written. */ - /* set entire read buffer with the constant 2 */ - HDmemset(rdata,2,(size_t)(dset_size*sizeof(int))); - /* Independently read the entire dataset back */ - ret = H5Dread(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rdata); - VRFY((ret >= 0), "H5Dread succeeded"); - /* Verify all data read are the fill value 0 */ - trdata = rdata; - err_num = 0; - for(i = 0; i < (int)dset_dims[0]; i++) + /* Create DXPL for I/O */ + dxpl = H5Pcreate(H5P_DATASET_XFER); + VRFY((dxpl >= 0), "H5Pcreate succeeded"); + +#ifdef H5_HAVE_INSTRUMENTED_LIBRARY + hbool_t prop_value; + prop_value = H5D_XFER_COLL_RANK0_BCAST_DEF; + ret = H5Pinsert2(dxpl, H5D_XFER_COLL_RANK0_BCAST_NAME, H5D_XFER_COLL_RANK0_BCAST_SIZE, &prop_value, + NULL, NULL, NULL, NULL, NULL, NULL); + VRFY((ret >= 0),"testing property list inserted succeeded"); +#endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ + + for(ii = 0; ii < 2; ii++) { + + if(ii == 0) + ret = H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_INDEPENDENT); + else + ret = H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_COLLECTIVE); + VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); + + /* set entire read buffer with the constant 2 */ + HDmemset(rdata,2,(size_t)(dset_size*sizeof(int))); + + /* Read the entire dataset back */ + ret = H5Dread(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, dxpl, rdata); + VRFY((ret >= 0), "H5Dread succeeded"); + +#ifdef H5_HAVE_INSTRUMENTED_LIBRARY + prop_value = FALSE; + ret = H5Pget(dxpl, H5D_XFER_COLL_RANK0_BCAST_NAME, &prop_value); + VRFY((ret >= 0), "testing property list get succeeded"); + if(ii == 0) + VRFY((prop_value == FALSE), "correctly handled rank 0 Bcast"); + else + VRFY((prop_value == TRUE), "correctly handled rank 0 Bcast"); +#endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ + + /* Verify all data read are the fill value 0 */ + trdata = rdata; + err_num = 0; + for(i = 0; i < (int)dset_dims[0]; i++) for(j = 0; j < (int)dset_dims[1]; j++) - for(k = 0; k < (int)dset_dims[2]; k++) - for(l = 0; l < (int)dset_dims[3]; l++, twdata++, trdata++) - if(*trdata != 0) - if(err_num++ < MAX_ERR_REPORT || VERBOSE_MED) - printf("Dataset Verify failed at [%d][%d][%d][%d]: expect 0, got %d\n", i, j, k, l, *trdata); - if(err_num > MAX_ERR_REPORT && !VERBOSE_MED) + for(k = 0; k < (int)dset_dims[2]; k++) + for(l = 0; l < (int)dset_dims[3]; l++, twdata++, trdata++) + if(*trdata != 0) + if(err_num++ < MAX_ERR_REPORT || VERBOSE_MED) + printf("Dataset Verify failed at [%d][%d][%d][%d]: expect 0, got %d\n", i, j, k, l, *trdata); + if(err_num > MAX_ERR_REPORT && !VERBOSE_MED) printf("[more errors ...]\n"); - if(err_num){ + if(err_num) { printf("%d errors found in check_value\n", err_num); - nerrors++; + nerrors++; + } } /* Barrier to ensure all processes have completed the above test. */ @@ -681,10 +729,6 @@ void dataset_fillvalue(void) ret = H5Sselect_hyperslab(memspace, H5S_SELECT_SET, req_start, NULL, req_count, NULL); VRFY((ret >= 0), "H5Sselect_hyperslab succeeded on memory dataspace"); - /* Create DXPL for collective I/O */ - dxpl = H5Pcreate(H5P_DATASET_XFER); - VRFY((dxpl >= 0), "H5Pcreate succeeded"); - ret = H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_COLLECTIVE); VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); if(dxfer_coll_type == DXFER_INDEPENDENT_IO) { @@ -711,37 +755,64 @@ void dataset_fillvalue(void) /* * Read dataset after partial write. */ - /* set entire read buffer with the constant 2 */ - HDmemset(rdata,2,(size_t)(dset_size*sizeof(int))); - /* Independently read the entire dataset back */ - ret = H5Dread(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rdata); - VRFY((ret >= 0), "H5Dread succeeded"); - /* Verify correct data read */ - twdata=wdata; - trdata=rdata; - err_num=0; - for(i=0; i<(int)dset_dims[0]; i++) +#ifdef H5_HAVE_INSTRUMENTED_LIBRARY + prop_value = H5D_XFER_COLL_RANK0_BCAST_DEF; + ret = H5Pset(dxpl, H5D_XFER_COLL_RANK0_BCAST_NAME, &prop_value); + VRFY((ret >= 0), " H5Pset succeeded"); +#endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ + + for(ii = 0; ii < 2; ii++) { + + if(ii == 0) + ret = H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_INDEPENDENT); + else + ret = H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_COLLECTIVE); + VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); + + /* set entire read buffer with the constant 2 */ + HDmemset(rdata,2,(size_t)(dset_size*sizeof(int))); + + /* Read the entire dataset back */ + ret = H5Dread(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, dxpl, rdata); + VRFY((ret >= 0), "H5Dread succeeded"); + +#ifdef H5_HAVE_INSTRUMENTED_LIBRARY + prop_value = FALSE; + ret = H5Pget(dxpl, H5D_XFER_COLL_RANK0_BCAST_NAME, &prop_value); + VRFY((ret >= 0), "testing property list get succeeded"); + if(ii == 0) + VRFY((prop_value == FALSE), "correctly handled rank 0 Bcast"); + else + VRFY((prop_value == TRUE), "correctly handled rank 0 Bcast"); +#endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ + + /* Verify correct data read */ + twdata=wdata; + trdata=rdata; + err_num=0; + for(i=0; i<(int)dset_dims[0]; i++) for(j=0; j<(int)dset_dims[1]; j++) - for(k=0; k<(int)dset_dims[2]; k++) - for(l=0; l<(int)dset_dims[3]; l++, twdata++, trdata++) - if(i MAX_ERR_REPORT && !VERBOSE_MED) + for(k=0; k<(int)dset_dims[2]; k++) + for(l=0; l<(int)dset_dims[3]; l++, twdata++, trdata++) + if(i MAX_ERR_REPORT && !VERBOSE_MED) printf("[more errors ...]\n"); - if(err_num){ + if(err_num){ printf("%d errors found in check_value\n", err_num); - nerrors++; + nerrors++; + } } - + /* Close all file objects */ ret = H5Dclose(dataset); VRFY((ret >= 0), "H5Dclose succeeded"); @@ -856,7 +927,7 @@ void collective_group_write(void) if(!((m+1) % 10)) { printf("created %d groups\n", m+1); MPI_Barrier(MPI_COMM_WORLD); - } + } #endif /* BARRIER_CHECKS */ } diff --git a/testpar/t_pread.c b/testpar/t_pread.c index 19ccf56..ac33b88 100644 --- a/testpar/t_pread.c +++ b/testpar/t_pread.c @@ -17,6 +17,7 @@ */ #include "testpar.h" +#include "H5Dprivate.h" /* The collection of files is included below to aid * an external "cleanup" process if required. @@ -34,6 +35,8 @@ const char *FILENAMES[NFILENAME + 1]={"reloc_t_pread_data_file", #define COUNT 1000 +#define LIMIT_NPROC 6 + hbool_t pass = true; static const char *random_hdf5_text = "Now is the time for all first-time-users of HDF5 to read their \ @@ -46,7 +49,7 @@ completely foolproof is to underestimate the ingenuity of complete\n\ fools.\n"; static int generate_test_file(MPI_Comm comm, int mpi_rank, int group); -static int test_parallel_read(MPI_Comm comm, int mpi_rank, int group); +static int test_parallel_read(MPI_Comm comm, int mpi_rank, int mpi_size, int group); static char *test_argv0 = NULL; @@ -108,6 +111,9 @@ generate_test_file( MPI_Comm comm, int mpi_rank, int group_id ) hid_t fapl_id = -1; hid_t dxpl_id = -1; hid_t dset_id = -1; + hid_t dset_id_ch = -1; + hid_t dcpl_id = H5P_DEFAULT; + hsize_t chunk[1]; float nextValue; float *data_slice = NULL; @@ -272,6 +278,55 @@ generate_test_file( MPI_Comm comm, int mpi_rank, int group_id ) } } + + /* create a chunked dataset */ + chunk[0] = COUNT/8; + + if ( pass ) { + if ( (dcpl_id = H5Pcreate (H5P_DATASET_CREATE)) < 0 ) { + pass = false; + failure_mssg = "H5Pcreate() failed.\n"; + } + } + + if ( pass ) { + if ( (H5Pset_chunk (dcpl_id, 1, chunk) ) < 0 ) { + pass = false; + failure_mssg = "H5Pset_chunk() failed.\n"; + } + } + + if ( pass ) { + + if ( (dset_id_ch = H5Dcreate2(file_id, "dataset0_chunked", H5T_NATIVE_FLOAT, + filespace, H5P_DEFAULT, dcpl_id, + H5P_DEFAULT)) < 0 ) { + pass = false; + failure_mssg = "H5Dcreate2() failed.\n"; + } + } + + if ( pass ) { + if ( (H5Dwrite(dset_id_ch, H5T_NATIVE_FLOAT, memspace, + filespace, dxpl_id, data_slice)) < 0 ) { + pass = false; + failure_mssg = "H5Dwrite() failed.\n"; + } + } + if ( pass || (dcpl_id != -1)) { + if ( H5Pclose(dcpl_id) < 0 ) { + pass = false; + failure_mssg = "H5Pclose(dcpl_id) failed.\n"; + } + } + + if ( pass || (dset_id_ch != -1)) { + if ( H5Dclose(dset_id_ch) < 0 ) { + pass = false; + failure_mssg = "H5Dclose(dset_id_ch) failed.\n"; + } + } + /* close file, etc. */ if ( pass || (dset_id != -1)) { if ( H5Dclose(dset_id) < 0 ) { @@ -413,7 +468,7 @@ generate_test_file( MPI_Comm comm, int mpi_rank, int group_id ) * Function: test_parallel_read * * Purpose: This actually tests the superblock optimization - * and covers the two primary cases we're interested in. + * and covers the three primary cases we're interested in. * 1). That HDF5 files can be opened in parallel by * the rank 0 process and that the superblock * offset is correctly broadcast to the other @@ -423,6 +478,10 @@ generate_test_file( MPI_Comm comm, int mpi_rank, int group_id ) * subgroups of MPI_COMM_WORLD and that each * subgroup operates as described in (1) to * collectively read the data. + * 3). Testing proc0-read-and-MPI_Bcast using + * sub-communicators, and reading into + * a memory space that is different from the + * file space, and chunked datasets. * * The global MPI rank is used for reading and * writing data for process specific data in the @@ -444,7 +503,7 @@ generate_test_file( MPI_Comm comm, int mpi_rank, int group_id ) *------------------------------------------------------------------------- */ static int -test_parallel_read(MPI_Comm comm, int mpi_rank, int group_id) +test_parallel_read(MPI_Comm comm, int mpi_rank, int mpi_size, int group_id) { const char *failure_mssg; const char *fcn_name = "test_parallel_read()"; @@ -457,8 +516,13 @@ test_parallel_read(MPI_Comm comm, int mpi_rank, int group_id) hid_t fapl_id = -1; hid_t file_id = -1; hid_t dset_id = -1; + hid_t dset_id_ch = -1; + hid_t dxpl_id = H5P_DEFAULT; hid_t memspace = -1; hid_t filespace = -1; + hid_t filetype = -1; + size_t filetype_size; + hssize_t dset_size; hsize_t i; hsize_t offset; hsize_t count = COUNT; @@ -552,6 +616,14 @@ test_parallel_read(MPI_Comm comm, int mpi_rank, int group_id) } } + /* open the chunked data set */ + if ( pass ) { + if ( (dset_id_ch = H5Dopen2(file_id, "dataset0_chunked", H5P_DEFAULT)) < 0 ) { + pass = FALSE; + failure_mssg = "H5Dopen2() failed\n"; + } + } + /* setup memspace */ if ( pass ) { dims[0] = count; @@ -606,14 +678,6 @@ test_parallel_read(MPI_Comm comm, int mpi_rank, int group_id) } } - /* close file, etc. */ - if ( pass || (dset_id != -1) ) { - if ( H5Dclose(dset_id) < 0 ) { - pass = false; - failure_mssg = "H5Dclose(dset_id) failed.\n"; - } - } - if ( pass || (memspace != -1) ) { if ( H5Sclose(memspace) < 0 ) { pass = false; @@ -628,6 +692,330 @@ test_parallel_read(MPI_Comm comm, int mpi_rank, int group_id) } } + /* free data_slice if it has been allocated */ + if ( data_slice != NULL ) { + HDfree(data_slice); + data_slice = NULL; + } + + /* + * Test reading proc0-read-and-bcast with sub-communicators + */ + + /* Don't test with more than LIMIT_NPROC processes to avoid memory issues */ + + if( group_size <= LIMIT_NPROC ) { +#ifdef H5_HAVE_INSTRUMENTED_LIBRARY + hbool_t prop_value; +#endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ + + if ( (filespace = H5Dget_space(dset_id )) < 0 ) { + pass = FALSE; + failure_mssg = "H5Dget_space failed.\n"; + } + + if ( (dset_size = H5Sget_simple_extent_npoints(filespace)) < 0 ) { + pass = FALSE; + failure_mssg = "H5Sget_simple_extent_npoints failed.\n"; + } + + if ( (filetype = H5Dget_type(dset_id)) < 0 ) { + pass = FALSE; + failure_mssg = "H5Dget_type failed.\n"; + } + + if ( (filetype_size = H5Tget_size(filetype)) == 0 ) { + pass = FALSE; + failure_mssg = "H5Tget_size failed.\n"; + } + + if ( H5Tclose(filetype) < 0 ) { + pass = FALSE; + failure_mssg = "H5Tclose failed.\n"; + }; + + if ( (data_slice = (float *)HDmalloc((size_t)dset_size*filetype_size)) == NULL ) { + pass = FALSE; + failure_mssg = "malloc of data_slice failed.\n"; + } + + if ( pass ) { + if ( (dxpl_id = H5Pcreate(H5P_DATASET_XFER)) < 0 ) { + pass = FALSE; + failure_mssg = "H5Pcreate(H5P_DATASET_XFER) failed.\n"; + } + } + + if ( pass ) { + if ( (H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE)) < 0 ) { + pass = FALSE; + failure_mssg = "H5Pset_dxpl_mpio() failed.\n"; + } + } + +#ifdef H5_HAVE_INSTRUMENTED_LIBRARY + if ( pass ) { + prop_value = H5D_XFER_COLL_RANK0_BCAST_DEF; + if(H5Pinsert2(dxpl_id, H5D_XFER_COLL_RANK0_BCAST_NAME, H5D_XFER_COLL_RANK0_BCAST_SIZE, &prop_value, + NULL, NULL, NULL, NULL, NULL, NULL) < 0) { + pass = FALSE; + failure_mssg = "H5Pinsert2() failed\n"; + } + } +#endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ + + /* read H5S_ALL section */ + if ( pass ) { + if ( (H5Dread(dset_id, H5T_NATIVE_FLOAT, H5S_ALL, + H5S_ALL, dxpl_id, data_slice)) < 0 ) { + pass = FALSE; + failure_mssg = "H5Dread() failed\n"; + } + } + +#ifdef H5_HAVE_INSTRUMENTED_LIBRARY + if ( pass ) { + prop_value = FALSE; + if(H5Pget(dxpl_id, H5D_XFER_COLL_RANK0_BCAST_NAME, &prop_value) < 0) { + pass = FALSE; + failure_mssg = "H5Pget() failed\n"; + } + if (pass) { + if(prop_value != TRUE) { + pass = FALSE; + failure_mssg = "rank 0 Bcast optimization was mistakenly not performed\n"; + } + } + } +#endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ + + /* verify the data */ + if ( pass ) { + + if ( comm == MPI_COMM_WORLD ) /* test 1 */ + nextValue = 0; + else if ( group_id == 0 ) /* test 2 group 0 */ + nextValue = 0; + else /* test 2 group 1 */ + nextValue = (float)((hsize_t)( mpi_size / 2 )*count); + + i = 0; + while ( ( pass ) && ( i < (hsize_t)dset_size ) ) { + /* what we really want is data_slice[i] != nextValue -- + * the following is a circumlocution to shut up the + * the compiler. + */ + if ( ( data_slice[i] > nextValue ) || + ( data_slice[i] < nextValue ) ) { + pass = FALSE; + failure_mssg = "Unexpected dset contents.\n"; + } + nextValue += 1; + i++; + } + } + + /* read H5S_ALL section for the chunked dataset */ + +#ifdef H5_HAVE_INSTRUMENTED_LIBRARY + if ( pass ) { + prop_value = H5D_XFER_COLL_RANK0_BCAST_DEF; + if(H5Pset(dxpl_id, H5D_XFER_COLL_RANK0_BCAST_NAME, &prop_value) < 0) { + pass = FALSE; + failure_mssg = "H5Pset() failed\n"; + } + } +#endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ + + for ( i = 0; i < (hsize_t)dset_size; i++) { + data_slice[i] = 0; + } + if ( pass ) { + if ( (H5Dread(dset_id_ch, H5T_NATIVE_FLOAT, H5S_ALL, + H5S_ALL, dxpl_id, data_slice)) < 0 ) { + pass = FALSE; + failure_mssg = "H5Dread() failed\n"; + } + } + +#ifdef H5_HAVE_INSTRUMENTED_LIBRARY + if ( pass ) { + prop_value = FALSE; + if(H5Pget(dxpl_id, H5D_XFER_COLL_RANK0_BCAST_NAME, &prop_value) < 0) { + pass = FALSE; + failure_mssg = "H5Pget() failed\n"; + } + if (pass) { + if(prop_value == TRUE) { + pass = FALSE; + failure_mssg = "rank 0 Bcast optimization was mistakenly performed for chunked dataset\n"; + } + } + } +#endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ + + /* verify the data */ + if ( pass ) { + + if ( comm == MPI_COMM_WORLD ) /* test 1 */ + nextValue = 0; + else if ( group_id == 0 ) /* test 2 group 0 */ + nextValue = 0; + else /* test 2 group 1 */ + nextValue = (float)((hsize_t)( mpi_size / 2 )*count); + + i = 0; + while ( ( pass ) && ( i < (hsize_t)dset_size ) ) { + /* what we really want is data_slice[i] != nextValue -- + * the following is a circumlocution to shut up the + * the compiler. + */ + if ( ( data_slice[i] > nextValue ) || + ( data_slice[i] < nextValue ) ) { + pass = FALSE; + failure_mssg = "Unexpected chunked dset contents.\n"; + } + nextValue += 1; + i++; + } + } + + if ( pass || (filespace != -1) ) { + if ( H5Sclose(filespace) < 0 ) { + pass = false; + failure_mssg = "H5Sclose(filespace) failed.\n"; + } + } + + /* free data_slice if it has been allocated */ + if ( data_slice != NULL ) { + HDfree(data_slice); + data_slice = NULL; + } + + /* + * Read an H5S_ALL filespace into a hyperslab defined memory space + */ + + if ( (data_slice = (float *)HDmalloc((size_t)(dset_size*2)*filetype_size)) == NULL ) { + pass = FALSE; + failure_mssg = "malloc of data_slice failed.\n"; + } + + /* setup memspace */ + if ( pass ) { + dims[0] = (hsize_t)dset_size*2; + if ( (memspace = H5Screate_simple(1, dims, NULL)) < 0 ) { + pass = FALSE; + failure_mssg = "H5Screate_simple(1, dims, NULL) failed\n"; + } + } + if ( pass ) { + offset = (hsize_t)dset_size; + if ( (H5Sselect_hyperslab(memspace, H5S_SELECT_SET, + &offset, NULL, &offset, NULL)) < 0 ) { + pass = FALSE; + failure_mssg = "H5Sselect_hyperslab() failed\n"; + } + } + +#ifdef H5_HAVE_INSTRUMENTED_LIBRARY + if ( pass ) { + prop_value = H5D_XFER_COLL_RANK0_BCAST_DEF; + if(H5Pset(dxpl_id, H5D_XFER_COLL_RANK0_BCAST_NAME, &prop_value) < 0) { + pass = FALSE; + failure_mssg = "H5Pset() failed\n"; + } + } +#endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ + + /* read this processes section of the data */ + if ( pass ) { + if ( (H5Dread(dset_id, H5T_NATIVE_FLOAT, memspace, + H5S_ALL, dxpl_id, data_slice)) < 0 ) { + pass = FALSE; + failure_mssg = "H5Dread() failed\n"; + } + } + +#ifdef H5_HAVE_INSTRUMENTED_LIBRARY + if ( pass ) { + prop_value = FALSE; + if(H5Pget(dxpl_id, H5D_XFER_COLL_RANK0_BCAST_NAME, &prop_value) < 0) { + pass = FALSE; + failure_mssg = "H5Pget() failed\n"; + } + if (pass) { + if(prop_value != TRUE) { + pass = FALSE; + failure_mssg = "rank 0 Bcast optimization was mistakenly not performed\n"; + } + } + } +#endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ + + /* verify the data */ + if ( pass ) { + + if ( comm == MPI_COMM_WORLD ) /* test 1 */ + nextValue = 0; + else if ( group_id == 0 ) /* test 2 group 0 */ + nextValue = 0; + else /* test 2 group 1 */ + nextValue = (float)((hsize_t)(mpi_size / 2)*count); + + i = (hsize_t)dset_size; + while ( ( pass ) && ( i < (hsize_t)dset_size ) ) { + /* what we really want is data_slice[i] != nextValue -- + * the following is a circumlocution to shut up the + * the compiler. + */ + if ( ( data_slice[i] > nextValue ) || + ( data_slice[i] < nextValue ) ) { + pass = FALSE; + failure_mssg = "Unexpected dset contents.\n"; + } + nextValue += 1; + i++; + } + } + + if ( pass || (memspace != -1) ) { + if ( H5Sclose(memspace) < 0 ) { + pass = false; + failure_mssg = "H5Sclose(memspace) failed.\n"; + } + } + + /* free data_slice if it has been allocated */ + if ( data_slice != NULL ) { + HDfree(data_slice); + data_slice = NULL; + } + + if ( pass || (dxpl_id != -1) ) { + if ( H5Pclose(dxpl_id) < 0 ) { + pass = false; + failure_mssg = "H5Pclose(dxpl_id) failed.\n"; + } + } + } + + /* close file, etc. */ + if ( pass || (dset_id != -1) ) { + if ( H5Dclose(dset_id) < 0 ) { + pass = false; + failure_mssg = "H5Dclose(dset_id) failed.\n"; + } + } + + if ( pass || (dset_id_ch != -1) ) { + if ( H5Dclose(dset_id_ch) < 0 ) { + pass = false; + failure_mssg = "H5Dclose(dset_id_ch) failed.\n"; + } + } + if ( pass || (file_id != -1) ) { if ( H5Fclose(file_id) < 0 ) { pass = false; @@ -668,17 +1056,9 @@ test_parallel_read(MPI_Comm comm, int mpi_rank, int group_id) HDfprintf(stdout, "%s: failure_mssg = \"%s\"\n", fcn_name, failure_mssg); } - HDremove(reloc_data_filename); } - /* free data_slice if it has been allocated */ - if ( data_slice != NULL ) { - HDfree(data_slice); - data_slice = NULL; - } - - return( ! pass ); } /* test_parallel_read() */ @@ -810,7 +1190,7 @@ main( int argc, char **argv) } /* Now read the generated test file (stil using MPI_COMM_WORLD) */ - nerrs += test_parallel_read( MPI_COMM_WORLD, mpi_rank, which_group); + nerrs += test_parallel_read( MPI_COMM_WORLD, mpi_rank, mpi_size, which_group); if ( nerrs > 0 ) { if ( mpi_rank == 0 ) { @@ -826,7 +1206,7 @@ main( int argc, char **argv) } /* run the 2nd set of tests */ - nerrs += test_parallel_read(group_comm, mpi_rank, which_group); + nerrs += test_parallel_read(group_comm, mpi_rank, mpi_size, which_group); if ( nerrs > 0 ) { if ( mpi_rank == 0 ) { -- cgit v0.12 From 534e535823f0bbe584f353421bf53f0153882377 Mon Sep 17 00:00:00 2001 From: "M. Scot Breitenfeld" Date: Fri, 11 Jan 2019 09:27:17 -0600 Subject: misc. merge clean-up --- src/H5CXprivate.h | 1 - src/H5Dmpio.c | 7 ----- src/H5FDmpio.c | 79 ++++++++++++++++++++++++++++--------------------------- 3 files changed, 40 insertions(+), 47 deletions(-) diff --git a/src/H5CXprivate.h b/src/H5CXprivate.h index 6b8e3b2..a091292 100644 --- a/src/H5CXprivate.h +++ b/src/H5CXprivate.h @@ -128,7 +128,6 @@ H5_DLL void H5CX_set_mpio_actual_chunk_opt(H5D_mpio_actual_chunk_opt_mode_t chun H5_DLL void H5CX_set_mpio_actual_io_mode(H5D_mpio_actual_io_mode_t actual_io_mode); H5_DLL void H5CX_set_mpio_local_no_coll_cause(uint32_t mpio_local_no_coll_cause); H5_DLL void H5CX_set_mpio_global_no_coll_cause(uint32_t mpio_global_no_coll_cause); -H5_DLL void H5CX_set_mpio_Proc0_BCast(hbool_t mpio_Proc0_BCast); #ifdef H5_HAVE_INSTRUMENTED_LIBRARY H5_DLL herr_t H5CX_test_set_mpio_coll_chunk_link_hard(int mpio_coll_chunk_link_hard); H5_DLL herr_t H5CX_test_set_mpio_coll_chunk_multi_hard(int mpio_coll_chunk_multi_hard); diff --git a/src/H5Dmpio.c b/src/H5Dmpio.c index 20b2066..7352375 100644 --- a/src/H5Dmpio.c +++ b/src/H5Dmpio.c @@ -82,11 +82,6 @@ /* Macros to represent the regularity of the selection for multiple chunk IO case. */ #define H5D_CHUNK_SELECT_REG 1 -/* Macros for reasons to not enable read-proc-and-bcast. */ -#define H5D_MPIO_PROC0_BCAST 0x00 -#define H5D_MPIO_NOT_H5S_ALL 0x01 -#define H5D_MPIO_GREATER_THAN_2GB 0x02 - /******************/ /* Local Typedefs */ /******************/ @@ -299,7 +294,6 @@ H5D__mpio_opt_possible(const H5D_io_info_t *io_info, const H5S_t *file_space, unsigned global_cause[2] = {0,0}; /* Global reason(s) for breaking collective mode */ htri_t is_vl_storage; /* Whether the dataset's datatype is stored in a variable-length form */ htri_t ret_value = SUCCEED; /* Return value */ - hbool_t H5FD_MPIO_Proc0_BCast; /* Flag if dataset is both: H5S_ALL and < 2GB */ FUNC_ENTER_PACKAGE @@ -419,7 +413,6 @@ H5D__mpio_opt_possible(const H5D_io_info_t *io_info, const H5S_t *file_space, */ if(MPI_SUCCESS != (mpi_code = MPI_Allreduce(&local_cause, &global_cause, 2, MPI_UNSIGNED, MPI_BOR, io_info->comm))) HMPI_GOTO_ERROR(FAIL, "MPI_Allreduce failed", mpi_code) - } /* end else */ /* Set the local & global values of no-collective-cause in the API context */ diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c index 689f43c..c6bc8e4 100644 --- a/src/H5FDmpio.c +++ b/src/H5FDmpio.c @@ -1414,33 +1414,35 @@ static herr_t H5FD_mpio_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNUSED dxpl_id, haddr_t addr, size_t size, void *buf/*out*/) { - H5FD_mpio_t *file = (H5FD_mpio_t*)_file; - MPI_Offset mpi_off; - MPI_Status mpi_stat; /* Status from I/O operation */ - int mpi_code; /* mpi return code */ - MPI_Datatype buf_type = MPI_BYTE; /* MPI description of the selection in memory */ - int size_i; /* Integer copy of 'size' to read */ + H5FD_mpio_t *file = (H5FD_mpio_t*)_file; + MPI_Offset mpi_off; + MPI_Status mpi_stat; /* Status from I/O operation */ + int mpi_code; /* mpi return code */ + MPI_Datatype buf_type = MPI_BYTE; /* MPI description of the selection in memory */ + int size_i; /* Integer copy of 'size' to read */ #if MPI_VERSION >= 3 - MPI_Count bytes_read = 0; /* Number of bytes read in */ - MPI_Count type_size; /* MPI datatype used for I/O's size */ - MPI_Count io_size; /* Actual number of bytes requested */ - MPI_Count n; + MPI_Count bytes_read = 0; /* Number of bytes read in */ + MPI_Count type_size; /* MPI datatype used for I/O's size */ + MPI_Count io_size; /* Actual number of bytes requested */ + MPI_Count n; #else - int bytes_read = 0; /* Number of bytes read in */ - int type_size; /* MPI datatype used for I/O's size */ - int io_size; /* Actual number of bytes requested */ - int n; + int bytes_read = 0; /* Number of bytes read in */ + int type_size; /* MPI datatype used for I/O's size */ + int io_size; /* Actual number of bytes requested */ + int n; #endif hbool_t use_view_this_time = FALSE; hbool_t rank0_bcast = FALSE; /* If read-with-rank0-and-bcast flag was used */ herr_t ret_value = SUCCEED; - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC #ifdef H5FDmpio_DEBUG - if (H5FD_mpio_Debug[(int)'t']) - fprintf(stdout, "Entering H5FD_mpio_read\n" ); + if(H5FD_mpio_Debug[(int)'t']) + HDfprintf(stdout, "%s: Entering\n", FUNC); #endif + + /* Sanity checks */ HDassert(file); HDassert(H5FD_MPIO==file->pub.driver_id); HDassert(buf); @@ -1449,16 +1451,15 @@ H5FD_mpio_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, HDmemset(&mpi_stat,0,sizeof(MPI_Status)); /* some numeric conversions */ - if (H5FD_mpi_haddr_to_MPIOff(addr, &mpi_off/*out*/)<0) + if(H5FD_mpi_haddr_to_MPIOff(addr, &mpi_off/*out*/) < 0) HGOTO_ERROR(H5E_INTERNAL, H5E_BADRANGE, FAIL, "can't convert from haddr to MPI off") size_i = (int)size; - if ((hsize_t)size_i != size) + if((hsize_t)size_i != size) HGOTO_ERROR(H5E_INTERNAL, H5E_BADRANGE, FAIL, "can't convert from size to size_i") #ifdef H5FDmpio_DEBUG - if (H5FD_mpio_Debug[(int)'r']) - fprintf(stdout, "in H5FD_mpio_read mpi_off=%ld size_i=%d\n", - (long)mpi_off, size_i ); + if(H5FD_mpio_Debug[(int)'r']) + HDfprintf(stdout, "%s: mpi_off = %ld size_i = %d\n", FUNC, (long)mpi_off, size_i); #endif /* Only look for MPI views for raw data transfers */ @@ -1475,7 +1476,7 @@ H5FD_mpio_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, * us to test that btype=ftype=MPI_BYTE (or even MPI_TYPE_NULL, which * could mean "use MPI_BYTE" by convention). */ - if(xfer_mode==H5FD_MPIO_COLLECTIVE) { + if(xfer_mode == H5FD_MPIO_COLLECTIVE) { MPI_Datatype file_type; /* Remember that views are used */ @@ -1503,8 +1504,8 @@ H5FD_mpio_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, H5FD_mpio_collective_opt_t coll_opt_mode; #ifdef H5FDmpio_DEBUG - if (H5FD_mpio_Debug[(int)'t']) - fprintf(stdout, "H5FD_mpio_read: using MPIO collective mode\n"); + if(H5FD_mpio_Debug[(int)'r']) + HDfprintf(stdout, "%s: using MPIO collective mode\n", FUNC); #endif /* Get the collective_opt property to check whether the application wants to do IO individually. */ if(H5CX_get_mpio_coll_opt(&coll_opt_mode) < 0) @@ -1512,8 +1513,8 @@ H5FD_mpio_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, if(coll_opt_mode == H5FD_MPIO_COLLECTIVE_IO) { #ifdef H5FDmpio_DEBUG - if(H5FD_mpio_Debug[(int)'t']) - fprintf(stdout, "H5FD_mpio_read: doing MPI collective IO\n"); + if(H5FD_mpio_Debug[(int)'r']) + HDfprintf(stdout, "%s: doing MPI collective IO\n", FUNC); #endif /* Check whether we should read from rank 0 and broadcast to other ranks */ if(H5CX_get_mpio_rank0_bcast()) { @@ -1537,8 +1538,8 @@ H5FD_mpio_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, } /* end if */ else { #ifdef H5FDmpio_DEBUG - if(H5FD_mpio_Debug[(int)'t']) - fprintf(stdout, "H5FD_mpio_read: doing MPI independent IO\n"); + if(H5FD_mpio_Debug[(int)'r']) + HDfprintf(stdout, "%s: doing MPI independent IO\n", FUNC); #endif if(MPI_SUCCESS != (mpi_code = MPI_File_read_at(file->f, mpi_off, buf, size_i, buf_type, &mpi_stat))) @@ -1550,10 +1551,10 @@ H5FD_mpio_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, */ if(MPI_SUCCESS != (mpi_code = MPI_File_set_view(file->f, (MPI_Offset)0, MPI_BYTE, MPI_BYTE, H5FD_mpi_native_g, file->info))) HMPI_GOTO_ERROR(FAIL, "MPI_File_set_view failed", mpi_code) - } else { + } /* end if */ + else if(MPI_SUCCESS != (mpi_code = MPI_File_read_at(file->f, mpi_off, buf, size_i, buf_type, &mpi_stat))) - HMPI_GOTO_ERROR(FAIL, "MPI_File_read_at failed", mpi_code) - } + HMPI_GOTO_ERROR(FAIL, "MPI_File_read_at failed", mpi_code) /* Only retrieve bytes read if this rank _actually_ participated in I/O */ if(!rank0_bcast || (rank0_bcast && file->mpi_rank == 0) ) { @@ -1582,25 +1583,25 @@ H5FD_mpio_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, #else if(MPI_SUCCESS != (mpi_code = MPI_Type_size(buf_type, &type_size))) #endif - HMPI_GOTO_ERROR(FAIL, "MPI_Type_size failed", mpi_code) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_size failed", mpi_code) /* Compute the actual number of bytes requested */ - io_size=type_size*size_i; + io_size = type_size * size_i; /* Check for read failure */ - if (bytes_read<0 || bytes_read>io_size) + if(bytes_read < 0 || bytes_read > io_size) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file read failed") /* * This gives us zeroes beyond end of physical MPI file. */ - if ((n=(io_size-bytes_read)) > 0) - HDmemset((char*)buf+bytes_read, 0, (size_t)n); + if((n = (io_size - bytes_read)) > 0) + HDmemset((char*)buf+bytes_read, 0, (size_t)n); done: #ifdef H5FDmpio_DEBUG - if (H5FD_mpio_Debug[(int)'t']) - fprintf(stdout, "Leaving H5FD_mpio_read\n" ); + if(H5FD_mpio_Debug[(int)'t']) + HDfprintf(stdout, "%s: Leaving\n", FUNC); #endif FUNC_LEAVE_NOAPI(ret_value) -- cgit v0.12 From 4bb965d9ec4d3d759e1bb4374f8391cf694e14bb Mon Sep 17 00:00:00 2001 From: "M. Scot Breitenfeld" Date: Fri, 11 Jan 2019 10:59:48 -0600 Subject: misc. merge clean-up --- release_docs/RELEASE.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 06c7c6a..3d93b80 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -408,8 +408,6 @@ Bug Fixes since HDF5-1.10.3 release (MSB, 2018/12/04, HDFFV-10511) -<<<<<<< HEAD -======= - Fixed issue with Fortran not returning h5o_info_t field values meta_size%attr%index_size and meta_size%attr%heap_size. @@ -685,7 +683,6 @@ Bug Fixes since HDF5-1.10.2 release Testing ------- ->>>>>>> fb5d83e146... Updated Fortran Bug and new features Supported Platforms =================== -- cgit v0.12 From c9cf24506b4fb1c50ea553d3ab962c37ac86760a Mon Sep 17 00:00:00 2001 From: Jordan Henderson Date: Tue, 8 Jan 2019 09:45:35 -0600 Subject: Fix for parallel filters tests with new read-proc0-broadcast feature --- src/H5Dio.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/H5Dio.c b/src/H5Dio.c index c762389..2ce47f9 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -1172,15 +1172,19 @@ H5D__ioinfo_adjust(H5D_io_info_t *io_info, const H5D_t *dset, hbool_t local_error_message_previously_written = FALSE; hbool_t global_error_message_previously_written = FALSE; size_t index; - char local_no_collective_cause_string[256] = ""; - char global_no_collective_cause_string[256] = ""; + size_t cause_strings_len; + char local_no_collective_cause_string[512] = ""; + char global_no_collective_cause_string[512] = ""; const char *cause_strings[] = { "independent I/O was requested", "datatype conversions were required", "data transforms needed to be applied", "optimized MPI types flag wasn't set", "one of the dataspaces was neither simple nor scalar", "dataset was not contiguous or chunked", - "parallel writes to filtered datasets are disabled" }; + "parallel writes to filtered datasets are disabled", + "an error occurred while checking if collective I/O was possible" }; + + cause_strings_len = sizeof(cause_strings) / sizeof(cause_strings[0]); if(H5CX_get_mpio_local_no_coll_cause(&local_no_collective_cause) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get local no collective cause value") @@ -1189,7 +1193,7 @@ H5D__ioinfo_adjust(H5D_io_info_t *io_info, const H5D_t *dset, /* Append each of the "reason for breaking collective I/O" error messages to the * local and global no collective cause strings */ - for (cause = 1, index = 0; cause < H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE; cause <<= 1, index++) { + for (cause = 1, index = 0; (cause < H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE) && (index < cause_strings_len); cause <<= 1, index++) { size_t cause_strlen = HDstrlen(cause_strings[index]); if (cause & local_no_collective_cause) { -- cgit v0.12